用Python写map,reduce
#!/usr/bin/env python
import sys
for line in sys.stdin:
line= line.strip()
words= line.split()
for word in words:
print'%s\t%s' % (word,1)
--map.py
#!/usr/bin/env python
from operator import itemgetter
import sys
word2count = {}
for line in sys.stdin:
line = line.strip()
word, count =line.split('\t', 1)
try:
count =int(count)
word2count[word]=word2count.get(word,0)+count
except ValueError:
pass
sorted_word2count = sorted(word2count.items(),key=itemgetter(0))
for word,count in sorted_word2count:
print'%s\t%s' % (word, count)
--
测试：
echo "foo fooquux labs foo bar quux" | ./mapper.py | sort | ./reducer.py
提交HADOOP集群运行：
hadoop jar hadoop-1.2.1/contrib/streaming/hadoop-streaming-1.2.1.jar -input input -output output-streaming-python -mapper /home/mapper.py -reducer /home/reducer.py
