Python-сообщество

 import urllib.request
text=urllib.request.urlopen("http://apetrov.me/assets/war_peace_1.txt").read().decode()
print(text[:80])

 second_names = ['безухов', ....]
def map_wordcount_filter(line):
    for word in line:
        if word in ....:
            yield(word, 1)
lines = ....
result = result = run_mapreduce(lines, map_wordcount_filter, ....)
for key, value in sorted( ....

  
>>> import re
>>> 
>>> def generate_lines(text):
...     word_re = re.compile(r"[а-яё]+")
...     lines = text.strip().lower().split("\n")
...     result = []
...     for line in lines:
...         result.append(word_re.findall(line))
...     return result
... 
>>> from collections import defaultdict
>>> 
>>> def run_mapreduce(input, map_fnc, reduce_fnc):
...     map_result = defaultdict(lambda: [])
...     for obj in input:
...         for key, value in map_fnc(obj):
...             map_result[key].append(value)
...     reduce_result = []
...     for key in map_result:
...         for key, value in reduce_fnc(key, map_result[key]):
...             reduce_result.append((key, value))
...     return reduce_result
... 
>>> def map_wordcount(line):
...     for word in line:
...         yield word, 1
... 
>>> def reduce_wordcount(word, counts):
...     yield word, sum(counts)
... 
>>> 
>>> second_names = ['безухов', 'ростова', 'болконский']
>>> 
>>> def map_wordcount_filter(line):
...     for word in line.split():
...         if word in second_names:
...             yield word, 1
... 
>>> 
>>> import urllib.request
>>> 
>>> text = urllib.request.urlopen("http://apetrov.me/assets/war_peace_1.txt").read().decode()
>>> 
>>> lines = text.lower().splitlines()
>>> result = run_mapreduce(lines, map_wordcount_filter, reduce_wordcount)
>>> for key, value in sorted(result):
...     print(key, value)
... 
безухов 10
болконский 27
ростова 43
>>>

#!/usr/bin/env python3

from matplotlib import pyplot
import numpy as np

def draw_points():
    x = np.array([1, 2, 3, 4, 5])
    y = np.array([7, 12, 3, 15, 8])
    
    pyplot.plot(x, y, 'bo')
    
    words = ('', 'word1', 'word2', 'word3', 'word4', 'word5', '')
    pyplot.xticks(range(7), words)
    pyplot.yticks(range(20))
    
    pyplot.savefig('image.png')
    
    pyplot.show()

def main():
    draw_points()

if __name__ == '__main__':
    main()

 lines = generate_lines(text)
result = run_mapreduce(lines, map_wordcount, reduce_wordcount)
for key, value in sorted(result, key=lambda x: -x[1])[:5]:
    print(key, value)

 #!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import sys
from collections import Counter
import numpy as np
import matplotlib.pyplot as plt
 
FILENAME = sys.argv[1][0:sys.argv[1].find('.')]
 
WORDS = []
 
with open(sys.argv[1], 'rb') as textFile:
    WORDS = (re.findall(re.compile(r'\w+'), (textFile.read()).lower()))
 
LABELS, VALUES = zip(*list(reversed(sorted(Counter(WORDS).items(), key=lambda x: x[1]))))
INDEXES = np.arange(len(LABELS))
 
plt.plot(INDEXES, VALUES)
 
plt.xlim(-(len(LABELS))*5/100)
plt.ylim(-(max(VALUES)*5)/100)
 
plt.xlabel('Rank')
plt.ylabel('Frequency')
 
plt.grid()
plt.savefig('figures/' + FILENAME + '_zipfs', dpi=500)

#!/usr/bin/env python3

from matplotlib import pyplot
import numpy as np

def draw_curve():

    pairs = (('word1', 15),
             ('word2', 13),
             ('word3', 11),
             ('word4', 7),
             ('word5', 2))

    words = [i for i, _ in pairs]
    freqs = [i for _, i in pairs]

    totalnum = sum(freqs)

    x = np.arange(len(words))
    y = np.array([i / totalnum for i in freqs])
    maxy = 1
    
    pyplot.plot(x, y, 'bo')

    pyplot.xticks(x, words)
    pyplot.yticks(np.arange(0, maxy, 1 / len(freqs)))

    pyplot.savefig('image_freq.png')

    pyplot.show()

def main():
    draw_curve()

if __name__ == '__main__':
    main()