Python-сообщество

import os
import os.path
import sys
from math import log
RUSSIAN_ALPHABET = set("йфяцычувскамепинртгоьшлбщдюзжхэъёЙФЯЦЫЧУВСКАМЕПИНРТГОЬШЛБЩДЮЗЖХЭЪЁ")
NUMERIC_SYMBOLS  = set("\u0303҂ΑΓιξωѿѕ0123456789")
DIACRITICS       = set("\u0300\u0301\u0302\u0308")
def moderniseOrthographyAndReturnWordlist(nameOfFile):
    """Функция, которая модернизирует орфографию и возвращает список слов из файла."""
    wordList = []
    with open("C:/Texts/File1.txt", "r", encoding = "utf-8") as inp:
        for line in inp:
            if line[0] == "№":
                continue
            numericSymbolInWord = False
            tempWord = []
            txt = line.lower()
            for char in txt:
                if char in DIACRITICS:
                    continue
                elif char in NUMERIC_SYMBOLS:
                    numericSymbolInWord = True
                    continue
                elif char == "ѣ" or char == "є":
                    tempWord.append("е")
                elif char == "і" or char == "ѵ":
                    tempWord.append("и")
                elif char == "ѳ":
                    tempWord.append("ф")
                elif char == "-":
                    if len(tempWord) == 0:
                        continue
                    else:
                        if tempWord[-1] == "ъ":
                            tempWord[-1] = "-"
                        else:
                            tempWord.append("-")
                elif char not in RUSSIAN_ALPHABET:
                    if len(tempWord) > 0 and not numericSymbolInWord:
                        if tempWord[-1] == "ъ":    
                            wordList.append("".join(tempWord[:-1]))
                            tempWord = []
                        else:
                            wordList.append("".join(tempWord))
                            tempWord = []
                    else:
                        tempWord = []
                        numericSymbolInWord = False
                else:
                    tempWord.append(char)
            if len(tempWord) > 0 and not numericSymbolInWord:
                if tempWord[-1] == "ъ":    
                    wordList.append("".join(tempWord[:-1]))
                else:
                    wordList.append("".join(tempWord))
    return wordList
moarw = moderniseOrthographyAndReturnWordlist
def computeAllFrequencies(dirName):
    allfiles      = []
    frequencyDict = {} 
    for root, subs, filenames in os.walk(dirName):
        for name in filenames:
            if name.endswith("txt") and not name.startswith("."):
                allfiles.append(os.path.join(root, name))
    for name in allfiles:
        wordList = moarw(name)
        for item in wordList:
            if item in frequencyDict:
                frequencyDict[item] += 1
            else:
                frequencyDict[item] = 1
    return frequencyDict
def computeFrequencies(filename):
    frequencyDict = {}
    wordList      = moarw(filename)
    for item in wordList:
        if item in frequencyDict:
            frequencyDict[item] += 1
        else:
            frequencyDict[item] = 1
    return frequencyDict
def computeNDocs(dirName):
    allfiles = []
    nDocDict = {} 
    for root, subs, filenames in os.walk(dirName):
        for name in filenames:
            if name.endswith("txt") and not name.startswith("."):
                allfiles.append(os.path.join(root, name))
    for name in allfiles:
        wordSet = set(moarw(name))
        for item in wordSet:
            if item in nDocDict:
                nDocDict[item] += 1
            else:
                nDocDict[item] = 1
    return nDocDict
def top_tf_idf(nwords, fd, N, nDocDict):
    fdItems = sorted(fd.items(), key = lambda x: x[1], reverse = True)
    maxFreq = fdItems[0][1]
    keywordlist = []
    for key in fd:
        tf = 0.5 + (0.5 * fd[key] / maxFreq)
        if key in nDocDict:
            nDoc = nDocDict[key] + 1
        else:
            nDoc = 1
        idf = log(N / nDoc)
        tfidf = tf * idf
        keywordlist.append((key, tfidf))
    keywordlist.sort(key = lambda x: x[1], reverse = True)
    return keywordlist[:nwords]
dirname  = sys.argv[1]
nDocDict = computeNDocs(dirname)
allFiles = []
for root, subs, filenames in os.walk(dirname):
    for name in filenames:
        if name.endswith("txt") and not name.startswith("."):
            allFiles.append(os.path.join(root, name))
N = len(allFiles)
keyWordDict = {}
fd = computeFrequencies(allFiles[1])
print("\n".join(str(item) for item in top_tf_idf(10, fd, N, nDocDict)))