Python-сообщество

 import string,re
import nltk
from nltk import FreqDist 
word_tokenize = re.compile(r"([^\w_\u2019\u2010\u002F\u0027-]|[+])")  
punctuation = string.punctuation + '\u2014\u2013\u2012\u2010\u2212' + '«»‹›‘’“”„`'
book = open('text.txt')
text = book.read()
def nltk_tokenizer(text):
    for token in nltk.word_tokenize(text):
        if token and not token.isspace() and not token in punctuation:
            yield token 
 
tokens = [word for word in nltk_tokenizer(text)] 
tokens = FreqDist(tokens).items()
for one_by_one in tokens:
	print one_by_one