Python-сообщество

 '''
Parse LiveInternet directory: http://www.liveinternet.ru/rating/
It has limits for requests from one IP:
----------------
Too many requests from one IP.
Access denied for 60 minutes.
Limit: 1,000 requests per 10 minutes.
----------------    
So, add thread sleep:
time.sleep(60*10/1000)
'''
from bs4 import BeautifulSoup
import time
category_sites = []
categories = [
   # 'http://www.liveinternet.ru/rating/ru/#period=week;geo=ru;group=rest;'
    'http://www.liveinternet.ru/rating/rest/'
]
def main():
    for c in categories:
        process_category(c)
def process_category(category):
    print '---------------------'
    print 'Processing category: %s ' % category
    start_time = time.time()
    pages_processed, sites_parsed = do_process_category(category)
    elapsed_time = time.time() - start_time
    print 'Elapsed time, secs: %f' % elapsed_time
    print 'Pages processed: %d' % pages_processed
    print 'Sites parsed: %d' % sites_parsed
    print '---------------------'
def do_process_category(category):
    global category_sites
    category_sites = []
    category_name = extract_category_name(category)
    # from BeautifulSoup import BeautifulSoup
    import urllib2
    html = urllib2.urlopen(category).read()
    pages_qty = get_pages_qty(html)
    parse_sites(html)
    for page in range(2, pages_qty+1):
        u = category + ('page=%d;' % page)
        print '----> %s ' % u
        html = urllib2.urlopen(u).read()
        parse_sites(html)
        time.sleep(600/1000)
    sites_parsed = len(category_sites)
    write_sites(category_name)
    return pages_qty, sites_parsed
def parse_sites(html):
    global category_sites
    soup = BeautifulSoup(html, "html.parser")
    trs = soup.findAll('tr', {'class': 'high', 'align': 'top'})
    for tr in trs:
        try:
            aa = tr.findAll('a', {'target': '_blank', 'onclick': 'cl(this)'})
            a = aa[0]
            href = a['href']
            category_sites.append(href)
        except:
            pass
def write_sites(category_name):
    f = open(category_name+'.txt','wb')
    for s in category_sites:
        f.write(s.encode('utf-8'))
        f.write('\n')
    f.close()
def get_pages_qty(html):
    #token = '«previous '
    token = '«'
    i = html.find(token)
    if i == -1:
        print 'Token not found'
        return 1
    j = html.rfind('</a>', i-100, i)
    if j == -1:
        return 1
    k = j-1
    while html[k] != '>':
        k = k-1
    srr = html[k+1:j]
    return int(srr)
def extract_category_name(category):
    s = category
    if s.endswith('/'): s = s[0:len(s)-1]
    i = s.rfind('/')
    return s[i+1:]
if __name__ == "__main__":
    main()