from bs4 import BeautifulSoup
from urllib.request import urlopen
import bisect
site = 'http://crate.io{0}'
page = BeautifulSoup(urlopen(site.format('/?has_releases=on&q=django')))
table = []
while True:
    for result in page.find_all('div', attrs={'class':'result'}): # строка с именем пакета  и количеством загрузок
        count = result.find('span', attrs={'class':'count'}).string # загрузки
        count = int(count.replace(',', ''))
        package_name = result.find(attrs={'class':'package-name'}).string # имя пакета
        couple = [count, package_name]
        bisect.insort_right(table, couple) # добавляю в список через сортировку
    li = page.find('li', attrs={'class':'next'}) # строка с ссылку на след страницу
    print(li['class'])
    if 'disabled' in li['class']: break # если найден disabled, то это последняя страница - выход
    page = BeautifulSoup(urlopen(site.format(li.a['href']))) # перехожу на след страницу
table.reverse() # хочу видеть на первых позициях пакет с большим кол загрузок
for num, i in enumerate(table): print(num, i)

from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import as_completed
from operator import itemgetter
from queue import Queue
import requests
from lxml import html
SITE = "http://crate.io{}"
class IterableQueue(Queue):
    def __iter__(self):
        return iter(self.get, StopIteration)
    def close(self):
        self.put(StopIteration)
def page(url, page_queue, page_set):
    _page = html.fromstring(requests.get(url).text)
    for a in _page.cssselect("li.active ~ li a"):
        link = a.get("href")
        try:
            num = int(a.text)
            if num not in page_set:
                page_set.add(num)
                link = SITE.format(link)
                page_queue.put(link)
        except ValueError:
            if link == "#":
                page_queue.put(StopIteration)
    packages = []
    for item in _page.cssselect("div.result.row"):
        name = item.cssselect("a")[0].text
        count = item.cssselect("span.count")[0].text.replace(",", "")
        packages.append((name, count))
    return packages
if "__main__" == __name__:
    page_queue = IterableQueue()
    page_set = set()
    result = []
    page_queue.put("https://crate.io/?has_releases=on&q=django")
    with ThreadPoolExecutor(max_workers=8) as pool:
        results = [pool.submit(page, url, page_queue, page_set) for url in page_queue]
        for task in as_completed(results):
            result.extend(task.result())
    for name, count in sorted(result, key=itemgetter(1), reverse=True):
        print(name, count)