Форум сайта python.su
Есть код, который парсит страницы из списка и напарсенное сохраняет в базу sqlite3.
async def main(): authors = json.load(open(r'test/authors_list.txt'))[:100] # для теста async with aiohttp.ClientSession() as session: for x in authors: url = urljoin(config.SAMLIB_DOMAIN, x[0]); print(url) html = await crawler.fetch(session, url) if parser.SI_parse_index_counters(html): _upddate, _counters = parser.SI_parse_index_counters(html) _upddate = hooks.convert_date(_upddate) else: _upddate, _counters = '', [0, 0] # author_link, author_name, author_text, author_descr, last_update, works_count, works_size, last_check db.update_authors_counters([ x[0], x[1], x[2], x[3], _upddate, _counters[1], _counters[0], datetime.datetime.now() ])
def update_authors_counters(args): '''author_link, author_name, author_text, author_descr, last_update, works_count, last_check''' print(*args) query = ''' INSERT INTO samlib_testdb ( author_link, author_name, author_text, author_descr, last_update, works_count, works_size, last_check ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ? ); ''' dbcurr.execute(query, args) dbconn.commit()
Офлайн