import requests
from bs4 import BeautifulSoup
import pymysql
def get_html(url):
    r = requests.get(url)
    return r.text
# < Получаем ссылки
def get_resource_links(resource_page,links_rule,resource_domain):
    resource_links = []
    soup = BeautifulSoup(resource_page,'lxml')
    resource_links_blocks = soup.findAll(links_rule[0],{links_rule[1]:links_rule[2]})
    for resource_link_block in resource_links_blocks:
        a_tag = resource_link_block .find("a")
        if a_tag:
            link = a_tag.get("href")
            resource_links.append(resource_domain + link)
    return resource_links
# < Собираем заголовки.
def get_item_title(item_page,title_rule):
    soup = BeautifulSoup(item_page,'lxml')
    item_title = soup.find(title_rule[0],{title_rule[1]:title_rule[2]})
    return item_title
# < Собираем даты.
def get_item_datetime(item_page,datetime_rule,datetime1_rule):
    soup = BeautifulSoup(item_page,'lxml')
    item_datetime = soup.find(datetime_rule[0],{datetime_rule[1]:datetime_rule[2]})
    item_datetime1= soup.find(datetime1_rule[0],{datetime1_rule[1]:datetime1_rule[2]})
    return item_datetime,item_datetime1
# < Подключение к базе данных.
connection = pymysql.connect(host='localhost',
                             user='root',
                             password='',
                             db='news_portal',
                             charset='utf8',
                             autocommit=True)
cursor = connection.cursor()
# < Запрос правил выдергивания контента.
cursor.execute('SELECT * FROM `resource`')
resources=cursor.fetchall()
# < Цикл для перебора из кортежа.
for resource in resources:
    resource_name=resource[1]
    resource_link=resource[2]
    resource_url=resource[3]
    link_rule=resource[4]
    title_rule=resource[8]
    datetime_rule=resource[9]
    datetime1_rule=resource[10]
    text_rule=resource[11]
    text1_rule=resource[12]
    print(resource_name)
    resource_domain=resource_link
# < Разбиваю данные из кортежа в массив
    links_rule=link_rule.split(',')
    title_rule=title_rule.split(',')
    datetime_rule=datetime_rule.split(',')
    datetime1_rule=datetime1_rule.split(',')
    text_rule=text_rule.split(',')
    text1_rule=text1_rule.split(',')
    resource_page = get_html(resource_url)
    resource_links = get_resource_links(resource_page,links_rule,resource_domain)
    print('кол-во ссылок: '+str(len(resource_links)))
    for resource_link in resource_links:
        item_page = get_html(resource_link)
        item_title = get_item_title(item_page,title_rule)
        item_datetime= get_item_datetime(item_page,datetime_rule,datetime1_rule)
        print(item_datetime)
connection.close()