Python-сообщество

 import requests
import pymysql
import dateparser
import time
import schedule
from bs4 import BeautifulSoup
# < Логер для подсчета времени работы скрипта
created = time.strftime('%Y-%m-%d %H:%M:%S')
start_time = time.time()
# < Получаем html код.
def get_html(url):
    r = requests.get(url)
    return r.text
# < Получаем ссылки.
def get_resource_links(resource_page,links_rule,resource_domain):
    resource_links = []
    soup = BeautifulSoup(resource_page,'lxml')
    resource_links_blocks = soup.find_all(links_rule[0],{links_rule[1]:links_rule[2]})
    for resource_link_block in resource_links_blocks:
        a_tag = resource_link_block .find("a")
        if a_tag:
            link = a_tag.get("href")
            resource_links.append(resource_domain + link)
    return resource_links
# < Собираем заголовки с страницы.
def get_item_title(item_page,title_rule):
    soup = BeautifulSoup(item_page, 'lxml')
    item_title = soup.find(title_rule[0],{title_rule[1]:title_rule[2]})
    return item_title['content']
# < Собираем даты с страницы.
def get_item_datetime(item_page,datetime_rule,datetime1_rule):
    soup = BeautifulSoup(item_page, 'lxml')
    item_datetime = soup.find(datetime_rule[0],{datetime_rule[1]:datetime_rule[2]})
    if item_datetime is not None:
        item_datetime = soup.find(datetime_rule[0],{datetime_rule[1]:datetime_rule[2]}).text
        item_datetime = dateparser.parse(item_datetime, date_formats=['%d %B %Y %H'])
    else:
        if (len(datetime1_rule) == 3):
            item_datetime = soup.find(datetime1_rule[0],{datetime1_rule[1]:datetime1_rule[2]}).text
            item_datetime = dateparser.parse(item_datetime, date_formats=['%d %B %Y %H'])
        else:
            item_datetime = ''
    return item_datetime
# < Собираем контент с страницы.
def get_text_content(item_page,text_rule,text1_rule):
    soup = BeautifulSoup(item_page, 'lxml')
    item_text = soup.find(text_rule[0],{text_rule[1]:text_rule[2]})
    if item_text is not None:
        item_text = soup.find(text_rule[0],{text_rule[1]:text_rule[2]}).text
    else:
        if (len(text1_rule) == 3):
            item_text = soup.find(text1_rule[0],{text1_rule[1]:text1_rule[2]}).text
        else:
            item_text = ''
    return item_text
# < Подключение к базе данных.
connection = pymysql.connect(host = 'localhost',
                             user = 'root',
                             password = '',
                             db = 'news_portal',
                             charset = 'utf8',
                             autocommit = True)
cursor = connection.cursor()
# < Запрос правил выдергивания из таблицы resource контента.
cursor.execute('SELECT * FROM `resource`')
resources = cursor.fetchall()
# < Вызов всех функций.
def call_all_func(resources):
    # < Цикл для перебора из кортежа.
    for resource in resources:
        resource_name = resource[1]
        resource_link = resource[2]
        resource_url = resource[3]
        link_rule = resource[4]
        title_rule = resource[5]
        datetime_rule = resource[6]
        datetime1_rule = resource[7]
        text_rule = resource[8]
        text1_rule = resource[9]
        print(resource_name)
        resource_domain=resource_link
        # < Разбиваю данные из кортежа в массив.
        links_rule = link_rule.split(',')
        title_rule = title_rule.split(',')
        datetime_rule = datetime_rule.split(',')
        datetime1_rule = datetime1_rule.split(',')
        text_rule = text_rule.split(',')
        text1_rule = text1_rule.split(',')
        resource_page = get_html(resource_url)
        resource_links = get_resource_links(resource_page,links_rule,resource_domain)
        print('кол-во ссылок: '+str(len(resource_links)))
        # < Цикл для вызова функции.
        for resource_link in resource_links:
            item_page = get_html(resource_link)
            item_title = get_item_title(item_page,title_rule)
            item_datetime = get_item_datetime(item_page,datetime_rule,datetime1_rule)
            item_text_content = get_text_content(item_page,text_rule,text1_rule)
            try:
                # < Запись новостей в БД.
                sql = "insert into items (`item_link`,`item_title`,`item_datetime`,`item_text_content`) values (%s,%s,%s,%s)"
                cursor=connection.cursor()
                cursor.execute(sql,(str(resource_link),str(item_title),str(item_datetime),str(item_text_content)))
                print('Запись в базу данных успешно завершена!')
            except pymysql.err.IntegrityError:
                print('ah shit ! duplicate error!')
                break
            except pymysql.err.InternalError:
                print('ah shit ! error')
                break
call_all_func(resources)
schedule.every(120).seconds.do(call_all_func, resources)
while True:
    schedule.run_pending()
    time.sleep(1)
# < Логер для подсчета времени работы скрипта
duration = time.time()-start_time # время работы в секундах,
finished = time.strftime('%Y-%m-%d %H:%M:%S')
sql = "insert into parser_logs (`created`,`finished`,`duration`) values (%s,%s,%s)"
cursor=connection.cursor()
cursor.execute(sql,(str(created),str(finished),str(duration)))
print('Запись логов успешно завершена!')
connection.close()

  schedule.run_pending()

 with connection.cursor() as cursor:
    ....

 if  not conn.open:
    ....

 Traceback (most recent call last):
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 639, in _update_chunk_length
    self.chunk_left = int(line, 16)
ValueError: invalid literal for int() with base 16: b''
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 397, in _error_catcher
    yield
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 704, in read_chunked
    self._update_chunk_length()
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 643, in _update_chunk_length
    raise httplib.IncompleteRead(line)
http.client.IncompleteRead: IncompleteRead(0 bytes read)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\models.py", line 750, in generate
    for chunk in self.raw.stream(chunk_size, decode_content=True):
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 527, in stream
    for line in self.read_chunked(amt, decode_content=decode_content):
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 732, in read_chunked
    self._original_response.close()
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\contextlib.py", line 130, in __exit__
    self.gen.throw(type, value, traceback)
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\urllib3\response.py", line 415, in _error_catcher
    raise ProtocolError('Connection broken: %r' % e, e)
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "C:/Users/Администратор/PycharmProjects/Task/sql.py", line 121, in <module>
    call_all_func(resources)
  File "C:/Users/Администратор/PycharmProjects/Task/sql.py", line 104, in call_all_func
    item_page = get_html(resource_link)
  File "C:/Users/Администратор/PycharmProjects/Task/sql.py", line 14, in get_html
    r = requests.get(url)
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\api.py", line 60, in request
    return session.request(method=method, url=url, **kwargs)
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\sessions.py", line 686, in send
    r.content
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\models.py", line 828, in content
    self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
  File "C:\Users\Администратор\AppData\Local\Programs\Python\Python37-32\lib\site-packages\requests\models.py", line 753, in generate
    raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
Process finished with exit code 1