import urllib
from bs4 import BeautifulSoup
import re
import csv


URL = 'https://jobs.tut.by/search/vacancy?text=%D0%9B%D0%BE%D0%B3%D0%B8%D1%81%D1%82&only_with_salary=false&area=16&enable_snippets=true&clusters=true&salary='


def get_page_count(html):

    soup = BeautifulSoup(html, "lxml")
    list = soup.find('div', class_='l-cell').find_all('li')[-1].text
    return int(list)

def get_html(url):
    response = urllib.request.urlopen(url)
    return response.read()

def parse(html):
    soup = BeautifulSoup(html, "lxml")
    table = soup.find('table', class_='l l_auto')

    projects = []

    for row in table.find_all('div', class_='search-result-description__item search-result-description__item_primary'):
        nazv = row.find_all('div', class_='search-result-item__head')
        region = row.find_all('span', class_='searchresult__address')
        date = row.find_all('span', class_='b-vacancy-list-date')
        company = row.find_all('div', class_='search-result-item__company')
        salary = (row.find_all('meta', itemprop='baseSalary'))

        projects.append({
            'vacancy': nazv[0].a.text,
            'region': region[0].text,
            'date':(date[0].text),
            'company': company[0].a.text.strip(),
            # 'salary' : salary[-1]['content']
            'salary': salary
        })

    for project in projects:
        project = project
    return projects

def save(projects, path):

    with open(path, 'w') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(('Вакансии','Регион','Дата','Компания','Зарплата'))

        for project in projects:

            # writer.writerow((project['vacancy'], ', '.join(project['region']), project['date'], project['company'], project['salary']))
            writer.writerow((project['vacancy'], project['region'], project['date'], project['company'], project['salary']))

def main():
    page_count = get_page_count(get_html(URL))
    print('Всего найдено страниц %d' %page_count)

    projects = []

    for page in range(1, page_count):
       print('Parsing %d%%' %(page/page_count*100))
       projects.extend(parse(get_html(URL + '?page=%d' % page)))

    for project in projects:
        project = re.sub(r'\\xa0', ' ', str(project))
        project = re.sub(r'(\[<meta content=")', '', str(project))
        project = re.sub(r'(\" itemprop="baseSalary"/>])', '', str(project))

        print(project)

    save(projects, 'vacancies.csv')

if __name__ == '__main__':
    main()