Python-сообщество

from selenium import webdriver
driver = webdriver.Chrome()
driver.get("http://www.neopoliscasa.ru/catalog.html")
categories = driver.find_elements_by_xpath(
    '//div[contains(@class, "listFilter")]/ul/li')[1:]
for category in categories:
    category.click()
    url = driver.current_url
    print(url)

from selenium import webdriver
import time
driver = webdriver.Chrome()
driver.get("http://www.neopoliscasa.ru/catalog.html")
categories = driver.find_elements_by_xpath(
    '//div[contains(@class, "listFilter")]/ul/li')[1:]
category_links = []
for category in categories:
    category.click()
    time.sleep(3)
    category_links.append(driver.current_url)
print(category_links)

import requests
import bs4
CATALOG_URLS = [
    "http://www.neopoliscasa.ru/catalog/mebel-gostinaya.html",
    "http://www.neopoliscasa.ru/catalog/mebel-spalna.html",
    "http://www.neopoliscasa.ru/catalog/mebel-stolovaya.html",
]
BASE_URL = "http://www.neopoliscasa.ru"
categories = set()
def page_url(base, page):
    """Prepare page URL.
        Example : (/krovat.html, 7) to /krovat_7.html
    """
    return "{}_{}.html".format(base.rsplit('.', 1)[0], page)
def parse_single_category_page(url):
    """Parses single page and returns list of product links"""
    page = requests.get(url).content
    soup = bs4.BeautifulSoup(page)
    catalog_items = soup.find_all("div", class_="catalogItem")
    return [item.find("a")["href"] for item in catalog_items]
def parse_furniture_dir(url):
    """Parse base catalog URL and add it to global
        categories set
    """
    page = requests.get(url).content
    soup = bs4.BeautifulSoup(page)
    for category in soup.find_all("div", class_="itemTypeIcoon"):
        categories.add(category.find("a")["href"])
def parse_category_dir(category):
    """Parse items in category and returns its
        URLs list
    """
    category_base_url = "{}{}".format(BASE_URL,
                                      category)
    page = requests.get(category_base_url).\
        content
    soup = bs4.BeautifulSoup(page)
    pages_count_ = soup.find('ul', class_="pagesContainer")
    pages_count = int(pages_count_.find_all('li')[-2].getText())
    products_list = parse_single_category_page(category_base_url)
    for page in range(2, pages_count):
        products_list += parse_single_category_page(page_url(category_base_url,
                                                             page))
    return products_list
for base_category_url in CATALOG_URLS:
    parse_furniture_dir(base_category_url)
for category in categories:
    for page in parse_category_dir(category):
        print(page)