Python-сообщество

 # coding: utf-8
from urllib.parse import urljoin
from urllib.parse import urlsplit
from grab.spider import Spider, Task
class Parser(Spider):
    initial_urls = ['http://ellunium.ru/']
    def prepare(self):
        self.urls = []
        self.urls_404 = []
        self.urls_200 = []
        self.result_counter = 0
    def task_initial(self,grab,task):
        print('task_initial')
        try:
            links = grab.xpath_list('//a')
        except:
            print("Данные для парсинга не найдены  - %s" % task.url)
        self.domain = urlsplit(task.url)[1]
        print(task.url)
        print(links)
        for link in links:
            if ('href' in link.attrib):
                url = urljoin(task.url, link.attrib['href']).split('#')[0]
                if (not (url in self.urls) and ((urlsplit(url)[1] == self.domain) or (urlsplit(url)[1] == "www." + self.domain))):
                    self.urls.append(url)
                   
bot = Parser()
bot.run()