Python-сообщество

# -*- coding: utf-8 -*-
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.loader.processor import TakeFirst
from scrapy.contrib.loader import XPathItemLoader
from scrapy.selector import HtmlXPathSelector
from orphanage.items import OrphanageItem
 
 
class OrphanSpider(CrawlSpider):
    name = "detskiedomiki"
    allowed_domains = ["www.detskiedomiki.ru"]
    start_urls = ["http://www.detskiedomiki.ru/guide/child/"]
 
    rules = (
        #Rule(SgmlLinkExtractor(allow=('act=home_reg', 'act=home_zone')), follow=True),
        Rule(SgmlLinkExtractor(allow=('http://detskiedomiki.ru/?act=home_more&id=6278&z_id=3&part_id=65')), callback='parse_item'),
    )
 
    def parse_item(self, response):
        hxs = HtmlXPathSelector(response)
        l = OrphanLoader(OrphanageItem(), hxs)
 
        #
        l.add_xpath('id', "//td[text()='%s']/following-sibling::td/text()" % u"Рег. номер:")
        l.add_xpath('region', "//td[text()='%s']/following-sibling::td/text()" % u"Регион:")
        l.add_xpath('district', "//td[text()='%s']/following-sibling::td/text()" % u"Район:")
        l.add_xpath('type', "//td[text()='%s']/following-sibling::td/text()" % u"Тип учреждения:")
        l.add_xpath('name', "//td[text()='%s']/following-sibling::td/strong/text()" % u"Название:")
        return l.load_item()
[b]pipeline[/b]:
import json
class FasttorrentPipeline(object):
    def __init__(self):
        self.file = open('items.txt', 'wb')
    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + "\n"
        self.file.write(line)
        
        return item

[code python][/code]

class OrphanLoader(ItemLoader):
    default_output_processor = TakeFirst()
    ...

class OrphanLoader(ItemLoader):
    name = TakeFirst()
    ...

json.dumps(..., ensure_ascii=False)

json.dumps(..., ensure_ascii=False)

class FastTor(CrawlSpider):
    name = 'FastTor1'
    allowed_domains = ['fast-torrent.ru']
    start_urls = ['http://fast-torrent.ru/russian/']
    rules = [Rule(SgmlLinkExtractor(allow=['/russian/4.html']), 'parse_torrent')]
    def parse_torrent(self, response):
        x = HtmlXPathSelector(response)
        i=0
        torrent = FasttorrentItem()
        torrent['name'] = x.select("//tr[@class='tr-type-video']/td[@colspan='3']/h2[1]/text()").extract()
        torrent['rating'] = x.select("//tr[@class='tr-type-video']/td[@colspan='3']/div/span/ul/li").extract()
        torrent['description']=['']
        name=torrent['name']
        rating=torrent['rating']
        print (len(name))
        description=torrent['description']
        p_name=re.compile('^(\()(\w)')
        p_rating=re.compile(r'([0-9]\.\d+)')
        while i < len(name):
            if p_name.findall(name[i]):
                del name[i]
            i=i+1
        i=0
        while i < len(rating):
            if p_rating.findall(rating[i]):
                rating[i]=p_rating.findall(rating[i])
                rating[i]=re.sub('[\[\]\'\']','',str(rating[i]))
                rating[i]=re.sub('[u]','',rating[i])
            else:
                rating[i]='0 '
            i=i+1
        i=0
        while i < len(rating):
            description.append(rating[i]+' '+name[i])
            i=i+1
        return torrent