Python-сообщество

from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request
from workopolis.items import Node
import time
import re
class apply(Spider):
	name = "apply_spider"
	lang = ''
	rowFrom = ''
	location = ''
	title = ''
	city = ''
	description = ''
	category = ''
	job_id = ''
	driver = None
	start_urls = [
		'https://apply.firstgroupcareers.com/vacancy/find/results/'
		]
	
	url_app = 'https://apply.firstgroupcareers.com/vacancy/jobecode/description/'
	def parse(self, response):
		
		hxs = Selector(response)
		
		jobs = hxs.xpath('//*[@class="rowContainerHolder"]')
		items = []
		
		for job in jobs:
			item = Node()
			x = job
		
		item['title'] = hxs.xpath("string(.//*[contains(@class,'rowHeader')])").extract()[0]
		url = html.xpath("string(.//*[contains(@class,'rcMenu')]//a").extract()[0]
		item['job_id'] = re.search('jobId=([0-9]+)', url).groups()[0]
		item['apply_url'] = self.url_app.replace('jobecode',item['job_id'])
		g = item['title'].split('-')
		item['title'] = g[1]
		item['city'] = g[0]
						
					
		print	item['apply_url']
					
		d_url = item['apply_url']
				
				
					
		request = Request(d_url, callback=self.parse_details)
		request.meta['item'] = item
		items.append(request)
				
		for item in items:
			
			yield item
	def parse_details(self,response):
		html = Selector(response)
		item = response.meta['item']
		item['description'] = html.xpath(".//*[contains(@class,'earcu_posdescriptionnote')]").extract()[0]
		item['language'] = 'en'
		item['state'] = html.xpath(".//*[contains(@class,'jobSumValue')]").extract()[0]
		
		
		
		
		return item