Python-сообщество

from __future__ import absolute_import
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'main.settings')
from django.conf import settings
app = Celery('main')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

BROKER_URL = 'amqp://guest:guest@localhost//'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

BROKER_URL = 'amqp://guest:guest@localhost//'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

from __future__ import absolute_import
from celery import shared_task
from my_scrapy import actions
@shared_task
def run_spider():
    actions.domain_crawl()

import scrapy
from ..items import DmozItem
class DmozSpider(scrapy.Spider):
    name = "dmoz"
    allowed_domains = ["dmoz.org"]
    start_urls = [
        # "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://localhost:8000/#/note/",
        # "http://localhost:8000/#/note/2/",
        # "http://localhost:8000/#/note/3/",
        # "http://localhost:8000/#/note/category/",
    ]
    def parse(self, response):
        # for h3 in response.xpath('//*[@id="bd-cross"]/span[3]/img').extract():
        #     yield {"desc": h3}
        #
        for a in response.xpath('//a/@href').extract():
            yield {"link": a}