for url, etag, l_mod in rss_urls:
    rss_feed = process_rss(url, etag, l_mod) # Read url with last etag, l_mod values
    if not rss:
        continue
        
    for new_item in rss_feed: # Iterate via *new* items in feed
        element = fetch_content(new_item) # Direct https request, download HTML source
        if not element:
            continue
            
        images = extract_images(element)
        goodImages = []
        for img in images:
            if img_qualify(img): # Download and analyze image if it could be used as a thumbnail
                goodImages.append(img)

from gevent import  queue

wqueue = queue.Queue()
for i in range(10):
    wqueue.put(('first_task', i, wqueue)) # очередь сюда передается, чтобы задача могла добавлять туда задачи второго этапа

from gevent import pool
wpool = pool.Pool(max_workers)

workers = {"first_task": do_first, "second_task": do_second}

    # dispatch tasks
    while True:
        if not wpool.full():
            while True:
                try:
                    task = wqueue.get_nowait()
                except queue.Empty:
                    break
                LOG.debug("do_it: starting task %s" % task[0])
                wpool.spawn(workers[task[0]], task)
            if wpool.free_count() == max_workers:
               break
            wpool.join()

        gevent.sleep(1)

def do_first(task):
    task_id, wqueue = task[1:]
    LOG.debug("do_first %d", task_id)
    # ... do something useful here 
    # add second step tasks to the same queue
    for i in range(randint(1, 10)):
        wqueue.put(("second_task", "%d-%d" % (task_id, i)))
    gevent.sleep(randint(1, 10))
    LOG.debug("do_first %d done" % task_id)

def do_second(task):
    task_id = task[1]
    LOG.debug("do_second %s", task_id)
    gevent.sleep(randint(1, 2))
    LOG.debug("do_second %s", task_id)

import gevent
import logging as log
import time
import uuid

from random import randint
from gevent import queue, pool, monkey
from functools import wraps

monkey.patch_all()

log.basicConfig(filename='runtime.txt', level=log.DEBUG)

start_time = time.time()
jobs_done = 0
jobs_processed = 0

def with_timeout(timeout):
    def factory(fn):
        @wraps(fn)
        def decorator(task):
            global jobs_done, jobs_processed # just for testing purpose, will gone in production
            try:
                jobs_done += 1 
                jobs_processed += 1
                with gevent.Timeout(timeout):
                    return fn(task)
            except gevent.Timeout:
                jobs_done -= 1
                log.debug("Task `{0}:{1}` timed out".format(task[1], task[2]))
        return decorator
    return factory

@with_timeout(6) 
def get_feed(task):
    task_id, job_queue = task[2:4]
    log.debug("get_feed {0} downloading...".format(task_id))
    # Feed downloaded, new items extracted
    for i in range(randint(1, 10)):
        job_queue.put((2, 'get_item', uuid.uuid4(), job_queue))
    gevent.sleep(randint(0, 5))
    log.debug('get_feed {0} done'.format(task_id))

@with_timeout(3)
def get_item(task):
    task_id, job_queue = task[2:4]
    log.debug("get_item {0} downloading...".format(task_id))
    # Item downloaded, images extracted
    for i in range(randint(1, 50)):
        job_queue.put((1, 'get_image', uuid.uuid4(), job_queue))
    gevent.sleep(randint(0, 5))
    log.debug('get_item {0} done'.format(task_id))

@with_timeout(3)
def get_image(task):
    task_id, job_queue = task[2:4]
    log.debug("get_image {0} downloading...".format(task_id))
    # Downloading image...
    gevent.sleep(randint(0, 5))
    log.debug('get_item {0} done'.format(task_id))

MAX_WORKERS = 100
workers = {
    'get_feed': get_feed,
    'get_item': get_item,
    'get_image': get_image,
}

job_queue = queue.PriorityQueue()
job_pool = pool.Pool(MAX_WORKERS)

# add all feeds to queue
for i in xrange(5):
    job_queue.put((3, 'get_feed', uuid.uuid4(), job_queue))

def dispatch():
    while True:
        if not job_pool.full():
            while True:
                try:
                    task = job_queue.get_nowait()
                except queue.Empty:
                    break
                log.debug("Starting task {0}:{1}".format(task[1], task[2]))
                job_pool.spawn(workers[task[1]], task)
            if job_pool.free_count() == MAX_WORKERS:
                break
            job_pool.join()

        gevent.sleep(1)

dispatch()
log.debug("Total jobs created: {0}".format(jobs_processed))
log.debug("Total jobs done: {0}".format(jobs_done))
log.debug("Processed in {:.3} sec".format(time.time() - start_time))