class Parser(threading.Thread):
    def __init__(self,config):
        threading.Thread.__init__(self)
        self.config = config

    def run(self):
        while True:
            try: item = queue.get_nowait()
            except: break

            try: self.posting(item)
            except: traceback.print_exc()
            time.sleep(0.5)

            queue.task_done()
    def append_to_base(self,links):
        #with mutex:
            for link in links:
                #print link
                link = link.replace("&","&")
                if not "http"==link[:4]: continue
                o = urlparse(link)
                host = o.netloc
                query = "SELECT * FROM main WHERE LOCATE('{0}',link);".format(esc_str(host))
                self.db.query(query)
                if not self.db.store_result().num_rows():
                    print link,"has been added."
                    query = "INSERT INTO main (link,status) VALUES('{0}','{1}')".format(esc_str(link),0)
                    self.db.query(query)
    def posting(self,row):
        t_id,link,status = row[0]
        print link,"start."
        url = link
        post = Post()
        the_page = post.get(url)

        for regex in self.regulars:
            if not the_page: continue
            links = re.findall(regex,the_page)
            self.append_to_base(links)
        #with mutex:
            query = "UPDATE main set status='1' where id='{0}';".format(t_id)
            self.db.query(query)
def main():
    config = ConfigParser.RawConfigParser()
    config.read('settings.cfg')
    threads = int(config.get("Basic","threads"))
    regulars = [x.strip() for x in open(config.get("Basic","regulars"))]
    args = dict(config.items('MySQL'))
    db = _mysql.connect(host=args['host'], user=args['user'], passwd=args['passwd'], db=args['db'])
    while True:
        query_string = "SELECT * FROM main WHERE status='0' LIMIT 0,1000;"
        db.query(query_string)
        res = db.store_result()
        if res.num_rows()<1:break
        for i in xrange(res.num_rows()):
            queue.put(res.fetch_row()) # заносим данные в очередь
            #break
        for i in xrange(threads):
            t = Parser(config) # создаем нить
            t.db = db
            t.regulars = regulars
            t.start() # стартуем
            time.sleep(0.1)
        queue.join()
        print "Next round"
    print "Done"
if __name__ == '__main__':
    main()