#!/usr/bin/env python2.7
#-*- coding: utf-8 -*-
import sys, time, urllib2
def chckdomain(listdomains, ftrue, ferror):
	headers={'User-agent':'Googlebot/2.1 (+http://www.googlebot.com/bot.html)'}
	while True:
		try:
			domain = listdomains.get()
			if not domain:
				sys.exit()
			
			request = urllib2.Request(domain, None, headers)
			request.get_method = lambda : 'HEAD'
			data = urllib2.urlopen(request).info().get('X-Powered-By', '')
			if data:
				ftrue.write(data+";;"+domain+";;true\n") 
			else:
				ftrue.write("0;;"+domain+";;false\n") 
		except urllib2.HTTPError, e:
			ftrue.write("0;;"+domain+";;"+str(e.code)+";;error\n") )
			ferror.flush()
			del  e
		except Exception, e:
			ferror.write("0;;"+domain+";;"+str(e)+"\n")
			ferror.flush()
			del e
		ftrue.flush()
		
		del domain
if __name__ == '__main__':
	from multiprocessing import Process, Queue
	listdomains = Queue(0)
	for domain in open(sys.argv[1], 'r'):
		listdomains.put(domain.rstrip())
	print "crawler"
	for idz in xrange(int(sys.argv[2])): #requests
		Process(target=chckdomain, args=(listdomains, open('./work/out/'+str(idz)+'.txt', 'w'),open('./work/error/'+str(idz)+'.txt', 'w'),)).start()	

with open(path, "r") as file: