Python-сообщество

#!/usr/bin/env python
# -*- coding: ascii -*-
#
# autor: Nikolay aka toksin
# radiotoksin@gmail.com
# http://illusory-planet.blogspot.com/

import urllib
import threading
import re
import time
from urllib import FancyURLopener

#for send correct header
class MyOpener(FancyURLopener):
	version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11'


#class request in google
class GoogleReq():
	
	def __init__(self):
		self.req = 'http://www.google.com.ua/search?q='
		
	#search by site or domain	
	def g_site(self, site):
		self.req = self.req + 'site:' + site + '+'

	#search by file type
	def g_filetype(self, ftype):
		self.req = self.req + 'filetype:' + ftype + '+'
	
	#returns formed querystring
	def get_str(self, pages=1): #pages equal to the number of pages
		result = []
		result.append(self.req[:-1] + '&client=firefox')
		try:
			if int(pages) and pages>1:
				for i in range(1,pages+1):
					result.append(self.req[:-1] + '&start=' + str(i*10) + '&client=firefox')
		except Exception, err:
			print err
		return result
	
	#private method parse links in the results of a query in Google
	def _regex(self, content):
		m = re.findall(r'<h3\sclass="r"><a\shref\=\"(http://\S*)\"\sclass=', content)
		if m:
			return m
			
	#executes a query on Google, parses the links and returns them
	def exec_req(self, listurl):
		allurl = []
		for page_url in listurl:
			try:
				myopener = MyOpener()
				content = myopener.open(page_url).read()
				allurl += self._regex(content)
			except Exception, err:
				print err
		return allurl
			
#phrase search
SEARCH = 'user'

#class creates streams for parsing pages
class Parser(threading.Thread):
	
	def __init__(self, url):
		self.url = url
		threading.Thread.__init__(self)
	
	def _parse(self):
		try:
			content = urllib.urlopen(self.url).read()
			#code to find specific information on the site
			if SEARCH in content:
				print self.url
		except Exception, err:
			print err
	
	#start thread
	def run(self):
		self._parse()


def main():
	req_str = GoogleReq()
	req_str.g_site('ua')
	req_str.g_filetype('log')
	request = req_str.get_str(1)
	res = []
	if request:
		res += req_str.exec_req(request)
		i = 0
		for link in res:
			i += 1
			if i % 10 == 0:
				time.sleep(1)
			Parser(link).start()
	
	
if __name__ == "__main__":
	main()