Python-сообщество

<form class="b-domik b-domik_type_popup g-js  b-domik_type_popup_js_inited" method="POST">
<i class="b-domik__roof">
</i><table class="b-domik__shadow"><tbody><tr><td class="b-domik__shadow__lt">&nbsp;</td>
<td class="b-domik__shadow__t">
</td><td class="b-domik__shadow__rt">&nbsp;</td></tr><tr><td class="b-domik__shadow__l">&nbsp;</td><td class="b-domik__shadow__m"><div class="b-domik__form"><div class="b-domik__title">
Вход</div>
<div class="b-domik__username">
<label for="b-domik_popup-username" class="b-hint-input g-js b-hint-input_js_inited" onclick="return {name:'b-hint-input'}" style="display: block; ">
логин</label><div class="b-input">
<input class="b-input__text" id="b-domik_popup-username" name="login" value="" tabindex="11"></div></div><div class="b-domik__password">
<label for="b-domik_popup-password" class="b-hint-input g-js b-hint-input_js_inited" onclick="return {name:'b-hint-input'}">
пароль</label><div class="b-input">
<input class="b-input__text" id="b-domik_popup-password" name="passwd" value="" type="password" tabindex="11">
</div></div><div class="b-domik__permanent">
<input class="b-domik__check" id="b-domik_popup-permanent" name="twoweeks" type="checkbox" value="yes" tabindex="11"><label for="b-domik_popup-permanent">запомнить меня</label> 
<span class="b-domik__help">(<a class="b-domik__link" href="http://help.yandex.ru/passport/?id=922493" tabindex="12">что&nbsp;это</a>)</span></div><div class="b-domik__button">
<input class="b-domik__submit" type="submit" value="Войти" tabindex="11" disabled=""><input class="b-domik__submit" type="button" value="Отмена" tabindex="11"></div><div class="b-domik__register">
<a class="b-domik__link" href="http://passport.yandex.ru/passport?mode=register&amp;from=wow&amp;retpath=http%3A%2F%2Fesaltheone.ya.ru%2Fregistration.xml%3Fretpath%3Dhttp%253A%252F%252Fesaltheone.ya.ru%252F" tabindex="11">
Зарегистрироваться</a></div>
<div class="b-domik__remember"><a class="b-domik__remind" href="http://passport.yandex.ru/passport?mode=remember" tabindex="11">Напомнить пароль</a></div></div></td><td class="b-domik__shadow__r">&nbsp;</td>
</tr><tr><td class="b-domik__shadow__lb">&nbsp;</td><td class="b-domik__shadow__b"></td><td class="b-domik__shadow__rb">&nbsp;</td></tr></tbody></table><iframe class="b-popup-under" frameborder="0"></iframe><input type="hidden" name="timestamp">
</form>

# -*- coding: cp1251 -*-
import urllib2
from BeautifulSoup import *
import urlparse, os
import sqlite3 as sqlite

dbname = 'esalfr.db'
starturl = 'http://pravda-vsay.ya.ru/profile_friends_friendsmutual.xml'

connection = sqlite.connect(dbname)
cursor = connection.cursor()

cursor.execute('CREATE TABLE IF NOT EXISTS esal (id INTEGER PRIMARY KEY, name VARCHAR(256), url VARCHAR(256))')

cursor.execute('CREATE TABLE IF NOT EXISTS friends     (id INTEGER PRIMARY KEY, parent INTEGER, name VARCHAR(256), url VARCHAR(256))')
cursor.execute('CREATE TABLE IF NOT EXISTS queue       (id INTEGER PRIMARY KEY, parent INTEGER, depth INTEGER, url VARCHAR(256))')
connection.commit()

cursor.execute("INSERT INTO queue VALUES ((?), (?), (?), (?))", (None, 0, 0, starturl))
connection.commit()

page = urllib2.urlopen(starturl)
soup = BeautifulSoup(page.read(), fromEncoding="utf-8")

queue = []
for i in soup.findAll(True, 'b-yauser'):
    try:
        urlfriend = i('a')[0]['href']
        queue.append(urlfriend)
        #print urlfriend    #for 
        url = urlparse.urlparse(i('a')[0]['href'])
        splittedurl = url[1].split('.')[0]
        #print splittedurl  #scyplyaeva
        cursor.execute("INSERT INTO esal VALUES((?),(?),(?))", (None, splittedurl, urlfriend))
        connection.commit()
    except:        
        continue

queuefr = []
for stuff in queue:
	urlfr = urlparse.urlparse(stuff)
	splfr = urlfr[1].split('.')[0]
	cursor.execute("SELECT id FROM esal WHERE name=?", [splfr])
	parentid = cursor.fetchone()
	
	urlstuff = stuff + 'profile_friends_friendsmutual.xml'
	print 'indexing friends of %s' % urlstuff
	page = urllib2.urlopen(urlstuff)
	soup = BeautifulSoup(page.read(), fromEncoding="utf-8")
	for i in soup.findAll(True, 'b-yauser'):
	    try:
	        urlfriend = i('a')[0]['href']
	        queuefr.append(urlfriend)
	        #print 'fr %s' %s urlfriend    #http://scyplyaeva.ya.ru/
	        url = urlparse.urlparse(urlfriend)
	        splittedurl = url[1].split('.')[0]
	        print '       %s has parentid %s' % (splittedurl, parentid)   #scyplyaeva
	        cursor.execute("INSERT INTO friends VALUES( (?), (?), (?), (?) )", (None, parentid, splittedurl, urlfriend))
	        connection.commit()
	    except:        
	        continue

con = sqlite.connect(dbname)
with open('dump.sql', 'w') as f:
    for line in con.iterdump():
        f.write('%s\n' % line)

# build opener with HTTPCookieProcessor
o = urllib2.build_opener( urllib2.HTTPCookieProcessor() )
urllib2.install_opener(o)
 
# assuming the site expects 'user' and 'pass' as query params
p = urllib.urlencode({ 'username': 'me', 'password': 'mypass' })
 
# perform login with params
f = o.open('http://www.mysite.com/login/',  p)
data = f.read()
f.close()
 
# second request should automatically pass back any
# cookies received during login... thanks to the HTTPCookieProcessor
f = o.open( 'http://www.mysite.com/protected/area/' )
data = f.read()

# -*- coding: cp1251 -*-
import urllib
import urllib2
import re
import cookielib
import urlparse
cookieJar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
headers = {"User-Agent":"Mozilla/5.0 (X11; U; Linux i686; ru; rv:1.9.2.6) Gecko/20100628 Ubuntu/10.04 (lucid) Firefox/3.6.6", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Keep-Alive": "115", "Connection":"keep-alive"}
account=['XXXXXX', '**************'] 
params = urllib.urlencode({'login':account[0], 'passwd':account[1]})
req = urllib2.Request('http://pda-passport.yandex.ru/passport?mode=mauth', params, headers)
opener.open(req)
keywordregex = re.compile('<span class="b-yauser"><a\shref="(.*?)">')
req = urllib2.Request('http://esaltheone.ya.ru/profile_friends_friendsmutual.xml', headers=headers)
f=opener.open(req)
data=f.read()
keywordlist = keywordregex.findall(data)
for k in keywordlist:
    print(k)