#!/usr/bin/env python
import urllib
import re
from bs4 import BeautifulSoup
html = urllib.urlopen('http://habrahabr.ru')
print "Scraping url: %s" % html.code
bt = BeautifulSoup(html.read(), "lxml")
print bt.title
allLinks = bt.find_all('a')
counter = 0
topX = 40
for item in allLinks:
        if (re.search('class="post_title"', str(item)) and (counter < topX)):
                   print item.string 
                   counter = counter + 1

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
import sys
from lxml.html import fromstring
raw_html = open('h.html').read()
doc = fromstring(raw_html)
link_list = doc.xpath('.//a[@class="post_title"]')
if not link_list:
	print 'Can`t parse link list'
	sys.exit(0)
for link in link_list:
	print link.get('href'), link.text.encode('utf-8')

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
import sys
from lxml.html import fromstring
raw_html = open('test.html').read()
doc = fromstring(raw_html)
content_div = doc.find('.//div[@class="content html_format"]')
if not content_div:
	print 'Can`t parse content div'
	sys.exit(0)
link_list = content_div.xpath('.//a')
if not link_list:
	print 'Can`t parse link'
	sys.exit(0)
link_count = len(link_list)
for link in link_list:
	print link.get('href')