Python-сообщество

 def extract_data(www,link):
	soup = BeautifulSoup(www)
	
	name = soup.find('h1',{'class':'name'}).text.strip()
	# off_name = 
	try:
		size = soup.find('li',{'class':'company-size'}).find('p').text.strip()
	except:
		size = ''
	try:
		country = soup.find('li',{'class':'vcard hq'}).find('span',{'class':'country-name'}).text.strip()
	except:
		country = ''
	try:
		address = soup.find('li',{'class':'vcard hq'}).find('span',{'class':'street-address'}).text.strip()
	except:
		address = ''
	try:
		c1 = soup.find('li',{'class':'vcard hq'}).find('span',{'class':'locality'}).text.strip().replace(',','')
	except:
		c1 = ''
	try:
		c2 = soup.find('li',{'class':'vcard hq'}).find('abbr',{'class':'region'}).text.strip()
	except:
		try:
			c2 = soup.find('li',{'class':'vcard hq'}).find('span',{'class':'region'}).text.strip()
		except:
			c2 = ''
               try:
                        specialities  = soup.find('li',{'class':'specialties'}).find('p').text.strip()
                  except:
                          specialties = ''
	try:
		industry = soup.find('li',{'class':'industry'}).find('p').text.strip()
	except:
		industry = ''
	try:
		web = soup.find('li',{'class':'website'}).find('a').text.strip()
	except:
		web = ''
	try:
		zip_code = soup.find('li',{'class':'vcard hq'}).find('span',{'class':'postal-code'}).text.strip()
	except:
		zip_code = ''
	try:
		desc = soup.find('div',{'class':'basic-info-description'}).text.strip()
	except:
		desc = ''
	# print([link, name, size, country, address, c1, c2, specialties, industry, web, zip_code, desc])
	return [link, name, size, country, address, c1, c2, specialties, industry, web, zip_code, desc]

  
def find(tag, attrs, tag1, attrs1):
   try:
       return soup.find(tag, attrs).find(tag1, attrs1).text.strip()
   except:
       return ""

  
c2 = find('li',{'class':'vcard hq'}, 'span',{'class':'region'})
zip_code = find('li',{'class':'vcard hq'}, 'span',{'class':'postal-code'})
# и так далее

 with open("scraped.pkl","wb") as p:
		pickle.dump(out, p)
	out2 = []
	for k in out:
		out2.append(k.split('?')[0])
	out3 = list(set(out2))
	with open('companies.txt','w',encoding='utf-8') as f:
		f.write('\n'.join(out3))
except Exception as ex:
	print(ex)
	print(traceback.format_exc())
	with open("err result.pkl","wb") as p:
		pickle.dump(out, p)
		print('error dump OK')
		input()
		
finally:
	driver.quit()

 from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import unittest, time, re
import traceback
import os
import pickle
import csv
from bs4 import BeautifulSoup
import pprint
import random
from account import settings
links = [i.strip() for i in open('links.txt')]
random.shuffle(links)
try:
	if 1:
		driver = webdriver.Chrome()
		driver.set_window_size(1200, 800)
		driver.implicitly_wait(4)
	else:
		profile=webdriver.FirefoxProfile()
		profile.set_preference("intl.accept_languages", "en")
		firefox_capabilities = DesiredCapabilities.FIREFOX
		firefox_capabilities['marionette'] = True
		driver = webdriver.Firefox(firefox_profile=profile,capabilities=firefox_capabilities)
		driver.set_window_size(1300, 1000)
		driver.implicitly_wait(30)
	driver.get("http://www.linkedin.com")
	time.sleep(3)
	emailBox = driver.find_element_by_xpath("//input[@id='login-email']")
	emailBox.send_keys(settings['login'])
	time.sleep(2)
	passwordBox = driver.find_element_by_xpath("//input[@id='login-password']")
	passwordBox.send_keys(settings['pass'])
	time.sleep(2)
	passwordBox.submit()
	time.sleep(5)
	
	out = []
	cntr = 1
	for link in links:
		print('Link %s/%s' % (cntr, len(links)))
		cntr += 1
		buf = []
		driver.get(link)
		time.sleep(3)
		pcnt = 1
		while True:
			pag = driver.find_elements_by_xpath("//div[@id='results-pagination']/ul[@class='pagination']/li[@class='next']")
			print("Page: %s" % pcnt)
			pcnt += 1
			if len(pag) > 0:
				print('next page button')
				urls = driver.find_elements_by_xpath("//div[@id='results-container']/ol[@id='results']/li/div[@class='bd']/h3/a")
				if urls:
					hrefs = [i.get_attribute('href') for i in urls]
					buf += hrefs
				driver.find_element_by_xpath("//div[@id='results-pagination']/ul[@class='pagination']/li[@class='next']").click()
			else:
				print('no next page button')
				urls = driver.find_elements_by_xpath("//div[@id='results-container']/ol[@id='results']/li/div[@class='bd']/h3/a")
				if urls:
					hrefs = [i.get_attribute('href') for i in urls]
					buf += hrefs
				break
			time.sleep(3)
		out.append([link, buf])
			time.sleep(3)
	with open("scraped.pkl","wb") as p:
		pickle.dump(out, p)
	out2 = []
	for k in out:
		out2.append(k.split('?')[0])
	out3 = list(set(out2))
	with open('companies.txt','w',encoding='utf-8') as f:
		f.write('\n'.join(out3))
except Exception as ex:
	print(ex)
	print(traceback.format_exc())
	with open("err result.pkl","wb") as p:
		pickle.dump(out, p)
		print('error dump OK')
		input()
		
finally:
	driver.quit()