Python-сообщество

 #
from selenium import webdriver
from bs4 import BeautifulSoup as BS
def get_pages(urls):
    driver = webdriver.Firefox(executable_path='/usr/local/bin/geckodriver')
    driver.set_window_size(0,0)
    for url in urls:
        driver.get(url)
        print pars(driver.page_source)
    driver.quit()
    
def pars(html):
    soup = BS(html, 'lxml')
    return soup.find('div', class_='info-date ng-binding ng-scope').text.strip()
if __name__ == '__main__':
    urls = ['http://bus.gov.ru/pub/agency/8968/tasks/3472492',
          'http://bus.gov.ru/pub/agency/8968/plans',
          'http://bus.gov.ru/pub/agency/8968/operations/'
          ]
    get_pages(urls)

 //get_page.js
var system = require('system');
if (system.args.length < 3 ) {
 console.log('No arguments');
 slimer.exit();
}
else {
 var url = system.args[1];
 var filename = system.args[2];
 var fs = require('fs');
 var page = require('webpage').create();
 page.open(url, function (status) {
   console.log(url,':',status);
//   if (status == 'success') fs.write(filename,page.plainText,'w');
   if (status == 'success') fs.write(filename,page.content,'w');
   slimer.exit();
 });
}

 slimerjs get_page.js 'http://example.com' 'example.html'

 //get_page.js
var system = require('system');
if (system.args.length < 2 ) {
 system.stderr.write('No arguments');
 slimer.exit();
}
else {
 var url = system.args[1];
 var page = require('webpage').create();
 page.open(url, function (status) {
   system.stderr.write(url,':',status);
   //if (status == 'success') system.stdout.write(page.plainText);
   if (status == 'success') system.stdout.write(page.content);
   slimer.exit();
 });
}

 slimerjs get_page.js 'http://example.com' > example.html

 from selenium import webdriver
from bs4 import BeautifulSoup as BS
def get_pages(urls):
    driver = webdriver.Chrome()
    driver.set_window_size(0, 0)
    driver.set_window_position(1000, 0)
    for url in urls:
        driver.get(url)
        while True:
            req = pars(driver.page_source)
            if req != '-':
                break
        print(req)
    driver.quit()
def pars(html):
    soup = BS(html, 'lxml')
    try:
        r = soup.find('div', class_='info-container info-container_np').find('div', class_='info-date ng-binding ng-scope').text.strip()
        return r
    except:
        try:
            r = soup.find('div', class_='info-date').find('span', class_='ng-binding ng-scope').text.strip()
            return r
        except:
            return '-'
if __name__ == '__main__':
    urls = ['http://bus.gov.ru/pub/agency/206030/tasks/3573224',
            'http://bus.gov.ru/pub/agency/206030/plans',
            'http://bus.gov.ru/pub/agency/206030/operations/',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-F0503721',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-F0503730',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-f0503737',
            'http://bus.gov.ru/pub/agency/206030/reports',
            'http://bus.gov.ru/pub/agency/206030/measures'
            ]
    get_pages(urls)

 #!/usr/bin/env python
# coding:utf-8
from subprocess import Popen,PIPE
def get_page(url):
    child = Popen(['/usr/local/bin/slimerjs','get_page.js',url],stdout=PIPE)
    child.wait()
    out = child.stdout.read().split('\n')
    for i in out:
        s = i.decode('utf-8')
        if u'Дата публикации' in s:
            print s
            break
if __name__ == '__main__':
    urls = ['http://bus.gov.ru/pub/agency/206030/tasks/3573224',
            'http://bus.gov.ru/pub/agency/206030/plans',
            'http://bus.gov.ru/pub/agency/206030/operations/',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-F0503721',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-F0503730',
            'http://bus.gov.ru/pub/agency/206030/annual-balances-f0503737',
            'http://bus.gov.ru/pub/agency/206030/reports',
            'http://bus.gov.ru/pub/agency/206030/measures'
            ]
    for url in urls: get_page(url)

 //get_page.js
'use strict'
var system = require('system');
if (system.args.length < 2 ) {
 console.log('No arguments');
 slimer.exit();
}
else {
 var url = system.args[1];
 var page = require('webpage').create();
 page.open(url, function (status) {
   if (status == 'success') console.log(page.plainText);
   slimer.exit();
 });
}