Python-сообщество

 out = list(filter(None, map(str.strip, soup.get_text().splitlines())))

 main_tag = soup.find(class_=[re.compile('[Cc]ontent'), 'article',   'main',  'entry'])
p_tags_list = main_tag.find_all(['h2', 'h3', 'p', 'ul', 'figure', 'img'])
outs = list(filter(None, map(get_text_in_tag, p_tags_list)))
def get_text_in_tag(tag):
    if tag.find('img') is not None:
        return str(tag.find('img'))
    elif tag.find('li') is not None:
        return str(tag.get_text()).strip()
    else:
        return str(tag.get_text()).strip()

<article>
  <h2>text</h2>
  <p>text</p>
  <p><img src="pics.jpg"></p>
  <h2>text</h2>
  <p>text
    <ul>
      <li>
        text
      </li>
        text
      <li>
        text
      </li>
    </ul>
  </p>
  <h2>text</h2>
  <p>text</p>
</article>

 tags1 = tree.xpath("//*[starts-with(@class, 'entry')]/descendant-or-self::node()")
tags2 = tree.xpath("//div[starts-with(@class, 'content')]/descendant-or-self::node()")
tags3 = tree.xpath("//*[contains(@id, 'content')]/descendant-or-self::node()")
tags4 = tree.xpath('//h1/parent::*/descendant-or-self::node()')

  
>>> def get_text_type(text):
...     out = 'undefined'
...     if text.startswith('<article>'):
...         out = 'article'
...     elif text.startswith('<div>'):
...         out = 'div'
...     return out
... 
>>> def parse_article(text, article_type):
...     out = None
...     if article_type == 'article':
...         out = parse_article_article(text)
...     elif article_type == 'div':
...         out = parse_article_div(text)
...     return out
... 
>>> def parse_article_article(text):
...     out = {'title': text.split('\n')[0][9:], 'content': text.split('\n')[1]}
...     return out
... 
>>> def parse_article_div(text):
...     out = {'title': text.split('\n')[0][5:], 'content': text.split('\n')[1]}
...     return out
... 
>>> def process_text(text):
...     text_type = get_text_type(text)
...     if text_type != 'undefined':
...         article = parse_article(text, text_type)
...         print('{}\n  {}'.format(article['title'], article['content']))
...     else:
...         print('It is not an article.')
... 
>>> process_text('article\ncontent of article')
It is not an article.
>>> process_text('<article>Article\nContent of Article')
Article
  Content of Article
>>> process_text('div\ncontent of div')
It is not an article.
>>> process_text('<div>Div\nContent of Div')
Div
  Content of Div
>>>

     tag1 = soup.find('article')
    tag2 = soup.find('section')
    tag3 = soup.find('div', itemtype='http://schema.org/Article')
    tag4 = soup.find_all('div', class_=re.compile('[Cc]ontent'))
    tag5 = soup.find_all('div', class_=re.compile('entry'))
    tag6 = soup.find('div', id='content')
    tag7 = soup.find_all('div', class_=re.compile('container'))
    if tag1:
        if tag1.has_attr('itemtype') or tag1.find('p'):
            article_tag = tag1
    elif tag2 and tag2.find('p'):
        article_tag = tag2
    elif tag3 and tag3.find('p'):
        article_tag = tag3
    elif tag4:
        for tag in tag4:
            if len(tag.find_all('p')) > 3:
                article_tag = tag
    elif tag5:
        for tag in tag5:
            if len(tag.find_all('p')) > 3:
                article_tag = tag
    elif tag6 and tag6.find('p'):
        article_tag = tag6
    elif tag7:
        for tag in tag7:
            if len(tag.find_all('p')) > 3:
                article_tag = tag
    try:
        article_tags_list = article_tag.find_all(['h2', 'h3', 'h4', 'p', 'ul', 'ol', 'figure'])
        print(article_tag.name, article_tag.attrs)
    except:
        print('Тег статьи не определен!')

     tag1 = soup.find('article')
    tag2 = soup.find_all('section')
    tag3 = soup.find_all('div', itemtype='http://schema.org/Article')
    tag4 = soup.find_all('div', class_=re.compile('[Cc]ontent'))
    tag5 = soup.find_all('div', class_=re.compile('entry'))
    tag6 = soup.find_all('div', id='content')
    tag7 = soup.find_all('div', class_=re.compile('container'))
    tags_list = []
    tags_list.append(tag2)
    tags_list.append(tag3)
    tags_list.append(tag4)
    tags_list.append(tag5)
    tags_list.append(tag6)
    tags_list.append(tag7)
    if tag1:
        if len(tag1.find_all('p')) > 3 or tag1.has_attr(itemtype='http://schema.org/Article'):
            article_tag = tag1
    else:
        for tags in tags_list:
            if tags:
                for tag in tags:
                    if len(tag.find_all('p')) > 3:
                        article_tag = tag

 tags_list = soup.find_all('div')
print('tags_list ', len(tags_list))  # 116 тегов
for tag in tags_list:
    if len(tag.find_all('p')) > 5:
        article_tag = tag