Read Lenta.ru: Новости on your iPad or Kindle in no time. Click download to load the free ebook on your reader.
Check out all the available public recipes or write your own with these quick start guides. ReadBeam is built on calibre, so everything in the docs and the fora applies here as well.
Ежедневная интернет-газета. Новости со всего мира на русском языке
Language: ru
Requires Subscription: No, it's available as free ebook
Schedule Every morning
ReadBeam
#!/usr/bin/env python
'''
Lenta.ru
'''
from calibre.web.feeds.feedparser import parse
from calibre.ebooks.BeautifulSoup import Tag
from calibre.web.feeds.news import BasicNewsRecipe
import re
class LentaRURecipe(BasicNewsRecipe):
title = u'Lenta.ru: \u041d\u043e\u0432\u043e\u0441\u0442\u0438'
__author__ = 'Nikolai Kotchetkov'
publisher = 'lenta.ru'
category = 'news, Russia'
description = u'''\u0415\u0436\u0435\u0434\u043d\u0435\u0432\u043d\u0430\u044f
\u0438\u043d\u0442\u0435\u0440\u043d\u0435\u0442-\u0433\u0430\u0437\u0435\u0442\u0430.
\u041d\u043e\u0432\u043e\u0441\u0442\u0438 \u0441\u043e
\u0432\u0441\u0435\u0433\u043e \u043c\u0438\u0440\u0430 \u043d\u0430
\u0440\u0443\u0441\u0441\u043a\u043e\u043c
\u044f\u0437\u044b\u043a\u0435'''
description = u'Ежедневная интернет-газета. Новости со всего мира на русском языке'
oldest_article = 3
max_articles_per_feed = 100
masthead_url = u'http://img.lenta.ru/i/logowrambler.gif'
cover_url = u'http://img.lenta.ru/i/logowrambler.gif'
#Add feed names if you want them to be sorted (feeds of this list appear first)
sortOrder = [u'_default', u'В России', u'б.СССР', u'В мире']
encoding = 'cp1251'
language = 'ru'
no_stylesheets = True
remove_javascript = True
recursions = 0
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
keep_only_tags = [dict(name='td', attrs={'class':['statya','content']})]
remove_tags_after = [dict(name='p', attrs={'class':'links'}), dict(name='div', attrs={'id':'readers-block'})]
remove_tags = [dict(name='table', attrs={'class':['vrezka','content']}), dict(name='div', attrs={'class':'b240'}), dict(name='div', attrs={'id':'readers-block'}), dict(name='p', attrs={'class':'links'})]
feeds = [u'http://lenta.ru/rss/']
extra_css = 'h1 {font-size: 1.2em; margin: 0em 0em 0em 0em;} h2 {font-size: 1.0em; margin: 0em 0em 0em 0em;} h3 {font-size: 0.8em; margin: 0em 0em 0em 0em;}'
def parse_index(self):
try:
feedData = parse(self.feeds[0])
if not feedData:
raise NotImplementedError
self.log("parse_index: Feed loaded successfully.")
if feedData.feed.has_key('title'):
self.title = feedData.feed.title
self.log("parse_index: Title updated to: ", self.title)
if feedData.feed.has_key('image'):
self.log("HAS IMAGE!!!!")
def get_virtual_feed_articles(feed):
if feeds.has_key(feed):
return feeds[feed][1]
self.log("Adding new feed: ", feed)
articles = []
feeds[feed] = (feed, articles)
return articles
feeds = {}
#Iterate feed items and distribute articles using tags
for item in feedData.entries:
link = item.get('link', '');
title = item.get('title', '');
if '' == link or '' == title:
continue
article = {'title':title, 'url':link, 'description':item.get('description', ''), 'date':item.get('date', ''), 'content':''};
if not item.has_key('tags'):
get_virtual_feed_articles('_default').append(article)
continue
for tag in item.tags:
addedToDefault = False
term = tag.get('term', '')
if '' == term:
if (not addedToDefault):
get_virtual_feed_articles('_default').append(article)
continue
get_virtual_feed_articles(term).append(article)
#Get feed list
#Select sorted feeds first of all
result = []
for feedName in self.sortOrder:
if (not feeds.has_key(feedName)): continue
result.append(feeds[feedName])
del feeds[feedName]
result = result + feeds.values()
return result
except Exception, err:
self.log(err)
raise NotImplementedError
def preprocess_html(self, soup):
return self.adeify_images(soup)
def postprocess_html(self, soup, first_fetch):
#self.log('Original: ', soup.prettify())
contents = Tag(soup, 'div')
#Extract tags with given attributes
extractElements = {'div' : [{'id' : 'readers-block'}]}
#Remove all elements that were not extracted before
for tag, attrs in extractElements.iteritems():
for attr in attrs:
garbage = soup.findAll(tag, attr)
if garbage:
for pieceOfGarbage in garbage:
pieceOfGarbage.extract()
#Find article text using header
#and add all elements to contents
element = soup.find({'h1' : True, 'h2' : True})
if (element):
element.name = 'h1'
while element:
nextElement = element.nextSibling
element.extract()
contents.insert(len(contents.contents), element)
element = nextElement
#Place article date after header
dates = soup.findAll(text=re.compile('\d{2}\.\d{2}\.\d{4}, \d{2}:\d{2}:\d{2}'))
if dates:
for date in dates:
for string in date:
parent = date.parent
if (parent and isinstance(parent, Tag) and 'div' == parent.name and 'dt' == parent['class']):
#Date div found
parent.extract()
parent['style'] = 'font-size: 0.5em; color: gray; font-family: monospace;'
contents.insert(1, parent)
break
#Place article picture after date
pic = soup.find('img')
if pic:
picDiv = Tag(soup, 'div')
picDiv['style'] = 'width: 100%; text-align: center;'
pic.extract()
picDiv.insert(0, pic)
title = pic.get('title', None)
if title:
titleDiv = Tag(soup, 'div')
titleDiv['style'] = 'font-size: 0.5em;'
titleDiv.insert(0, title)
picDiv.insert(1, titleDiv)
contents.insert(2, picDiv)
body = soup.find('td', {'class':['statya','content']})
if body:
body.replaceWith(contents)
#self.log('Result: ', soup.prettify())
return soup