osc的rss不是全文输出的,不开心，所以就有了python抓取osc最新博客生成rss
# -*- coding: utf-8 -*-
from bs4 import beautifulsoup
import urllib2
import datetime
import time
import pyrss2gen
from email.utils import formatdate
import re
import sys
import os
reload(sys)
sys.setdefaultencoding(‘utf-8′)
class rssspider():
def __init__(self):
self.myrss = pyrss2gen.rss2(title=’oschina’,
link=’http://my.oschina.net’,
description=str(datetime.date.today()),
pubdate=datetime.datetime.now(),
lastbuilddate = datetime.datetime.now(),
items=[]
)
self.xmlpath=r’/var/www/myrss/oschina.xml’
self.baseurl=”http://www.oschina.net/blog”
#if os.path.isfile(self.xmlpath):
#os.remove(self.xmlpath)
def useragent(self,url):
i_headers = {“user-agent”:”mozilla/5.0 (windows nt 6.1; wow64)
applewebkit/537.36 (khtml, like gecko) chrome/36.0.1985.125 safari/537.36″,
“referer”: ‘http://baidu.com/’}
req = urllib2.request(url, headers=i_headers)
html = urllib2.urlopen(req).read()
return html
def enterpage(self,url):
pattern = re.compile(r’d{4}sd{2}sd{2}sd{2}sd{2}’)
rsp=self.useragent(url)
soup=beautifulsoup(rsp)
timespan=soup.find(‘p’,{‘class’:’blogstat’})
timespan=str(timespan).strip().replace(‘n’,”).decode(‘utf-8′)
match=re.search(r’d{4}sd{2}sd{2}sd{2}sd{2}’,timespan)
timestr=str(datetime.date.today())
if match:
timestr=match.group()
#print timestr
ititle=soup.title.string
p=soup.find(‘p’,{‘class’:’blogcontent’})
rss=pyrss2gen.rssitem(
title=ititle,
link=url,
description = str(p),
pubdate = timestr
)
return rss
def getcontent(self):
rsp=self.useragent(self.baseurl)
soup=beautifulsoup(rsp)
ul=soup.find(‘p’,{‘id’:’recentblogs’})
for li in ul.findall(‘li’):
p=li.find(‘p’)
if p is not none:
alink=p.find(‘a’)
if alink is not none:
link=alink.get(‘href’)
print link
html=self.enterpage(link)
self.myrss.items.append(html)
def saverssfile(self,filename):
finallxml=self.myrss.to_xml(encoding=’utf-8′)
file=open(self.xmlpath,’w’)
file.writelines(finallxml)
file.close()
if __name__==’__main__’:
rssspider=rssspider()
rssspider.getcontent()
rssspider.saverssfile(‘oschina.xml’)
以上所述就是本文的全部内容了，希望大家能够喜欢。