本文实例讲述了python实现批量下载新浪博客的方法。分享给大家供大家参考。具体实现方法如下：
# coding=utf-8
import urllib2
import sys, os
import re
import string
from beautifulsoup import beautifulsoup
def encode(s):
return s.decode(‘utf-8’).encode(sys.stdout.encoding, ‘ignore’)
def gethtml(url):
#proxy_handler = urllib2.proxyhandler({‘http’:’http://211.138.124.211:80′})
#opener = urllib2.build_opener(proxy_handler)
#urllib2.install_opener(opener)
req = urllib2.request(url)
response = urllib2.urlopen(req, timeout=15)
return beautifulsoup(response, convertentities=beautifulsoup.html_entities)
def visible(element):
”’抓取可见的文本元素”’
if element.parent.name in [‘style’, ‘script’, ‘[document]’, ‘head’, ‘title’]:
return false
elif re.match(”, str(element)):
return false
elif element == u’\xa0′:
return false
return true
def delreturn(element):
”’删除元素内的换行”’
return re.sub(‘(?”|\xa0]’, ”, filename)
def writetofile(text, filename, dirname):
if not os.path.exists(dirname):
os.makedirs(dirname)
print encode(‘保存到目录’), dirname
filename = validfilename(filename)
print encode(‘保存文章’), filename
path = os.path.join(dirname, filename)
if not os.path.exists(path):
f = open(path, ‘w’)
f.write(text)
f.close()
else:
print filename, encode(‘已经存在’)
def formatcontent(url, title=”):
”’格式化文章内容”’
page = gethtml(url)
content = page.find(‘p’, {‘class’:’articalcontent’})
art_id = re.search(‘blog_(\w+)\.html’, url).group(1)
blog_name = page.find(‘span’, ).string
if title == ”:
title = page.find(‘h2′, ^t_’)).string
temp_data = filter(visible, content.findall(text=true)) # 去掉不可见元素
temp_data = ”.join(map(delreturn, temp_data)) # 删除元素内的换行符
temp_data = temp_data.strip() # 删除文章首尾的空行
temp_data = re.sub(‘\n{2,}’, ‘\n\n’, temp_data) # 删除文章内过多的空行
# 输出到文件
# 编码问题
temp_data = ‘本文地址:’.decode(‘utf-8’) + url + ‘\n\n’ + temp_data
op_text = temp_data.encode(‘utf-8’)
op_file = title + ‘_’ + art_id +’.txt’
writetofile(op_text, op_file, blog_name)
def articlelist(url):
articles = {}
page = gethtml(url)
pages = page.find(‘ul’, {‘class’:’sg_pages’}).span.string
page_num = int(re.search(‘(\d+)’, pages).group(1))
for i in range(1, page_num+1):
print encode(‘生成第%d页文章索引’%i)
if i != 1:
url = re.sub(‘(_)\d+(\.html)$’, ‘\g’+str(i)+’\g’, url)
page = gethtml(url)
article = page.findall(‘span’, {‘class’:’atc_title’})
for art in article:
art_title = art.a[‘title’]
art_href = art.a[‘href’]
articles[art_title] = art_href
return articles
def blog_dld(articles):
if not isinstance(articles, dict):
return false
print encode(‘开始下载文章’)
for art_title, art_href in articles.items():
formatcontent(art_href, art_title)
if __name__ == ‘__main__’:
sel = raw_input(encode(‘你要下载的是(1)全部文章还是(2)单篇文章，输入1或者2: ‘))
if sel == ‘1’:
#articlelist_url = ‘http://blog.sina.com.cn/s/articlelist_1303481411_0_1.html’
articlelist_url = raw_input(encode(‘请输入博客文章目录链接: ‘))
articles = articlelist(articlelist_url)
blog_dld(articles)
else:
#article_url = ‘http://blog.sina.com.cn/s/blog_4db18c430100gxc5.html’
article_url = raw_input(encode(‘请输入博客文章链接: ‘))
formatcontent(article_url)
希望本文所述对大家的python程序设计有所帮助。