本文实例讲述了python实现提取百度搜索结果的方法。分享给大家供大家参考。具体实现方法如下：
# coding=utf8
import urllib2
import string
import urllib
import re
import random
#设置多个user_agents，防止百度限制ip
user_agents = [‘mozilla/5.0 (windows nt 6.1; wow64; rv:23.0) gecko/20130406 firefox/23.0’, \
‘mozilla/5.0 (windows nt 6.1; wow64; rv:18.0) gecko/20100101 firefox/18.0’, \
‘mozilla/5.0 (windows; u; windows nt 6.1; en-us) applewebkit/533+ \
(khtml, like gecko) element browser 5.0’, \
‘ibm webexplorer /v0.94’, ‘galaxy/1.0 [en] (mac os x 10.5.6; u; en)’, \
‘mozilla/5.0 (compatible; msie 10.0; windows nt 6.1; wow64; trident/6.0)’, \
‘opera/9.80 (windows nt 6.0) presto/2.12.388 version/12.14’, \
‘mozilla/5.0 (ipad; cpu os 6_0 like mac os x) applewebkit/536.26 (khtml, like gecko) \
version/6.0 mobile/10a5355d safari/8536.25’, \
‘mozilla/5.0 (windows nt 6.1) applewebkit/537.36 (khtml, like gecko) \
chrome/28.0.1468.0 safari/537.36’, \
‘mozilla/5.0 (compatible; msie 9.0; windows nt 6.0; trident/5.0; theworld)’]
def baidu_search(keyword,pn):
p= {‘wd’: keyword}
res=urllib2.urlopen((“http://www.baidu.com/s?”+urllib.urlencode(p)+”&pn={0}&cl=3&rn=100”).format(pn))
html=res.read()
return html
def getlist(regex,text):
arr = []
res = re.findall(regex, text)
if res:
for r in res:
arr.append(r)
return arr
def getmatch(regex,text):
res = re.findall(regex, text)
if res:
return res[0]
return “”
def cleartag(text):
p = re.compile(u’]+>’)
retval = p.sub(“”,text)
return retval
def geturl(keyword):
for page in range(10):
pn=page*100+1
html = baidu_search(keyword,pn)
content = unicode(html, ‘utf-8′,’ignore’)
arrlist = getlist(u”(.*?)”
link = getmatch(regex,item)
url = link[0]
#获取标题
#title = cleartag(link[1]).encode(‘utf8’)
try:
domain=urllib2.request(url)
r=random.randint(0,11)
domain.add_header(‘user-agent’, user_agents[r])
domain.add_header(‘connection’,’keep-alive’)
response=urllib2.urlopen(domain)
uri=response.geturl()
print uri
except:
continue
if __name__==’__main__’:
geturl(‘python’)
希望本文所述对大家的python程序设计有所帮助。