本文实例讲述了python实现爬取千万淘宝商品的方法。分享给大家供大家参考。具体实现方法如下：
import time
import leveldb
from urllib.parse import quote_plus
import re
import json
import itertools
import sys
import requests
from queue import queue
from threading import thread
url_base = ‘http://s.m.taobao.com/search?q={}&n=200&m=api4h5&
def url_get(url):
# print(‘get ‘ + url)
header = dict()
header[‘accept’] = ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8’
header[‘accept-encoding’] = ‘gzip,deflate,sdch’
header[‘accept-language’] = ‘en-us,en;q=0.8’
header[‘connection’] = ‘keep-alive’
header[‘dnt’] = ‘1’
#header[‘user-agent’] = ‘mozilla/5.0 (macintosh; intel mac os x 10_8_0) applewebkit/537.36 (khtml, like gecko) chrome/28.0.1500.71 safari/537.36’
header[‘user-agent’] = ‘mozilla/12.0 (compatible; msie 8.0; windows nt)’
return requests.get(url, timeout = 5, headers = header).text
def item_thread(cate_queue, db_cate, db_item):
while true:
try:
cate = cate_queue.get()
post_exist = true
try:
state = db_cate.get(cate.encode(‘utf-8′))
if state != b’ok’: post_exist = false
except:
post_exist = false
if post_exist == true:
print(‘cate-{}: {} already exists … ignore’.format(cate, title))
continue
db_cate.put(cate.encode(‘utf-8′), b’crawling’)
for item_page in itertools.count(1):
url = url_base.format(quote_plus(cate), item_page)
for tr in range(5):
try:
items_obj = json.loads(url_get(url))
break
except keyboardinterrupt:
quit()
except exception as e:
if tr == 4: raise e
if len(items_obj[‘listitem’]) == 0: break
for item in items_obj[‘listitem’]:
item_obj = dict(
_id = int(item[‘itemnumid’]),
name = item[‘name’],
price = float(item[‘price’]),
query = cate,
category = int(item[‘category’]) if item[‘category’] != ” else 0,
nick = item[‘nick’],
area = item[‘area’])
db_item.put(str(item_obj[‘_id’]).encode(‘utf-8’),
json.dumps(item_obj, ensure_ascii = false).encode(‘utf-8’))
print(‘get {} items from {}: {}’.format(len(items_obj[‘listitem’]), cate, item_page))
if ‘nav’ in items_obj:
for na in items_obj[‘nav’][‘navcatlist’]:
try:
db_cate.get(na[‘name’].encode(‘utf-8’))
except:
db_cate.put(na[‘name’].encode(‘utf-8′), b’waiting’)
db_cate.put(cate.encode(‘utf-8′), b’ok’)
print(cate, ‘ok’)
except keyboardinterrupt:
break
except exception as e:
print(‘an {} exception occured’.format(e))
def cate_thread(cate_queue, db_cate):
while true:
try:
for key, value in db_cate.rangeiter():
if value != b’ok’:
print(‘catethread: put {} into queue’.format(key.decode(‘utf-8’)))
cate_queue.put(key.decode(‘utf-8’))
time.sleep(10)
except keyboardinterrupt:
break
except exception as e:
print(‘catethread: {}’.format(e))
if __name__ == ‘__main__’:
db_cate = leveldb.leveldb(‘./taobao-cate’)
db_item = leveldb.leveldb(‘./taobao-item’)
orig_cate = ‘正装’
try:
db_cate.get(orig_cate.encode(‘utf-8’))
except:
db_cate.put(orig_cate.encode(‘utf-8′), b’waiting’)
cate_queue = queue(maxsize = 1000)
cate_th = thread(target = cate_thread, args = (cate_queue, db_cate))
cate_th.start()
item_th = [thread(target = item_thread, args = (cate_queue, db_cate, db_item)) for _ in range(5)]
for item_t in item_th:
item_t.start()
cate_th.join()
希望本文所述对大家的python程序设计有所帮助。