#coding=utf8??
import?urllib2??
import?re??
import?MySQLdb??
import?requests
from?compiler.pycodegen?import?EXCEPT
import?random
import?time
import?datetime
from?lxml?import?etree
import?sys
reload（sys）
sys.setdefaultencoding（‘utf8‘）

#程序進行前準(zhǔn)備：要先創(chuàng)建個數(shù)據(jù)庫將1688database名字的數(shù)據(jù)庫，然后創(chuàng)建好urltable表，用sqlyong工具將店鋪網(wǎng)址都放到urltable表里面去
#將urltable表的url店鋪地址存儲到url_list列表中
#?url的格式：https://shop1470760677060.1688.com/page/offerlist.htm?spm=a261y.7663282.0.0.Su0VBS
def?get_url_list（）:
????url_list=[]?
????get_url_sql?=?‘SELECT?url?FROM?urltable;‘
????count??=?cur.execute（get_url_sql）?
????print?u‘?有??%s?個店鋪地址??‘?%?count
????urlresults?=?cur.fetchall（）??
????result=list（urlresults）?
????for?url?in?result:
????????print?url[0]
????????url_list.append（url[0]）
????return?url_list

def?get_all_goods_url（page）:
????begin?=?datetime.datetime.now（）
????
????#?如果出現(xiàn)異常，嘗試次數(shù)5次，還是錯誤，則判斷，頁碼超出范圍，停止采集。
????page?=?page
????print?u‘.................第%s頁...........‘?%page
????count?=?0
????conut_net?=?0
???
????DD?=?True
????while?DD:
????????print?‘conut_net-->‘conut_net
????????try:
????????????proxyHost?=?“proxy.abuyun.com“
????????????proxyPort?=?“9020“
????????????proxyUser?=?“H4073W6H9EJ29Z4D“
????????????proxyPass?=?“32D3D1294745B2B2“
????????????proxymeta?=?“http://%（user）s:%（pass）s@%（host）s:%（port）s“?%?{
??????????????????“host“?:?proxyHost
??????????????????“port“?:?proxyPort
??????????????????“user“?:?proxyUser
??????????????????“pass“?:?proxyPass
????????????}
????????????proxies?=?{
????????????????????“http“??:?proxymeta
????????????????????“https“?:?proxymeta
????????????}
????????????headers?=?{
????????????????#‘:authority‘:‘bertoys.1688.com‘
????????????????#‘:method‘:‘GET‘
????????????????#‘:path‘:‘/page/offerlist.htm?spm=a2615.7691456.0.0.0MNLge&tradenumFilter=false&sampleFilter=false&mixFilter=false&privateFilter=false&mobileOfferFilter=%24mobileOfferFilter&groupFilter=false&sortType=tradenumdown&pageNum=6‘
????????????????#‘:scheme‘:‘https‘
????????????????‘a(chǎn)ccept‘:‘text/htmlapplication/xhtml+xmlapplication/xml;q=0.9image/webp*/*;q=0.8‘
????????????????‘a(chǎn)ccept-encoding‘:‘gzip?deflate?sdch?br‘
????????????????‘a(chǎn)ccept-language‘:‘zh-CNzh;q=0.8‘
????????????????‘referer‘:url
????????????????‘upgrade-insecure-requests‘:‘1‘
????????????????‘user-agent‘:useragent
????????????????????????????}??
????????????????
????????????shop_url?=?url.split（‘?‘）[0]
????????????spm?=?url.split（‘?‘）[1]
??????????????????
????????????parameter?=?{
????????????????????‘spm‘:spm
????????????????????‘tradenumFilter‘:‘false‘
????????????????????‘sampleFilter‘:‘false‘
????????????????????‘mixFilter‘:‘false‘
????????????????????‘privateFilter‘:‘false‘
????????????????????‘mobileOfferFilter‘:‘$mobileOfferFilter‘
????????????????????‘groupFilter‘:‘false‘
????????????????????‘sortType‘:‘tradenumdown‘
????????????????????‘pageNum‘:str（page）
????????????????}
????????????
????????????#測試