import?datetime
import?os
import?os.path
import?random
import?sqlite3
import?threading
import?time
import?ujson
from?concurrent.futures?import?ThreadPoolExecutor

import?numpy?as?np
import?requests
from?retrying?import?retry

from?modules.ProxyProvider?import?ProxyProvider


class?Crawler:
????def?__init__（self）:
????????self.start_time?=?datetime.datetime.now（）
????????self.csv_path?=?“./db/“?+?datetime.datetime.now（）.strftime（“%Y%m%d“）
????????os.makedirs（self.csv_path?exist_ok=True）
????????self.csv_name?=?self.csv_path?+?“/“?+?datetime.datetime.now（）.strftime（“%Y%m%d-%H%M%S“）?+?‘.csv‘
????????self.db_name?=?“./temp.db“
????????self.lock?=?threading.Lock（）
????????self.proxyProvider?=?ProxyProvider（）
????????self.total?=?0
????????self.done?=?0

????def?get_nearby_bikes（self?args）:
????????try:
????????????url?=?“https://mwx.mobike.com/mobike-api/rent/nearbyBikesInfo.do“

????????????payload?=?“latitude=%s&longitude=%s&errMsg=getMapCenterLocation“?%?（args[0]?args[1]）

????????????headers?=?{
????????????????‘charset‘:?“utf-8“
????????????????‘platform‘:?“4“
????????????????“referer“:“https://servicewechat.com/wx40f112341ae33edb/1/“
????????????????‘content-type‘:?“application/x-www-form-urlencoded“
????????????????‘user-agent‘:?“MicroMessenger/6.5.4.1000?NetType/WIFI?Language/zh_CN“
????????????????‘host‘:?“mwx.mobike.com“
????????????????‘connection‘:?“Keep-Alive“
????????????????‘accept-encoding‘:?“gzip“
????????????????‘cache-control‘:?“no-cache“
????????????}

????????????self.request（headers?payload?args?url）
????????except?Exception?as?ex:
????????????print（ex）

????def?request（self?headers?payload?args?url）:
????????while?True:
????????????proxy?=?self.proxyProvider.pick（）
????????????try:
????????????????response?=?requests.request（
????????????????????“POST“?url?data=payload?headers=headers
????????????????????proxies={“https“:?proxy.url}
????????????????????timeout=5verify=False
????????????????）

????????????????with?self.lock:
????????????????????with?sqlite3.connect（self.db_name）?as?c:
????????????????????????try:
????????????????????????????print（response.text）
????????????????????????????decoded?=?ujson.decode（response.text）[‘object‘]
????????????????????????????self.done?+=?1
????????????????????????????for?x?in?decoded:
????????????????????????????????c.execute（“INSERT?INTO?mobike?VALUES?（%d‘%s‘%d%d%s%s%f%f）“?%?（
????????????????????????????????????int（time.time（））?*?1000?x[‘bikeIds‘]?int（x[‘biketype‘]）?int（x[‘distId‘]）
????????????????????????????????????x[‘distNum‘]?x[‘type‘]?x[‘distX‘]
????????????????????????????????????x[‘distY‘]））

????????????????????????????timespend?=?datetime.datetime.now（）?-?self.start_time
????????????????????????????percent?=?self.done?/?self.total
????????????????????????????total?=?timespend?/?percent
????????????????????????????print（args?self.done?percent?*?100?self.done?/?timespend.total_seconds（）?*?60?total
??????????????????????????????????total?-?

?屬性????????????大小?????日期????時間???名稱
-----------?---------??----------?-----??----
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\
?????文件????????1513??2017-04-17?15:39??mobike-crawler-master\README.md
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\analysis\
?????文件??????190964??2017-04-17?15:39??mobike-crawler-master\analysis\analysis.ipynb
?????文件????????4794??2017-04-17?15:39??mobike-crawler-master\crawler.py
?????文件????????2718??2017-04-17?15:39??mobike-crawler-master\importToDb.py
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\
?????文件?????????353??2017-04-17?15:39??mobike-crawler-master\influx-importer\build.gradle
?????文件??????????38??2017-04-17?15:39??mobike-crawler-master\influx-importer\settings.gradle
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\main\
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\main\java\
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\main\java\com\
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\main\java\com\april1985\
?????文件????????3868??2017-04-17?15:39??mobike-crawler-master\influx-importer\src\main\java\com\april1985\Application.java
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\modules\
?????文件?????????436??2017-04-17?15:39??mobike-crawler-master\modules\Proxy.py
?????文件????????1178??2017-04-17?15:39??mobike-crawler-master\modules\ProxyProvider.py
?????文件???????????0??2017-04-17?15:39??mobike-crawler-master\modules\__init__.py
?????文件?????????228??2017-04-17?15:39??mobike-crawler-master\sql.sql
?????文件?????????113??2017-04-17?15:39??mobike-crawler-master\start.sh
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\web\
?????文件????????5992??2017-04-17?15:39??mobike-crawler-master\web\server.py
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\web\static\
?????文件?????????180??2017-04-17?15:39??mobike-crawler-master\web\static\.gitignore
?????文件???????57790??2017-04-17?15:39??mobike-crawler-master\web\static\README.md
?????文件?????????539??2017-04-17?15:39??mobike-crawler-master\web\static\package.json
?????目錄???????????0??2017-04-17?15:39??mobike-crawler-master\web\static\public\
?????文件????????6722??2017-04-17?15:39??mobike-crawler-master\web\static\public\icon.png
?????文件????????1332??2017-04-17?15:39??mobike-crawler-master\web\static\public\index.html
?????文件????????2418??2017-04-17?15:39??mobike-crawler-master\web\static\public\select.png
............此處省略11個文件信息