#!/usr/bin/env?python
#?-*-?coding:?utf-8?-*-
#?今日頭條新聞分類數據爬取

import?requests
import?json
import?time
import?random

‘‘‘
100?民生?故事?news_story
101?文化?文化?news_culture
102?娛樂?娛樂?news_entertainment
103?體育?體育?news_sports
104?財經?財經?news_finance
105?時政?新時代?nineteenth
106?房產?房產?news_house
107?汽車?汽車?news_car
108?教育?教育?news_edu?
109?科技?科技?news_tech
110?軍事?軍事?news_military
111?宗教?無，鳳凰佛教等來源
112?旅游?旅游?news_travel
113?國際?國際?news_world
114?證券?股票?stock
115?農業?三農?news_agriculture
116?電競?游戲?news_game
‘‘‘

g_cnns?=?[
[100?‘民生?故事‘?‘news_story‘]
[101?‘文化?文化‘?‘news_culture‘]
[102?‘娛樂?娛樂‘?‘news_entertainment‘]
[103?‘體育?體育‘?‘news_sports‘]
[104?‘財經?財經‘?‘news_finance‘]
#?[105?‘時政?新時代‘?‘nineteenth‘]
[106?‘房產?房產‘?‘news_house‘]
[107?‘汽車?汽車‘?‘news_car‘]
[108?‘教育?教育‘?‘news_edu‘?]
[109?‘科技?科技‘?‘news_tech‘]
[110?‘軍事?軍事‘?‘news_military‘]
#?[111?宗教?無，鳳凰佛教等來源]
[112?‘旅游?旅游‘?‘news_travel‘]
[113?‘國際?國際‘?‘news_world‘]
[114?‘證券?股票‘?‘stock‘]
[115?‘農業?三農‘?‘news_agriculture‘]
[116?‘電競?游戲‘?‘news_game‘]
]

g_ua?=?‘Dalvik/1.6.0?（Linux;?U;?Android?4.4.4;?MuMu?Build/V417IR）?NewsArticle/6.3.1?okhttp/3.7.0.2‘


g_id_cache?=?{}
g_count?=?0

def?get_data（tup）:
????global?g_id_cache
????global?g_count
????cid?=?tup[0]
????cname?=?tup[2]
????url?=?“http://it.snssdk.com/api/news/feed/v63/“

????t?=?int（time.time（）/10000）
????t?=?random.randint（6*t?10*t）
????querystring?=?{“category“:cname“concern_id“:“6215497896830175745““refer“:“1““count“:“20““max_behot_time“:t“last_refresh_sub_entrance_interval“:“1524907088““loc_mode“:“5““tt_from“:“pre_load_more““cp“:“51a5ee4f38c50q1““plugin_enable“:“0““iid“:“31047425023““device_id“:“51425358841““ac“:“wifi““channel“:“tengxun““aid“:“13““app_name“:“news_article““version_code“:“631““version_name“:“6.3.1““device_platform“:“android““ab_version“:“33311629797931749833655629582732504623909732428317098833543233209832519833644333063229705827620328621231321932861533204132935832232132753733571033388333510233482832867032400731707733430528077333567131996033398533171933645221406931643332881333968318434207253266310321519247847281298328218335998325618333327336199323429287591288418260650326188324614335477271178326588326524326532““ab_client“:“a1c4e1f2g2f7““ab_feature“:“94563102749““abflag“:“3““ssmix“:“a““device_type“:“MuMu““device_brand“:“Android““language“:“zh““os_api“:“19““os_version“:“4.4.4““uuid“:“008796762094657““openudid“:“b7215ea70ca32066““manifest_version_code“:“631““resolution“:“1280*720““dpi“:“240““update_version_code“:“6310““_rticket“:“1524907088018““plugin“:“256“}

????headers?=?{
????????‘cache-control‘:?“no-cache“
????????‘postman-token‘:?“26530547-e697-1e8b-fd82-7c6014b3ee86“
????????‘User-Agent‘:?g_ua
????????}

????response?=?requests.request（“GET“?url?headers=headers?params=querystring）


????jj?=?json.loads（response.text）
????with?open（‘toutiao_cat_data.txt‘?‘a‘）?as?fp:
????????for?item?in?jj[‘data‘]:
????????

?屬性????????????大小?????日期????時間???名稱
-----------?---------??----------?-----??----
?????目錄???????????0??2018-08-27?01:51??toutiao-text-classfication-dataset-master\
?????文件????????4785??2018-08-27?01:51??toutiao-text-classfication-dataset-master\get_data.py
?????文件????????2764??2018-08-27?01:51??toutiao-text-classfication-dataset-master\readme.md
?????文件????26912069??2018-08-27?01:51??toutiao-text-classfication-dataset-master\toutiao_cat_data.txt.zip