"""
====================================================================
项目名称: 唯品会商品数据爬取
项目描述: 通过requests框架获取网页数据
项目环境: pycharm && python3.8
作者所属: 几许
====================================================================
1. 对主页抓包 , 鼠标移动到视频位置视频自动播放获得视频数据包
2. 对视频数据包地址进行解析 , 复制链接 , 进行检索
3. 获得视频的数据表
4. 转化为字典类型 , 通过取值获得相应数据
====================================================================
"""
import requests
import json
import os
from tqdm import tqdm
class Spider():
def __init__(self):
self.name = input("用户名称:")
self.url = 'https://www.douyin.com/aweme/v1/web/aweme/post/?device_plat' \
'form=webapp&aid=6383&channel=channel_pc_web&sec_user_id=MS4wLj' \
'ABAAAARSI_YyerRex7gC78FWY-wnL9LA0VLV0-hw4MsHQXKQc&max_cursor=0&' \
'locate_item_id=7319002747116719394&locate_query=false&show_live_' \
'replay_strategy=1&need_time_list=1&time_list_query=0&whale_cut_t' \
'oken=&cut_version=1&count=18&publish_video_strategy_type=2&pc_cli' \
'ent_type=1&version_code=170400&version_name=17.4.0&cookie_enabled=tr' \
'ue&screen_width=1739&screen_height=1087&browser_language=zh-CN&browser' \
'_platform=Win32&browser_name=Chrome&browser_version=120.0.0.0&browser' \
'_online=true&engine_name=Blink&engine_version=120.0.0.0&os_name=Windo' \
'ws&os_version=10&cpu_core_num=16&device_memory=8&platform=PC&downlink' \
'=10&effective_type=4g&round_trip_time=50&webid=7317298922984064552&ms' \
'Token=41miuagnXWbrfJA3fKZcTLpK0uZqsPeVIPFNrcpnIa_qqykpK8IUO85mYnyTrEK' \
'grcP_ZMaiDJQmc_yPjtSsGWaRtHX9RUkMff5DWnIlxahYCAHBDBD4wC1IuDBE1Q==&X-Bo' \
'gus=DFSzswVObKiANy5nt74z2vB9PimP'
self.headers = {
'authority': 'www.douyin.com',
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9',
'cookie': 'ttwid=1%7CetvXY3q8LycHG8S_Jbdkn0coDJBB1g4jDT8rUsZxkgE%7C1703691433%7C25a10db3f3cb2286bc61c5df891762b53ed86642718f6514a971a1dd5b926b60; dy_swidth=1739; dy_sheight=1087; volume_info=%7B%22isUserMute%22%3Afalse%2C%22isMute%22%3Atrue%2C%22volume%22%3A0.6%7D; s_v_web_id=verify_lqnxw3p8_LHhIVuoi_xT4U_4GAO_Bso3_NXXogcEL5P6L; passport_csrf_token=a400b27cd87ff688e2ed491b78c86cec; passport_csrf_token_default=a400b27cd87ff688e2ed491b78c86cec; xgplayer_user_id=85464987351; bd_ticket_guard_client_web_domain=2; ttcid=982e11696ecb4394a427ca8a8bc61b8712; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%2C%22isForcePopClose%22%3A1%7D; passport_assist_user=CkGdWr_4NXuB6776SugOMjnvz8kqR8AAMU-Oo472dCf56_f5plVV3JVgBZS9xHMtKkUpDNjW4YE-wNMhS2l7KCGLaBpKCjy7KERoTX9NxbrlQ4fwf80IWklA7yrQ68hpu4tGNpmPPb5DgZY1oISMsOHOHsdhCZyB4S2tzcky5UgdolAQ-b_FDRiJr9ZUIAEiAQMCSFAU; n_mh=aOe6MoW0EhRYuUYuxK7zU5ABLi2JtbyKLjBKJVaPHY4; sso_uid_tt=6ed47ea04759b3a853fbad85bf16f1c0; sso_uid_tt_ss=6ed47ea04759b3a853fbad85bf16f1c0; toutiao_sso_user=be81bb8ea7faacdcbc9dd5db7758dcde; toutiao_sso_user_ss=be81bb8ea7faacdcbc9dd5db7758dcde; sid_ucp_sso_v1=1.0.0-KDExYWI4ZTY3MzIwZDMxOTFkZmZiNDY1NTE4ODMzOGI4OTM4MTJmODYKHwi9tIDt1YzkAxChhMusBhjvMSAMMMDC14IGOAZA9AcaAmxmIiBiZTgxYmI4ZWE3ZmFhY2RjYmM5ZGQ1ZGI3NzU4ZGNkZQ; ssid_ucp_sso_v1=1.0.0-KDExYWI4ZTY3MzIwZDMxOTFkZmZiNDY1NTE4ODMzOGI4OTM4MTJmODYKHwi9tIDt1YzkAxChhMusBhjvMSAMMMDC14IGOAZA9AcaAmxmIiBiZTgxYmI4ZWE3ZmFhY2RjYmM5ZGQ1ZGI3NzU4ZGNkZQ; passport_auth_status=66f3029dfff0bfcbcc4a88214c47101a%2C; passport_auth_status_ss=66f3029dfff0bfcbcc4a88214c47101a%2C; uid_tt=30af4edf2be0abcf7863968a5619085f; uid_tt_ss=30af4edf2be0abcf7863968a5619085f; sid_tt=2876f1470127b791f0cf9a72cd86dfe8; sessionid=2876f1470127b791f0cf9a72cd86dfe8; sessionid_ss=2876f1470127b791f0cf9a72cd86dfe8; publish_badge_show_info=%220%2C0%2C0%2C1704116788119%22; LOGIN_STATUS=1; _bd_ticket_crypt_doamin=2; _bd_ticket_crypt_cookie=e7315ef4e091ee513451063fa2da5513; __security_server_data_status=1; sid_guard=2876f1470127b791f0cf9a72cd86dfe8%7C1704116794%7C5183978%7CFri%2C+01-Mar-2024+13%3A46%3A12+GMT; sid_ucp_v1=1.0.0-KDhiOTQ5YTE0NDA1ZmJjMWViOWZmNmZmZjFmNTc5NTA2MTU1MGM1OWUKGwi9tIDt1YzkAxC6hMusBhjvMSAMOAZA9AdIBBoCbGYiIDI4NzZmMTQ3MDEyN2I3OTFmMGNmOWE3MmNkODZkZmU4; ssid_ucp_v1=1.0.0-KDhiOTQ5YTE0NDA1ZmJjMWViOWZmNmZmZjFmNTc5NTA2MTU1MGM1OWUKGwi9tIDt1YzkAxC6hMusBhjvMSAMOAZA9AdIBBoCbGYiIDI4NzZmMTQ3MDEyN2I3OTFmMGNmOWE3MmNkODZkZmU4; store-region=cn-ah; store-region-src=uid; download_guide=%223%2F20240101%2F1%22; my_rd=2; EnhanceDownloadGuide=%220_0_0_0_1_1704117357%22; pwa2=%220%7C0%7C3%7C0%22; strategyABtestKey=%221704267225.226%22; SEARCH_RESULT_LIST_TYPE=%22single%22; douyin.com; device_web_cpu_core=16; device_web_memory_size=8; architecture=amd64; csrf_session_id=3af6123a976801945da40296beb67c81; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1739%2C%5C%22screen_height%5C%22%3A1087%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A16%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A10%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A50%7D%22; __ac_nonce=06595391400381d9d7250; __ac_signature=_02B4Z6wo00f01kfyV6wAAIDBeltNM7xgII5H0lMAAPR4zvgmgRl4bZQiw8Hy7m1HJQyGxv-69JHZ7ep7bv06SKWB3HnU902JGVt5BoxWU57grcNKrgwl0D1h-fnclHqOBW1bw14QCmbhuaeCea; FOLLOW_NUMBER_YELLOW_POINT_INFO=%22MS4wLjABAAAAH_la9atp1nq0NfBl06KGYylxI45qL-L_V1YPgFrYa2a44J05vyu0abQGp13pO8_m%2F1704297600000%2F1704267233660%2F0%2F1704279492177%22; tt_scid=jaahDcXkVC9dvqUIH5uGmlzxfBiNsOFnKCsQFb1O8AspLavoMj5sv3PBOnnmhNLAd099; passport_fe_beating_status=true; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A0%2C%5C%22is_mute%5C%22%3A1%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A0%7D%22; xg_device_score=7.664966583511937; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCUGltTkFHc1RpTkhpNmNyeFhOSExXb2tMQ3duQWE2RnQyNGZNSlVnbVVlOTBzQ3NSUGpqVSt2ajVjUUl0czBtOEFBNGl4K1Y2MWRTOHorTGVnY2FBeE09IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoxfQ%3D%3D; msToken=tyKrHlga8UZGJDBEwpGp7urIvXmrT9jeFWRj35PdBv6_f1bvA1RrKwQKipwekauBXqeeSDLP3tyH32ow7LFDhA50p8N1dAmoMScRT-2JNrl0wQ4HW-jGM_-imzkb; msToken=F4m6s_dH7A-07AFYB0Xd0TvUzrjRVYE6ZHvFsVzD8h3O6IU_0foQiXoFw0zFG1D1alVcOEsFgU0NbraMql0b3JAZX7EWmsvhkNPem0iik9HWek_CFpw2lY8kS9gX; odin_tt=3ecfdb9d2b86cb25ea7999936d0bac85c7423d8abe2fcab0ac0b4e89a71be53a73c178c90455d05d6454792b4a4d822d90ec2afdfb6edd932ca068a1cb79e012; IsDouyinActive=true; home_can_add_dy_2_desktop=%220%22',
'referer': 'https://www.douyin.com/user/MS4wLjABAAAAkzRSrOuSsM4Z1Ricsddumx_aSvX0jmOPcQR2qTs3PEtImBD8BomLrqvtIOBKOL0P',
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' }
def prase_data(self):
msg = requests.get(self.url, headers=self.headers).text
json_msg = json.loads(msg)
dict_msg = json_msg['aweme_list']
for dict in tqdm(dict_msg):
if dict['desc'] == '':
title = dict['aweme_id']
elif dict['desc'].split()[0]:
title = dict['desc'].split()[0]
else:
title = dict['desc']
link = dict['video']['play_addr']['url_list'][0]
if title == '':
title = dict['aweme_id']
movie = requests.get(link).content
self.keep_data(title,movie)
def keep_data(self,title,link):
if not os.path.exists(self.name):
os.makedirs(self.name)
with open(f'{self.name}\{title}.mp4','wb')as f:
f.write(link)
if __name__ == '__main__':
spider = Spider()
spider.prase_data()