怎么用Python爬取抖音小视频? 资深程序员都这样爬取的(附源码)
简介
抖音，是一款可以拍短视频的音乐创意短视频社交软件，该软件于2016年9月上线，是一个专注年轻人的15秒音乐短视频社区。用户可以通过这款软件选择歌曲，拍摄15秒的音乐短视频，形成自己的作品。此APP已在Android各大应用商店和APP Store均有上线。
今天咱们就用Python爬取抖音视频
准备：
环境：Python3.6 Windows
IDE：你开行就好，喜欢用哪个就用哪个
模块：
1 from splinter.driver.webdriver.chrome import Options, Chrome2 from splinter.browser import Browser3 from contextlib import closing4 import requests, json, time, re, os, sys, time5 from bs4 import BeautifulSoup
获得视频播放地址
查询的用户ID
视频名字列表
视频链接列表
用户昵称
1 def get_video_urls(self, user_id): 2 3 video_names = [] 4 video_urls = [] 5 unique_id = '' 6 while unique_id != user_id: 7 search_url = 'https://api.amemv.com/aweme/v1/discover/search/?cursor=0&keyword=%s&count=10&type=1&retry_type=no_retry&iid=17900846586&device_id=34692364855&ac=wifi&channel=xiaomi&aid=1128&app_name=aweme&version_code=162&version_name=1.6.2&device_platform=android&ssmix=a&device_type=MI 5&device_brand=Xiaomi&os_api=24&os_version=7.0&uuid=861945034132187&openudid=dc451556fc0eeadb&manifest_version_code=162&resolution=1080*1920&dpi=480&update_version_code=1622' % user_id 8 req = requests.get(url = search_url, verify = False) 9 html = json.loads(req.text)10 aweme_count = html['user_list'][0]['user_info']['aweme_count']11 uid = html['user_list'][0]['user_info']['uid']12 nickname = html['user_list'][0]['user_info']['nickname']13 unique_id = html['user_list'][0]['user_info']['unique_id']14 user_url = 'https://www.douyin.com/aweme/v1/aweme/post/?user_id=%s&max_cursor=0&count=%s' % (uid, aweme_count)15 req = requests.get(url = user_url, verify = False)16 html = json.loads(req.text)17 i = 118 for each in html['aweme_list']:19 share_desc = each['share_info']['share_desc']20 if '抖音-原创音乐短视频社区' == share_desc:21 video_names.append(str(i) '.mp4')22 i = 123 else:24 video_names.append(share_desc '.mp4')25 video_urls.append(each['share_info']['share_url'])26 27 return video_names, video_urls, nickname
获得带水印的视频播放地址
video_url：带水印的视频播放地址
download_url: 带水印的视频下载地址
1 def get_download_url(self, video_url):2 3 req = requests.get(url = video_url, verify = False)4 bf = BeautifulSoup(req.text, 'lxml')5 script = bf.find_all('script')[-1]6 video_url_js = re.findall('var data = \[(. )\];', str(script))[0]7 video_html = json.loads(video_url_js)8 download_url = video_html['video']['play_addr']['url_list'][0]9 return download_url
视频下载
video_url: 带水印的视频地址
video_name: 视频名
watermark_flag: 是否下载不带水印的视频
1 def video_downloader(self, video_url, video_name, watermark_flag=True): 2 ''' 3 视频下载 4 Parameters: 5 video_url: 带水印的视频地址 6 video_name: 视频名 7 watermark_flag: 是否下载不带水印的视频 8 Returns: 9 无10 '''11 size = 012 if watermark_flag == True:13 video_url = self.remove_watermark(video_url)14 else:15 video_url = self.get_download_url(video_url)16 with closing(requests.get(video_url, stream=True, verify = False)) as response:17 chunk_size = 102418 content_size = int(response.headers['content-length']) 19 if response.status_code == 200:20 sys.stdout.write(' [文件大小]:%0.2f MB\n' % (content_size / chunk_size / 1024))21 22 with open(video_name, 'wb') as file: 23 for data in response.iter_content(chunk_size = chunk_size):24 file.write(data)25 size = len(data)26 file.flush()27 28 sys.stdout.write(' [下载进度]:%.2f%%' % float(size / content_size * 100) '\r')29 sys.stdout.flush()
获得无水印的视频播放地址
1 def remove_watermark(self, video_url): 2 ''' 3 获得无水印的视频播放地址 4 Parameters: 5 video_url: 带水印的视频地址 6 Returns: 7 无水印的视频下载地址 8 ''' 9 self.driver.visit('http://douyin.iiilab.com/')10 self.driver.find_by_tag('input').fill(video_url)11 self.driver.find_by_xpath('//button[@class='btn btn-default']').click()12 html = self.driver.find_by_xpath('//div[@class='thumbnail']/div/p')[0].html13 bf = BeautifulSoup(html, 'lxml')14 return bf.find('a').get('href')
下载视频
1 def run(self): 2 ''' 3 运行函数 4 Parameters: 5 None 6 Returns: 7 None 8 ''' 9 self.hello()10 user_id = input('请输入ID(例如40103580):')11 video_names, video_urls, nickname = self.get_video_urls(user_id)12 if nickname not in os.listdir():13 os.mkdir(nickname)14 print('视频下载中:共有%d个作品!\n' % len(video_urls))15 for num in range(len(video_urls)):16 print(' 解析第%d个视频链接 [%s] 中，请稍后!\n' % (num 1, video_urls[num]))17 if '\\' in video_names[num]:18 video_name = video_names[num].replace('\\', '')19 elif '/' in video_names[num]:20 video_name = video_names[num].replace('/', '')21 else:22 video_name = video_names[num]23 self.video_downloader(video_urls[num], os.path.join(nickname, video_name))24 print('\n')25 26 print('下载完成!')
全部代码
1 # -*- coding:utf-8 -*- 2 3 Python学习交流群：125240963 4 Python学习交流群：125240963 5 Python学习交流群：125240963 6 7 from splinter.driver.webdriver.chrome import Options, Chrome 8 from splinter.browser import Browser 9 from contextlib import closing 10 import requests, json, time, re, os, sys, time 11 from bs4 import BeautifulSoup 12 13 class DouYin(object): 14 def __init__(self, width = 500, height = 300): 15 ''' 16 抖音App视频下载 17 ''' 18 # 无头浏览器 19 chrome_options = Options() 20 chrome_options.add_argument('user-agent='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'') 21 self.driver = Browser(driver_name='chrome', executable_path='D:/chromedriver', options=chrome_options, headless=True) 22 23 def get_video_urls(self, user_id): 24 ''' 25 获得视频播放地址 26 Parameters: 27 user_id：查询的用户ID 28 Returns: 29 video_names: 视频名字列表 30 video_urls: 视频链接列表 31 nickname: 用户昵称 32 ''' 33 video_names = [] 34 video_urls = [] 35 unique_id = '' 36 while unique_id != user_id: 37 search_url = 'https://api.amemv.com/aweme/v1/discover/search/?cursor=0&keyword=%s&count=10&type=1&retry_type=no_retry&iid=17900846586&device_id=34692364855&ac=wifi&channel=xiaomi&aid=1128&app_name=aweme&version_code=162&version_name=1.6.2&device_platform=android&ssmix=a&device_type=MI 5&device_brand=Xiaomi&os_api=24&os_version=7.0&uuid=861945034132187&openudid=dc451556fc0eeadb&manifest_version_code=162&resolution=1080*1920&dpi=480&update_version_code=1622' % user_id 38 req = requests.get(url = search_url, verify = False) 39 html = json.loads(req.text) 40 aweme_count = html['user_list'][0]['user_info']['aweme_count'] 41 uid = html['user_list'][0]['user_info']['uid'] 42 nickname = html['user_list'][0]['user_info']['nickname'] 43 unique_id = html['user_list'][0]['user_info']['unique_id'] 44 user_url = 'https://www.douyin.com/aweme/v1/aweme/post/?user_id=%s&max_cursor=0&count=%s' % (uid, aweme_count) 45 req = requests.get(url = user_url, verify = False) 46 html = json.loads(req.text) 47 i = 1 48 for each in html['aweme_list']: 49 share_desc = each['share_info']['share_desc'] 50 if '抖音-原创音乐短视频社区' == share_desc: 51 video_names.append(str(i) '.mp4') 52 i = 1 53 else: 54 video_names.append(share_desc '.mp4') 55 video_urls.append(each['share_info']['share_url']) 56 57 return video_names, video_urls, nickname 58 59 def get_download_url(self, video_url): 60 ''' 61 获得带水印的视频播放地址 62 Parameters: 63 video_url：带水印的视频播放地址 64 Returns: 65 download_url: 带水印的视频下载地址 66 ''' 67 req = requests.get(url = video_url, verify = False) 68 bf = BeautifulSoup(req.text, 'lxml') 69 script = bf.find_all('script')[-1] 70 video_url_js = re.findall('var data = \[(. )\];', str(script))[0] 71 video_html = json.loads(video_url_js) 72 download_url = video_html['video']['play_addr']['url_list'][0] 73 return download_url 74 75 def video_downloader(self, video_url, video_name, watermark_flag=True): 76 ''' 77 视频下载 78 Parameters: 79 video_url: 带水印的视频地址 80 video_name: 视频名 81 watermark_flag: 是否下载不带水印的视频 82 Returns: 83 无 84 ''' 85 size = 0 86 if watermark_flag == True: 87 video_url = self.remove_watermark(video_url) 88 else: 89 video_url = self.get_download_url(video_url) 90 with closing(requests.get(video_url, stream=True, verify = False)) as response: 91 chunk_size = 1024 92 content_size = int(response.headers['content-length']) 93 if response.status_code == 200: 94 sys.stdout.write(' [文件大小]:%0.2f MB\n' % (content_size / chunk_size / 1024)) 95 96 with open(video_name, 'wb') as file: 97 for data in response.iter_content(chunk_size = chunk_size): 98 file.write(data) 99 size = len(data)100 file.flush()101 102 sys.stdout.write(' [下载进度]:%.2f%%' % float(size / content_size * 100) '\r')103 sys.stdout.flush()104 105 106 def remove_watermark(self, video_url):107 '''108 获得无水印的视频播放地址109 Parameters:110 video_url: 带水印的视频地址111 Returns:112 无水印的视频下载地址113 '''114 self.driver.visit('http://douyin.iiilab.com/')115 self.driver.find_by_tag('input').fill(video_url)116 self.driver.find_by_xpath('//button[@class='btn btn-default']').click()117 html = self.driver.find_by_xpath('//div[@class='thumbnail']/div/p')[0].html118 bf = BeautifulSoup(html, 'lxml')119 return bf.find('a').get('href')120 121 def run(self):122 '''123 运行函数124 Parameters:125 None126 Returns:127 None128 '''129 self.hello()130 user_id = input('请输入ID(例如40103580):')131 video_names, video_urls, nickname = self.get_video_urls(user_id)132 if nickname not in os.listdir():133 os.mkdir(nickname)134 print('视频下载中:共有%d个作品!\n' % len(video_urls))135 for num in range(len(video_urls)):136 print(' 解析第%d个视频链接 [%s] 中，请稍后!\n' % (num 1, video_urls[num]))137 if '\\' in video_names[num]:138 video_name = video_names[num].replace('\\', '')139 elif '/' in video_names[num]:140 video_name = video_names[num].replace('/', '')141 else:142 video_name = video_names[num]143 self.video_downloader(video_urls[num], os.path.join(nickname, video_name))144 print('\n')145 146 print('下载完成!')147 148 def hello(self):149 '''150 打印欢迎界面151 Parameters:152 None153 Returns:154 None155 '''156 print('*' * 100)157 print('\t\t\t\t抖音App视频下载小助手')158 print('\t\t作者:Python学习交流群：125240963')159 print('*' * 100)160 161 162 if __name__ == '__main__':163 douyin = DouYin()164 douyin.run()
赞 (0)