|
|
import threading
|
|
|
import time
|
|
|
import datetime
|
|
|
import csv
|
|
|
import json
|
|
|
import requests
|
|
|
from lxml import etree
|
|
|
|
|
|
from entity.BilibiliVideo import BilibiliVideo
|
|
|
|
|
|
|
|
|
class SpyderController:
|
|
|
# Bilibili视频爬虫控制器,打开网页，爬取符合BilibiliVideo.py中的数据,将其下载下来，保存为csv文件
|
|
|
def getBilibiliVideoList(self, videoCount, threadCount, waitTime):
|
|
|
"""
|
|
|
整个爬虫的调用程序
|
|
|
:param videoCount: 爬取的视频数量，若不限制可设置为999
|
|
|
:param threadCount: 爬取的线程并发数量
|
|
|
:param waitTime:float: 每个线程的等待时间，单位秒（避免爬取过快）
|
|
|
:return: list[BilibiliVideo] 返回处理完成后的videoList
|
|
|
"""
|
|
|
all_data_list = []
|
|
|
videoCount = int(videoCount)
|
|
|
threadCount = int(threadCount)
|
|
|
waitTime = float(waitTime)
|
|
|
if videoCount < threadCount:
|
|
|
threadCount = videoCount
|
|
|
if videoCount > 100:
|
|
|
videoCount = 100
|
|
|
json_url = "https://api.bilibili.com/x/web-interface/ranking/v2?"
|
|
|
headers = {
|
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0'
|
|
|
}
|
|
|
#f = open("file_3.csv", "a", encoding="UTF-8", newline="")
|
|
|
#csv_writer = csv.writer(f)
|
|
|
|
|
|
rank_text = requests.get(url=json_url, headers=headers).text
|
|
|
# 将含有"万"的数据转换为数字
|
|
|
def crawl_data(start_index, end_index):
|
|
|
def convert_to_number(lst):
|
|
|
result = []
|
|
|
for item in lst:
|
|
|
if '万' in item:
|
|
|
number = int(float(item.replace('万', '')) * 10000)
|
|
|
result.append(str(number))
|
|
|
else:
|
|
|
result.append(item)
|
|
|
return result
|
|
|
|
|
|
# url = "https://api.bilibili.com/x/web-interface/ranking/v2?"
|
|
|
# headers = {
|
|
|
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0'
|
|
|
# }
|
|
|
f = open("file_3.csv", "a", encoding="GB18030", newline="")
|
|
|
csv_writer = csv.writer(f)
|
|
|
#
|
|
|
# rank_text = requests.get(url=url, headers=headers).text
|
|
|
for i in range(start_index, end_index):
|
|
|
parsed_data = json.loads(rank_text)
|
|
|
aid = str(parsed_data["data"]["list"][i]["aid"])
|
|
|
print(aid)
|
|
|
|
|
|
bvId = [str(parsed_data['data']['list'][i]['bvid'])]
|
|
|
topNo = [str(i+1)]
|
|
|
|
|
|
url = "https://www.bilibili.com/video/av" + aid + "/?"
|
|
|
video_text = requests.get(url=url, headers=headers).text
|
|
|
tree = etree.HTML(video_text)
|
|
|
#print(video_text)
|
|
|
title = tree.xpath('//div[@class="video-info-title-inner"]//text()')
|
|
|
uploadTime = [str(parsed_data["data"]["list"][i]["ctime"])]
|
|
|
viewCount = tree.xpath('//div[@class="view item"]/div/text()')
|
|
|
likeCount = tree.xpath('//div[@class="video-like video-toolbar-left-item"]/span/text()')
|
|
|
coinCount = tree.xpath('//div[@class="video-coin video-toolbar-left-item"]/span/text()')
|
|
|
favoriteCount = tree.xpath('//div[@class="video-fav video-toolbar-left-item"]/span/text()')
|
|
|
bulletCount = tree.xpath('//div[@class="dm-text"]/text()')
|
|
|
creatorFanCount = tree.xpath('//div[@class="default-btn follow-btn b-gz not-follow"]/span/text()')
|
|
|
viewCount = convert_to_number(viewCount)
|
|
|
likeCount = convert_to_number(likeCount)
|
|
|
coinCount = convert_to_number(coinCount)
|
|
|
favoriteCount = convert_to_number(favoriteCount)
|
|
|
bulletCount = convert_to_number(bulletCount)
|
|
|
# match = re.search(r'\d+', text)
|
|
|
# number = match.group()
|
|
|
if not creatorFanCount:
|
|
|
creatorFanCount = [str(1)]
|
|
|
else :
|
|
|
followers_str = creatorFanCount[0].strip().split()[1]
|
|
|
followers_num = float(followers_str.replace('万', '')) * 10000
|
|
|
# 转化为整数
|
|
|
followers_num = int(followers_num)
|
|
|
creatorFanCount = [str(followers_num)]
|
|
|
commentCount = [str(parsed_data['data']['list'][i]['stat']['reply'])]
|
|
|
creatorId = [str(parsed_data['data']['list'][i]['owner']['mid'])]
|
|
|
creatorName = [str(parsed_data['data']['list'][i]['owner']['name'])]
|
|
|
bulletCount = convert_to_number(bulletCount)
|
|
|
|
|
|
#up_url = "https://space.bilibili.com/" + creatorId[0] + "?"
|
|
|
up_url = "https://space.bilibili.com/401742377?spm_id_from=333.788.0.0"
|
|
|
up_text = requests.get(url=up_url, headers=headers).text
|
|
|
tree = etree.HTML(up_text)
|
|
|
#print(up_text)
|
|
|
all_data = bvId + title + [url] + uploadTime + topNo + viewCount + likeCount + coinCount + favoriteCount + bulletCount + commentCount + creatorId + creatorName + creatorFanCount
|
|
|
|
|
|
csv_writer.writerow(all_data)
|
|
|
|
|
|
# all_data_list = all_data_list + [topNo, bvId, title, [video_url], uploadTime, viewCount, likeCount, coinCount, favoriteCount, bulletCount, commentCount, creatorId, creatorName, creatorFanCount]
|
|
|
video = BilibiliVideo(bvId[0], title[0], url, int(uploadTime[0]), int(topNo[0]), int(viewCount[0]), int(likeCount[0]), int(coinCount[0]),
|
|
|
int(favoriteCount[0]), int(commentCount[0]), int(bulletCount[0]), creatorId[0], creatorName[0], int(creatorFanCount[0]))
|
|
|
all_data_list.append(video)
|
|
|
f.close()
|
|
|
# print(all_data_list)
|
|
|
return all_data_list
|
|
|
|
|
|
thread_list = []
|
|
|
video_per_thread = videoCount // threadCount
|
|
|
|
|
|
for i in range(threadCount):
|
|
|
start_index = i * video_per_thread
|
|
|
end_index = (i + 1) * video_per_thread if (i + 1) < threadCount else videoCount
|
|
|
thread = threading.Thread(target=crawl_data, args=(start_index, end_index))
|
|
|
thread_list.append(thread)
|
|
|
thread.start()
|
|
|
time.sleep(waitTime)
|
|
|
|
|
|
for thread in thread_list:
|
|
|
thread.join()
|
|
|
print(all_data_list)
|
|
|
return all_data_list
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
f = open("file_3.csv", "w", encoding="GB18030", newline="")
|
|
|
csv_writer = csv.writer(f)
|
|
|
csv_writer.writerow(
|
|
|
["bvId", "title", "url", "uploadTime", "topNo", "viewCount", "likeCount", "coinCount",
|
|
|
"favoriteCount", "commentCount", "bulletCount", "creatorId", "creatorName", "creatorFanCount"])
|
|
|
f.close()
|
|
|
spyderController = SpyderController()
|
|
|
spyderController.getBilibiliVideoList(6, 2, 0.3) # 设置线程数为2 |
