xx_mmc
2 months ago
commit
28fe6a727c
@ -0,0 +1,220 @@
|
||||
headers = {
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36",
|
||||
"Cookie": "i-wanna-go-back=-1; buvid_fp_plain=undefined; CURRENT_BLACKGAP=0; blackside_state=0; LIVE_BUVID=AUTO5216539051785441; buvid4=BF640363-932C-9859-2DEB-9D5332BED8BA14521-022050118-RBQaCti2N%2FgbXXvSImVESA%3D%3D; buvid3=EA6B6EE5-CF42-44F0-8BF1-0E035F5182C9167646infoc; DedeUserID=506881997; DedeUserID__ckMd5=6816981dbd4223e9; CURRENT_FNVAL=4048; rpdid=|(u))kRlJJ)u0J'uYY)l~u)~J; CURRENT_QUALITY=80; hit-new-style-dyn=1; CURRENT_PID=150df130-cdea-11ed-9e61-390f799e5bb1; _uuid=68159E9C-3BA8-49EE-A1C6-D7E510610D865E40530infoc; nostalgia_conf=-1; b_ut=5; FEED_LIVE_VERSION=V8; hit-dyn-v2=1; home_feed_column=5; browser_resolution=1530-712; header_theme_version=CLOSE; fingerprint=93340026c1ba350713aeadf8766000e1; SESSDATA=5c25a608%2C1709466512%2Cc7e4b%2A92gDhsEFKTVzRobJkJtk9Sk1ph71ufczEtnhZVk3UyXcKE4ChKGDta46HuRUO_g-u_Rbl2OgAAYQA; bili_jct=37f8d40c6076352e8e44a85bbbeb65a4; sid=7px9659x; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTQxODU3MzksImlhdCI6MTY5MzkyNjUzOSwicGx0IjotMX0.ek0FRkjhs25UswbCHtI0R25Otecvf_5FppkCkYoDMCE; bili_ticket_expires=1694185739; PVID=3; b_nut=100; buvid_fp=93340026c1ba350713aeadf8766000e1; b_lsid=109D53E510_18A6AC7C071; bp_video_offset_506881997=838254238965432390",
|
||||
}
|
||||
|
||||
chrome_driver_path = 'D:\chromedriver-win64\chromedriver.exe' # ChromeDriver的路径
|
||||
|
||||
def get_urls(query, number):
|
||||
"""
|
||||
获取指定关键字的 Bilibili 视频网址。
|
||||
|
||||
参数:
|
||||
query (str): 搜索关键字。
|
||||
number (int): 需要获取的网址数量。
|
||||
chrome_driver_path (str):
|
||||
|
||||
返回:
|
||||
set: 包含视频网址的集合。
|
||||
"""
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.common.by import By
|
||||
|
||||
# 设置 ChromeDriver 服务和选项, 并初始化 WebDriver
|
||||
service = Service(chrome_driver_path)
|
||||
options = Options()
|
||||
options.add_argument('--headless') # 启用无头模式，不显示浏览器窗口
|
||||
driver = webdriver.Chrome(service=service, options=options)
|
||||
|
||||
url_set = set() # 存储网址的集合
|
||||
|
||||
#循环搜索每一页，获取视频链接
|
||||
for page in range(1, 100):
|
||||
search_url = f'https://search.bilibili.com/video?keyword={query}&page={page}'
|
||||
driver.get(search_url) # 打开搜索结果页面
|
||||
|
||||
# 查找所有符合选择器的 <a> 标签
|
||||
elements = driver.find_elements(By.CSS_SELECTOR, ".video-list.row div.bili-video-card > div > a")
|
||||
|
||||
# 提取每个 <a> 标签的 href 属性（即网址），并加入集合
|
||||
for element in elements:
|
||||
url_set.add(element.get_attribute('href'))
|
||||
if len(url_set) >= number: # 达到数量要求
|
||||
break
|
||||
if len(url_set) >= number: # 达到数量要求
|
||||
break
|
||||
# print(f"成功打开{len(url_set)}")
|
||||
|
||||
driver.quit()
|
||||
return url_set
|
||||
|
||||
|
||||
def url_to_bv(url):
|
||||
"""
|
||||
从给定的哔哩哔哩视频网址中提取BVID。
|
||||
|
||||
参数:
|
||||
url (str): 哔哩哔哩视频的网址。
|
||||
|
||||
返回:
|
||||
str: 提取的BV号。
|
||||
"""
|
||||
import re
|
||||
return re.findall('https://www.bilibili.com/video/(.*?)/', url)[0]
|
||||
|
||||
def rand_sleep():
|
||||
"""
|
||||
暂停执行一段随机时间,范围在1到5秒之间,包含小数部分以增加随机性。
|
||||
|
||||
返回:
|
||||
None
|
||||
"""
|
||||
import random
|
||||
import time
|
||||
sleep_time = random.randint(1, 4) + random.random()
|
||||
time.sleep(sleep_time)
|
||||
|
||||
def bv_to_cid(bvid):
|
||||
"""
|
||||
通过向哔哩哔哩的视频播放器页面列表接口发送请求,获取指定BV号的CID。
|
||||
|
||||
参数:
|
||||
bvid (str): 哔哩哔哩视频的BV号。
|
||||
|
||||
返回:
|
||||
int: 视频的CID。
|
||||
"""
|
||||
import json
|
||||
import requests
|
||||
|
||||
# 定义API请求的URL
|
||||
url = "https://api.bilibili.com/x/player/pagelist?bvid=" + str(bvid) + "&jsonp=jsonp"
|
||||
|
||||
# 向哔哩哔哩API发起请求
|
||||
video_logo = requests.get(url=url, headers=headers)
|
||||
|
||||
# 将响应文本解析为JSON格式
|
||||
video_name = video_logo.text
|
||||
name = json.loads(video_name)
|
||||
|
||||
# 从JSON响应中提取CID
|
||||
cid = name['data'][0]['cid']
|
||||
|
||||
return cid
|
||||
|
||||
|
||||
def cid_to_danmu(cid):
|
||||
"""
|
||||
根据给定的CID, 从哔哩哔哩获取弹幕数据。
|
||||
|
||||
参数:
|
||||
cid (Union[int, str]): 哔哩哔哩视频的CID。
|
||||
|
||||
返回:
|
||||
list: 包含弹幕文本的列表。
|
||||
"""
|
||||
import requests
|
||||
import re
|
||||
|
||||
if isinstance(cid, int) :
|
||||
cid = str(cid)
|
||||
# 构造API请求URL
|
||||
url = 'https://api.bilibili.com/x/v1/dm/list.so?oid=' + cid
|
||||
|
||||
# 发起GET请求
|
||||
response = requests.get(url=url, headers=headers)
|
||||
response.encoding = response.apparent_encoding
|
||||
|
||||
# 使用正则表达式提取弹幕文本
|
||||
data_list = re.findall('<d p=".*?">(.*?)</d>', response.text)
|
||||
return data_list
|
||||
|
||||
def get_danmu(query, number, display_progress=False):
|
||||
"""
|
||||
获取指定查询条件下的视频弹幕数据。
|
||||
|
||||
参数:
|
||||
query (str): 搜索关键词。
|
||||
number (int): 要获取的视频数量。
|
||||
display_progress (bool): 是否显示进度信息。
|
||||
|
||||
返回:
|
||||
list: 包含所有视频弹幕的列表。
|
||||
"""
|
||||
# 根据查询条件获取指定数量的视频链接
|
||||
url_set = get_urls(query, number)
|
||||
|
||||
if display_progress:
|
||||
print(f"成功获取 {len(url_set)} 个链接")
|
||||
|
||||
danmu = []
|
||||
for index, url in enumerate(url_set):
|
||||
rand_sleep() # 随机延时，避免请求过于频繁
|
||||
bv = url_to_bv(url) # 将视频URL转换为BV号
|
||||
cid = bv_to_cid(bv) # 将BV号转换为CID
|
||||
danmu.extend(cid_to_danmu(cid)) # 获取弹幕并添加到列表中
|
||||
|
||||
if display_progress:
|
||||
# 打印当前进度信息
|
||||
print(f"\r当前进度 {index + 1}/{len(url_set)}, 共获取 {len(danmu)} 个弹幕", end='')
|
||||
|
||||
return danmu
|
||||
|
||||
|
||||
|
||||
def get_danmu_contains_keywords(query, number, keywords, display_progress=False):
|
||||
"""
|
||||
根据查询条件和关键词从视频中获取包含关键词的弹幕数据。
|
||||
|
||||
参数:
|
||||
query (str): 搜索关键词。
|
||||
number (int): 要获取的视频数量。
|
||||
keywords (list): 需要匹配的关键词列表。
|
||||
display_progress (bool): 是否显示进度信息。
|
||||
|
||||
返回:
|
||||
list: 包含所有符合关键词条件的弹幕的列表。
|
||||
"""
|
||||
import jieba
|
||||
|
||||
def contains_keywords(text, keywords):
|
||||
"""
|
||||
判断文本是否包含任意一个关键词。
|
||||
|
||||
参数:
|
||||
text (str): 待检测的文本。
|
||||
keywords (list): 关键词列表。
|
||||
|
||||
返回:
|
||||
bool: 如果文本包含关键词则返回True, 否则返回False。
|
||||
"""
|
||||
for word in list(jieba.cut(text)):
|
||||
for keyword in keywords:
|
||||
if word == keyword:
|
||||
return True
|
||||
return False
|
||||
|
||||
# 获取指定查询条件的视频链接
|
||||
url_set = get_urls(query, number)
|
||||
|
||||
if display_progress:
|
||||
print(f"成功获取 {len(url_set)} 个链接")
|
||||
|
||||
danmu = []
|
||||
for index, url in enumerate(url_set):
|
||||
rand_sleep() # 随机睡眠，避免过于频繁的请求
|
||||
bv = url_to_bv(url) # 将视频URL转换为BV号
|
||||
cid = bv_to_cid(bv) # 将BV号转换为CID
|
||||
|
||||
# 获取弹幕并筛选包含关键词的弹幕
|
||||
for item in cid_to_danmu(cid):
|
||||
if contains_keywords(item, keywords):
|
||||
danmu.append(item)
|
||||
|
||||
if display_progress:
|
||||
# 打印当前进度
|
||||
print(f"\r当前进度 {index + 1}/{len(url_set)}, 共获取 {len(danmu)} 个有关弹幕", end='')
|
||||
|
||||
return danmu
|
||||
@ -0,0 +1,86 @@
|
||||
import bilibili_spider
|
||||
import matplotlib.pyplot as plt
|
||||
from wordcloud import WordCloud
|
||||
import jieba
|
||||
import pandas
|
||||
from openpyxl import Workbook
|
||||
|
||||
def contains_keywords(text, keywords):
|
||||
for word in list(jieba.cut(text)):
|
||||
for keyword in keywords:
|
||||
if word == keyword:
|
||||
return True
|
||||
return False
|
||||
|
||||
ai_keywords = [
|
||||
"机器学习", "深度学习", "自然语言处理", "计算机视觉", "图像识别",
|
||||
"语音识别", "强化学习", "生成对抗网络", "智能推荐系统", "数据挖掘",
|
||||
"模式识别", "智能机器人", "自动驾驶", "预测分析", "数据清洗",
|
||||
"异常检测", "知识图谱", "人工智能伦理", "智能合约", "虚拟助手",
|
||||
"语义分析", "图像生成", "文本生成", "情感分析", "决策支持系统",
|
||||
"人脸识别", "智能搜索", "自然语言生成", "人工神经网络", "模型优化",
|
||||
"智能监控", "医疗影像分析", "自动化", "智能制造", "虚拟现实",
|
||||
"增强现实", "智能家居", "边缘计算", "云计算", "数据隐私",
|
||||
"算法公平性", "知识推理", "智能交通", "聊天机器人", "自动化客服",
|
||||
"智能推荐引擎", "生物识别", "机器人过程自动化", "多模态学习", "量子计算",
|
||||
"自适应系统", "算法优化", "智能数据分析", "虚拟角色", "环境感知",
|
||||
"ai", "AI", "人工智能"
|
||||
]
|
||||
|
||||
def list_to_dict(list):
|
||||
# 遍历列表中的每个元素
|
||||
count_dict = {}
|
||||
for item in list:
|
||||
if item in count_dict:
|
||||
count_dict[item] += 1
|
||||
else:
|
||||
count_dict[item] = 1
|
||||
return count_dict
|
||||
|
||||
|
||||
def main():
|
||||
query = '2024巴黎奥运会'
|
||||
number = 300
|
||||
|
||||
# 获取弹幕列表
|
||||
danmu_list = bilibili_spider.get_danmu(query=query, number=number, display_progress=True)
|
||||
# danmu_list = ["test", "ai", "noai"]
|
||||
|
||||
# 筛选其中包含AI关键词的弹幕
|
||||
ai_danmu_list = []
|
||||
for danmu in danmu_list:
|
||||
if contains_keywords(danmu, ai_keywords):
|
||||
ai_danmu_list.append(danmu)
|
||||
|
||||
ai_danmu_dict = list_to_dict(ai_danmu_list)
|
||||
ai_danmu_dict = dict(sorted(ai_danmu_dict.items(), key=lambda item: item[1], reverse=True))
|
||||
|
||||
#输出数量排名前8的弹幕
|
||||
first_8_ai_danmu = list(ai_danmu_dict.items())[:8]
|
||||
for item in first_8_ai_danmu:
|
||||
print(f"{item[0]} : 出现{item[1]}次数")
|
||||
|
||||
# 将所有弹幕数量写入 Excel 文件
|
||||
danmu_dict = list_to_dict(danmu_list)
|
||||
danmu_dict = dict(sorted(danmu_dict.items(), key=lambda item: item[1], reverse=True))
|
||||
Workbook().save('output.xlsx')
|
||||
with pandas.ExcelWriter('output.xlsx', engine='openpyxl', mode='a', if_sheet_exists='replace') as writer:
|
||||
pandas.DataFrame(list(danmu_dict.items())).to_excel(writer, sheet_name='所有弹幕', index=False)
|
||||
|
||||
# 将ai弹幕数量写入 Excel 文件
|
||||
with pandas.ExcelWriter('output.xlsx', engine='openpyxl', mode='a', if_sheet_exists='replace') as writer:
|
||||
pandas.DataFrame(list(ai_danmu_dict.items())).to_excel(writer, sheet_name='ai弹幕', index=False)
|
||||
|
||||
# 制作词云图
|
||||
font_path = "C:\Windows\Fonts\SimHei.ttf"
|
||||
wordcloud = WordCloud(font_path=font_path, width=800, height=400, background_color='white').generate(' '.join(ai_danmu_list))
|
||||
|
||||
plt.figure(figsize=(10, 5))
|
||||
plt.imshow(wordcloud, interpolation='bilinear')
|
||||
plt.axis('off')
|
||||
|
||||
plt.savefig('wordcloud.png', format='png') # 保存为 PNG 文件
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Loading…
Reference in new issue