|
|
|
|
|
import requests
|
|
|
import re
|
|
|
import xml.etree.ElementTree as ET
|
|
|
from DrissionPage import ChromiumPage
|
|
|
from bs4 import BeautifulSoup
|
|
|
from collections import OrderedDict
|
|
|
from collections import Counter
|
|
|
import pandas as pd
|
|
|
from PIL import Image
|
|
|
import numpy as np
|
|
|
import jieba
|
|
|
from wordcloud import WordCloud
|
|
|
import matplotlib.pyplot as plt
|
|
|
|
|
|
|
|
|
headers = {
|
|
|
# cookie用户信息：用于检测用户是否登录账号
|
|
|
"cookie": "buvid4=B1349383-F2A6-E4E5-ED2C-A5B428CC0ED955473-022061918-9v4qmUz9VFkZrXyhSPgawQ%3D%3D; buvid_fp_plain=undefined; LIVE_BUVID=AUTO2216562263398747; is-2022-channel=1; CURRENT_FNVAL=4048; DedeUserID=406267828; DedeUserID__ckMd5=a7899a47ba8a07ab; enable_web_push=DISABLE; rpdid=|(u))kkYu||u0J'u~|JkJR)ul; FEED_LIVE_VERSION=V_WATCHLATER_PIP_WINDOW3; CURRENT_QUALITY=80; buvid3=3D0F285C-7560-9E88-C05A-1AFF4D9CAD7617643infoc; b_nut=1718944717; _uuid=BAAADDA5-4DDD-184B-DB310-EC4105D97D51E17974infoc; header_theme_version=CLOSE; fingerprint=5d019e2bddbce042baa44b97851bb2fd; buvid_fp=5d019e2bddbce042baa44b97851bb2fd; PVID=3; SESSDATA=e97fd178%2C1741840358%2C6c71f%2A91CjB7QCyKSGsn3CjY0C0HRTQF_8a0AYz9r9QLq2FL2YiXLfQFehx5MdXHV3EDcePi4mwSVlM3TVE3LWcyczhLbXoycnhOTWtwa2g5OEl4RVdxOVRQZjljWGJ5d2czbFZpTkVhd3hfRnFmYTE4TVRISTlkdXFpemd1SmhOejNnVm1mYlJ0UDZEcTNBIIEC; bili_jct=4b351ffc7724b0bb10c029efdfad7f75; sid=6jnrbe6l; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjY1NDc1NzEsImlhdCI6MTcyNjI4ODMxMSwicGx0IjotMX0.l-PQPoCUFvsMgRpDwqUKwUY9jLLjI-p-HZ1Qaq7AIjI; bili_ticket_expires=1726547511; b_lsid=1564BAD10_191F00CC884; bp_t_offset_406267828=976996173829111808; home_feed_column=5; browser_resolution=2048-1030",
|
|
|
# user-agent用户代理：表示浏览器设备的基本信息
|
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0"
|
|
|
}
|
|
|
|
|
|
def get_video_details(search_url):
|
|
|
|
|
|
response = requests.get(search_url, headers=headers)
|
|
|
# 确保请求成功
|
|
|
response.raise_for_status()
|
|
|
|
|
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
|
|
|
|
# 匹配 BV 号
|
|
|
bv_pattern = re.compile(r'BV[1-9A-Za-z]{10}')
|
|
|
bv_l=[]
|
|
|
for tag in soup.find_all('a', href=True):
|
|
|
if 'BV' in tag['href']:
|
|
|
bv = bv_pattern.search(tag['href'])
|
|
|
bv_l.append(bv.group())
|
|
|
bv_l = list(OrderedDict.fromkeys(bv_l))# 去重
|
|
|
|
|
|
return bv_l
|
|
|
|
|
|
def get_video_info(bv_id):
|
|
|
""" 获取视频信息并解析CID """
|
|
|
url = f'https://api.bilibili.com/x/web-interface/view?bvid={bv_id}'
|
|
|
response = requests.get(url, headers=headers)
|
|
|
response.encoding = 'utf-8'
|
|
|
response.raise_for_status()
|
|
|
data = response.json()
|
|
|
cid = data['data']['cid']
|
|
|
return cid
|
|
|
|
|
|
|
|
|
def get_danmaku_xml(cid):
|
|
|
""" 根据CID获取弹幕XML数据 """
|
|
|
url = f'https://comment.bilibili.com/{cid}.xml'
|
|
|
response = requests.get(url, headers=headers)
|
|
|
response.encoding = 'utf-8'
|
|
|
# 确保请求成功
|
|
|
response.raise_for_status()
|
|
|
return response.text
|
|
|
|
|
|
|
|
|
def parse_danmaku_xml(xml_str):
|
|
|
""" 解析弹幕XML数据 """
|
|
|
root = ET.fromstring(xml_str)
|
|
|
danmaku_list = [d.text for d in root.findall('.//d') if d.text]
|
|
|
return danmaku_list
|
|
|
|
|
|
|
|
|
def save_danmaku_to_excel(danmaku_content, file_name='danmaku_content.xlsx'):
|
|
|
"""
|
|
|
将弹幕内容保存到 Excel 文件中
|
|
|
:param danmaku_content: 包含弹幕内容的列表
|
|
|
:param file_name: Excel 文件名
|
|
|
"""
|
|
|
# 检查是否已经存在文件
|
|
|
try:
|
|
|
# 尝试读取现有文件
|
|
|
existing_df = pd.read_excel(file_name,engine='openpyxl')
|
|
|
except FileNotFoundError:
|
|
|
# 文件不存在，则创建新的 DataFrame
|
|
|
existing_df = pd.DataFrame(columns=['content'])
|
|
|
|
|
|
# 将新弹幕内容转换为 DataFrame，并自动生成 'id'
|
|
|
new_df = pd.DataFrame(danmaku_content, columns=['content'])
|
|
|
# new_df['id'] = range(len(existing_df), len(existing_df) + len(new_df))
|
|
|
|
|
|
# 合并现有内容和新内容
|
|
|
updated_df = pd.concat([existing_df, new_df], ignore_index=True)
|
|
|
|
|
|
# 保存到 Excel 文件
|
|
|
updated_df.to_excel(file_name, index=False)
|
|
|
|
|
|
|
|
|
def savef(video_url):
|
|
|
""" 主函数，获取并展示弹幕数据 """
|
|
|
|
|
|
# 提取BV号
|
|
|
bv_id = re.search(r'BV\w+', video_url).group(0)
|
|
|
|
|
|
# 获取视频CID
|
|
|
cid = get_video_info(bv_id)
|
|
|
# print(f"提取的CID: {cid}")
|
|
|
|
|
|
# 获取弹幕数据
|
|
|
xml_data = get_danmaku_xml(cid)
|
|
|
danmakus = parse_danmaku_xml(xml_data)
|
|
|
|
|
|
save_danmaku_to_excel(danmakus, file_name='danmaku_content.xlsx')
|
|
|
|
|
|
# # 保存并打印弹幕
|
|
|
# for i, danmaku in enumerate(danmakus):
|
|
|
# # 遍历爬取的弹幕内容并逐条保存
|
|
|
#
|
|
|
# print(f"弹幕 {i + 1}: {danmaku}")
|
|
|
# file_name = 'danmaku_content.xlsx'
|
|
|
# print(f"弹幕内容已保存到 {file_name}")
|
|
|
|
|
|
# 网址
|
|
|
driver = ChromiumPage()
|
|
|
url = "https://search.bilibili.com/all?keyword=2024%E5%B7%B4%E9%BB%8E%E5%A5%A5%E8%BF%90%E4%BC%9A&from_source=webtop_search&spm_id_from=333.1007&search_source=3&page=2&o=36"
|
|
|
driver.get('https://search.bilibili.com/all?keyword=2024%E5%B7%B4%E9%BB%8E%E5%A5%A5%E8%BF%90%E4%BC%9A&from_source=webtop_search&spm_id_from=333.1007&search_source=3&page=2&o=36')
|
|
|
j=0
|
|
|
for page in range(8):
|
|
|
bv_list = get_video_details(url)
|
|
|
for bv in bv_list:
|
|
|
if j==300:
|
|
|
break
|
|
|
j+=1
|
|
|
# 函数入口，定义函数需要调用
|
|
|
video_url = f'https://www.bilibili.com/video/{bv}/?spm_id_from=333.337'
|
|
|
# 调用保存的函数
|
|
|
print(f"v{j}")
|
|
|
savef(video_url)
|
|
|
# 点击下一页
|
|
|
driver.ele('css:vui_button vui_pagenation--btn vui_pagenation--btn-side').click()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 读取xlsx文件
|
|
|
df = pd.read_excel('danmaku_content.xlsx',engine='openpyxl')
|
|
|
|
|
|
# 假设弹幕数据在名为'danmaku'的列中
|
|
|
danmaku_column = df['content']
|
|
|
|
|
|
def filter_ai_danmaku(danmaku_series, keywords):
|
|
|
ai_danmaku = []
|
|
|
for danmaku in danmaku_series:
|
|
|
if pd.notna(danmaku): # 检查弹幕是否为空
|
|
|
danmaku_str = str(danmaku) # 转换为字符串
|
|
|
if any(keyword in danmaku_str for keyword in keywords):
|
|
|
ai_danmaku.append(danmaku_str)
|
|
|
return ai_danmaku
|
|
|
|
|
|
ai_keywords = ["AI", "AI技术", "机器学习", "深度学习", "智能", "VR/AR", "全景直播", "360度", "3D", "追踪" ,"虚拟", "数字", "人工智能", "面部识别", "云技术", "安保", "检测", "监测", "福州大学"]
|
|
|
# 筛选与AI相关的弹幕
|
|
|
filtered_danmaku = filter_ai_danmaku(danmaku_column, ai_keywords)
|
|
|
|
|
|
# print(len(filtered_danmaku))
|
|
|
|
|
|
# 统计每种弹幕的数量
|
|
|
danmaku_count = Counter(filtered_danmaku)
|
|
|
|
|
|
# 排序并获取前n项
|
|
|
sorted_danmaku = danmaku_count.most_common(15)
|
|
|
|
|
|
# print(sorted_danmaku)
|
|
|
|
|
|
# 输出排名前n的弹幕及数量
|
|
|
for i, (danmaku, count) in enumerate(sorted_danmaku, 1):
|
|
|
print(f"排名 {i}: {danmaku} - 数量: {count}")
|
|
|
|
|
|
# 将数据转化为DataFrame
|
|
|
danmaku_df = pd.DataFrame(sorted_danmaku, columns=['弹幕内容', '数量'])
|
|
|
|
|
|
# 导出到Excel文件
|
|
|
danmaku_df.to_excel('ai_danmaku_statistics.xlsx', index=False, engine='openpyxl')
|
|
|
|
|
|
|
|
|
def generate_beautiful_wordcloud(file_name='ai_danmaku_statistics.xlsx', mask_image_path=None,font_path=r"C:\Users\wyk93\Desktop\苹方黑体-准-简.ttf"):
|
|
|
# 读取 Excel 文件
|
|
|
df = pd.read_excel(file_name)
|
|
|
|
|
|
# 合并所有弹幕内容为一个大字符串
|
|
|
text = ' '.join(df['弹幕内容'].dropna().astype(str))
|
|
|
|
|
|
# 使用 jieba 进行分词
|
|
|
seg_list = jieba.cut(text, cut_all=True)
|
|
|
segmented_text = " ".join(seg_list)
|
|
|
|
|
|
# # 打印合并后的文本以检查内容
|
|
|
# print("Text content:")
|
|
|
# print(text[:500]) # 打印前500个字符以检查
|
|
|
|
|
|
# 读取自定义形状图片
|
|
|
if mask_image_path:
|
|
|
mask_image = np.array(Image.open(mask_image_path).convert('L'))
|
|
|
mask_image[mask_image > 200] = 255
|
|
|
mask_image[mask_image <= 200] = 0
|
|
|
else:
|
|
|
mask_image = None
|
|
|
|
|
|
background_image = Image.open(mask_image_path)
|
|
|
|
|
|
# 创建词云对象
|
|
|
wordcloud = WordCloud(
|
|
|
background_color='white',# 透明背景
|
|
|
mask=mask_image,
|
|
|
contour_color=None,
|
|
|
contour_width=2,
|
|
|
width=2000, # 词云图的宽度
|
|
|
height=2000, # 词云图的高度
|
|
|
colormap='viridis' , # 使用 'viridis' 颜色映射
|
|
|
max_words=20000,
|
|
|
# min_font_size=10,
|
|
|
# max_font_size=100,
|
|
|
# prefer_horizontal=0.9, # 更高的水平优先级
|
|
|
font_path=font_path # 如果需要显示中文，需要指定中文字体路径
|
|
|
).generate(segmented_text) # 这里替换为你自己的文本数据
|
|
|
|
|
|
# 显示词云图
|
|
|
plt.figure(figsize=(10, 8))
|
|
|
plt.imshow(background_image, interpolation='bilinear')
|
|
|
plt.imshow(wordcloud, interpolation='bilinear', alpha=0.4) # 设置透明度
|
|
|
plt.axis('off') # 不显示坐标轴
|
|
|
plt.show()
|
|
|
|
|
|
# 调用函数生成词云
|
|
|
# 可选: 使用自定义形状的掩膜图像
|
|
|
generate_beautiful_wordcloud(mask_image_path=r"C:\Users\wyk93\Desktop\图片素材\sucai3.jpg")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
