diff --git a/time_bilibili.py b/time_bilibili.py deleted file mode 100644 index e6ca80f..0000000 --- a/time_bilibili.py +++ /dev/null @@ -1,248 +0,0 @@ -import requests -import json -from bs4 import BeautifulSoup -from collections import Counter -import pandas as pd -from wordcloud import WordCloud -import matplotlib.pyplot as plt -import time -import jieba - -# B站搜索API URL -search_url = 'https://api.bilibili.com/x/web-interface/wbi/search/type' - -# B站视频详情API URL，用于获取视频的cid -video_info_url = 'https://api.bilibili.com/x/web-interface/view' - -# B站弹幕API URL -danmu_url = 'https://api.bilibili.com/x/v1/dm/list.so' - -def search_bilibili(query, total_results): - num_per_page = 42 # 每页最大视频数 - pages_needed = (total_results + num_per_page - 1) // num_per_page # 计算需要多少页 - video_list = [] - - for page in range(1, pages_needed + 1): - params = { - '__refresh__': 'true', - '_extra': '', - 'context': '', - 'page_size': num_per_page, - 'from_source': '', - 'from_spmid': '333.337', - 'platform': 'pc', - 'highlight': '1', - 'single_column': '0', - 'keyword': query, - 'qv_id': '0EnOHi82F62j2usODhMghThN7EvXEZmj', - 'source_tag': '3', - 'dynamic_offset': 30, - 'search_type': 'video', - 'w_rid': '16f27d62ff40f1a5f935a6af26432c81', - 'wts': '1726306000', - 'page': page # 设置页码 - } - headers = { - 'accept': 'application/json, text/plain, */*', - 'accept-encoding': 'gzip, deflate, br, zstd', - 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,en-GB;q=0.6', - 'cookie': 'DedeUserID=1075156086; DedeUserID__ckMd5=7460241d769e1da4; buvid4=9980B4C0-302E-C6A9-122A-0EFE06E4B5F435899-022102715-X83v1qigvaWQdhtSeo%2BvYQ%3D%3D; enable_web_push=DISABLE; buvid3=0DD4B4A8-5B28-59F0-F5EB-9EB31F483AF226299infoc; b_nut=1699086426; _uuid=1FCED779-E59E-F3CA-81A8-817C10CCF3105C25422infoc; header_theme_version=CLOSE; PVID=1; buvid_fp=395bc05f8612d5e47df093ecc1b2bd8e; rpdid=|(J|)Y)JlmJJ0J\'u~|~m|lJ|Y; CURRENT_FNVAL=4048; CURRENT_QUALITY=80; FEED_LIVE_VERSION=V_HEADER_LIVE_NO_POP; home_feed_column=5; browser_resolution=1528-738; bsource=search_bing; bp_t_offset_1075156086=976968501354823680; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjY3MTY5MzMsImlhdCI6MTcyNjQ1NzY3MywicGx0IjotMX0.7WQjSxEb__Z8q6mXZZVKcYfGj_p_EP-8VkK9httVQQA; bili_ticket_expires=1726716873; b_lsid=A255A8C5_191FF65B3BE; SESSDATA=0e66c2c1%2C1742120673%2Cd251f%2A92CjClS9jPOjTyWfjKmoc1Qved4Vfi9N1Jb4KXprWc3-K-qETxsCKQP47sEElvDz-dK0kSVjNHZTNRUUhDSS1DUUJfVzQ3VlQ2NW44YktqbmpLN2hSR2VGQUVIajlfMFAxeERvWlhlWEQ5M1FkX2gxV19FT2wwYjNIcWMwVVRTcElteFpLbkZvRnBRIIEC; bili_jct=0409648e28f719911ffba1058edc4d6d; sid=gq4mtedj', - 'origin': 'https://search.bilibili.com', - 'referer': 'https://search.bilibili.com/all', - 'sec-ch-ua': '"Chromium";v="128", "Not;A=Brand";v="24", "Microsoft Edge";v="128"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-site', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0' - } - - response = requests.get(search_url, params=params, headers=headers) - print(f"Page {page} HTTP Status Code: {response.status_code}") - - if response.status_code == 412: - print("请求被阻止，等待1秒重试...") - time.sleep(1) - continue - - try: - data = response.json() - print(f"Page {page} Parsed JSON Data:") - print(data) - except json.JSONDecodeError: - print(f"Page {page} 无法解析 JSON 数据") - continue - - if data['code'] != 0: - print(f"Page {page} Failed to fetch data from Bilibili API") - continue - - videos = data['data']['result'] - for video in videos: - video_id = video['bvid'] - video_list.append(video_id) - if len(video_list) >= total_results: - break - - if len(video_list) >= total_results: - break - - return video_list - - -def get_video_cid(bvid): - # 请求视频的详情信息，获取cid - params = { - 'bvid': bvid - } - headers = { - 'accept': 'application/json, text/plain, */*', - 'accept-encoding': 'gzip', - 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,en-GB;q=0.6', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0' - } - - response = requests.get(video_info_url, params=params, headers=headers) - print(f"Video Info HTTP Status Code: {response.status_code}") - - if response.status_code != 200: - print(f"Failed to fetch video info for {bvid}") - return None - - try: - data = response.json() - if 'cid' in data['data']: - return data['data']['cid'] - else: - print(f"CID not found for video {bvid}") - return None - except json.JSONDecodeError: - print("无法解析视频信息的 JSON 数据") - return None - - -def fetch_danmu(cid): - params = { - 'oid': cid - } - headers = { - 'accept': 'application/xml, text/xml, */*', - 'accept-encoding': 'gzip', - 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7,en-GB;q=0.6', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0' - } - - response = requests.get(danmu_url, params=params, headers=headers) - print(f"Danmu HTTP Status Code: {response.status_code}") - - if response.status_code != 200: - print(f"Failed to fetch danmu for CID {cid}") - return [] - - content = response.content.decode('utf-8') - print("Danmu Response Content:") - print(content) - - soup = BeautifulSoup(content, 'xml') - danmu_texts = [d.text for d in soup.find_all('d')] - - return danmu_texts - - -def count_and_rank_danmu(danmu_texts): - ai_keywords = ['人工智能', '机器学习', '深度学习', '自然语言处理', '计算机视觉', '智能算法', '大数据', 'AI', '智能制造', '智能家居', '智能医疗', '物联网', '云计算', '智能服务', '自动化','ai'] - top_n = 8 - - # 统计每种弹幕的频率 - counter = Counter(danmu_texts) - - # 统计与 AI 技术应用相关的弹幕频率 - ai_counter = Counter() - keyword_counter = Counter() - - for text, count in counter.items(): - # 统计 AI 关键词的出现次数 - for keyword in ai_keywords: - if keyword in text: - ai_counter[text] += count - keyword_counter[keyword] += count - - # 排名前 top_n 的弹幕 - ranked_ai_danmu = ai_counter.most_common(top_n) - - # 输出每种 AI 关键词的出现次数 - print("AI 技术应用关键词的出现次数:") - for keyword, count in keyword_counter.items(): - print(f"{keyword}: {count} 次") - - # 输出排名前 top_n 的弹幕及其频率 - print(f"\n排名前 {top_n} 的 AI 技术应用弹幕:") - for text, count in ranked_ai_danmu: - print(f"弹幕: {text} - 频率: {count} 次") - - # 将统计结果导出到 Excel - export_to_excel(ranked_ai_danmu, keyword_counter) - - -def export_to_excel(ranked_ai_danmu, keyword_counter): - # 创建 DataFrame，不进行分词，保持原始弹幕 - df_danmu = pd.DataFrame(ranked_ai_danmu, columns=['弹幕', '频率']) - df_keywords = pd.DataFrame(keyword_counter.items(), columns=['关键词', '出现次数']) - - # 保存到 Excel 文件 - with pd.ExcelWriter('danmu_statistics.xlsx') as writer: - df_danmu.to_excel(writer, sheet_name='AI 技术应用弹幕', index=False) - df_keywords.to_excel(writer, sheet_name='AI 技术关键词', index=False) - - print("统计结果已导出到 danmu_statistics.xlsx") - - # 在生成词云图时进行分词 - generate_wordcloud(df_danmu) - - -def generate_wordcloud(df_danmu): - # 进行分词 - processed_texts = [] - for text in df_danmu['弹幕']: - words = jieba.cut(text) # 使用 jieba 分词 - processed_texts.append(' '.join(words)) # 分词结果拼接为字符串 - - # 创建词云图的文本数据 - text = ' '.join(processed_texts) - - # 生成词云图 - wordcloud = WordCloud(font_path='simhei.ttf', width=800, height=600, background_color='white').generate(text) - - # 保存词云图 - wordcloud.generate(text) - wordcloud.to_file('词云.png') - - -import time - -def main(): - query = '2024巴黎奥运会' - total_results = 300 # 设定要爬取的总视频数量 - - start_time = time.time() - video_list = search_bilibili(query, total_results) - print(f"搜索并获取视频列表所需时间: {time.time() - start_time:.2f} 秒") - - all_danmu_texts = [] - start_time = time.time() - for bvid in video_list: - cid = get_video_cid(bvid) - if cid: - danmu_texts = fetch_danmu(cid) - all_danmu_texts.extend(danmu_texts) - print(f"获取所有弹幕文本所需时间: {time.time() - start_time:.2f} 秒") - - start_time = time.time() - count_and_rank_danmu(all_danmu_texts) - print(f"统计和排序弹幕所需时间: {time.time() - start_time:.2f} 秒") - - - -if __name__ == '__main__': - main()