bilibili-spider/Paris Olympic Games/my_cprofile.py

import collections  # 用于词频统计
import json  # 用于处理JSON数据
import requests  # 用于发送HTTP请求
import re  # 正则表达式模块，用于解析弹幕
import time  # 用于时间相关操作
import openpyxl  # 用于处理Excel文件
import pandas as pd  # 用于数据处理
from concurrent.futures import ThreadPoolExecutor, as_completed  # 用于并发操作
import cProfile

profile = cProfile.Profile()
profile.enable()


# 定义开始和结束日期，用于生成日期范围
startdate = '20240710'
enddate = '20240910'
date = [x for x in pd.date_range(startdate, enddate).strftime('%Y-%m-%d')]  # 生成日期列表

# 定义Excel文件名，用于保存弹幕数据
file_xlsx = '我的全部弹幕.xlsx'

# 创建Excel工作簿和工作表，并添加标题行
total_workbook = openpyxl.Workbook()
total_sheet = total_workbook.active
total_sheet.append(['弹幕'])

# 定义B站弹幕API的基础URL，{number}是占位符，用于填充视频的cid号
tempApi = 'https://api.bilibili.com/x/v1/dm/list.so?oid={number}'

# 定义请求头，包含cookie和user-agent，用于伪装请求
headers = {        
    'cookie':"buvid3=D65868DE-AFD5-34A4-1714-A1C0F783C5DC27124infoc; b_nut=1725930527; _uuid=FF569C27-D2C6-10814-36A8-48AA8141364924857infoc; CURRENT_FNVAL=4048; buvid_fp=2ba89565eab107e1e14c7982fc1ef9ea; buvid4=FAB9A58B-B8F5-8DAF-2AC4-4E874D3D1F0E28371-024091001-a%2FA7nVxQVETBwJOeuHlVsQ%3D%3D; rpdid=|(u))kkYu|lu0J'u~klmJ|lkm; SESSDATA=e8f35e7e%2C1741482645%2Cb3572%2A91CjC7hBYEVq-d38AwweerB9sclbgqT78LR6aribbsaBRVlJ0BoUjCMidR-nm82eDlo70SVlVibjl1UnQ0Y0NzSFFCb21DRGNNSXp4YnRSbFdzMXo0NjR4QkM0TlBKejUweW1TbDJkT0g3Z2Z6bTdmQVJzdmpvVHZmR1JWOEhtbnFGZmpuQUt6WXZnIIEC; bili_jct=7d37b038ea7714a0c41ec3d26603737b; DedeUserID=1917958039; DedeUserID__ckMd5=eaa26b970b7e3104; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjYxOTIwNTMsImlhdCI6MTcyNTkzMjc5MywicGx0IjotMX0.82V6_w7kGoSvzDy9rT-9DpsL7U_BrB24GefbBM0Vvb8; bili_ticket_expires=1726191993; header_theme_version=CLOSE; enable_web_push=DISABLE; home_feed_column=5; browser_resolution=1536-730; b_lsid=953CBCA8_191E441EE95; bp_t_offset_1917958039=976131738646347776; sid=hl295qcj",
    'user-agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
}

# 定义函数，获取搜索结果中的bvid（视频的唯一标识符）
def get_bvid(page_number, number):
    # 构造搜索API的URL，page_number是页码，number是该页中的视频编号
    url = f'https://api.bilibili.com/x/web-interface/search/type?page={page_number}&page_size=50&keyword=2024%E5%B7%B4%E9%BB%8E%E5%A5%A5%E8%BF%90%E4%BC%9A&search_type=video'
    response = requests.get(url=url, headers=headers)  # 发送请求
    try:
        # 解析返回的JSON数据，提取视频的bvid
        json_data = json.loads(response.text)
        print(json_data)
        bvid = json_data['data']['result'][number]['bvid']
        print(f"获取到bvid: {bvid}")
        return bvid  # 返回bvid
    except (KeyError, IndexError, json.JSONDecodeError, requests.RequestException) as e:
        print(f"获取bvid时出错: {e}")
        # 捕获错误并返回None，防止程序崩溃
        return None

# 定义函数，根据bvid获取视频的cid（弹幕对应的唯一标识符）
def get_cid(bvid):
    try:
        # 通过bvid构造获取cid的API请求URL
        url = f'https://api.bilibili.com/x/player/pagelist?bvid={bvid}&jsonp=jsonp'
        response = requests.get(url, headers=headers)  # 发送请求
        if response.status_code != 200:
            # 如果请求状态码不是200，返回None
            return None
        # 解析返回的JSON数据，提取cid
        json_dict = json.loads(response.text)
        return json_dict['data'][0]['cid']  # 返回cid
    except (KeyError, IndexError, json.JSONDecodeError, requests.RequestException):
        return None  # 捕获错误并返回None

# 定义函数，获取并保存某个视频的弹幕
def fetch_and_save_bulletchat(cid):
    # 用cid替换API中的占位符
    url = tempApi.replace("{number}", str(cid))
    try:
        # 发送请求获取弹幕数据
        response = requests.get(url, headers=headers)
        response.encoding = response.apparent_encoding  # 设置编码
        # 使用正则表达式解析弹幕内容
        data = re.findall('<d p=".*?">(.*?)</d>', response.text)
        if data:
            return data  # 返回弹幕列表
    except requests.RequestException:
        return []  # 如果请求失败，返回空列表

# 定义函数，批量获取bvid和cid，并创建并发任务
def put_api():
    tasks = []
    # 使用ThreadPoolExecutor创建线程池，用于并发请求
    with ThreadPoolExecutor(max_workers=10) as executor:
        # 控制页码范围（1到5页），每页50个视频
        for i in range(1, 7):
            for j in range(50):
                bvid = get_bvid(i, j)  # 获取bvid
                if bvid:
                    cid = get_cid(bvid)  # 获取cid
                    if cid:
                        # 提交弹幕抓取任务到线程池
                        tasks.append(executor.submit(fetch_and_save_bulletchat, cid))
    return tasks  # 返回任务列表

# 定义函数，处理并发任务，收集所有弹幕数据
def get_data(tasks):
    all_bulletchats = []
    # 遍历所有完成的任务，获取结果
    for task in as_completed(tasks):
        bulletchat_data = task.result()
        if bulletchat_data:
            all_bulletchats.extend(bulletchat_data)  # 将弹幕数据加入总列表
    return all_bulletchats  # 返回所有弹幕数据

# 定义函数，将弹幕数据保存到文件和Excel中
def save_to_file(bulletchats):
    # 打开文本文件，将弹幕逐行写入
    with open('我的全部弹幕.txt', 'a', encoding='utf-8') as file_txt:
        for index in bulletchats:
            file_txt.write(index + '\n')
            total_sheet.append([index])  # 将弹幕写入Excel表格
    total_workbook.save(file_xlsx)  # 保存Excel文件

# 定义函数，计算弹幕频次，并保存到Excel
def calculate_frequency():
    try:
        # 读取弹幕Excel文件
        fd = pd.read_excel(file_xlsx)
        lines = fd['弹幕']
        # 将所有弹幕拼接成一个字符串
        text = ' '.join(lines.astype(str))
        words = text.split()  # 将弹幕分割为单词
        word_counts = collections.Counter(words)  # 统计单词频次
        sorted_word_counts = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)  # 按频次排序
        
        # 创建新的Excel工作簿用于保存频次统计结果
        workbook = openpyxl.Workbook()
        sheet = workbook.active
        sheet.append(['弹幕', '频次'])  # 添加标题行
        
        # 将排序后的词频结果写入Excel
        for word, count in sorted_word_counts:
            sheet.append([word, count])
        
        workbook.save('我的统计弹幕出现次数.xlsx')  # 保存频次统计的Excel文件
    except Exception as e:
        print(f"计算频次时出错: {e}")

# 主函数，负责执行整个流程
def main():
    tasks = put_api()  # 获取bvid和cid并创建并发任务
    bulletchats = get_data(tasks)  # 获取所有弹幕数据
    save_to_file(bulletchats)  # 保存弹幕数据到文件和Excel
    calculate_frequency()  # 计算弹幕频次
    print("Finished")  # 输出流程结束信息
    

# 如果此脚本被直接运行，则调用main函数
if __name__ == '__main__':
    main()

profile.disable()
# Save the profiling data to a file
profile.dump_stats('./output.prof')