ADD file via upload

2 months ago · 78068460a8
parent 3399f513fc
commit 78068460a8
1 changed files with 191 additions and 0 deletions
--- a/弹幕-爬虫-数据处理-词云图.py
+++ b/弹幕-爬虫-数据处理-词云图.py
@ -0,0 +1,191 @@
+import requests
+import re
+import pandas as pds
+import time
+import jieba  
+import matplotlib.pyplot as plt
+from wordcloud import WordCloud
+from PIL import Image  
+from bs4 import BeautifulSoup 
+from collections import Counter
+from openpyxl import Workbook
+
+# 定义全局变量
+bvid_list = []  # 存储BV号的列表
+oid_list = []   # 存储OID号的列表
+content_list = []  # 存储弹幕内容的列表
+s = set([])  # 存储不重复的BV号集合
+count = 300  # 爬取视频数量
+pages = 15  # 爬取网页页数
+
+
+#爬取bv号
+def bvid_get(page,count):
+    try:
+        for i in range(1,page): # 爬取指定页数
+            url = f'https://search.bilibili.com/video?keyword=2024%E5%B7%B4%E9%BB%8E%E5%A5%A5%E8%BF%90%E4%BC%9A&from_source=webtop_search&spm_id_from=333.1007&search_source=3&page={i}'
+            headers = {
+                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                    "cookie": "buvid4=B85AEC97-F69C-C637-E3C2-C663D774232B87096-022081317-Mk4wjKcJQ45KiYrEq7vrds65fs3pUAqkDfUodWgIWmefZ%2B8Gkn9QRg%3D%3D; CURRENT_FNVAL=4048; buvid_fp=ae1a9ed67ce4b4b81601b1428b3217f9; rpdid=|(k|lk~umYk)0J'u~um~Y|lku; SESSDATA=19c9c747%2C1735394649%2Cde0b5%2A72CjCGWwVQmSRSWbmZZg7l2IVicrrImP6KbqADlDdybFU35A2pNwxPiQvGuSWVPFCvg4sSVjNCOVZTU3ZjeXhGcTYwc1RZSWo1U2E2clc2WkJwdE0zREh2SnRnaldDcGhDUTZOVnF2TVdIWVZvVXBuNnJGeWs5MlhvNFdraGZpaGFwbVNvSXlWSERRIIEC; bili_jct=b7b181986a9d1a649b30eb2fa5929d52; DedeUserID=1604567014; DedeUserID__ckMd5=e19ded6239c7cd47; buvid3=3D25DBF7-1426-D2BB-E102-DB9D3B6CE68458040infoc; b_nut=1719933858; enable_web_push=DISABLE; header_theme_version=CLOSE; LIVE_BUVID=AUTO8417216581143780; fingerprint=4a6a977133f772625df973e6fe4b955c; bp_video_offset_1604567014=968870267503771648; hit-dyn-v2=1; CURRENT_QUALITY=80; _uuid=DEA107613-17109-A8AA-9E27-E8BA6FBFC103E89699infoc; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjU3OTUyODcsImlhdCI6MTcyNTUzNjAyNywicGx0IjotMX0.4xQbg9vPDx7yeSRK4xst2CzX32eMjPSMdlfFyeUn6_g; bili_ticket_expires=1725795227; PVID=8; bp_t_offset_1604567014=974079272559837184; b_lsid=351B10593_191C81CD95D; home_feed_column=4; browser_resolution=978-740; sid=6zbnnd1f"
+                }
+            response = requests.get(url=url, headers=headers)
+            response.encoding = 'utf-8'
+            #print(response.text)
+            temp=re.findall(r'bvid:"(.*?)"',response.text) #存入temp列表
+            #print(temp)
+            for i in temp:
+                s.add(i)
+                if len(bvid_list)>=count: # 爬取指定数量的视频
+                    break
+                if (len(s)>=count):
+                    bvid_list.extend(list(s))
+                    break
+        #print(bvid_list)
+        print("爬取",len(bvid_list),"个视频的bvid")
+    except requests.RequestException as e:
+        print(e)
+    
+    
+
+#爬取视频oid号
+def oid_get():
+    try:
+        for i in bvid_list:
+            url = f'https://api.bilibili.com/x/player/pagelist?bvid={i}&jsonp=jsonp'
+            headers = {
+                        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                        "cookie": "buvid4=B85AEC97-F69C-C637-E3C2-C663D774232B87096-022081317-Mk4wjKcJQ45KiYrEq7vrds65fs3pUAqkDfUodWgIWmefZ%2B8Gkn9QRg%3D%3D; CURRENT_FNVAL=4048; buvid_fp=ae1a9ed67ce4b4b81601b1428b3217f9; rpdid=|(k|lk~umYk)0J'u~um~Y|lku; SESSDATA=19c9c747%2C1735394649%2Cde0b5%2A72CjCGWwVQmSRSWbmZZg7l2IVicrrImP6KbqADlDdybFU35A2pNwxPiQvGuSWVPFCvg4sSVjNCOVZTU3ZjeXhGcTYwc1RZSWo1U2E2clc2WkJwdE0zREh2SnRnaldDcGhDUTZOVnF2TVdIWVZvVXBuNnJGeWs5MlhvNFdraGZpaGFwbVNvSXlWSERRIIEC; bili_jct=b7b181986a9d1a649b30eb2fa5929d52; DedeUserID=1604567014; DedeUserID__ckMd5=e19ded6239c7cd47; buvid3=3D25DBF7-1426-D2BB-E102-DB9D3B6CE68458040infoc; b_nut=1719933858; enable_web_push=DISABLE; header_theme_version=CLOSE; LIVE_BUVID=AUTO8417216581143780; fingerprint=4a6a977133f772625df973e6fe4b955c; bp_video_offset_1604567014=968870267503771648; hit-dyn-v2=1; CURRENT_QUALITY=80; _uuid=DEA107613-17109-A8AA-9E27-E8BA6FBFC103E89699infoc; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjU3OTUyODcsImlhdCI6MTcyNTUzNjAyNywicGx0IjotMX0.4xQbg9vPDx7yeSRK4xst2CzX32eMjPSMdlfFyeUn6_g; bili_ticket_expires=1725795227; PVID=8; bp_t_offset_1604567014=974079272559837184; b_lsid=351B10593_191C81CD95D; home_feed_column=4; browser_resolution=978-740; sid=6zbnnd1f"
+                    }
+            response = requests.get(url=url, headers=headers)
+            response.encoding = 'utf-8'
+            #print(response.text)
+            oid_list.extend(re.findall(r'"cid":(.*?),',response.text)) #存入oid_list列表
+            #print(oid_list)
+        #print('爬取',len(oid_list),'个视频的oid')
+    except requests.RequestException as e:
+        print(e)
+    
+
+
+#爬取弹幕
+def danmu_get():
+    try:
+        for i in oid_list:
+            #print(i)
+            url = f'https://api.bilibili.com/x/v1/dm/list.so?oid={i}'
+            headers = {
+                        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                        "cookie": "buvid4=B85AEC97-F69C-C637-E3C2-C663D774232B87096-022081317-Mk4wjKcJQ45KiYrEq7vrds65fs3pUAqkDfUodWgIWmefZ%2B8Gkn9QRg%3D%3D; CURRENT_FNVAL=4048; buvid_fp=ae1a9ed67ce4b4b81601b1428b3217f9; rpdid=|(k|lk~umYk)0J'u~um~Y|lku; SESSDATA=19c9c747%2C1735394649%2Cde0b5%2A72CjCGWwVQmSRSWbmZZg7l2IVicrrImP6KbqADlDdybFU35A2pNwxPiQvGuSWVPFCvg4sSVjNCOVZTU3ZjeXhGcTYwc1RZSWo1U2E2clc2WkJwdE0zREh2SnRnaldDcGhDUTZOVnF2TVdIWVZvVXBuNnJGeWs5MlhvNFdraGZpaGFwbVNvSXlWSERRIIEC; bili_jct=b7b181986a9d1a649b30eb2fa5929d52; DedeUserID=1604567014; DedeUserID__ckMd5=e19ded6239c7cd47; buvid3=3D25DBF7-1426-D2BB-E102-DB9D3B6CE68458040infoc; b_nut=1719933858; enable_web_push=DISABLE; header_theme_version=CLOSE; LIVE_BUVID=AUTO8417216581143780; fingerprint=4a6a977133f772625df973e6fe4b955c; bp_video_offset_1604567014=968870267503771648; hit-dyn-v2=1; CURRENT_QUALITY=80; _uuid=DEA107613-17109-A8AA-9E27-E8BA6FBFC103E89699infoc; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MjU3OTUyODcsImlhdCI6MTcyNTUzNjAyNywicGx0IjotMX0.4xQbg9vPDx7yeSRK4xst2CzX32eMjPSMdlfFyeUn6_g; bili_ticket_expires=1725795227; PVID=8; bp_t_offset_1604567014=974079272559837184; b_lsid=351B10593_191C81CD95D; home_feed_column=4; browser_resolution=978-740; sid=6zbnnd1f"
+                    }
+            response = requests.get(url=url, headers=headers)
+            response.encoding = 'utf-8'
+            #print(response.text)
+            for j in re.findall(r'<d p=".*?">(.*?)</d>',response.text): #存入content_list列表
+                content_list.append(j)
+            #print(content_list)
+            
+            time.sleep(0.5)
+
+        print("弹幕数量",len(content_list))
+    except requests.RequestException as e:
+        print(e) 
+    
+#生成弹幕txt
+def create_txt():
+    for content in content_list:
+            with open('弹幕567.txt', mode='a', encoding='utf-8') as f:
+                f.write(content)
+                f.write('\n')
+                #print(content)
+
+#生成弹幕excel
+def create_excel():
+    writeData = {  
+    '弹幕': content_list
+    }
+    fwrite = pds.DataFrame(writeData)  
+    fwrite.to_excel("./弹幕567.xlsx", index=False)
+
+#寻找AI相关弹幕以及前8弹幕
+def AI_danmu_and_top_eight_danmu():
+    # 设置文件路径和表名  
+    file_path = '弹幕567.xlsx'  
+    sheet_name = 'Sheet1'  
+    
+    # 定义要搜索的关键词  
+    keyword = 'AI'  
+    
+    # 读取Excel文件  
+    df = pds.read_excel(file_path, sheet_name=sheet_name)  
+    
+    # 筛选包含关键词的行  
+    filtered_df = df[df['弹幕'].str.contains(keyword, na=False, case=False)]  
+    
+    # 计算每种内容的出现次数（不去重，直接计数）  
+    content_counts = filtered_df['弹幕'].value_counts()  
+    
+    # 将结果转换为DataFrame，并重置索引  
+    content_counts_df = content_counts.reset_index()  
+    content_counts_df.columns = ['Text', 'Total_Count']  
+        
+    # 写入新的Excel文件  
+    output_file_path = 'AI相关弹幕.xlsx'  
+    content_counts_df.to_excel(output_file_path, index=False)  
+        
+    print(f"包含关键词'{keyword}'的内容及其总数量已写入到'{output_file_path}'")
+
+    # 对出现次数进行排序，并选择前八个  
+    top_8_counts = content_counts.nlargest(8) 
+
+    # 将结果转换为DataFrame，并重置索引  
+    content_counts_df1 = top_8_counts.reset_index()  
+    content_counts_df1.columns = ['Text', 'Total_Count']  
+    
+    # 写入新的Excel文件  
+    output_file_path = 'AI相关弹幕_前8.xlsx'  
+    content_counts_df1.to_excel(output_file_path, index=False)  
+    
+    print(f"包含关键词'{keyword}'的内容及其前八个总数量已写入到'{output_file_path}'")
+
+
+#词云图
+def create_wordcloud():
+    # 需要排除的关键词列表  
+    exclude_words = set(['的', '了', '啊','在', '是', '我', '你', '他', '她', '它', '有', '和', '这', '那', '这个','不是','真的','我们', '你们', '他们'])  
+    #mask = Image.open(r"D:\b站弹幕爬虫\mask.png").convert("L")   
+    # 读取文件内容  
+    with open('弹幕567.txt', encoding='utf-8') as f:  
+        txt = f.read()  
+    
+    # 分词并排除指定关键词  
+    words = jieba.lcut(txt)  
+    filtered_words = [word for word in words if word not in exclude_words and len(word) > 1]  # 排除太短或指定的词  
+    string = ' '.join(filtered_words)  
+    
+    # 创建词云对象  
+    wc = WordCloud(  
+        width=1000,  
+        height=700,  
+        background_color='white',  
+        colormap='viridis',  
+        #mask=mask,
+        font_path='msyh.ttc'    
+    )  
+    
+    # 生成词云  
+    wc.generate(string)  
+    plt.imshow(wc, interpolation='bilinear')
+    plt.axis('off')  # 不显示坐标轴
+    plt.show()
+    # 保存词云到文件  
+    wc.to_file('词云567.png')  
+    
+
+if __name__ == '__main__':
+    bvid_get(pages,count)
+    oid_get()
+    danmu_get()
+    create_txt()
+    create_excel()
+    AI_danmu_and_top_eight_danmu()
+    create_wordcloud()