ADD file via upload

crawler.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+import requests
+from bs4 import BeautifulSoup
+import time
+import os
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+class BilibiliDanmakuCrawler:
+    def __init__(self, keyword, video_count, save_dir):
+        self.keyword = keyword
+        self.video_count = video_count
+        self.save_dir = save_dir
+        os.makedirs(save_dir, exist_ok=True)
+        
+        # 带重试机制的会话（解决网络波动）
+        self.session = requests.Session()
+        retry = Retry(
+            total=3,
+            backoff_factor=1,
+            status_forcelist=[412, 429, 500, 502, 503, 504]
+        )
+        self.session.mount('http://', HTTPAdapter(max_retries=retry))
+        self.session.mount('https://', HTTPAdapter(max_retries=retry))
+        
+        # 请求头（替换为你的Cookie）
+        self.headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+            "Referer": "https://search.bilibili.com/",
+            "Cookie": "buvid3=E8B6A22C-2C45-4243-1A50-0B7887C84A4500588infoc; rpdid=|(u))kkYu|um0J'u~k)~mYm|u; b_nut=100; b_lsid=BD6D42CE_19A76AED18B; bsource=search_sougo; _uuid=BD2459B10-A5107-5424-57D5-D3E3C2510222859761infoc; buvid_fp=01d6b98373d0ee5ae25897960af410a8; home_feed_column=5; browser_resolution=1699-941; buvid4=FD45C162-3221-ACF9-4FB1-19AE076778C860860-025111214-xHcqiQBrPSLcJVMwvAmRJw%3D%3D; CURRENT_QUALITY=0; csrf_state=e8a98437773776f3420329dc3758a34c; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3NjMxOTAwMzEsImlhdCI6MTc2MjkzMDc3MSwicGx0IjotMX0.wLjhNhZSAz-VJZCGAEltHftitM-90-C1nSRr_JtoVII; bili_ticket_expires=1763189971; SESSDATA=36fd0664%2C1778482862%2Ce1d71%2Ab1CjBUqeAMzYv1RDPYIrJSzYNj9v_TLDaM3RXdELRfLIrNqKJHA7i5yvvzwA3AiDKvw4ISVkN3Q0lpNFcwU2pLNk9ZakczZklvOWsxTWZvcm9KMzNBR1R1d05kOTcwU1BSSWY1RVdBWmtmUHY3VURDektoVHNJVTVFSF9vQlFmSlY4ZDRERkFfaF9nIIEC; bili_jct=3380c36097411cca708626cb4e7a81d6; DedeUserID=409262887; DedeUserID__ckMd5=234150aa4a4b4661; bp_t_offset_409262887=1134268504589991936; theme-tip-show=SHOWED; CURRENT_FNVAL=4048; sid=5uxkxypo"
+        }
+
+    def get_top_bv_ids(self):
+        """获取视频BV号列表"""
+        bv_list = []
+        page = 1
+        max_pages = 20  # 最大页数
+        print(f"开始获取「{self.keyword}」相关视频BV号...")
+        
+        while len(bv_list) < self.video_count and page <= max_pages:
+            api_url = (
+                f"https://api.bilibili.com/x/web-interface/search/type"
+                f"?keyword={self.keyword}&search_type=video&order=totalrank&page={page}"
+            )
+            try:
+                resp = self.session.get(api_url, headers=self.headers, timeout=15)
+                resp.raise_for_status()
+                data = resp.json()
+                
+                if data.get("code") != 0:
+                    print(f"第{page}页错误：{data.get('message')}")
+                    page += 1
+                    time.sleep(2)
+                    continue
+                
+                # 解析BV号并去重
+                new_count = 0
+                for item in data["data"]["result"]:
+                    bv = item.get("bvid")
+                    if bv and bv not in bv_list:
+                        bv_list.append(bv)
+                        new_count += 1
+                        if len(bv_list) >= self.video_count:
+                            break
+                
+                print(f"第{page}页处理完成，新增{new_count}个，共{len(bv_list)}个")
+                page += 1
+                time.sleep(1.5)
+                
+            except Exception as e:
+                print(f"第{page}页请求失败：{str(e)}，重试...")
+                time.sleep(3)
+        
+        # 保存BV号
+        with open(f"{self.save_dir}/bv_list.txt", "w", encoding="utf-8") as f:
+            f.write("\n".join(bv_list))
+        print(f"BV号获取完成，共{len(bv_list)}个")
+        return bv_list
+
+    def get_cid_by_bv(self, bv):
+        """通过BV号获取CID"""
+        try:
+            url = f"https://api.bilibili.com/x/web-interface/view?bvid={bv}"
+            resp = self.session.get(url, headers=self.headers, timeout=15)
+            return resp.json()["data"]["cid"]
+        except Exception as e:
+            print(f"BV号「{bv}」获取CID失败：{str(e)}")
+            return None
+
+    def crawl_danmaku(self, cid):
+        """爬取弹幕并过滤噪声"""
+        if not cid:
+            return []
+        try:
+            url = f"https://comment.bilibili.com/{cid}.xml"
+            resp = self.session.get(url, headers=self.headers, timeout=15)
+            resp.encoding = "utf-8"
+            soup = BeautifulSoup(resp.text, "xml")
+            danmu_tags = soup.find_all("d")
+            
+            # 过滤规则
+            noise = {"666", "哈哈哈", "点赞", "投币", "收藏", "打卡", "来了"}
+            valid = [
+                tag.text.strip() for tag in danmu_tags
+                if tag.text.strip() and len(tag.text.strip()) > 1
+                and not any(n in tag.text.strip() for n in noise)
+            ]
+            return valid
+        except Exception as e:
+            print(f"CID「{cid}」爬取失败：{str(e)}")
+            return []
+
+    def run(self):
+        """执行爬取流程"""
+        bv_list = self.get_top_bv_ids()
+        all_danmu = []
+        
+        for i, bv in enumerate(bv_list, 1):
+            print(f"处理第{i}/{len(bv_list)}个视频（BV：{bv}）")
+            cid = self.get_cid_by_bv(bv)
+            all_danmu.extend(self.crawl_danmaku(cid))
+            time.sleep(1)
+        
+        # 保存弹幕
+        with open(f"{self.save_dir}/all_danmu.txt", "w", encoding="utf-8") as f:
+            f.write("\n".join(all_danmu))
+        print(f"爬取完成，共{len(all_danmu)}条有效弹幕")
+        return all_danmu