parent
d90479a134
commit
fa607c4f1f
@ -0,0 +1,191 @@
|
|||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import time
|
||||||
|
import random
|
||||||
|
import openpyxl
|
||||||
|
from collections import Counter
|
||||||
|
import jieba
|
||||||
|
import wordcloud
|
||||||
|
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
'Connection': 'keep-alive',
|
||||||
|
'Cache-Control': 'max-age=0',
|
||||||
|
'DNT': '1',
|
||||||
|
'Upgrade-Insecure-Requests': '1',
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4083.0 Safari/537.36 Edg/82.0.458.0',
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
|
||||||
|
'Sec-Fetch-Site': 'none',
|
||||||
|
'Sec-Fetch-Mode': 'navigate',
|
||||||
|
'Sec-Fetch-User': '?1',
|
||||||
|
'Sec-Fetch-Dest': 'document',
|
||||||
|
'Accept-Language': 'en-US,en;q=0.9'
|
||||||
|
}
|
||||||
|
def get_video_ids(api_urls):
|
||||||
|
video_ids=[]
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0',
|
||||||
|
'cookie' : 'BIDUPSID=B217BCFBC37D845BE7576B36283B7200; PSTM=1694936551; BAIDUID_BFESS=B217BCFBC37D845B41B0963C72424B9E:FG=1; ZFY=8a5uERFpWfnCU59:Bf7xfjug61O89yJEG1n4GF:BPaY1c:C; BDRCVFR[bPTzwF-RsLY]=mk3SLVN4HKm; H_PS_PSSID=60724_60360_60799; BD_HOME=1; BD_UPN=12314753; BA_HECTOR=058ka00hahalak0gak21a1akapkf6v1jelat81u'
|
||||||
|
}
|
||||||
|
#获取前300个热门视频bvid并保存
|
||||||
|
# 设定计数器记录id的个数
|
||||||
|
cnt = 0
|
||||||
|
# 从多页数据中获取视频信息
|
||||||
|
for page in range(1, 22):
|
||||||
|
# 执行翻页操作
|
||||||
|
api_url = api_urls + str(page)
|
||||||
|
# 获取Datas
|
||||||
|
response = requests.get(api_url, headers=headers)
|
||||||
|
response.encoding = 'utf-8'
|
||||||
|
Json = response.json()
|
||||||
|
Datas = Json['data']['result']
|
||||||
|
for Data in Datas:
|
||||||
|
# 通过try except跳过B站设置的断点
|
||||||
|
try:
|
||||||
|
bvids = Data['bvid']
|
||||||
|
video_ids.append(bvids)
|
||||||
|
with open("bv.txt", mode='a', encoding='utf-8') as f: # 执行写入操作
|
||||||
|
f.write(bvids + '\n')
|
||||||
|
cnt += 1
|
||||||
|
if (cnt >= 300):
|
||||||
|
break
|
||||||
|
except:
|
||||||
|
continue
|
||||||
|
if (cnt >= 300):
|
||||||
|
break
|
||||||
|
# print(bvids)
|
||||||
|
|
||||||
|
return video_ids
|
||||||
|
|
||||||
|
|
||||||
|
# API URLs
|
||||||
|
api_urls = 'https://api.bilibili.com/x/web-interface/search/type?search_type=video&keyword=巴黎奥运会&page='
|
||||||
|
|
||||||
|
video_ids = get_video_ids(api_urls)
|
||||||
|
#从bvid获取单个cid的方法
|
||||||
|
def get_cid_from_bv(bv_number, p_number=0):
|
||||||
|
try:
|
||||||
|
url = 'https://api.bilibili.com/x/player/pagelist?bvid={}&jsonp=jsonp'
|
||||||
|
response = requests.get(url.format(bv_number), headers=headers)
|
||||||
|
|
||||||
|
if response.status_code == 200:
|
||||||
|
data = response.json()['data']
|
||||||
|
|
||||||
|
if p_number < len(data):
|
||||||
|
return data[p_number]['cid']
|
||||||
|
else:
|
||||||
|
print(f'Error: Part number out of range for BV code {bv_number}.')
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
print(f'Error: Failed to retrieve CID for BV code {bv_number}. Status code: {response.status_code}')
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f'Error: {str(e)} for BV code {bv_number}.')
|
||||||
|
return None
|
||||||
|
|
||||||
|
#批量转换bvid到cid
|
||||||
|
def get_cids_from_bv_list(video_ids):
|
||||||
|
cid_list = []
|
||||||
|
for bv_code in video_ids:
|
||||||
|
cid = get_cid_from_bv(bv_code)
|
||||||
|
if cid is not None:
|
||||||
|
cid_list.append(cid)
|
||||||
|
else:
|
||||||
|
cid_list.append(None)
|
||||||
|
return cid_list
|
||||||
|
|
||||||
|
cids = get_cids_from_bv_list(video_ids)
|
||||||
|
|
||||||
|
print(f'The corresponding CIDs for the provided BV codes are: {cids}')
|
||||||
|
|
||||||
|
#通过cid批量获取弹幕链接
|
||||||
|
def get_danmu(cids):
|
||||||
|
url = f'https://api.bilibili.com/x/v1/dm/list.so?oid={cids}'
|
||||||
|
headers = {
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
|
||||||
|
}
|
||||||
|
response = requests.get(url, headers=headers)
|
||||||
|
response.encoding = 'utf-8'
|
||||||
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
danmu_list = []
|
||||||
|
for danmu in soup.find_all('d'):
|
||||||
|
danmu_list.append(danmu.text)
|
||||||
|
return danmu_list
|
||||||
|
|
||||||
|
# 批量爬取弹幕
|
||||||
|
with open('all_danmu.txt', 'w', encoding='utf-8') as f:
|
||||||
|
for video_id in cids:
|
||||||
|
danmu_list = get_danmu(video_id)
|
||||||
|
print(f'视频 {video_id} 的弹幕数量: {len(danmu_list)}')
|
||||||
|
|
||||||
|
# 写入当前视频的弹幕到文件
|
||||||
|
f.write(f'== 开始保存视频 {video_id} 的弹幕 ==\n')
|
||||||
|
f.write('\n'.join(danmu_list) + '\n')
|
||||||
|
|
||||||
|
# 随机休眠1-3秒,防止被ban
|
||||||
|
time.sleep(random.uniform(1, 3))
|
||||||
|
|
||||||
|
print('所有视频的弹幕已保存到 all_danmu.txt 文件中')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# 1. 读取弹幕数据文件
|
||||||
|
filename = 'all_danmu.txt'
|
||||||
|
|
||||||
|
with open(filename, 'r', encoding='utf-8') as f:
|
||||||
|
danmus = f.readlines()
|
||||||
|
|
||||||
|
# 2. 过滤与AI技术相关的弹幕
|
||||||
|
ai_keywords = ['AI', '人工智能']
|
||||||
|
ai_related_danmus = [danmu for danmu in danmus if any(keyword in danmu for keyword in ai_keywords)]
|
||||||
|
|
||||||
|
# 3. 统计每种弹幕数量
|
||||||
|
danmu_counter = Counter(ai_related_danmus)
|
||||||
|
|
||||||
|
# 4. 获取数量排名前8的弹幕
|
||||||
|
top_danmus = danmu_counter.most_common(8)
|
||||||
|
|
||||||
|
# 5. 将统计结果写入Excel表
|
||||||
|
excel_filename = 'AI相关弹幕统计.xlsx'
|
||||||
|
wb = openpyxl.Workbook()
|
||||||
|
sheet = wb.active
|
||||||
|
sheet.title = 'AI相关弹幕统计'
|
||||||
|
|
||||||
|
sheet['A1'] = '弹幕内容'
|
||||||
|
sheet['B1'] = '出现次数'
|
||||||
|
|
||||||
|
# 写入前8的弹幕和次数
|
||||||
|
for idx, (danmu, count) in enumerate(top_danmus, start=2):
|
||||||
|
sheet[f'A{idx}'] = danmu.strip() # 去除弹幕末尾的换行符
|
||||||
|
sheet[f'B{idx}'] = count
|
||||||
|
|
||||||
|
# 保存Excel文件
|
||||||
|
wb.save(excel_filename)
|
||||||
|
print(f'AI相关弹幕统计已保存到 {excel_filename}')
|
||||||
|
|
||||||
|
|
||||||
|
# 读取弹幕数据文件
|
||||||
|
filename = 'all_danmu.txt'
|
||||||
|
|
||||||
|
with open(filename, 'r', encoding='utf-8') as f:
|
||||||
|
text = f.read()
|
||||||
|
|
||||||
|
# 使用 jieba 分词
|
||||||
|
word_list = jieba.lcut(text)
|
||||||
|
text_str = ' '.join(word_list)
|
||||||
|
|
||||||
|
# 生成词云
|
||||||
|
wc = wordcloud.WordCloud(font_path='C:/Windows/Fonts/微软雅黑/msyh.ttc', # 指定中文字体文件
|
||||||
|
width=800, height=600, # 设置词云大小
|
||||||
|
background_color='white', # 设置背景颜色
|
||||||
|
stopwords={'哈哈','哈哈哈','的','是','了','我','和','这','也','你','啊','吧','就是','这个','吗','他','不是','真的','都','在','现在','感觉','看','有','不'},
|
||||||
|
contour_width=1, contour_color='blue')
|
||||||
|
wc.generate(text_str)
|
||||||
|
|
||||||
|
# 保存词云图到当前文件夹
|
||||||
|
|
||||||
|
wc.to_file('wordcloud.png')
|
||||||
|
|
||||||
Loading…
Reference in new issue