Scrapy/python1111.py

import pandas
import requests  #发送请求
import pandas as pd
import numpy as np#导入matplotlib库，绘制图像
import matplotlib.pyplot as plt#读取数据集到phone_data中
import seaborn as sns
import jieba
from collections import Counter
from wordcloud import WordCloud
 
url_dict = {
	'全站': 'https://api.bilibili.com/x/web-interface/ranking/v2?rid=0&type=all',
	'音乐': 'https://api.bilibili.com/x/web-interface/ranking/v2?rid=3&type=all',
	'舞蹈': 'https://api.bilibili.com/x/web-interface/ranking/v2?rid=129&type=all',
	'生活': 'https://api.bilibili.com/x/web-interface/ranking/v2?rid=160&type=all',
	'美食': 'https://api.bilibili.com/x/web-interface/ranking/v2?rid=211&type=all',
}
 
headers = {
    "upgrade-insecure-requests": "1",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
    "Referer": "https://www.bilibili.com/v/popular/rank/all/",
}
 
for i in url_dict.items():
	url = i[1]  # url地址
	tab_name = i[0]  # tab页名称
	title_list = []
	play_cnt_list = []  # 播放数
	danmu_cnt_list = []  # 弹幕数
	coin_cnt_list = []  # 投币数
	like_cnt_list = []  # 点赞数
	share_cnt_list = []  # 分享数
	favorite_cnt_list = []  # 收藏数
	author_list = []
	video_url = []
	try:
		r = requests.get(url, headers=headers)
		print(r.status_code)#获取目标数据所在数据并转成字典类型
		json_data = r.json()
		list_data = json_data['data']['list']
 
		for data in list_data:
			title_list.append(data['title'])
			play_cnt_list.append(data['stat']['view'])
			danmu_cnt_list.append(data['stat']['danmaku'])
			coin_cnt_list.append(data['stat']['coin'])
			like_cnt_list.append(data['stat']['like'])
			# dislike_cnt_list.append(data['stat']['dislike'])
			share_cnt_list.append(data['stat']['share'])
			favorite_cnt_list.append(data['stat']['favorite'])
			author_list.append(data['owner']['name'])
			# score_list.append(data['score'])
			video_url.append('https://www.bilibili.com/video/' + data['bvid'])
			# print('*' * 10)
	except Exception as e:
		print("爬取失败:{}".format(str(e)))
	#创建dataframe保存数据
	df = pd.DataFrame(
		{'视频标题': title_list,
		 '视频地址': video_url,
		 '作者': author_list,
		 '播放数': play_cnt_list,
		 '弹幕数': danmu_cnt_list,
		 '投币数': coin_cnt_list,
		 '点赞数': like_cnt_list,
		 '分享数': share_cnt_list,
		 '收藏数': favorite_cnt_list,
		 })
	#print(df.head())
	#将数据保存到本地
	df.to_csv('B站TOP100-{}.csv'.format(tab_name), index=False,encoding='utf_8_sig')  # utf_8_sig修复乱码问题
	print('写入成功: ' + 'B站TOP100-{}.csv'.format(tab_name))
data_meishi=pd.read_csv('B站TOP100-美食.csv')
data_quanzhan=pd.read_csv('B站TOP100-全站.csv')
data_shenghuo=pd.read_csv('B站TOP100-生活.csv')
data_wudao=pd.read_csv('B站TOP100-舞蹈.csv')
data_yinyue=pd.read_csv('B站TOP100-音乐.csv')
 
plt.rcParams['font.family'] = 'SimHei'

#弹幕数与播放数的散点图
plt.figure(figsize=(8, 6))   # 调整图像大小：确认图像没有被显示在可视区域外,尝试设置图像的大小利用plt.figure(figsize=(,))
sns.scatterplot(x='弹幕数', y='播放数', data=data_quanzhan)
plt.show() # 调用 plt.show()显示结果

numeric_data = data_quanzhan.select_dtypes(include=['number'])
cor = numeric_data.corr(method='spearman')
 
sns.heatmap(cor,
            annot=True,  # 显示相关系数的数据
            center=0.5,  # 居中
            fmt='.2f',  # 只显示两位小数
            linewidth=0.5,  # 设置每个单元格的距离
            linecolor='blue',  # 设置间距线的颜色
            vmin=0, vmax=1,  # 设置数值最小值和最大值
            xticklabels=True, yticklabels=True,  # 显示x轴和y轴
            square=True,  # 每个方格都是正方形
            cbar=True,  # 绘制颜色条
            cmap='coolwarm_r',  # 设置热力图颜色
            )
plt.show() #显示图片

mean_list = [data_quanzhan['播放数'].mean(),data_wudao['播放数'].mean(),data_shenghuo['播放数'].mean(),data_meishi['播放数'].mean(),data_yinyue['播放数'].mean()]
name_list=['全站','舞蹈','生活','美食','音乐']
plt.bar(name_list,mean_list)
plt.ylabel('播放数')
plt.show()

var_list = [data_quanzhan['播放数'].var(),data_wudao['播放数'].var(),data_shenghuo['播放数'].var(),data_meishi['播放数'].var(),data_yinyue['播放数'].var()]
plt.bar(name_list,var_list)
plt.ylabel('播放数方差')

# 文本预处理
data_quanzhan['视频标题'] = data_quanzhan['视频标题'].apply(lambda x: ' '.join(jieba.lcut(x))) # 分词
# 读取停用词
with open('./停用词表.txt', 'r', encoding='utf-8') as f:
    stop_words = f.read().split('\n')
    data_quanzhan['视频标题'] = data_quanzhan['视频标题'].apply(lambda x: ' '.join([word for word in x.split() if word not in stop_words])) # 去除停用词
# 统计单词出现频率
words = []
for comment in data_quanzhan['视频标题']:
    words += comment.split()
    word_count = Counter(words).most_common(100)
# 转成字典
dct = {k:v for k, v in word_count}
print(dct)
# 生成词云图
font_path = r'C:\Windows\Fonts\msyh.ttc' # 指定微软雅黑字体路径
wordcloud = WordCloud(width=800, height=600, background_color='white',font_path=font_path).generate_from_frequencies(dct)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.show()

def panduan(x):
    if x in data_meishi['作者'].tolist():
        return '美食'
    elif x in data_shenghuo['作者'].tolist():
        return '生活'
    elif x in data_yinyue['作者'].tolist():
        return '音乐'
    elif x in data_wudao['作者'].tolist():
        return '舞蹈'
    else:
        return '其他'
data_quanzhan['类别'] = data_quanzhan['作者'].apply(panduan)
data_pie = data_quanzhan['类别'].value_counts()
plt.pie(data_pie.values,labels=data_pie.index,autopct="%3.1f%%") 
plt.show()