|
|
import requests
|
|
|
from urllib.parse import urlencode
|
|
|
from pyquery import PyQuery as pq
|
|
|
import time
|
|
|
import xlwt
|
|
|
import jieba
|
|
|
from wordcloud import WordCloud
|
|
|
import matplotlib.pyplot as plt
|
|
|
from collections import Counter
|
|
|
|
|
|
# 设置代理等（新浪微博的数据是用ajax异步下拉加载的，network->xhr）
|
|
|
host = 'm.weibo.cn'
|
|
|
base_url = f'https://{host}/api/container/getIndex?'
|
|
|
|
|
|
# 设置请求头
|
|
|
headers = {
|
|
|
'Host': host,
|
|
|
'Referer': 'https://m.weibo.cn/search?containerid=231522type%3D1%26q%3D%23%E7%BE%8E%E5%9B%BD%E7%96%AB%E6%83%85%23',
|
|
|
'User-Agent': 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Mobile Safari/537.36'
|
|
|
}
|
|
|
|
|
|
# 按页数抓取数据
|
|
|
def get_single_page(page):
|
|
|
"""
|
|
|
获取单个页面的数据。
|
|
|
|
|
|
:param page: 页面编号
|
|
|
:return: JSON 数据
|
|
|
"""
|
|
|
params = {
|
|
|
'containerid': '231522type=1&q=#人工智能#',
|
|
|
'page_type': 'searchall',
|
|
|
'page': page
|
|
|
}
|
|
|
url = base_url + urlencode(params)
|
|
|
try:
|
|
|
response = requests.get(url, headers=headers)
|
|
|
if response.status_code == 200:
|
|
|
return response.json()
|
|
|
except requests.ConnectionError as e:
|
|
|
print('抓取错误:', e.args)
|
|
|
|
|
|
# 解析页面返回的JSON数据
|
|
|
count = 0
|
|
|
|
|
|
def parse_page(json_data):
|
|
|
"""
|
|
|
解析页面返回的JSON数据。
|
|
|
|
|
|
:param json_data: JSON 数据
|
|
|
:return: 解析后的数据
|
|
|
"""
|
|
|
global count
|
|
|
items = json_data.get('data').get('cards')
|
|
|
for item in items:
|
|
|
mblog = item.get('mblog')
|
|
|
if mblog:
|
|
|
data = {
|
|
|
'id': mblog.get('id'),
|
|
|
'created': mblog.get('created_at'),
|
|
|
'text': pq(mblog.get('text')).text() # 提取内容中的文本
|
|
|
}
|
|
|
yield data
|
|
|
count += 1
|
|
|
|
|
|
def segment_text(text):
|
|
|
"""
|
|
|
对文本进行分词处理。
|
|
|
|
|
|
:param text: 输入文本
|
|
|
:return: 分词后的列表
|
|
|
"""
|
|
|
seg_list = jieba.cut(text.strip())
|
|
|
return list(seg_list)
|
|
|
|
|
|
def generate_wordcloud(words):
|
|
|
"""
|
|
|
生成词云图。
|
|
|
|
|
|
:param words: 分词后的单词列表
|
|
|
"""
|
|
|
wordcloud = WordCloud(
|
|
|
font_path='simhei.ttf', # 指定字体路径
|
|
|
background_color='white',
|
|
|
width=800,
|
|
|
height=600
|
|
|
).generate(' '.join(words))
|
|
|
|
|
|
# 显示词云图
|
|
|
plt.imshow(wordcloud, interpolation='bilinear')
|
|
|
plt.axis('off')
|
|
|
plt.show()
|
|
|
|
|
|
# 保存词云图
|
|
|
wordcloud.to_file('fjt.png')
|
|
|
|
|
|
def predict_event_probability(words, event_keywords):
|
|
|
"""
|
|
|
根据关键词预测事件发生的概率。
|
|
|
|
|
|
:param words: 分词后的单词列表
|
|
|
:param event_keywords: 事件关键词列表
|
|
|
:return: 事件发生的概率
|
|
|
"""
|
|
|
word_count = Counter(words)
|
|
|
total_words = sum(word_count.values())
|
|
|
event_word_count = sum(word_count[word] for word in event_keywords if word in word_count)
|
|
|
probability = event_word_count / total_words if total_words > 0 else 0
|
|
|
return probability
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
workbook = xlwt.Workbook(encoding='utf-8')
|
|
|
worksheet = workbook.add_sheet('人工智能')
|
|
|
|
|
|
# 设置列标题
|
|
|
worksheet.write(0, 0, '创建时间')
|
|
|
worksheet.write(0, 1, '文本')
|
|
|
worksheet.write(0, 2, '分词结果')
|
|
|
worksheet.write(0, 3, '事件概率')
|
|
|
|
|
|
row = 1
|
|
|
|
|
|
all_words = []
|
|
|
event_keywords = ['人工智能']
|
|
|
|
|
|
for page in range(1, 5):
|
|
|
json_data = get_single_page(page)
|
|
|
results = parse_page(json_data)
|
|
|
tmp_list = []
|
|
|
|
|
|
for result in results:
|
|
|
created = result.get('created').strip('\n')
|
|
|
text = result.get('text').strip('\n')
|
|
|
segmented_text = segment_text(text)
|
|
|
|
|
|
# 写入创建时间和文本
|
|
|
worksheet.write(row, 0, label=created)
|
|
|
worksheet.write(row, 1, label=text)
|
|
|
|
|
|
# 写入分词结果
|
|
|
for idx, word in enumerate(segmented_text):
|
|
|
worksheet.write(row + idx, 2, label=word)
|
|
|
|
|
|
all_words.extend(segmented_text) # 添加分词结果到列表
|
|
|
|
|
|
# 计算事件概率
|
|
|
event_probability = predict_event_probability(segmented_text, event_keywords)
|
|
|
worksheet.write(row, 3, label=event_probability)
|
|
|
|
|
|
row += len(segmented_text) # 更新行号
|
|
|
|
|
|
time.sleep(1) # 爬取时间间隔
|
|
|
workbook.save('fjt.xls')
|
|
|
|
|
|
# 生成词云图
|
|
|
generate_wordcloud(all_words)
|
