|
|
import requests
|
|
|
from pylab import mpl
|
|
|
import pandas as pd
|
|
|
import time
|
|
|
from datetime import datetime, timedelta, date
|
|
|
|
|
|
mpl.rcParams["font.sans-serif"] = ["SimHei"]
|
|
|
mpl.rcParams["axes.unicode_minus"] = False
|
|
|
|
|
|
|
|
|
class DownloadBaiDuIndex(object):
|
|
|
# 创建一个类来下载百度指数
|
|
|
def __init__(self, cookie):
|
|
|
self.cookie = cookie
|
|
|
# 配置请求头
|
|
|
self.headers = {
|
|
|
"Connection":
|
|
|
"keep-alive",
|
|
|
"Accept":
|
|
|
"application/json, text/plain, */*",
|
|
|
"User-Agent":
|
|
|
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
|
|
|
"Sec-Fetch-Site":
|
|
|
"same-origin",
|
|
|
"Sec-Fetch-Mode":
|
|
|
"cors",
|
|
|
"Sec-Fetch-Dest":
|
|
|
"empty",
|
|
|
"Referer":
|
|
|
"https://index.baidu.com/v2/main/index.html",
|
|
|
"Accept-Language":
|
|
|
"zh-CN,zh;q=0.9",
|
|
|
'Cookie':
|
|
|
self.cookie,
|
|
|
"Host":
|
|
|
"index.baidu.com",
|
|
|
"X-Requested-With":
|
|
|
"XMLHttpRequest",
|
|
|
"Cipher-Text":
|
|
|
"1656572408684_1656582701256_Nvm1pABkNsfD7V9VhZSzzFiFKylr3l5NR3YDrmHmH9yfFicm+Z9kmmwKVqVV6unvzAEh5hgXmgelP+OyOeaK8F21LyRVX1BDjxm+ezsglwoe1yfp6lEpuvu5Iggg1dz3PLF8e2II0e80ocXeU0jQFBhSbnB2wjhKl57JggTej12CzuL+h9eeVWdaMO4DSBWU2XX6PfbN8pv9+cdfFhVRHCzb0BJBU3iccoFczwNQUvzLn0nZsu0YPtG5DxDkGlRlZrCfKMtqKAe1tXQhg3+Oww4N3CQUM+6A/tKZA7jfRE6CGTFetC7QQyKlD7nxabkQ5CReAhFYAFAVYJ+sEqmY5pke8s3+RZ6jR7ASOih6Afl35EArbJzzLpnNPgrPCHoJiDUlECJveul7P5vvXl/O/Q==",
|
|
|
}
|
|
|
|
|
|
def decrypt(self, ptbk, index_data):
|
|
|
n = len(ptbk) // 2
|
|
|
a = dict(zip(ptbk[:n], ptbk[n:]))
|
|
|
return "".join([a[s] for s in index_data])
|
|
|
|
|
|
def get_index_data_json(self, keys, start=None, end=None):
|
|
|
words = [[{"name": key, "wordType": 1}] for key in keys]
|
|
|
words = str(words).replace(" ", "").replace("'", "\"")
|
|
|
url = f'http://index.baidu.com/api/SearchApi/index?area=0&word={words}&area=0&startDate={start}&endDate={end}'
|
|
|
res = requests.get(url, headers=self.headers)
|
|
|
html = res.content.decode("UTF-8")
|
|
|
data = res.json()['data']
|
|
|
uniqid = data['uniqid']
|
|
|
url = f'http://index.baidu.com/Interface/ptbk?uniqid={uniqid}'
|
|
|
# print(url)
|
|
|
res = requests.get(url, headers=self.headers)
|
|
|
html2 = res.content.decode("UTF-8")
|
|
|
time.sleep(3)
|
|
|
ptbk = res.json()['data']
|
|
|
result = {}
|
|
|
result["startDate"] = start
|
|
|
result["endDate"] = end
|
|
|
for userIndexe in data['userIndexes']:
|
|
|
name = userIndexe['word'][0]['name']
|
|
|
tmp = {}
|
|
|
index_all = userIndexe['all']['data']
|
|
|
index_all_data = [
|
|
|
int(e) for e in self.decrypt(ptbk, index_all).split(",")
|
|
|
]
|
|
|
tmp["all"] = index_all_data
|
|
|
index_pc = userIndexe['pc']['data']
|
|
|
index_pc_data = [
|
|
|
int(e) for e in self.decrypt(ptbk, index_pc).split(",")
|
|
|
]
|
|
|
tmp["pc"] = index_pc_data
|
|
|
index_wise = userIndexe['wise']['data']
|
|
|
index_wise_data = [
|
|
|
int(e) for e in self.decrypt(ptbk, index_wise).split(",")
|
|
|
]
|
|
|
tmp["wise"] = index_wise_data
|
|
|
result[name] = tmp
|
|
|
return result
|
|
|
|
|
|
def GetIndex(self, keys, start=None, end=None):
|
|
|
today = date.today()
|
|
|
if start is None:
|
|
|
start = str(today - timedelta(days=8))
|
|
|
if end is None:
|
|
|
end = str(today - timedelta(days=2))
|
|
|
try:
|
|
|
raw_data = self.get_index_data_json(keys=keys,
|
|
|
start=start,
|
|
|
end=end)
|
|
|
raw_data = pd.DataFrame(raw_data[keys[0]])
|
|
|
raw_data.index = pd.date_range(start=start, end=end)
|
|
|
except Exception as e:
|
|
|
print(e)
|
|
|
raw_data = pd.DataFrame({'all': [], 'pc': [], 'wise': []})
|
|
|
# 分别表示总计，PC端，移动端
|
|
|
finally:
|
|
|
return raw_data
|
|
|
|
|
|
|
|
|
def get_baidu_index():
|
|
|
cookie = 'BIDUPSID=84B8FDC3134DE2D8E0E6B86E2BFCC3DC; \
|
|
|
PSTM=1697213335; \
|
|
|
BAIDUID=84B8FDC3134DE2D8E0E6B86E2BFCC3DC:SL=0:NR=10:FG=1; BAIDUID_BFESS=84B8FDC3134DE2D8E0E6B86E2BFCC3DC:SL=0:NR=10:FG=1; Hm_lvt_d101ea4d2a5c67dab98251f0b5de24dc=1701483117; BDUSS=RUU3ZtM0RwcU9VeW0zV0ltMGhWZXNvd3hoMXc3YmtoZmxOOXktTDNFM3JMNUpsRUFBQUFBJCQAAAAAAQAAAAEAAADwtxh-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOuiamXromplSH; SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a04514997999zSyIXXcI1QTeZqm4c8hyxlWksvkordeK7x1ZPceY2CR3NLufUujm7MOZ3p6TYUaUvd3Qjet3M3JcQfM5hy8%2FuP9HNu4dCG7B6RoS3S4L25PQZlnh3joEA0cArzaShqjtNyIlDOFD7nF4m%2FHL%2FxUXMnks0IYh6ZyO0xZ1iCY3pJruPDK3dBKJPJ%2BTsLIUPckisDLv5o4FBynumqVmNrIcRJauvv%2BcQtioTBjGMshtfwaZjDT2WCz713NtlH6uxabBdf8gRHMu6r8uSWjXKPG3dAflk5ycDG%2F1BoioLYK697k%3D91877884685963653296273632513192; __cas__rn__=451499799; __cas__st__212=b5f51a7b5b20cb36d3ced6764c8b0e567b436d1a2aa46e1f861833387e9d43267ac11419a4d630081274b162; __cas__id__212=51862268; CPTK_212=1671659797; CPID_212=51862268; bdindexid=473uetvtav5o3d1jfb3m9s3d34; RT="z=1&dm=baidu.com&si=0751b751-3767-4525-9566-4b5f1cd26e3a&ss=lpnhlcxe&sl=8&tt=fr3&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf"; Hm_lpvt_d101ea4d2a5c67dab98251f0b5de24dc=1701490081; ab_sr=1.0.1_MjQ2ODNmNmI4NzI5MzFhZDAxYzIzZDQzYmMyZDAwOTZiYWE5NDY4OGQxMDNkYzA0NGM4OGU1ZDk5YjZmYjdkMTkyNTYxMDJiZmVlMjllNGU1MWQ1YjgwYTAzZGQxMWFkYzEyMDQ3ZjYxMThkNWI1NTg1ZTliOWVmYTQ1M2E3NjhmMDUzNTllNjU3YzYwNDlhOTU0ODRhMzJlZDAwMWY5Yg==; BDUSS_BFESS=RUU3ZtM0RwcU9VeW0zV0ltMGhWZXNvd3hoMXc3YmtoZmxOOXktTDNFM3JMNUpsRUFBQUFBJCQAAAAAAQAAAAEAAADwtxh-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOuiamXromplSH'
|
|
|
|
|
|
# 初始化一个实例
|
|
|
downloadbaiduindex = DownloadBaiDuIndex(cookie=cookie)
|
|
|
# key = input('请输入关键词')
|
|
|
key = '流感'
|
|
|
# 获取当天时间
|
|
|
|
|
|
# from datetime import date
|
|
|
today = str(date.today())
|
|
|
data = downloadbaiduindex.get_index_data_json(keys=[key],
|
|
|
start='2012-01-01',
|
|
|
end=today)
|
|
|
liugan_data = (data['流感']['all'])
|
|
|
|
|
|
# 设定起始日期和终止日期
|
|
|
start_date = date(2012, 1, 1)
|
|
|
end_date = datetime.now().date() + timedelta(days=7)
|
|
|
|
|
|
# 创建日期列表，间隔为一周
|
|
|
date_list = []
|
|
|
current_date = start_date
|
|
|
while current_date <= end_date:
|
|
|
date_list.append(current_date)
|
|
|
current_date += timedelta(weeks=1) # 每次增加一周
|
|
|
date_list = date_list[:len(liugan_data)]
|
|
|
|
|
|
df = pd.DataFrame({'date': date_list, 'liugan_index': liugan_data})
|
|
|
df = df.drop(df.index[-1])
|
|
|
print(df)
|
|
|
# 数据保存
|
|
|
df.to_csv('./test/data/baidu_index.csv', encoding='utf-8')
|
|
|
print('成功爬取百度流感指数并储存在baidu_index.csv')
|
|
|
|
|
|
|
|
|
# 调用函数
|
|
|
get_baidu_index()
|
