|
|
|
|
|
import requests
|
|
|
|
|
|
import random
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
from lxml import etree
|
|
|
|
|
|
import time
|
|
|
|
|
|
import re
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
|
|
from tqdm import *
|
|
|
|
|
|
from multiprocessing.pool import ThreadPool
|
|
|
|
|
|
from user_agents_pool import *
|
|
|
|
|
|
|
|
|
|
|
|
url_1 = ['https://ivdc.chinacdc.cn/cnic/zyzx/lgzb/index.htm']
|
|
|
|
|
|
url_list2 = [
|
|
|
|
|
|
f'https://ivdc.chinacdc.cn/cnic/zyzx/lgzb/index_{i}.htm'
|
|
|
|
|
|
for i in range(1, 4)
|
|
|
|
|
|
]
|
|
|
|
|
|
url_list = url_1 + url_list2
|
|
|
|
|
|
|
|
|
|
|
|
user_Agent = random.choice(agent_list)
|
|
|
|
|
|
headers = {
|
|
|
|
|
|
"User-Agent": user_Agent,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_Link(url):
|
|
|
|
|
|
link_list = []
|
|
|
|
|
|
response = requests.get(url=url, headers=headers)
|
|
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
html = response.content.decode("utf-8")
|
|
|
|
|
|
tree = etree.HTML(html)
|
|
|
|
|
|
li_list = tree.xpath('/html/body/div[2]/div/div[1]/div/div[2]/ul/li')
|
|
|
|
|
|
# print(len(li_list))
|
|
|
|
|
|
for table in li_list:
|
|
|
|
|
|
link = table.xpath("./span[1]/a/@href")[0]
|
|
|
|
|
|
link = link.replace('.', '')
|
|
|
|
|
|
url_head = "https://ivdc.chinacdc.cn/cnic/zyzx/lgzb"
|
|
|
|
|
|
link = url_head + link
|
|
|
|
|
|
link = link.replace('htm', '.htm')
|
|
|
|
|
|
link_list.append(link)
|
|
|
|
|
|
return link_list
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_content(link):
|
|
|
|
|
|
response = requests.get(url=link, headers=headers)
|
|
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
html = response.content.decode("utf-8")
|
|
|
|
|
|
# print(html)
|
|
|
|
|
|
tree = etree.HTML(html)
|
|
|
|
|
|
date = tree.xpath(
|
|
|
|
|
|
'/html/body/div[2]/div/div[1]/div/div[2]/div/div/div/p[1]/span/text()'
|
|
|
|
|
|
)[1]
|
|
|
|
|
|
# print(time)
|
|
|
|
|
|
year = tree.xpath(
|
|
|
|
|
|
'/html/body/div[2]/div/div[1]/div/div[2]/div/div/div/p[1]/span/span/text()'
|
|
|
|
|
|
)[0]
|
|
|
|
|
|
# print(year)
|
|
|
|
|
|
date = year + date
|
|
|
|
|
|
date = date.replace('）', '')
|
|
|
|
|
|
date_format = '%Y年%m月%d日'
|
|
|
|
|
|
target_date = datetime.strptime(date, date_format)
|
|
|
|
|
|
# print(target_date)
|
|
|
|
|
|
start_time = '2023年2月18日'
|
|
|
|
|
|
start_date = datetime.strptime(start_time, date_format)
|
|
|
|
|
|
if target_date > start_date:
|
|
|
|
|
|
specific_number = re.search(
|
|
|
|
|
|
r'(.?<=font-size: 10pt;\">|<span lang=\"EN-US\">)(\d+)(?=</span>起|起)',
|
|
|
|
|
|
html)
|
|
|
|
|
|
number = specific_number.group(2) if specific_number else None
|
|
|
|
|
|
if number == None:
|
|
|
|
|
|
pattern = r'<span lang="EN-US" style="font-size: 10pt;">(\d+)</span><span style="font-size: 10pt'
|
|
|
|
|
|
number_list = re.findall(pattern, html)
|
|
|
|
|
|
if number_list:
|
|
|
|
|
|
number = number_list[0]
|
|
|
|
|
|
else:
|
|
|
|
|
|
number = 0
|
|
|
|
|
|
# print(html)
|
|
|
|
|
|
return [date, number]
|
|
|
|
|
|
else:
|
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_liuganzhoubao():
|
|
|
|
|
|
link_list_all = []
|
|
|
|
|
|
for url in url_list:
|
|
|
|
|
|
link_list_all += get_Link(url)
|
|
|
|
|
|
link_list_all = list(reversed(link_list_all))
|
|
|
|
|
|
|
|
|
|
|
|
data_all = []
|
|
|
|
|
|
# 使用多进程处理
|
|
|
|
|
|
pool = ThreadPool(30)
|
|
|
|
|
|
data_list = pool.map(get_content, link_list_all)
|
|
|
|
|
|
pool.close()
|
|
|
|
|
|
pool.join()
|
|
|
|
|
|
for data in data_list:
|
|
|
|
|
|
if data:
|
|
|
|
|
|
data_all.append(data)
|
|
|
|
|
|
print(data_all)
|
|
|
|
|
|
|
|
|
|
|
|
df = pd.DataFrame(data_all, columns=['date', 'infection_number'])
|
|
|
|
|
|
# 将日期列转换为日期时间类型
|
|
|
|
|
|
df['date'] = pd.to_datetime(df['date'], format='%Y年%m月%d日')
|
|
|
|
|
|
# 将日期时间类型列格式化为所需的字符串格式
|
|
|
|
|
|
df['date'] = df['date'].dt.strftime('%Y-%m-%d')
|
|
|
|
|
|
print(df)
|
|
|
|
|
|
df.to_csv('liugan_zhoubao.csv', encoding='utf-8')
|
|
|
|
|
|
print('流感周报数据已经储存在liugan_zhoubao.csv中')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 调用函数
|
|
|
|
|
|
get_liuganzhoubao()
|
