mux4etwkg
/
p4fkfpicj
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

一份完美的封装成函数的爬取并存储国内疫情概况的代码

Browse Source
dev_kfq
Chinaholmesfans 5 years ago
parent 1dab372b20
commit 423815b295
1 changed files with 124 additions and 121 deletions
Show Stats Download Patch File Download Diff File

169
InsideSummary.py
Unescape View File

@ -1,124 +1,127 @@
import re
import json import json
import requests import requests
import pymysql import pymysql
import datetime
################################################################################################## #######################################################################################################################
#爬取数据 #爬取国内疫情数据概况
# 国内疫情数据概况 def InsideSummary():
class InsideSummary:
class class_InsideSummary:
def __init__(self): def __init__(self):
self.curConfirm = 0 #现有确诊 self.curConfirm = 0 # 现有确诊
self.curConfirmRelative = 0#较昨日新增确诊 self.curConfirmRelative = 0 # 较昨日新增确诊
self.asymptomatic = 0 #无症状感染 self.asymptomatic = 0 # 无症状感染
self.asymptomaticRelative = 0 #较昨日新增无症状感染 self.asymptomaticRelative = 0 # 较昨日新增无症状感染
self.unconfirmed = 0 #现有疑似 self.unconfirmed = 0 # 现有疑似
self.unconfirmedRelative = 0#较昨日疑似新增 self.unconfirmedRelative = 0 # 较昨日疑似新增
self.icu = 0 #现有重症 self.icu = 0 # 现有重症
self.icuRelative = 0 #较昨日重症病例新增 self.icuRelative = 0 # 较昨日重症病例新增
self.confirmed = 0 #累计确诊 self.confirmed = 0 # 累计确诊
self.confirmedRelative = 0 #较昨日累计确诊新增 self.confirmedRelative = 0 # 较昨日累计确诊新增
self.overseasInput = 0 #累计境外输入 self.overseasInput = 0 # 累计境外输入
self.overseasInputRelative = 0#较昨日累计境外输入新 增 self.overseasInputRelative = 0 # 较昨日累计境外输入新 增
self.cured = 0 #累计治愈 self.cured = 0 # 累计治愈
self.curedRelative = 0 #较昨日累计治愈新增 self.curedRelative = 0 # 较昨日累计治愈新增
self.died = 0 #累计死亡 self.died = 0 # 累计死亡
self.diedRelative = 0#较昨日累计死亡新增 self.diedRelative = 0 # 较昨日累计死亡新增
self.updatedTime = 0 #发布时间 self.updatedTime = 0 # 发布时间
# 数据库接口
# 返回元组
def get_inside_summary_tuple(self): def get_inside_summary_tuple(self):
return ((self.curConfirm, self.curConfirmRelative, self.asymptomatic, self.asymptomaticRelative, \ return ((self.curConfirm, self.curConfirmRelative, self.asymptomatic, self.asymptomaticRelative, \
self.unconfirmed, self.unconfirmedRelative, self.icu, self.icuRelative, self.confirmed, \ self.unconfirmed, self.unconfirmedRelative, self.icu, self.icuRelative, self.confirmed, \
self.confirmedRelative, self.overseasInput, self.overseasInputRelative, self.cured, self.curedRelative, \ self.confirmedRelative, self.overseasInput, self.overseasInputRelative, self.cured,
self.curedRelative, \
self.died, self.diedRelative, self.updatedTime)) self.died, self.diedRelative, self.updatedTime))
# 输出接口 # 输出接口
def __str__(self): def __str__(self):
return '%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s' % ( return '%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s' % (
self.curConfirm, self.curConfirmRelative, self.asymptomatic, self.asymptomaticRelative, self.unconfirmed, self.curConfirm, self.curConfirmRelative, self.asymptomatic, self.asymptomaticRelative,
self.unconfirmed,
self.unconfirmedRelative, self.icu, self.icuRelative, self.confirmed, self.confirmedRelative, self.unconfirmedRelative, self.icu, self.icuRelative, self.confirmed, self.confirmedRelative,
self.overseasInput, self.overseasInputRelative, self.cured, self.curedRelative, self.died, self.diedRelative, self.overseasInput, self.overseasInputRelative, self.cured, self.curedRelative, self.died,
self.diedRelative,
self.updatedTime) self.updatedTime)
#######################################################################################################
#业务逻辑类
# 爬取国内疫情数据
res=requests.get('https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5')
#解码
res=res.content.decode('utf-8')
dict=json.loads(res) #str->dict
#规范数据 def get_text():# 爬取国内疫情数据文本
for key in dict: res = requests.get('https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5')
res = res.content.decode('utf-8')
dict = json.loads(res) # str->dict
# 规范数据
for key in dict:
try: try:
dict[key] = dict[key].replace('\\', '') dict[key] = dict[key].replace('\\', '')
key=key.replace('\\','') key = key.replace('\\', '')
except: except:
pass pass
#新生成data字典 data = json.loads(dict['data'])
data=json.loads(dict['data'])
return data
# 国内疫情数据实例化
# # #赋值
def fetch_inside_summary(data):
insideSummary = InsideSummary()
insideSummary.curConfirm = int(data['chinaTotal']['nowConfirm']) # 国内疫情数据赋值
insideSummary.curConfirmRelative = int(data['chinaAdd']['nowConfirm']) def fetch_inside_summary():
insideSummary.asymptomatic = int(data['chinaTotal']['noInfect'])
insideSummary.asymptomaticRelative = int(data['chinaAdd']['noInfect']) dataf=get_text()
insideSummary.unconfirmed = int(data['chinaTotal']['suspect']) insideSummary = class_InsideSummary()
insideSummary.unconfirmedRelative = int(data['chinaAdd']['suspect']) insideSummary.curConfirm = int(dataf['chinaTotal']['nowConfirm'])
insideSummary.icu = int(data['chinaTotal']['nowSevere']) insideSummary.curConfirmRelative = int(dataf['chinaAdd']['nowConfirm'])
insideSummary.icuRelative = int(data['chinaAdd']['nowSevere']) insideSummary.asymptomatic = int(dataf['chinaTotal']['noInfect'])
insideSummary.confirmed = int(data['chinaTotal']['confirm']) insideSummary.asymptomaticRelative = int(dataf['chinaAdd']['noInfect'])
insideSummary.updatedTime = data['lastUpdateTime'] insideSummary.unconfirmed = int(dataf['chinaTotal']['suspect'])
insideSummary.confirmedRelative = int(data['chinaAdd']['confirm']) insideSummary.unconfirmedRelative = int(dataf['chinaAdd']['suspect'])
insideSummary.overseasInput = int(data['chinaTotal']['importedCase']) insideSummary.icu = int(dataf['chinaTotal']['nowSevere'])
insideSummary.overseasInputRelative = int(data['chinaAdd']['importedCase']) insideSummary.icuRelative = int(dataf['chinaAdd']['nowSevere'])
insideSummary.cured = int(data['chinaTotal']['heal']) insideSummary.confirmed = int(dataf['chinaTotal']['confirm'])
insideSummary.curedRelative = int(data['chinaAdd']['heal']) insideSummary.updatedTime = dataf['lastUpdateTime']
insideSummary.died = int(data['chinaTotal']['dead']) insideSummary.confirmedRelative = int(dataf['chinaAdd']['confirm'])
insideSummary.diedRelative = int(data['chinaAdd']['dead']) insideSummary.overseasInput = int(dataf['chinaTotal']['importedCase'])
insideSummary.overseasInputRelative = int(dataf['chinaAdd']['importedCase'])
insideSummary.cured = int(dataf['chinaTotal']['heal'])
insideSummary.curedRelative = int(dataf['chinaAdd']['heal'])
insideSummary.died = int(dataf['chinaTotal']['dead'])
insideSummary.diedRelative = int(dataf['chinaAdd']['dead'])
return insideSummary return insideSummary
##########################################################################################################
#存储数据于数据库
#保存国内疫情概况数据
def insert(res):
# 保存国内疫情概况数据
def insert(res):
# 创建连接，并且返回连接对象 # 创建连接，并且返回连接对象
conn =pymysql.connect(host='localhost',user='root',db='covid19_datas_guangxi',passwd='kfq991122',charset='utf8') conn = pymysql.connect(host='localhost', user='root', db='covid19', passwd='kfq991122')
# 创建游标对象 # 创建游标对象
cursor = conn.cursor() cursor = conn.cursor()
date = data['lastUpdateTime']
date = data['lastUpdateTime'].split(' ',1)[0] sql = 'delete from home_realtime_datas where updatedTime like "%s"' % (date + '%')
print(date) print('delete old_insidesummary_datas successful')
sql = 'delete from home_realtime_datas where updatedTime like "%s"'%(date+'%')
print('The update was successful')
try: try:
cursor.execute(sql) cursor.execute(sql)
conn.commit() conn.commit()
except Exception as e: except Exception as e:
print(e) print(e)
sql = 'insert into home_realtime_datas(curConfirm,curConfirmRelative,asymptomatic,asymptomaticRelative,unconfirmed,unconfirmedRelative,icu,icuRelative,confirmed,confirmedRelative,overseasInput,overseasInputRelative,cured,curedRelative,died,diedRelative,updatedTime) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql='insert into home_realtime_datas(curConfirm,curConfirmRelative,asymptomatic,asymptomaticRelative,unconfirmed,unconfirmedRelative,icu,icuRelative,confirmed,confirmedRelative,overseasInput,overseasInputRelative,cured,curedRelative,died,diedRelative,updatedTime) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
try: try:
cursor.execute(sql, res) cursor.execute(sql, res)
conn.commit() conn.commit()
print("+++ save_insidesummary_datas successful")
print('+++ save_outsideSummary_datas, data len: %d' % len(res))
except Exception as e: except Exception as e:
print(e) print(e)
print('+++ save_province_datas is over.') print('+++ save_insidesummary_datas fail.')
print('+++ save_insidesummary_datas is over.')
cursor.close() cursor.close()
conn.close() conn.close()
insert(fetch_inside_summary(data).get_inside_summary_tuple()) ########InsideSummary__main__####################
data = get_text()
insert(fetch_inside_summary().get_inside_summary_tuple())
conn = pymysql.connect('localhost','root','kfq991122','covid19_datas_guangxi') InsideSummary()
cursor = conn.cursor()
sql = 'select * from home_realtime_datas'
cursor.execute(sql)
results = cursor.fetchall()
print(results)
Write Preview
Loading…
Cancel
Save