修改了省份表和城市表的结构，解决重复省份不能插入问题

5 years ago · c23a6852e1
parent fffe76ee5a
commit c23a6852e1
6 changed files with 35 additions and 39 deletions
--- a/数据存储/.idea/workspace.xml
+++ b/数据存储/.idea/workspace.xml
@ -3,7 +3,6 @@
  <component name="ChangeListManager">
    <list default="true" id="fcea9d71-f4b2-41ab-b680-ecae8402b966" name="Default Changelist" comment="">
      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/DataFrame.py" beforeDir="false" afterPath="$PROJECT_DIR$/DataFrame.py" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/GrapData.py" beforeDir="false" afterPath="$PROJECT_DIR$/GrapData.py" afterDir="false" />
      <change beforePath="$PROJECT_DIR$/Grapmain.py" beforeDir="false" afterPath="$PROJECT_DIR$/Grapmain.py" afterDir="false" />
    </list>
@ -80,22 +79,26 @@
    <servers />
  </component>
  <component name="WindowStateProjectService">
-    <state width="1814" height="241" key="GridCell.Tab.0.bottom" timestamp="1596610032433">
+    <state width="1814" height="241" key="GridCell.Tab.0.bottom" timestamp="1596675433318">
      <screen x="67" y="27" width="1853" height="1053" />
    </state>
-    <state width="1814" height="241" key="GridCell.Tab.0.bottom/67.27.1853.1053@67.27.1853.1053" timestamp="1596610032433" />
-    <state width="1814" height="241" key="GridCell.Tab.0.center" timestamp="1596610032431">
+    <state width="1814" height="241" key="GridCell.Tab.0.bottom/67.27.1853.1053@67.27.1853.1053" timestamp="1596675433318" />
+    <state width="1814" height="241" key="GridCell.Tab.0.center" timestamp="1596675433318">
      <screen x="67" y="27" width="1853" height="1053" />
    </state>
-    <state width="1814" height="241" key="GridCell.Tab.0.center/67.27.1853.1053@67.27.1853.1053" timestamp="1596610032431" />
-    <state width="1814" height="241" key="GridCell.Tab.0.left" timestamp="1596610032430">
+    <state width="1814" height="241" key="GridCell.Tab.0.center/67.27.1853.1053@67.27.1853.1053" timestamp="1596675433318" />
+    <state width="1814" height="241" key="GridCell.Tab.0.left" timestamp="1596675433317">
      <screen x="67" y="27" width="1853" height="1053" />
    </state>
-    <state width="1814" height="241" key="GridCell.Tab.0.left/67.27.1853.1053@67.27.1853.1053" timestamp="1596610032430" />
-    <state width="1814" height="241" key="GridCell.Tab.0.right" timestamp="1596610032432">
+    <state width="1814" height="241" key="GridCell.Tab.0.left/67.27.1853.1053@67.27.1853.1053" timestamp="1596675433317" />
+    <state width="1814" height="241" key="GridCell.Tab.0.right" timestamp="1596675433318">
      <screen x="67" y="27" width="1853" height="1053" />
    </state>
-    <state width="1814" height="241" key="GridCell.Tab.0.right/67.27.1853.1053@67.27.1853.1053" timestamp="1596610032432" />
+    <state width="1814" height="241" key="GridCell.Tab.0.right/67.27.1853.1053@67.27.1853.1053" timestamp="1596675433318" />
+    <state x="481" y="302" key="com.intellij.ide.util.TipDialog" timestamp="1596673278210">
+      <screen x="67" y="27" width="1853" height="1053" />
+    </state>
+    <state x="481" y="302" key="com.intellij.ide.util.TipDialog/67.27.1853.1053@67.27.1853.1053" timestamp="1596673278210" />
    <state x="1111" y="25" key="search.everywhere.popup" timestamp="1596606706048">
      <screen x="67" y="27" width="1853" height="1053" />
    </state>
--- a/数据存储/GrapData.py
+++ b/数据存储/GrapData.py
@ -66,22 +66,10 @@ class Spider():
                citysinfo.city=cj['city']
                citysinfo.confirmed=int(cj['confirmed'])
                #需要对缺失数据进行处理
-                if cj['died']=='':
-                    citysinfo.died=0
-                else:
-                    citysinfo.died=int(cj['died'])
-                if cj['crued']=='':
-                    citysinfo.cured=0
-                else:
-                    citysinfo.cured=int(cj['crued'])
-                citysinfo.confirmedRelative=int(cj['confirmedRelative'])
-                try:
-                    if cj['curConfirm']=='':
-                        citysinfo.curConfirm=0
-                    else:
-                        citysinfo.curConfirm=int(cj['curConfirm'])
-                except:
-                    citysinfo.curConfirm = 0
+                citysinfo.died=int(self.dealData(cj,'died'))
+                citysinfo.cured=int(self.dealData(cj,'crued'))
+                citysinfo.confirmedRelative=int(self.dealData(cj,'confirmedRelative'))
+                citysinfo.curConfirm=int(self.dealData(cj,'curConfirm'))
                province.subList.append(citysinfo)
                ctuple=(citysinfo.city,province.area,citysinfo.confirmed,citysinfo.died,citysinfo.cured,citysinfo.confirmed,
                        citysinfo.curConfirm,province.pub_date)
@ -122,8 +110,6 @@ class Spider():
            foreignTuples.append(ftuples)
        return foreignTuples

-
-
    #抓取国外疫情信息概况
    def grapSummaryForeign(self):
        data=requests.get(self.url,headers=self.headers)
@ -152,7 +138,6 @@ class Spider():
                Outside.cured,Outside.curedRelative,Outside.died,Outside.diedRelative,
                Outside.updatedTime)

-
    #抓取国内疫情信息概况
    def grapSummaryChina(self):
        data=requests.get(self.url,headers=self.headers)
--- a/数据存储/Grapmain.py
+++ b/数据存储/Grapmain.py
@ -1,16 +1,24 @@
 from GrapData import Spider,SaveToDB

-def main():
+#抓取所有数据填入数据库
+def updateDB():
    spider=Spider()
+    saveDb=SaveToDB()
    spider.url='https://voice.baidu.com/act/newpneumonia/newpneumonia/'
-    #spider.grapProvince()
-    savedb=SaveToDB()
-    #foreignData=spider.grapForeign()
-    #国内疫情概述
-    #InsideData=spider.grapSummaryChina()
-    #国外疫情汇总
+    proData,cityData=spider.grapProvince()
+    saveDb.InsertIntoProvince(proData)
+    saveDb.InsertToCity(cityData)
+    foreignData = spider.grapForeign()
+    saveDb.InsertForeignCountry(foreignData)
+    InsideData=spider.grapSummaryChina()
+    saveDb.InsertSummaryChina(InsideData)
    OutsideData=spider.grapSummaryForeign()
-    savedb.InsertSummaryForeign(OutsideData)
+    saveDb.InsertSummaryForeign(OutsideData)
+
+
+
+def main():
+    updateDB()

 if __name__ == '__main__':
    main()
--- a/数据存储/pycache/DataFrame.cpython-36.pyc
+++ b/数据存储/pycache/DataFrame.cpython-36.pyc
--- a/数据存储/pycache/GrapData.cpython-36.pyc
+++ b/数据存储/pycache/GrapData.cpython-36.pyc
--- a/数据存储/xinguan.sql
+++ b/数据存储/xinguan.sql
@ -10,7 +10,7 @@ create table ForeignCountry(
 	country varchar(30) unique,
 	curConfirm int,
 	confirmedRelative int,
-	pub_data DATETIME not null
+	pub_date DATETIME not null
 	);
 	
 create table InsideChina(
@ -56,7 +56,6 @@ create table CityTable(
 	confirmedRelative int,
 	curConfirm int,
 	pub_date DATETIME,
-	foreign key(provinceName) references ProvinceTable(area)
 	);
 	
 create table ProvinceTable(
@ -70,8 +69,9 @@ create table ProvinceTable(
 	asymptomatic int,
 	asymptomaticRelative int,
 	curConfirm int,
+	curConfirmRelative int,
 	icuDisable int,
-	area varchar(10) unique,
+	area varchar(10),
 	pub_date DATETIME not null
 	);