plxhw7zgq/spider.py

import requests,json
from pymysql import *
from bs4 import BeautifulSoup
import datetime,xingZheng
def hello():
    return '我是hello'
def connectDB():
    try:
        db = connect(host='localhost', port=3306, user='root', password='759521', db='FeiYan')
        print("数据库连接成功")
        return db
    except Exception as e:
        print(e)
    return


db = connectDB()

def insert_city(db,updata_time,province_name,name,now_confirm,confirm,dead):
    cursor=db.cursor()
    try:
        cursor.execute("insert into city(update_time,province_name,name,now_confirm,confirm,dead) values ('%s','%s','%s','%s','%s','%s')"%(updata_time,province_name,name,now_confirm,confirm,dead))
        print('成功插入')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        print(e)
        db.rollback()
    return False
def insert_country(dazhou_name,name,confirm,dead,now_confirm):
    cursor=db.cursor()
    try:
        cursor.execute("insert into country(dazhou_name,name,confirm,dead,now_confirm)values ('%s','%s','%s','%s','%s')"%(dazhou_name,name,confirm,dead,now_confirm))
        print('插入成功')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        print(e)
        db.rollback()
    return False
#城市疫情表插入数据

def insert_rumor(neirong,lunduan,zhengwen,bool):
    cursor = db.cursor()
    try:
        cursor.execute(
            "insert into rumor(neirong,lunduan,zhengwen,bool)values ('%s','%s','%s','%s')" % (neirong,lunduan,zhengwen,bool))
        print('插入成功')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        db.rollback()
    return False
def insert_hospital(name,dizhi,cityname,shuxing):
    cursor = db.cursor()
    try:
        cursor.execute(
            "insert into hospital(hos_name,location,city_name,shuxing)values ('%s','%s','%s','%s')" % (name,dizhi,cityname,shuxing))
        print('插入成功')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        print(e)
        db.rollback()
def insert_prov(province_name,city_name):
    cursor = db.cursor()
    try:
        cursor.execute(
            "insert into province(province_name,city_name)values ('%s','%s')" % (province_name,city_name))
        print('插入成功')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        print(e)
        db.rollback()

def insert_dazhou(dazhou_name,country_name):
    cursor = db.cursor()
    try:
        cursor.execute(
            "insert into dazhou(dazhou_name,country_name)values ('%s','%s')" % (dazhou_name,country_name))
        print('插入成功')
        db.commit()
        cursor.close()
        return True
    except Exception as e:
        print(e)
        db.rollback()
def spider_city():
    one=requests.get(url='https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5').json()
    print(one)
    data=json.loads(one['data'])
    update_time=data['lastUpdateTime']
    all_counties=data['areaTree']
    print(data)
    print(update_time)
    print(all_counties)
    all_list=[]
    for country_data in all_counties:
        if country_data['name']=='中国':
            all_provinces=country_data['children']
            for province_data in all_provinces:
                province_name= province_data['name']
                all_cities = province_data['children']
                for city_data in all_cities:
                    city_name = city_data['name']
                    city_total = city_data['total']
                    province_result = {'province': province_name, 'city': city_name, 'update_time': update_time}
                    province_result.update(city_total)
                    all_list.append(province_result)
    for i in all_list:
        print(i)
        insert_city(db,updata_time=i.get('update_time'),province_name=i.get('province'),name=i.get('city'),now_confirm=i.get('nowConfirm'),confirm=i.get('confirm'),dead=i.get('dead'))


    print(all_list)
    print('国内完毕')
    return

def spider_foreign():
    url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0'  # 请求地址
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}  # 创建头部信息
    response = requests.get(url, headers=headers)  # 发送网络请求
    content = response.content.decode('utf-8')
    soup = BeautifulSoup(content, 'html.parser')

    # 世界确诊getAreaStat
    listB = soup.find_all(name='script', attrs={"id": "getListByCountryTypeService2true"})

    world_messages = str(listB)[95:-21]

    print(world_messages)
    world_messages_json = json.loads(world_messages)
    worldList = []
    now_time = datetime.datetime.now().strftime('%Y-%m-%d')
    for k in world_messages_json:
        name=k.get('provinceName')
        dazhou=k.get('continents')
        now_confirm=k.get('currentConfirmedCount')
        confirm=k.get('confirmedCount')
        dead=k.get('deadCount')
        insert_country(dazhou_name=dazhou,name=name,confirm=confirm,dead=dead,now_confirm=now_confirm)
        print(k)
    print('国外完毕')
    return
def spider_rumor():
    url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia_rumors?target=5&from=dxy'  # 请求地址
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}  # 创建头部信息
    response = requests.get(url, headers=headers)  # 发送网络请求
    content = response.content.decode('utf-8')
    soup = BeautifulSoup(content, 'html.parser')

    listB = soup.find_all(name='script', attrs={"id": "getIndexRumorList"})
    print(listB)
    cc=str(listB)[65:-21]

    js=json.loads(cc)
    print(js)
    for c in js:
        neirong=c.get('title')
        lunduan=c.get('mainSummary')
        zhengwen=c.get('body')
        zhenjia=c.get('rumorType')
        insert_rumor(neirong=neirong,lunduan=lunduan,zhengwen=zhengwen,bool=zhenjia)
        print(c)
    print('谣言完毕')
    return
def spider_hospital():
    a=1
    for i in range(11,66):
        for j in range(0,25):
            print('id=%d'%(a))
            a+=1
            url = 'https://card.wecity.qq.com/feverHosp/feverHospList?cityCode=%d%02d00&pageIndex=1&pageSize=999&partnerType=4&lat=0&lng=0&searchKey=&sort=distance'  %(i,j)# 请求地址
            headers = {
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}  # 创建头部信息
            response = requests.get(url, headers=headers)  # 发送网络请求

            content = response.content.decode('utf-8')
            if len(content)<=60:
                continue



            cc=json.loads(content)
            dd=cc['data']
            ee=dd['data']
            print(ee)
            for k in ee:
                name=k.get('orgName')
                dizhi=k.get('orgAddr')
                shiming=xingZheng.use(int(k.get('cityCode')))
                shuxing='发热门诊'
                insert_hospital(name=name,dizhi=dizhi,cityname=shiming,shuxing=shuxing)
            if i in (11, 12, 31, 50):#4个直辖市
                break


    print('结束了')
    return
def chushihua_prov():

        one = requests.get(url='https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5').json()
        print(one)
        data = json.loads(one['data'])
        update_time = data['lastUpdateTime']
        all_counties = data['areaTree']
        print(data)
        print(update_time)
        print(all_counties)
        all_list = []
        for country_data in all_counties:
            if country_data['name'] == '中国':
                all_provinces = country_data['children']
                for province_data in all_provinces:
                    province_name = province_data['name']
                    all_cities = province_data['children']
                    for city_data in all_cities:
                        city_name = city_data['name']
                        city_total = city_data['total']
                        province_result = {'province': province_name, 'city': city_name, 'update_time': update_time}
                        province_result.update(city_total)
                        all_list.append(province_result)
        for i in all_list:
            print(i)
            insert_prov(province_name=i.get('province'),city_name=i.get('city'))

        print(all_list)
        print('省市完毕')
        return

def chushihua_dazhou():
    url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0'  # 请求地址
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}  # 创建头部信息
    response = requests.get(url, headers=headers)  # 发送网络请求
    content = response.content.decode('utf-8')
    soup = BeautifulSoup(content, 'html.parser')

    # 世界确诊getAreaStat
    listB = soup.find_all(name='script', attrs={"id": "getListByCountryTypeService2true"})

    world_messages = str(listB)[95:-21]

    print(world_messages)
    world_messages_json = json.loads(world_messages)
    worldList = []
    now_time = datetime.datetime.now().strftime('%Y-%m-%d')
    for k in world_messages_json:
        name = k.get('provinceName')
        dazhou = k.get('continents')
        now_confirm = k.get('currentConfirmedCount')
        confirm = k.get('confirmedCount')
        dead = k.get('deadCount')
        insert_dazhou(dazhou_name=dazhou,country_name=name)
        print(k)
    print('大洲初始完毕')
    return