''' 负责处理入库业务 ''' from Model import DataDB as db from Controller import DataFetch import logging from datetime import datetime scrawler = DataFetch.Scrawler() '''入库业务''' '''1.新闻入库 insertData(tableName,source,ctime,title,url):''' def newsAdd(id): if id=='0': '''工大的地址''' source="工大新闻" url = 'http://news.hut.edu.cn/' count=0 for i in range(0,5): ctime = scrawler.reqbsGetText(url, cssselector='.news-list .date')[i] title = scrawler.reqbsGetText(url, cssselector='.news-list .info h2')[i] urls = url+scrawler.reqbsGetHref(url, cssselector='.news-list .info a')[i] if db.insertData('news',source,ctime,title,urls): count=count+1 else:count=0 if count==5:return True else:return False elif id=='1': '''北大新闻''' source = "北大新闻" url = 'https://news.pku.edu.cn/ttxw/index.htm' count = 0 for i in range(0, 5): ctime = scrawler.reqbsGetText(url, cssselector='.articleList01 .item-date')[i] title = scrawler.reqbsGetText(url, cssselector='.articleList01 .item-txt h3')[i] urls = url + scrawler.reqbsGetHref(url, cssselector='.articleList01 .item-txt h3 a')[i] if db.insertData('news', source, ctime, title, urls): count = count + 1 else: count = 0 if count == 5: return True else: return False pass '''2.疫情数据入库''' def yqAdd(source): if source=='1': # 来源是新浪疫情 url = 'http://zj.sina.com.cn/zt_d/zjyiqing/?hk=/*' res = scrawler.reqbsGetJSON(url=url) ctime = res['data']['mtime'] cnumber = res['data']['gntotal'] gnumber = res['data']['deathtotal'] if db.insertData('yq',source,ctime,cnumber,gnumber): return True else:return False elif source=='0': # 来源是腾讯疫情 url = 'https://www.qq.com/' res = scrawler.reqbsGetJSON(url=url) ctime = res['data']['mtime'] cnumber = res['data']['gntotal'] gnumber = res['data']['deathtotal'] if db.insertData('yq',source,ctime,cnumber,gnumber): return True else:return False ''' 日志相关处理方法 ''' class MyLogHandler(logging.Handler, object): """ 自定义日志handler """ def __init__(self): logging.Handler.__init__(self) def emit(self, record): """ record为一个消息类对象,包括name,asctime、lineno、funcname等属性 emit函数为自定义handler类时必重写的函数,这里可以根据需要对日志消息做一些处理,比如发送日志到服务器 发出记录(Emit a record)""" try: db.log2db(str(datetime.now()),record.lineno, record.funcName,record.getMessage()) except Exception: self.handleError(record) def mylog(): # 创建一个日志记录器 log = logging.getLogger("test_logger") log.setLevel(logging.DEBUG) # 创建一个日志处理器 logHandler = MyLogHandler() logHandler.setLevel(logging.INFO) # 创建一个日志格式器 formats = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d: %(message)s') # 将日志格式器添加到日志处理器中 logHandler.setFormatter(formats) # 将日志处理器添加到日志记录器中 log.addHandler(logHandler) return log if __name__=='__main__': print(newsAdd('工大新闻',0)) print(yqAdd('新浪')) # print(db.getAllData('yq'))