进一步完善爬取结果写入数据库

master
wufayuan 3 years ago
parent 2dbf99feda
commit 0c45592b8f

@ -374,3 +374,9 @@
2022-03-29 11:09:56.282 | INFO | dcs.tests.server:run:36 - [REQUEST] end 2022-03-29 11:09:56.282 | INFO | dcs.tests.server:run:36 - [REQUEST] end
2022-03-29 11:09:56.282 | WARNING | dcs.tests.server:run:37 - communication over! 2022-03-29 11:09:56.282 | WARNING | dcs.tests.server:run:37 - communication over!
2022-03-29 11:09:56.283 | WARNING | __main__:<module>:21 - Overing... 2022-03-29 11:09:56.283 | WARNING | __main__:<module>:21 - Overing...
2022-03-29 11:26:08.833 | INFO | __main__:<module>:9 - reading config args...
2022-03-29 11:26:08.834 | INFO | __main__:<module>:16 - starting the server...
2022-03-29 11:26:47.684 | INFO | __main__:<module>:9 - reading config args...
2022-03-29 11:26:47.684 | INFO | __main__:<module>:16 - starting the server...
2022-03-29 11:34:44.764 | INFO | __main__:<module>:9 - reading config args...
2022-03-29 11:34:44.764 | INFO | __main__:<module>:16 - starting the server...

@ -2,8 +2,9 @@
from dcs.tests.server import Server from dcs.tests.server import Server
from configparser import ConfigParser from configparser import ConfigParser
from loguru import logger from loguru import logger
from dcs.tests.database import create_user_info
create_user_info()
logger.add('./dcs.log', rotation='10 MB', enqueue=True, backtrace=True, diagnose=True) logger.add('./dcs.log', rotation='10 MB', enqueue=True, backtrace=True, diagnose=True)
logger.info('reading config args...') logger.info('reading config args...')
configFile = '../conf/settings.ini' configFile = '../conf/settings.ini'

@ -18,7 +18,7 @@ def register(u_name, u_pwd):
# 获取数据库操作cursor游标 # 获取数据库操作cursor游标
cur = conn.cursor() cur = conn.cursor()
# 编写查询的sql语句 # 编写查询的sql语句
select_sql = f'select user_password from userinfotest where user_name = "{u_name}"' select_sql = f'select user_password from user_info where user_name = "{u_name}"'
# 执行sql语句 # 执行sql语句
cur.execute(select_sql) cur.execute(select_sql)
# 获取执行结果 fetch_one(),判断结果 # 获取执行结果 fetch_one(),判断结果
@ -28,7 +28,7 @@ def register(u_name, u_pwd):
info = '用户名已存在,注册失败' info = '用户名已存在,注册失败'
else: else:
# 注册-> 插入数据手动commit # 注册-> 插入数据手动commit
insert_sql = 'insert into userinfotest (user_name, user_password) values (%s,%s)' insert_sql = 'insert into user_info (user_name, user_password, create_time, login_state) values (%s,%s,now(),false)'
insert_params = [u_name, sha_pwd] insert_params = [u_name, sha_pwd]
cur.execute(insert_sql, insert_params) cur.execute(insert_sql, insert_params)
conn.commit() conn.commit()
@ -48,7 +48,7 @@ def login(u_name, u_pwd):
try: try:
conn = mysql_conn() conn = mysql_conn()
cur = conn.cursor() cur = conn.cursor()
select_sql = f'select user_password from userinfotest where user_name = "{u_name}"' select_sql = f'select user_password from user_info where user_name = "{u_name}"'
cur.execute(select_sql) cur.execute(select_sql)
res = cur.fetchone() res = cur.fetchone()
if res is None: if res is None:
@ -59,6 +59,9 @@ def login(u_name, u_pwd):
m_pwd = res[0] m_pwd = res[0]
if m_pwd == sha_pwd: if m_pwd == sha_pwd:
info = '用户' + u_name + '登录成功' info = '用户' + u_name + '登录成功'
update_sql = f'update user_info set login_state = true where username = {u_name}'
cur.execute(update_sql)
conn.commit()
else: else:
info = '密码错误,登录失败' info = '密码错误,登录失败'
# 关闭连接 # 关闭连接
@ -73,7 +76,7 @@ def cancel(u_name):
try: try:
conn = mysql_conn() conn = mysql_conn()
cur = conn.cursor() cur = conn.cursor()
select_sql = f'delete from userinfotest where username = "{u_name}"' select_sql = f'delete from user_info where user_name = "{u_name}"'
cur.execute(select_sql) cur.execute(select_sql)
cur.close() cur.close()
conn.close() conn.close()
@ -116,20 +119,10 @@ def drop_table(table_name: str):
print(e) print(e)
def create_crawl_result_table(table_name: str): def create_table(create_sql: str):
try: try:
conn = mysql_conn() conn = mysql_conn()
cur = conn.cursor() cur = conn.cursor()
create_sql = f'create table if not exists {table_name} (' \
f'id int primary key not null auto_increment,' \
f'crawl_id int not null,' \
f'time timestamp not null,' \
f'name varchar(100),' \
f'college varchar(200),' \
f'major varchar(200),' \
f'paper varchar(200)' \
f')'
cur.execute(create_sql) cur.execute(create_sql)
conn.commit() conn.commit()
cur.close() cur.close()
@ -138,6 +131,30 @@ def create_crawl_result_table(table_name: str):
print(e) print(e)
def create_crawl_result_table(table_name: str):
create_sql = f'create table if not exists {table_name} (' \
f'id int primary key not null auto_increment,' \
f'crawl_id int not null,' \
f'time timestamp not null,' \
f'name varchar(100),' \
f'college varchar(200),' \
f'major varchar(200),' \
f'paper varchar(200)' \
f')'
create_table(create_sql)
def create_user_info(table_name: str = 'user_info'):
create_sql = f'create table if not exists {table_name} (' \
f'id int primary key not null auto_increment,' \
f'create_time timestamp not null default now(),' \
f'user_name varchar(100),' \
f'user_password varchar(200),' \
f'login_state boolean default false' \
f')'
create_table(create_sql)
def write_result2database(res: list, table_name: str, last_crawl_id: int): def write_result2database(res: list, table_name: str, last_crawl_id: int):
try: try:
conn = mysql_conn() conn = mysql_conn()

Loading…
Cancel
Save