最后一次正确的测试

master
wufayuan 2 years ago
parent 3c186535e9
commit ad427ef9bc

@ -1,5 +1,5 @@
[server]
ip = 127.0.0.1
ip = 192.168.43.241
port = 7777
daemon = True
buffer_size = 8 * 1024 * 1024
@ -9,8 +9,8 @@ edge_driver_path = G:\course\yykf\dcs\bin\msedgedriver.exe
max_count_of_crawlers = 10
[database]
ip = 127.0.0.1
ip = 192.168.43.64
user = root
password = xwdjzwy5252
password = 427318Aa
database = test

@ -4,7 +4,7 @@ from loguru import logger
from dcs.clients.client import Client
start = 9000
ip = '127.0.0.1'
ip = '192.168.43.241'
port = 7777
local_ip = ip
local_port = None

File diff suppressed because one or more lines are too long

@ -146,17 +146,17 @@ class Spider_task(threading.Thread):
def compose_result(self):
logger.info('[COMPOSE] composing task...')
logger.info(f'[RESULT] {self.results}')
remote_result = get_crawl_result(self.request_map['cookie'])
for result_map in list(remote_result):
create_crawl_result_table(table_name=self.table_name)
for _, data in result_map.items():
write2results(Paper(data['title'], [Author(data['name'], data['college'], data['major'])]), self.results)
logger.info(f'[RESULT] {self.results}')
logger.info(f'[DATABASE] writing crawl results to database...')
write_results2database(self.results, self.table_name, self.last_crawl_id)
result = {'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name, 'data': self.results}
result = {'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name} # , 'data': self.results}
global_var.communicator.add_response('response', self.client_socket, result)
def run(self) -> None:

Loading…
Cancel
Save