|
|
@ -146,17 +146,17 @@ class Spider_task(threading.Thread):
|
|
|
|
|
|
|
|
|
|
|
|
def compose_result(self):
|
|
|
|
def compose_result(self):
|
|
|
|
logger.info('[COMPOSE] composing task...')
|
|
|
|
logger.info('[COMPOSE] composing task...')
|
|
|
|
logger.info(f'[RESULT] {self.results}')
|
|
|
|
|
|
|
|
remote_result = get_crawl_result(self.request_map['cookie'])
|
|
|
|
remote_result = get_crawl_result(self.request_map['cookie'])
|
|
|
|
for result_map in list(remote_result):
|
|
|
|
for result_map in list(remote_result):
|
|
|
|
create_crawl_result_table(table_name=self.table_name)
|
|
|
|
create_crawl_result_table(table_name=self.table_name)
|
|
|
|
for _, data in result_map.items():
|
|
|
|
for _, data in result_map.items():
|
|
|
|
write2results(Paper(data['title'], [Author(data['name'], data['college'], data['major'])]), self.results)
|
|
|
|
write2results(Paper(data['title'], [Author(data['name'], data['college'], data['major'])]), self.results)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
logger.info(f'[RESULT] {self.results}')
|
|
|
|
logger.info(f'[DATABASE] writing crawl results to database...')
|
|
|
|
logger.info(f'[DATABASE] writing crawl results to database...')
|
|
|
|
write_results2database(self.results, self.table_name, self.last_crawl_id)
|
|
|
|
write_results2database(self.results, self.table_name, self.last_crawl_id)
|
|
|
|
|
|
|
|
|
|
|
|
result = {'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name, 'data': self.results}
|
|
|
|
result = {'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name} # , 'data': self.results}
|
|
|
|
global_var.communicator.add_response('response', self.client_socket, result)
|
|
|
|
global_var.communicator.add_response('response', self.client_socket, result)
|
|
|
|
|
|
|
|
|
|
|
|
def run(self) -> None:
|
|
|
|
def run(self) -> None:
|
|
|
|