diff --git a/conf/config.py b/conf/config.py index 8c1cfc8..bedbfe8 100644 --- a/conf/config.py +++ b/conf/config.py @@ -21,6 +21,7 @@ class CUI: class global_var: """需要定义全局变量的放在这里""" + connection = None free_spiders = [] current_user_info: list[CUI] = [] requester = None diff --git a/dcs/client/client.py b/dcs/client/client.py index 55316de..0d2cc6e 100644 --- a/dcs/client/client.py +++ b/dcs/client/client.py @@ -1,23 +1,71 @@ import json import socket import struct -from json import JSONDecoder import threading +from json import JSONDecoder +from time import sleep +from loguru import logger +from msedge.selenium_tools import Edge +from msedge.selenium_tools import EdgeOptions + +from dcs.tests.zhiwang import * from dcs.tools import message_process as mp from dcs.tools.message_process import parse_request, generate_response +def crawl_zhiwang(word, pages_start, pages_end): + edge_options = EdgeOptions() + edge_options.use_chromium = True + # edge_options.add_argument('--headless') + driver = Edge(options=edge_options, executable_path=r'G:\course\yykf\dcs\bin\msedgedriver.exe') + + soup = driver_open(driver, word) + papers = [] # 用于保存爬取到的论文 + paper_id = 0 + res = {} # 保存终端爬取结果 + + # 爬取第一篇 + if pages_start == 1: + spider(driver, soup, papers) + logger.debug(res) + pages_start += 1 + while paper_id < len(papers): + write2res(papers[paper_id], res) + paper_id += 1 + + while pages_start < pages_end: + content = change_page(driver, pages_start) + spider(driver, content, papers) + while paper_id < len(papers): + write2res(papers[paper_id], res) + paper_id += 1 + pages_start += 1 + driver.close() + logger.debug("here") + return res + + +def write2res(paper: Paper, res): + for author in paper.authors: + if author.name: + res.update( + {len(res): {'name': author.name, 'college': author.college, 'major': author.major, 'title': paper.title}}) + + class Crawl(threading.Thread): def __init__(self): super(Crawl, self).__init__() self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.server_socket.bind(('', 9000)) + self.server_socket.bind(('', 9999)) @staticmethod - def crawl() -> dict: - result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}} + def crawl(request_map) -> dict: + result_map = crawl_zhiwang(request_map['word'], request_map['pages_start'], request_map['pages_end']) + # result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, + # 1: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}} + logger.debug(result_map) return result_map def run(self) -> None: @@ -26,8 +74,8 @@ class Crawl(threading.Thread): client_socket, _ = self.server_socket.accept() request_map = parse_request(client_socket) if request_map['type'] == 'request': - print("receiving help request:\n"+json.dumps(request_map, ensure_ascii=False)) - response_map = self.crawl() + print("receiving help request:\n" + json.dumps(request_map, ensure_ascii=False)) + response_map = self.crawl(request_map) response_map.update({'cookie': request_map['cookie']}) client_socket.sendall(generate_response(response_map)) if request_map['type'] == 'response': @@ -37,42 +85,41 @@ class Crawl(threading.Thread): crawl = Crawl() crawl.start() -# crawl.join() with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_server: - socket_to_server.bind(('127.0.0.1', 9000)) + socket_to_server.bind(('127.0.0.1', 9999)) socket_to_server.connect(('127.0.0.1', 7777)) - request = {'action': 'register', 'user': 'wufayuan', 'password': '113818'} + # request = {'action': 'register', 'user': 'wufayuan', 'password': '113818'} + # socket_to_server.sendall(mp.generate_request(request)) + # responseJson = JSONDecoder().decode( + # mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( + # "utf-8")) + # print(responseJson) + + request = {'action': 'login', 'user': 'wufayuan', 'password': '113818'} socket_to_server.sendall(mp.generate_request(request)) responseJson = JSONDecoder().decode( mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( "utf-8")) + cookie = responseJson['cookie'] print(responseJson) - request = {'action': 'login', 'user': 'wufayuan', 'password': '113818'} + request = {'action': 'report_free', 'cookie': cookie} socket_to_server.sendall(mp.generate_request(request)) responseJson = JSONDecoder().decode( mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( "utf-8")) - cookie = responseJson['cookie'] print(responseJson) - request = {'action': 'report_free', 'cookie': cookie} + request = {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3, + 'cookie': cookie} socket_to_server.sendall(mp.generate_request(request)) responseJson = JSONDecoder().decode( mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( "utf-8")) print(responseJson) - # request = {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, - # 'cookie': cookie} - # socket_to_server.sendall(mp.generate_request(request)) - # responseJson = JSONDecoder().decode( - # mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( - # "utf-8")) - # print(responseJson) - request = {'action': 'end'} socket_to_server.sendall(mp.generate_request(request)) diff --git a/dcs/dcs.log b/dcs/dcs.log index 1df9c75..9dcd690 100644 --- a/dcs/dcs.log +++ b/dcs/dcs.log @@ -7586,3 +7586,901 @@ 2022-06-30 20:28:30.694 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... 2022-06-30 20:28:31.008 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} 2022-06-30 20:28:31.196 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} +2022-06-30 20:57:45.189 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:57:48.343 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000) +2022-06-30 20:57:48.484 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:57:48.516 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:57:48.516 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:57:48.799 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:57:49.254 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:57:49.599 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 3b7f7ace06dc34e5881edc41ced6e9c742966bb0 +2022-06-30 20:57:49.615 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '3b7f7ace06dc34e5881edc41ced6e9c742966bb0'} +2022-06-30 20:57:49.866 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:57:49.929 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-06-30 20:57:49.929 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 3b7f7ace06dc34e5881edc41ced6e9c742966bb0 +2022-06-30 20:57:49.975 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 3b7f7ace06dc34e5881edc41ced6e9c742966bb0'} +2022-06-30 20:57:50.007 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-06-30 20:57:50.007 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)! +2022-06-30 20:58:00.443 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:58:00.585 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:58:00.648 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:58:00.648 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:00.805 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:01.323 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:01.998 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: f9efa9fde188db0826ddede55b520b7f63c8c19e +2022-06-30 20:58:02.014 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'} +2022-06-30 20:58:02.030 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:58:02.171 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:58:14.065 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:58:14.097 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:14.521 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 20:58:14.537 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 20:58:14.537 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, , , ] +2022-06-30 20:58:14.552 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-06-30 20:58:14.568 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:58:14.537 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 20:58:14.584 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task +2022-06-30 20:58:14.584 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000) +2022-06-30 20:58:14.615 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:58:14.709 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:58:14.725 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:14.725 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:58:14.756 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:14.772 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:58:14.788 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.070 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.102 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.149 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.494 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:58:15.494 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.494 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.541 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:58:15.541 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.541 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.651 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:58:15.651 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.651 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.745 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.745 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.761 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.761 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:15.965 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:15.997 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:16.122 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:16.232 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:16.389 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:16.420 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:16.577 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 20:58:16.671 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:58:17.017 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:17.362 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 20:58:17.488 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 20:58:22.854 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-06-30 20:58:22.869 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}} +2022-06-30 20:58:22.885 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 20:58:22.885 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:23.465 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:23.921 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:24.329 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:58:24.329 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:25.035 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:58:25.035 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:58:25.427 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:58:25.505 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} +2022-06-30 21:04:02.332 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 21:04:02.442 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:02.991 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 21:04:03.039 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:03.039 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, , , ] +2022-06-30 21:04:03.039 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 21:04:03.039 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:04:03.039 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-06-30 21:04:03.086 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 21:04:03.133 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-06-30 21:04:03.180 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.196 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:04:03.196 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.243 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:04:03.290 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.290 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:04:03.305 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.541 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.667 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:03.761 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.059 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.122 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:04:04.122 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:04.122 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.546 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:04:04.546 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:04.546 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.656 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:04.656 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.671 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:04:04.671 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:04.671 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:04.938 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:04:04.938 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:04.938 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:05.080 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:05.080 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:05.127 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:05.237 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:05.347 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:05.378 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:05.519 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:05.865 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:05.880 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:06.053 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:06.147 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:06.336 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:06.696 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:06.901 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:07.026 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:07.026 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:04:07.073 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:07.089 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:07.497 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:07.764 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:07.795 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 21:04:07.795 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:07.811 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:08.078 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:04:08.141 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:08.141 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:08.502 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:08.784 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:08.816 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:08.894 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:09.067 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:04:09.067 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:09.240 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 11: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 14: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 21:04:09.318 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:04:09.366 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:09.789 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-06-30 21:04:09.789 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:10.056 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-06-30 21:04:10.056 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:04:10.229 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 11: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 14: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 21:04:10.292 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 11: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 14: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} +2022-06-30 21:05:28.201 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000) +2022-06-30 21:05:28.217 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 21:05:28.217 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 21:05:28.217 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:28.452 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:28.782 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:29.143 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 82eb7583e79ef09a2e111aef019638372a3b7f86 +2022-06-30 21:05:29.222 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '82eb7583e79ef09a2e111aef019638372a3b7f86'} +2022-06-30 21:05:29.300 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 21:05:29.332 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-06-30 21:05:29.332 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 82eb7583e79ef09a2e111aef019638372a3b7f86 +2022-06-30 21:05:29.332 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 82eb7583e79ef09a2e111aef019638372a3b7f86'} +2022-06-30 21:05:29.441 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-06-30 21:05:29.441 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)! +2022-06-30 21:05:40.662 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 21:05:40.678 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:41.228 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 21:05:41.244 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:05:41.244 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, , , ] +2022-06-30 21:05:41.244 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 21:05:41.244 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task +2022-06-30 21:05:41.244 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000) +2022-06-30 21:05:41.244 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-06-30 21:05:41.291 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 21:05:41.337 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-06-30 21:05:41.385 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:05:41.495 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:41.510 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:05:41.541 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:41.542 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 21:05:41.573 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.075 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.138 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.217 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.374 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:05:42.389 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:42.389 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.939 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:42.939 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.955 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:05:42.955 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:42.955 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:42.955 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 21:05:42.955 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:42.955 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:43.253 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:43.253 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:43.489 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:43.489 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:43.614 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:43.614 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:43.818 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:43.834 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 21:05:43.881 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:43.975 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:44.163 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:05:44.540 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:05:44.760 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': 'f9efa9fde188db0826ddede55b520b7f63c8c19e'}) +2022-06-30 21:05:49.280 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-06-30 21:05:49.280 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, '1': {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}} +2022-06-30 21:05:49.280 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 21:05:49.280 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:49.672 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:50.331 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:50.787 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:51.461 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:52.073 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:52.702 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 11: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 12: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 21:05:52.702 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:53.549 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 11: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 12: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 21:05:53.549 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 21:05:53.972 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 11: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 12: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 21:05:54.004 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, '1': {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 10: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 11: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 12: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} +2022-07-02 09:46:06.130 | DEBUG | __main__::15 - reading config args... +2022-07-02 09:52:17.122 | DEBUG | __main__::16 - reading config args... +2022-07-02 09:53:10.807 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 09:53:10.854 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 09:53:10.870 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 09:53:12.172 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 64e1150a93f2d15d194d5f16379a76845abb75db +2022-07-02 09:53:12.172 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '64e1150a93f2d15d194d5f16379a76845abb75db'} +2022-07-02 09:53:12.423 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 09:53:12.471 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 09:53:12.533 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 64e1150a93f2d15d194d5f16379a76845abb75db +2022-07-02 09:53:12.549 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 64e1150a93f2d15d194d5f16379a76845abb75db'} +2022-07-02 09:53:12.612 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 09:53:12.612 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 09:53:25.997 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-07-02 09:53:26.123 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 09:53:26.138 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 09:53:26.908 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 7ef690c44c8a91e6fa53098fcb0505eae7a75e34 +2022-07-02 09:53:26.970 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '7ef690c44c8a91e6fa53098fcb0505eae7a75e34'} +2022-07-02 09:53:27.002 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-02 09:53:27.064 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'} +2022-07-02 09:53:42.349 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-07-02 09:53:43.479 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 09:53:43.573 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '7ef690c44c8a91e6fa53098fcb0505eae7a75e34'}) +2022-07-02 09:53:43.573 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, , , ] +2022-07-02 09:53:43.573 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 09:53:43.573 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 09:53:43.589 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 09:53:43.620 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-07-02 09:53:43.636 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-02 09:53:43.746 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 09:53:43.746 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-07-02 09:53:43.856 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 09:53:43.950 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 09:53:44.593 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 09:53:44.656 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.064 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.127 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 09:53:45.205 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.566 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 09:53:45.566 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.566 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.645 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:45.817 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': '7ef690c44c8a91e6fa53098fcb0505eae7a75e34'}) +2022-07-02 09:53:45.975 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:46.117 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:46.431 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 09:53:46.496 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '7ef690c44c8a91e6fa53098fcb0505eae7a75e34'}) +2022-07-02 09:53:47.343 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '7ef690c44c8a91e6fa53098fcb0505eae7a75e34'}) +2022-07-02 10:06:19.914 | DEBUG | __main__::16 - reading config args... +2022-07-02 10:07:15.237 | DEBUG | __main__::15 - reading config args... +2022-07-02 10:07:22.707 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 10:07:22.864 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:07:22.895 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 10:07:22.895 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:23.099 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:23.397 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:23.852 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: eed26c74e744eed648a1aebdacbe211689c35a47 +2022-07-02 10:07:23.868 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'eed26c74e744eed648a1aebdacbe211689c35a47'} +2022-07-02 10:07:23.993 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:07:24.009 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 10:07:24.009 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked eed26c74e744eed648a1aebdacbe211689c35a47 +2022-07-02 10:07:24.009 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked eed26c74e744eed648a1aebdacbe211689c35a47'} +2022-07-02 10:07:24.088 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 10:07:24.088 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 10:07:33.991 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-07-02 10:07:34.133 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:07:34.148 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 10:07:34.148 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:34.337 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:34.697 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:07:35.089 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 8b0c9086811e907706bfb08f9abfb152dc2aac68 +2022-07-02 10:07:35.152 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '8b0c9086811e907706bfb08f9abfb152dc2aac68'} +2022-07-02 10:07:35.168 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-02 10:07:35.215 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'} +2022-07-02 10:07:52.167 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-07-02 10:07:52.261 | WARNING | dcs.tests.requestHandler:run:27 - user info error! +2022-07-02 10:08:19.723 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-07-02 10:08:19.865 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:20.462 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 10:08:20.509 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '8b0c9086811e907706bfb08f9abfb152dc2aac68'}) +2022-07-02 10:08:20.509 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, , , ] +2022-07-02 10:08:20.509 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 10:08:20.541 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 10:08:20.541 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 10:08:20.603 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-07-02 10:08:20.619 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-02 10:08:20.634 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:08:20.651 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:20.682 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:08:20.697 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:20.745 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-07-02 10:08:20.776 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:08:20.807 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:21.074 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:21.137 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:21.451 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:21.750 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 10:08:21.765 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:21.765 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:22.032 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 10:08:22.032 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:22.032 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:22.314 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:22.314 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:22.379 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:22.410 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:22.489 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 10:08:22.803 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:22.959 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:23.257 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:23.257 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:23.351 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:23.806 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:24.073 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '8b0c9086811e907706bfb08f9abfb152dc2aac68'}) +2022-07-02 10:08:24.104 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:24.276 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:24.559 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': '8b0c9086811e907706bfb08f9abfb152dc2aac68'}) +2022-07-02 10:08:25.156 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 10:08:25.171 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:26.066 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '8b0c9086811e907706bfb08f9abfb152dc2aac68'}) +2022-07-02 10:08:36.278 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 10:08:36.278 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 10:08:36.341 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 10:08:36.341 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:36.740 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:37.003 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 10:08:37.003 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:37.396 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 10:08:37.396 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:08:37.820 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 10:08:37.898 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 10:11:45.755 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 10:11:45.834 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:11:45.849 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 10:11:45.849 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:11:46.179 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:11:46.587 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:11:46.650 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: da452e861d5bd34fb3638d8f919c041683e3736a +2022-07-02 10:11:46.650 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'da452e861d5bd34fb3638d8f919c041683e3736a'} +2022-07-02 10:11:46.716 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:11:46.873 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 10:11:46.949 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked da452e861d5bd34fb3638d8f919c041683e3736a +2022-07-02 10:11:46.987 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked da452e861d5bd34fb3638d8f919c041683e3736a'} +2022-07-02 10:11:47.097 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 10:11:47.112 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 10:17:33.047 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 10:17:33.094 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:17:33.204 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 10:17:33.299 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:17:33.487 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:17:33.769 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:17:33.989 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 0265b402d5574f0145c6eee0b72ef5de891f61a3 +2022-07-02 10:17:33.989 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '0265b402d5574f0145c6eee0b72ef5de891f61a3'} +2022-07-02 10:17:33.989 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:17:34.020 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 10:17:34.020 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 0265b402d5574f0145c6eee0b72ef5de891f61a3 +2022-07-02 10:17:34.020 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 0265b402d5574f0145c6eee0b72ef5de891f61a3'} +2022-07-02 10:17:34.020 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 10:17:34.020 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 10:59:41.997 | DEBUG | __main__::15 - reading config args... +2022-07-02 10:59:45.620 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 10:59:45.668 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:59:45.700 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 10:59:45.700 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:59:45.954 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:59:46.241 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:59:46.400 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: c71dc5a41334d3d5a14565a803507f8a7a76bc09 +2022-07-02 10:59:46.400 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'c71dc5a41334d3d5a14565a803507f8a7a76bc09'} +2022-07-02 10:59:46.479 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 10:59:46.597 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 10:59:46.657 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked c71dc5a41334d3d5a14565a803507f8a7a76bc09 +2022-07-02 10:59:46.688 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked c71dc5a41334d3d5a14565a803507f8a7a76bc09'} +2022-07-02 10:59:46.793 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 10:59:47.663 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 10:59:47.726 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'c71dc5a41334d3d5a14565a803507f8a7a76bc09'}) +2022-07-02 10:59:47.726 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, , , , , , , , ] +2022-07-02 10:59:47.726 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 10:59:47.804 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 10:59:47.835 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 10:59:47.945 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 10:59:47.976 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 10:59:47.976 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 10:59:48.055 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.116 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.209 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.270 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.318 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.460 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.523 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 10:59:48.602 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:00:13.822 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 3, 'pages_end': 4, 'cookie': 'c71dc5a41334d3d5a14565a803507f8a7a76bc09'}) +2022-07-02 11:00:13.854 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 6, 'pages_end': 7, 'cookie': 'c71dc5a41334d3d5a14565a803507f8a7a76bc09'}) +2022-07-02 11:00:13.916 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:23.664 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:01:26.343 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:01:26.422 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:01:26.453 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:01:26.453 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:01:26.690 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:01:26.928 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:01:27.118 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 268d6df0f9e6d3a1c02086d78f129b9c092fc27f +2022-07-02 11:01:27.118 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '268d6df0f9e6d3a1c02086d78f129b9c092fc27f'} +2022-07-02 11:01:27.181 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:01:27.197 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:01:27.197 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 268d6df0f9e6d3a1c02086d78f129b9c092fc27f +2022-07-02 11:01:27.244 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 268d6df0f9e6d3a1c02086d78f129b9c092fc27f'} +2022-07-02 11:01:27.307 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:01:27.737 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:01:27.753 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': '268d6df0f9e6d3a1c02086d78f129b9c092fc27f'}) +2022-07-02 11:01:27.753 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, , , , , , , , ] +2022-07-02 11:01:27.753 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:01:27.754 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:01:27.754 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:01:27.754 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:01:27.801 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:01:27.927 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:27.927 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:01:28.039 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:01:28.055 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.103 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, '1': {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}} +2022-07-02 11:01:28.327 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.359 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.469 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.533 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.645 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:01:28.787 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:02:14.483 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:02:33.480 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:02:33.655 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:02:33.671 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:02:33.672 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:33.895 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:34.179 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:34.275 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 3c475781317650dcff2514094e966185419e7b4a +2022-07-02 11:02:34.275 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '3c475781317650dcff2514094e966185419e7b4a'} +2022-07-02 11:02:34.290 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:02:34.353 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:02:34.353 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 3c475781317650dcff2514094e966185419e7b4a +2022-07-02 11:02:34.368 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 3c475781317650dcff2514094e966185419e7b4a'} +2022-07-02 11:02:34.402 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:34.409 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:02:34.448 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '3c475781317650dcff2514094e966185419e7b4a'}) +2022-07-02 11:02:34.448 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:02:34.449 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:02:34.449 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:02:34.464 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:02:34.471 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:02:34.468 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:02:34.472 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:02:34.559 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:02:34.873 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:35.942 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:36.668 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:02:36.668 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:02:36.668 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:37.330 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:02:37.362 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:37.787 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:02:37.787 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:02:38.210 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '3c475781317650dcff2514094e966185419e7b4a'}) +2022-07-02 11:04:45.200 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:04:48.170 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:04:48.298 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:04:48.330 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:04:48.346 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:48.601 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:48.919 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:49.285 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 3acce67497cb6d0b8b440f3e04812887a97defc9 +2022-07-02 11:04:49.382 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '3acce67497cb6d0b8b440f3e04812887a97defc9'} +2022-07-02 11:04:49.493 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:04:49.494 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:04:49.494 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 3acce67497cb6d0b8b440f3e04812887a97defc9 +2022-07-02 11:04:49.556 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 3acce67497cb6d0b8b440f3e04812887a97defc9'} +2022-07-02 11:04:49.652 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:50.228 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:04:50.324 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:04:50.372 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:04:50.372 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:04:50.372 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '3acce67497cb6d0b8b440f3e04812887a97defc9'}) +2022-07-02 11:04:50.373 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:04:50.373 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:04:50.373 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:04:50.373 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:04:50.404 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:04:50.419 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:50.771 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:51.591 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:04:51.606 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:04:51.686 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:52.513 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:04:52.514 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:53.079 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:04:53.079 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:04:53.633 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '3acce67497cb6d0b8b440f3e04812887a97defc9'}) +2022-07-02 11:05:02.033 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:05:02.049 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:05:02.049 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:05:02.364 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:05:02.904 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:05:03.109 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:05:03.188 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:06:02.832 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:06:06.620 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:06:06.636 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:06:06.668 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:06:06.668 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:06.796 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:06.956 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:07.305 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 55ebfabbff02b09e7f7967665eb2eefef370e486 +2022-07-02 11:06:07.321 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '55ebfabbff02b09e7f7967665eb2eefef370e486'} +2022-07-02 11:06:07.369 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:06:07.401 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:06:07.401 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 55ebfabbff02b09e7f7967665eb2eefef370e486 +2022-07-02 11:06:07.544 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 55ebfabbff02b09e7f7967665eb2eefef370e486'} +2022-07-02 11:06:07.623 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:08.151 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:06:08.167 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '55ebfabbff02b09e7f7967665eb2eefef370e486'}) +2022-07-02 11:06:08.167 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:06:08.168 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:06:08.168 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:06:08.168 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:06:08.168 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:06:08.183 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:06:08.183 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:06:08.183 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:06:08.199 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:08.631 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:08.979 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:06:08.980 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:06:09.011 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:09.325 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:06:09.434 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:10.634 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:06:10.634 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:11.266 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '55ebfabbff02b09e7f7967665eb2eefef370e486'}) +2022-07-02 11:06:18.506 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:06:18.712 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:06:18.744 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:06:18.760 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:19.445 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:06:19.859 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:06:19.891 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:09:28.734 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:09:31.030 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:09:31.078 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:09:31.126 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:09:31.126 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:31.443 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:31.619 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:31.954 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: bb7f790418710d64df16d3547d803e1fe78bd5c7 +2022-07-02 11:09:31.970 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'bb7f790418710d64df16d3547d803e1fe78bd5c7'} +2022-07-02 11:09:32.050 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:09:32.114 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:09:32.114 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked bb7f790418710d64df16d3547d803e1fe78bd5c7 +2022-07-02 11:09:32.162 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked bb7f790418710d64df16d3547d803e1fe78bd5c7'} +2022-07-02 11:09:32.178 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:32.498 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:09:32.514 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': 'bb7f790418710d64df16d3547d803e1fe78bd5c7'}) +2022-07-02 11:09:32.514 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:09:32.514 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:09:32.530 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:09:32.546 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:09:32.610 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:09:32.611 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:09:32.611 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:09:32.706 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:09:32.753 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:33.183 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:34.496 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:09:34.590 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:09:34.606 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:35.172 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:09:35.172 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:35.711 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:09:35.711 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:36.416 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'bb7f790418710d64df16d3547d803e1fe78bd5c7'}) +2022-07-02 11:09:42.763 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:09:42.938 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:09:43.050 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:09:43.050 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:43.639 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:09:44.372 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:09:44.420 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:11:36.963 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:11:39.223 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:11:39.367 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:11:39.399 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:11:39.399 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:39.702 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:39.830 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:40.261 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: a673144faa1cbf41a4c66eee0d13c5f44c91be21 +2022-07-02 11:11:40.341 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'a673144faa1cbf41a4c66eee0d13c5f44c91be21'} +2022-07-02 11:11:40.389 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:11:40.437 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:11:40.485 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked a673144faa1cbf41a4c66eee0d13c5f44c91be21 +2022-07-02 11:11:40.485 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked a673144faa1cbf41a4c66eee0d13c5f44c91be21'} +2022-07-02 11:11:40.565 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:40.964 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:11:40.980 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': 'a673144faa1cbf41a4c66eee0d13c5f44c91be21'}) +2022-07-02 11:11:40.980 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:11:40.981 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:11:40.981 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:11:40.981 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:11:40.996 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:11:41.012 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:11:41.012 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:11:41.028 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:11:41.044 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:41.603 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:42.176 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:11:42.270 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:11:42.476 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:43.337 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:11:43.337 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:43.979 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:11:43.979 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:44.729 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'a673144faa1cbf41a4c66eee0d13c5f44c91be21'}) +2022-07-02 11:11:50.292 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:11:50.337 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:11:50.337 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:11:50.464 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:51.006 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:11:51.469 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:11:51.517 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:14:37.412 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:14:37.539 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:14:37.555 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:14:37.587 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:38.019 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:38.643 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:39.281 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 49c307a8e8066679b550f6f28accee1ebb8cdd69 +2022-07-02 11:14:39.409 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '49c307a8e8066679b550f6f28accee1ebb8cdd69'} +2022-07-02 11:14:39.457 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:14:39.473 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:14:39.473 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 49c307a8e8066679b550f6f28accee1ebb8cdd69 +2022-07-02 11:14:39.553 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 49c307a8e8066679b550f6f28accee1ebb8cdd69'} +2022-07-02 11:14:39.600 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:40.317 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:14:40.333 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '49c307a8e8066679b550f6f28accee1ebb8cdd69'}) +2022-07-02 11:14:40.333 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:14:40.334 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:14:40.334 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:14:40.349 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:14:40.381 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:14:40.493 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:14:40.557 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:14:40.589 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:14:40.637 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:41.387 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:41.691 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:14:41.692 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:14:41.692 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:41.704 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:14:41.704 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:41.787 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:14:41.787 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:41.897 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '49c307a8e8066679b550f6f28accee1ebb8cdd69'}) +2022-07-02 11:14:51.038 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:14:51.133 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:14:51.149 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:14:51.227 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:51.576 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:14:51.864 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:14:51.943 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:16:44.939 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:16:49.941 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:16:50.004 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:16:50.020 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:16:50.036 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:50.276 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:50.676 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:50.995 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e +2022-07-02 11:16:51.027 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e'} +2022-07-02 11:16:51.107 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:16:51.123 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:16:51.123 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e +2022-07-02 11:16:51.155 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e'} +2022-07-02 11:16:51.219 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:51.411 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:16:51.427 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e'}) +2022-07-02 11:16:51.427 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:16:51.427 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:16:51.428 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:16:51.428 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:16:51.491 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:16:51.523 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:16:51.523 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:51.619 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:16:51.619 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:16:52.003 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:52.572 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:16:52.574 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:16:52.575 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:53.456 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:16:53.583 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:54.188 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:16:54.188 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:16:55.007 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '1cd23bbbf8f9367cf21bf6f82b3ebcc4e2b97b6e'}) +2022-07-02 11:17:01.078 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:17:01.094 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:17:01.094 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:17:01.236 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:17:01.519 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:17:01.867 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-02 11:17:02.024 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'} +2022-07-02 11:19:16.201 | DEBUG | __main__::15 - reading config args... +2022-07-02 11:19:19.800 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:19:19.848 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:19:19.849 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:19:19.849 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:20.102 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:20.262 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:20.995 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 05fe10ea7c58258d45d1cb494a45dcff50ba043c +2022-07-02 11:19:21.027 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '05fe10ea7c58258d45d1cb494a45dcff50ba043c'} +2022-07-02 11:19:21.107 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:19:21.155 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:19:21.155 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 05fe10ea7c58258d45d1cb494a45dcff50ba043c +2022-07-02 11:19:21.155 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 05fe10ea7c58258d45d1cb494a45dcff50ba043c'} +2022-07-02 11:19:21.188 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:21.537 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:19:21.569 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:19:21.585 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:19:21.585 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:19:21.601 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3, 'cookie': '05fe10ea7c58258d45d1cb494a45dcff50ba043c'}) +2022-07-02 11:19:21.601 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:19:21.697 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:19:21.713 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:19:21.713 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:19:21.873 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:19:21.920 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:22.368 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:22.991 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:19:23.052 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:23.344 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:23.767 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:23.813 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:24.761 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:24.761 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:25.950 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 2, 'pages_end': 3, 'cookie': '05fe10ea7c58258d45d1cb494a45dcff50ba043c'}) +2022-07-02 11:19:31.561 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:19:31.624 | DEBUG | dcs.requester:run:68 - result: {} +2022-07-02 11:19:31.686 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:19:31.862 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:31.862 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:19:31.894 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:32.293 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:32.405 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:32.707 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:19:32.707 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:32.707 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:33.315 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:33.315 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:33.937 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:19:33.937 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:19:34.225 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 2, 'cookie': '05fe10ea7c58258d45d1cb494a45dcff50ba043c', 'type': 'request'}) +2022-07-02 11:21:44.207 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-02 11:21:44.349 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:21:44.350 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-02 11:21:44.350 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:44.637 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:44.844 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:45.639 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 2a67d5bc16ee739e78686854ed3eb186fa5e202d +2022-07-02 11:21:45.655 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '2a67d5bc16ee739e78686854ed3eb186fa5e202d'} +2022-07-02 11:21:45.703 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-02 11:21:45.719 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-02 11:21:45.719 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 2a67d5bc16ee739e78686854ed3eb186fa5e202d +2022-07-02 11:21:45.720 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 2a67d5bc16ee739e78686854ed3eb186fa5e202d'} +2022-07-02 11:21:45.831 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:46.212 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-02 11:21:46.244 | INFO | dcs.tests.spider_task:distribute_task:137 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3, 'cookie': '2a67d5bc16ee739e78686854ed3eb186fa5e202d'}) +2022-07-02 11:21:46.244 | DEBUG | dcs.tests.spider_task:distribute_task:150 - [, ] +2022-07-02 11:21:46.244 | DEBUG | dcs.tests.spider_task:run:189 - [] +2022-07-02 11:21:46.245 | DEBUG | dcs.tests.spider_task:run:201 - generating remote task +2022-07-02 11:21:46.245 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-02 11:21:46.292 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-02 11:21:46.323 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-07-02 11:21:46.323 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9999)! +2022-07-02 11:21:46.323 | DEBUG | dcs.tests.spider_task:run:209 - generating local task +2022-07-02 11:21:46.370 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:47.069 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:47.274 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling... +2022-07-02 11:21:47.275 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:21:47.275 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:47.306 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:21:47.306 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:47.401 | INFO | dcs.tests.spider_task:write2database:16 - writing to database: test +2022-07-02 11:21:47.402 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:21:47.588 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 2, 'pages_end': 3, 'cookie': '2a67d5bc16ee739e78686854ed3eb186fa5e202d'}) +2022-07-02 11:22:32.922 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-02 11:22:33.001 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': '王芳', 'college': '南开大学', 'major': '图书情报与数字图书馆;新闻与传媒;计算机软件及计算机应用;', 'title': '科学数据管理政策发展比较研究'}, '1': {'name': '卜昊昊', 'college': '中国人民大学', 'major': '新闻与传媒;', 'title': '科学数据管理政策发展比较研究'}, '2': {'name': '贾宝余', 'college': '中国科学院创新发展研究中心', 'major': '高等教育;科学研究管理;中国共产党;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '3': {'name': '杨明', 'college': '中国科学院科技促进发展局', 'major': '科学研究管理;有机化工;图书情报与数字图书馆;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '4': {'name': '应验', 'college': '北京大学', 'major': '行政学及国家行政管理;经济体制改革;思想政治教育;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '5': {'name': '左玉河', 'college': '中国社会科学院', 'major': '中国近现代史;史学理论;政治学;', 'title': '中国科学社与民国学术体制之创建'}, '6': {'name': '张萍', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '7': {'name': 'Ding Lin', 'college': '美国俄亥俄州州立大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '8': {'name': '徐祯', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '9': {'name': '朱小玉', 'college': '中国科学技术大学附属第一医院', 'major': '肿瘤学;心血管系统疾病;临床医学;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '10': {'name': '邬惟为', 'college': '山西医科大学', 'major': '妇产科学;医学教育与医学边缘学科;感染性疾病及传染病;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '11': {'name': '官方霖', 'college': '西安交通大学', 'major': '医学教育与医学边缘学科;生物学;公安;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '12': {'name': '朱蔚彤', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '13': {'name': '孙瑞娟', 'college': '国家自然科学基金委员会', 'major': '医学教育与医学边缘学科;科学研究管理;基础医学;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '14': {'name': '徐岩英', 'college': '国家自然科学基金委员会生命科学部', 'major': '口腔科学;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '15': {'name': '张凤珠', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '16': {'name': '冯勇', 'college': '国家自然科学基金委员会综合计划局', 'major': '科学研究管理;自然科学理论与方法;图书情报与数字图书馆;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '17': {'name': '谢焕英', 'college': '国家自然科学基金委员会', 'major': '自然科学理论与方法;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '18': {'name': '蔡乾和', 'college': '华北理工大学', 'major': '科学研究管理;自然科学理论与方法;哲学;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '19': {'name': '苑怡', 'college': '中国国家科技评估中心', 'major': '自然科学理论与方法;科学研究管理;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '20': {'name': '邢怀滨', 'college': '中国科学技术部科技评估中心', 'major': '科学研究管理;自然科学理论与方法;宏观经济管理与可持续发展;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '21': {'name': '田德录', 'college': '中国科学技术部科技评估中心', 'major': '企业经济;科学研究管理;经济体制改革;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '22': {'name': '郎赟超', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;地球物理学;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '23': {'name': '丁虎', 'college': '上海市应用数学和力学研究所', 'major': '力学;工业通用技术及设备;电力工业;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '24': {'name': '韩晓昆', 'college': '天津大学', 'major': '环境科学与资源利用;地质学;地球物理学;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '25': {'name': '剌伟', 'college': '天津大学', 'major': '环境科学与资源利用;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '26': {'name': '刘丛强', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;矿业工程;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '27': {'name': '韩晓彤', 'college': '北京理工大学', 'major': '宏观经济管理与可持续发展;科学研究管理;自然科学理论与方法;', 'title': '科学推动下技术机会发现方法研究'}, '28': {'name': '朱东华', 'college': '北京理工大学', 'major': '科学研究管理;计算机软件及计算机应用;企业经济;', 'title': '科学推动下技术机会发现方法研究'}, '29': {'name': '汪雪锋', 'college': '北京理工大学', 'major': '科学研究管理;图书情报与数字图书馆;计算机软件及计算机应用;', 'title': '科学推动下技术机会发现方法研究'}, '30': {'name': '陆丽娜', 'college': '山东理工大学', 'major': '图书情报与数字图书馆;计算机软件及计算机应用;新闻与传媒;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '31': {'name': '尹丽红', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '32': {'name': '于啸', 'college': '东北农业大学', 'major': '计算机软件及计算机应用;生物学;农业基础科学;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '33': {'name': '尹居峰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '34': {'name': '尚士杰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '35': {'name': '吕凤先', 'college': '中国科学院文献情报中心', 'major': '科学研究管理;图书情报与数字图书馆;物理学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '36': {'name': '朱相丽', 'college': '中国科学院文献情报中心', 'major': '图书情报与数字图书馆;科学研究管理;材料科学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '37': {'name': '刘小平', 'college': '中国科学院大学', 'major': '图书情报与数字图书馆;科学研究管理;物理学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '38': {'name': '白云朴', 'college': '南京航空航天大学', 'major': '信息经济与邮政经济;企业经济;经济体制改革;', 'title': '科学数据共享研究的演化路径分析'}, '39': {'name': '李果', 'college': '南京邮电大学', 'major': '信息经济与邮政经济;经济体制改革;人才学与劳动科学;', 'title': '科学数据共享研究的演化路径分析'}, '40': {'name': 'Andrew Chi-Chih YAO', 'college': '清华大学', 'major': '计算机硬件技术;高等教育;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '41': {'name': 'Xia YIN', 'college': '清华大学', 'major': '计算机硬件技术;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '42': {'name': 'Yinqun LIU', 'college': '清华大学', 'major': '计算机硬件技术;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '43': {'name': 'Guoliang LI', 'college': '清华大学', 'major': '计算机软件及计算机应用;计算机硬件技术;冶金工业;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '44': {'name': '包秋华', 'college': '内蒙古农业大学', 'major': '生物学;轻工业手工业;中医学;', 'title': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, '45': {'name': '刘倩宇', 'college': '中国农业科学院植物保护研究所', 'major': '植物保护;化学;图书情报与数字图书馆;', 'title': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, '46': {'name': '阎晓阳', 'college': '北京大学', 'major': '马克思主义;中国共产党;政治学;', 'title': '如何理解“科学社会主义的入门”？——基于《社会主义从空想到科学的发展》'}, '47': {'name': '张建琴', 'college': '嘉兴学院', 'major': '自然科学理论与方法;科学研究管理;非线性科学与系统科学;', 'title': '评析贾祖璋的科普思想及实践'}, '48': {'name': '高铸烨', 'college': '中国中医科学院西苑医院', 'major': '中医学;心血管系统疾病;中药学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '49': {'name': '张冬梅', 'college': '北京中医药大学东直门医院', 'major': '中药学;中医学;心血管系统疾病;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '50': {'name': '赵京霞', 'college': '首都医科大学附属北京中医医院', 'major': '中药学;临床医学;中医学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '51': {'name': '徐鹏', 'college': '长春中医药大学附属医院', 'major': '中医学;神经病学;中药学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '52': {'name': '毕明刚', 'college': '国家自然科学基金委员会', 'major': '中药学;中医学;肿瘤学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '53': {'name': '刘睿', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '54': {'name': '左蕾', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '55': {'name': '张鹏', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '56': {'name': '赵宗超', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;自动化技术;地质学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '57': {'name': '陶刚', 'college': '成都理工大学', 'major': '地质学;石油天然气工业;地球物理学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '58': {'name': '王坤', 'college': '中国矿业大学', 'major': '地质学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '59': {'name': '陶东平', 'college': '山东理工大学', 'major': '矿业工程;有机化工;无机化工;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '60': {'name': '张紫璇', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'title': '脑科学技术在针刺研究中的应用'}, '61': {'name': '李少源', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;内分泌腺及全身性疾病;', 'title': '脑科学技术在针刺研究中的应用'}, '62': {'name': '王瑜', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;临床医学;', 'title': '脑科学技术在针刺研究中的应用'}, '63': {'name': '王艺霏', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'title': '脑科学技术在针刺研究中的应用'}, '64': {'name': '赵亚楠', 'college': '中国中医科学院针灸研究所', 'major': '中医学;计算机软件及计算机应用;', 'title': '脑科学技术在针刺研究中的应用'}, '65': {'name': '荣培晶', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;中药学;', 'title': '脑科学技术在针刺研究中的应用'}, '66': {'name': '熊璋', 'college': '中国教育部', 'major': '计算机软件及计算机应用;电信技术;互联网技术;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '67': {'name': '赵健', 'college': '华东师范大学', 'major': '教育理论与教育管理;中等教育;计算机软件及计算机应用;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '68': {'name': '陆海丰', 'college': '浙江省海宁市教育局', 'major': '中等教育;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '69': {'name': '欧阳元新', 'college': '北京航空航天大学', 'major': '计算机软件及计算机应用;计算机硬件技术;电信技术;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '70': {'name': '李梦柯', 'college': '中国科学院大学', 'major': '机械工业;科学研究管理;宏观经济管理与可持续发展;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, '71': {'name': '王芳', 'college': '中国科学院地理科学与资源研究所', 'major': '临床医学;环境科学与资源利用;儿科学;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, '72': {'name': '赵兰香', 'college': '中国科学院科技战略咨询研究院', 'major': '科学研究管理;企业经济;宏观经济管理与可持续发展;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}} +2022-07-02 11:22:33.064 | DEBUG | dcs.tests.spider_task:compose_result:160 - composing task... +2022-07-02 11:22:33.112 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:33.295 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:33.397 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:33.681 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:34.026 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:34.217 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:34.456 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:34.789 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:34.959 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:35.464 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:35.607 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:36.067 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:36.938 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:37.273 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:37.670 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:38.085 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:38.515 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:39.161 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:39.366 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:39.541 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:39.934 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:40.233 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:40.615 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:40.867 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:41.051 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:41.485 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:41.851 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:42.388 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:42.814 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:43.357 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:43.816 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:44.088 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:44.295 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:44.548 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:45.041 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:45.312 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:45.678 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:46.060 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:46.423 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:46.852 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:47.215 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:47.550 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:47.853 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:48.151 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:48.388 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:48.672 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:48.908 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:49.178 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:49.575 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:50.034 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:50.415 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:50.638 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:51.116 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:51.705 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:52.088 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:52.456 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:53.011 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:53.234 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:53.724 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:54.089 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:54.454 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:54.739 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:55.120 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:55.406 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:55.566 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:55.836 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:56.248 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:56.648 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:56.902 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:57.359 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:57.551 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:57.835 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:58.169 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:58.378 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:58.569 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-07-02 11:22:59.171 | DEBUG | dcs.tests.spider_task:compose_result:178 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '王芳', 'college': '南开大学', 'major': '图书情报与数字图书馆;新闻与传媒;计算机软件及计算机应用;', 'paper': '科学数据管理政策发展比较研究'}, 5: {'name': '卜昊昊', 'college': '中国人民大学', 'major': '新闻与传媒;', 'paper': '科学数据管理政策发展比较研究'}, 6: {'name': '贾宝余', 'college': '中国科学院创新发展研究中心', 'major': '高等教育;科学研究管理;中国共产党;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 7: {'name': '杨明', 'college': '中国科学院科技促进发展局', 'major': '科学研究管理;有机化工;图书情报与数字图书馆;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 8: {'name': '应验', 'college': '北京大学', 'major': '行政学及国家行政管理;经济体制改革;思想政治教育;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 9: {'name': '左玉河', 'college': '中国社会科学院', 'major': '中国近现代史;史学理论;政治学;', 'paper': '中国科学社与民国学术体制之创建'}, 10: {'name': '张萍', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 11: {'name': 'Ding Lin', 'college': '美国俄亥俄州州立大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 12: {'name': '徐祯', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 13: {'name': '朱小玉', 'college': '中国科学技术大学附属第一医院', 'major': '肿瘤学;心血管系统疾病;临床医学;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 14: {'name': '邬惟为', 'college': '山西医科大学', 'major': '妇产科学;医学教育与医学边缘学科;感染性疾病及传染病;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 15: {'name': '官方霖', 'college': '西安交通大学', 'major': '医学教育与医学边缘学科;生物学;公安;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 16: {'name': '朱蔚彤', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 17: {'name': '孙瑞娟', 'college': '国家自然科学基金委员会', 'major': '医学教育与医学边缘学科;科学研究管理;基础医学;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 18: {'name': '徐岩英', 'college': '国家自然科学基金委员会生命科学部', 'major': '口腔科学;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 19: {'name': '张凤珠', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 20: {'name': '冯勇', 'college': '国家自然科学基金委员会综合计划局', 'major': '科学研究管理;自然科学理论与方法;图书情报与数字图书馆;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 21: {'name': '谢焕英', 'college': '国家自然科学基金委员会', 'major': '自然科学理论与方法;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 22: {'name': '蔡乾和', 'college': '华北理工大学', 'major': '科学研究管理;自然科学理论与方法;哲学;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 23: {'name': '苑怡', 'college': '中国国家科技评估中心', 'major': '自然科学理论与方法;科学研究管理;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 24: {'name': '邢怀滨', 'college': '中国科学技术部科技评估中心', 'major': '科学研究管理;自然科学理论与方法;宏观经济管理与可持续发展;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 25: {'name': '田德录', 'college': '中国科学技术部科技评估中心', 'major': '企业经济;科学研究管理;经济体制改革;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 26: {'name': '郎赟超', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;地球物理学;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 27: {'name': '丁虎', 'college': '上海市应用数学和力学研究所', 'major': '力学;工业通用技术及设备;电力工业;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 28: {'name': '韩晓昆', 'college': '天津大学', 'major': '环境科学与资源利用;地质学;地球物理学;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 29: {'name': '剌伟', 'college': '天津大学', 'major': '环境科学与资源利用;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 30: {'name': '刘丛强', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;矿业工程;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 31: {'name': '韩晓彤', 'college': '北京理工大学', 'major': '宏观经济管理与可持续发展;科学研究管理;自然科学理论与方法;', 'paper': '科学推动下技术机会发现方法研究'}, 32: {'name': '朱东华', 'college': '北京理工大学', 'major': '科学研究管理;计算机软件及计算机应用;企业经济;', 'paper': '科学推动下技术机会发现方法研究'}, 33: {'name': '汪雪锋', 'college': '北京理工大学', 'major': '科学研究管理;图书情报与数字图书馆;计算机软件及计算机应用;', 'paper': '科学推动下技术机会发现方法研究'}, 34: {'name': '陆丽娜', 'college': '山东理工大学', 'major': '图书情报与数字图书馆;计算机软件及计算机应用;新闻与传媒;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 35: {'name': '尹丽红', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 36: {'name': '于啸', 'college': '东北农业大学', 'major': '计算机软件及计算机应用;生物学;农业基础科学;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 37: {'name': '尹居峰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 38: {'name': '尚士杰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 39: {'name': '吕凤先', 'college': '中国科学院文献情报中心', 'major': '科学研究管理;图书情报与数字图书馆;物理学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 40: {'name': '朱相丽', 'college': '中国科学院文献情报中心', 'major': '图书情报与数字图书馆;科学研究管理;材料科学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 41: {'name': '刘小平', 'college': '中国科学院大学', 'major': '图书情报与数字图书馆;科学研究管理;物理学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 42: {'name': '白云朴', 'college': '南京航空航天大学', 'major': '信息经济与邮政经济;企业经济;经济体制改革;', 'paper': '科学数据共享研究的演化路径分析'}, 43: {'name': '李果', 'college': '南京邮电大学', 'major': '信息经济与邮政经济;经济体制改革;人才学与劳动科学;', 'paper': '科学数据共享研究的演化路径分析'}, 44: {'name': 'Andrew Chi-Chih YAO', 'college': '清华大学', 'major': '计算机硬件技术;高等教育;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 45: {'name': 'Xia YIN', 'college': '清华大学', 'major': '计算机硬件技术;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 46: {'name': 'Yinqun LIU', 'college': '清华大学', 'major': '计算机硬件技术;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 47: {'name': 'Guoliang LI', 'college': '清华大学', 'major': '计算机软件及计算机应用;计算机硬件技术;冶金工业;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 48: {'name': '包秋华', 'college': '内蒙古农业大学', 'major': '生物学;轻工业手工业;中医学;', 'paper': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, 49: {'name': '刘倩宇', 'college': '中国农业科学院植物保护研究所', 'major': '植物保护;化学;图书情报与数字图书馆;', 'paper': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, 50: {'name': '阎晓阳', 'college': '北京大学', 'major': '马克思主义;中国共产党;政治学;', 'paper': '如何理解“科学社会主义的入门”？——基于《社会主义从空想到科学的发展》'}, 51: {'name': '张建琴', 'college': '嘉兴学院', 'major': '自然科学理论与方法;科学研究管理;非线性科学与系统科学;', 'paper': '评析贾祖璋的科普思想及实践'}, 52: {'name': '高铸烨', 'college': '中国中医科学院西苑医院', 'major': '中医学;心血管系统疾病;中药学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 53: {'name': '张冬梅', 'college': '北京中医药大学东直门医院', 'major': '中药学;中医学;心血管系统疾病;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 54: {'name': '赵京霞', 'college': '首都医科大学附属北京中医医院', 'major': '中药学;临床医学;中医学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 55: {'name': '徐鹏', 'college': '长春中医药大学附属医院', 'major': '中医学;神经病学;中药学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 56: {'name': '毕明刚', 'college': '国家自然科学基金委员会', 'major': '中药学;中医学;肿瘤学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 57: {'name': '刘睿', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 58: {'name': '左蕾', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 59: {'name': '张鹏', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 60: {'name': '赵宗超', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;自动化技术;地质学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 61: {'name': '陶刚', 'college': '成都理工大学', 'major': '地质学;石油天然气工业;地球物理学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 62: {'name': '王坤', 'college': '中国矿业大学', 'major': '地质学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 63: {'name': '陶东平', 'college': '山东理工大学', 'major': '矿业工程;有机化工;无机化工;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 64: {'name': '张紫璇', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'paper': '脑科学技术在针刺研究中的应用'}, 65: {'name': '李少源', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;内分泌腺及全身性疾病;', 'paper': '脑科学技术在针刺研究中的应用'}, 66: {'name': '王瑜', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;临床医学;', 'paper': '脑科学技术在针刺研究中的应用'}, 67: {'name': '王艺霏', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'paper': '脑科学技术在针刺研究中的应用'}, 68: {'name': '赵亚楠', 'college': '中国中医科学院针灸研究所', 'major': '中医学;计算机软件及计算机应用;', 'paper': '脑科学技术在针刺研究中的应用'}, 69: {'name': '荣培晶', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;中药学;', 'paper': '脑科学技术在针刺研究中的应用'}, 70: {'name': '熊璋', 'college': '中国教育部', 'major': '计算机软件及计算机应用;电信技术;互联网技术;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 71: {'name': '赵健', 'college': '华东师范大学', 'major': '教育理论与教育管理;中等教育;计算机软件及计算机应用;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 72: {'name': '陆海丰', 'college': '浙江省海宁市教育局', 'major': '中等教育;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 73: {'name': '欧阳元新', 'college': '北京航空航天大学', 'major': '计算机软件及计算机应用;计算机硬件技术;电信技术;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 74: {'name': '李梦柯', 'college': '中国科学院大学', 'major': '机械工业;科学研究管理;宏观经济管理与可持续发展;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 75: {'name': '王芳', 'college': '中国科学院地理科学与资源研究所', 'major': '临床医学;环境科学与资源利用;儿科学;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 76: {'name': '赵兰香', 'college': '中国科学院科技战略咨询研究院', 'major': '科学研究管理;企业经济;宏观经济管理与可持续发展;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}} +2022-07-02 11:22:59.315 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9999): {'0': {'name': '王芳', 'college': '南开大学', 'major': '图书情报与数字图书馆;新闻与传媒;计算机软件及计算机应用;', 'title': '科学数据管理政策发展比较研究'}, '1': {'name': '卜昊昊', 'college': '中国人民大学', 'major': '新闻与传媒;', 'title': '科学数据管理政策发展比较研究'}, '2': {'name': '贾宝余', 'college': '中国科学院创新发展研究中心', 'major': '高等教育;科学研究管理;中国共产党;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '3': {'name': '杨明', 'college': '中国科学院科技促进发展局', 'major': '科学研究管理;有机化工;图书情报与数字图书馆;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '4': {'name': '应验', 'college': '北京大学', 'major': '行政学及国家行政管理;经济体制改革;思想政治教育;', 'title': '高水平科技自立自强视野中重大科技项目选题机制研究'}, '5': {'name': '左玉河', 'college': '中国社会科学院', 'major': '中国近现代史;史学理论;政治学;', 'title': '中国科学社与民国学术体制之创建'}, '6': {'name': '张萍', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '7': {'name': 'Ding Lin', 'college': '美国俄亥俄州州立大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '8': {'name': '徐祯', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'title': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, '9': {'name': '朱小玉', 'college': '中国科学技术大学附属第一医院', 'major': '肿瘤学;心血管系统疾病;临床医学;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '10': {'name': '邬惟为', 'college': '山西医科大学', 'major': '妇产科学;医学教育与医学边缘学科;感染性疾病及传染病;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '11': {'name': '官方霖', 'college': '西安交通大学', 'major': '医学教育与医学边缘学科;生物学;公安;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '12': {'name': '朱蔚彤', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '13': {'name': '孙瑞娟', 'college': '国家自然科学基金委员会', 'major': '医学教育与医学边缘学科;科学研究管理;基础医学;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '14': {'name': '徐岩英', 'college': '国家自然科学基金委员会生命科学部', 'major': '口腔科学;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '15': {'name': '张凤珠', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'title': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, '16': {'name': '冯勇', 'college': '国家自然科学基金委员会综合计划局', 'major': '科学研究管理;自然科学理论与方法;图书情报与数字图书馆;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '17': {'name': '谢焕英', 'college': '国家自然科学基金委员会', 'major': '自然科学理论与方法;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '18': {'name': '蔡乾和', 'college': '华北理工大学', 'major': '科学研究管理;自然科学理论与方法;哲学;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '19': {'name': '苑怡', 'college': '中国国家科技评估中心', 'major': '自然科学理论与方法;科学研究管理;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '20': {'name': '邢怀滨', 'college': '中国科学技术部科技评估中心', 'major': '科学研究管理;自然科学理论与方法;宏观经济管理与可持续发展;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '21': {'name': '田德录', 'college': '中国科学技术部科技评估中心', 'major': '企业经济;科学研究管理;经济体制改革;', 'title': '国家自然科学基金重大项目绩效评价探析及政策思考'}, '22': {'name': '郎赟超', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;地球物理学;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '23': {'name': '丁虎', 'college': '上海市应用数学和力学研究所', 'major': '力学;工业通用技术及设备;电力工业;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '24': {'name': '韩晓昆', 'college': '天津大学', 'major': '环境科学与资源利用;地质学;地球物理学;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '25': {'name': '剌伟', 'college': '天津大学', 'major': '环境科学与资源利用;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '26': {'name': '刘丛强', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;矿业工程;', 'title': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, '27': {'name': '韩晓彤', 'college': '北京理工大学', 'major': '宏观经济管理与可持续发展;科学研究管理;自然科学理论与方法;', 'title': '科学推动下技术机会发现方法研究'}, '28': {'name': '朱东华', 'college': '北京理工大学', 'major': '科学研究管理;计算机软件及计算机应用;企业经济;', 'title': '科学推动下技术机会发现方法研究'}, '29': {'name': '汪雪锋', 'college': '北京理工大学', 'major': '科学研究管理;图书情报与数字图书馆;计算机软件及计算机应用;', 'title': '科学推动下技术机会发现方法研究'}, '30': {'name': '陆丽娜', 'college': '山东理工大学', 'major': '图书情报与数字图书馆;计算机软件及计算机应用;新闻与传媒;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '31': {'name': '尹丽红', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '32': {'name': '于啸', 'college': '东北农业大学', 'major': '计算机软件及计算机应用;生物学;农业基础科学;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '33': {'name': '尹居峰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '34': {'name': '尚士杰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'title': '基于区块链的农业科学数据管理场景模型构建研究'}, '35': {'name': '吕凤先', 'college': '中国科学院文献情报中心', 'major': '科学研究管理;图书情报与数字图书馆;物理学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '36': {'name': '朱相丽', 'college': '中国科学院文献情报中心', 'major': '图书情报与数字图书馆;科学研究管理;材料科学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '37': {'name': '刘小平', 'college': '中国科学院大学', 'major': '图书情报与数字图书馆;科学研究管理;物理学;', 'title': '美国能源基础科学竞争力分析及对我国的启示'}, '38': {'name': '白云朴', 'college': '南京航空航天大学', 'major': '信息经济与邮政经济;企业经济;经济体制改革;', 'title': '科学数据共享研究的演化路径分析'}, '39': {'name': '李果', 'college': '南京邮电大学', 'major': '信息经济与邮政经济;经济体制改革;人才学与劳动科学;', 'title': '科学数据共享研究的演化路径分析'}, '40': {'name': 'Andrew Chi-Chih YAO', 'college': '清华大学', 'major': '计算机硬件技术;高等教育;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '41': {'name': 'Xia YIN', 'college': '清华大学', 'major': '计算机硬件技术;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '42': {'name': 'Yinqun LIU', 'college': '清华大学', 'major': '计算机硬件技术;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '43': {'name': 'Guoliang LI', 'college': '清华大学', 'major': '计算机软件及计算机应用;计算机硬件技术;冶金工业;', 'title': 'Perspectives from the second Global Forum on Development of Computer Science'}, '44': {'name': '包秋华', 'college': '内蒙古农业大学', 'major': '生物学;轻工业手工业;中医学;', 'title': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, '45': {'name': '刘倩宇', 'college': '中国农业科学院植物保护研究所', 'major': '植物保护;化学;图书情报与数字图书馆;', 'title': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, '46': {'name': '阎晓阳', 'college': '北京大学', 'major': '马克思主义;中国共产党;政治学;', 'title': '如何理解“科学社会主义的入门”？——基于《社会主义从空想到科学的发展》'}, '47': {'name': '张建琴', 'college': '嘉兴学院', 'major': '自然科学理论与方法;科学研究管理;非线性科学与系统科学;', 'title': '评析贾祖璋的科普思想及实践'}, '48': {'name': '高铸烨', 'college': '中国中医科学院西苑医院', 'major': '中医学;心血管系统疾病;中药学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '49': {'name': '张冬梅', 'college': '北京中医药大学东直门医院', 'major': '中药学;中医学;心血管系统疾病;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '50': {'name': '赵京霞', 'college': '首都医科大学附属北京中医医院', 'major': '中药学;临床医学;中医学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '51': {'name': '徐鹏', 'college': '长春中医药大学附属医院', 'major': '中医学;神经病学;中药学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '52': {'name': '毕明刚', 'college': '国家自然科学基金委员会', 'major': '中药学;中医学;肿瘤学;', 'title': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, '53': {'name': '刘睿', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '54': {'name': '左蕾', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '55': {'name': '张鹏', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '56': {'name': '赵宗超', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;自动化技术;地质学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '57': {'name': '陶刚', 'college': '成都理工大学', 'major': '地质学;石油天然气工业;地球物理学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '58': {'name': '王坤', 'college': '中国矿业大学', 'major': '地质学;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '59': {'name': '陶东平', 'college': '山东理工大学', 'major': '矿业工程;有机化工;无机化工;', 'title': '纳米地质学——量子科学走进地质学的桥梁'}, '60': {'name': '张紫璇', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'title': '脑科学技术在针刺研究中的应用'}, '61': {'name': '李少源', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;内分泌腺及全身性疾病;', 'title': '脑科学技术在针刺研究中的应用'}, '62': {'name': '王瑜', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;临床医学;', 'title': '脑科学技术在针刺研究中的应用'}, '63': {'name': '王艺霏', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'title': '脑科学技术在针刺研究中的应用'}, '64': {'name': '赵亚楠', 'college': '中国中医科学院针灸研究所', 'major': '中医学;计算机软件及计算机应用;', 'title': '脑科学技术在针刺研究中的应用'}, '65': {'name': '荣培晶', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;中药学;', 'title': '脑科学技术在针刺研究中的应用'}, '66': {'name': '熊璋', 'college': '中国教育部', 'major': '计算机软件及计算机应用;电信技术;互联网技术;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '67': {'name': '赵健', 'college': '华东师范大学', 'major': '教育理论与教育管理;中等教育;计算机软件及计算机应用;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '68': {'name': '陆海丰', 'college': '浙江省海宁市教育局', 'major': '中等教育;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '69': {'name': '欧阳元新', 'college': '北京航空航天大学', 'major': '计算机软件及计算机应用;计算机硬件技术;电信技术;', 'title': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, '70': {'name': '李梦柯', 'college': '中国科学院大学', 'major': '机械工业;科学研究管理;宏观经济管理与可持续发展;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, '71': {'name': '王芳', 'college': '中国科学院地理科学与资源研究所', 'major': '临床医学;环境科学与资源利用;儿科学;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, '72': {'name': '赵兰香', 'college': '中国科学院科技战略咨询研究院', 'major': '科学研究管理;企业经济;宏观经济管理与可持续发展;', 'title': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '王芳', 'college': '南开大学', 'major': '图书情报与数字图书馆;新闻与传媒;计算机软件及计算机应用;', 'paper': '科学数据管理政策发展比较研究'}, 5: {'name': '卜昊昊', 'college': '中国人民大学', 'major': '新闻与传媒;', 'paper': '科学数据管理政策发展比较研究'}, 6: {'name': '贾宝余', 'college': '中国科学院创新发展研究中心', 'major': '高等教育;科学研究管理;中国共产党;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 7: {'name': '杨明', 'college': '中国科学院科技促进发展局', 'major': '科学研究管理;有机化工;图书情报与数字图书馆;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 8: {'name': '应验', 'college': '北京大学', 'major': '行政学及国家行政管理;经济体制改革;思想政治教育;', 'paper': '高水平科技自立自强视野中重大科技项目选题机制研究'}, 9: {'name': '左玉河', 'college': '中国社会科学院', 'major': '中国近现代史;史学理论;政治学;', 'paper': '中国科学社与民国学术体制之创建'}, 10: {'name': '张萍', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 11: {'name': 'Ding Lin', 'college': '美国俄亥俄州州立大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 12: {'name': '徐祯', 'college': '北京师范大学', 'major': '物理学;高等教育;中等教育;', 'paper': '2011—2020年间美国国家科学基金对物理教育研究领域资助情况分析'}, 13: {'name': '朱小玉', 'college': '中国科学技术大学附属第一医院', 'major': '肿瘤学;心血管系统疾病;临床医学;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 14: {'name': '邬惟为', 'college': '山西医科大学', 'major': '妇产科学;医学教育与医学边缘学科;感染性疾病及传染病;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 15: {'name': '官方霖', 'college': '西安交通大学', 'major': '医学教育与医学边缘学科;生物学;公安;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 16: {'name': '朱蔚彤', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 17: {'name': '孙瑞娟', 'college': '国家自然科学基金委员会', 'major': '医学教育与医学边缘学科;科学研究管理;基础医学;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 18: {'name': '徐岩英', 'college': '国家自然科学基金委员会生命科学部', 'major': '口腔科学;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 19: {'name': '张凤珠', 'college': '国家自然科学基金委员会', 'major': '科学研究管理;医学教育与医学边缘学科;医药卫生方针政策与法律法规研究;', 'paper': '2019—2020年度人口与健康领域国家自然科学基金联合基金项目申请与资助回顾及思考'}, 20: {'name': '冯勇', 'college': '国家自然科学基金委员会综合计划局', 'major': '科学研究管理;自然科学理论与方法;图书情报与数字图书馆;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 21: {'name': '谢焕英', 'college': '国家自然科学基金委员会', 'major': '自然科学理论与方法;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 22: {'name': '蔡乾和', 'college': '华北理工大学', 'major': '科学研究管理;自然科学理论与方法;哲学;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 23: {'name': '苑怡', 'college': '中国国家科技评估中心', 'major': '自然科学理论与方法;科学研究管理;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 24: {'name': '邢怀滨', 'college': '中国科学技术部科技评估中心', 'major': '科学研究管理;自然科学理论与方法;宏观经济管理与可持续发展;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 25: {'name': '田德录', 'college': '中国科学技术部科技评估中心', 'major': '企业经济;科学研究管理;经济体制改革;', 'paper': '国家自然科学基金重大项目绩效评价探析及政策思考'}, 26: {'name': '郎赟超', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;地球物理学;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 27: {'name': '丁虎', 'college': '上海市应用数学和力学研究所', 'major': '力学;工业通用技术及设备;电力工业;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 28: {'name': '韩晓昆', 'college': '天津大学', 'major': '环境科学与资源利用;地质学;地球物理学;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 29: {'name': '剌伟', 'college': '天津大学', 'major': '环境科学与资源利用;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 30: {'name': '刘丛强', 'college': '天津大学', 'major': '地质学;环境科学与资源利用;矿业工程;', 'paper': '地球系统科学观下的滨海湿地生态系统保护和恢复科学'}, 31: {'name': '韩晓彤', 'college': '北京理工大学', 'major': '宏观经济管理与可持续发展;科学研究管理;自然科学理论与方法;', 'paper': '科学推动下技术机会发现方法研究'}, 32: {'name': '朱东华', 'college': '北京理工大学', 'major': '科学研究管理;计算机软件及计算机应用;企业经济;', 'paper': '科学推动下技术机会发现方法研究'}, 33: {'name': '汪雪锋', 'college': '北京理工大学', 'major': '科学研究管理;图书情报与数字图书馆;计算机软件及计算机应用;', 'paper': '科学推动下技术机会发现方法研究'}, 34: {'name': '陆丽娜', 'college': '山东理工大学', 'major': '图书情报与数字图书馆;计算机软件及计算机应用;新闻与传媒;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 35: {'name': '尹丽红', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 36: {'name': '于啸', 'college': '东北农业大学', 'major': '计算机软件及计算机应用;生物学;农业基础科学;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 37: {'name': '尹居峰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 38: {'name': '尚士杰', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;', 'paper': '基于区块链的农业科学数据管理场景模型构建研究'}, 39: {'name': '吕凤先', 'college': '中国科学院文献情报中心', 'major': '科学研究管理;图书情报与数字图书馆;物理学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 40: {'name': '朱相丽', 'college': '中国科学院文献情报中心', 'major': '图书情报与数字图书馆;科学研究管理;材料科学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 41: {'name': '刘小平', 'college': '中国科学院大学', 'major': '图书情报与数字图书馆;科学研究管理;物理学;', 'paper': '美国能源基础科学竞争力分析及对我国的启示'}, 42: {'name': '白云朴', 'college': '南京航空航天大学', 'major': '信息经济与邮政经济;企业经济;经济体制改革;', 'paper': '科学数据共享研究的演化路径分析'}, 43: {'name': '李果', 'college': '南京邮电大学', 'major': '信息经济与邮政经济;经济体制改革;人才学与劳动科学;', 'paper': '科学数据共享研究的演化路径分析'}, 44: {'name': 'Andrew Chi-Chih YAO', 'college': '清华大学', 'major': '计算机硬件技术;高等教育;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 45: {'name': 'Xia YIN', 'college': '清华大学', 'major': '计算机硬件技术;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 46: {'name': 'Yinqun LIU', 'college': '清华大学', 'major': '计算机硬件技术;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 47: {'name': 'Guoliang LI', 'college': '清华大学', 'major': '计算机软件及计算机应用;计算机硬件技术;冶金工业;', 'paper': 'Perspectives from the second Global Forum on Development of Computer Science'}, 48: {'name': '包秋华', 'college': '内蒙古农业大学', 'major': '生物学;轻工业手工业;中医学;', 'paper': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, 49: {'name': '刘倩宇', 'college': '中国农业科学院植物保护研究所', 'major': '植物保护;化学;图书情报与数字图书馆;', 'paper': '基于Web of Science细菌活但非可培养态研究文献的可视化分析'}, 50: {'name': '阎晓阳', 'college': '北京大学', 'major': '马克思主义;中国共产党;政治学;', 'paper': '如何理解“科学社会主义的入门”？——基于《社会主义从空想到科学的发展》'}, 51: {'name': '张建琴', 'college': '嘉兴学院', 'major': '自然科学理论与方法;科学研究管理;非线性科学与系统科学;', 'paper': '评析贾祖璋的科普思想及实践'}, 52: {'name': '高铸烨', 'college': '中国中医科学院西苑医院', 'major': '中医学;心血管系统疾病;中药学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 53: {'name': '张冬梅', 'college': '北京中医药大学东直门医院', 'major': '中药学;中医学;心血管系统疾病;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 54: {'name': '赵京霞', 'college': '首都医科大学附属北京中医医院', 'major': '中药学;临床医学;中医学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 55: {'name': '徐鹏', 'college': '长春中医药大学附属医院', 'major': '中医学;神经病学;中药学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 56: {'name': '毕明刚', 'college': '国家自然科学基金委员会', 'major': '中药学;中医学;肿瘤学;', 'paper': '国家自然科学基金血瘀证与活血化瘀领域资助情况与研究趋势分析'}, 57: {'name': '刘睿', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 58: {'name': '左蕾', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 59: {'name': '张鹏', 'college': '山东理工大学', 'major': '地质学;矿业工程;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 60: {'name': '赵宗超', 'college': '山东理工大学', 'major': '计算机软件及计算机应用;自动化技术;地质学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 61: {'name': '陶刚', 'college': '成都理工大学', 'major': '地质学;石油天然气工业;地球物理学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 62: {'name': '王坤', 'college': '中国矿业大学', 'major': '地质学;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 63: {'name': '陶东平', 'college': '山东理工大学', 'major': '矿业工程;有机化工;无机化工;', 'paper': '纳米地质学——量子科学走进地质学的桥梁'}, 64: {'name': '张紫璇', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'paper': '脑科学技术在针刺研究中的应用'}, 65: {'name': '李少源', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;内分泌腺及全身性疾病;', 'paper': '脑科学技术在针刺研究中的应用'}, 66: {'name': '王瑜', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;临床医学;', 'paper': '脑科学技术在针刺研究中的应用'}, 67: {'name': '王艺霏', 'college': '中国中医科学院针灸研究所', 'major': '中医学;内分泌腺及全身性疾病;图书情报与数字图书馆;', 'paper': '脑科学技术在针刺研究中的应用'}, 68: {'name': '赵亚楠', 'college': '中国中医科学院针灸研究所', 'major': '中医学;计算机软件及计算机应用;', 'paper': '脑科学技术在针刺研究中的应用'}, 69: {'name': '荣培晶', 'college': '中国中医科学院针灸研究所', 'major': '中医学;基础医学;中药学;', 'paper': '脑科学技术在针刺研究中的应用'}, 70: {'name': '熊璋', 'college': '中国教育部', 'major': '计算机软件及计算机应用;电信技术;互联网技术;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 71: {'name': '赵健', 'college': '华东师范大学', 'major': '教育理论与教育管理;中等教育;计算机软件及计算机应用;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 72: {'name': '陆海丰', 'college': '浙江省海宁市教育局', 'major': '中等教育;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 73: {'name': '欧阳元新', 'college': '北京航空航天大学', 'major': '计算机软件及计算机应用;计算机硬件技术;电信技术;', 'paper': '义务教育阶段信息科技课程的时代性与科学性——《义务教育信息科技课程标准(2022年版)》解读'}, 74: {'name': '李梦柯', 'college': '中国科学院大学', 'major': '机械工业;科学研究管理;宏观经济管理与可持续发展;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 75: {'name': '王芳', 'college': '中国科学院地理科学与资源研究所', 'major': '临床医学;环境科学与资源利用;儿科学;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 76: {'name': '赵兰香', 'college': '中国科学院科技战略咨询研究院', 'major': '科学研究管理;企业经济;宏观经济管理与可持续发展;', 'paper': '科学知识特征对基于科学的创新绩效的影响——基于仪器产业的分析'}, 'type': 'response'} diff --git a/dcs/tests/spider_task.py b/dcs/tests/spider_task.py index 9c40819..1962e4e 100644 --- a/dcs/tests/spider_task.py +++ b/dcs/tests/spider_task.py @@ -1,4 +1,3 @@ -import csv import socket import threading from typing import Optional @@ -8,9 +7,9 @@ from msedge.selenium_tools import Edge from msedge.selenium_tools import EdgeOptions from conf.config import global_var, get_free_sockets, get_crawl_result, get_by_cookie, set_state_socket +from dcs.tests.zhiwang import * from dcs.tools.database import get_crawl_result_by_crawl_id, write_result2database from dcs.tools.database import get_last_crawl_id, create_crawl_result_table -from dcs.tests.zhiwang import * def write2database(paper: Paper, table_name: str, last_crawl_id: int): @@ -47,7 +46,7 @@ class Crawler(threading.Thread): def crawl_zhiwang(self, user_name=None): edge_options = EdgeOptions() edge_options.use_chromium = True - # edge_options.add_argument('headless') + edge_options.add_argument('--headless') driver = Edge(options=edge_options, executable_path=r'G:\course\yykf\dcs\bin\msedgedriver.exe') soup = driver_open(driver, self.partial_task.word) # 搜索word @@ -130,7 +129,7 @@ class Spider_task(threading.Thread): self.client_socket = client_socket self.request_map = request_map self.partial_tasks: list[Spider_partial_task] = [] - self.const_page = 3 + self.const_page = 1 def distribute_task(self): # distribute tasks, 3 pages as a task diff --git a/dcs/tests/zhiwang.py b/dcs/tests/zhiwang.py index d08dab4..fba9211 100644 --- a/dcs/tests/zhiwang.py +++ b/dcs/tests/zhiwang.py @@ -6,6 +6,7 @@ import time import requests # 定义论文类 +from loguru import logger from msedge.selenium_tools import webdriver from msedge.selenium_tools.service import Service from selenium.webdriver.common.by import By @@ -30,21 +31,15 @@ class Author: # 进入知网首页并搜索关键词 def driver_open(driver, key_word): - from msedge.selenium_tools import EdgeOptions - url = "https://www.cnki.net/" - edge_options = EdgeOptions() - edge_options.use_chromium = True - # 设置无界面模式，也可以添加其它设置 - edge_options.add_argument('--headless') url = "https://www.cnki.net/" driver.get(url) - time.sleep(2) + time.sleep(1) driver.find_element(by=By.CSS_SELECTOR, value='#txt_SearchText').send_keys(key_word) - time.sleep(2) + # time.sleep(2) # 点击搜索按钮 driver.find_element(by=By.CSS_SELECTOR, value='body > div.wrapper.section1 > div.searchmain > div > div.input-box > input.search-btn').click() - time.sleep(5) + time.sleep(5) # 必须要等待 content = driver.page_source.encode('utf-8') # driver.close() soup = BeautifulSoup(content, 'lxml') @@ -55,7 +50,8 @@ def spider(driver, soup, papers): tbody = soup.find_all('tbody') try: tbody = BeautifulSoup(str(tbody[0]), 'lxml') - except: + except Exception as e: + logger.error(str(e)) return tr = tbody.find_all('tr') for item in tr: @@ -86,7 +82,7 @@ def spider(driver, soup, papers): print('\n') paper = Paper(title, authors) papers.append(paper) - time.sleep(1) # 每调一次spider休息1s + # time.sleep(1) # 每调一次spider休息1s # pn表示当前要爬的页数 diff --git a/dcs/tools/database.py b/dcs/tools/database.py index f8a445a..176ebde 100644 --- a/dcs/tools/database.py +++ b/dcs/tools/database.py @@ -159,6 +159,7 @@ def get_crawl_result_by_crawl_id(table_name: str, crawl_id: int): def create_table(create_sql: str): try: conn = mysql_conn() + cur = conn.cursor() cur.execute(create_sql) conn.commit()