|
|
|
|
@ -6801,3 +6801,603 @@
|
|
|
|
|
2022-04-21 20:42:59.230 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-04-21 20:42:59.476 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-04-21 20:42:59.497 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-28 10:40:07.596 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 10:40:07.611 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 10:40:07.611 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 10:40:07.611 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 10:40:07.627 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 10:40:07.658 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 10:40:47.217 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 10:40:47.327 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 10:40:47.358 | INFO | dcs.tests.user_request_handler:register:37 - [REQUEST] register
|
|
|
|
|
2022-06-28 11:26:41.089 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 11:26:41.089 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 11:26:41.089 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 11:26:41.089 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 11:26:41.105 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 11:26:41.152 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 11:26:44.118 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 11:26:44.228 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 11:26:44.291 | INFO | dcs.tests.user_request_handler:register:37 - [REQUEST] register
|
|
|
|
|
2022-06-28 11:28:58.088 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 11:28:58.088 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 11:28:58.104 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 11:28:58.104 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 11:28:58.120 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 11:28:58.151 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 11:29:02.389 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 11:29:02.483 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 11:29:02.499 | INFO | dcs.tests.user_request_handler:register:37 - [REQUEST] register
|
|
|
|
|
2022-06-28 11:29:03.189 | INFO | dcs.tests.user_request_handler:register:45 - [RESPONSE] register: 注册成功
|
|
|
|
|
2022-06-28 11:29:03.268 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '注册成功'}
|
|
|
|
|
2022-06-28 11:29:03.299 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 11:29:03.331 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 11:29:04.005 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: 30b7a709cb34e1eeb1bb8dfdade7ea458354a17f
|
|
|
|
|
2022-06-28 11:29:04.037 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '30b7a709cb34e1eeb1bb8dfdade7ea458354a17f'}
|
|
|
|
|
2022-06-28 11:29:04.068 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 11:29:04.084 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-28 11:29:04.084 | INFO | dcs.tests.user_request_handler:report_state:23 - [RESPONSE] report free: success marked 30b7a709cb34e1eeb1bb8dfdade7ea458354a17f
|
|
|
|
|
2022-06-28 11:29:04.146 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 30b7a709cb34e1eeb1bb8dfdade7ea458354a17f'}
|
|
|
|
|
2022-06-28 11:29:04.193 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': '30b7a709cb34e1eeb1bb8dfdade7ea458354a17f'})
|
|
|
|
|
2022-06-28 11:29:04.193 | DEBUG | dcs.tests.spider_task:distribute_task:167 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000002134D847D90>, <dcs.tests.spider_task.Spider_partial_task object at 0x000002134D847DC0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000002134D847E20>]
|
|
|
|
|
2022-06-28 11:29:04.193 | DEBUG | dcs.tests.spider_task:run:203 - [<socket.socket fd=400, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-28 11:29:04.193 | DEBUG | dcs.tests.spider_task:run:215 - generating remote task
|
|
|
|
|
2022-06-28 11:29:04.193 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 11:29:04.193 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 11:29:04.225 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-28 11:29:04.225 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-28 11:29:04.241 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 11:29:04.303 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 11:29:04.413 | DEBUG | dcs.requester:run:70 - receiving remote task result, saving...
|
|
|
|
|
2022-06-28 11:29:04.492 | DEBUG | dcs.requester:run:74 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-28 11:29:05.151 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 11:29:05.151 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:05.496 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 11:29:05.496 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:05.763 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:06.140 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:06.517 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:06.627 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 11:29:06.768 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': '30b7a709cb34e1eeb1bb8dfdade7ea458354a17f'})
|
|
|
|
|
2022-06-28 11:29:07.113 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': '30b7a709cb34e1eeb1bb8dfdade7ea458354a17f'})
|
|
|
|
|
2022-06-28 11:29:07.145 | DEBUG | dcs.tests.spider_task:compose_result:177 - composing task...
|
|
|
|
|
2022-06-28 11:29:07.396 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 11:29:07.600 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 11:29:07.647 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-28 14:49:12.246 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 14:49:12.246 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 14:49:12.246 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 14:49:12.246 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 14:49:12.261 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 14:49:12.371 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 14:49:17.048 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 14:49:17.063 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 14:49:17.110 | INFO | dcs.tests.user_request_handler:register:37 - [REQUEST] register
|
|
|
|
|
2022-06-28 14:49:17.597 | INFO | dcs.tests.user_request_handler:register:45 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-28 14:49:17.659 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-28 14:49:17.753 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 14:49:17.769 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 14:49:18.947 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: 9fef547cebc2f8673e826daa21337ce6f9b5ffcb
|
|
|
|
|
2022-06-28 14:49:18.962 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '9fef547cebc2f8673e826daa21337ce6f9b5ffcb'}
|
|
|
|
|
2022-06-28 14:49:18.978 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 14:49:19.010 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-28 14:49:19.025 | INFO | dcs.tests.user_request_handler:report_state:23 - [RESPONSE] report free: success marked 9fef547cebc2f8673e826daa21337ce6f9b5ffcb
|
|
|
|
|
2022-06-28 14:49:19.056 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 9fef547cebc2f8673e826daa21337ce6f9b5ffcb'}
|
|
|
|
|
2022-06-28 14:49:19.135 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': '9fef547cebc2f8673e826daa21337ce6f9b5ffcb'})
|
|
|
|
|
2022-06-28 14:49:19.135 | DEBUG | dcs.tests.spider_task:distribute_task:167 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001BADF133FD0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001BADF16C400>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001BADF16C1C0>]
|
|
|
|
|
2022-06-28 14:49:19.135 | DEBUG | dcs.tests.spider_task:run:203 - [<socket.socket fd=1016, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-28 14:49:19.135 | DEBUG | dcs.tests.spider_task:run:215 - generating remote task
|
|
|
|
|
2022-06-28 14:49:19.135 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 14:49:19.166 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 14:49:19.198 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-28 14:49:19.198 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-28 14:49:19.198 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 14:49:19.370 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 14:49:19.386 | DEBUG | dcs.requester:run:70 - receiving remote task result, saving...
|
|
|
|
|
2022-06-28 14:49:19.700 | DEBUG | dcs.requester:run:74 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-28 14:49:20.453 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 14:49:20.453 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:20.595 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 14:49:20.595 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:20.846 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:20.956 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:21.364 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:21.505 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 14:49:21.725 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': '9fef547cebc2f8673e826daa21337ce6f9b5ffcb'})
|
|
|
|
|
2022-06-28 14:49:21.960 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': '9fef547cebc2f8673e826daa21337ce6f9b5ffcb'})
|
|
|
|
|
2022-06-28 14:49:22.023 | DEBUG | dcs.tests.spider_task:compose_result:177 - composing task...
|
|
|
|
|
2022-06-28 14:49:22.588 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 14:49:23.122 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 14:49:23.153 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-28 14:54:28.848 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52306)
|
|
|
|
|
2022-06-28 14:56:06.566 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 14:56:06.566 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 14:56:06.581 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 14:56:06.581 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 14:56:06.597 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 14:56:06.660 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 14:57:35.539 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52347)
|
|
|
|
|
2022-06-28 14:59:09.516 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52356)
|
|
|
|
|
2022-06-28 15:01:29.819 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:01:29.819 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:01:29.819 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:01:29.819 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:01:29.835 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:01:29.945 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:01:42.873 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52363)
|
|
|
|
|
2022-06-28 15:01:57.533 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:01:57.533 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:01:57.533 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:01:57.533 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:01:57.548 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:01:57.611 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:02:50.367 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52367)
|
|
|
|
|
2022-06-28 15:02:50.414 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:02:50.446 | INFO | dcs.tests.user_request_handler:register:37 - [REQUEST] register
|
|
|
|
|
2022-06-28 15:02:50.979 | INFO | dcs.tests.user_request_handler:register:45 - [RESPONSE] register: 注册成功
|
|
|
|
|
2022-06-28 15:02:50.995 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52367): {'register': '注册成功'}
|
|
|
|
|
2022-06-28 15:02:59.519 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52370)
|
|
|
|
|
2022-06-28 15:02:59.550 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:02:59.597 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:03:00.335 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: 9c8c82096cada9adb7f6fbec6a7d568567186932
|
|
|
|
|
2022-06-28 15:03:00.351 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52370): {'cookie': '9c8c82096cada9adb7f6fbec6a7d568567186932'}
|
|
|
|
|
2022-06-28 15:03:42.009 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52374)
|
|
|
|
|
2022-06-28 15:03:42.056 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:03:42.072 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:03:43.044 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: d6c6088e46bdc7573af8e25de200f0cb88ca1e35
|
|
|
|
|
2022-06-28 15:03:43.076 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52374): {'cookie': 'd6c6088e46bdc7573af8e25de200f0cb88ca1e35'}
|
|
|
|
|
2022-06-28 15:04:21.446 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52387)
|
|
|
|
|
2022-06-28 15:05:13.998 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52389)
|
|
|
|
|
2022-06-28 15:05:14.077 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:05:14.092 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:05:15.081 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: 88ab5f7581216778aed56970f68532cca5c2a18a
|
|
|
|
|
2022-06-28 15:05:15.081 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52389): {'cookie': '88ab5f7581216778aed56970f68532cca5c2a18a'}
|
|
|
|
|
2022-06-28 15:05:58.183 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52393)
|
|
|
|
|
2022-06-28 15:05:58.324 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:05:58.340 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:05:58.936 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] cookie: 9b4349c2a1c3baffbf0dd923f92fc44b8aca505d
|
|
|
|
|
2022-06-28 15:05:58.983 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52393): {'cookie': '9b4349c2a1c3baffbf0dd923f92fc44b8aca505d'}
|
|
|
|
|
2022-06-28 15:07:49.742 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:07:49.742 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:07:49.742 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:07:49.742 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:07:49.757 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:07:49.835 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:07:52.488 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52401)
|
|
|
|
|
2022-06-28 15:07:52.614 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:07:52.614 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:07:53.445 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] login: 86b46a0f4bbf9ccbe998ace76ac57c5e2f1d056a
|
|
|
|
|
2022-06-28 15:07:53.461 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52401): {'cookie': '86b46a0f4bbf9ccbe998ace76ac57c5e2f1d056a'}
|
|
|
|
|
2022-06-28 15:09:06.121 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52408)
|
|
|
|
|
2022-06-28 15:09:06.153 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:09:06.200 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:09:07.110 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] login: 762010ca505017652bd14ae5734779f67b3b2007
|
|
|
|
|
2022-06-28 15:09:07.158 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52408): {'cookie': '762010ca505017652bd14ae5734779f67b3b2007'}
|
|
|
|
|
2022-06-28 15:10:42.794 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52414)
|
|
|
|
|
2022-06-28 15:10:42.872 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:10:42.872 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:10:43.815 | INFO | dcs.tests.user_request_handler:login:34 - [RESPONSE] login: d04b5b59c6bed3c9f87e2488ff13c6319c71a9f2
|
|
|
|
|
2022-06-28 15:10:43.830 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52414): {'cookie': 'd04b5b59c6bed3c9f87e2488ff13c6319c71a9f2'}
|
|
|
|
|
2022-06-28 15:11:22.808 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:11:22.808 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:11:22.808 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:11:22.808 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:11:22.824 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:11:22.871 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:11:25.603 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52426)
|
|
|
|
|
2022-06-28 15:11:25.634 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:11:25.677 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:11:26.273 | INFO | dcs.tests.user_request_handler:login:35 - [RESPONSE] login: 81f7d8aeadc116583a7602b11088ca49ef37291d
|
|
|
|
|
2022-06-28 15:11:26.305 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52426): {'cookie': '81f7d8aeadc116583a7602b11088ca49ef37291d', 'login': '81f7d8aeadc116583a7602b11088ca49ef37291d'}
|
|
|
|
|
2022-06-28 15:11:36.207 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52430)
|
|
|
|
|
2022-06-28 15:13:27.254 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:13:27.286 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:13:27.301 | INFO | dcs.tests.user_request_handler:register:38 - [REQUEST] register
|
|
|
|
|
2022-06-28 15:13:27.897 | INFO | dcs.tests.user_request_handler:register:46 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-28 15:13:27.928 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-28 15:13:27.944 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:13:27.976 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:13:29.012 | INFO | dcs.tests.user_request_handler:login:35 - [RESPONSE] login: 7c947cd7366e9fdb7320e5450bf336b0448865f2
|
|
|
|
|
2022-06-28 15:13:29.059 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '7c947cd7366e9fdb7320e5450bf336b0448865f2', 'login': '7c947cd7366e9fdb7320e5450bf336b0448865f2'}
|
|
|
|
|
2022-06-28 15:13:29.184 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:13:29.231 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-28 15:13:29.326 | INFO | dcs.tests.user_request_handler:report_state:23 - [RESPONSE] report free: success marked 7c947cd7366e9fdb7320e5450bf336b0448865f2
|
|
|
|
|
2022-06-28 15:13:29.451 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 7c947cd7366e9fdb7320e5450bf336b0448865f2'}
|
|
|
|
|
2022-06-28 15:13:29.624 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': '7c947cd7366e9fdb7320e5450bf336b0448865f2'})
|
|
|
|
|
2022-06-28 15:13:29.624 | DEBUG | dcs.tests.spider_task:distribute_task:167 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5C3D0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5C310>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5C430>]
|
|
|
|
|
2022-06-28 15:13:29.624 | DEBUG | dcs.tests.spider_task:run:203 - [<socket.socket fd=308, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-28 15:13:29.624 | DEBUG | dcs.tests.spider_task:run:215 - generating remote task
|
|
|
|
|
2022-06-28 15:13:29.624 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:13:29.655 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:13:29.671 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 15:13:29.859 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:13:29.875 | DEBUG | dcs.requester:run:70 - receiving remote task result, saving...
|
|
|
|
|
2022-06-28 15:13:29.875 | DEBUG | dcs.requester:run:74 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-28 15:13:29.906 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-28 15:13:29.906 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-28 15:13:30.644 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:13:30.644 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:30.958 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:13:30.958 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:31.020 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:31.350 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:31.460 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:31.774 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': '7c947cd7366e9fdb7320e5450bf336b0448865f2'})
|
|
|
|
|
2022-06-28 15:13:31.978 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:13:32.449 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': '7c947cd7366e9fdb7320e5450bf336b0448865f2'})
|
|
|
|
|
2022-06-28 15:13:32.449 | DEBUG | dcs.tests.spider_task:compose_result:177 - composing task...
|
|
|
|
|
2022-06-28 15:13:32.967 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:13:33.453 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:13:33.501 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-28 15:16:17.728 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:16:17.806 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:16:17.822 | INFO | dcs.tests.user_request_handler:register:38 - [REQUEST] register
|
|
|
|
|
2022-06-28 15:16:18.419 | INFO | dcs.tests.user_request_handler:register:46 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-28 15:16:18.450 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-28 15:16:18.513 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:16:18.529 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:16:19.470 | INFO | dcs.tests.user_request_handler:login:35 - [RESPONSE] login: e3939532880fb270dbec8a63ba01fda943df5e14
|
|
|
|
|
2022-06-28 15:16:19.486 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14', 'login': 'e3939532880fb270dbec8a63ba01fda943df5e14'}
|
|
|
|
|
2022-06-28 15:16:19.502 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:16:19.517 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-28 15:16:19.517 | INFO | dcs.tests.user_request_handler:report_state:23 - [RESPONSE] report free: success marked e3939532880fb270dbec8a63ba01fda943df5e14
|
|
|
|
|
2022-06-28 15:16:19.533 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked e3939532880fb270dbec8a63ba01fda943df5e14'}
|
|
|
|
|
2022-06-28 15:16:19.721 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 15:16:19.737 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-28 15:16:19.737 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14'})
|
|
|
|
|
2022-06-28 15:16:19.768 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-28 15:16:19.816 | DEBUG | dcs.tests.spider_task:distribute_task:167 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5C1F0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5D2A0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001EEF8A5D120>]
|
|
|
|
|
2022-06-28 15:16:19.816 | DEBUG | dcs.tests.spider_task:run:203 - [<socket.socket fd=1172, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-28 15:16:19.816 | DEBUG | dcs.tests.spider_task:run:215 - generating remote task
|
|
|
|
|
2022-06-28 15:16:19.816 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:16:19.972 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:16:19.988 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:16:20.004 | DEBUG | dcs.requester:run:70 - receiving remote task result, saving...
|
|
|
|
|
2022-06-28 15:16:20.004 | DEBUG | dcs.requester:run:74 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-28 15:16:20.977 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:16:20.977 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:20.993 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:16:20.993 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:21.338 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:21.448 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:22.045 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:22.343 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:22.531 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14'})
|
|
|
|
|
2022-06-28 15:16:22.547 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:16:22.782 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14'})
|
|
|
|
|
2022-06-28 15:16:22.782 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:16:22.799 | DEBUG | dcs.tests.spider_task:compose_result:177 - composing task...
|
|
|
|
|
2022-06-28 15:16:23.050 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:16:23.332 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:16:23.332 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:23.348 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:16:23.395 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-28 15:16:23.599 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:23.740 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:16:23.740 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:24.039 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:24.133 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:24.525 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:16:24.557 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14'})
|
|
|
|
|
2022-06-28 15:16:24.871 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'e3939532880fb270dbec8a63ba01fda943df5e14'})
|
|
|
|
|
2022-06-28 15:17:07.523 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:17:07.523 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:17:07.523 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:17:07.523 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:17:07.532 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:17:07.579 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:17:39.064 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52487)
|
|
|
|
|
2022-06-28 15:19:21.217 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:19:21.217 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:19:21.217 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:19:21.217 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:19:21.232 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:19:21.327 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:19:24.922 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52491)
|
|
|
|
|
2022-06-28 15:22:48.102 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:22:48.117 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:22:48.117 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:22:48.117 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:22:48.133 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:22:48.227 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:22:52.372 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52497)
|
|
|
|
|
2022-06-28 15:25:25.911 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-28 15:25:25.911 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-28 15:25:25.911 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-28 15:25:25.911 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-28 15:25:25.920 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-28 15:25:25.951 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-28 15:25:30.566 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52502)
|
|
|
|
|
2022-06-28 15:25:30.613 | INFO | dcs.spider:run:45 - processing spider request...
|
|
|
|
|
2022-06-28 15:25:30.629 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 52502), {'action': 'crawl zhiwang', 'word': 'computer'})
|
|
|
|
|
2022-06-28 15:25:30.629 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52502): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 15:28:17.011 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 52509)
|
|
|
|
|
2022-06-28 15:28:17.058 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:28:17.074 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:28:18.251 | INFO | dcs.tests.user_request_handler:login:35 - [RESPONSE] login: 89cd39c2e271951c496fb7ae7e6043814832741b
|
|
|
|
|
2022-06-28 15:28:18.299 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 52509): {'cookie': '89cd39c2e271951c496fb7ae7e6043814832741b', 'login': '89cd39c2e271951c496fb7ae7e6043814832741b'}
|
|
|
|
|
2022-06-28 15:28:59.537 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:28:59.631 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:28:59.662 | INFO | dcs.tests.user_request_handler:register:38 - [REQUEST] register
|
|
|
|
|
2022-06-28 15:29:00.196 | INFO | dcs.tests.user_request_handler:register:46 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-28 15:29:00.212 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-28 15:29:00.274 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:29:00.290 | INFO | dcs.tests.user_request_handler:login:26 - [REQUEST] login
|
|
|
|
|
2022-06-28 15:29:01.342 | INFO | dcs.tests.user_request_handler:login:35 - [RESPONSE] login: e81ec809257de9e5b0ac79eb4a134d3015add49d
|
|
|
|
|
2022-06-28 15:29:01.389 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': 'e81ec809257de9e5b0ac79eb4a134d3015add49d', 'login': 'e81ec809257de9e5b0ac79eb4a134d3015add49d'}
|
|
|
|
|
2022-06-28 15:29:01.452 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-28 15:29:01.483 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-28 15:29:01.483 | INFO | dcs.tests.user_request_handler:report_state:23 - [RESPONSE] report free: success marked e81ec809257de9e5b0ac79eb4a134d3015add49d
|
|
|
|
|
2022-06-28 15:29:01.483 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked e81ec809257de9e5b0ac79eb4a134d3015add49d'}
|
|
|
|
|
2022-06-28 15:29:01.515 | INFO | dcs.spider:run:45 - processing spider request...
|
|
|
|
|
2022-06-28 15:29:01.515 | INFO | dcs.tests.spider_task:distribute_task:154 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'e81ec809257de9e5b0ac79eb4a134d3015add49d'})
|
|
|
|
|
2022-06-28 15:29:01.515 | DEBUG | dcs.tests.spider_task:distribute_task:167 - [<dcs.tests.spider_task.Spider_partial_task object at 0x0000024C9A01C9D0>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000024C9A01C790>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000024C9A01C400>]
|
|
|
|
|
2022-06-28 15:29:01.531 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-28 15:29:01.546 | DEBUG | dcs.tests.spider_task:run:203 - [<socket.socket fd=1048, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-28 15:29:01.625 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-28 15:29:01.641 | DEBUG | dcs.tests.spider_task:run:215 - generating remote task
|
|
|
|
|
2022-06-28 15:29:01.672 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-28 15:29:01.719 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-28 15:29:01.845 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:29:01.923 | DEBUG | dcs.tests.spider_task:run:223 - generating local task
|
|
|
|
|
2022-06-28 15:29:01.923 | DEBUG | dcs.requester:run:70 - receiving remote task result, saving...
|
|
|
|
|
2022-06-28 15:29:01.923 | DEBUG | dcs.requester:run:74 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-28 15:29:02.803 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:29:02.818 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:02.913 | DEBUG | dcs.tests.spider_task:test_simulation:102 - simulation crawling...
|
|
|
|
|
2022-06-28 15:29:02.913 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:03.195 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:03.383 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:03.446 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:03.964 | INFO | dcs.tests.spider_task:write2database:35 - writing to database: test
|
|
|
|
|
2022-06-28 15:29:04.027 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'e81ec809257de9e5b0ac79eb4a134d3015add49d'})
|
|
|
|
|
2022-06-28 15:29:04.828 | INFO | dcs.tests.spider_task:run:118 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'e81ec809257de9e5b0ac79eb4a134d3015add49d'})
|
|
|
|
|
2022-06-28 15:29:04.874 | DEBUG | dcs.tests.spider_task:compose_result:177 - composing task...
|
|
|
|
|
2022-06-28 15:29:05.189 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:29:05.471 | DEBUG | dcs.tests.spider_task:compose_result:192 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-28 15:29:05.487 | INFO | dcs.communicate:run:33 - sending response to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-29 19:47:45.641 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 19:47:45.641 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 19:47:45.641 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 19:47:45.641 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 19:47:45.657 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 19:47:45.735 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 19:47:55.071 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 53943)
|
|
|
|
|
2022-06-29 19:47:55.318 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 19:47:55.333 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
|
|
|
|
|
2022-06-29 19:47:55.881 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 注册成功
|
|
|
|
|
2022-06-29 19:47:55.992 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 53943): {'register': '注册成功'}
|
|
|
|
|
2022-06-29 19:56:51.157 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54028)
|
|
|
|
|
2022-06-29 19:56:51.188 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 19:56:51.188 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 19:56:52.391 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 5e3651e55fc608b75fbbe505b896f046b4c6891a
|
|
|
|
|
2022-06-29 19:56:52.407 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54028): {'cookie': '5e3651e55fc608b75fbbe505b896f046b4c6891a'}
|
|
|
|
|
2022-06-29 19:57:51.720 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54039)
|
|
|
|
|
2022-06-29 19:57:51.798 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 19:57:51.813 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 19:57:53.173 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 9e717ab6d5a3862c7698c66c5546d3c9911e271a
|
|
|
|
|
2022-06-29 19:57:53.188 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54039): {'cookie': '9e717ab6d5a3862c7698c66c5546d3c9911e271a'}
|
|
|
|
|
2022-06-29 20:00:29.847 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 20:00:29.847 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 20:00:29.847 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 20:00:29.847 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 20:00:29.862 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 20:00:29.956 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 20:00:32.644 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54069)
|
|
|
|
|
2022-06-29 20:00:32.722 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:00:32.753 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:00:33.534 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 6ec8d6d01310d5b9b96b987e19569dad310fb1cc
|
|
|
|
|
2022-06-29 20:00:33.565 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54069): {'login': '6ec8d6d01310d5b9b96b987e19569dad310fb1cc'}
|
|
|
|
|
2022-06-29 20:01:16.042 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54084)
|
|
|
|
|
2022-06-29 20:01:16.073 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:01:16.073 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:01:17.385 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: c787a940860f59040b4d1e02921fa223cb46a869
|
|
|
|
|
2022-06-29 20:01:17.401 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54084): {'login': 'c787a940860f59040b4d1e02921fa223cb46a869'}
|
|
|
|
|
2022-06-29 20:01:17.495 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54088)
|
|
|
|
|
2022-06-29 20:01:17.604 | INFO | dcs.spider:run:22 - processing spider request...
|
|
|
|
|
2022-06-29 20:01:17.651 | INFO | dcs.tests.spider_task:distribute_task:136 - distributing task: (('127.0.0.1', 54088), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'c787a940860f59040b4d1e02921fa223cb46a869'})
|
|
|
|
|
2022-06-29 20:01:17.651 | DEBUG | dcs.tests.spider_task:distribute_task:149 - [<dcs.tests.spider_task.Spider_partial_task object at 0x0000026DE102FBE0>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000026DE102FA60>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000026DE102FE80>]
|
|
|
|
|
2022-06-29 20:01:17.651 | DEBUG | dcs.tests.spider_task:run:185 - []
|
|
|
|
|
2022-06-29 20:01:17.651 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:01:17.667 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54088): {'crawling state': 'starting, please wait...'}
|
|
|
|
|
2022-06-29 20:01:17.667 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:01:17.760 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:01:18.745 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:01:18.776 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:19.104 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:01:19.182 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:19.542 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:19.745 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:01:19.745 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:20.135 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:20.245 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:20.635 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:20.995 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54088), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'c787a940860f59040b4d1e02921fa223cb46a869'})
|
|
|
|
|
2022-06-29 20:01:21.260 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:21.729 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54088), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': 'c787a940860f59040b4d1e02921fa223cb46a869'})
|
|
|
|
|
2022-06-29 20:01:21.917 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:01:22.917 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54088), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'c787a940860f59040b4d1e02921fa223cb46a869'})
|
|
|
|
|
2022-06-29 20:01:22.932 | DEBUG | dcs.tests.spider_task:compose_result:159 - composing task...
|
|
|
|
|
2022-06-29 20:01:23.323 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:01:23.713 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:01:24.292 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:04:41.308 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 20:04:41.308 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 20:04:41.308 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 20:04:41.308 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 20:04:41.323 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 20:04:41.355 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 20:04:44.276 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54136)
|
|
|
|
|
2022-06-29 20:04:44.339 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:04:44.355 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:04:45.417 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: e3b5b304e274e2cd45e284af54d653b75343bd4f
|
|
|
|
|
2022-06-29 20:04:45.511 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54136): {'login': 'e3b5b304e274e2cd45e284af54d653b75343bd4f'}
|
|
|
|
|
2022-06-29 20:04:45.526 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54140)
|
|
|
|
|
2022-06-29 20:04:45.558 | INFO | dcs.spider:run:22 - processing spider request...
|
|
|
|
|
2022-06-29 20:04:45.573 | INFO | dcs.tests.spider_task:distribute_task:136 - distributing task: (('127.0.0.1', 54140), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'e3b5b304e274e2cd45e284af54d653b75343bd4f'})
|
|
|
|
|
2022-06-29 20:04:45.573 | DEBUG | dcs.tests.spider_task:distribute_task:149 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001D154DA2C50>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001D154DA3220>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001D154DA2B90>]
|
|
|
|
|
2022-06-29 20:04:45.573 | DEBUG | dcs.tests.spider_task:run:185 - []
|
|
|
|
|
2022-06-29 20:04:45.573 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:04:45.605 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54140): {'crawl zhiwang': 'starting, please wait...'}
|
|
|
|
|
2022-06-29 20:04:45.636 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:04:45.683 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:04:46.698 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:04:46.698 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:46.745 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:04:46.745 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.167 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.183 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:04:47.183 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.245 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.448 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.917 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:47.964 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:48.011 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:04:48.198 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54140), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'e3b5b304e274e2cd45e284af54d653b75343bd4f'})
|
|
|
|
|
2022-06-29 20:04:48.511 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54140), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'e3b5b304e274e2cd45e284af54d653b75343bd4f'})
|
|
|
|
|
2022-06-29 20:04:48.589 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54140), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': 'e3b5b304e274e2cd45e284af54d653b75343bd4f'})
|
|
|
|
|
2022-06-29 20:04:48.589 | DEBUG | dcs.tests.spider_task:compose_result:159 - composing task...
|
|
|
|
|
2022-06-29 20:04:49.386 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:04:50.026 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:04:50.995 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:23:25.116 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-29 20:23:25.225 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:23:25.241 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
|
|
|
|
|
2022-06-29 20:23:25.616 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-29 20:24:21.600 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 20:24:21.600 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 20:24:21.600 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 20:24:21.600 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 20:24:21.632 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 20:24:21.725 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 20:24:34.241 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-29 20:24:34.335 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:24:34.335 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
|
|
|
|
|
2022-06-29 20:24:35.382 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-29 20:24:35.444 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-29 20:24:35.569 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:24:35.569 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:24:37.116 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: e093dd66585292b073070fdcee8efeec9a135a30
|
|
|
|
|
2022-06-29 20:24:37.319 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'login': 'e093dd66585292b073070fdcee8efeec9a135a30'}
|
|
|
|
|
2022-06-29 20:25:00.038 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 20:25:00.038 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 20:25:00.038 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 20:25:00.038 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 20:25:00.054 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 20:25:00.148 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 20:25:02.132 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-29 20:25:02.226 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:25:02.273 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
|
|
|
|
|
2022-06-29 20:25:03.210 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
|
|
|
|
|
2022-06-29 20:25:03.398 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
|
|
|
|
|
2022-06-29 20:25:03.523 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:25:03.538 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:25:04.351 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: db157a90920fc62a1a6001a64838144c8cb7c684
|
|
|
|
|
2022-06-29 20:25:04.413 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'login': 'db157a90920fc62a1a6001a64838144c8cb7c684'}
|
|
|
|
|
2022-06-29 20:25:04.445 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:25:04.445 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
|
|
|
|
|
2022-06-29 20:25:04.445 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked db157a90920fc62a1a6001a64838144c8cb7c684
|
|
|
|
|
2022-06-29 20:25:04.554 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked db157a90920fc62a1a6001a64838144c8cb7c684'}
|
|
|
|
|
2022-06-29 20:25:04.585 | INFO | dcs.spider:run:22 - processing spider request...
|
|
|
|
|
2022-06-29 20:25:04.601 | INFO | dcs.tests.spider_task:distribute_task:136 - distributing task: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'db157a90920fc62a1a6001a64838144c8cb7c684'})
|
|
|
|
|
2022-06-29 20:25:04.601 | DEBUG | dcs.tests.spider_task:distribute_task:149 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001686B3331C0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001686B3333A0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001686B333340>]
|
|
|
|
|
2022-06-29 20:25:04.601 | DEBUG | dcs.tests.spider_task:run:185 - [<socket.socket fd=980, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
|
|
|
|
|
2022-06-29 20:25:04.601 | DEBUG | dcs.tests.spider_task:run:197 - generating remote task
|
|
|
|
|
2022-06-29 20:25:04.601 | INFO | dcs.requester:get:44 - sending crawl request to ('127.0.0.1', 9000)
|
|
|
|
|
2022-06-29 20:25:04.616 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 9000): {'crawl zhiwang': 'starting, please wait...'}
|
|
|
|
|
2022-06-29 20:25:04.632 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:25:04.632 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end
|
|
|
|
|
2022-06-29 20:25:04.632 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9000)!
|
|
|
|
|
2022-06-29 20:25:04.632 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:25:04.695 | DEBUG | dcs.requester:run:68 - receiving remote task result, saving...
|
|
|
|
|
2022-06-29 20:25:04.835 | DEBUG | dcs.requester:run:72 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
|
|
|
|
|
2022-06-29 20:25:05.507 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:25:05.507 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:05.944 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:25:05.944 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:06.148 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:06.351 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:06.882 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:07.038 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:07.335 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'db157a90920fc62a1a6001a64838144c8cb7c684'})
|
|
|
|
|
2022-06-29 20:25:08.194 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 9000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'db157a90920fc62a1a6001a64838144c8cb7c684'})
|
|
|
|
|
2022-06-29 20:25:08.257 | DEBUG | dcs.tests.spider_task:compose_result:159 - composing task...
|
|
|
|
|
2022-06-29 20:25:09.226 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:25:10.304 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:25:10.570 | INFO | dcs.communicate:run:47 - sending info to ('127.0.0.1', 9000): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'type': 'response'}
|
|
|
|
|
2022-06-29 20:25:20.007 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54314)
|
|
|
|
|
2022-06-29 20:25:20.163 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:25:20.179 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:25:21.101 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: d49d977e3441b1f14823be39ebde5a56664f8c4b
|
|
|
|
|
2022-06-29 20:25:21.163 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54314): {'login': 'd49d977e3441b1f14823be39ebde5a56664f8c4b'}
|
|
|
|
|
2022-06-29 20:25:21.179 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54318)
|
|
|
|
|
2022-06-29 20:25:21.304 | INFO | dcs.spider:run:22 - processing spider request...
|
|
|
|
|
2022-06-29 20:25:21.320 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54318): {'crawl zhiwang': 'starting, please wait...'}
|
|
|
|
|
2022-06-29 20:25:21.335 | INFO | dcs.tests.spider_task:distribute_task:136 - distributing task: (('127.0.0.1', 54318), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'd49d977e3441b1f14823be39ebde5a56664f8c4b'})
|
|
|
|
|
2022-06-29 20:25:21.335 | DEBUG | dcs.tests.spider_task:distribute_task:149 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001686B332D10>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001686B332F20>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001686B3337C0>]
|
|
|
|
|
2022-06-29 20:25:21.335 | DEBUG | dcs.tests.spider_task:run:185 - []
|
|
|
|
|
2022-06-29 20:25:21.335 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:25:21.507 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:25:21.601 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:25:22.507 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:25:22.507 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:22.726 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:25:22.726 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:22.726 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:22.757 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:25:22.757 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:22.976 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:23.116 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:23.148 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:23.226 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54318), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'd49d977e3441b1f14823be39ebde5a56664f8c4b'})
|
|
|
|
|
2022-06-29 20:25:23.538 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:23.913 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54318), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'd49d977e3441b1f14823be39ebde5a56664f8c4b'})
|
|
|
|
|
2022-06-29 20:25:24.163 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:25:25.194 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54318), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': 'd49d977e3441b1f14823be39ebde5a56664f8c4b'})
|
|
|
|
|
2022-06-29 20:25:25.210 | DEBUG | dcs.tests.spider_task:compose_result:159 - composing task...
|
|
|
|
|
2022-06-29 20:25:26.351 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:25:26.929 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:25:27.413 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 8: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 9: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:27:37.836 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54346)
|
|
|
|
|
2022-06-29 20:27:38.024 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:27:38.024 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:27:39.446 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 3d05e7a36b5e59554c3dc905df5ee05af132a8fa
|
|
|
|
|
2022-06-29 20:27:53.321 | DEBUG | __main__:<module>:14 - reading config args...
|
|
|
|
|
2022-06-29 20:27:53.321 | DEBUG | __main__:<module>:21 - starting the main server...
|
|
|
|
|
2022-06-29 20:27:53.321 | DEBUG | __main__:<module>:26 - starting the requester server...
|
|
|
|
|
2022-06-29 20:27:53.321 | DEBUG | __main__:<module>:31 - starting the spider server...
|
|
|
|
|
2022-06-29 20:27:53.336 | DEBUG | __main__:<module>:36 - starting the user server...
|
|
|
|
|
2022-06-29 20:27:53.508 | DEBUG | __main__:<module>:41 - starting the communicator server...
|
|
|
|
|
2022-06-29 20:28:01.180 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54352)
|
|
|
|
|
2022-06-29 20:28:01.227 | INFO | dcs.user_process:run:18 - processing user request...
|
|
|
|
|
2022-06-29 20:28:01.242 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
|
|
|
|
|
2022-06-29 20:28:02.321 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: b0d50e48b0a3431deff0be2c27528254e1fe8041
|
|
|
|
|
2022-06-29 20:28:02.337 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54352): {'login': 'b0d50e48b0a3431deff0be2c27528254e1fe8041'}
|
|
|
|
|
2022-06-29 20:28:02.384 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 54356)
|
|
|
|
|
2022-06-29 20:28:02.493 | INFO | dcs.spider:run:22 - processing spider request...
|
|
|
|
|
2022-06-29 20:28:02.524 | INFO | dcs.tests.spider_task:distribute_task:136 - distributing task: (('127.0.0.1', 54356), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 10, 'cookie': 'b0d50e48b0a3431deff0be2c27528254e1fe8041'})
|
|
|
|
|
2022-06-29 20:28:02.524 | DEBUG | dcs.tests.spider_task:distribute_task:149 - [<dcs.tests.spider_task.Spider_partial_task object at 0x000001A4A28328C0>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001A4A2832980>, <dcs.tests.spider_task.Spider_partial_task object at 0x000001A4A2832620>]
|
|
|
|
|
2022-06-29 20:28:02.571 | DEBUG | dcs.tests.spider_task:run:185 - []
|
|
|
|
|
2022-06-29 20:28:02.571 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:28:02.587 | INFO | dcs.communicate:run:40 - sending response to ('127.0.0.1', 54356): {'crawl zhiwang': 'starting, please wait...'}
|
|
|
|
|
2022-06-29 20:28:02.602 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:28:02.680 | DEBUG | dcs.tests.spider_task:run:205 - generating local task
|
|
|
|
|
2022-06-29 20:28:03.196 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:28:03.196 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:03.430 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:28:03.430 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:03.618 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:03.962 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:04.087 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:04.149 | DEBUG | dcs.tests.spider_task:test_simulation:84 - simulation crawling...
|
|
|
|
|
2022-06-29 20:28:04.149 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:04.259 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:04.415 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:04.524 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54356), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 7, 'cookie': 'b0d50e48b0a3431deff0be2c27528254e1fe8041'})
|
|
|
|
|
2022-06-29 20:28:04.743 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54356), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 7, 'pages_end': 10, 'cookie': 'b0d50e48b0a3431deff0be2c27528254e1fe8041'})
|
|
|
|
|
2022-06-29 20:28:05.165 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
|
|
|
|
|
2022-06-29 20:28:05.774 | INFO | dcs.tests.spider_task:run:100 - partial crawl task finished: (('127.0.0.1', 54356), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': 'b0d50e48b0a3431deff0be2c27528254e1fe8041'})
|
|
|
|
|
2022-06-29 20:28:05.805 | DEBUG | dcs.tests.spider_task:compose_result:159 - composing task...
|
|
|
|
|
2022-06-29 20:28:06.493 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:28:06.790 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
2022-06-29 20:28:07.477 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
|
|
|
|
|
|
