From 1757411834c5619f89a51b9cf78d36bb3d858168 Mon Sep 17 00:00:00 2001 From: wufayuan <2858767122@qq.com> Date: Thu, 30 Jun 2022 20:29:17 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BE=83=E4=B8=BA=E5=AE=8C=E6=95=B4=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dcs/communicate.py | 4 - dcs/dcs.log | 185 ++++++++++++++++++++++++++++++ dcs/main.py | 2 +- dcs/tests/user_request_handler.py | 6 +- dcs/tools/database.py | 12 +- 5 files changed, 198 insertions(+), 11 deletions(-) diff --git a/dcs/communicate.py b/dcs/communicate.py index bdd5194..07939d1 100644 --- a/dcs/communicate.py +++ b/dcs/communicate.py @@ -37,7 +37,3 @@ class Communicator(threading.Thread): except Exception as e: logger.error(str(e)) self.info_list.remove(info) - socket_to_client.connect(info[0]) - logger.info(f'sending response to {info[0]}: {info[1]}') - socket_to_client.sendall(generate_request(info[1])) - self.info_list.remove(info) diff --git a/dcs/dcs.log b/dcs/dcs.log index 9458f52..1df9c75 100644 --- a/dcs/dcs.log +++ b/dcs/dcs.log @@ -7401,3 +7401,188 @@ 2022-06-29 20:28:06.493 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} 2022-06-29 20:28:06.790 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} 2022-06-29 20:28:07.477 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-06-30 20:19:17.387 | DEBUG | __main__::14 - reading config args... +2022-06-30 20:19:17.387 | DEBUG | __main__::21 - starting the servers... +2022-06-30 20:20:10.039 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:21:36.072 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:21:56.306 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:22:27.265 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:23:01.160 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000) +2022-06-30 20:23:01.238 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:23:01.254 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-06-30 20:23:01.960 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-06-30 20:23:01.991 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'} +2022-06-30 20:23:02.132 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:23:02.149 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:23:03.042 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 26ac224dd3592ae5268a4d611fad8c4d3ed80855 +2022-06-30 20:23:03.059 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'login': '26ac224dd3592ae5268a4d611fad8c4d3ed80855'} +2022-06-30 20:23:03.090 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:23:03.105 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'login': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:24:59.253 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:25:02.812 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000) +2022-06-30 20:25:02.859 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:25:02.891 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-06-30 20:25:02.891 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:03.236 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:03.628 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-06-30 20:25:03.785 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'} +2022-06-30 20:25:03.816 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:25:03.864 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:25:03.864 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:04.146 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:04.413 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:04.774 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 4160fbfae8b6795314683eb9feec1947cd1d3c00 +2022-06-30 20:25:04.806 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '4160fbfae8b6795314683eb9feec1947cd1d3c00'} +2022-06-30 20:25:04.947 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:25:04.978 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-06-30 20:25:04.978 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 4160fbfae8b6795314683eb9feec1947cd1d3c00 +2022-06-30 20:25:04.978 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 4160fbfae8b6795314683eb9feec1947cd1d3c00'} +2022-06-30 20:25:05.072 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-06-30 20:25:05.072 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)! +2022-06-30 20:25:32.582 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:25:32.802 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:25:32.818 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:25:32.818 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:33.398 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:33.870 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:34.231 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 4c42ae113c7bd392a10dea1951837f1dea3c4d24 +2022-06-30 20:25:34.262 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'} +2022-06-30 20:25:34.278 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:25:34.419 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:25:49.452 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:25:49.530 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:49.813 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 20:25:49.845 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, ] +2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task +2022-06-30 20:25:49.845 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000) +2022-06-30 20:25:49.876 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:25:49.923 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:49.939 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-06-30 20:25:49.939 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}} +2022-06-30 20:25:49.970 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-06-30 20:25:50.002 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:25:50.111 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:25:50.488 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:50.912 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:25:50.912 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:25:50.912 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:51.258 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:25:51.258 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:51.697 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:25:51.697 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:52.247 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:25:52.247 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 20:25:52.247 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:52.812 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:53.408 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:25:53.581 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:25:53.643 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} +2022-06-30 20:27:29.898 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:27:29.929 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:30.385 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 20:27:30.432 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, ] +2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:27:30.683 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:30.683 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:27:30.715 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:31.044 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:31.060 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:31.342 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:27:31.357 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:31.357 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:31.625 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:27:31.625 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:31.625 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:32.189 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:32.221 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:32.221 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:32.252 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:33.132 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:33.132 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:33.320 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:33.320 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:33.430 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:27:33.869 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:27:33.869 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:27:33.932 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:33.932 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 20:27:33.932 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:34.466 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:34.482 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:34.859 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:27:34.859 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:34.859 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:35.093 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:35.235 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:35.235 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:35.408 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-06-30 20:27:35.408 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:35.706 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:27:35.706 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:27:35.722 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:27:35.879 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}) +2022-06-30 20:27:58.681 | DEBUG | __main__::15 - reading config args... +2022-06-30 20:28:02.274 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000) +2022-06-30 20:28:02.462 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:28:02.478 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-06-30 20:28:02.478 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:02.682 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:03.262 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-06-30 20:28:03.278 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'} +2022-06-30 20:28:03.310 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:28:03.310 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:28:03.310 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:03.545 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:03.875 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:04.079 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 2e96ef02c0922f39228fc4754129e0ddf1d356b6 +2022-06-30 20:28:04.110 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '2e96ef02c0922f39228fc4754129e0ddf1d356b6'} +2022-06-30 20:28:04.141 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:28:04.157 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-06-30 20:28:04.189 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 2e96ef02c0922f39228fc4754129e0ddf1d356b6 +2022-06-30 20:28:04.314 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 2e96ef02c0922f39228fc4754129e0ddf1d356b6'} +2022-06-30 20:28:04.346 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end +2022-06-30 20:28:04.346 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)! +2022-06-30 20:28:12.741 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:28:13.024 | INFO | dcs.user_process:run:18 - processing user request... +2022-06-30 20:28:13.040 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-06-30 20:28:13.040 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:13.401 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:13.699 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:14.154 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 8fa7771b805ce10cceee5b9d02684bf4c84fd8f7 +2022-06-30 20:28:14.170 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'} +2022-06-30 20:28:14.185 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:28:14.248 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:28:24.198 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002) +2022-06-30 20:28:24.277 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:24.810 | INFO | dcs.spider:run:22 - processing spider request... +2022-06-30 20:28:24.888 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'}) +2022-06-30 20:28:24.888 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [, ] +2022-06-30 20:28:24.904 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'} +2022-06-30 20:28:25.061 | DEBUG | dcs.tests.spider_task:run:190 - [] +2022-06-30 20:28:25.108 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-06-30 20:28:25.108 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task +2022-06-30 20:28:25.202 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000) +2022-06-30 20:28:25.281 | DEBUG | dcs.tests.spider_task:run:210 - generating local task +2022-06-30 20:28:25.281 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'} +2022-06-30 20:28:25.297 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:25.532 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-06-30 20:28:25.532 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}} +2022-06-30 20:28:26.489 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:27.352 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling... +2022-06-30 20:28:27.352 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:28:27.352 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:27.980 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:28:27.980 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:28.497 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-06-30 20:28:28.576 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:28.984 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'}) +2022-06-30 20:28:29.031 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task... +2022-06-30 20:28:29.031 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:30.051 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:30.694 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database... +2022-06-30 20:28:31.008 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}} +2022-06-30 20:28:31.196 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'} diff --git a/dcs/main.py b/dcs/main.py index 3cf8703..3fb1998 100644 --- a/dcs/main.py +++ b/dcs/main.py @@ -9,6 +9,7 @@ from conf.config import global_var from dcs.user_process import UP from dcs.communicate import Communicator +logger.debug('starting the servers...') create_user_info() logger.add('./dcs.log', rotation='10 MB', enqueue=True, backtrace=True, diagnose=True) logger.debug('reading config args...') @@ -18,7 +19,6 @@ con.read(configFile, encoding='utf-8') items = con.items('server') items = dict(items) -logger.debug('starting the servers...') global_var.server = Server(int(items['port']), eval(items['buffer_size'])) global_var.server.daemon = items['daemon'] global_var.server.start() diff --git a/dcs/tests/user_request_handler.py b/dcs/tests/user_request_handler.py index c2287bd..3d1a6bb 100644 --- a/dcs/tests/user_request_handler.py +++ b/dcs/tests/user_request_handler.py @@ -26,10 +26,10 @@ class Urh(threading.Thread): database.mysql_conn() response = database.login(user, password, st) response = { - 'login': response + 'cookie': response } - global_var.communicator.add_response('cookie', self.client_socket, response) - logger.info(f"[RESPONSE] login: {response['login']}") + global_var.communicator.add_response('login', self.client_socket, response) + logger.info(f"[RESPONSE] login: {response['cookie']}") def register(self, user, password): logger.info(f"[REQUEST] register") diff --git a/dcs/tools/database.py b/dcs/tools/database.py index 4d239c8..f8a445a 100644 --- a/dcs/tools/database.py +++ b/dcs/tools/database.py @@ -1,13 +1,19 @@ from hashlib import * import pymysql +from loguru import logger + import conf.config as config import dcs.tools.cookie as cookie # 获取数据库连接对象 -def mysql_conn(host='192.168.43.65', user='root', passwd='427318Aa', db='test'): - conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db) - return conn +def mysql_conn(host='127.0.0.1', user='root', passwd='xwdjzwy5252', db='test'): + try: + logger.debug('connecting to database...') + conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db) + return conn + except Exception as e: + logger.error(str(e)) def register(u_name, u_pwd):