pl5xfp7y8
/
dcs
forked from p3t2ja9zs/dcs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

较为完整的代码

Browse Source
master
wufayuan 2 years ago
parent da9136e30d
commit 1757411834
5 changed files with 198 additions and 11 deletions
Show Stats Download Patch File Download Diff File

4
dcs/communicate.py
Unescape View File

@ -37,7 +37,3 @@ class Communicator(threading.Thread):
except Exception as e:
logger.error(str(e))
self.info_list.remove(info)
socket_to_client.connect(info[0])
logger.info(f'sending response to {info[0]}: {info[1]}')
socket_to_client.sendall(generate_request(info[1]))
self.info_list.remove(info)

185
dcs/dcs.log
Unescape View File

@ -7401,3 +7401,188 @@
2022-06-29 20:28:06.493 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
2022-06-29 20:28:06.790 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
2022-06-29 20:28:07.477 | DEBUG | dcs.tests.spider_task:compose_result:174 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
2022-06-30 20:19:17.387 | DEBUG | __main__:<module>:14 - reading config args...
2022-06-30 20:19:17.387 | DEBUG | __main__:<module>:21 - starting the servers...
2022-06-30 20:20:10.039 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:21:36.072 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:21:56.306 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:22:27.265 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:23:01.160 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
2022-06-30 20:23:01.238 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:23:01.254 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
2022-06-30 20:23:01.960 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
2022-06-30 20:23:01.991 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
2022-06-30 20:23:02.132 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:23:02.149 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
2022-06-30 20:23:03.042 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 26ac224dd3592ae5268a4d611fad8c4d3ed80855
2022-06-30 20:23:03.059 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'login': '26ac224dd3592ae5268a4d611fad8c4d3ed80855'}
2022-06-30 20:23:03.090 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes
2022-06-30 20:23:03.105 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'login': 'unpack requires a buffer of 8 bytes'}
2022-06-30 20:24:59.253 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:25:02.812 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
2022-06-30 20:25:02.859 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:25:02.891 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
2022-06-30 20:25:02.891 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:03.236 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:03.628 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
2022-06-30 20:25:03.785 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
2022-06-30 20:25:03.816 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:25:03.864 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
2022-06-30 20:25:03.864 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:04.146 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:04.413 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:04.774 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 4160fbfae8b6795314683eb9feec1947cd1d3c00
2022-06-30 20:25:04.806 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '4160fbfae8b6795314683eb9feec1947cd1d3c00'}
2022-06-30 20:25:04.947 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:25:04.978 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
2022-06-30 20:25:04.978 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 4160fbfae8b6795314683eb9feec1947cd1d3c00
2022-06-30 20:25:04.978 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 4160fbfae8b6795314683eb9feec1947cd1d3c00'}
2022-06-30 20:25:05.072 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end
2022-06-30 20:25:05.072 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)!
2022-06-30 20:25:32.582 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002)
2022-06-30 20:25:32.802 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:25:32.818 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
2022-06-30 20:25:32.818 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:33.398 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:33.870 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:34.231 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 4c42ae113c7bd392a10dea1951837f1dea3c4d24
2022-06-30 20:25:34.262 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'}
2022-06-30 20:25:34.278 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes
2022-06-30 20:25:34.419 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'}
2022-06-30 20:25:49.452 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002)
2022-06-30 20:25:49.530 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:49.813 | INFO | dcs.spider:run:22 - processing spider request...
2022-06-30 20:25:49.845 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [<dcs.tests.spider_task.Spider_partial_task object at 0x0000025B7C65C730>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000025B7C65C580>]
2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:run:190 - [<socket.socket fd=944, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
2022-06-30 20:25:49.845 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task
2022-06-30 20:25:49.845 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000)
2022-06-30 20:25:49.876 | DEBUG | dcs.tests.spider_task:run:210 - generating local task
2022-06-30 20:25:49.923 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:49.939 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving...
2022-06-30 20:25:49.939 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}}
2022-06-30 20:25:49.970 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'}
2022-06-30 20:25:50.002 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes
2022-06-30 20:25:50.111 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'}
2022-06-30 20:25:50.488 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:50.912 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling...
2022-06-30 20:25:50.912 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:25:50.912 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:51.258 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:25:51.258 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:51.697 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:25:51.697 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:52.247 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:25:52.247 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task...
2022-06-30 20:25:52.247 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:52.812 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:53.408 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:25:53.581 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
2022-06-30 20:25:53.643 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'}
2022-06-30 20:27:29.898 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002)
2022-06-30 20:27:29.929 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:30.385 | INFO | dcs.spider:run:22 - processing spider request...
2022-06-30 20:27:30.432 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [<dcs.tests.spider_task.Spider_partial_task object at 0x0000025B7C65CF70>, <dcs.tests.spider_task.Spider_partial_task object at 0x0000025B7C65D1B0>]
2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:run:190 - []
2022-06-30 20:27:30.432 | DEBUG | dcs.tests.spider_task:run:210 - generating local task
2022-06-30 20:27:30.683 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:30.683 | DEBUG | dcs.tests.spider_task:run:210 - generating local task
2022-06-30 20:27:30.715 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:31.044 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:31.060 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:31.342 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling...
2022-06-30 20:27:31.357 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:31.357 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:31.625 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling...
2022-06-30 20:27:31.625 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:31.625 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:32.189 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:32.221 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:32.221 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:32.252 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:33.132 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:33.132 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:33.320 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:33.320 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:33.430 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:27:33.869 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:27:33.869 | DEBUG | dcs.tests.spider_task:run:210 - generating local task
2022-06-30 20:27:33.932 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:33.932 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task...
2022-06-30 20:27:33.932 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:34.466 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:34.482 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:34.859 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling...
2022-06-30 20:27:34.859 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:34.859 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:35.093 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:35.235 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:35.235 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:35.408 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}}
2022-06-30 20:27:35.408 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:35.706 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:27:35.706 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:27:35.722 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
2022-06-30 20:27:35.879 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 4, 'cookie': '4c42ae113c7bd392a10dea1951837f1dea3c4d24'})
2022-06-30 20:27:58.681 | DEBUG | __main__:<module>:15 - reading config args...
2022-06-30 20:28:02.274 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9000)
2022-06-30 20:28:02.462 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:28:02.478 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register
2022-06-30 20:28:02.478 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:02.682 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:03.262 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败
2022-06-30 20:28:03.278 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'register': '用户名已存在，注册失败'}
2022-06-30 20:28:03.310 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:28:03.310 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
2022-06-30 20:28:03.310 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:03.545 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:03.875 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:04.079 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 2e96ef02c0922f39228fc4754129e0ddf1d356b6
2022-06-30 20:28:04.110 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'cookie': '2e96ef02c0922f39228fc4754129e0ddf1d356b6'}
2022-06-30 20:28:04.141 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:28:04.157 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free
2022-06-30 20:28:04.189 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 2e96ef02c0922f39228fc4754129e0ddf1d356b6
2022-06-30 20:28:04.314 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9000): {'report_free': 'success marked 2e96ef02c0922f39228fc4754129e0ddf1d356b6'}
2022-06-30 20:28:04.346 | INFO | dcs.tests.requestHandler:run:21 - [REQUEST] end
2022-06-30 20:28:04.346 | DEBUG | dcs.tests.requestHandler:run:22 - communication over from ('127.0.0.1', 9000)!
2022-06-30 20:28:12.741 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002)
2022-06-30 20:28:13.024 | INFO | dcs.user_process:run:18 - processing user request...
2022-06-30 20:28:13.040 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login
2022-06-30 20:28:13.040 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:13.401 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:13.699 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:14.154 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 8fa7771b805ce10cceee5b9d02684bf4c84fd8f7
2022-06-30 20:28:14.170 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'}
2022-06-30 20:28:14.185 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes
2022-06-30 20:28:14.248 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'login': 'unpack requires a buffer of 8 bytes'}
2022-06-30 20:28:24.198 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9002)
2022-06-30 20:28:24.277 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:24.810 | INFO | dcs.spider:run:22 - processing spider request...
2022-06-30 20:28:24.888 | INFO | dcs.tests.spider_task:distribute_task:138 - distributing task: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 5, 'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'})
2022-06-30 20:28:24.888 | DEBUG | dcs.tests.spider_task:distribute_task:151 - [<dcs.tests.spider_task.Spider_partial_task object at 0x00000194BB603730>, <dcs.tests.spider_task.Spider_partial_task object at 0x00000194BB6030A0>]
2022-06-30 20:28:24.904 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawling state': 'starting, please wait...'}
2022-06-30 20:28:25.061 | DEBUG | dcs.tests.spider_task:run:190 - [<socket.socket fd=1012, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('127.0.0.1', 7777), raddr=('127.0.0.1', 9000)>]
2022-06-30 20:28:25.108 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes
2022-06-30 20:28:25.108 | DEBUG | dcs.tests.spider_task:run:202 - generating remote task
2022-06-30 20:28:25.202 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9000)
2022-06-30 20:28:25.281 | DEBUG | dcs.tests.spider_task:run:210 - generating local task
2022-06-30 20:28:25.281 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9002): {'crawl zhiwang': 'unpack requires a buffer of 8 bytes'}
2022-06-30 20:28:25.297 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:25.532 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving...
2022-06-30 20:28:25.532 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}}
2022-06-30 20:28:26.489 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:27.352 | DEBUG | dcs.tests.spider_task:test_simulation:85 - simulation crawling...
2022-06-30 20:28:27.352 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:28:27.352 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:27.980 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:28:27.980 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:28.497 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test
2022-06-30 20:28:28.576 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:28.984 | INFO | dcs.tests.spider_task:run:101 - partial crawl task finished: (('127.0.0.1', 9002), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 4, 'pages_end': 5, 'cookie': '8fa7771b805ce10cceee5b9d02684bf4c84fd8f7'})
2022-06-30 20:28:29.031 | DEBUG | dcs.tests.spider_task:compose_result:161 - composing task...
2022-06-30 20:28:29.031 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:30.051 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:30.694 | DEBUG | dcs.tools.database:mysql_conn:12 - connecting to database...
2022-06-30 20:28:31.008 | DEBUG | dcs.tests.spider_task:compose_result:179 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}}
2022-06-30 20:28:31.196 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9002): {'0': {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'paper': 'remote'}, 'type': 'response'}

2
dcs/main.py
Unescape View File

@ -9,6 +9,7 @@ from conf.config import global_var
from dcs.user_process import UP
from dcs.communicate import Communicator
logger.debug('starting the servers...')
create_user_info()
logger.add('./dcs.log', rotation='10 MB', enqueue=True, backtrace=True, diagnose=True)
logger.debug('reading config args...')
@ -18,7 +19,6 @@ con.read(configFile, encoding='utf-8')
items = con.items('server')
items = dict(items)
logger.debug('starting the servers...')
global_var.server = Server(int(items['port']), eval(items['buffer_size']))
global_var.server.daemon = items['daemon']
global_var.server.start()

6
dcs/tests/user_request_handler.py
Unescape View File

@ -26,10 +26,10 @@ class Urh(threading.Thread):
database.mysql_conn()
response = database.login(user, password, st)
response = {
'login': response
'cookie': response
}
global_var.communicator.add_response('cookie', self.client_socket, response)
logger.info(f"[RESPONSE] login: {response['login']}")
global_var.communicator.add_response('login', self.client_socket, response)
logger.info(f"[RESPONSE] login: {response['cookie']}")
def register(self, user, password):
logger.info(f"[REQUEST] register")

12
dcs/tools/database.py
Unescape View File

@ -1,13 +1,19 @@
from hashlib import *
import pymysql
from loguru import logger
import conf.config as config
import dcs.tools.cookie as cookie
# 获取数据库连接对象
def mysql_conn(host='192.168.43.65', user='root', passwd='427318Aa', db='test'):
conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db)
return conn
def mysql_conn(host='127.0.0.1', user='root', passwd='xwdjzwy5252', db='test'):
try:
logger.debug('connecting to database...')
conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db)
return conn
except Exception as e:
logger.error(str(e))
def register(u_name, u_pwd):

Write Preview
Loading…
Cancel
Save