From 2f4fa14b2b49ae257d19a1d73af055b29a12acb4 Mon Sep 17 00:00:00 2001 From: wufayuan <2858767122@qq.com> Date: Mon, 4 Jul 2022 21:21:00 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E5=86=99=E4=BA=86connect=E9=80=9A?= =?UTF-8?q?=E4=BF=A1=E7=A8=8B=E5=BA=8F=E4=B8=8E=E6=9C=8D=E5=8A=A1=E5=99=A8?= =?UTF-8?q?=E9=80=9A=E4=BF=A1=E7=B3=BB=E7=BB=9F=EF=BC=8C=E7=9B=AE=E5=89=8D?= =?UTF-8?q?=E5=8F=AF=E7=94=A8=E6=80=A7=E5=BA=94=E5=BD=93=E5=A4=A7=E5=B9=85?= =?UTF-8?q?=E6=8F=90=E5=8D=87=EF=BC=8C=E8=BF=90=E8=A1=8C=E6=83=85=E5=86=B5?= =?UTF-8?q?=E6=AD=A3=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dcs/clients/client.py | 136 +++++++++++ dcs/communicate.py | 21 +- dcs/dcs.log | 445 ++++++++++++++++++++++++++++++++++++ dcs/tests/requestHandler.py | 4 + dcs/tests/spider_task.py | 3 +- dcs/tools/database.py | 5 +- ui/connect.py | 55 +++-- ui/login.bat | 2 +- ui/login.js | 11 +- 9 files changed, 640 insertions(+), 42 deletions(-) create mode 100644 dcs/clients/client.py diff --git a/dcs/clients/client.py b/dcs/clients/client.py new file mode 100644 index 0000000..040d7cb --- /dev/null +++ b/dcs/clients/client.py @@ -0,0 +1,136 @@ +import json +import socket +import struct +import threading +from configparser import ConfigParser +from json import JSONDecoder + +from msedge.selenium_tools import Edge +from msedge.selenium_tools import EdgeOptions + +from dcs.tests.zhiwang import * +from dcs.tools import message_process as mp +from dcs.tools.message_process import parse_request, generate_response + + +def crawl_zhiwang(word, pages_start, pages_end): + edge_options = EdgeOptions() + edge_options.use_chromium = True + No_Image_loading = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2} + edge_options.add_experimental_option("prefs", No_Image_loading) + edge_options.add_argument('--headless') + configFile = '../../conf/settings.ini' + con = ConfigParser() + con.read(configFile, encoding='utf-8') + items = con.items('crawler') + items = dict(items)['edge_driver_path'] + print(items) + driver = Edge(options=edge_options, executable_path=items) + + soup = driver_open(driver, word) + papers = [] # 用于保存爬取到的论文 + paper_id = 0 + res = {} # 保存终端爬取结果 + + # 爬取第一篇 + if pages_start == 1: + spider(driver, soup, papers) + logger.debug(res) + pages_start += 1 + while paper_id < len(papers): + write2res(papers[paper_id], res) + paper_id += 1 + + while pages_start < pages_end: + content = change_page(driver, pages_start) + spider(driver, content, papers) + while paper_id < len(papers): + write2res(papers[paper_id], res) + paper_id += 1 + pages_start += 1 + driver.close() + # logger.debug("here") + return res + + +def write2res(paper: Paper, res): + for author in paper.authors: + if author.name: + res.update( + {len(res): {'name': author.name, 'college': author.college, 'major': author.major, + 'title': paper.title}}) + + +class Crawl(threading.Thread): + def __init__(self): + super(Crawl, self).__init__() + self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.server_socket.bind(('', local_port)) + + @staticmethod + def crawl(request_map) -> dict: + result_map = crawl_zhiwang(request_map['word'], request_map['pages_start'], request_map['pages_end']) + # result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'}, + # 1: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}} + logger.debug(result_map) + return result_map + + def run(self) -> None: + self.server_socket.listen() + while True: + client_socket, _ = self.server_socket.accept() + request_map = parse_request(client_socket) + if request_map['type'] == 'request': + print("receiving help request:\n" + json.dumps(request_map, ensure_ascii=False)) + response_map = self.crawl(request_map) + response_map.update({'cookie': request_map['cookie']}) + client_socket.sendall(generate_response(response_map)) + + report_map = {'action': 'report_free', 'cookie': cookie} + logger.debug(send_request(socket_to_server, report_map)) + if request_map['type'] == 'response': + print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False)) + # break + + +def send_request(socket2server, req): + socket2server.sendall(mp.generate_request(req)) + responseJson = JSONDecoder().decode( + mp.read_bytes(socket2server, struct.unpack('!Q', socket2server.recv(8))[0]).decode( + "utf-8")) + return responseJson + + +server_ip = '127.0.0.1' +server_port = 7777 +local_port = 9999 +crawl = Crawl() +crawl.start() +# res = crawl.crawl({'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 2, 'cookie': '123'}) +# logger.debug(res) + +socket_to_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) +socket_to_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +socket_to_server.bind(('', local_port)) +socket_to_server.connect((server_ip, server_port)) + +request = {'action': 'register', 'user': 'liuxiaoyu', 'password': '113818'} +logger.debug(send_request(socket_to_server, request)) + +request = {'action': 'login', 'user': 'liuxiaoyu', 'password': '113818'} +response = send_request(socket_to_server, request) +logger.debug(response) +cookie = response['cookie'] + +request = {'action': 'report_free', 'cookie': cookie} +logger.debug(send_request(socket_to_server, request)) + +# request = {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3, +# 'cookie': cookie} +# logger.debug(send_request(socket_to_server, request)) + +request = {'action': 'end'} +socket_to_server.sendall(mp.generate_request(request)) + +crawl.join() diff --git a/dcs/communicate.py b/dcs/communicate.py index 07939d1..4e591e9 100644 --- a/dcs/communicate.py +++ b/dcs/communicate.py @@ -13,6 +13,7 @@ class Communicator(threading.Thread): self.info_list: list[tuple[tuple, dict]] = [] def add_response(self, response_type: str, client_socket: socket.socket, response_map: dict): + response_map.update({'type': response_type}) self.responser_list.append((response_type, client_socket, response_map)) def add_info(self, info_type: str, address: tuple, info_map: dict): @@ -27,13 +28,13 @@ class Communicator(threading.Thread): client_socket.sendall(generate_response(response_map)) self.responser_list.remove(responser) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_client: - for info in self.info_list: - try: - logger.info(f'sending info to {info[0]}: {info[1]}') - socket_to_client.connect(info[0]) - socket_to_client.sendall(generate_request(info[1])) - self.info_list.remove(info) - except Exception as e: - logger.error(str(e)) - self.info_list.remove(info) + # with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_client: + # for info in self.info_list: + # try: + # logger.info(f'sending info to {info[0]}: {info[1]}') + # socket_to_client.connect(info[0]) + # socket_to_client.sendall(generate_request(info[1])) + # self.info_list.remove(info) + # except Exception as e: + # logger.error(str(e)) + # self.info_list.remove(info) diff --git a/dcs/dcs.log b/dcs/dcs.log index 32f8a0b..d856b96 100644 --- a/dcs/dcs.log +++ b/dcs/dcs.log @@ -9550,3 +9550,448 @@ from unknown error: web view not found 2022-07-04 17:24:15.751 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 12: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 13: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 14: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} 2022-07-04 17:24:16.227 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 12: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 13: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 14: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} 2022-07-04 17:24:16.274 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9014): {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 7: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 12: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 13: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 14: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 'crawl_id': 12, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 17:27:37.873 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-04 17:27:37.904 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 17:27:37.920 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 17:27:38.877 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-07-04 17:27:38.908 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'register': '用户名已存在，注册失败'} +2022-07-04 17:27:38.956 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 17:27:38.971 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 17:27:40.556 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 949d5c739e70f2be85ba75b725a21c51a67fa542 +2022-07-04 17:27:40.572 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '949d5c739e70f2be85ba75b725a21c51a67fa542'} +2022-07-04 17:27:40.618 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 17:27:40.634 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-04 17:27:40.634 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 949d5c739e70f2be85ba75b725a21c51a67fa542 +2022-07-04 17:27:40.791 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 949d5c739e70f2be85ba75b725a21c51a67fa542'} +2022-07-04 17:27:40.807 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 17:27:40.807 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9999)! +2022-07-04 17:27:45.403 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9014) +2022-07-04 17:27:46.559 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 17:27:46.639 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9014): {'crawling state': 'starting, please wait...'} +2022-07-04 17:27:46.655 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 9014), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': 'fb90de22c26723e4d7172fcf1db124f4db91fa30'}) +2022-07-04 17:27:46.687 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 17:27:46.718 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 17:27:46.814 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 17:27:46.814 | DEBUG | dcs.tests.spider_task:run:211 - generating remote task +2022-07-04 17:27:46.815 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-04 17:27:46.926 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 17:27:48.541 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 17:27:48.557 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 17:27:49.582 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 17:27:51.131 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 17:27:52.454 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 9014), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'fb90de22c26723e4d7172fcf1db124f4db91fa30'}) +2022-07-04 17:28:02.880 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-04 17:28:02.896 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 17:28:02.896 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 17:28:16.274 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 12: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 13: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 14: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 15: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 16: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 17: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 18: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 19: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 17:28:16.352 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9014): {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 12: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 13: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 14: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 15: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 16: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 17: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 18: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 19: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 'crawl_id': 13, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 19:57:58.966 | DEBUG | __main__::15 - reading config args... +2022-07-04 19:59:04.397 | DEBUG | __main__::15 - reading config args... +2022-07-04 19:59:20.354 | DEBUG | __main__::15 - reading config args... +2022-07-04 20:01:25.127 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-04 20:01:25.190 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:01:25.190 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:01:25.536 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 注册成功 +2022-07-04 20:01:25.536 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'register': '注册成功'} +2022-07-04 20:01:25.614 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:01:25.614 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:01:26.462 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: d2f764d6168aad82403a7fb28b22431f2e3e37a7 +2022-07-04 20:01:26.478 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'd2f764d6168aad82403a7fb28b22431f2e3e37a7'} +2022-07-04 20:01:26.509 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:01:26.525 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-04 20:01:26.525 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked d2f764d6168aad82403a7fb28b22431f2e3e37a7 +2022-07-04 20:01:26.572 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked d2f764d6168aad82403a7fb28b22431f2e3e37a7'} +2022-07-04 20:01:26.949 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 20:01:26.949 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3, 'cookie': 'd2f764d6168aad82403a7fb28b22431f2e3e37a7'}) +2022-07-04 20:01:26.949 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 20:01:26.949 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 20:01:26.949 | DEBUG | dcs.tests.spider_task:run:211 - generating remote task +2022-07-04 20:01:26.949 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-04 20:01:27.012 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 20:01:27.028 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'crawling state': 'starting, please wait...'} +2022-07-04 20:01:27.201 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:01:27.201 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9999)! +2022-07-04 20:01:28.221 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 20:01:28.221 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:01:28.897 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:01:29.368 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:01:30.050 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 9999), {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 2, 'pages_end': 3, 'cookie': 'd2f764d6168aad82403a7fb28b22431f2e3e37a7'}) +2022-07-04 20:02:04.244 | DEBUG | __main__::15 - reading config args... +2022-07-04 20:02:07.368 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-04 20:02:07.462 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:02:07.493 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:02:08.216 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-07-04 20:02:08.263 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'register': '用户名已存在，注册失败'} +2022-07-04 20:02:08.373 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:02:08.388 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:02:09.205 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: cf651c98a80bef654417ecd679a9e0d1fd12f203 +2022-07-04 20:02:09.237 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': 'cf651c98a80bef654417ecd679a9e0d1fd12f203'} +2022-07-04 20:02:09.284 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:02:09.315 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-04 20:02:09.315 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked cf651c98a80bef654417ecd679a9e0d1fd12f203 +2022-07-04 20:02:09.362 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked cf651c98a80bef654417ecd679a9e0d1fd12f203'} +2022-07-04 20:02:09.378 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:02:09.378 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9999)! +2022-07-04 20:03:57.043 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9014) +2022-07-04 20:03:57.201 | WARNING | dcs.tests.requestHandler:run:26 - user info error! +2022-07-04 20:05:06.806 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9014) +2022-07-04 20:05:07.438 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 20:05:07.555 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 9014), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': 'cf651c98a80bef654417ecd679a9e0d1fd12f203'}) +2022-07-04 20:05:07.556 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9014): {'crawling state': 'starting, please wait...'} +2022-07-04 20:05:07.587 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 20:05:07.621 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:05:07.621 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 20:05:07.621 | DEBUG | dcs.tests.spider_task:run:211 - generating remote task +2022-07-04 20:05:07.621 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-04 20:05:07.770 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 20:05:08.560 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 20:05:08.560 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:05:08.560 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:05:08.592 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:05:08.733 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 9014), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': 'cf651c98a80bef654417ecd679a9e0d1fd12f203'}) +2022-07-04 20:05:17.531 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-04 20:05:17.659 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 20:05:17.659 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 20:05:22.901 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 20:05:22.979 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 9014): {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 'crawl_id': 2, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 20:05:25.231 | ERROR | dcs.communicate:run:38 - [WinError 10061] 由于目标计算机积极拒绝，无法连接。 +2022-07-04 20:09:38.421 | DEBUG | __main__::15 - reading config args... +2022-07-04 20:09:41.277 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-04 20:09:41.340 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:09:41.340 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:09:41.843 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-07-04 20:09:41.843 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'register': '用户名已存在，注册失败'} +2022-07-04 20:09:41.905 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:09:41.921 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:09:43.035 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 7049a31300eec1b5d73206e777dfbac6147235ad +2022-07-04 20:09:43.192 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '7049a31300eec1b5d73206e777dfbac6147235ad'} +2022-07-04 20:09:43.286 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:09:43.380 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-04 20:09:43.380 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 7049a31300eec1b5d73206e777dfbac6147235ad +2022-07-04 20:09:43.443 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 7049a31300eec1b5d73206e777dfbac6147235ad'} +2022-07-04 20:09:43.459 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:09:43.459 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9999)! +2022-07-04 20:16:08.022 | DEBUG | __main__::15 - reading config args... +2022-07-04 20:16:16.860 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 9999) +2022-07-04 20:16:16.891 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:16:16.922 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:16:17.707 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 注册成功 +2022-07-04 20:16:17.723 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'register': '注册成功'} +2022-07-04 20:16:17.739 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:16:17.754 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:16:18.445 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 5251a4b7af210db6a3d4da01eadff73dbadae4b2 +2022-07-04 20:16:18.460 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'cookie': '5251a4b7af210db6a3d4da01eadff73dbadae4b2'} +2022-07-04 20:16:18.523 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:16:18.539 | INFO | dcs.tests.user_request_handler:report_state:16 - [REQUEST] report free +2022-07-04 20:16:18.539 | INFO | dcs.tests.user_request_handler:report_state:22 - [RESPONSE] report free: success marked 5251a4b7af210db6a3d4da01eadff73dbadae4b2 +2022-07-04 20:16:18.790 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 9999): {'report_free': 'success marked 5251a4b7af210db6a3d4da01eadff73dbadae4b2'} +2022-07-04 20:16:18.806 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:16:18.806 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 9999)! +2022-07-04 20:23:11.464 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10000) +2022-07-04 20:23:12.233 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 20:23:12.249 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 10000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '5251a4b7af210db6a3d4da01eadff73dbadae4b2'}) +2022-07-04 20:23:12.249 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 20:23:12.249 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 20:23:12.249 | DEBUG | dcs.tests.spider_task:run:211 - generating remote task +2022-07-04 20:23:12.249 | INFO | dcs.requester:get:40 - sending crawl request to ('127.0.0.1', 9999) +2022-07-04 20:23:12.265 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 20:23:12.280 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10000): {'crawling state': 'starting, please wait...'} +2022-07-04 20:23:12.296 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:23:14.172 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 20:23:14.401 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:23:15.628 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:23:16.271 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:23:17.040 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 10000), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '5251a4b7af210db6a3d4da01eadff73dbadae4b2'}) +2022-07-04 20:23:25.476 | DEBUG | dcs.requester:run:64 - receiving remote task result, saving... +2022-07-04 20:23:25.492 | DEBUG | dcs.requester:run:68 - result: {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 20:23:25.508 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 20:23:30.467 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}} +2022-07-04 20:23:30.499 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 10000): {'0': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '1': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '2': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '3': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '4': {'name': '刘思源', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '5': {'name': '冯蕾霖', 'college': None, 'major': None, 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '6': {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, '7': {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'title': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 5: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 6: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 7: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 8: {'name': '刘思源', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 9: {'name': '冯蕾霖', 'college': 'None', 'major': 'None', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 10: {'name': '朱章黔', 'college': '陆军勤务学院', 'major': '数学;互联网技术;高等教育;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 11: {'name': '贾韬', 'college': '西南大学', 'major': '科学研究管理;计算机软件及计算机应用;自动化技术;', 'paper': '基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析'}, 'crawl_id': 1, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 20:23:32.572 | ERROR | dcs.communicate:run:38 - [WinError 10061] 由于目标计算机积极拒绝，无法连接。 +2022-07-04 20:25:12.916 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10000) +2022-07-04 20:25:13.026 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:25:13.041 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:25:13.968 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 用户名错误，登录失败 +2022-07-04 20:25:14.015 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10000): {'cookie': '用户名错误，登录失败'} +2022-07-04 20:25:14.078 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:26:34.201 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10001) +2022-07-04 20:26:34.311 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:26:34.327 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:26:35.033 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 用户名错误，登录失败 +2022-07-04 20:26:35.205 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10001): {'cookie': '用户名错误，登录失败'} +2022-07-04 20:26:35.221 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:27:35.176 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10002) +2022-07-04 20:27:35.271 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:27:35.286 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:27:36.307 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 注册成功 +2022-07-04 20:27:36.354 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10002): {'register': '注册成功'} +2022-07-04 20:27:36.448 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:31:36.434 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 63204) +2022-07-04 20:31:36.450 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:31:36.465 | INFO | dcs.tests.user_request_handler:register:35 - [REQUEST] register +2022-07-04 20:31:45.504 | INFO | dcs.tests.user_request_handler:register:42 - [RESPONSE] register: 用户名已存在，注册失败 +2022-07-04 20:31:45.536 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 63204): {'register': '用户名已存在，注册失败'} +2022-07-04 20:31:45.583 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:31:45.583 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 63204)! +2022-07-04 20:33:53.805 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 63217) +2022-07-04 20:33:53.915 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:33:53.963 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:33:55.642 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: bdfd4bba3310c3e36dd76fe6f583fe2545c81c74 +2022-07-04 20:33:55.721 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 63217): {'cookie': 'bdfd4bba3310c3e36dd76fe6f583fe2545c81c74'} +2022-07-04 20:33:55.925 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:33:55.925 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 63217)! +2022-07-04 20:35:38.658 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10002) +2022-07-04 20:35:38.674 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:35:38.689 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:35:39.882 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 84a9db99fa29cc949673ed903e2dae96cc3fefe2 +2022-07-04 20:35:39.882 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10002): {'cookie': '84a9db99fa29cc949673ed903e2dae96cc3fefe2'} +2022-07-04 20:35:39.929 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:35:39.929 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 10002)! +2022-07-04 20:37:42.000 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 63232) +2022-07-04 20:37:42.079 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:39:26.856 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10002) +2022-07-04 20:39:26.934 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 20:39:26.950 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 20:39:27.861 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3 +2022-07-04 20:39:27.876 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10002): {'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'} +2022-07-04 20:39:27.924 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 20:39:27.924 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 10002)! +2022-07-04 20:40:15.146 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10003) +2022-07-04 20:40:15.555 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 20:40:15.570 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 10003), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 20:40:15.570 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 20:40:15.570 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 20:40:15.570 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 20:40:15.570 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 10003): {'crawling state': 'starting, please wait...'} +2022-07-04 20:40:15.601 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 20:40:15.696 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 20:40:16.982 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 20:40:16.982 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:17.265 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:17.327 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 20:40:17.327 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:17.736 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:17.892 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:17.971 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 10003), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 20:40:19.085 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 20:40:19.572 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 10003), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 20:40:19.572 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 20:40:20.435 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 20:40:20.765 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 20:40:20.780 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 10003): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 1, 'table_name': 'yuu_crawl_result', 'type': 'response'} +2022-07-04 20:40:22.899 | ERROR | dcs.communicate:run:38 - [WinError 10061] 由于目标计算机积极拒绝，无法连接。 +2022-07-04 20:41:05.553 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 10004) +2022-07-04 20:41:05.631 | ERROR | dcs.tests.requestHandler:run:37 - unpack requires a buffer of 8 bytes +2022-07-04 21:00:31.384 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62419) +2022-07-04 21:00:31.400 | ERROR | dcs.tests.requestHandler:run:32 - no action start! +2022-07-04 21:00:31.416 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62419): {'start': 'no action start!'} +2022-07-04 21:00:31.776 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:00:31.792 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62419), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 21:00:31.792 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:00:31.792 | DEBUG | dcs.tests.spider_task:run:199 - [] +2022-07-04 21:00:31.792 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 21:00:31.824 | DEBUG | dcs.tests.spider_task:run:219 - generating local task +2022-07-04 21:00:31.855 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62419): {'crawling state': 'starting, please wait...'} +2022-07-04 21:00:32.844 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:00:32.844 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:32.907 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:00:32.907 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:33.315 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:33.597 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:33.785 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:34.131 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:00:34.241 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62419), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 21:00:34.775 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62419), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '04e49cbb6fff9bb3109132e4fa8a7c5fb7190fc3'}) +2022-07-04 21:00:34.790 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:00:35.309 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:00:35.576 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:00:35.576 | INFO | dcs.communicate:run:33 - sending info to ('127.0.0.1', 62419): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 2, 'table_name': 'yuu_crawl_result', 'type': 'response'} +2022-07-04 21:00:37.616 | ERROR | dcs.communicate:run:38 - [WinError 10061] 由于目标计算机积极拒绝，无法连接。 +2022-07-04 21:01:23.874 | DEBUG | __main__::15 - reading config args... +2022-07-04 21:01:28.881 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62450) +2022-07-04 21:01:28.944 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:01:28.944 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62450)! +2022-07-04 21:05:56.554 | ERROR | dcs.tests.requestHandler:run:40 - [WinError 10054] 远程主机强迫关闭了一个现有的连接。 +2022-07-04 21:06:02.674 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62465) +2022-07-04 21:06:02.705 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:06:02.705 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62465)! +2022-07-04 21:08:00.716 | ERROR | dcs.tests.requestHandler:run:40 - [WinError 10054] 远程主机强迫关闭了一个现有的连接。 +2022-07-04 21:08:06.632 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62474) +2022-07-04 21:08:06.695 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:08:06.695 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62474)! +2022-07-04 21:08:52.470 | DEBUG | __main__::15 - reading config args... +2022-07-04 21:08:55.483 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62481) +2022-07-04 21:08:55.593 | INFO | dcs.tests.requestHandler:run:23 - [REQUEST] start +2022-07-04 21:08:55.593 | DEBUG | dcs.tests.requestHandler:run:24 - communication begin from ('127.0.0.1', 62481)! +2022-07-04 21:08:55.593 | DEBUG | dcs.tests.requestHandler:run:37 - request over! +2022-07-04 21:10:21.726 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62485) +2022-07-04 21:10:21.773 | INFO | dcs.tests.requestHandler:run:23 - [REQUEST] start +2022-07-04 21:10:21.773 | DEBUG | dcs.tests.requestHandler:run:24 - communication begin from ('127.0.0.1', 62485)! +2022-07-04 21:10:21.773 | DEBUG | dcs.tests.requestHandler:run:37 - request over! +2022-07-04 21:12:44.445 | DEBUG | __main__::15 - reading config args... +2022-07-04 21:12:47.364 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62499) +2022-07-04 21:12:47.427 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:12:47.490 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62499)! +2022-07-04 21:12:47.490 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:12:47.506 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:12:47.506 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 21:12:47.553 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 21:12:48.981 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 用户名错误，登录失败 +2022-07-04 21:12:49.044 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62499): {'cookie': '用户名错误，登录失败'} +2022-07-04 21:12:49.091 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:12:49.091 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62499)! +2022-07-04 21:13:32.814 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62511) +2022-07-04 21:13:33.034 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:13:33.034 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62511)! +2022-07-04 21:13:33.034 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:13:33.034 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:13:33.034 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 21:13:33.081 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 21:13:35.734 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 2894186347f89f1508c5669eca68830339aa9736 +2022-07-04 21:13:35.750 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62511): {'cookie': '2894186347f89f1508c5669eca68830339aa9736'} +2022-07-04 21:13:35.875 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:13:35.875 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62511)! +2022-07-04 21:13:51.067 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62516) +2022-07-04 21:13:51.287 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:13:51.287 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62516)! +2022-07-04 21:13:51.287 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:13:51.570 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:13:51.602 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:13:51.617 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62516): {'crawling state': 'starting, please wait...'} +2022-07-04 21:13:51.712 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62516), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:13:51.712 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:13:51.712 | DEBUG | dcs.tests.spider_task:run:200 - [] +2022-07-04 21:13:51.712 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:13:51.821 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:13:53.250 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:13:53.250 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:54.035 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:54.364 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:13:54.364 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:55.007 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:55.274 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:55.933 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62516), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:13:56.028 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:13:57.079 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62516), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:13:57.142 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:13:57.754 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:13:58.460 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:13:58.492 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62516): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 3, 'table_name': 'liuxiaoyu_crawl_result'} +2022-07-04 21:13:58.523 | ERROR | dcs.tests.requestHandler:run:41 - unpack requires a buffer of 8 bytes +2022-07-04 21:15:12.496 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62533) +2022-07-04 21:15:12.559 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:15:12.559 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62533)! +2022-07-04 21:15:12.559 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:15:13.611 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:15:13.626 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:15:13.673 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62533), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:13.673 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:15:13.673 | DEBUG | dcs.tests.spider_task:run:200 - [] +2022-07-04 21:15:13.673 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:15:13.689 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62533): {'crawling state': 'starting, please wait...'} +2022-07-04 21:15:13.720 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:15:15.431 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:15:15.431 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:15.559 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:15:15.559 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:16.099 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:16.715 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:17.501 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:17.987 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62533), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:18.034 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:19.180 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62533), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:19.180 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:15:19.918 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:15:20.467 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:15:20.514 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62533): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 5, 'table_name': 'liuxiaoyu_crawl_result'} +2022-07-04 21:15:20.530 | ERROR | dcs.tests.requestHandler:run:41 - unpack requires a buffer of 8 bytes +2022-07-04 21:15:46.925 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62550) +2022-07-04 21:15:47.050 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:15:47.176 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62550)! +2022-07-04 21:15:47.176 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:15:48.087 | DEBUG | dcs.tests.requestHandler:run:38 - request over! +2022-07-04 21:15:48.102 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:15:48.118 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62550), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:48.118 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:15:48.118 | DEBUG | dcs.tests.spider_task:run:200 - [] +2022-07-04 21:15:48.118 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:15:48.149 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:15:48.149 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62550): {'crawling state': 'starting, please wait...'} +2022-07-04 21:15:49.577 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:15:49.577 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:49.703 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:15:49.703 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:50.205 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:50.613 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:50.833 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:51.696 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:15:51.852 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62550), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:53.077 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62550), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '2894186347f89f1508c5669eca68830339aa9736'}) +2022-07-04 21:15:53.093 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:15:54.128 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:15:54.929 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:15:54.929 | INFO | dcs.communicate:run:26 - sending response to ('127.0.0.1', 62550): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 6, 'table_name': 'liuxiaoyu_crawl_result'} +2022-07-04 21:15:55.007 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:15:55.007 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62550)! +2022-07-04 21:16:11.409 | DEBUG | __main__::15 - reading config args... +2022-07-04 21:16:44.919 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62585) +2022-07-04 21:16:44.981 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:16:44.981 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62585)! +2022-07-04 21:16:45.013 | INFO | dcs.user_process:run:18 - processing user request... +2022-07-04 21:16:45.044 | INFO | dcs.tests.user_request_handler:login:25 - [REQUEST] login +2022-07-04 21:16:47.382 | INFO | dcs.tests.user_request_handler:login:32 - [RESPONSE] login: 8f607bcea67d4f62475fcc710e2f5aff794a4833 +2022-07-04 21:16:47.398 | INFO | dcs.communicate:run:27 - sending response to ('127.0.0.1', 62585): {'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833', 'type': 'login'} +2022-07-04 21:16:47.429 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:16:47.429 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62585)! +2022-07-04 21:17:23.356 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62604) +2022-07-04 21:17:23.529 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:17:23.607 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62604)! +2022-07-04 21:17:24.114 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:17:24.162 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62604), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:17:24.162 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:17:24.162 | DEBUG | dcs.tests.spider_task:run:200 - [] +2022-07-04 21:17:24.163 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:17:24.163 | INFO | dcs.communicate:run:27 - sending response to ('127.0.0.1', 62604): {'crawling state': 'starting, please wait...', 'type': 'crawling state'} +2022-07-04 21:17:24.239 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:17:26.452 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:17:26.452 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:27.091 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:17:27.091 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:27.394 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:28.144 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:29.277 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:29.293 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:17:29.753 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62604), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:17:30.645 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62604), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:17:30.676 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:17:31.378 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:17:32.112 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:17:32.128 | INFO | dcs.communicate:run:27 - sending response to ('127.0.0.1', 62604): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 7, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 21:17:32.160 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:17:32.160 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62604)! +2022-07-04 21:18:39.808 | DEBUG | dcs.server:run:25 - connected to client ('127.0.0.1', 62630) +2022-07-04 21:18:39.901 | INFO | dcs.tests.requestHandler:run:24 - [REQUEST] start +2022-07-04 21:18:39.933 | DEBUG | dcs.tests.requestHandler:run:25 - communication begin from ('127.0.0.1', 62630)! +2022-07-04 21:18:40.997 | INFO | dcs.spider:run:22 - processing spider request... +2022-07-04 21:18:41.013 | INFO | dcs.tests.spider_task:distribute_task:145 - distributing task: (('127.0.0.1', 62630), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 3, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:18:41.013 | DEBUG | dcs.tests.spider_task:distribute_task:158 - [, ] +2022-07-04 21:18:41.013 | DEBUG | dcs.tests.spider_task:run:200 - [] +2022-07-04 21:18:41.014 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:18:41.045 | INFO | dcs.communicate:run:27 - sending response to ('127.0.0.1', 62630): {'crawling state': 'starting, please wait...', 'type': 'crawling state'} +2022-07-04 21:18:41.045 | DEBUG | dcs.tests.spider_task:run:220 - generating local task +2022-07-04 21:18:42.433 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:18:42.433 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:43.357 | DEBUG | dcs.tests.spider_task:test_simulation:90 - simulation crawling... +2022-07-04 21:18:43.357 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:43.645 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:43.997 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:44.317 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:45.322 | INFO | dcs.tests.spider_task:write2database:17 - writing to database: test +2022-07-04 21:18:45.481 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62630), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 1, 'pages_end': 2, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:18:46.310 | INFO | dcs.tests.spider_task:run:108 - partial crawl task finished: (('127.0.0.1', 62630), {'action': 'crawl zhiwang', 'word': 'computer', 'pages_start': 2, 'pages_end': 3, 'cookie': '8f607bcea67d4f62475fcc710e2f5aff794a4833'}) +2022-07-04 21:18:46.342 | DEBUG | dcs.tests.spider_task:compose_result:168 - composing task... +2022-07-04 21:18:47.091 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:18:48.185 | DEBUG | dcs.tests.spider_task:compose_result:186 - {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}} +2022-07-04 21:18:48.185 | INFO | dcs.communicate:run:27 - sending response to ('127.0.0.1', 62630): {1: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 2: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 3: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 4: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 5: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 6: {'name': 'test', 'college': 'test', 'major': 'test', 'paper': 'test'}, 'crawl_id': 8, 'table_name': 'liuxiaoyu_crawl_result', 'type': 'response'} +2022-07-04 21:18:48.217 | INFO | dcs.tests.requestHandler:run:20 - [REQUEST] end +2022-07-04 21:18:48.217 | DEBUG | dcs.tests.requestHandler:run:21 - communication over from ('127.0.0.1', 62630)! diff --git a/dcs/tests/requestHandler.py b/dcs/tests/requestHandler.py index fbd89d8..293b4e1 100644 --- a/dcs/tests/requestHandler.py +++ b/dcs/tests/requestHandler.py @@ -20,6 +20,9 @@ class RequestHandler(threading.Thread): logger.info(f"[REQUEST] end") logger.debug(f"communication over from {self.client_socket.getpeername()}!") break + elif request_map['action'] == 'start': + logger.info(f"[REQUEST] start") + logger.debug(f"communication begin from {self.client_socket.getpeername()}!") elif request_map['action'] == 'crawl zhiwang': chk_res = check(request_map) if chk_res is None: @@ -32,6 +35,7 @@ class RequestHandler(threading.Thread): logger.error(f"no action {request_map['action']}!") global_var.communicator.add_response('error', self.client_socket, {request_map['action']: f"no action {request_map['action']}!"}) + # logger.debug('request over!') # break except Exception as e: logger.error(str(e)) diff --git a/dcs/tests/spider_task.py b/dcs/tests/spider_task.py index 7b1b0d3..2f51dcb 100644 --- a/dcs/tests/spider_task.py +++ b/dcs/tests/spider_task.py @@ -188,7 +188,8 @@ class Spider_task(threading.Thread): result.update({'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name}) - global_var.communicator.add_info('response', self.client_socket.getpeername(), result) + # global_var.communicator.add_info('response', self.client_socket.getpeername(), result) + global_var.communicator.add_response('response', self.client_socket, result) def run(self) -> None: global_var.communicator.add_response('crawling state', self.client_socket, diff --git a/dcs/tools/database.py b/dcs/tools/database.py index 4c68523..fa036cf 100644 --- a/dcs/tools/database.py +++ b/dcs/tools/database.py @@ -7,7 +7,10 @@ import dcs.tools.cookie as cookie # 获取数据库连接对象 -def mysql_conn(host='192.168.43.64', user='root', passwd='427318Aa', db='test'): +# def mysql_conn(host='10.129.16.173', user='root', passwd='427318Aa', db='test'): +# def mysql_conn(host='10.129.16.155', user='root', passwd='427318Aa', db='test'): +# def mysql_conn(host='192.168.43.64', user='root', passwd='427318Aa', db='test'): +def mysql_conn(host='127.0.0.1', user='root', passwd='xwdjzwy5252', db='test'): try: # logger.debug('connecting to database...') conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db) diff --git a/ui/connect.py b/ui/connect.py index c709e39..df2aced 100644 --- a/ui/connect.py +++ b/ui/connect.py @@ -37,35 +37,29 @@ def read_bytes(s: 'socket.socket', size: 'int') -> 'bytes': return data -def send_request(ip, port, request_info): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_server: - socket_to_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - socket_to_server.bind(('', 9014)) - socket_to_server.connect((ip, int(port))) +def send_request(request_info, socket_to_server): + full_request = generate_request(request_info) - full_request = generate_request(request_info) + socket_to_server.sendall(full_request) - socket_to_server.sendall(full_request) + if request_info['action'] == 'end' or request_info['action'] == 'start': + return - responseJson = JSONDecoder().decode( - read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( - "utf-8")) + responseJson = JSONDecoder().decode( + read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode( + "utf-8")) - return responseJson + return responseJson -def receive_response(): - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(('', 9014)) - server_socket.listen() +def receive_response(server_socket): # while True: - client_socket, _ = server_socket.accept() - request_map = parse_request(client_socket) - if request_map['type'] == 'response': - print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False)) - with open('result.txt', 'w', encoding='utf-8') as f: - json.dump(request_map, f, ensure_ascii=False, indent=4) + # client_socket, _ = server_socket.accept() + request_map = parse_request(server_socket) + # if request_map['type'] == 'response': + print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False)) + with open('result.json', 'w', encoding='utf-8') as f: + json.dump(request_map, f, ensure_ascii=False, indent=4) if __name__ == '__main__': @@ -93,15 +87,26 @@ if __name__ == '__main__': args = parser.parse_args() - request = dict() + local_port = 10004 + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # server_socket.bind(('', local_port)) + server_socket.connect((args.ip, int(args.port))) + # server_socket.listen() + request = {'action': 'start'} + send_request(request, server_socket) if args.action == 'crawling': request = {'action': 'crawl zhiwang', 'word': args.word, 'pages_start': args.pages_start, 'pages_end': args.pages_end, 'cookie': args.cookie} elif args.action == 'login' or args.action == 'register': request = {'action': args.action, 'user': args.user, 'password': args.password} - response = send_request(args.ip, args.port, request) + response = send_request(request, server_socket) print(response) if args.action == 'crawling': - receive_response() + receive_response(server_socket) + + request = {'action': 'end'} + send_request(request, server_socket) + server_socket.close() diff --git a/ui/login.bat b/ui/login.bat index 1c4e06d..6ad6b23 100644 --- a/ui/login.bat +++ b/ui/login.bat @@ -1 +1 @@ -python .\connect.py --ip 127.0.0.1 --port 7777 login --user yuu --password yuu +python .\connect.py --ip 127.0.0.1 --port 7777 login --user liuxiaoyu --password 113818 diff --git a/ui/login.js b/ui/login.js index 0d508d7..d41be97 100644 --- a/ui/login.js +++ b/ui/login.js @@ -10,9 +10,12 @@ var fs = require("fs"); * 配置MySql */ var connection = mysql.createConnection({ - host : '192.168.43.64', + // host : '192.168.43.64', + // host : '10.129.16.173', + host : '127.0.0.1', user : 'root', - password : '427318Aa', + // password : '427318Aa', + password : 'xwdjzwy5252', database : 'test', port:'3306' }); @@ -134,8 +137,8 @@ function execute(cmd) { //调用cmd命令 }) } app.post('/check', function (req, res) { - execute('python connect.py --ip 127.0.0.1 --port 7777 crawling --word computer --cookie fb90de22c26723e4d7172fcf1db124f4db91fa30 --pages_start 1 --pages_end 3'); - fs.readFile('./result.txt', 'utf-8', function (err, data) { + execute('python connect.py --ip 127.0.0.1 --port 7777 crawling --word computer --cookie 8f607bcea67d4f62475fcc710e2f5aff794a4833 --pages_start 1 --pages_end 3'); + fs.readFile('./result.json', 'utf-8', function (err, data) { if (err) { console.error(err); }