重写了connect通信程序与服务器通信系统，彻底重写了终端节点集群，对整个系统进行了较大幅度的优化,优化集群为多进程，增加轮询间隔

dcs/communicate.py

@@ -1,5 +1,6 @@
 import socket
 import threading
+from time import sleep
 
 from loguru import logger
 
@@ -27,3 +28,5 @@ class Communicator(threading.Thread):
                 logger.info(f'[COMMUNICATE] sending response to {client_socket.getpeername()}: {response_map}')
                 client_socket.sendall(generate_response(response_map))
                 self.responser_list.remove(responser)
+
+            sleep(1)

dcs/spider.py

@@ -1,5 +1,6 @@
 import socket
 import threading
+from time import sleep
 
 from loguru import logger
 
@@ -21,3 +22,5 @@ class Spider(threading.Thread):
                 logger.info(f'[REQUEST HANDLER] processing spider request...')
                 task.start()
                 self.tasks.remove(task)
+
+            sleep(1)

dcs/tests/spider_task.py

@@ -92,8 +92,8 @@ class Crawler(threading.Thread):
 
     def run(self) -> None:
         try:
-            self.crawl_zhiwang(user_name=self.partial_task.cui.user_name)
-            # self.test_simulation(user_name=self.partial_task.cui.user_name)
+            # self.crawl_zhiwang(user_name=self.partial_task.cui.user_name)
+            self.test_simulation(user_name=self.partial_task.cui.user_name)
         except Exception as e:
             logger.error(f'[ERROR] {str(e)}')
         finally:

dcs/tools/database.py

@@ -200,6 +200,7 @@ def create_user_info(table_name: str = 'user_info'):
 
 def write_result2database(res: list, table_name: str, last_crawl_id: int):
     try:
+        logger.debug('writing to database...')
         conn = mysql_conn()
         cur = conn.cursor()
         insert_sql = f"insert into {table_name} (name,college,major,paper,crawl_id,time) values ('%s','%s','%s','%s',%s,now())" % (

dcs/user_process.py

@@ -1,5 +1,7 @@
 import socket
 import threading
+from time import sleep
+
 from dcs.tests.user_request_handler import Urh
 from loguru import logger
 
@@ -19,4 +21,4 @@ class UP(threading.Thread):
                 urh = Urh(request[1], request[0])
                 urh.start()
                 self.requests.remove(request)
-        pass
+            sleep(1)