重写了connect通信程序与服务器通信系统,彻底重写了终端节点集群,对整个系统进行了较大幅度的优化

master
wufayuan 2 years ago
parent 4fbc6cc294
commit ae894c1fc0

@ -1,21 +1,15 @@
import socket
from collections import deque
# from dcs.tests.spider import Spider
# from dcs.tests.requester import Requester
# from dcs.tests.user_process import UP
# from dcs.tests.response import Responser
from typing import Any
class CUI:
def __init__(self, user_name, login_time, login_state, state, cookie, st=None):
def __init__(self, user_name, login_time, login_state, state, cookie, address):
self.user_name = user_name
self.login_time = login_time
self.login_state = login_state
self.state = state
self.cookie = cookie
self.socket = st
self.address = address
self.crawl_result = deque()
@ -23,7 +17,7 @@ class global_var:
"""需要定义全局变量的放在这里"""
connection = None
free_spiders = []
current_user_info: list[CUI] = []
current_user_info: list[CUI] = [CUI('god', None, None, None, 'god', None)]
requester = None
server = None
spider = None
@ -34,11 +28,11 @@ class global_var:
test = None
def get_free_sockets() -> tuple[socket.socket]:
fs: list[socket.socket] = []
def get_free_addresses() -> tuple[Any, ...]:
fs = []
for i in global_var.current_user_info:
if i.state == 'free':
fs.append(i.socket)
fs.append(i.address)
return tuple(fs)
@ -49,13 +43,13 @@ def exists(cookie):
return False
def add_user(user_name, login_time, login_state, state, cookie, st=None):
global_var.current_user_info.append(CUI(user_name, login_time, login_state, state, cookie, st))
def add_user(user_name, login_time, login_state, state, cookie, address=None):
global_var.current_user_info.append(CUI(user_name, login_time, login_state, state, cookie, address))
def set_state_socket(sockett, state):
def set_state_client(cookie, state):
for i in global_var.current_user_info:
if i.socket == sockett:
if i.cookie == cookie:
i.state = state
break
@ -64,14 +58,12 @@ def set_crawl_result(cookie, result):
for i in global_var.current_user_info:
if i.cookie == cookie:
i.crawl_result.append(result)
# print(f'hhh: {cookie}', result)
break
def get_crawl_result(cookie):
for i in global_var.current_user_info:
if i.cookie == cookie:
# print(f'hgf: {cookie}', i.crawl_result)
return i.crawl_result

@ -1,4 +1,5 @@
[server]
ip = 127.0.0.1
port = 7777
daemon = True
buffer_size = 8 * 1024 * 1024

@ -1,137 +0,0 @@
import json
import socket
import struct
import threading
from configparser import ConfigParser
from json import JSONDecoder
from time import sleep
from loguru import logger
from msedge.selenium_tools import Edge
from msedge.selenium_tools import EdgeOptions
from dcs.tests.zhiwang import *
from dcs.tools import message_process as mp
from dcs.tools.message_process import parse_request, generate_response
def crawl_zhiwang(word, pages_start, pages_end):
edge_options = EdgeOptions()
edge_options.use_chromium = True
No_Image_loading = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2}
edge_options.add_experimental_option("prefs", No_Image_loading)
edge_options.add_argument('--headless')
configFile = '../../conf/settings.ini'
con = ConfigParser()
con.read(configFile, encoding='utf-8')
items = con.items('crawler')
items = dict(items)['edge_driver_path']
print(items)
driver = Edge(options=edge_options, executable_path=items)
soup = driver_open(driver, word)
papers = [] # 用于保存爬取到的论文
paper_id = 0
res = {} # 保存终端爬取结果
# 爬取第一篇
if pages_start == 1:
spider(driver, soup, papers)
logger.debug(res)
pages_start += 1
while paper_id < len(papers):
write2res(papers[paper_id], res)
paper_id += 1
while pages_start < pages_end:
content = change_page(driver, pages_start)
spider(driver, content, papers)
while paper_id < len(papers):
write2res(papers[paper_id], res)
paper_id += 1
pages_start += 1
driver.close()
# logger.debug("here")
return res
def write2res(paper: Paper, res):
for author in paper.authors:
if author.name:
res.update(
{len(res): {'name': author.name, 'college': author.college, 'major': author.major, 'title': paper.title}})
class Crawl(threading.Thread):
def __init__(self):
super(Crawl, self).__init__()
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_socket.bind(('', 9999))
@staticmethod
def crawl(request_map) -> dict:
result_map = crawl_zhiwang(request_map['word'], request_map['pages_start'], request_map['pages_end'])
# result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'},
# 1: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}}
logger.debug(result_map)
return result_map
def run(self) -> None:
self.server_socket.listen()
while True:
client_socket, _ = self.server_socket.accept()
request_map = parse_request(client_socket)
if request_map['type'] == 'request':
print("receiving help request:\n" + json.dumps(request_map, ensure_ascii=False))
response_map = self.crawl(request_map)
response_map.update({'cookie': request_map['cookie']})
client_socket.sendall(generate_response(response_map))
if request_map['type'] == 'response':
print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False))
# break
crawl = Crawl()
crawl.start()
# res = crawl.crawl({'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 2, 'cookie': '123'})
# logger.debug(res)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_server:
socket_to_server.bind(('127.0.0.1', 9999))
socket_to_server.connect(('127.0.0.1', 7777))
request = {'action': 'register', 'user': 'liuxiaoyu', 'password': '113818'}
socket_to_server.sendall(mp.generate_request(request))
responseJson = JSONDecoder().decode(
mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode(
"utf-8"))
print(responseJson)
request = {'action': 'login', 'user': 'liuxiaoyu', 'password': '113818'}
socket_to_server.sendall(mp.generate_request(request))
responseJson = JSONDecoder().decode(
mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode(
"utf-8"))
cookie = responseJson['cookie']
print(responseJson)
request = {'action': 'report_free', 'cookie': cookie}
socket_to_server.sendall(mp.generate_request(request))
responseJson = JSONDecoder().decode(
mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode(
"utf-8"))
print(responseJson)
# request = {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3,
# 'cookie': cookie}
# socket_to_server.sendall(mp.generate_request(request))
# responseJson = JSONDecoder().decode(
# mp.read_bytes(socket_to_server, struct.unpack('!Q', socket_to_server.recv(8))[0]).decode(
# "utf-8"))
# print(responseJson)
request = {'action': 'end'}
socket_to_server.sendall(mp.generate_request(request))
crawl.join()

@ -1,19 +1,21 @@
import json
import socket
import struct
import sys
import threading
from configparser import ConfigParser
from json import JSONDecoder
from msedge.selenium_tools import Edge
from msedge.selenium_tools import EdgeOptions
sys.path.append(r'F:\Users\28587\dcs')
from dcs.tests.zhiwang import *
from dcs.tools import message_process as mp
from dcs.tools.message_process import parse_request, generate_response
def crawl_zhiwang(word, pages_start, pages_end):
logger.info(f'[CRAWLER] crawling pages {pages_start}-{pages_end} of keyword {word}...')
edge_options = EdgeOptions()
edge_options.use_chromium = True
No_Image_loading = {"profile.managed_default_content_settings.images": 2, 'permissions.default.stylesheet': 2}
@ -24,7 +26,7 @@ def crawl_zhiwang(word, pages_start, pages_end):
con.read(configFile, encoding='utf-8')
items = con.items('crawler')
items = dict(items)['edge_driver_path']
print(items)
# print(items)
driver = Edge(options=edge_options, executable_path=items)
soup = driver_open(driver, word)
@ -35,7 +37,6 @@ def crawl_zhiwang(word, pages_start, pages_end):
# 爬取第一篇
if pages_start == 1:
spider(driver, soup, papers)
logger.debug(res)
pages_start += 1
while paper_id < len(papers):
write2res(papers[paper_id], res)
@ -49,7 +50,6 @@ def crawl_zhiwang(word, pages_start, pages_end):
paper_id += 1
pages_start += 1
driver.close()
# logger.debug("here")
return res
@ -61,39 +61,6 @@ def write2res(paper: Paper, res):
'title': paper.title}})
class Crawl(threading.Thread):
def __init__(self):
super(Crawl, self).__init__()
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_socket.bind(('', local_port))
@staticmethod
def crawl(request_map) -> dict:
result_map = crawl_zhiwang(request_map['word'], request_map['pages_start'], request_map['pages_end'])
# result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'},
# 1: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}}
logger.debug(result_map)
return result_map
def run(self) -> None:
self.server_socket.listen()
while True:
client_socket, _ = self.server_socket.accept()
request_map = parse_request(client_socket)
if request_map['type'] == 'request':
print("receiving help request:\n" + json.dumps(request_map, ensure_ascii=False))
response_map = self.crawl(request_map)
response_map.update({'cookie': request_map['cookie']})
client_socket.sendall(generate_response(response_map))
report_map = {'action': 'report_free', 'cookie': cookie}
logger.debug(send_request(socket_to_server, report_map))
if request_map['type'] == 'response':
print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False))
# break
def send_request(socket2server, req):
socket2server.sendall(mp.generate_request(req))
responseJson = JSONDecoder().decode(
@ -102,35 +69,72 @@ def send_request(socket2server, req):
return responseJson
server_ip = '127.0.0.1'
server_port = 7777
local_port = 9999
crawl = Crawl()
crawl.start()
# res = crawl.crawl({'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 2, 'cookie': '123'})
# logger.debug(res)
def crawl(request_map) -> dict:
result_map = crawl_zhiwang(request_map['word'], request_map['pages_start'], request_map['pages_end'])
# result_map = {0: {'name': 'remote', 'college': 'remote', 'major': 'remote', 'title': 'remote'},
# 1: {'name': 'remote1', 'college': 'remote1', 'major': 'remote', 'title': 'remote'}}
return result_map
class Client(threading.Thread):
def __init__(self, server_ip, server_port, local_ip, local_port):
super(Client, self).__init__()
self.server_ip = server_ip
self.server_port = server_port
self.local_ip = local_ip
self.local_port = local_port
self.client_name = f'client_{self.local_port}'
self.client_password = f'client_{self.local_port}'
def run(self) -> None:
ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ssocket.bind((self.local_ip, self.local_port)) # ip 不能是'' !
ssocket.listen()
socket_to_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
socket_to_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
socket_to_server.connect((self.server_ip, self.server_port))
request = {'action': 'register', 'user': self.client_name, 'password': self.client_password}
logger.info(f'[RESPONSE] {send_request(socket_to_server, request)}')
request = {'action': 'login', 'user': self.client_name, 'password': self.client_password, 'address': (self.local_ip, self.local_port)}
response = send_request(socket_to_server, request)
logger.info(f'[RESPONSE] {response}')
cookie = response['cookie']
request = {'action': 'report_free', 'cookie': cookie}
logger.info(f'[RESPONSE] {send_request(socket_to_server, request)}')
while True:
try:
client_socket, _ = ssocket.accept()
request_map = parse_request(client_socket)
socket_to_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
socket_to_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
socket_to_server.bind(('', local_port))
socket_to_server.connect((server_ip, server_port))
if request_map['type'] == 'request':
logger.info("[REQUEST] receiving help request: " + json.dumps(request_map, ensure_ascii=False))
response_map = crawl(request_map)
response_map.update({'cookie': request_map['cookie']})
client_socket.sendall(generate_response(response_map))
logger.info(f'[RESPONSE] sending client result {response_map}...')
request = {'action': 'register', 'user': 'liuxiaoyu', 'password': '113818'}
logger.debug(send_request(socket_to_server, request))
request = {'action': 'report_free', 'cookie': cookie}
logger.info(f'[RESPONSE] {send_request(socket_to_server, request)}')
request = {'action': 'login', 'user': 'liuxiaoyu', 'password': '113818'}
response = send_request(socket_to_server, request)
logger.debug(response)
cookie = response['cookie']
elif request_map['type'] == 'end':
logger.info(f"[REQUEST] end")
logger.debug(f"communication end from {client_socket.getpeername()}!")
request = {'action': 'report_free', 'cookie': cookie}
logger.debug(send_request(socket_to_server, request))
request = {'action': 'end'}
socket_to_server.sendall(mp.generate_request(request))
# request = {'action': 'crawl zhiwang', 'word': 'science', 'pages_start': 1, 'pages_end': 3,
# 'cookie': cookie}
# logger.debug(send_request(socket_to_server, request))
break
except Exception as e:
logger.error(str(e))
request = {'action': 'end'}
socket_to_server.sendall(mp.generate_request(request))
crawl.join()
if __name__ == '__main__':
client = Client('127.0.0.1', 7777, '127.0.0.1', 9998)
client.start()
client.join()

@ -0,0 +1,25 @@
# 分布式节点集群服务器
from loguru import logger
from dcs.clients.client import Client
start = 9000
ip = '127.0.0.1'
port = 7777
local_ip = '127.0.0.1'
local_port = None
# 开启的分布节点数量
count = 5
if __name__ == '__main__':
clients = []
socket_to_servers = []
for i in range(start, start + count):
client = Client(ip, port, local_ip, i)
client.daemon = True
clients.append(client)
[c.start() for c in clients]
logger.info('[CLIENTS] starting all client nodes...')
[c.join() for c in clients]

@ -1,9 +1,9 @@
import threading
import socket
import threading
from loguru import logger
from dcs.tools.message_process import generate_response, generate_request
from dcs.tools.message_process import generate_response
class Communicator(threading.Thread):
@ -24,17 +24,6 @@ class Communicator(threading.Thread):
while True:
for responser in self.responser_list:
response_type, client_socket, response_map = responser[0], responser[1], responser[2]
logger.info(f'sending response to {client_socket.getpeername()}: {response_map}')
logger.info(f'[COMMUNICATE] sending response to {client_socket.getpeername()}: {response_map}')
client_socket.sendall(generate_response(response_map))
self.responser_list.remove(responser)
# with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_client:
# for info in self.info_list:
# try:
# logger.info(f'sending info to {info[0]}: {info[1]}')
# socket_to_client.connect(info[0])
# socket_to_client.sendall(generate_request(info[1]))
# self.info_list.remove(info)
# except Exception as e:
# logger.error(str(e))
# self.info_list.remove(info)

File diff suppressed because one or more lines are too long

@ -9,10 +9,10 @@ from conf.config import global_var
from dcs.user_process import UP
from dcs.communicate import Communicator
logger.debug('starting the servers...')
logger.info('[SERVER] starting the servers...')
create_user_info()
logger.add('./dcs.log', rotation='10 MB', enqueue=True, backtrace=True, diagnose=True)
logger.debug('reading config args...')
logger.info('[SERVER] reading config args...')
configFile = '../conf/settings.ini'
con = ConfigParser()
con.read(configFile, encoding='utf-8')
@ -20,7 +20,7 @@ global_var.configs = con
items = con.items('server')
items = dict(items)
global_var.server = Server(int(items['port']), eval(items['buffer_size']))
global_var.server = Server(str(items['ip']), int(items['port']), eval(items['buffer_size']))
global_var.server.daemon = items['daemon']
global_var.server.start()

@ -37,7 +37,7 @@ class Requester(threading.Thread):
pass
def get(self, client_address, task: Spider_partial_task):
logger.info(f'sending crawl request to {str(client_address)}')
# logger.info(f'[REQUESTER] sending crawl request to {str(client_address)}')
req = Req(client_address, task)
self.reqs.append(req)
req.start()
@ -53,20 +53,20 @@ class Req(threading.Thread):
def run(self) -> None:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as socket_to_client:
socket_to_client.connect(self.client_address)
socket_to_client.connect(tuple(self.client_address))
self.request_map.update({'type': 'request'})
logger.info(f'[REQUESTER] sending request {self.request_map} to client {self.client_address}...')
socket_to_client.sendall(generate_request(self.request_map))
self.responseJson = JSONDecoder().decode(
read_bytes(socket_to_client, struct.unpack('!Q', socket_to_client.recv(8))[0]).decode(
"utf-8"))
cookie = self.responseJson['cookie']
del self.responseJson['cookie']
logger.debug('receiving remote task result, saving...')
logger.info(f'[REMOTE] receiving remote task result {self.responseJson} from {self.client_address}, saving...')
set_crawl_result(cookie, self.responseJson)
self.task.pages_start = self.task.pages_end # finished
self.task.thread = None
logger.debug("result: "+str(self.responseJson))
# global_var.requester.set_req_state(self.client_address, self.request_map, True)
if __name__ == '__main__':
@ -77,10 +77,5 @@ if __name__ == '__main__':
res = deque()
requester = Requester()
requester.start()
# requester.get(address, my_request)
# requester.get(address1, my_request1)
sleep(2)
# print(requester1.get())
# print(requester.get())
print(res)

@ -8,13 +8,13 @@ from conf.config import global_var
class Server(threading.Thread): # 将监听和处理分离, 以便同时响应多个客户端
def __init__(self, port: 'int', buffer_size: 'int'):
def __init__(self, ip: 'str', port: 'int', buffer_size: 'int'):
super().__init__()
self.port: 'int' = port
self.buffer_size = buffer_size
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_socket.bind(('', port))
self.server_socket.bind((ip, port))
global_var.server_socket = self.server_socket
self.client_sockets: list[socket.socket] = []
@ -22,7 +22,7 @@ class Server(threading.Thread): # 将监听和处理分离, 以便同时响
self.server_socket.listen()
while True:
client_socket, _ = self.server_socket.accept()
logger.debug(f'connected to client {client_socket.getpeername()}')
logger.info(f'[SERVER] connected to client {client_socket.getpeername()}')
self.client_sockets.append(client_socket)
r = RequestHandler(client_socket)
r.start()

@ -9,7 +9,6 @@ from dcs.tests.spider_task import Spider_task
class Spider(threading.Thread):
def __init__(self):
super(Spider, self).__init__()
# self.tasks: list[tuple[socket.socket, dict]] = []
self.tasks: list[Spider_task] = []
self.daemon = True
@ -19,6 +18,6 @@ class Spider(threading.Thread):
def run(self) -> None:
while True:
for task in self.tasks:
logger.info(f'processing spider request...')
logger.info(f'[REQUEST HANDLER] processing spider request...')
task.start()
self.tasks.remove(task)

@ -17,17 +17,16 @@ class RequestHandler(threading.Thread):
request_map = parse_request(self.client_socket)
if request_map['action'] == 'end':
logger.info(f"[REQUEST] end")
logger.debug(f"communication over from {self.client_socket.getpeername()}!")
logger.info(f"[REQUEST] end: communication over from {self.client_socket.getpeername()}!")
break
elif request_map['action'] == 'start':
logger.info(f"[REQUEST] start")
logger.debug(f"communication begin from {self.client_socket.getpeername()}!")
logger.info(f"[REQUEST] start: communication begin from {self.client_socket.getpeername()}!")
elif request_map['action'] == 'crawl zhiwang':
chk_res = check(request_map)
if chk_res is None:
logger.warning("user info error!")
break
if request_map['cookie'] != 'god':
logger.warning("[ERROR] user info error!")
break
global_var.spider.add_task(request_map, self.client_socket)
elif request_map['action'] in ['report_free', 'login', 'register']:
global_var.up.add_request(request_map, self.client_socket)
@ -35,8 +34,5 @@ class RequestHandler(threading.Thread):
logger.error(f"no action {request_map['action']}!")
global_var.communicator.add_response('error', self.client_socket,
{request_map['action']: f"no action {request_map['action']}!"})
# logger.debug('request over!')
# break
except Exception as e:
logger.error(str(e))
# global_var.communicator.add_response('error', self.client_socket, {request_map['action']: str(e)})

@ -1,20 +1,17 @@
import socket
import threading
from typing import Optional
import multiprocessing
from loguru import logger
from msedge.selenium_tools import Edge
from msedge.selenium_tools import EdgeOptions
from conf.config import global_var, get_free_sockets, get_crawl_result, get_by_cookie, set_state_socket
from conf.config import global_var, get_free_addresses, get_crawl_result, get_by_cookie, set_state_client
from dcs.tests.zhiwang import *
from dcs.tools.database import get_crawl_result_by_crawl_id, write_result2database
from dcs.tools.database import get_last_crawl_id, create_crawl_result_table
def write2database(paper: Paper, table_name: str, last_crawl_id: int):
logger.info(f'writing to database: {paper.title}')
logger.info(f'[DATABASE] writing to database: {paper.title}')
for author in paper.authors:
if author.name:
write_result2database([author.name, author.college, author.major, paper.title], table_name, last_crawl_id)
@ -53,13 +50,10 @@ class Crawler(threading.Thread):
edge_options.add_experimental_option("prefs", No_Image_loading)
driver = Edge(options=edge_options, executable_path=self.edge_driver_path)
soup = driver_open(driver, self.partial_task.word) # 搜索word
# logger.debug(self.last_crawl_id)
# logger.debug(self.partial_task.word)
papers = [] # 用于保存爬取到的论文
table_name = f'{user_name}_crawl_result'
create_crawl_result_table(table_name=table_name)
# last_crawl_id = get_last_crawl_id(table_name=table_name)
self.partial_task.crawl_id = self.last_crawl_id + 1
paper_id = 0
@ -101,13 +95,10 @@ class Crawler(threading.Thread):
# self.crawl_zhiwang(user_name=self.partial_task.cui.user_name)
self.test_simulation(user_name=self.partial_task.cui.user_name)
except Exception as e:
logger.error(str(e))
logger.error(e.__traceback__.tb_frame.f_globals["__file__"]) # 发生异常所在的文件
logger.error(e.__traceback__.tb_lineno) # 发生异常所在的行数
logger.error(f'[ERROR] {str(e)}')
finally:
logger.info(f'partial crawl task finished: {str(self.partial_task)}')
logger.info(f'[TASK] local partial crawl task finished: {str(self.partial_task)}')
self.partial_task.thread = None
# self.partial_task.pages_start = self.partial_task.pages_end
class Spider_partial_task:
@ -142,7 +133,7 @@ class Spider_task(threading.Thread):
def distribute_task(self):
# distribute tasks, 3 pages as a task
# [pages_start, pages_end), like [1,3) means 1,2 page
logger.info(f'distributing task: {self.client_socket.getpeername(), self.request_map}')
logger.info(f'[TASK] distributing task: {self.client_socket.getpeername(), self.request_map}')
pages_start = self.request_map['pages_start']
pages_end = self.request_map['pages_end']
while pages_start < pages_end:
@ -153,28 +144,25 @@ class Spider_task(threading.Thread):
else:
pages_start = pages_end
tmp['pages_end'] = pages_start
# self.tasks.append((client_socket, tmp))
self.partial_tasks.append(Spider_partial_task(self, tmp))
logger.debug(self.partial_tasks)
logger.info(f'[TASK] all tasks: {self.partial_tasks}')
def is_all_task_crawled(self):
for task in self.partial_tasks:
# print(task.task_type, task.is_partial_task_crawl_completely())
if not task.is_partial_task_crawl_completely():
return False
return True
def compose_result(self):
logger.debug('composing task...')
logger.info('[COMPOSE] composing task...')
result = dict()
remote_result = get_crawl_result(self.request_map['cookie'])
for result_map in remote_result:
for result_map in list(remote_result):
result.update(result_map)
create_crawl_result_table(table_name=self.table_name)
for id, data in result_map.items():
for _, data in result_map.items():
write_result2database([data['name'], data['college'], data['major'], data['title']], self.table_name, self.last_crawl_id)
for task in self.partial_tasks:
# print(task.task_type)
if task.task_type == 'local':
local_result = dict()
local_result_database = get_crawl_result_by_crawl_id(
@ -183,12 +171,11 @@ class Spider_task(threading.Thread):
initial_id = local_result_database[0][0]
for res in local_result_database:
local_result.update({res[0]-initial_id+1: {'name': res[1], 'college': res[2], 'major': res[3], 'paper': res[4]}})
logger.debug(local_result)
logger.info(f'[RESULT] {local_result}')
result.update(local_result)
result.update({'crawl_id': self.last_crawl_id+1, 'table_name': self.table_name})
# global_var.communicator.add_info('response', self.client_socket.getpeername(), result)
global_var.communicator.add_response('response', self.client_socket, result)
def run(self) -> None:
@ -196,9 +183,9 @@ class Spider_task(threading.Thread):
{'crawling state': 'starting, please wait...'})
self.distribute_task()
free_remote_nodes = list(get_free_sockets())
logger.debug(free_remote_nodes)
while True: # necessary otherwise for-cycle only executed once
free_remote_nodes = list(get_free_addresses())
logger.info(f'[REMOTE] free nodes: {free_remote_nodes}')
while True:
for task in self.partial_tasks:
if task.is_partial_task_crawl_completely():
continue
@ -206,24 +193,20 @@ class Spider_task(threading.Thread):
current_task_thread = task.thread
if current_task_thread is None:
for f_node in free_remote_nodes:
# print(free_remote_nodes)
address = f_node.getpeername()
# address = (address[0], address[1] + 1)
logger.debug('generating remote task')
address = f_node
logger.info('[TASK] generating remote task')
task.thread = global_var.requester
task.task_type = 'remote'
global_var.requester.get(address, task)
free_remote_nodes.remove(f_node) # TODO
set_state_socket(f_node, 'busy')
free_remote_nodes.remove(f_node)
set_state_client(f_node, 'busy')
break
else:
logger.debug('generating local task')
logger.info('[TASK] generating local task')
crawler = Crawler(task, self.last_crawl_id)
task.thread = crawler
task.task_type = 'local'
crawler.start()
if self.is_all_task_crawled():
break
# sleep(3)
self.compose_result()

@ -9,22 +9,22 @@ from conf.config import global_var
class Urh(threading.Thread):
def __init__(self, request_map: dict, client_socket: 'socket.socket'):
super().__init__()
self.request_map = request_map
self.request_map: dict = request_map
self.client_socket = client_socket
def report_state(self, state):
logger.info(f"[REQUEST] report free")
config.set_state_socket(self.client_socket, state)
config.set_state_client(self.request_map['cookie'], state)
response = {
'report_free': 'success marked ' + str(self.request_map['cookie'])
}
global_var.communicator.add_response('report_free', self.client_socket, response)
logger.info(f"[RESPONSE] report free: {response['report_free']}")
def login(self, user, password, st):
def login(self, user, password, address):
logger.info(f"[REQUEST] login")
database.mysql_conn()
response = database.login(user, password, st)
response = database.login(user, password, address)
response = {
'cookie': response
}
@ -48,7 +48,11 @@ class Urh(threading.Thread):
if self.request_map['action'] == 'report_free':
self.report_state('free')
elif self.request_map['action'] == 'login':
self.login(self.request_map['user'], self.request_map['password'], self.client_socket)
if self.request_map.__contains__('address'):
address = self.request_map['address']
else:
address = None
self.login(self.request_map['user'], self.request_map['password'], address)
elif self.request_map['action'] == 'register':
self.register(self.request_map['user'], self.request_map['password'])
elif self.request_map['action'] == 'get task process':

@ -47,16 +47,16 @@ def driver_open(driver, key_word):
def spider(driver, soup, papers):
logger.debug("crawling a soup...")
logger.info("[CRAWLER] crawling a soup...")
tbody = soup.find_all('tbody')
try:
tbody = BeautifulSoup(str(tbody[0]), 'lxml')
except Exception as e:
logger.error(str(e))
logger.error(f'[ERROR] {str(e)}')
return
tr = tbody.find_all('tr')
for item in tr:
logger.debug("crawling an item...")
logger.info("[CRAWLER] crawling an item...")
tr_bf = BeautifulSoup(str(item), 'lxml')
td_name = tr_bf.find_all('td', class_='name')
@ -64,7 +64,7 @@ def spider(driver, soup, papers):
a_name = td_name_bf.find_all('a')
# get_text()是获取标签中的所有文本,包含其子标签中的文本
title = a_name[0].get_text().strip()
print("title : " + title)
# print("title : " + title)
td_author = tr_bf.find_all('td', class_='author')
td_author_bf = BeautifulSoup(str(td_author), 'lxml')
@ -75,17 +75,16 @@ def spider(driver, soup, papers):
skey, code = get_skey_code(author) # 获取作者详情页url的skey和code
name = author.get_text().strip() # 获取学者的名字
# print('name : ' + name)
print('name : ' + name)
college, major = get_author_info(skey, code) # 在作者详情页获取大学和专业, major是一个数组
au = Author(name, college, major) # 创建一个学者对象
authors.append(au)
# print('\n')
print('\n')
# print('\n')
paper = Paper(title, authors)
papers.append(paper)
papers.append(paper)
break
# break # TODO: this is to shorten time of crawling
# time.sleep(1) # 每调一次spider休息1s

@ -1,4 +1,3 @@
from hashlib import *
import pymysql
from loguru import logger
@ -16,7 +15,7 @@ def mysql_conn(host='127.0.0.1', user='root', passwd='xwdjzwy5252', db='test'):
conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db)
return conn
except Exception as e:
logger.error(str(e))
logger.error(f'[ERROR] {str(e)}')
def register(u_name, u_pwd):
@ -50,7 +49,7 @@ def register(u_name, u_pwd):
conn.close()
return info
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def get_now():
@ -65,10 +64,10 @@ def get_now():
conn.close()
return res[0]
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def login(u_name, u_pwd, st):
def login(u_name, u_pwd, address):
# s1 = sha1()
# s1.update(u_pwd.encode())
# sha_pwd = s1.hexdigest()
@ -89,7 +88,7 @@ def login(u_name, u_pwd, st):
# info = '用户' + u_name + '登录成功'
time = str(get_now())
info = cookie.Cookie(u_name, time, 'true').generate_cookie()
config.add_user(u_name, time, 'true', 'busy', info, st)
config.add_user(u_name, time, 'true', 'busy', info, address)
else:
info = '密码错误,登录失败'
# 关闭连接
@ -97,7 +96,7 @@ def login(u_name, u_pwd, st):
conn.close()
return info
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def cancel(u_name):
@ -109,7 +108,7 @@ def cancel(u_name):
cur.close()
conn.close()
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def get_last_crawl_id(table_name: str) -> int:
@ -130,7 +129,7 @@ def get_last_crawl_id(table_name: str) -> int:
conn.close()
return last_crawl_id
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
return 0
@ -145,7 +144,7 @@ def drop_table(table_name: str):
cur.close()
conn.close()
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def get_crawl_result_by_crawl_id(table_name: str, crawl_id: int):
@ -159,7 +158,7 @@ def get_crawl_result_by_crawl_id(table_name: str, crawl_id: int):
conn.close()
return result
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def create_table(create_sql: str):
@ -172,7 +171,7 @@ def create_table(create_sql: str):
cur.close()
conn.close()
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
def create_crawl_result_table(table_name: str):
@ -211,7 +210,7 @@ def write_result2database(res: list, table_name: str, last_crawl_id: int):
conn.close()
info = '插入成功'
except Exception as e:
print(e)
logger.error(f'[ERROR] {str(e)}')
info = '插入失败'
return info

@ -2,6 +2,9 @@ import socket
import json
import struct
from json import JSONEncoder
from loguru import logger
from conf.config import exists
@ -37,7 +40,8 @@ def check(cookie):
if exists(cookie['cookie']):
return cookie
return None
except:
except Exception as e:
logger.error(f'[ERROR] {str(e)}')
return None

@ -15,7 +15,7 @@ class UP(threading.Thread):
def run(self) -> None:
while True:
for request in self.requests:
logger.info(f'processing user request...')
logger.info(f'[REQUEST HANDLER] processing user request...')
urh = Urh(request[1], request[0])
urh.start()
self.requests.remove(request)

@ -10,6 +10,7 @@ def parse_request(client_socket: socket.socket):
request_map = json.JSONDecoder().decode(read_bytes(client_socket, request_header_size).decode("utf-8"))
return request_map
def generate_request(request_info) -> 'bytes':
"""
根据传入的dict生成请求
@ -53,10 +54,7 @@ def send_request(request_info, socket_to_server):
def receive_response(server_socket):
# while True:
# client_socket, _ = server_socket.accept()
request_map = parse_request(server_socket)
# if request_map['type'] == 'response':
print("receiving response:\n" + json.dumps(request_map, ensure_ascii=False))
with open('result.json', 'w', encoding='utf-8') as f:
json.dump(request_map, f, ensure_ascii=False, indent=4)
@ -90,9 +88,7 @@ if __name__ == '__main__':
local_port = 10004
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# server_socket.bind(('', local_port))
server_socket.connect((args.ip, int(args.port)))
# server_socket.listen()
request = {'action': 'start'}
send_request(request, server_socket)

@ -137,7 +137,7 @@ function execute(cmd) { //调用cmd命令
})
}
app.post('/check', function (req, res) {
execute('python connect.py --ip 127.0.0.1 --port 7777 crawling --word computer --cookie 8f607bcea67d4f62475fcc710e2f5aff794a4833 --pages_start 1 --pages_end 3');
execute('python connect.py --ip 127.0.0.1 --port 7777 crawling --word computer --cookie god --pages_start 1 --pages_end 3');
fs.readFile('./result.json', 'utf-8', function (err, data) {
if (err) {
console.error(err);

@ -1,41 +1,821 @@
{
"0": {
"name": "孙亚康",
"college": null,
"major": null,
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"1": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "郭红领",
"college": "清华大学",
"major": "建筑科学与工程;计算机软件及计算机应用;安全科学与灾害防治;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"2": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "罗柱邦",
"college": "清华大学",
"major": "计算机软件及计算机应用;建筑科学与工程;基础医学;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"3": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "张智慧",
"college": "清华大学",
"major": "环境科学与资源利用;建筑科学与工程;工业经济;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"4": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "孙亚康",
"college": null,
"major": null,
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"5": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "郭红领",
"college": "清华大学",
"major": "建筑科学与工程;计算机软件及计算机应用;安全科学与灾害防治;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"6": {
"name": "test",
"college": "test",
"major": "test",
"paper": "test"
"name": "罗柱邦",
"college": "清华大学",
"major": "计算机软件及计算机应用;建筑科学与工程;基础医学;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"7": {
"name": "张智慧",
"college": "清华大学",
"major": "环境科学与资源利用;建筑科学与工程;工业经济;",
"title": "基于计算机视觉的预制墙板临时支撑安全合规性检查"
},
"8": {
"name": "刘思源",
"college": "西南大学",
"major": "计算机硬件技术;高等教育;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"9": {
"name": "冯蕾霖",
"college": "西南大学",
"major": "计算机硬件技术;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"10": {
"name": "朱章黔",
"college": "陆军勤务学院",
"major": "数学;计算机硬件技术;计算机软件及计算机应用;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"11": {
"name": "贾韬",
"college": "西南大学",
"major": "科学研究管理;计算机软件及计算机应用;自动化技术;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"12": {
"name": "刘思源",
"college": "西南大学",
"major": "计算机硬件技术;高等教育;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"13": {
"name": "冯蕾霖",
"college": "西南大学",
"major": "计算机硬件技术;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"14": {
"name": "朱章黔",
"college": "陆军勤务学院",
"major": "数学;计算机硬件技术;计算机软件及计算机应用;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"15": {
"name": "贾韬",
"college": "西南大学",
"major": "科学研究管理;计算机软件及计算机应用;自动化技术;",
"title": "基于大规模课程大纲数据的中美高校计算机相关课程考核的比较分析"
},
"16": {
"name": "吴迪",
"college": "承德医学院附属医院",
"major": "外科学;临床医学;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"17": {
"name": "司丽娜",
"college": "承德医学院",
"major": "基础医学;生物学;外科学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"18": {
"name": "武丽珠",
"college": "承德市中心医院",
"major": "外科学;心血管系统疾病;急救医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"19": {
"name": "王建华",
"college": "承德医学院附属医院",
"major": "外科学;计算机软件及计算机应用;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"20": {
"name": "罗金伟",
"college": "承德医学院附属医院",
"major": "外科学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"21": {
"name": "常乾坤",
"college": "承德医学院附属医院",
"major": "外科学;临床医学;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"22": {
"name": "吕永明",
"college": "承德医学院附属医院",
"major": "外科学;内分泌腺及全身性疾病;临床医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"23": {
"name": "杜元良",
"college": "承德医学院附属医院",
"major": "外科学;内分泌腺及全身性疾病;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"24": {
"name": "吴迪",
"college": "承德医学院附属医院",
"major": "外科学;临床医学;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"25": {
"name": "司丽娜",
"college": "承德医学院",
"major": "基础医学;生物学;外科学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"26": {
"name": "武丽珠",
"college": "承德市中心医院",
"major": "外科学;心血管系统疾病;急救医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"27": {
"name": "王建华",
"college": "承德医学院附属医院",
"major": "外科学;计算机软件及计算机应用;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"28": {
"name": "罗金伟",
"college": "承德医学院附属医院",
"major": "外科学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"29": {
"name": "常乾坤",
"college": "承德医学院附属医院",
"major": "外科学;临床医学;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"30": {
"name": "吕永明",
"college": "承德医学院附属医院",
"major": "外科学;内分泌腺及全身性疾病;临床医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"31": {
"name": "杜元良",
"college": "承德医学院附属医院",
"major": "外科学;内分泌腺及全身性疾病;基础医学;",
"title": "3D打印技术与计算机辅助设计应用在全膝关节置换治疗重度膝关节骨关节炎中的可行性"
},
"32": {
"name": "康熙",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;农业工程;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"33": {
"name": "刘刚",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;自动化技术;农业工程;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"34": {
"name": "初梦苑",
"college": "河北农业大学",
"major": "计算机软件及计算机应用;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"35": {
"name": "李前",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"36": {
"name": "王彦超",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"37": {
"name": "康熙",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;农业工程;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"38": {
"name": "刘刚",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;自动化技术;农业工程;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"39": {
"name": "初梦苑",
"college": "河北农业大学",
"major": "计算机软件及计算机应用;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"40": {
"name": "李前",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"41": {
"name": "王彦超",
"college": "中国农业大学",
"major": "计算机软件及计算机应用;畜牧与动物医学;",
"title": "基于计算机视觉的奶牛生理参数监测与疾病诊断研究进展及挑战"
},
"42": {
"name": "杨骏",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;自动化技术;电信技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"43": {
"name": "项炜",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"44": {
"name": "敬思远",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;自动化技术;计算机硬件技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"45": {
"name": "苏炳均",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;计算机硬件技术;电信技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"46": {
"name": "杨骏",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;自动化技术;电信技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"47": {
"name": "项炜",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"48": {
"name": "敬思远",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;自动化技术;计算机硬件技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"49": {
"name": "苏炳均",
"college": "乐山师范学院",
"major": "计算机软件及计算机应用;计算机硬件技术;电信技术;",
"title": "地方高校计算机专业人才工程能力培养策略"
},
"50": {
"name": "张娜娜",
"college": "上海思博职业技术学院",
"major": "计算机软件及计算机应用;轻工业手工业;",
"title": "基于计算机视觉传达的皮革缺陷检测算法研究"
},
"51": {
"name": "张娜娜",
"college": "上海思博职业技术学院",
"major": "计算机软件及计算机应用;轻工业手工业;",
"title": "基于计算机视觉传达的皮革缺陷检测算法研究"
},
"52": {
"name": "李晓辉",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"53": {
"name": "杨洪伟",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"54": {
"name": "蒋兰玲",
"college": "辽宁省农业科学院",
"major": "计算机软件及计算机应用;计算机硬件技术;园艺;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"55": {
"name": "张芳",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"56": {
"name": "李晓辉",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"57": {
"name": "杨洪伟",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"58": {
"name": "蒋兰玲",
"college": "辽宁省农业科学院",
"major": "计算机软件及计算机应用;计算机硬件技术;园艺;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"59": {
"name": "张芳",
"college": "沈阳农业大学",
"major": "计算机软件及计算机应用;计算机硬件技术;高等教育;",
"title": "新农科背景下大学生计算机应用能力提升教学研究与实践"
},
"60": {
"name": "贾小军",
"college": "嘉兴学院",
"major": "计算机软件及计算机应用;自动化技术;教育理论与教育管理;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"61": {
"name": "张春花",
"college": "嘉兴学院",
"major": "计算机硬件技术;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"62": {
"name": "刘子豪",
"college": "嘉兴学院",
"major": "计算机软件及计算机应用;自动化技术;仪器仪表工业;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"63": {
"name": "贾小军",
"college": "嘉兴学院",
"major": "计算机软件及计算机应用;自动化技术;教育理论与教育管理;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"64": {
"name": "张春花",
"college": "嘉兴学院",
"major": "计算机硬件技术;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"65": {
"name": "刘子豪",
"college": "嘉兴学院",
"major": "计算机软件及计算机应用;自动化技术;仪器仪表工业;",
"title": "基于“复盘”模式的计算机通识课程管理研究"
},
"66": {
"name": "周学勇",
"college": "信阳师范学院",
"major": "数学;高等教育;生物学;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"67": {
"name": "路振国",
"college": "信阳师范学院",
"major": "数学;高等教育;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"68": {
"name": "程晓明",
"college": "信阳师范学院",
"major": "数学;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"69": {
"name": "周学勇",
"college": "信阳师范学院",
"major": "数学;高等教育;生物学;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"70": {
"name": "路振国",
"college": "信阳师范学院",
"major": "数学;高等教育;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"71": {
"name": "程晓明",
"college": "信阳师范学院",
"major": "数学;",
"title": "一类分数阶计算机病毒模型的稳定性分析"
},
"72": {
"name": "李娜",
"college": "中国科学院大学",
"major": "生物学;计算机软件及计算机应用;生物医学工程;",
"title": "计算机辅助血管介入技术进展综述"
},
"73": {
"name": "贺建安",
"college": "东南大学",
"major": "生物学;计算机软件及计算机应用;生物医学工程;",
"title": "计算机辅助血管介入技术进展综述"
},
"74": {
"name": "陈阳",
"college": "东南大学",
"major": "计算机软件及计算机应用;临床医学;计算机硬件技术;",
"title": "计算机辅助血管介入技术进展综述"
},
"75": {
"name": "周寿军",
"college": "中国科学院深圳先进技术研究院",
"major": "计算机软件及计算机应用;生物医学工程;外科学;",
"title": "计算机辅助血管介入技术进展综述"
},
"76": {
"name": "李娜",
"college": "中国科学院大学",
"major": "生物学;计算机软件及计算机应用;生物医学工程;",
"title": "计算机辅助血管介入技术进展综述"
},
"77": {
"name": "贺建安",
"college": "东南大学",
"major": "生物学;计算机软件及计算机应用;生物医学工程;",
"title": "计算机辅助血管介入技术进展综述"
},
"78": {
"name": "陈阳",
"college": "东南大学",
"major": "计算机软件及计算机应用;临床医学;计算机硬件技术;",
"title": "计算机辅助血管介入技术进展综述"
},
"79": {
"name": "周寿军",
"college": "中国科学院深圳先进技术研究院",
"major": "计算机软件及计算机应用;生物医学工程;外科学;",
"title": "计算机辅助血管介入技术进展综述"
},
"80": {
"name": "李正凡",
"college": "云南华能澜沧江水电有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"81": {
"name": "谭良良",
"college": "南京市南瑞继保工程技术有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"82": {
"name": "杨渊",
"college": "西安热工研究院有限公司",
"major": "电力工业;电信技术;水利水电工程;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"83": {
"name": "陈映喜",
"college": "云南华能澜沧江水电有限公司",
"major": "电力工业;水利水电工程;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"84": {
"name": "周喜",
"college": "云南华能澜沧江水电有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"85": {
"name": "杨伟",
"college": "南京市南瑞继保工程技术有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"86": {
"name": "李正凡",
"college": "云南华能澜沧江水电有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"87": {
"name": "谭良良",
"college": "南京市南瑞继保工程技术有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"88": {
"name": "杨渊",
"college": "西安热工研究院有限公司",
"major": "电力工业;电信技术;水利水电工程;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"89": {
"name": "陈映喜",
"college": "云南华能澜沧江水电有限公司",
"major": "电力工业;水利水电工程;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"90": {
"name": "周喜",
"college": "云南华能澜沧江水电有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"91": {
"name": "杨伟",
"college": "南京市南瑞继保工程技术有限公司",
"major": "水利水电工程;电力工业;自动化技术;",
"title": "基于时间序列的国产化水电站计算机监控系统异常行为报警方法应用"
},
"92": {
"name": "刘文",
"college": "上海大学",
"major": "航空航天科学与工程;计算机软件及计算机应用;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"93": {
"name": "陆小锋",
"college": "上海大学",
"major": "计算机软件及计算机应用;临床医学;神经病学;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"94": {
"name": "毛建华",
"college": "上海大学",
"major": "计算机软件及计算机应用;自然地理学和测绘学;自动化技术;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"95": {
"name": "方思凯",
"college": "Shanghai University",
"major": "航空航天科学与工程;计算机软件及计算机应用;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"96": {
"name": "钱国",
"college": "上海宝冶冶金工程有限公司",
"major": "计算机软件及计算机应用;建筑科学与工程;航空航天科学与工程;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"97": {
"name": "刘文",
"college": "上海大学",
"major": "航空航天科学与工程;计算机软件及计算机应用;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"98": {
"name": "陆小锋",
"college": "上海大学",
"major": "计算机软件及计算机应用;临床医学;神经病学;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"99": {
"name": "毛建华",
"college": "上海大学",
"major": "计算机软件及计算机应用;自然地理学和测绘学;自动化技术;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"100": {
"name": "方思凯",
"college": "Shanghai University",
"major": "航空航天科学与工程;计算机软件及计算机应用;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"101": {
"name": "钱国",
"college": "上海宝冶冶金工程有限公司",
"major": "计算机软件及计算机应用;建筑科学与工程;航空航天科学与工程;",
"title": "基于机载计算机的无人机智能巡检方案"
},
"102": {
"name": "黄立鹤",
"college": "同济大学",
"major": "中国语言文字;外国语言文字;高等教育;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"103": {
"name": "曲惠宇",
"college": "同济大学",
"major": "中国语言文字;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"104": {
"name": "杨晶晶",
"college": "同济大学",
"major": "中国语言文字;精神病学;中国政治与国际政治;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"105": {
"name": "黄立鹤",
"college": "同济大学",
"major": "中国语言文字;外国语言文字;高等教育;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"106": {
"name": "曲惠宇",
"college": "同济大学",
"major": "中国语言文字;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"107": {
"name": "杨晶晶",
"college": "同济大学",
"major": "中国语言文字;精神病学;中国政治与国际政治;",
"title": "老年话语的计算机自动文本分析:进展与前景"
},
"108": {
"name": "杨乐",
"college": "山西工程科技职业大学",
"major": "互联网技术;高等教育;",
"title": "关于高校计算机实验室网络安全管理的研究"
},
"109": {
"name": "杨乐",
"college": "山西工程科技职业大学",
"major": "互联网技术;高等教育;",
"title": "关于高校计算机实验室网络安全管理的研究"
},
"110": {
"name": "孟云飞",
"college": "哈尔滨职业技术学院",
"major": "计算机软件及计算机应用;轻工业手工业;教育理论与教育管理;",
"title": "计算机系统与信息管理融合发展研究"
},
"111": {
"name": "孟云飞",
"college": "哈尔滨职业技术学院",
"major": "计算机软件及计算机应用;轻工业手工业;教育理论与教育管理;",
"title": "计算机系统与信息管理融合发展研究"
},
"112": {
"name": "郝玉成",
"college": "合肥学院",
"major": "化学;无机化工;材料科学;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"113": {
"name": "张全争",
"college": "合肥学院",
"major": "化学;无机化工;有机化工;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"114": {
"name": "林文海",
"college": "合肥学院",
"major": "无线电电子学;高等教育;电力工业;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"115": {
"name": "秦广超",
"college": "合肥学院",
"major": "无机化工;轻工业手工业;化学;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"116": {
"name": "郝玉成",
"college": "合肥学院",
"major": "化学;无机化工;材料科学;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"117": {
"name": "张全争",
"college": "合肥学院",
"major": "化学;无机化工;有机化工;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"118": {
"name": "林文海",
"college": "合肥学院",
"major": "无线电电子学;高等教育;电力工业;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"119": {
"name": "秦广超",
"college": "合肥学院",
"major": "无机化工;轻工业手工业;化学;",
"title": "计算机在材料科学中的应用课程改革与应用型人才教学实践研究"
},
"120": {
"name": "谭毅飞",
"college": "上海市东亚联合控股(集团)有限公司",
"major": "建筑科学与工程;计算机软件及计算机应用;自动化技术;",
"title": "计算机视觉与传感技术下全新劳务计酬模式探究"
},
"121": {
"name": "谭毅飞",
"college": "上海市东亚联合控股(集团)有限公司",
"major": "建筑科学与工程;计算机软件及计算机应用;自动化技术;",
"title": "计算机视觉与传感技术下全新劳务计酬模式探究"
},
"122": {
"name": "吴涤清",
"college": "徐州医科大学",
"major": "口腔科学;肿瘤学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"123": {
"name": "李志萍",
"college": "徐州市中心医院",
"major": "口腔科学;肿瘤学;外科学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"124": {
"name": "孟箭",
"college": "徐州市中心医院",
"major": "口腔科学;肿瘤学;临床医学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"125": {
"name": "陈彬",
"college": "徐州市中心医院",
"major": "口腔科学;仪器仪表工业;肿瘤学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"126": {
"name": "吴涤清",
"college": "徐州医科大学",
"major": "口腔科学;肿瘤学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"127": {
"name": "李志萍",
"college": "徐州市中心医院",
"major": "口腔科学;肿瘤学;外科学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"128": {
"name": "孟箭",
"college": "徐州市中心医院",
"major": "口腔科学;肿瘤学;临床医学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"129": {
"name": "陈彬",
"college": "徐州市中心医院",
"major": "口腔科学;仪器仪表工业;肿瘤学;",
"title": "计算机辅助测量唇腭裂继发颌骨畸形患者手术前后口腔及气道容积的变化"
},
"130": {
"name": "张新星",
"college": "衢州职业技术学院",
"major": "电力工业;机械工业;外科学;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"131": {
"name": "赵英杰",
"college": "衢州职业技术学院",
"major": "外科学;自动化技术;生物医学工程;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"132": {
"name": "陈超",
"college": "衢州职业技术学院",
"major": "外科学;自动化技术;生物医学工程;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"133": {
"name": "张新星",
"college": "衢州职业技术学院",
"major": "电力工业;机械工业;外科学;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"134": {
"name": "赵英杰",
"college": "衢州职业技术学院",
"major": "外科学;自动化技术;生物医学工程;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"135": {
"name": "陈超",
"college": "衢州职业技术学院",
"major": "外科学;自动化技术;生物医学工程;",
"title": "计算机辅助骨科手术机器人技术发展及应用综述"
},
"crawl_id": 8,
"table_name": "liuxiaoyu_crawl_result",
"crawl_id": 34,
"table_name": "god_crawl_result",
"type": "response"
}
Loading…
Cancel
Save