update via git

master
wkyuu 3 years ago
parent 43423c6ab3
commit 2a3b2ee52f

@ -63,7 +63,7 @@ def write2csv(category, response): # 写入csv文件
filename_csv = os.getcwd() + "\\Catalogues\\" + FILENAME_CSV.get(category) filename_csv = os.getcwd() + "\\Catalogues\\" + FILENAME_CSV.get(category)
pipelines.write2csv(response, filename_csv) pipelines.write2csv(response, filename_csv)
class milkSpider: class milkSpider: # 定义一个爬虫类
def __init__(self, url): def __init__(self, url):
self.url = url self.url = url
self.category = getCategory(url) self.category = getCategory(url)
@ -73,7 +73,7 @@ class milkSpider:
print("write2csv for '{}' was started.".format(self.url)) print("write2csv for '{}' was started.".format(self.url))
write2csv(self.category, self.response) write2csv(self.category, self.response)
def mainThread(): def mainThread(): # 主线程,配置线程池状况,实例化爬虫对象
try: try:
with ThreadPoolExecutor(max_workers = 8) as thread: with ThreadPoolExecutor(max_workers = 8) as thread:
while True: while True:

Loading…
Cancel
Save