|
|
@ -63,7 +63,7 @@ def write2csv(category, response): # 写入csv文件
|
|
|
|
filename_csv = os.getcwd() + "\\Catalogues\\" + FILENAME_CSV.get(category)
|
|
|
|
filename_csv = os.getcwd() + "\\Catalogues\\" + FILENAME_CSV.get(category)
|
|
|
|
pipelines.write2csv(response, filename_csv)
|
|
|
|
pipelines.write2csv(response, filename_csv)
|
|
|
|
|
|
|
|
|
|
|
|
class milkSpider:
|
|
|
|
class milkSpider: # 定义一个爬虫类
|
|
|
|
def __init__(self, url):
|
|
|
|
def __init__(self, url):
|
|
|
|
self.url = url
|
|
|
|
self.url = url
|
|
|
|
self.category = getCategory(url)
|
|
|
|
self.category = getCategory(url)
|
|
|
@ -73,7 +73,7 @@ class milkSpider:
|
|
|
|
print("write2csv for '{}' was started.".format(self.url))
|
|
|
|
print("write2csv for '{}' was started.".format(self.url))
|
|
|
|
write2csv(self.category, self.response)
|
|
|
|
write2csv(self.category, self.response)
|
|
|
|
|
|
|
|
|
|
|
|
def mainThread():
|
|
|
|
def mainThread(): # 主线程,配置线程池状况,实例化爬虫对象
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
with ThreadPoolExecutor(max_workers = 8) as thread:
|
|
|
|
with ThreadPoolExecutor(max_workers = 8) as thread:
|
|
|
|
while True:
|
|
|
|
while True:
|
|
|
|