master
wkyuu 3 years ago
parent 3e41a90ac6
commit f014993c2b

@ -50,7 +50,7 @@ def precheck() -> bool: # 检查redis队列情况
return True return True
elif check == 'q': elif check == 'q':
print("Exit.") print("Exit.")
exit() return
else: print("invalid input!") else: print("invalid input!")
return True return True

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import time import time
import middlewares import middlewares
@ -42,6 +43,11 @@ def View():
def milkSpider(): def milkSpider():
dir = 'Catalogues'
if os.path.exists(dir):
if len(os.listdir(dir)):
print("检测到缓存目录下已有数据调用milkSpider将有可能导致数据重复和冗余。")
print("注意调用milkSpider将启动selenium以及requests进程因为爬取数据量较大往往会占用较多时间确定吗[c]continue or [q]quit", end = '') print("注意调用milkSpider将启动selenium以及requests进程因为爬取数据量较大往往会占用较多时间确定吗[c]continue or [q]quit", end = '')
flag = str(input()) flag = str(input())
if flag == "q": if flag == "q":

@ -174,6 +174,11 @@ def write2csv(response, filename_csv): # 写入到csv文件
fd.write(itemString) fd.write(itemString)
try: try:
dir = "Catalogues"
if not os.path.exists(dir):
os.mkdir(dir)
if os.path.exists(filename_csv): if os.path.exists(filename_csv):
with open(filename_csv, 'a+', encoding = 'utf-8-sig') as fd: # 存在,文件尾追加 with open(filename_csv, 'a+', encoding = 'utf-8-sig') as fd: # 存在,文件尾追加
writer(fd) writer(fd)

Loading…
Cancel
Save