update

downloader.py

@@ -31,3 +31,36 @@ def getsource(url):
 
     driver.close()
     return response
+
+def manmanbuy(url):
+    initChrome = Options()
+
+    initChrome.add_argument('--no-sandbox')
+    initChrome.add_argument('--headless')
+    initChrome.add_argument('--disable-gpu')
+    initChrome.add_argument("disable-cache")
+    initChrome.add_argument('disable-infobars')
+    initChrome.add_argument('log-level=3')    # INFO = 0 WARNING = 1 LOG_ERROR = 2 LOG_FATAL = 3 default is 0
+    initChrome.add_experimental_option("excludeSwitches",['enable-automation','enable-logging'])
+
+    driver = webdriver.Chrome(chrome_options = initChrome, executable_path = './chromedriver.exe')
+    driver.implicitly_wait(10)
+    driver.get(url)
+
+    response = etree.HTML(driver.page_source)
+    response = etree.tostring(response, encoding = "utf-8", pretty_print = True, method = "html")
+    response = response.decode('utf-8')
+
+    driver.close()
+    return response
+
+
+def buy():
+    jdurl = "https://item.jd.com/10047511027349.html"
+    url = "https://tool.manmanbuy.com/HistoryLowest.aspx?url=" + jdurl
+    # print(url)
+    response = getsource(url)
+    print(response)
+
+if __name__ == "__main__":
+    buy()

middlewares.py

@@ -106,12 +106,7 @@ def mainThread(threadlines = 16, flag = flag):    # 线程数默认为3
         exit()
 
 if __name__ == '__main__':
-    if precheck():
-        start_time = time.time()
-        mainThread()
-        print("Totally spend " + str(time.time() - start_time) + "secends")
-    print("done.")
-
+    pass
 
 
 

milkSpider.py

@@ -1,16 +1,17 @@
 # -*- coding: utf-8 -*-
 
-import settings
 import middlewares
 import time
 
-if __name__ == "__main__":
+def milkSpider():
+    if middlewares.precheck():
+        start_time = time.time()
+        middlewares.mainThread()
+        print("Totally spend " + str(time.time() - start_time) + "secends")
+    print("milkSpider done.")
+
 
-    # category = "冰淇淋"   # 要爬取的种类，默认是牛奶，详见settings.py
-    start_time = time.time()
 
-    for category in settings.BASEURL.keys():
-        middlewares.singleSpider(category)
+if __name__ == "__main__":
     
-    end_time = time.time()
-    print(start_time - end_time)
+    milkSpider()

pipelines.py

@@ -160,4 +160,4 @@ def write2csv(response, filename_csv):    # 写入到csv文件
         print("sth wrong in pipelines.write2csv")
 
 if __name__ == "__main__":
-    write2csv()
+    pass