milkSpider/downloader.py

# -*- coding: utf-8 -*-

import json
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from lxml import etree
import random
import settings
import time

headers = {
    'User-Agent': random.choice(settings.USER_AGENT)
}

def getsource(url):
    initChrome = Options()

    initChrome.add_argument('--no-sandbox')
    initChrome.add_argument('--headless')
    initChrome.add_argument('--disable-gpu')
    initChrome.add_argument("disable-cache")
    initChrome.add_argument('disable-infobars')
    initChrome.add_argument('log-level=3')    # INFO = 0 WARNING = 1 LOG_ERROR = 2 LOG_FATAL = 3 default is 0
    initChrome.add_experimental_option("excludeSwitches",['enable-automation','enable-logging'])

    driver = webdriver.Chrome(chrome_options = initChrome, executable_path = './chromedriver.exe')
    driver.implicitly_wait(10)
    driver.get(url)

    response = etree.HTML(driver.page_source)
    response = etree.tostring(response, encoding = "utf-8", pretty_print = True, method = "html")
    response = response.decode('utf-8')

    driver.close()
    return response

def manmanbuy(url):
    initChrome = Options()
    '''
    initChrome.add_argument('--no-sandbox')
    initChrome.add_argument('--headless')
    initChrome.add_argument('--disable-gpu')
    initChrome.add_argument("disable-cache")
    initChrome.add_argument('disable-infobars')
    initChrome.add_argument('log-level=3')    # INFO = 0 WARNING = 1 LOG_ERROR = 2 LOG_FATAL = 3 default is 0
    initChrome.add_experimental_option("excludeSwitches",['enable-automation','enable-logging'])
    '''

    # driver = webdriver.Chrome(chrome_options = initChrome, executable_path = './chromedriver.exe')
    driver = webdriver.Chrome(executable_path = './chromedriver.exe')
    # driver.get(url)
    # time.sleep(10)
    with open('cookies.json', 'r', encoding='utf-8') as fd:
            listCookies = json.loads(fd.read())
    for cookie in listCookies:
        cookies = {
            'domain': cookie['domain'],
            'httpOnly': cookie['httpOnly'],
            'name':cookie['name'],
            'path':'/',
            'secure': cookie['secure'],
            'value':cookie['value'],
        }
        print(cookies)
        driver.add_cookie(cookies)
    driver.get(url)
    time.sleep(10)
    exit()
    driver.implicitly_wait(10)
    driver.get(url)

    response = etree.HTML(driver.page_source)
    response = etree.tostring(response, encoding = "utf-8", pretty_print = True, method = "html")
    response = response.decode('utf-8')

    driver.close()
    return response

def saveCookies(driver):
    jsonCookies = json.dumps(driver.get_cookies())
    with open('cookies.json', 'w', encoding='utf-8') as fd:
        fd.write(jsonCookies)
    fd.close()


def buy():
    jdurl = "https://item.jd.com/10047511027349.html"
    url = "https://tool.manmanbuy.com/HistoryLowest.aspx?url=" + jdurl
    # print(url)
    response = manmanbuy(url)
    print(response)

if __name__ == "__main__":
    buy()