milkSpider/downloader.py

# -*- coding: utf-8 -*-

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from lxml import etree
import random
import settings

headers = {
    'User-Agent': random.choice(settings.USER_AGENT)
}

def getsource(url):
    initChrome = Options()

    initChrome.add_argument('--no-sandbox')
    initChrome.add_argument('--headless')
    initChrome.add_argument('--disable-gpu')
    initChrome.add_argument("disable-cache")
    initChrome.add_argument('disable-infobars')
    initChrome.add_argument('log-level=3')    # INFO = 0 WARNING = 1 LOG_ERROR = 2 LOG_FATAL = 3 default is 0
    initChrome.add_experimental_option("excludeSwitches",['enable-automation','enable-logging'])

    driver = webdriver.Chrome(chrome_options = initChrome, executable_path = './chromedriver.exe')
    driver.implicitly_wait(10)
    driver.get(url)

    response = etree.HTML(driver.page_source)
    response = etree.tostring(response, encoding = "utf-8", pretty_print = True, method = "html")
    response = response.decode('utf-8')

    driver.close()
    return response