You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
31 lines
1.0 KiB
31 lines
1.0 KiB
3 years ago
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
from selenium import webdriver
|
||
|
|
from selenium.webdriver.chrome.options import Options
|
||
|
|
from lxml import etree
|
||
|
|
|
||
|
|
headers = {
|
||
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36'
|
||
|
|
}
|
||
|
|
|
||
|
|
def getsource(url):
|
||
|
|
init = Options()
|
||
|
|
|
||
|
|
init.add_argument('--no-sandbox')
|
||
|
|
init.add_argument('--headless')
|
||
|
|
init.add_argument('--disable-gpu')
|
||
|
|
init.add_argument("disable-cache")
|
||
|
|
init.add_argument('disable-infobars')
|
||
|
|
init.add_argument('log-level=3') # INFO = 0 WARNING = 1 LOG_ERROR = 2 LOG_FATAL = 3 default is 0
|
||
|
|
init.add_experimental_option("excludeSwitches",['enable-automation','enable-logging'])
|
||
|
|
|
||
|
|
driver = webdriver.Chrome(chrome_options = init)
|
||
|
|
driver.implicitly_wait(10)
|
||
|
|
driver.get(url)
|
||
|
|
|
||
|
|
response = etree.HTML(driver.page_source)
|
||
|
|
response = etree.tostring(response, encoding = "utf-8", pretty_print = True, method = "html")
|
||
|
|
response = response.decode('utf-8')
|
||
|
|
|
||
|
|
driver.close()
|
||
|
|
return response
|