from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.edge.options import Options
import time
import json

# 设置 Edge 以无头模式运行
options = Options()
options.add_argument("--headless")
options.add_argument("--disable-gpu")

# 实例化浏览器
driver = webdriver.Edge(options=options)

# driver = webdriver.Edge()

ip_list = []
ip_port_dict = {}
proxies = []

a = 0
for i in range(1, 10):
    # 打开网页
    driver.get(f'https://www.kuaidaili.com/free/inha/{i}/')
    # 获取页面源代码
    html = driver.page_source
    # 使用BeautifulSoup解析HTML内容
    soup = BeautifulSoup(html, 'lxml')

    all_l = soup.select('tbody>tr')

    for all_2 in all_l:
        # 确保tr标签内确实有td标签
        if len(all_2.select('td')) >= 2:
            a += 1
            ip_l = all_2.select('td')[0].text.strip()
            port_l = all_2.select('td')[1].text.strip()
            # 分别添加到列表和字典中
            ip_list.append(ip_l)
            ip_port_dict[ip_l] = port_l
            print(f'IP代理池正在加载第:{a}个')
            time.sleep(0.4)

print('IP代理池加载完成')

# 构建proxies列表
for ip in ip_list:
    proxies.append({'http': f'http://{ip}:{ip_port_dict[ip]}'})

# 写入JSON文件
with open('ip代理池.json', 'w', encoding='utf-8') as f:
    json.dump(proxies, f, ensure_ascii=False, indent=4)
    print("写入文件完成：ip代理池.json")

# 写入txt文件
# with open('ip代理池.txt', 'w', encoding='UTF-8') as f:
#     for i in proxies:
#         f.write(str(i) + ',' + "\n")
#     print("写入文件完成：ip代理池.txt")

driver.quit()
# time.sleep(999)