import json
import sys
import requests
from bs4 import BeautifulSoup
import random
import time

class IpManage(object):

    def __init__(self, max_ip_num):
        self.ip_pool = []   # ip池数组

        const_path = sys.path[0].split("quant-on-volume")[0] + "quant-on-volume"
        f = open(const_path + '\\const.json', "r", encoding="utf8")
        consts = json.loads(f.read())
        self.check_url = consts['check_url1']        # 校验ip是否可用的 url
        self.type = consts['type']                  # https / http
        self.pages = []                             # 已爬取的 page 数组
        self.max_ip_num = max_ip_num                # 要爬取的 ip 数量上限

    # 抓取单页的ip
    def craw_ips_by_page(self, page):
        url = "https://www.xicidaili.com/nn/%s" % page
        html_content = self.requests_get(url, 'html')
        
        if html_content == "0":
            print("获取ip池失败， 请检查网络设置， 并确认本机ip没有被封禁")
            return
        soup = BeautifulSoup(html_content, 'html.parser')
        all_trs = soup.find("table", id="ip_list").find_all('tr')
        for tr in all_trs[1:]:
            tds = tr.find_all('td')
            ip = {
                'ip': tds[1].get_text(),
                'port': tds[2].get_text(),
                'type': tds[5].get_text()
            }
            # 检查ip是否可用
            if self.check_ip(ip):
                self.ip_pool.append(ip)
            if len(self.ip_pool) >= self.max_ip_num:
                break
    
    # 获取 IP 池
    def craw_ips(self):
        page = self.get_random_page()
        self.pages.append(page)
        print("当前爬取的ip页码为： ", page)
        self.craw_ips_by_page(page)
        
        # 判断 ip 数量是否已足够
        if len(self.ip_pool) < self.max_ip_num:
            self.craw_ips()
        else:
            with open(sys.path[0] + "\\ip_pool.json", "w", encoding="utf-8") as f:
                json.dump(self.ip_pool, f)
            print("共抓取 %s 条ip" % len(self.ip_pool))
        
    # 检查代理ip是否可用
    def check_ip(self, ip):
        proxy_temp = {
            "http": "http://%s:%s" % (ip['ip'], ip['port']),
            "https": "http://%s:%s" % (ip['ip'], ip['port'])
        }
        show_info = self.check_url + "---" + "http://%s:%s  [%s]" % (ip['ip'], ip['port'], time.perf_counter())
        try:
            # print(show_info)
            res = requests.get(self.check_url, timeout=1, proxies=proxy_temp)
            if res.status_code == 200:
                print(show_info, ":SUCCESS")
                return True
            else:
                print(show_info, ":FAIL")
                return False
        except:
            print(show_info, ":FAIL")
            return False

    # get 请求
    def requests_get(self, url, type, data=None):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'
        }
        response = requests.get(url=url, headers=headers, data=data)
        if response.status_code == 200:
            if type == "img":
                # 获取图片
                return response.content
            if type == "html":
                html = response.content
                html_content = html.decode("utf-8", "ignore")
                return html_content 
            if type == "text":
                return response.text
        else:
            print("Request Falied For Code: %s" % response.status_code)
            return "0"

    # 生成随机页码
    def get_random_page(self):
        max_page = 3000
        page = 0
        while True:
            page = random.randint(0, max_page)
            if page not in self.pages:
                break
        return page

# if __name__ == "__main__":
#     ip_manage = IpManage(10)
#     ip_manage.craw_ips();