import requests
import re
import os #生成文件
import parsel #解析数据
from threading import Thread
from fake_useragent import UserAgent
import requests
from time import sleep
from queue import Queue


class MyThread(Thread):
    def __init__(self,url_queue_no,file_novel):
        Thread.__init__(self)
        self.url_queue_no = url_queue_no
        self.file_novel = file_novel
    def run(self):
        while not self.url_queue_no.empty():#有坑，会卡进程，但是影响不大
            url = self.url_queue_no.get(timeout = 1)
            headers = {
                'User-Agent': UserAgent().chrome
            }
            sleep(2)#防止被封IP,控制爬取速度
            try:
                response2 = requests.get(url, headers=headers, timeout=5)
                selector_html2 = parsel.Selector(response2.text)
                #getall返回文本列表
                title = selector_html2.css('#main > h1::text').get()
                title = re.sub(r'[\\/:*?"<>|]', '', title)#过滤字符
                #注意 由于window系统对于文件名敏感，不允许‘\ / : * ? " < > |’出现，需要过滤
                book = selector_html2.css('#content::text').getall()
                # print(book)
                book = '\n'.join(book)
                with open(self.file_novel + title + '.txt', mode = 'a', encoding='utf-8') as f:
                    f.write(book)
                    print(title + '下载中')
            except Exception as e:
                print(e)
                
def mkfile_novel(file_name):
    if not os.path.exists(file_name):
        os.makedirs(file_name)
        
def spider_novel_start(id,url_queue_no,file_novel):
    headers  ={
        'User-Agent': UserAgent().chrome
    }
    # id = input('书名的id号:')
    url = 'https://www.qb5.ch/book_' + id + '/'

    response = requests.get(url, headers=headers)

    selector_html1 = parsel.Selector(response.text)
    div1 = selector_html1.css('body > div.zjbox > dl > dd')
    # 方法2 bs4
    # bs4html = BeautifulSoup(response.text, 'lxml')
    # div2 = bs4html.select('body > div.listmain > dl > dd:nth-child(n+2) > a'
    for div in div1:
        href = div.css('a::attr(href)').get()
        if href == None:#避免空标签导致对请求地址的修改
            continue
        url2 = url + href
        url_queue_no.put(url2)#多线程
    for i in range(4):#线程数量
        t1 = MyThread(url_queue_no, file_novel)
        t1.start()
            
# if __name__ == '__main__':
#     url_queue_no = Queue()#用队列保证线程安全
#     # file_name = input('请输\入保存的文件夹名:') + '\\'
#     file_novel = 'end\\book\\'
#     mkfile_novel(file_novel)
#     # id = input('请输入小说的id号:')
#     id = '116659'
#     spider_novel_start(id,url_queue_no)