import requests
import csv
from bs4 import BeautifulSoup

url = "https://www.biquge7.xyz/50227"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0"
}


# 1. 编写函数get_source
def get_source(url, headers):
        response = requests.get(url, headers=headers)
        return response.text
        return None

    # 2. 解析网页源代码，获取原序和所有章节的网页地址


def parse_chapter_links(source):
    soup = BeautifulSoup(source, 'html.parser')
    # 假设章节链接在a标签的href属性中，并且这些a标签在某个特定的div或列表中
    # 这里的CSS选择器需要根据实际的HTML结构来修改
    chapter_links = [base_url + link.get('href') for link in soup.select('div.list > ul > li > a')]
    #print(chapter_links)
    return chapter_links


# 3. 访问每一章节页面，获取每一章节的网页源代码
def fetch_chapter_sources(chapter_links, headers):
    chapter_sources = {}
    for link in chapter_links:
        #print(link)
        source = get_source(link, headers)
        if source:
            chapter_sources[link] = source
    #print(chapter_sources)
    return chapter_sources


# 4. 解析每一章小说的正文，和每章小说的标题
def parse_chapter_content(source):
    soup = BeautifulSoup(source, 'html.parser')
    # 假设标题和正文分别位于特定的HTML元素中
    # 这里的CSS选择器需要根据实际的HTML结构来修改
    title = soup.select_one('h1.list_tit').get_text(strip=True)
    content = soup.select_one('div.text').get_text(strip=True, separator='\n')
    print(title)
    return title, content


# 5. 将每一章节的小说正文存储到本地的txt文件中
def save_chapters_to_txt(chapters):
    for link, source in chapters.items():
        title, content = parse_chapter_content(source)
        with open(f'{title}.txt', 'w', encoding='utf-8') as f:
            f.write(content)

        # 6. 将每章的网页地址、标题和正文长度写入到csv中


def save_to_csv(chapters, csv_filename):
    with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
        fieldnames = ['链接', '标题', '正文长度']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        for link, source in chapters.items():
            title, content = parse_chapter_content(source)
            writer.writerow({'链接': link, '标题': title, '正文长度': len(content)})

        # 主程序


if __name__ == "__main__":
    base_url = "https://www.biquge7.xyz"
    source = get_source(url, headers)
    # print(source)
    if source:
        chapter_links = parse_chapter_links(source)
        # print(chapter_links)
        chapters = fetch_chapter_sources(chapter_links, headers)
        # print(chapters)
        save_chapters_to_txt(chapters)
        save_to_csv(chapters, '偷偷藏不住.csv')