import pandas as pd
import requests
from bs4 import BeautifulSoup

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'


def get_world_comments():
    # 因为没设代理池，在复现时请开代理，不然有些外网url无法访问
    urls = [
        "https://www.nytimes.com/athletic/5646415/2024/07/25/2024-paris-olympics-ai-omega/",
        "https://www.news.ufl.edu/2024/07/ai-olympics/",
        "https://www.thepaper.cn/newsDetail_forward_28287864",
        "https://www.wicongress.org.cn/2024/zh/article/6268",
        "https://cn.technode.com/post/2024-08-06/paris-2024-esg-ai-digital-new-tech/",
    ]
    headers = {
        'User-Agent': USER_AGENT,
    }
    data = []
    for url in urls:
        response = requests.get(url, headers=headers,verify=False)
        if response.status_code == 200:
            soup = BeautifulSoup(response.content, 'html.parser')
            if url == "https://www.nytimes.com/athletic/5646415/2024/07/25/2024-paris-olympics-ai-omega/":
                strong_tags = soup.find_all('strong')
                count = 0
                for tag in strong_tags:
                    print(tag.get_text())
                    data.append({
                        'url': url,
                        'content': tag.get_text(),
                        'tag': 'strong',
                    })
                    count += 1
                    if count == 5:
                        break
                h1_tags = soup.find_all('h1')
                for tag in h1_tags:
                    print(tag.get_text())
                    data.append({
                        'url': url,
                        'content': tag.get_text(),
                        'tag': 'h1',
                    })
            if url == "https://www.news.ufl.edu/2024/07/ai-olympics/":
                h1_tags = soup.find_all('h1')
                for tag in h1_tags:
                    print(tag.get_text())
                    data.append({
                        'url': url,
                        'content': tag.get_text(),
                        'tag': 'h1',
                    })
                h2_tags = soup.find_all('h2')
                for tag in h2_tags:
                    print(tag.get_text())
                    data.append({
                        'url': url,
                        'content': tag.get_text(),
                        'tag': 'h2',
                    })
            if url == "https://www.thepaper.cn/newsDetail_forward_28287864":
                p_tags = soup.find_all('p')
                for tag in p_tags:
                    content = tag.get_text()
                    if content.startswith("技术"):
                        print(content)
                        data.append({
                            'url': url,
                            'content': content,
                            'tag': 'p',
                        })

            if url == "https://www.wicongress.org.cn/2024/zh/article/6268" :
                strong_tags = soup.find_all('strong')
                for tag in strong_tags:
                    print(tag.get_text())
                    data.append({
                        'url': url,
                        'content': tag.get_text(),
                        'tag': 'strong',
                    })
            if url == "https://cn.technode.com/post/2024-08-06/paris-2024-esg-ai-digital-new-tech/":
                h2_tags = soup.find_all('h2')
                for tag in h2_tags:
                    content = tag.get_text().strip()
                    if content:
                        print(content)
                        data.append({
                            'url': url,
                            'content': tag.get_text(),
                            'tag': 'h2',
                        })
        else:
            print(f"无法访问 {url}")
    df = pd.DataFrame(data)
    df.to_csv('world_comment.csv', index=False, encoding='utf-8')


if __name__ == '__main__':
    get_world_comments()