import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
from datetime import datetime
import random

def get_movie_data(page):
    # Use a list of different User-Agents to avoid detection
    user_agents = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
    ]
    
    url = f"https://movie.douban.com/top250?start={page}"
    headers = {
        'User-Agent': random.choice(user_agents),
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Connection': 'keep-alive',
        'Referer': 'https://movie.douban.com/top250',
        'Cache-Control': 'max-age=0',
        'Upgrade-Insecure-Requests': '1'
    }
    
    try:
        print(f"\n正在请求页面: {url}")
        response = requests.get(url, headers=headers, timeout=10)
        print(f"响应状态码: {response.status_code}")
        response.raise_for_status()
        
        # Check if we got a valid response
        if '验证码' in response.text:
            print(f"检测到验证码，等待10秒后重试...")
            time.sleep(10)
            return []
            
        soup = BeautifulSoup(response.text, 'html.parser')
        print(f"页面标题: {soup.title.string if soup.title else '无标题'}")
        
        movies = []
        
        # Print the HTML structure to debug
        print("\n页面结构:")
        print(soup.prettify()[:1000])  # Print first 1000 characters
        
        for item in soup.find_all('div', class_='item'):
            print("\n找到一个电影项:")
            movie_data = {}
            
            # Get title (both Chinese and English)
            title_elem = item.find('span', class_='title')
            if title_elem:
                print(f"标题: {title_elem.text}")
                movie_data['中文标题'] = title_elem.text.strip()
                
                # Also get English title if available
                eng_title_elem = item.find('span', class_='other')
                if eng_title_elem:
                    print(f"英文标题: {eng_title_elem.text}")
                    movie_data['英文标题'] = eng_title_elem.text.strip().replace('/', '').strip()
            
            # Get year
            year_elem = item.find('div', class_='bd').find('p')
            if year_elem:
                year_span = year_elem.find('span', class_='year')
                if year_span:
                    print(f"年份: {year_span.text}")
                    movie_data['年份'] = year_span.text.strip('()')
            
            # Get rating
            rating_elem = item.find('span', class_='rating_num')
            if rating_elem:
                print(f"评分: {rating_elem.text}")
                movie_data['评分'] = float(rating_elem.text)
            
            # Get number of votes
            star_div = item.find('div', class_='star')
            if star_div:
                votes_elem = star_div.find_all('span')
                if votes_elem and len(votes_elem) >= 4:
                    votes_text = votes_elem[3].text.strip()
                    if '人评价' in votes_text:
                        print(f"评价人数: {votes_text}")
                        try:
                            movie_data['评价人数'] = int(votes_text.replace('人评价', '').strip())
                        except ValueError:
                            print(f"警告: 无法转换评价人数: {votes_text}")
                            movie_data['评价人数'] = None
            
            # Get director and actors
            info_elem = item.find('div', class_='bd').find('p')
            if info_elem:
                info_text = info_elem.text.strip()
                if '导演' in info_text and '主演' in info_text:
                    director = info_text.split('导演: ')[1].split('主演: ')[0].strip()
                    actors = info_text.split('主演: ')[1].strip()
                    print(f"导演: {director}")
                    print(f"主演: {actors}")
                    movie_data['导演'] = director
                    movie_data['主演'] = actors
            
            if movie_data:  # Only add if we got some data
                print("\n收集到的电影数据:")
                print(movie_data)
                movies.append(movie_data)
        
        print(f"\n本页收集到的电影数量: {len(movies)}")
        return movies
        
    except Exception as e:
        print(f"获取电影数据时出错: {str(e)}")
        return []

def main():
    all_movies = []
    
    try:
        # Scrape multiple pages (each page contains 25 movies)
        for page in range(0, 250, 25):
            print(f"\n正在抓取第 {page//25 + 1} 页...")
            movies = get_movie_data(page)
            all_movies.extend(movies)
            time.sleep(random.uniform(1, 2))  # Reduced delay to 1-2 seconds
            
            # Print progress
            print(f"已收集电影数量: {len(all_movies)}")
            
            # If we got no movies, wait longer before next attempt
            if not movies:
                print("等待更长时间以避免检测...")
                time.sleep(random.uniform(5, 8))  # Reduced long wait to 5-8 seconds
        
        if all_movies:
            # Create DataFrame and save to CSV
            df = pd.DataFrame(all_movies)
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            df.to_csv(f'豆瓣电影_{timestamp}.csv', index=False, encoding='utf-8-sig')
            print(f"\n成功保存 {len(all_movies)} 部电影到 CSV 文件!")
        else:
            print("\n没有收集到任何电影数据。请检查错误信息。")
            
    except Exception as e:
        print(f"主程序出错: {str(e)}")

if __name__ == "__main__":
    main()