import requests from bs4 import BeautifulSoup import pandas as pd import random def requests_get(url): user_agent_list = [ \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", \ "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \ "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3" ] user_agent = random.choice(user_agent_list) headers = { "User-Agent": user_agent } try: response = requests.get(url, headers=headers) response.raise_for_status() return response.text except requests.exceptions.RequestException as e: print(f"请求失败: {e}") return None def get_movie_info(url): # BeautifulSoup的三个方法find: # 返回的是第一个符合条件的数据 # 。可以根据title的值来找到对应的标签对象 # ·根据class的值来找，需要在class下添加下划线find_a11 # ·返回的是一个列表，并且返回了匹配的所有的标签 # select # 有了如上基础，对源代码进行标签路径的分析，就可以取值了。 soup = BeautifulSoup(requests_get(url), 'html.parser') item_elements = soup.find_all("div", class_="item") # print(item_elements) movie_list = [] for item in item_elements: movie = {} # 获取电影排名 movie['ranking'] = item.find('em', class_='').get_text(strip=True) # 获取电影名称 movie['title'] = item.find('span', class_='title').get_text(strip=True) # 获取电影详情链接 movie['link'] = item.find('a', href=True)['href'] # 获取电影评分 movie['rating'] = item.find('span', class_='rating_num').get_text(strip=True) # 获取电影简介 try: movie['profile'] = item.find('span', class_='inq').get_text(strip=True) except AttributeError: movie['profile'] = '无' # 获取电影评价人数 span_list_len = len(item.find_all('span')) if span_list_len >= 6: num_reviews = item.find_all('span')[span_list_len - 2].get_text(strip=True).replace('人评价', '') movie['num_reviews'] = num_reviews # 获取电影海报链接 poster = item.find('img', class_='')['src'] movie['poster'] = poster # 获取电影类型与首映国家与上映年份 movie_infos = item.select('p', class_="")[0].text.strip() # print(movie_infos) if movie['title'] == '大闹天宫': movie['type'] = movie_infos.split('\n')[1].split('/')[4].strip() movie['country'] = movie_infos.split('\n')[1].split('/')[3].split(' ')[0].strip() else: movie['type'] = movie_infos.split('\n')[1].split('/')[2].strip() movie['country'] = movie_infos.split('\n')[1].split('/')[1].split(' ')[0].strip() # print(movie['type']) # print(movie['country']) movie['year'] = movie_infos.split('\n')[1].split('/')[0].strip(' ').strip('(中国大陆)') # print(movie['year']) # print(movie['country']) # 获取电影导演与上映时间 # 注意：因为时间想要一个年月日，所以直接进入详情页获取数据 # 且top页，一些导演名称太长，导致遮挡了主演的全称 # 这些都需要我们重新请求电影详情链接 movie_soup = BeautifulSoup(requests_get(movie['link']), 'html.parser') movie['director'] = movie_soup.find("a", rel="v:directedBy").get_text() # movie['year'] = movi_infose_soup.find("span", property="v:initialReleaseDate").get_text().split('(')[0] #获取片长 movie['time'] = movie_soup.find("span", property="v:runtime").get_text(strip=True).replace('片长','') #获取主演 #主演有些为空，所以需要我们定义异常处理 try: movie['actor'] = movie_soup.find("a", rel="v:starring").get_text(strip=True) except AttributeError: movie['actor'] = '无' # print(movie) movie_list.append(movie) return movie_list if __name__ == '__main__': print(requests_get('https://movie.douban.com/top250')) base_url = 'https://movie.douban.com/top250' movie_data = [] for page in range(10): start = page * 25 # url ='https://movie.douban.com/top250?start=50' url = f'{base_url}?start={start}' print(f"开始爬取第 {page + 1} 页: {url}") movies = get_movie_info(url) movie_data.extend(movies) print('爬取完成') df = pd.DataFrame(movie_data) # # 将数据保存为CSV文件 csv_file = 'flaskProject/data/export.csv' df.to_csv(csv_file, index=False, encoding='utf-8') print('存储完成')