|
|
import pymysql as pymysql
|
|
|
import requests
|
|
|
from bs4 import BeautifulSoup
|
|
|
import re
|
|
|
from lxml import etree
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def getDB():
|
|
|
'''连接数据库'''
|
|
|
db = pymysql.connect(host='localhost',user='root',password='shengji.',database='douban',)
|
|
|
return db
|
|
|
|
|
|
|
|
|
def Agent_info():
|
|
|
''' 用于保存cookies，url，user-agent信息'''
|
|
|
headers = {
|
|
|
'Cookie': 'douban-fav-remind=1; bid=LP2o54Mcp34; ll="118268"; ap_v=0,6.0',
|
|
|
'Host':'movie.douban.com',
|
|
|
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62'
|
|
|
}
|
|
|
return headers
|
|
|
#获取电影详情url地址列表和外国电影名字
|
|
|
def get_url(url):
|
|
|
print('抓取网址：',url)
|
|
|
headers = Agent_info()
|
|
|
request = requests.get(url,headers=headers)
|
|
|
soup = BeautifulSoup(request.text,'lxml')
|
|
|
pic = soup.find_all(attrs={'class':'pic'})
|
|
|
film_urls = [] #电影详情url地址列表
|
|
|
for x in pic:
|
|
|
href = x.a.get('href')
|
|
|
film_urls.append(href)
|
|
|
movie_list = [] #外国电影名字
|
|
|
div_list = soup.find_all('div',class_='hd')
|
|
|
for each in div_list:
|
|
|
movie = each.a.contents[3].text.strip()
|
|
|
movie = movie[2:]
|
|
|
movie_list.append(movie)
|
|
|
|
|
|
return film_urls,movie_list
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#获取电影信息
|
|
|
def get_url_info(film_url,film_name_en,id):
|
|
|
print('抓取网址：',film_url)
|
|
|
headers = Agent_info()
|
|
|
request = requests.get(film_url, headers=headers)
|
|
|
soup = BeautifulSoup(request.text, 'lxml')
|
|
|
#排名
|
|
|
ranks = soup.find(attrs={'class':"top250-no"}).text.split(".")[1]
|
|
|
#电影中文名
|
|
|
film_name = soup.find(attrs={'property':'v:itemreviewed'}).text.split(' ')[0]
|
|
|
#导演
|
|
|
director = soup.find(attrs={'id':"info"}).text.split('\n')[1].split(':')[1].strip()
|
|
|
#编剧
|
|
|
scriptwriter = soup.find(attrs={'id':"info"}).text.split('\n')[2].split(':')[1].strip()
|
|
|
#主演
|
|
|
actor = soup.find(attrs={'id':"info"}).text.split('\n')[3].split(':')[1].strip()
|
|
|
# 类型
|
|
|
filmtype = soup.find(attrs={'id': "info"}).text.split('\n')[4].split(':')[1].strip()
|
|
|
types = filmtype.split("/")
|
|
|
if soup.find(attrs={'id': "info"}).text.split('\n')[5].split(':')[0] =='官方网站':
|
|
|
# 制片国家/地区
|
|
|
area = soup.find(attrs={'id': "info"}).text.split('\n')[6].split(':')[1].strip()
|
|
|
# 语言
|
|
|
language = soup.find(attrs={'id': "info"}).text.split('\n')[7].split(':')[1].strip()
|
|
|
# 上映日期
|
|
|
initialrReleaseDate = soup.find(attrs={'id': "info"}).text.split('\n')[8].split(':')[1].split('(')[0].strip()
|
|
|
else:
|
|
|
# 制片国家/地区
|
|
|
area = soup.find(attrs={'id': "info"}).text.split('\n')[5].split(':')[1].strip()
|
|
|
# 语言
|
|
|
language = soup.find(attrs={'id': "info"}).text.split('\n')[6].split(':')[1].strip()
|
|
|
# 上映日期
|
|
|
initialrReleaseDate = soup.find(attrs={'id': "info"}).text.split('\n')[7].split(':')[1].split('(')[0].strip()
|
|
|
|
|
|
# 片长
|
|
|
runtime = soup.find(attrs={'property': "v:runtime"}).text
|
|
|
# 评分(平均分)
|
|
|
rating_num = soup.find(attrs={'property': "v:average"}).text
|
|
|
# 五星评分比例
|
|
|
stars5_rating_per = soup.find(attrs={'class': "rating_per"}).text
|
|
|
# 评价人数
|
|
|
rating_people = soup.find(attrs={'property': "v:votes"}).text
|
|
|
#剧情简介
|
|
|
summary = soup.find(attrs={'property': "v:summary"}).text
|
|
|
summary = pymysql.converters.escape_string(summary)
|
|
|
|
|
|
#存到数据库
|
|
|
sql = 'insert into movies(film_name,director,scriptwriter,actor,filmtype,area,language,initialrReleaseDate,ranks,runtime,rating_num,stars5_rating_per,rating_people,summary,film_name_en,links) values("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}");'.format(film_name,director,scriptwriter,actor,filmtype,area,language,initialrReleaseDate,ranks,runtime,rating_num,stars5_rating_per,rating_people,summary,film_name_en,film_url)
|
|
|
db = getDB()
|
|
|
try:
|
|
|
cursor = db.cursor()
|
|
|
cursor.execute(sql)
|
|
|
cursor.execute('insert into moviehash(movieid) values("{}")'.format(id))
|
|
|
for j in range(len(types)):
|
|
|
cursor.execute('insert into movietype(movieid,filmtype) values("{}","{}")'.format(id,types[j].strip()))
|
|
|
db.commit()
|
|
|
except Exception as e:
|
|
|
print(e)
|
|
|
db.rollback()
|
|
|
cursor.close()
|
|
|
db.close()
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
print('开始抓取')
|
|
|
db = getDB()
|
|
|
cursor = db.cursor()
|
|
|
for i in range(0,250,25):
|
|
|
film_urls, movie_list = get_url("https://movie.douban.com/top250?start="+str(i)+"&filter=")
|
|
|
for film_url in range(len(film_urls)):
|
|
|
id = re.search('\d\d+',film_urls[film_url]).group()
|
|
|
sql = 'select movieid from moviehash where movieid = {}'.format(id)
|
|
|
cursor.execute(sql)
|
|
|
data = cursor.fetchall()
|
|
|
if not data:
|
|
|
get_url_info(film_urls[film_url],movie_list[film_url],id)
|
|
|
# get_url_info("https://movie.douban.com/subject/1291561/","111","1291561") |
