|
|
import scrapy
|
|
|
import json
|
|
|
import re
|
|
|
from spider.spiders.Redis_Con import Redis_Con
|
|
|
|
|
|
|
|
|
class DoubanSpider(scrapy.Spider):
|
|
|
name = 'douban'
|
|
|
allowed_domains = ['movie.douban.com']
|
|
|
|
|
|
def start_requests(self):
|
|
|
|
|
|
#urls=['https://movie.douban.com/j/chart/top_list?type=0&interval_id=100%3A90&action=&start=0&limit=100', 'https://movie.douban.com/j/chart/top_list?type=0&interval_id=95%3A85&action=&start=0&limit=100']
|
|
|
#urls = [father_url.format(t, max, min) for t in range(1) for max, min in zip(range(100, 9, -5), range(90, -1, -5))]
|
|
|
#print(urls)
|
|
|
|
|
|
f = Redis_Con()
|
|
|
url = f.geturl()
|
|
|
print('当前请求的URL----->'+url)
|
|
|
yield scrapy.http.Request(url=url, callback=self.parse,dont_filter=True)
|
|
|
|
|
|
|
|
|
def parse(self, response):
|
|
|
|
|
|
|
|
|
result = response.xpath('/html/body/p/text()').extract()
|
|
|
# '\"RecruitPostName\":\".*?\"'
|
|
|
print(result)
|
|
|
'''json转义'''
|
|
|
fin = re.findall(r'{.*?}', str(result).replace('\\\\','\\')[1:-1])
|
|
|
|
|
|
|
|
|
for x in fin:
|
|
|
print (x)
|
|
|
need = {}
|
|
|
d_json = json.loads(x)
|
|
|
need['评分'] = d_json['score']
|
|
|
need['电影名称'] = d_json['title']
|
|
|
need['排名'] = d_json['rank']
|
|
|
need['类型'] = d_json['types']
|
|
|
need['演员'] = d_json['actors']
|
|
|
need['国家'] = d_json['regions']
|
|
|
need['上映时间'] = d_json['release_date']
|
|
|
need['type'] = re.findall('type=.*?&',response.url)[0][5:-1]
|
|
|
#正则获取喜爱度区间,便于之后区分
|
|
|
need['喜爱区间'] = re.findall('interval_id=.*?&',response.url)[0][12:-1]
|
|
|
print('###############yield need is OK ###############')
|
|
|
if result[0] != '[]':
|
|
|
yield need
|
|
|
|
|
|
# 检查当前页面是否有信息
|
|
|
#print(result)
|
|
|
if result[0] != '[]':
|
|
|
# 构造下一页的url，发送下次请求
|
|
|
nexturl=self.replace_page(response.url)
|
|
|
# 在parse方法中发送请求，请求完成后调用parse方法。
|
|
|
headers = {
|
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
|
|
|
'Referer': 'https://accounts.douban.com/'
|
|
|
}
|
|
|
#print(nexturl)
|
|
|
yield scrapy.Request(nexturl,callback=self.parse)
|
|
|
|
|
|
def replace_page(self,s):
|
|
|
index = s.find('start=')
|
|
|
# 正则拿出数字
|
|
|
res = re.findall('start=.*?&', s)
|
|
|
# 修改后放回去(+1000)
|
|
|
fin = res[0][6:-1]
|
|
|
sum = str(int(fin) + 1000)
|
|
|
new_s = 'start=' + sum
|
|
|
return s.replace(res[0][0:-1], new_s)
|
|
|
|
|
|
|
