|
|
import scrapy
|
|
|
import requests
|
|
|
from PythonScrapyWeather.items import PythonscrapyweatherItem
|
|
|
"""
|
|
|
多页面爬取有两种形式。
|
|
|
1）从某一个或者多个主页中获取多个子页面的url列表，parse()函数依次爬取列表中的各个子页面。
|
|
|
2）从递归爬取，这个相对简单。在scrapy中只要定义好初始页面以及爬虫规则rules，就能够实现自动化的递归爬取。
|
|
|
"""
|
|
|
|
|
|
class WeathersSpider(scrapy.Spider):
|
|
|
name = 'Weathers'
|
|
|
allowed_domains = ['tianqi.com']
|
|
|
start_urls = ['http://tianqi.com/']
|
|
|
|
|
|
def parse(self, response):
|
|
|
url = "https://www.tianqi.com"
|
|
|
allProvince_list = response.xpath('//div[@class="tqqgsf"]/p/a/text()').extract()
|
|
|
allCity_list = response.xpath('//div[@class="tqqgsf"]/p/a/@href').extract()
|
|
|
print("*************allCity_list*************", allCity_list)
|
|
|
for city_name in allCity_list:
|
|
|
city_url = city_name
|
|
|
print("*************city_url*************", city_url)
|
|
|
# 再通过省、直辖市的URL请求每个省所有市的URL（请求）
|
|
|
yield scrapy.Request(city_url, callback=self.subpage_content)
|
|
|
|
|
|
# 获取到每个省所有市的URL（响应）
|
|
|
def subpage_content(self, response):
|
|
|
print("response", response.status)
|
|
|
try:
|
|
|
# 实例化对象item
|
|
|
item = PythonscrapyweatherItem()
|
|
|
# 使用xpath方法遍历HTML所需要的元素
|
|
|
province_Data = response.xpath('//div[@class="left"]/div[5]')
|
|
|
# print("*************province_Data*************", province_Data)
|
|
|
for province_name in province_Data:
|
|
|
item["province_Name"] = province_name.xpath('//div[@class="left"]/div[5]/div/h2/text()').extract()[0]
|
|
|
province_Name = item["province_Name"]
|
|
|
print("*****************province_Name*******************", province_Name)
|
|
|
#获取每个省内的市区跳转
|
|
|
|
|
|
province_url = response.xpath('/html/body/div[7]/div[1]/div[5]/ul/li/a[1]/@href').extract()
|
|
|
print(province_url)
|
|
|
for city_url1 in province_url:
|
|
|
url_test = 'http://tianqi.com/' + city_url1
|
|
|
print(url_test)
|
|
|
yield scrapy.Request(url_test, callback=self.subpage_content_1)
|
|
|
# return item
|
|
|
|
|
|
# requests.get('http://tianqi.com/' + city_url)
|
|
|
|
|
|
# weather_Detail_Data = response.xpath('//div[@class="left"]')
|
|
|
# for weather_detail in weather_Detail_Data:
|
|
|
# # 获取item对象的属性值
|
|
|
# item["city_Name"] = weather_detail.xpath('dl/dd[@class ="name"]/h1/text()').extract()[0]
|
|
|
# city_Name1 = item["city_Name"]
|
|
|
# print("*************************************************111111", city_Name1)
|
|
|
# item["date"] = weather_detail.xpath('dl/dd[@class="week"]/text()').extract()[0]
|
|
|
# item["temperature"] = weather_detail.xpath('dl/dd[@class="weather"]/span/text()').extract()[0]
|
|
|
# item["weather_condition"] = weather_detail.xpath('dl/dd[@class="weather"]/span/b/text()').extract()[0]
|
|
|
# item["air_quality"] = weather_detail.xpath('dl/dd[@class="kongqi"]/h5/text()').extract()[0]
|
|
|
# return item
|
|
|
except:
|
|
|
print(response.status)
|
|
|
pass
|
|
|
|
|
|
def subpage_content_1(self, response):
|
|
|
print("response2", response.status)
|
|
|
try:
|
|
|
# 实例化对象item
|
|
|
item = PythonscrapyweatherItem()
|
|
|
weather_Detail_Data = response.xpath('//div[@class="left"]')
|
|
|
for weather_detail in weather_Detail_Data:
|
|
|
# 获取item对象的属性值
|
|
|
item["city_Name"] = weather_detail.xpath('dl/dd[@class ="name"]/h1/text()').extract()[0]
|
|
|
# city_Name1 = item["city_Name"]
|
|
|
# print("*************************************************111111", city_Name1)
|
|
|
item["date"] = weather_detail.xpath('dl/dd[@class="week"]/text()').extract()[0]
|
|
|
item["temperature"] = weather_detail.xpath('dl/dd[@class="weather"]/span/text()').extract()[0]
|
|
|
item["weather_condition"] = weather_detail.xpath('dl/dd[@class="weather"]/span/b/text()').extract()[0]
|
|
|
item["air_quality"] = weather_detail.xpath('dl/dd[@class="kongqi"]/h5/text()').extract()[0]
|
|
|
return item
|
|
|
except:
|
|
|
print(response.status)
|
|
|
pass
|
|
|
|
