import scrapy

from homework.items import ArticleItem
from scrapy import Request


class ZhangqiaokeyanSpider(scrapy.Spider):
    # 爬虫文件的名称：就是爬虫源文件的唯一标识
    name = "zhangqiaokeyan"
    # 允许的域名：用来限定start_urls列表中哪些url可以进行请求发送
    # allowed_domains = ["www.zhangqiaokeyan.com"]
    # 起始的URL列表：该列表中存放的URL会被scrapy自动进行请求的发送
    # start_urls = ["https://www.zhangqiaokeyan.com/article_item/list-1_1/"]

    def start_requests(self):
        for page_num in range(10):
            # url = "https://www.zhangqiaokeyan.com/article_item/list-1_" + str(page_num)
            # 默认调用下面的parse进行爬虫
            yield Request(url=f"https://www.zhangqiaokeyan.com/article/list-1_{page_num}")

    def parse(self, response):
        # print(response)
        # 使用xpath进行数据解析
        item_list = response.xpath('//div[@class="yc_right"]/ul/li')
        for index, item in enumerate(item_list):
            list = item.xpath('./div')
            # 解决一个bug
            if len(list) == 1:
                div = 'div[1]'
            else:
                div = 'div[2]'
            # 创建item对象
            article_item = ArticleItem()
            # xpath返回的是列表，但是列表元素是Selector类型的对象
            # extract 可以将Selector对象中data参数存储的字符串提取出来
            # 可以使用[0]，但是可能会空指针异常，所以可以使用extract_first，如果没有数据，会返回None
            article_item['title'] = item.xpath('./' + div + '/a/text()').extract_first()
            #列表调用了extract，表示将列表中每一个Selector对象中data对应的字符串提取了出来
            keyword = item.xpath('./' + div + '/h2/span/em/text()').extract()
            article_item['keyword'] = ','.join(keyword)
            article_item['time'] = item.xpath('./' + div + '/h2/span/time/text()').extract_first()

            # 获取url进行进一步爬取
            detail_url = item.xpath('./' + div + '/a/@href').extract_first()

            yield Request(url='https://www.zhangqiaokeyan.com' + detail_url, callback=self.parse_detail, cb_kwargs={'item' : article_item})
            # yield article_item  # 生成item，将item提交给管道

    def parse_detail(self, response, **kwargs):
        base = '//div[@class="yc_atice"]/div[1]'
        # 获取数据
        article_item = kwargs['item']

        line = response.xpath(base + '/p/span[1]/span[1]/text()').extract_first()
        if line != None and line[0] == '作':
            info_list = list(line.split())
            article_item['author'] = info_list[0][3:]
            # 有可能没有来源
            if len(info_list) == 2:
                article_item['source'] = info_list[1][3:]
        elif line != None and line[0] == '来':
            article_item['source'] = line[3:]

        content = response.xpath(base + '/p//text()').extract()
        content_filter = [element if '\r\n\t' not in element else element.strip().replace('\r\n\t', '') for element in content]
        content_filter = [element for element in content_filter if element != '']
        content_filter = [element for element in content_filter if '\r\n' not in element]
        content_filter = ['\n' if element == ' ' or element == '\xa0' else element for element in content_filter]
        if content_filter != None and len(content_filter) > 0 and ('作者：' or '来源：') in content_filter[0]:
            content_filter[0] += '\n'
        article_item['content'] = ''.join(content_filter)

        print("题目：" + article_item.get('title', '无'))
        print("作者：" + article_item.get('author', '无'))
        print("来源：" + article_item.get('source', '无'))
        print("关键字：" + article_item.get('keyword', '无'))
        print("时间：" + article_item.get('time', '无'))
        print()
        # print(content)
        # print(content_filter)
        # print(article_item.get('content', ''))
        yield article_item