#!/usr/bin/env python # encoding: utf-8 import re from lxml import etree from scrapy import Spider from scrapy.crawler import CrawlerProcess from scrapy.selector import Selector from scrapy.http import Request from scrapy.utils.project import get_project_settings from bot.items import TweetsItem, InformationItem, RelationshipsItem, CommentItem from bot.spiders.utils import time_fix from datetime import datetime class WeiboSpider(Spider): name = "weibo_spider" base_url = "https://weibo.cn" def start_requests(self): import pymysql.cursors # 连接数据库 connect = pymysql.connect( host='localhost', # 数据库地址 port=3306, # 数据库端口 db='WeiboAnalysisSystem', # 数据库名 user='root', # 数据库用户名 passwd='1234', # 数据库密码 charset='utf8mb4', # 编码方式 use_unicode=True) # 通过cursor执行增删查改 cursor = connect.cursor() cursor.execute("""SELECT uid from scrapydapi_target WHERE isScrapy=0""") uids = cursor.fetchall() start_uids = uids print('uid:::',uids) for uid in start_uids: cursor.execute("""UPDATE scrapydapi_target set isScrapy=1 WHERE uid=%s""" % uid) cursor.execute("commit") yield Request(url="https://weibo.cn/%s/info" % uid, callback=self.parse_information) connect.close() cursor.close() def parse_information(self, response): """ 抓取个人信息 """ information_item = InformationItem() information_item['crawl_time'] = datetime.now() selector = Selector(response) information_item['_id'] = re.findall('(\d+)/info', response.url)[0] image = selector.xpath('body/div[@class="c"]//img/@src').extract() text1 = ";".join(selector.xpath('body/div[@class="c"]//text()').extract()) # 获取标签里的所有text() nick_name = re.findall('昵称;?[：:]?(.*?);', text1) gender = re.findall('性别;?[：:]?(.*?);', text1) place = re.findall('地区;?[：:]?(.*?);', text1) briefIntroduction = re.findall('简介;?[：:]?(.*?);', text1) birthday = re.findall('生日;?[：:]?(.*?);', text1) sex_orientation = re.findall('性取向;?[：:]?(.*?);', text1) sentiment = re.findall('感情状况;?[：:]?(.*?);', text1) vip_level = re.findall('会员等级;?[：:]?(.*?);', text1) authentication = re.findall('认证;?[：:]?(.*?);', text1) labels = re.findall('标签;?[：:]?(.*?)更多>>', text1) if image and image[0]: information_item["Image"] = image[0] if nick_name and nick_name[0]: information_item["nick_name"] = nick_name[0].replace(u"\xa0", "") if gender and gender[0]: information_item["gender"] = gender[0].replace(u"\xa0", "") if place and place[0]: place = place[0].replace(u"\xa0", "").split(" ") information_item["province"] = place[0] if len(place) > 1: information_item["city"] = place[1] if briefIntroduction and briefIntroduction[0]: information_item["brief_introduction"] = briefIntroduction[0].replace(u"\xa0", "") if birthday and birthday[0]: try: birthday = datetime.datetime.strptime(birthday[0], "%Y-%m-%d") information_item['birthday'] = birthday - datetime.timedelta(hours=8) except Exception: information_item['constellation'] = birthday[0] # 有可能是星座，而非时间 if sex_orientation and sex_orientation[0]: if sex_orientation[0].replace(u"\xa0", "") == gender[0]: information_item["sex_orientation"] = "同性恋" else: information_item["sex_orientation"] = "异性恋" if sentiment and sentiment[0]: information_item["sentiment"] = sentiment[0].replace(u"\xa0", "") if vip_level and vip_level[0]: information_item["vip_level"] = vip_level[0].replace(u"\xa0", "") if authentication and authentication[0]: information_item["authentication"] = authentication[0].replace(u"\xa0", "") if labels and labels[0]: information_item["labels"] = labels[0].replace(u"\xa0", ",").replace(';', '').strip(',') request_meta = response.meta request_meta['item'] = information_item yield Request(self.base_url + '/u/{}'.format(information_item['_id']), callback=self.parse_further_information, meta=request_meta, dont_filter=True, priority=1) def parse_further_information(self, response): text = response.text information_item = response.meta['item'] tweets_num = re.findall('微博\[(\d+)\]', text) if tweets_num: information_item['tweets_num'] = int(tweets_num[0]) follows_num = re.findall('关注\[(\d+)\]', text) if follows_num: information_item['follows_num'] = int(follows_num[0]) fans_num = re.findall('粉丝\[(\d+)\]', text) if fans_num: information_item['fans_num'] = int(fans_num[0]) yield information_item # 获取该用户微博 yield Request(url=self.base_url + '/{}/profile?page=1'.format(information_item['_id']), callback=self.parse_tweet, priority=1) # # 获取关注列表 # yield Request(url=self.base_url + '/{}/follow?page=1'.format(information_item['_id']), # callback=self.parse_follow, # dont_filter=True) # # 获取粉丝列表 # yield Request(url=self.base_url + '/{}/fans?page=1'.format(information_item['_id']), # callback=self.parse_fans, # dont_filter=True) def parse_tweet(self, response): if response.url.endswith('page=1'): # 如果是第1页，一次性获取后面的所有页 all_page = re.search(r'/> 1/(\d+)页', response.text) if all_page: all_page = all_page.group(1) all_page = int(all_page) for page_num in range(2, all_page + 1): page_url = response.url.replace('page=1', 'page={}'.format(page_num)) yield Request(page_url, self.parse_tweet, dont_filter=True, meta=response.meta) """ 解析本页的数据 """ tree_node = etree.HTML(response.body) tweet_nodes = tree_node.xpath('//div[@class="c" and @id]') for tweet_node in tweet_nodes: try: tweet_item = TweetsItem() tweet_item['crawl_time'] = datetime.now() tweet_repost_url = tweet_node.xpath('.//a[contains(text(),"转发[")]/@href')[0] user_tweet_id = re.search(r'/repost/(.*?)\?uid=(\d+)', tweet_repost_url) tweet_item['weibo_url'] = 'https://weibo.com/{}/{}'.format(user_tweet_id.group(2), user_tweet_id.group(1)) tweet_item['user_id'] = user_tweet_id.group(2) tweet_item['_id'] = '{}_{}'.format(user_tweet_id.group(2), user_tweet_id.group(1)) create_time_info = tweet_node.xpath('.//span[@class="ct"]/text()')[-1] if "来自" in create_time_info: tweet_item['created_at'] = time_fix(create_time_info.split('来自')[0].strip()) else: tweet_item['created_at'] = time_fix(create_time_info.strip()) like_num = tweet_node.xpath('.//a[contains(text(),"赞[")]/text()')[-1] tweet_item['like_num'] = int(re.search('\d+', like_num).group()) repost_num = tweet_node.xpath('.//a[contains(text(),"转发[")]/text()')[-1] tweet_item['repost_num'] = int(re.search('\d+', repost_num).group()) comment_num = tweet_node.xpath( './/a[contains(text(),"评论[") and not(contains(text(),"原文"))]/text()')[-1] tweet_item['comment_num'] = int(re.search('\d+', comment_num).group()) tweet_content_node = tweet_node.xpath('.//span[@class="ctt"]')[0] # 检测由没有阅读全文: all_content_link = tweet_content_node.xpath('.//a[text()="全文"]') if all_content_link: all_content_url = self.base_url + all_content_link[0].xpath('./@href')[0] yield Request(all_content_url, callback=self.parse_all_content, meta={'item': tweet_item}, priority=1) else: all_content = tweet_content_node.xpath('string(.)').replace('\u200b', '').strip() tweet_item['content'] = all_content[1:] yield tweet_item # 抓取该微博的评论信息 # comment_url = self.base_url + '/comment/' + tweet_item['weibo_url'].split('/')[-1] + '?page=1' # yield Request(url=comment_url, callback=self.parse_comment, meta={'weibo_url': tweet_item['weibo_url']}) except Exception as e: self.logger.error(e) def parse_all_content(self, response): # 有阅读全文的情况，获取全文 tree_node = etree.HTML(response.body) tweet_item = response.meta['item'] content_node = tree_node.xpath('//div[@id="M_"]//span[@class="ctt"]')[0] all_content = content_node.xpath('string(.)').replace('\u200b', '').strip() tweet_item['content'] = all_content[1:] yield tweet_item def parse_follow(self, response): """ 抓取关注列表 """ # 如果是第1页，一次性获取后面的所有页 if response.url.endswith('page=1'): all_page = re.search(r'/> 1/(\d+)页', response.text) if all_page: all_page = all_page.group(1) all_page = int(all_page) for page_num in range(2, all_page + 1): page_url = response.url.replace('page=1', 'page={}'.format(page_num)) yield Request(page_url, self.parse_follow, dont_filter=True, meta=response.meta) selector = Selector(response) urls = selector.xpath('//a[text()="关注他" or text()="关注她" or text()="取消关注"]/@href').extract() uids = re.findall('uid=(\d+)', ";".join(urls), re.S) ID = re.findall('(\d+)/follow', response.url)[0] for uid in uids: relationships_item = RelationshipsItem() relationships_item['crawl_time'] = datetime.now() relationships_item["fan_id"] = ID relationships_item["followed_id"] = uid relationships_item["_id"] = ID + '-' + uid yield relationships_item def parse_fans(self, response): """ 抓取粉丝列表 """ # 如果是第1页，一次性获取后面的所有页 if response.url.endswith('page=1'): all_page = re.search(r'/> 1/(\d+)页', response.text) if all_page: all_page = all_page.group(1) all_page = int(all_page) for page_num in range(2, all_page + 1): page_url = response.url.replace('page=1', 'page={}'.format(page_num)) yield Request(page_url, self.parse_fans, dont_filter=True, meta=response.meta) selector = Selector(response) urls = selector.xpath('//a[text()="关注他" or text()="关注她" or text()="移除"]/@href').extract() uids = re.findall('uid=(\d+)', ";".join(urls), re.S) ID = re.findall('(\d+)/fans', response.url)[0] for uid in uids: relationships_item = RelationshipsItem() relationships_item['crawl_time'] = datetime.now() relationships_item["fan_id"] = uid relationships_item["followed_id"] = ID relationships_item["_id"] = uid + '-' + ID yield relationships_item def parse_comment(self, response): # 如果是第1页，一次性获取后面的所有页 if response.url.endswith('page=1'): all_page = re.search(r'/> 1/(\d+)页', response.text) if all_page: all_page = all_page.group(1) all_page = int(all_page) for page_num in range(2, all_page + 1): page_url = response.url.replace('page=1', 'page={}'.format(page_num)) yield Request(page_url, self.parse_comment, dont_filter=True, meta=response.meta) selector = Selector(response) comment_nodes = selector.xpath('//div[@class="c" and contains(@id,"C_")]') for comment_node in comment_nodes: comment_user_url = comment_node.xpath('.//a[contains(@href,"/u/")]/@href').extract_first() if not comment_user_url: continue comment_item = CommentItem() comment_item['crawl_time'] = datetime.now() comment_item['weibo_url'] = response.meta['weibo_url'] comment_item['comment_user_id'] = re.search(r'/u/(\d+)', comment_user_url).group(1) comment_item['content'] = comment_node.xpath('.//span[@class="ctt"]').xpath('string(.)').extract_first() comment_item['_id'] = comment_node.xpath('./@id').extract_first() created_at = comment_node.xpath('.//span[@class="ct"]/text()').extract_first() comment_item['created_at'] = time_fix(created_at.split('\xa0')[0]) yield comment_item if __name__ == "__main__": process = CrawlerProcess(get_project_settings()) process.crawl('weibo_spider') process.start()