web_crawler/Redis_Con.py

import random
import redis

class Redis_Con():
    def __init__(self):
        try:
            self.pool = redis.ConnectionPool(host='10.70.120.79', port=6379, decode_responses=True,db=0)
            self.r = redis.Redis(connection_pool=self.pool)
        except Exception as e:
            print('连接数据库失败')
            print(str(e))

    #上传需要爬取的url
    def puturls(self,url):
        try:
            self.r.lpush('requesturl',url)
            print('URL上传成功')
        except Exception as e:
            print('上传失败')

    #上传cookie到cookie池
    def putcookie(self,cookies:list):
        try:
            for cookie in cookies:
                self.r.lpush('cookies',cookie)
                print('Cookie上传成功')
        except Exception as e:
            print('上传失败')

   #爬虫获取一个url
    def geturl(self):
        u = self.r.lpop('requesturl')
        print('爬虫获取的url--->'+u)
        return u

    #爬虫获取cookie池中得值
    def getcookie(self):
        return self.r.lpop('cookies')

    #爬虫结束归还cookie
    def reuturncookie(self,cookie:str):
        self.r.lpush('cookies',cookie)

    #上传爬到的免费代理
    def putip(self,url):
        self.r.lpush('ip',url)

    #获取ip池中随机的ip
    def getip(self):
        all=self.r.llen('freeip')
        if all == 0:
            return None
        g_ip = self.r.lindex('freeip',random.randrange(0,all,1))
        return g_ip

    #返回Redis连接实例
    def pool(self):
        return self.r