You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
1.6 KiB
61 lines
1.6 KiB
import random
|
|
import redis
|
|
|
|
class Redis_Con():
|
|
def __init__(self):
|
|
try:
|
|
self.pool = redis.ConnectionPool(host='10.70.120.79', port=6379, decode_responses=True,db=0)
|
|
self.r = redis.Redis(connection_pool=self.pool)
|
|
except Exception as e:
|
|
print('连接数据库失败')
|
|
print(str(e))
|
|
|
|
#上传需要爬取的url
|
|
def puturls(self,url):
|
|
try:
|
|
self.r.lpush('requesturl',url)
|
|
print('URL上传成功')
|
|
except Exception as e:
|
|
print('上传失败')
|
|
|
|
#上传cookie到cookie池
|
|
def putcookie(self,cookies:list):
|
|
try:
|
|
for cookie in cookies:
|
|
self.r.lpush('cookies',cookie)
|
|
print('Cookie上传成功')
|
|
except Exception as e:
|
|
print('上传失败')
|
|
|
|
#爬虫获取一个url
|
|
def geturl(self):
|
|
u = self.r.lpop('requesturl')
|
|
print('爬虫获取的url--->'+u)
|
|
return u
|
|
|
|
#爬虫获取cookie池中得值
|
|
def getcookie(self):
|
|
return self.r.lpop('cookies')
|
|
|
|
#爬虫结束归还cookie
|
|
def reuturncookie(self,cookie:str):
|
|
self.r.lpush('cookies',cookie)
|
|
|
|
#上传爬到的免费代理
|
|
def putip(self,url):
|
|
self.r.lpush('ip',url)
|
|
|
|
#获取ip池中随机的ip
|
|
def getip(self):
|
|
all=self.r.llen('freeip')
|
|
if all == 0:
|
|
return None
|
|
g_ip = self.r.lindex('freeip',random.randrange(0,all,1))
|
|
return g_ip
|
|
|
|
#返回Redis连接实例
|
|
def pool(self):
|
|
return self.r
|
|
|
|
|