parent
f881ba4164
commit
067770177f
@ -0,0 +1,60 @@
|
||||
import random
|
||||
import redis
|
||||
|
||||
class Redis_Con():
|
||||
def __init__(self):
|
||||
try:
|
||||
self.pool = redis.ConnectionPool(host='10.70.120.79', port=6379, decode_responses=True,db=0)
|
||||
self.r = redis.Redis(connection_pool=self.pool)
|
||||
except Exception as e:
|
||||
print('连接数据库失败')
|
||||
print(str(e))
|
||||
|
||||
#上传需要爬取的url
|
||||
def puturls(self,url):
|
||||
try:
|
||||
self.r.lpush('requesturl',url)
|
||||
print('URL上传成功')
|
||||
except Exception as e:
|
||||
print('上传失败')
|
||||
|
||||
#上传cookie到cookie池
|
||||
def putcookie(self,cookies:list):
|
||||
try:
|
||||
for cookie in cookies:
|
||||
self.r.lpush('cookies',cookie)
|
||||
print('Cookie上传成功')
|
||||
except Exception as e:
|
||||
print('上传失败')
|
||||
|
||||
#爬虫获取一个url
|
||||
def geturl(self):
|
||||
u = self.r.lpop('requesturl')
|
||||
print('爬虫获取的url--->'+u)
|
||||
return u
|
||||
|
||||
#爬虫获取cookie池中得值
|
||||
def getcookie(self):
|
||||
return self.r.lpop('cookies')
|
||||
|
||||
#爬虫结束归还cookie
|
||||
def reuturncookie(self,cookie:str):
|
||||
self.r.lpush('cookies',cookie)
|
||||
|
||||
#上传爬到的免费代理
|
||||
def putip(self,url):
|
||||
self.r.lpush('ip',url)
|
||||
|
||||
#获取ip池中随机的ip
|
||||
def getip(self):
|
||||
all=self.r.llen('freeip')
|
||||
if all == 0:
|
||||
return None
|
||||
g_ip = self.r.lindex('freeip',random.randrange(0,all,1))
|
||||
return g_ip
|
||||
|
||||
#返回Redis连接实例
|
||||
def pool(self):
|
||||
return self.r
|
||||
|
||||
|
Loading…
Reference in new issue