forked from p3t2ja9zs/dcs
初步完善了服务器中存在的“当前用户信息表”，以及保存它的外部变量，同时修缮了数据库处理，以及提供了cookie认证机制，经过初步验证；此外初步实现了爬虫任务分发功能，还未验证，因为需要改写客户端文件。
parent
3d8e40bb5e
commit
e69f4ea071
@ -0,0 +1,43 @@
|
|||||||
|
import os
|
||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
#爬虫头数据
|
||||||
|
cookies = {
|
||||||
|
'SINAGLOBAL': '6797875236621.702.1603159218040',
|
||||||
|
'SUB': '_2AkMXbqMSf8NxqwJRmfkTzmnhboh1ygvEieKhMlLJJRMxHRl-yT9jqmg8tRB6PO6N_Rc_2FhPeZF2iThYO9DfkLUGpv4V',
|
||||||
|
'SUBP': '0033WrSXqPxfM72-Ws9jqgMF55529P9D9Wh-nU-QNDs1Fu27p6nmwwiJ',
|
||||||
|
'_s_tentry': 'www.baidu.com',
|
||||||
|
'UOR': 'www.hfut.edu.cn,widget.weibo.com,www.baidu.com',
|
||||||
|
'Apache': '7782025452543.054.1635925669528',
|
||||||
|
'ULV': '1635925669554:15:1:1:7782025452543.054.1635925669528:1627316870256',
|
||||||
|
}
|
||||||
|
headers = {
|
||||||
|
'Connection': 'keep-alive',
|
||||||
|
'Cache-Control': 'max-age=0',
|
||||||
|
'Upgrade-Insecure-Requests': '1',
|
||||||
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.6241 SLBChan/25',
|
||||||
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
|
||||||
|
'Sec-Fetch-Site': 'cross-site',
|
||||||
|
'Sec-Fetch-Mode': 'navigate',
|
||||||
|
'Sec-Fetch-User': '?1',
|
||||||
|
'Sec-Fetch-Dest': 'document',
|
||||||
|
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||||
|
}
|
||||||
|
params = (
|
||||||
|
('cate', 'realtimehot'),
|
||||||
|
)
|
||||||
|
#数据存储
|
||||||
|
fo = open("./微博热搜.txt",'a',encoding="utf-8")
|
||||||
|
#获取网页
|
||||||
|
response = requests.get('https://s.weibo.com/top/summary', headers=headers, params=params, cookies=cookies)
|
||||||
|
#解析网页
|
||||||
|
response.encoding='utf-8'
|
||||||
|
soup = BeautifulSoup(response.text, 'html.parser')
|
||||||
|
#爬取内容
|
||||||
|
content="#pl_top_realtimehot > table > tbody > tr > td.td-02 > a"
|
||||||
|
#清洗数据
|
||||||
|
a=soup.select(content)
|
||||||
|
for i in range(0,len(a)):
|
||||||
|
a[i] = a[i].text
|
||||||
|
fo.write(a[i]+'\n')
|
||||||
|
fo.close()
|
||||||
@ -0,0 +1,18 @@
|
|||||||
|
from hashlib import *
|
||||||
|
|
||||||
|
|
||||||
|
class Cookie:
|
||||||
|
def __init__(self, user_name: str, login_time: str, login_state: str, cookie=None):
|
||||||
|
self.user_name = user_name
|
||||||
|
self.login_time = login_time
|
||||||
|
self.login_state = login_state
|
||||||
|
self.cookie = cookie
|
||||||
|
|
||||||
|
def generate_cookie(self):
|
||||||
|
s1 = sha1()
|
||||||
|
s1.update(str(self.user_name+self.login_time+self.login_state).encode())
|
||||||
|
self.cookie = s1.hexdigest()
|
||||||
|
return self.cookie
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.cookie
|
||||||
Loading…
Reference in new issue