update via git

master
wkyuu 3 years ago
parent e5d47a8d55
commit 46dcfdb879

@ -232,10 +232,10 @@ redisconn = redis.Redis(host = '127.0.0.1', port = '6379', password = 'x', db =
12[【自动化】selenium设置请求头](https://www.jianshu.com/p/419eb4e00963)
13[https://blog.csdn.net/fox64194167/article/details/80542717](https://blog.csdn.net/fox64194167/article/details/80542717)
13[python selenium 保存cookie 读取cookie](https://blog.csdn.net/fox64194167/article/details/80542717)
14[Selenium添加Cookie的方法](https://cloud.tencent.com/developer/article/1616175)
15
15[requests库使用方法汇总笔记](https://wenku.baidu.com/view/fa71322401020740be1e650e52ea551810a6c928.html)
16

@ -35,13 +35,35 @@ def getsource(url):
return response
def useRequests(url):
pass
def saveCookies(response):
myCookies = {}
for key, value in response.cookies.items():
myCookies[key] = value
jsonCookies = json.dumps(myCookies)
with open(settings.COOKIES_FILENAME, mode = 'a', encoding = 'utf-8') as fd:
fd.write(jsonCookies)
print("Cookies saved!")
try:
res = requests.get(url, headers = headers)
res.raise_for_status() # 判断是不是200
res.encoding = res.apparent_encoding
print(res.cookies)
saveCookies(res)
return res
except BaseException as e:
print(e)
print("sth wrong in your downloader.useRequests. Exiting...")
exit()
if __name__ == "__main__":
jdurl = "https://item.jd.com/10036840192083.html"
url = "https://www.vveby.com/search?keyword=" + jdurl
with open('historyPrice.html', 'w+', encoding = 'utf-8') as fd:
fd.write(getsource(url))
fd.close()
# jdurl = r"https://item.jd.com/10036840192083.html"
jdurl = r"https://item.jd.com/59162092942.html"
url = r"https://www.vveby.com/search?keyword=" + jdurl
print(url)
with open('newhistoryPrice.html', 'w+', encoding = 'utf-8') as fd:
fd.write(useRequests(url).text)
print('done')

@ -7,16 +7,10 @@ import downloader
class historyPriceItem:
def __init__(self, url, ):
'''
if __name__ == '__main__':
pass
'''

@ -11,5 +11,4 @@ def milkSpider():
print("milkSpider done.")
if __name__ == "__main__":
milkSpider()

@ -37,3 +37,5 @@ USER_AGENT = [
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'
]
COOKIES_FILENAME = "cookies.json"
Loading…
Cancel
Save