From bbf66a01b4156aee7a4a2b5d18153ae42c78dfab Mon Sep 17 00:00:00 2001 From: wkyuu Date: Sat, 16 Apr 2022 17:31:56 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=BA=86=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E9=A5=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f79b530..47cc18d 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,9 @@ selenium + redis + 分布式 + xpath + etree + 可视化 任务：爬取京东网站上在售的各类牛奶品类的商品名称，简介，价格相关，评论区相关。并给出相应的价格波动趋势，精选好评，用python的可视化展示。计划任务自动爬取。 -![image-20220410095017421](README [Image]/image-20220410095017421.png) - -![image-20220410095022817](README [Image]/image-20220410095022817.png) - ## TODO - [x] 初始化 selenium 框架，编写好相应的爬取规则，初步实现小规模爬取内容 -- [ ] 考虑user-agent，ip池，cookie，token，实现更大规模爬取内容 - [ ] 从历史价格网页爬取历史价格，比对，给出价格波动趋势 - [x] 加入Redis分布式设计 - [ ] 数据可视化 @@ -31,7 +26,6 @@ selenium + redis + 分布式 + xpath + etree + 可视化 > > > > milkSpider.py 主文件，配置爬取设置，自动化等 > > -> > items.py 暂定 ### selenium @@ -86,8 +80,8 @@ git pull https://bdgit.educoder.net/mf942lkca/milkSpider.git git remote -v # 查看远程仓库信息 touch .gitignore # 创建忽略上传控制文件 -git commit -m "update" # 先添加一个commit git add *.py # 添加要push的本地内容到一个本地临时仓库 +git commit -m "update" # 先添加一个commit git push -u origin master # push, 出错就 -f(注意会造成不可回避的损失) ``` @@ -212,4 +206,6 @@ redisconn = redis.Redis(host = '127.0.0.1', port = '6379', password = 'x', db = 13，[https://blog.csdn.net/fox64194167/article/details/80542717](https://blog.csdn.net/fox64194167/article/details/80542717) -14， \ No newline at end of file +14，[Selenium：添加Cookie的方法](https://cloud.tencent.com/developer/article/1616175) + +15， \ No newline at end of file