parent
104a58dff1
commit
53b479b643
@ -0,0 +1,8 @@
|
|||||||
|
# Default ignored files
|
||||||
|
/shelf/
|
||||||
|
/workspace.xml
|
||||||
|
# Editor-based HTTP Client requests
|
||||||
|
/httpRequests/
|
||||||
|
# Datasource local storage ignored files
|
||||||
|
/dataSources/
|
||||||
|
/dataSources.local.xml
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
<component name="InspectionProjectProfileManager">
|
||||||
|
<settings>
|
||||||
|
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||||
|
<version value="1.0" />
|
||||||
|
</settings>
|
||||||
|
</component>
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module type="PYTHON_MODULE" version="4">
|
||||||
|
<component name="NewModuleRootManager">
|
||||||
|
<content url="file://$MODULE_DIR$">
|
||||||
|
<excludeFolder url="file://$MODULE_DIR$/venv" />
|
||||||
|
</content>
|
||||||
|
<orderEntry type="jdk" jdkName="Python 3.10" jdkType="Python SDK" />
|
||||||
|
<orderEntry type="sourceFolder" forTests="false" />
|
||||||
|
</component>
|
||||||
|
</module>
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
|
||||||
|
</project>
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectModuleManager">
|
||||||
|
<modules>
|
||||||
|
<module fileurl="file://$PROJECT_DIR$/.idea/jd-distributed-crawler.iml" filepath="$PROJECT_DIR$/.idea/jd-distributed-crawler.iml" />
|
||||||
|
</modules>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="VcsDirectoryMappings">
|
||||||
|
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
# 京东分布式爬虫
|
||||||
|
|
||||||
|
#### Description
|
||||||
|
selenium爬取搜索页
|
||||||
|
scrapy爬取详情页
|
||||||
|
|
||||||
|
#### Software Architecture
|
||||||
|
Software architecture description
|
||||||
|
|
||||||
|
#### Installation
|
||||||
|
|
||||||
|
1. xxxx
|
||||||
|
2. xxxx
|
||||||
|
3. xxxx
|
||||||
|
|
||||||
|
#### Instructions
|
||||||
|
|
||||||
|
1. xxxx
|
||||||
|
2. xxxx
|
||||||
|
3. xxxx
|
||||||
|
|
||||||
|
#### Contribution
|
||||||
|
|
||||||
|
1. Fork the repository
|
||||||
|
2. Create Feat_xxx branch
|
||||||
|
3. Commit your code
|
||||||
|
4. Create Pull Request
|
||||||
|
|
||||||
|
|
||||||
|
#### Gitee Feature
|
||||||
|
|
||||||
|
1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
|
||||||
|
2. Gitee blog [blog.gitee.com](https://blog.gitee.com)
|
||||||
|
3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
|
||||||
|
4. The most valuable open source project [GVP](https://gitee.com/gvp)
|
||||||
|
5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
|
||||||
|
6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
|
||||||
|
After Width: | Height: | Size: 4.9 KiB |
|
After Width: | Height: | Size: 116 KiB |
|
After Width: | Height: | Size: 116 KiB |
|
After Width: | Height: | Size: 92 KiB |
|
After Width: | Height: | Size: 149 KiB |
|
After Width: | Height: | Size: 149 KiB |
@ -0,0 +1,115 @@
|
|||||||
|
from time import sleep
|
||||||
|
import yaml
|
||||||
|
import redis
|
||||||
|
from selenium import webdriver
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
from pymongo import MongoClient
|
||||||
|
|
||||||
|
|
||||||
|
def scroll_by_xpath(webDriver, xpath):
|
||||||
|
"""
|
||||||
|
将webDriver里的网页下拉到xpath的位置
|
||||||
|
:param webDriver: web driver, 用于操作驱动后的浏览器
|
||||||
|
:param xpath: 要定位的元素的xpath, str
|
||||||
|
"""
|
||||||
|
target = webDriver.find_element(By.XPATH, xpath)
|
||||||
|
webDriver.execute_script("arguments[0].scrollIntoView();", target)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# 获取配置文件
|
||||||
|
yaml_file = "./settings.yaml"
|
||||||
|
with open(yaml_file, 'r') as f:
|
||||||
|
settings = yaml.safe_load(f)
|
||||||
|
|
||||||
|
# 获取redis连接
|
||||||
|
redis_connection = redis.Redis(
|
||||||
|
host=settings['redis']['host'],
|
||||||
|
port=settings['redis']['port'],
|
||||||
|
password=settings['redis']['password'],
|
||||||
|
db=settings['redis']['db'],
|
||||||
|
charset='UTF-8',
|
||||||
|
encoding='UTF-8')
|
||||||
|
|
||||||
|
mongo_connection = MongoClient('mongodb://{}:{}'.format(settings['mongodb']['host'], settings['mongodb']['port']),
|
||||||
|
username = settings['mongodb']['username'],
|
||||||
|
password = settings['mongodb']['password'])
|
||||||
|
|
||||||
|
# 打开京东商品搜索页
|
||||||
|
driver = webdriver.Chrome()
|
||||||
|
driver.get('https://search.jd.com/Search?keyword={}&enc=utf-8&wq={}'.format(settings['search']['keyword'], settings['search']['keyword']))
|
||||||
|
# 下拉到翻页导航的位置
|
||||||
|
scroll_by_xpath(driver, '//*[@id="J_bottomPage"]/span[1]/a[9]')
|
||||||
|
# 强制休眠5s再读取元素, 京东的搜索页需要下拉才加载全, 而下拉操作是异步的, 不匹配下拉时间读取不全元素
|
||||||
|
sleep(settings['search']['sleep'])
|
||||||
|
# 获取当前搜索的关键词的最大商品页
|
||||||
|
max_page = int(driver.find_element(By.XPATH, '//*[@id="J_bottomPage"]/span[2]/em[1]/b').text)
|
||||||
|
|
||||||
|
skuid_prev = []
|
||||||
|
skuid_url = []
|
||||||
|
page = 0
|
||||||
|
while page < max_page:
|
||||||
|
sku_price = []
|
||||||
|
|
||||||
|
# 下拉
|
||||||
|
scroll_by_xpath(driver, '//*[@id="J_bottomPage"]/span[1]/a[9]')
|
||||||
|
sleep(settings['search']['sleep'])
|
||||||
|
|
||||||
|
# 获取搜索页的商品的skuid和对应的价格
|
||||||
|
li_list = driver.find_elements(By.XPATH, '//*[@id="J_goodsList"]/ul/li')
|
||||||
|
for li in li_list:
|
||||||
|
li_text = li.text.split("\n")
|
||||||
|
result = 0
|
||||||
|
for text in li_text:
|
||||||
|
if text[1] in [str(i) for i in range(10)] and text[2] in [str(i) for i in range(10)]:
|
||||||
|
result = text[1:-1]
|
||||||
|
break
|
||||||
|
price = result
|
||||||
|
sku_price.append({li.get_attribute("data-sku"): price})
|
||||||
|
# 重复读取时, 重来
|
||||||
|
if page != 0 and sku_price[0] == skuid_prev[0]:
|
||||||
|
page -= 1
|
||||||
|
continue
|
||||||
|
# 防止未加载完就读取元素, 重新读取
|
||||||
|
while len(skuid_prev) < len(skuid_prev):
|
||||||
|
li_list = driver.find_elements(By.XPATH, '//*[@id="J_goodsList"]/ul/li')
|
||||||
|
for li in li_list:
|
||||||
|
price = li.find_element(By.XPATH, "//*[@class='gl-i-wrap']/*[@class='p-price']//strong/i")
|
||||||
|
sku_price.append({li.get_attribute("data-sku"): price})
|
||||||
|
|
||||||
|
#拼接成url, 入redis的start_url
|
||||||
|
for dic in sku_price:
|
||||||
|
for key in dic.keys():
|
||||||
|
skuid_url.append('https://item.jd.com/{}.html'.format(key))
|
||||||
|
for url in skuid_url:
|
||||||
|
redis_connection.rpush(settings['result']['rpush_key'], url)
|
||||||
|
|
||||||
|
#将sku和price放入mongodb
|
||||||
|
mongoDB = mongo_connection['admin']
|
||||||
|
mongoCollection = mongoDB['a']
|
||||||
|
|
||||||
|
for dic in sku_price:
|
||||||
|
document = {}
|
||||||
|
for key in dic.keys():
|
||||||
|
document['sku'] = key
|
||||||
|
document['price'] = dic[key]
|
||||||
|
# 更新MongoDB:
|
||||||
|
# 当数据库里没有, 插入
|
||||||
|
mongoCollection.update_one(
|
||||||
|
{'sku': document['sku']},
|
||||||
|
{
|
||||||
|
'$setOnInsert': document #生效需要upsert=True
|
||||||
|
},
|
||||||
|
upsert=True
|
||||||
|
)
|
||||||
|
|
||||||
|
print(str(page) + " " + str(len(skuid_url)) + " ", end="")
|
||||||
|
print(sku_price)
|
||||||
|
print(skuid_url)
|
||||||
|
sleep(settings['search']['sleep'])
|
||||||
|
|
||||||
|
skuid_prev = sku_price.copy()
|
||||||
|
skuid_url = []
|
||||||
|
page += 1
|
||||||
|
# 翻页
|
||||||
|
driver.find_element(By.XPATH, '//*[@id="J_bottomPage"]/span[1]/a[9]').click()
|
||||||
@ -0,0 +1,18 @@
|
|||||||
|
search:
|
||||||
|
keyword: '3080'
|
||||||
|
sleep: 5
|
||||||
|
|
||||||
|
redis:
|
||||||
|
host: '120.24.87.40'
|
||||||
|
port: '6379'
|
||||||
|
password: 'Guet@207'
|
||||||
|
db: 1
|
||||||
|
|
||||||
|
mongodb:
|
||||||
|
host: '120.24.87.40'
|
||||||
|
port: '27017'
|
||||||
|
username: "admin"
|
||||||
|
password: "123456"
|
||||||
|
|
||||||
|
result:
|
||||||
|
rpush_key: "jd:start_urls"
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
# Define here the models for your scraped items
|
||||||
|
#
|
||||||
|
# See documentation in:
|
||||||
|
# https://docs.scrapy.org/en/latest/topics/items.html
|
||||||
|
|
||||||
|
import scrapy
|
||||||
|
|
||||||
|
|
||||||
|
class JdskuspiderItem(scrapy.Item):
|
||||||
|
# define the fields for your item here like:
|
||||||
|
# name = scrapy.Field()
|
||||||
|
# sku-id
|
||||||
|
sku = scrapy.Field()
|
||||||
|
|
||||||
|
# title
|
||||||
|
title = scrapy.Field()
|
||||||
|
|
||||||
|
# 显卡型号
|
||||||
|
model = scrapy.Field()
|
||||||
|
|
||||||
|
# 显存类型
|
||||||
|
memoryType = scrapy.Field()
|
||||||
|
|
||||||
|
# 显存位宽
|
||||||
|
bitWidth = scrapy.Field()
|
||||||
|
|
||||||
|
# 显存容量
|
||||||
|
memoryCapacity = scrapy.Field()
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
# Define your item pipelines here
|
||||||
|
#
|
||||||
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
||||||
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||||
|
|
||||||
|
|
||||||
|
# useful for handling different item types with a single interface
|
||||||
|
from itemadapter import ItemAdapter
|
||||||
|
|
||||||
|
|
||||||
|
class JdskuspiderPipeline:
|
||||||
|
def process_item(self, item, spider):
|
||||||
|
print(item)
|
||||||
|
return item
|
||||||
@ -0,0 +1,160 @@
|
|||||||
|
# Scrapy settings for jdSkuSpider project
|
||||||
|
#
|
||||||
|
# For simplicity, this file contains only settings considered important or
|
||||||
|
# commonly used. You can find more settings consulting the documentation:
|
||||||
|
#
|
||||||
|
# https://docs.scrapy.org/en/latest/topics/settings.html
|
||||||
|
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||||
|
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||||
|
|
||||||
|
BOT_NAME = 'jdSkuSpider'
|
||||||
|
|
||||||
|
SPIDER_MODULES = ['jdSkuSpider.spiders']
|
||||||
|
NEWSPIDER_MODULE = 'jdSkuSpider.spiders'
|
||||||
|
|
||||||
|
#代理商提供的代理网址
|
||||||
|
PROXY_SERVER_URL = 'https://h.shanchendaili.com/api.html?action=get_ip&key=HU015c86520413222339Rp2a&time=10&count=1&protocol=http&type=text&textSep=1&only=1'
|
||||||
|
#用来测试当前的代理ip是否被封禁
|
||||||
|
TEST_PAGE = 'https://item.jd.com/100017846659.html'
|
||||||
|
#不去重(True代表不去重, False代表去重)
|
||||||
|
DONT_FILTER = True
|
||||||
|
|
||||||
|
REDIS_HOST = '120.24.87.40'
|
||||||
|
REDIS_PORT = '6379'
|
||||||
|
REDIS_DB = 1
|
||||||
|
REDIS_PARAMS = {
|
||||||
|
'password': 'Guet@207',
|
||||||
|
}
|
||||||
|
# 使用scrapy_redis中的调度器, 即保证每台主机爬取的URL地址都不同的Scheduler
|
||||||
|
SCHEDULER = 'scrapy_redis.scheduler.Scheduler'
|
||||||
|
# 配置scrapy使用的去重类, 即RFPDupeFilter
|
||||||
|
DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter'
|
||||||
|
# 序列化
|
||||||
|
SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"
|
||||||
|
#自动限速(控制爬取速度，降低对方服务器压力)
|
||||||
|
AUTOTHROTTLE_ENABLED = True
|
||||||
|
|
||||||
|
mongodb = {
|
||||||
|
'host': '120.24.87.40',
|
||||||
|
'port': 27017,
|
||||||
|
'username': "admin",
|
||||||
|
'password': "123456"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
cookies = {
|
||||||
|
'shshshfpa': '88442a32-4008-3016-2ed4-5f00ba6e02a6-1583306994',
|
||||||
|
'__jdu': '1645417385963897370071',
|
||||||
|
'shshshfpb': 'eKzbp55ekVbG%2BmXNii7FmnA%3D%3D',
|
||||||
|
'unpl': 'JF8EAKpnNSttC0hdDR9XGhMQTw5XW15YGURUPDQCVFwIHFEHGQoSExZ7XlVdXhRKFR9vZxRXXlNKUQ4fBCsSE3tdVV9cD0gVBWduNWRtW0tkBCsCHBcUTl1SXFQMQxABZm8DVltZSlIFKzIcEhl7bWRbXQlKFQVpZAVXbVl7VgQaAB8XFEJcU24cZk4XAmZlSFRaXU9RBR0AEhYYTF9dVlsKTRYCaWc1VlReQ1I1GA',
|
||||||
|
'__jdv': '76161171|baidu|-|organic|notset|1648984819662',
|
||||||
|
'areaId': '20',
|
||||||
|
'PCSYCityID': 'CN_450000_450300_0',
|
||||||
|
'ipLoc-djd': '20-1726-22884-51455',
|
||||||
|
'pinId': 'bxSvmv8CNc-KKJVv2AYoCLV9-x-f3wj7',
|
||||||
|
'pin': 'jd_701eaef2a29ff',
|
||||||
|
'unick': '%E6%9D%9C%E6%92%B0%E4%B8%AD%E7%9A%84%E6%9C%AA%E5%90%8D',
|
||||||
|
'_tp': 'Wy5hpT1Zse8ScnmqROf6D38%2BJ8IK3ESqvwXymUySsLE%3D',
|
||||||
|
'_pst': 'jd_701eaef2a29ff',
|
||||||
|
'shshshfp': '9dc45f7c100e4d56fc0ddb7d96bc59ab',
|
||||||
|
'__jdc': '122270672',
|
||||||
|
'__jda': '122270672.1645417385963897370071.1645417385.1649430078.1649491583.20',
|
||||||
|
'thor': 'FC76D1F441FDD23810222D65B21A1E62936594BACBBD52CF88EB830809C78B7F75DEEDA34458FE020C54AF1DD6E55F389B2063BCCBBD829B977631DD6FB67A17BA374FEFAA00AF1C8264A11F080FA449884B327D73A31031D1F35232730745A6BD1570FDB90E15A4002FCBF4C8CDED1F588BFA29B2272823A7263C9A88F289B84E2075F1A710BED202A651BD7ABBC09D354497FAFBB4A0A7CBDC9590803F6162',
|
||||||
|
'ceshi3.com': '000',
|
||||||
|
'token': '5a5e7384ead314efe3237efb8d9825fb,3,916384',
|
||||||
|
'__tk': 'OINnrcq4NDN5NiuEqcrdriKiOcJ5sINEsca4rfKiqIhgNLbhOIN4sG,3,916384',
|
||||||
|
'ip_cityCode': '1726',
|
||||||
|
'wlfstk_smdl': 'xrr3qwmzs7zoj7y6kbt5nfkai2clqo2n',
|
||||||
|
'shshshsID': '3c84311b6180b15c5bf06ba762b90ce2_7_1649491714829',
|
||||||
|
'__jdb': '122270672.15.1645417385963897370071|20.1649491583',
|
||||||
|
'3AB9D23F7A4B3C9B': '3WE3SSBAN2EJKLE3VP6ZK2I4UOCXTYVZTD7O46KFL2S7J2UVXVZ7IJGBJKC4S3RAWL2DRAMRLPN63TK3LHWTA5JVQQ',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||||
|
#USER_AGENT = 'jdSkuSpider (+http://www.yourdomain.com)'
|
||||||
|
|
||||||
|
# Obey robots.txt rules
|
||||||
|
ROBOTSTXT_OBEY = False
|
||||||
|
|
||||||
|
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
||||||
|
#CONCURRENT_REQUESTS = 32
|
||||||
|
|
||||||
|
# Configure a delay for requests for the same website (default: 0)
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
|
||||||
|
# See also autothrottle settings and docs
|
||||||
|
DOWNLOAD_DELAY = 1
|
||||||
|
# The download delay setting will honor only one of:
|
||||||
|
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
||||||
|
#CONCURRENT_REQUESTS_PER_IP = 16
|
||||||
|
|
||||||
|
# Disable cookies (enabled by default)
|
||||||
|
#COOKIES_ENABLED = False
|
||||||
|
|
||||||
|
# Disable Telnet Console (enabled by default)
|
||||||
|
#TELNETCONSOLE_ENABLED = False
|
||||||
|
|
||||||
|
# Override the default request headers:
|
||||||
|
DEFAULT_REQUEST_HEADERS = {
|
||||||
|
'authority': 'item.jd.com',
|
||||||
|
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
|
||||||
|
'accept-language': 'zh,zh-CN;q=0.9',
|
||||||
|
'cache-control': 'max-age=0',
|
||||||
|
# Requests sorts cookies= alphabetically
|
||||||
|
# 'cookie': 'shshshfpa=88442a32-4008-3016-2ed4-5f00ba6e02a6-1583306994; __jdu=1645417385963897370071; shshshfpb=eKzbp55ekVbG%2BmXNii7FmnA%3D%3D; unpl=JF8EAKpnNSttC0hdDR9XGhMQTw5XW15YGURUPDQCVFwIHFEHGQoSExZ7XlVdXhRKFR9vZxRXXlNKUQ4fBCsSE3tdVV9cD0gVBWduNWRtW0tkBCsCHBcUTl1SXFQMQxABZm8DVltZSlIFKzIcEhl7bWRbXQlKFQVpZAVXbVl7VgQaAB8XFEJcU24cZk4XAmZlSFRaXU9RBR0AEhYYTF9dVlsKTRYCaWc1VlReQ1I1GA; __jdv=76161171|baidu|-|organic|notset|1648984819662; areaId=20; PCSYCityID=CN_450000_450300_0; ipLoc-djd=20-1726-22884-51455; pinId=bxSvmv8CNc-KKJVv2AYoCLV9-x-f3wj7; pin=jd_701eaef2a29ff; unick=%E6%9D%9C%E6%92%B0%E4%B8%AD%E7%9A%84%E6%9C%AA%E5%90%8D; _tp=Wy5hpT1Zse8ScnmqROf6D38%2BJ8IK3ESqvwXymUySsLE%3D; _pst=jd_701eaef2a29ff; shshshfp=9dc45f7c100e4d56fc0ddb7d96bc59ab; __jdc=122270672; __jda=122270672.1645417385963897370071.1645417385.1649430078.1649491583.20; thor=FC76D1F441FDD23810222D65B21A1E62936594BACBBD52CF88EB830809C78B7F75DEEDA34458FE020C54AF1DD6E55F389B2063BCCBBD829B977631DD6FB67A17BA374FEFAA00AF1C8264A11F080FA449884B327D73A31031D1F35232730745A6BD1570FDB90E15A4002FCBF4C8CDED1F588BFA29B2272823A7263C9A88F289B84E2075F1A710BED202A651BD7ABBC09D354497FAFBB4A0A7CBDC9590803F6162; ceshi3.com=000; token=5a5e7384ead314efe3237efb8d9825fb,3,916384; __tk=OINnrcq4NDN5NiuEqcrdriKiOcJ5sINEsca4rfKiqIhgNLbhOIN4sG,3,916384; ip_cityCode=1726; wlfstk_smdl=xrr3qwmzs7zoj7y6kbt5nfkai2clqo2n; shshshsID=3c84311b6180b15c5bf06ba762b90ce2_7_1649491714829; __jdb=122270672.15.1645417385963897370071|20.1649491583; 3AB9D23F7A4B3C9B=3WE3SSBAN2EJKLE3VP6ZK2I4UOCXTYVZTD7O46KFL2S7J2UVXVZ7IJGBJKC4S3RAWL2DRAMRLPN63TK3LHWTA5JVQQ',
|
||||||
|
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
|
||||||
|
'sec-ch-ua-mobile': '?0',
|
||||||
|
'sec-ch-ua-platform': '"Windows"',
|
||||||
|
'sec-fetch-dest': 'document',
|
||||||
|
'sec-fetch-mode': 'navigate',
|
||||||
|
'sec-fetch-site': 'none',
|
||||||
|
'sec-fetch-user': '?1',
|
||||||
|
'upgrade-insecure-requests': '1',
|
||||||
|
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
|
||||||
|
}
|
||||||
|
|
||||||
|
# Enable or disable spider middlewares
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||||
|
#SPIDER_MIDDLEWARES = {
|
||||||
|
# 'jdSkuSpider.middlewares.JdskuspiderSpiderMiddleware': 543,
|
||||||
|
#}
|
||||||
|
|
||||||
|
# Enable or disable downloader middlewares
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||||
|
DOWNLOADER_MIDDLEWARES = {
|
||||||
|
'jdSkuSpider.middlewares.ProxyMiddleWare':542,
|
||||||
|
'jdSkuSpider.middlewares.JdskuspiderDownloaderMiddleware': 543,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Enable or disable extensions
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/extensions.html
|
||||||
|
#EXTENSIONS = {
|
||||||
|
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
||||||
|
#}
|
||||||
|
|
||||||
|
# Configure item pipelines
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||||
|
ITEM_PIPELINES = {
|
||||||
|
'scrapy_redis.pipelines.RedisPipeline':400,
|
||||||
|
'jdSkuSpider.pipelines.JdskuspiderPipeline': 300,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
|
||||||
|
#AUTOTHROTTLE_ENABLED = True
|
||||||
|
# The initial download delay
|
||||||
|
#AUTOTHROTTLE_START_DELAY = 5
|
||||||
|
# The maximum download delay to be set in case of high latencies
|
||||||
|
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||||
|
# The average number of requests Scrapy should be sending in parallel to
|
||||||
|
# each remote server
|
||||||
|
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||||
|
# Enable showing throttling stats for every response received:
|
||||||
|
#AUTOTHROTTLE_DEBUG = False
|
||||||
|
|
||||||
|
# Enable and configure HTTP caching (disabled by default)
|
||||||
|
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||||
|
#HTTPCACHE_ENABLED = True
|
||||||
|
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||||
|
#HTTPCACHE_DIR = 'httpcache'
|
||||||
|
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||||
|
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
# This package will contain the spiders of your Scrapy project
|
||||||
|
#
|
||||||
|
# Please refer to the documentation for information on how to create and manage
|
||||||
|
# your spiders.
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
# Automatically created by: scrapy startproject
|
||||||
|
#
|
||||||
|
# For more information about the [deploy] section see:
|
||||||
|
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||||
|
|
||||||
|
[settings]
|
||||||
|
default = jdSkuSpider.settings
|
||||||
|
|
||||||
|
[deploy]
|
||||||
|
#url = http://localhost:6800/
|
||||||
|
project = jdSkuSpider
|
||||||
Loading…
Reference in new issue