diff --git a/Scrapy-Redis-Zhihu/scrapy_redis/defaults.py b/Scrapy-Redis-Zhihu/scrapy_redis/defaults.py new file mode 100644 index 0000000..408a383 --- /dev/null +++ b/Scrapy-Redis-Zhihu/scrapy_redis/defaults.py @@ -0,0 +1,25 @@ +import redis + + +# For standalone use. +DUPEFILTER_KEY = 'dupefilter:%(timestamp)s' + +PIPELINE_KEY = '%(spider)s:items' + +REDIS_CLS = redis.StrictRedis +REDIS_ENCODING = 'utf-8' +# Sane connection defaults. +REDIS_PARAMS = { + 'socket_timeout': 30, + 'socket_connect_timeout': 30, + 'retry_on_timeout': True, + 'encoding': REDIS_ENCODING, +} + +SCHEDULER_QUEUE_KEY = '%(spider)s:requests' +SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue' +SCHEDULER_DUPEFILTER_KEY = '%(spider)s:dupefilter' +SCHEDULER_DUPEFILTER_CLASS = 'scrapy_redis.dupefilter.RFPDupeFilter' + +START_URLS_KEY = '%(name)s:start_urls' +START_URLS_AS_SET = False