You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
46 lines
1.5 KiB
46 lines
1.5 KiB
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
|
|
from hyperlink import URL
|
|
|
|
# 修改要生成的文件名，下面的是默认，注意要用.csv结尾
|
|
FILENAME_CSV = {
|
|
"牛奶": "milk.csv",
|
|
"苹果": "apple.csv",
|
|
"橙子": "orange.csv",
|
|
"芒果": "mango.csv",
|
|
"冰淇淋": "iceCream.csv"
|
|
}
|
|
|
|
# 几个默认的爬取目录
|
|
BASEURL = {
|
|
'牛奶': 'https://list.jd.com/list.html?cat=1320,1585,9434', # ok
|
|
'苹果': 'https://list.jd.com/list.html?cat=12218,12221,13554', # ok
|
|
'橙子': 'https://list.jd.com/list.html?cat=12218,12221,13555', # ok
|
|
'芒果': 'https://list.jd.com/list.html?cat=12218,12221,13558', # ok
|
|
'冰淇淋': 'https://list.jd.com/list.html?cat=12218,13598,13603' # ok
|
|
}
|
|
|
|
FILEPATH = {
|
|
'牛奶': os.getcwd() + '\\1320,1585,9434\\1320,1585,9434',
|
|
}
|
|
|
|
# REDIS 相关配置
|
|
REDIS_HOST = 'tencentCloud'
|
|
REDIS_PORT = '6379'
|
|
REDIS_PASSWORD = 'root'
|
|
REDIS_LISTNAME = "urlList"
|
|
|
|
# 下载器相关配置
|
|
USER_AGENT = [
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:97.0) Gecko/20100101 Firefox/97.0',
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586',
|
|
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36',
|
|
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'
|
|
]
|
|
|
|
COOKIES_FILENAME = "cookies.json"
|
|
|
|
# 历史价格查询网站 vveby.com
|
|
HISTORY_PRICE_URL = r"https://www.vveby.com/search?keyword=" |