p39718465
/
zjtj
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix some bug in Spider

Browse Source
master
fishermanykx 3 years ago
parent 8de470b5b0
commit 7dc98330bc
7 changed files with 307 additions and 30 deletions
Show Stats Download Patch File Download Diff File

252
src/backend/Data_Cleansing/createTestDB.sql
Unescape View File

@ -0,0 +1,252 @@
CREATE DATABASE IF NOT EXISTS test;
USE test;
DROP TABLE IF EXISTS `cpu`;
CREATE TABLE `cpu` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`tags` varchar(255) DEFAULT NULL,
`clock_speed` varchar(255) DEFAULT NULL,
`core_num` varchar(255) DEFAULT NULL,
`TDP` INT DEFAULT 0,
`socket` varchar(255) DEFAULT NULL,
`have_core_graphics_card` varchar(255) DEFAULT NULL,
`have_cpu_fan` varchar(255) DEFAULT NULL,
`generation` INT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate cpu;
DROP TABLE IF EXISTS `motherboard`;
CREATE TABLE `motherboard` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`tags` varchar(255) DEFAULT NULL,
`form_factor` varchar(255) DEFAULT NULL,
`platform` varchar(255) DEFAULT NULL,
`cpu_socket` varchar(255) DEFAULT NULL,
`m2_num` INT DEFAULT 0,
`slot_num` INT DEFAULT 0,
`ddr_gen` varchar(255) DEFAULT NULL,
`max_memory` INT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate motherboard;
DROP TABLE IF EXISTS `graphics_card`;
CREATE TABLE `graphics_card` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`tags` varchar(255) DEFAULT NULL,
`card_length` float DEFAULT 0,
`rgb` varchar(255) DEFAULT NULL,
`card_type` varchar(255) DEFAULT NULL,
`generation` INT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate graphics_card;
DROP TABLE IF EXISTS `memory`;
CREATE TABLE `memory` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`frequency` varchar(255) DEFAULT NULL,
`total_capacity` INT DEFAULT 0,
`memory_num` varchar(255) DEFAULT NULL,
`appearance` varchar(255) DEFAULT NULL,
`ddr_gen` varchar(255) DEFAULT NULL,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate memory;
DROP TABLE IF EXISTS `cpu_radiator`;
CREATE TABLE `cpu_radiator` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`height` INT DEFAULT 0,
`socket` varchar(255) DEFAULT NULL,
`radiator_size` INT DEFAULT 0,
`rgb` varchar(255) DEFAULT NULL,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate cpu_radiator;
DROP TABLE IF EXISTS `ssd`;
CREATE TABLE `ssd` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`interface` varchar(255) DEFAULT NULL,
`total_capacity` FLOAT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate ssd;
DROP TABLE IF EXISTS `hdd`;
CREATE TABLE `hdd` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`size` varchar(255) DEFAULT NULL,
`rotating_speed` varchar(255) DEFAULT NULL,
`total_capacity` FLOAT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate hdd;
DROP TABLE IF EXISTS `power_supply`;
CREATE TABLE `power_supply` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`tags` varchar(255) DEFAULT NULL,
`power` INT DEFAULT 0,
`size` varchar(255) DEFAULT NULL,
`modularization` varchar(255) DEFAULT NULL,
`transfer_efficiency` varchar(255) DEFAULT NULL,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate power_supply;
DROP TABLE IF EXISTS `computer_case`;
CREATE TABLE `computer_case` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`comment_num` INT DEFAULT 100,
`praise_rate` INT DEFAULT 90,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`brand` varchar(255) DEFAULT NULL,
`max_form_factor` varchar(255) DEFAULT NULL,
`max_card_len` INT DEFAULT 0,
`max_radiator_height` INT DEFAULT 0,
`supported_radiator` varchar(255) DEFAULT NULL,
`has_transparent_side_panel` INT DEFAULT 0,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate computer_case;
DROP TABLE IF EXISTS `board_u_suit`;
CREATE TABLE `board_u_suit` (
`id` int(11) DEFAULT 0,
`name` varchar(255) DEFAULT NULL,
`board` varchar(255) DEFAULT NULL,
`cpu` varchar(255) DEFAULT NULL,
`shop_name` varchar(255) DEFAULT NULL,
`price` INT DEFAULT 0,
`link` varchar(255) NOT NULL,
`introduction` JSON,
`Ptable_params` JSON,
`title_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`link`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate board_u_suit;
DROP TABLE IF EXISTS `trendings`;
CREATE TABLE `trendings` (
`id` varchar(255) NOT NULL,
`frequency` int(11) DEFAULT 0,
`cpu` varchar(255) NOT NULL,
`cpu_price` int(11) DEFAULT 0,
`cpu_link` varchar(255) NOT NULL,
`motherboard` varchar(255) NOT NULL,
`motherboard_price` int(11) DEFAULT 0,
`motherboard_link` varchar(255) NOT NULL,
`GPU` varchar(255) NOT NULL,
`GPU_price` int(11) DEFAULT 0,
`GPU_link` varchar(255) NOT NULL,
`memory` varchar(255) NOT NULL,
`memory_price` int(11) DEFAULT 0,
`memory_link` varchar(255) NOT NULL,
`CPURadiator` varchar(255) NOT NULL,
`CPURadiator_price` int(11) DEFAULT 0,
`CPURadiator_link` varchar(255) NOT NULL,
`ssd` varchar(255) NOT NULL,
`ssd_price` int(11) DEFAULT 0,
`ssd_link` varchar(255) NOT NULL,
`hdd` varchar(255) NOT NULL,
`hdd_price` int(11) DEFAULT 0,
`hdd_link` varchar(255) NOT NULL,
`powerSupply` varchar(255) NOT NULL,
`powerSupply_price` int(11) DEFAULT 0,
`powerSupply_link` varchar(255) NOT NULL,
`case` varchar(255) NOT NULL,
`case_price` int(11) DEFAULT 0,
`case_link` varchar(255) NOT NULL,
`totalPrice` int(11) DEFAULT 0,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=utf8mb4, COLLATE=utf8mb4_general_ci;
truncate trendings;

6
src/backend/Data_Cleansing/data_cleansing.py
Unescape View File

@ -331,7 +331,7 @@ class DataCleaning:
" modify card_length float default 0.0" " modify card_length float default 0.0"
row = cursor.execute(instruct) # 返回被影响的行数 row = cursor.execute(instruct) # 返回被影响的行数
# 重新写入 # 重新写入
sql_insert = "INSERT INTO video_card (id, name, comment_num, praise_rate, shop_name, price, link,"\ sql_insert = "INSERT INTO graphics_card (id, name, comment_num, praise_rate, shop_name, price, link,"\
"brand, tags, card_length, rgb, card_type, introduction, Ptable_params) VALUES (%(id)s, %(name)s, %(comment_num)s,"\ "brand, tags, card_length, rgb, card_type, introduction, Ptable_params) VALUES (%(id)s, %(name)s, %(comment_num)s,"\
"%(praise_rate)s, %(shop_name)s, %(price)s, %(link)s, %(brand)s, %(tags)s, %(card_length)s, "\ "%(praise_rate)s, %(shop_name)s, %(price)s, %(link)s, %(brand)s, %(tags)s, %(card_length)s, "\
"%(rgb)s, %(card_type)s, %(introduction)s, %(Ptable_params)s)" "%(rgb)s, %(card_type)s, %(introduction)s, %(Ptable_params)s)"
@ -1029,7 +1029,7 @@ class DataCleaning:
self.cleanCPU() self.cleanCPU()
elif t == 'motherboard': elif t == 'motherboard':
self.cleanMotherboard() self.cleanMotherboard()
elif t == 'video_card': elif t == 'graphics_card':
self.cleanGraphicsCard() self.cleanGraphicsCard()
elif t == 'cpu_radiator': elif t == 'cpu_radiator':
self.cleanCPURadiator() self.cleanCPURadiator()
@ -1050,6 +1050,6 @@ class DataCleaning:
if __name__ == "__main__": if __name__ == "__main__":
# washer = DataCleaning('power_supply') # washer = DataCleaning('power_supply')
washer = DataCleaning('video_card') washer = DataCleaning('graphics_card')
# washer = DataCleaning('motherboard') # washer = DataCleaning('motherboard')
washer.main() washer.main()

79
src/backend/JDSpiders/Spider/JDSpider.py
Unescape View File

@ -10,6 +10,7 @@ import re
import json import json
import time import time
import pymysql import pymysql
from pymysql.converters import escape_string
from selenium.common.exceptions import ElementNotInteractableException from selenium.common.exceptions import ElementNotInteractableException
from selenium.common.exceptions import ElementClickInterceptedException from selenium.common.exceptions import ElementClickInterceptedException
@ -553,13 +554,19 @@ class CPUSpider(JDSpider):
# 时钟频率 # 时钟频率
record['clock_speed'] = p_table['规格']['主频'] record['clock_speed'] = p_table['规格']['主频']
# 内核数 # 内核数
record['core_num'] = introd['核心数量'] try:
record['core_num'] = introd['核心数量']
except:
record['core_num'] = introd['核心数']
# 是否支持核显 # 是否支持核显
record['have_core_graphics_card'] = introd.get('是否支持核显', '不支持核显') record['have_core_graphics_card'] = introd.get('是否支持核显', '不支持核显')
# 是否自带风扇 # 是否自带风扇
record['have_cpu_fan'] = introd.get('是否自带风扇', '不带风扇') record['have_cpu_fan'] = introd.get('是否自带风扇', '不带风扇')
# TDP # TDP
record['TDP'] = int(p_table['规格']['功率'][:-1]) try:
record['TDP'] = int(p_table['规格']['功率'][:-1])
except:
record['TDP'] = -1
# 处理接口 # 处理接口
socket = introd["接口"] socket = introd["接口"]
if "1151" in socket: if "1151" in socket:
@ -569,7 +576,8 @@ class CPUSpider(JDSpider):
record["socket"] = socket record["socket"] = socket
# CPU 代数 # CPU 代数
# 分类标准：锐龙5000系和酷睿10代为最新代；锐龙3000和酷睿9代为上代；其余为上古版本 # 分类标准：
# 鉴于 Intel 11 代倒吸牙膏，认定锐龙5000系和酷睿10代为最新代；锐龙3000和酷睿11代或9代为上代；其余为上古版本
# 提取代数 # 提取代数
name = record['name'] name = record['name']
pat = r"\d+\w" pat = r"\d+\w"
@ -578,6 +586,8 @@ class CPUSpider(JDSpider):
if record['brand'] == 'INTEL': if record['brand'] == 'INTEL':
if res[:2] == "10": if res[:2] == "10":
record['generation'] = 3 record['generation'] = 3
elif res[:2] == '11':
record['generation'] = 2
elif res[0] == '9': elif res[0] == '9':
record['generation'] = 2 record['generation'] = 2
else: else:
@ -647,9 +657,9 @@ class CPUSpider(JDSpider):
cpu_link = "https://list.jd.com/list.html?cat=670%2C677%2C678&page=" cpu_link = "https://list.jd.com/list.html?cat=670%2C677%2C678&page="
cpu_link = "https://list.jd.com/list.html?cat=670%2C677%2C678&psort=3&psort=3&page=" cpu_link = "https://list.jd.com/list.html?cat=670%2C677%2C678&psort=3&psort=3&page="
page_num = 3 page_num = 1
start_page = 1 start_page = 1
self.productSpider(cpu_link, page_num, start_page) # self.productSpider(cpu_link, page_num, start_page)
self.cleanCPU() self.cleanCPU()
print("Successfully get CPU data!") print("Successfully get CPU data!")
@ -757,7 +767,10 @@ class MotherboardSpider(JDSpider):
print(introd) print(introd)
exit(1) exit(1)
# 加入 m2 接口数 列 # 加入 m2 接口数 列
record['m2_num'] = int(introd.get('M.2接口数量', 0)) try:
record['m2_num'] = int(introd.get('M.2接口数量', 0))
except:
record['m2_num'] = int(introd.get('M.2接口数量', "00")[:-1])
# 插槽数目 # 插槽数目
slot_num = p_table['内存'].get("内存插槽", 0) slot_num = p_table['内存'].get("内存插槽", 0)
try: try:
@ -811,9 +824,13 @@ class MotherboardSpider(JDSpider):
# 判定是否为板-U套装 # 判定是否为板-U套装
record = data[i] record = data[i]
if record['comment_num'] == 100: # 抓到板-U套装了 if record['comment_num'] == 100: # 抓到板-U套装了
continue continue
if '套装' in record['title_name']: try:
continue if '套装' in record['title_name']:
continue
except:
if '套装' in record['name']:
continue
# 清洗数据 # 清洗数据
new_data.append(self.handleSingleMotherboardRecord(record)) new_data.append(self.handleSingleMotherboardRecord(record))
new_data[index]['id'] = cnt new_data[index]['id'] = cnt
@ -833,12 +850,12 @@ class MotherboardSpider(JDSpider):
connection.commit() connection.commit()
def main(self): def main(self):
motherboard_link = "https://list.jd.com/list.html?cat=670%2C677%2C681&psort=3&psort=3&page=" motherboard_link = "https://list.jd.com/list.html?cat=670%2C677%2C681&psort=3&psort=3&pvid=e726d0ee460448b6a16cf24950c1dabb&page="
# 爬取数据 # 爬取数据
page_num = 26 # 一共爬了26页 # page_num = 26 # 一共爬了26页
# page_num = 1 # for testing page_num = 1 # for testing
start_page = 1 start_page = 1
self.productSpider(motherboard_link, page_num, start_page) # self.productSpider(motherboard_link, page_num, start_page)
# 清洗数据 # 清洗数据
self.cleanMotherboard() self.cleanMotherboard()
print("Successfully get Motherboard data!") print("Successfully get Motherboard data!")
@ -997,9 +1014,10 @@ class GraphicsCardSpider(JDSpider):
connection.commit() connection.commit()
def main(self): def main(self):
graphics_card_link = "https://list.jd.com/list.html?cat=670%2C677%2C679&psort=3&psort=3&page=" graphics_card_link = "https://list.jd.com/list.html?cat=670%2C677%2C679&psort=3&psort=3&pvid=e726d0ee460448b6a16cf24950c1dabb&page="
# 爬取数据 # 爬取数据
page_num = 30 # page_num = 30
page_num = 1
start_page = 1 start_page = 1
# self.productSpider(graphics_card_link, page_num, start_page) # self.productSpider(graphics_card_link, page_num, start_page)
# 清洗数据 # 清洗数据
@ -1169,17 +1187,18 @@ class MemorySpider(JDSpider):
"Ptable_params, title_name) VALUES (%(id)s, %(name)s, %(comment_num)s, %(praise_rate)s, %(shop_name)s, %(price)s"\ "Ptable_params, title_name) VALUES (%(id)s, %(name)s, %(comment_num)s, %(praise_rate)s, %(shop_name)s, %(price)s"\
", %(link)s, %(brand)s, %(frequency)s, %(total_capacity)s, %(memory_num)s, %(appearance)s, "\ ", %(link)s, %(brand)s, %(frequency)s, %(total_capacity)s, %(memory_num)s, %(appearance)s, "\
"%(ddr_gen)s, %(introduction)s, %(Ptable_params)s, %(title_name)s)" "%(ddr_gen)s, %(introduction)s, %(Ptable_params)s, %(title_name)s)"
sql_insert = pymysql.escape_string(sql_insert) # sql_insert = escape_string(sql_insert)
cursor.executemany(sql_insert, new_data) # cursor.executemany(sql_insert, new_data)
# for i in range(len(new_data)): for i in range(len(new_data)):
# cursor.execute(sql_insert, new_data[i]) cursor.execute(sql_insert, new_data[i])
connection.commit() connection.commit()
def main(self): def main(self):
memory_link = "https://list.jd.com/list.html?cat=670%2C677%2C680&psort=3&ev=210_1558%5E&psort=3&page=" memory_link = "https://list.jd.com/list.html?cat=670%2C677%2C680&psort=3&ev=210_1558%5E&psort=3&page="
# 爬取数据 # 爬取数据
page_num = 40 # page_num = 40
page_num = 1
start_page = 1 start_page = 1
# self.productSpider(memory_link, page_num, start_page) # self.productSpider(memory_link, page_num, start_page)
# 清洗数据 # 清洗数据
@ -1366,9 +1385,10 @@ class CPURadiatorSpider(JDSpider):
def main(self): def main(self):
radiator_link = "https://list.jd.com/list.html?cat=670%2C677%2C682&psort=3&ev=3680_97402%7C%7C97403%7C%7C106254%7C%7C106255%5E&psort=3&page=" radiator_link = "https://list.jd.com/list.html?cat=670%2C677%2C682&psort=3&ev=3680_97402%7C%7C97403%7C%7C106254%7C%7C106255%5E&psort=3&page="
# 爬取数据 # 爬取数据
page_num = 27 # page_num = 27
page_num = 1
start_page = 1 start_page = 1
# self.productSpider(radiator_link, page_num, start_page) self.productSpider(radiator_link, page_num, start_page)
# 清洗数据 # 清洗数据
self.cleanCPURadiator() self.cleanCPURadiator()
print("Successfully get CPU Radiator data!") print("Successfully get CPU Radiator data!")
@ -1538,7 +1558,8 @@ class SSDSpider(JDSpider):
def main(self): def main(self):
ssd_link = "https://list.jd.com/list.html?cat=670%2C677%2C11303&psort=3&psort=3&page=" ssd_link = "https://list.jd.com/list.html?cat=670%2C677%2C11303&psort=3&psort=3&page="
page_num = 36 # 一共爬了36页 # page_num = 36 # 一共爬了36页
page_num = 1 # 一共爬了36页
start_page = 1 start_page = 1
# self.productSpider(ssd_link, page_num, start_page) # self.productSpider(ssd_link, page_num, start_page)
@ -1715,7 +1736,8 @@ class HDDSpider(JDSpider):
def main(self): def main(self):
hdd_link = "https://list.jd.com/list.html?cat=670%2C677%2C683&psort=3&psort=3&page=" hdd_link = "https://list.jd.com/list.html?cat=670%2C677%2C683&psort=3&psort=3&page="
page_num = 11 # page_num = 11
page_num = 1
start_page = 1 start_page = 1
# self.productSpider(hdd_link, page_num, start_page) # self.productSpider(hdd_link, page_num, start_page)
@ -1871,7 +1893,8 @@ class PowerSupplySpider(JDSpider):
def main(self): def main(self):
power_supply_link = "https://list.jd.com/list.html?cat=670%2C677%2C691&psort=3&psort=3&page=" power_supply_link = "https://list.jd.com/list.html?cat=670%2C677%2C691&psort=3&psort=3&page="
page_num = 25 # 抓25页 # page_num = 25 # 抓25页
page_num = 1
start_page = 1 start_page = 1
# self.productSpider(power_supply_link, page_num, start_page) # self.productSpider(power_supply_link, page_num, start_page)
@ -2056,9 +2079,10 @@ class CaseSpider(JDSpider):
def main(self): def main(self):
case_link = "https://list.jd.com/list.html?cat=670%2C677%2C687&psort=3&psort=3&page=" case_link = "https://list.jd.com/list.html?cat=670%2C677%2C687&psort=3&psort=3&page="
page_num = 36 # page_num = 36
page_num = 1
start_page = 1 start_page = 1
self.productSpider(case_link, page_num, start_page) # self.productSpider(case_link, page_num, start_page)
self.cleanCase() self.cleanCase()
print("Successfully get Computer Case data!") print("Successfully get Computer Case data!")
@ -2066,6 +2090,7 @@ class CaseSpider(JDSpider):
if __name__ == "__main__": if __name__ == "__main__":
accessory_type = 'all' accessory_type = 'all'
accessory_type = 'motherboard'
if accessory_type == 'cpu': if accessory_type == 'cpu':
cpu_spider = CPUSpider('cpu') cpu_spider = CPUSpider('cpu')
cpu_spider.main() cpu_spider.main()

BIN
src/backend/JDSpiders/Spider/__pycache__/DBConfig.cpython-37.pyc
View File

Binary file not shown.

BIN
src/backend/JDSpiders/Spider/__pycache__/DBConfig.cpython-38.pyc
View File

Binary file not shown.

BIN
src/backend/JDSpiders/Spider/__pycache__/JDSpider.cpython-37-PYTEST.pyc
View File

Binary file not shown.

BIN
src/backend/JDSpiders/Spider/__pycache__/dataCleaning.cpython-37.pyc
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save