|
|
from selenium import webdriver
|
|
|
from selenium.webdriver.common.by import By
|
|
|
from selenium.webdriver.chrome.options import Options
|
|
|
import json
|
|
|
import csv
|
|
|
import threading
|
|
|
import random
|
|
|
import time
|
|
|
from time import sleep
|
|
|
from queue import Queue
|
|
|
|
|
|
def get_cookie():#登录后获取cookie
|
|
|
options = Options()
|
|
|
driver = webdriver.Chrome(options=options)
|
|
|
url = 'https://passport.jd.com/new/login.aspx?/'
|
|
|
driver.get(url)
|
|
|
print("请手动登录京东账号...")
|
|
|
input("登录完成后，请按回车键继续...")
|
|
|
|
|
|
cookie_list = driver.get_cookies()
|
|
|
cookie_str = json.dumps(cookie_list)
|
|
|
with open('Jdcookie.txt', 'w') as f:
|
|
|
f.write(cookie_str)
|
|
|
print('Cookie已保存到Jdcookie.txt')
|
|
|
driver.quit()
|
|
|
|
|
|
def crawl_page(start_page, end_page, current_url, queue):
|
|
|
options = Options()
|
|
|
driver = webdriver.Chrome(options=options)
|
|
|
|
|
|
# 设置隐式等待时间
|
|
|
driver.implicitly_wait(random.uniform(3, 6))
|
|
|
|
|
|
with open('Jdcookie.txt', mode='r', encoding='utf-8') as f:
|
|
|
cookie = f.read()
|
|
|
cookie = json.loads(cookie)
|
|
|
driver.get('https://www.jd.com/')
|
|
|
driver.delete_all_cookies()#清除cookie可以模拟用户每次访问网站时都是一个全新的会话，提高爬虫成功率
|
|
|
for item in cookie:
|
|
|
driver.add_cookie(item)
|
|
|
driver.refresh()
|
|
|
|
|
|
for page in range(start_page, end_page + 1):
|
|
|
url = current_url + f'&isList=0&page={page*2-1}'#根据分析url得出的规律
|
|
|
driver.get(url)
|
|
|
for x in range(1, 12, 2):#下拉页面
|
|
|
time.sleep(1)
|
|
|
j = x / 9
|
|
|
js = "document.documentElement.scrollTop=document.documentElement.scrollHeight * " + str(j)
|
|
|
driver.execute_script(js)
|
|
|
goods = driver.find_elements(By.CLASS_NAME, 'gl-i-wrap')
|
|
|
for good in goods:
|
|
|
try:
|
|
|
title = good.find_element(By.CSS_SELECTOR, '.p-name em').text.strip()#元素定位
|
|
|
price = good.find_element(By.CSS_SELECTOR, '.p-price strong').text.strip()
|
|
|
shop = good.find_element(By.CSS_SELECTOR, '.p-shop span a').text.strip()
|
|
|
comment = good.find_element(By.CSS_SELECTOR, '.p-commit strong a').text.strip()
|
|
|
print('title: ' + title)
|
|
|
print('price: ' + price)
|
|
|
print('shop: ' + shop)
|
|
|
print('comment: ' + comment)
|
|
|
with open('JD.csv', mode='a', encoding='UTF-8', newline='') as file:#信息写入csv文件
|
|
|
csv.writer(file).writerow([title, price, shop, comment])
|
|
|
except Exception as e:
|
|
|
print(f"Error: {e}")
|
|
|
print(f'第{page}页爬取完毕！')
|
|
|
|
|
|
driver.quit()
|
|
|
queue.put(1) # 爬取完成后，放入队列表示该线程完成任务
|
|
|
|
|
|
def main():
|
|
|
# get_cookie() # 首次登录请取消注释
|
|
|
options = Options()
|
|
|
driver = webdriver.Chrome(options=options)
|
|
|
with open('Jdcookie.txt', mode='r', encoding='utf-8') as f:
|
|
|
cookie = f.read()
|
|
|
cookie = json.loads(cookie)
|
|
|
driver.get('https://www.jd.com/')
|
|
|
driver.delete_all_cookies()
|
|
|
for item in cookie:
|
|
|
driver.add_cookie(item)
|
|
|
driver.refresh()
|
|
|
sleep(1)
|
|
|
driver.find_element(By.ID, 'key').send_keys("口红")
|
|
|
sleep(2)
|
|
|
driver.find_element(By.CLASS_NAME, 'button').click()
|
|
|
sleep(15)
|
|
|
current_url = driver.current_url
|
|
|
driver.quit()
|
|
|
|
|
|
queue = Queue() # 创建一个队列
|
|
|
|
|
|
with open('JD.csv',mode='w',encoding='UTF-8',newline='') as file:
|
|
|
csv.writer(file).writerow(['商品', '价格','店铺','评论数'])
|
|
|
|
|
|
# 创建4个线程，每个线程都使用独立的WebDriver实例
|
|
|
t1 = threading.Thread(target=crawl_page, args=(1, 5, current_url, queue))
|
|
|
t2 = threading.Thread(target=crawl_page, args=(6, 10, current_url, queue))
|
|
|
t3 = threading.Thread(target=crawl_page, args=(11, 15, current_url, queue))
|
|
|
t4 = threading.Thread(target=crawl_page, args=(16, 20, current_url, queue))
|
|
|
|
|
|
# 启动线程
|
|
|
t1.start()
|
|
|
t2.start()
|
|
|
t3.start()
|
|
|
t4.start()
|
|
|
|
|
|
# 等待线程完成第一页的爬取
|
|
|
queue.get()
|
|
|
queue.get()
|
|
|
queue.get()
|
|
|
queue.get()
|
|
|
|
|
|
# 继续爬取剩余页面
|
|
|
t1.join()
|
|
|
t2.join()
|
|
|
t3.join()
|
|
|
t4.join()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
main()
|
