Spider/jingdong.py

from selenium import webdriver
import time
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import json
import csv
from selenium.webdriver.common.by import By
from time import sleep
from multiprocessing import Process


driver = webdriver.Chrome()

#先手动登录，让程序获取到cookie，保存下来
def getcookie():
    url = 'https://passport.jd.com/new/login.aspx?/'
    #首先直接访问登录的页面 passport.jd.com
    driver.get(url)
    time.sleep(20)
    #扫码登录
    #登录之后的页面会跳转到这里，让浏览器等待，直到url完全匹配
    url='https://www.jd.com/'
    WebDriverWait(driver,20).until(EC.url_to_be(url))
    #登录之后停2秒
    time.sleep(2)
    #获取到的cookies是列表
    cookieList  = driver.get_cookies()
    #转成字符串
    cookieStr = json.dumps(cookieList)

    # print(cookieStr)
    with open('Jdcookie.txt', 'w') as f:
        f.write(cookieStr)

    print('cookie已写入')
    print(driver.current_url)
    driver.close()


if __name__ == '__main__':
    
    # getcookie()#首次登录时去除注释

    driver.get('https://www.jd.com/')
    with open('Jdcookie.txt',mode='r',encoding='utf-8') as f:
        cookie = f.read()

    #读取到的是字符串类型，loads之后就变成了python中的字典类型
    cookie = json.loads(cookie)
    
    #先把所有的cookie全部删掉
    driver.delete_all_cookies()
    for item in cookie:
        print(type(item))
        print(item)
        driver.add_cookie(item)
    #是一个列表内套字典的形式

    driver.refresh()
    
    driver.find_element(By.ID,'key').send_keys("口红")   #通过id找到搜索框，输入内容
    driver.find_element(By.CLASS_NAME, 'button').click()   #通过类名找到button提交，并点击
    driver.implicitly_wait(10)  
    with open('JD.csv',mode='w',encoding='UTF-8',newline='') as file:
        csv.writer(file).writerow(['商品', '价格','店铺','评论数'])
    for page in range(0,50):
        sleep(10)
        # 下滑页面
        # 滑动到最底部
        driver.execute_script('window.scrollTo(0,1000)')
        sleep(1)
        driver.execute_script('window.scrollTo(1000,2000)')
        sleep(1)
        driver.execute_script('window.scrollTo(2000,3000)')
        sleep(1)
        driver.execute_script('window.scrollTo(3000,4000)')
        sleep(1)
        driver.execute_script('window.scrollTo(4000,5000)')
        sleep(1)
        driver.execute_script('window.scrollTo(5000,6000)')
        sleep(1)
        driver.execute_script('window.scrollTo(6000,7000)')


        sleep(10)
        goods = driver.find_elements(By.CLASS_NAME,'gl-i-wrap')
        for good in goods:
            title = good.find_element(By.CSS_SELECTOR,'.p-name em').text.strip()
            price = good.find_element(By.CSS_SELECTOR,'.p-price strong').text.strip()
            shop = good.find_element(By.CSS_SELECTOR,'.p-shop span a').text.strip()
            comment = good.find_element(By.CSS_SELECTOR,'.p-commit strong a').text.strip()
            print('title: ' + title)
            print('price: ' + price)
            print('shop: ' + shop)
            print('comment: ' + comment)
            # 用a+模式创建csv文件并写入
            f = open('JD.csv', 'a+', encoding='utf-8')
            # 基于文件对象构建csv写入
            csv_a = csv.writer(f)
            # 将数据写入
            csv_a.writerow([title, price, shop,comment])
            # 关闭文件
            f.close()
        driver.find_element(By.CLASS_NAME,'pn-next').click()
        print(f'第{page+1}页爬取完毕！')
        print('下一页加载中……')
    # time.sleep(5)

    driver.close()