|
|
import csv
|
|
|
|
|
|
import requests
|
|
|
from lxml import etree
|
|
|
import re
|
|
|
from bs4 import BeautifulSoup
|
|
|
header={"User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0'}
|
|
|
url='https://shenmezhidedu.com/'
|
|
|
|
|
|
response=requests.get(url,headers=header)
|
|
|
source=response.text
|
|
|
#print(source)
|
|
|
all_link=[]
|
|
|
all_book=[]
|
|
|
soup = BeautifulSoup(source, "lxml")
|
|
|
#爬取心理学一栏的书名，作者，出版社，发行日期，内容简介和作者简介，存储到csv文件
|
|
|
books=soup.select('div.list-card')
|
|
|
for b in books:
|
|
|
book=b.select('div.h6')
|
|
|
for b1 in book:
|
|
|
if '心理学' in b1.get_text():
|
|
|
for group in b1.find_next_siblings('div', class_='list-grouped'):
|
|
|
links=group.find_all('di')
|
|
|
for l1 in links:
|
|
|
# print(l1['href'])
|
|
|
all_link.append(l1['href'])
|
|
|
#print(all_link)
|
|
|
for link in all_link:
|
|
|
response1 = requests.get(link, headers=header)
|
|
|
source1 = response1.text
|
|
|
#print(source1)
|
|
|
soup1 = BeautifulSoup(source1, "lxml")
|
|
|
book1=soup1.select('div.post')
|
|
|
for b in book1:
|
|
|
books_name=b.select('div>div.post-name')
|
|
|
for book_name in books_name:
|
|
|
title=book_name.get_text(strip=True)
|
|
|
all_book.append({title})
|
|
|
details= b.select('li')
|
|
|
for detail1 in details:
|
|
|
detail=detail1.get_text()
|
|
|
all_book.append({detail})
|
|
|
contents=b.select('div>p')
|
|
|
for content1 in contents:
|
|
|
content=content1.get_text()
|
|
|
all_book.append({content})
|
|
|
|
|
|
#print(all_book)
|
|
|
filename = "book" + ".csv"
|
|
|
with open(filename,"w",encoding='utf-8') as file:
|
|
|
writer = csv.writer(file)
|
|
|
for a1 in all_book:
|
|
|
writer.writerow(a1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
