You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
105 lines
3.9 KiB
105 lines
3.9 KiB
import requests
|
|
from bs4 import BeautifulSoup
|
|
from openpyxl import Workbook
|
|
import time
|
|
import random
|
|
|
|
wb = Workbook() #创建一个workbook
|
|
def createSheetObj():
|
|
sheet = wb.active #获取当前workbook的第一个worksheet
|
|
sheet.title = "58同城" #worksheet的标题
|
|
sheet.append(['短租房名称', '短租房链接', '短租房地址', "短租房价格 元/日", "短租房居住人数", "出租类型", "短租房户型", "面积/㎡", "用户评论数", "用户评分","用户满意度"]) # 添加一行表头
|
|
return sheet
|
|
|
|
def getpage1(): #返回网页源码
|
|
path = 'nn.html'
|
|
htmlfile = open(path, 'r', encoding='utf-8') #打开1.html文件
|
|
htmlhandle = htmlfile.read() #返回整个文件
|
|
soup = BeautifulSoup(htmlhandle, "lxml") #使用Beautifulsoup解析
|
|
#print(soup)
|
|
return soup
|
|
|
|
def getPage(url):
|
|
# 伪造身份
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363'
|
|
}
|
|
|
|
resp = requests.get(url, headers=headers)
|
|
print(resp.status_code)
|
|
resp.encoding = "utf-8"
|
|
html = resp.text
|
|
|
|
soup = BeautifulSoup(html, "lxml")
|
|
return soup
|
|
|
|
|
|
def parsePage():
|
|
soup = getpage1()
|
|
i = 0
|
|
lis = soup.find_all("a", attrs={"class": "ui-house-item-house-wrapper"}) #找到所有li的链接地址
|
|
for li in lis:
|
|
print("第" + str(i) + "条")
|
|
i = i + 1
|
|
house_name = li.find('div',class_='ui-house-item-content-area-title').get_text()
|
|
house_href = 'http:' + li.get('href')
|
|
house_location = li.find('div',class_='ui-house-item-content-area-introduce2').get_text()
|
|
house_price = li.find('span', class_='ui-house-item-price-area-nowprice-price').get_text()
|
|
house_unit = li.find('span', class_='ui-house-item-price-area-nowprice-unit').get_text()
|
|
|
|
z = house_unit.split('/')
|
|
|
|
if(z[1] == "月"):
|
|
house_price = int(house_price) // 30
|
|
house_person_type_huxing = li.find('div', class_='ui-house-item-content-area-introduce3').get_text()
|
|
#合住房间·4室1床·可住1人·7天起租
|
|
x = house_person_type_huxing.split('·')
|
|
house_type = x[0]
|
|
house_huxing = x[1]
|
|
house_person = x[2][2] #第三个字
|
|
#异常/缺省值处理
|
|
try:
|
|
house_score_comments = li.find('div', class_='ui-house-item-content-area-introduce1').get_text()
|
|
y = house_score_comments.split('/')
|
|
house_score = y[0][0] + y[0][1] + y[0][2]
|
|
house_comments = y[1][0]
|
|
|
|
except:
|
|
house_score = 0
|
|
house_comments =0
|
|
|
|
try:
|
|
house_satisfy = li.find('div', class_='ui-house-item-pic-degreeofsatisfaction-number').get_text()
|
|
s = house_satisfy.split(':')
|
|
house_satisfy_num = s[1]
|
|
except:
|
|
house_satisfy_num = 0
|
|
|
|
try:
|
|
soup1 = getPage(house_href)
|
|
house_square = soup1.find_all('div',class_='desc')[2].get_text()
|
|
#print(house_square)
|
|
house_square = house_square.strip('㎡')
|
|
#print(house_square)
|
|
except:
|
|
house_square = 0
|
|
|
|
print(house_name,house_href,house_location,house_price,
|
|
house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num)
|
|
|
|
sheet.append([house_name,house_href,house_location,house_price,
|
|
house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num])
|
|
|
|
time.sleep(random.random() * 3) #模拟人的行为
|
|
if (i > 2000):
|
|
print("爬取结束")
|
|
break
|
|
#wb.save("nanning.xlsx") #将最终的结果导出到本地文件
|
|
#wb.save("nanning.csv")
|
|
wb.save("nanning.json")
|
|
|
|
sheet = createSheetObj()
|
|
|
|
parsePage()
|
|
|