diff --git a/bs4nanningzufang0902.py b/bs4nanningzufang0902.py new file mode 100644 index 0000000..6583e22 --- /dev/null +++ b/bs4nanningzufang0902.py @@ -0,0 +1,104 @@ +import requests +from bs4 import BeautifulSoup +from openpyxl import Workbook +import time +import random + +wb = Workbook() #创建一个workbook +def createSheetObj(): + sheet = wb.active #获取当前workbook的第一个worksheet + sheet.title = "58同城" #worksheet的标题 + sheet.append(['短租房名称', '短租房链接', '短租房地址', "短租房价格 元/日", "短租房居住人数", "出租类型", "短租房户型", "面积/㎡", "用户评论数", "用户评分","用户满意度"]) # 添加一行表头 + return sheet + +def getpage1(): #返回网页源码 + path = 'nn.html' + htmlfile = open(path, 'r', encoding='utf-8') #打开1.html文件 + htmlhandle = htmlfile.read() #返回整个文件 + soup = BeautifulSoup(htmlhandle, "lxml") #使用Beautifulsoup解析 + #print(soup) + return soup + +def getPage(url): + # 伪造身份 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363' + } + + resp = requests.get(url, headers=headers) + print(resp.status_code) + resp.encoding = "utf-8" + html = resp.text + + soup = BeautifulSoup(html, "lxml") + return soup + + +def parsePage(): + soup = getpage1() + i = 0 + lis = soup.find_all("a", attrs={"class": "ui-house-item-house-wrapper"}) #找到所有li的链接地址 + for li in lis: + print("第" + str(i) + "条") + i = i + 1 + house_name = li.find('div',class_='ui-house-item-content-area-title').get_text() + house_href = 'http:' + li.get('href') + house_location = li.find('div',class_='ui-house-item-content-area-introduce2').get_text() + house_price = li.find('span', class_='ui-house-item-price-area-nowprice-price').get_text() + house_unit = li.find('span', class_='ui-house-item-price-area-nowprice-unit').get_text() + + z = house_unit.split('/') + + if(z[1] == "月"): + house_price = int(house_price) // 30 + house_person_type_huxing = li.find('div', class_='ui-house-item-content-area-introduce3').get_text() + #合住房间·4室1床·可住1人·7天起租 + x = house_person_type_huxing.split('·') + house_type = x[0] + house_huxing = x[1] + house_person = x[2][2] #第三个字 + #异常/缺省值处理 + try: + house_score_comments = li.find('div', class_='ui-house-item-content-area-introduce1').get_text() + y = house_score_comments.split('/') + house_score = y[0][0] + y[0][1] + y[0][2] + house_comments = y[1][0] + + except: + house_score = 0 + house_comments =0 + + try: + house_satisfy = li.find('div', class_='ui-house-item-pic-degreeofsatisfaction-number').get_text() + s = house_satisfy.split(':') + house_satisfy_num = s[1] + except: + house_satisfy_num = 0 + + try: + soup1 = getPage(house_href) + house_square = soup1.find_all('div',class_='desc')[2].get_text() + #print(house_square) + house_square = house_square.strip('㎡') + #print(house_square) + except: + house_square = 0 + + print(house_name,house_href,house_location,house_price, + house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num) + + sheet.append([house_name,house_href,house_location,house_price, + house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num]) + + time.sleep(random.random() * 3) #模拟人的行为 + if (i > 2000): + print("爬取结束") + break + #wb.save("nanning.xlsx") #将最终的结果导出到本地文件 + #wb.save("nanning.csv") + wb.save("nanning.json") + +sheet = createSheetObj() + +parsePage() +