ADD file via upload

4 months ago · 78e0e73666
parent 820f7d98a7
commit 78e0e73666
1 changed files with 104 additions and 0 deletions
--- a/bs4nanningzufang0902.py
+++ b/bs4nanningzufang0902.py
@ -0,0 +1,104 @@
+import requests
+from bs4 import BeautifulSoup
+from openpyxl import Workbook
+import time
+import random
+
+wb = Workbook() #创建一个workbook
+def createSheetObj():
+    sheet = wb.active #获取当前workbook的第一个worksheet
+    sheet.title = "58同城" #worksheet的标题
+    sheet.append(['短租房名称', '短租房链接', '短租房地址', "短租房价格 元/日", "短租房居住人数", "出租类型", "短租房户型", "面积/㎡", "用户评论数", "用户评分","用户满意度"])  # 添加一行表头
+    return sheet
+
+def getpage1(): #返回网页源码
+    path = 'nn.html'
+    htmlfile = open(path, 'r', encoding='utf-8') #打开1.html文件
+    htmlhandle = htmlfile.read() #返回整个文件
+    soup = BeautifulSoup(htmlhandle, "lxml") #使用Beautifulsoup解析
+    #print(soup)
+    return soup
+
+def getPage(url):
+    # 伪造身份
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363'
+    }
+
+    resp = requests.get(url, headers=headers)
+    print(resp.status_code)
+    resp.encoding = "utf-8"
+    html = resp.text
+
+    soup = BeautifulSoup(html, "lxml")
+    return soup
+
+
+def parsePage():
+    soup = getpage1()
+    i = 0
+    lis = soup.find_all("a", attrs={"class": "ui-house-item-house-wrapper"}) #找到所有li的链接地址
+    for li in lis:
+        print("第" + str(i) + "条")
+        i = i + 1
+        house_name = li.find('div',class_='ui-house-item-content-area-title').get_text()
+        house_href = 'http:' + li.get('href')
+        house_location = li.find('div',class_='ui-house-item-content-area-introduce2').get_text()
+        house_price = li.find('span', class_='ui-house-item-price-area-nowprice-price').get_text()
+        house_unit = li.find('span', class_='ui-house-item-price-area-nowprice-unit').get_text()
+
+        z = house_unit.split('/')
+
+        if(z[1] == "月"):
+            house_price = int(house_price) // 30
+        house_person_type_huxing = li.find('div', class_='ui-house-item-content-area-introduce3').get_text()
+        #合住房间·4室1床·可住1人·7天起租
+        x = house_person_type_huxing.split('·')
+        house_type = x[0]
+        house_huxing = x[1]
+        house_person = x[2][2] #第三个字
+		#异常/缺省值处理
+        try:
+            house_score_comments = li.find('div', class_='ui-house-item-content-area-introduce1').get_text()
+            y = house_score_comments.split('/')
+            house_score = y[0][0] + y[0][1] + y[0][2]
+            house_comments = y[1][0]
+
+        except:
+            house_score = 0
+            house_comments =0
+
+        try:
+            house_satisfy = li.find('div', class_='ui-house-item-pic-degreeofsatisfaction-number').get_text()
+            s = house_satisfy.split(':')
+            house_satisfy_num = s[1]
+        except:
+            house_satisfy_num = 0
+
+        try:
+            soup1 = getPage(house_href)
+            house_square = soup1.find_all('div',class_='desc')[2].get_text()
+            #print(house_square)
+            house_square = house_square.strip('㎡')
+            #print(house_square)
+        except:
+            house_square = 0
+
+        print(house_name,house_href,house_location,house_price,
+                      house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num)
+
+        sheet.append([house_name,house_href,house_location,house_price,
+                      house_person,house_type,house_huxing,house_square,house_comments,house_score,house_satisfy_num])
+
+        time.sleep(random.random() * 3) #模拟人的行为
+        if (i > 2000):
+            print("爬取结束")
+            break
+    #wb.save("nanning.xlsx") #将最终的结果导出到本地文件
+    #wb.save("nanning.csv")
+    wb.save("nanning.json")
+
+sheet = createSheetObj()
+
+parsePage()
+