ADD file via upload

spider.py

@@ -0,0 +1,230 @@
+from bs4 import BeautifulSoup
+import re
+import urllib.error,urllib.request
+import xlwt
+import sqlite3
+
+
+#对于豆瓣top250电影爬取内容的正则表达式
+findLink = re.compile(r'<a href="(.*?)">')
+findImgSrc = re.compile(r'<img.*src="(.*?)"',re.S)
+findTitle = re.compile(r'<span class="title">(.*)</span>')
+findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
+findJudge = re.compile(r'<span>(\d*)人评价</span>')
+findBd = re.compile(r'<p class="">(.*?)</p>',re.S)
+
+
+#对于豆瓣当前某城市正在上映的电影爬取内容的正则表达式
+findnowmovielink = re.compile(r'<a data-psource="poster" href="(.*?)" target="_blank">',re.S)
+findnowmovieimg = re.compile(r'src="(.*?)"/>')
+findnowmovietitle= re.compile(r'data-title="(.*?)"')
+findnowmovieactors= re.compile(r'data-actors="(.*?)"')
+findnowmovietime= re.compile(r'data-duration="(.*?)"')
+findnowmovieregion= re.compile(r'data-region="(.*?)"')
+findnowmoviedirector= re.compile(r'data-director="(.*?)"')
+findnowmovieid = re.compile(r'id="(.*?)"')
+
+
+
+def main():
+    a = input("Enter a number: ")
+    if a=="1":
+        baseurl = "https://movie.douban.com/top250?start="  #指定的网页超链接
+        datalist = getData1(baseurl)  #返回一个装有所有数据的列表
+        savepath = ".\\doubanmovietop250.xls"  #存储数据的文档路径
+        saveData1(datalist,savepath)  #存储数据
+        dbpath = "doubanmovietop250.db"
+        saveData2DB1(datalist,dbpath)
+    if a=="2":
+        b = input("Enter a city: ")
+        baseurl = "https://movie.douban.com/cinema/nowplaying/" + b + '/'
+        c = getData2(baseurl)
+        savepath = ".\\doubannowplaying.xls"
+        saveData2(c,savepath)
+
+
+
+
+
+#用于获得数据
+def getData1(baseurl):
+    datalist = []  #将获得的数据装于datalist这个列表
+    for i in range(0,10):  #网页一共10页，遍历爬取
+        url = baseurl + str(i*25)
+        html = askURL1(url)
+        soup = BeautifulSoup(html,"html.parser")  #解析
+        for item in soup.find_all('div',class_= "item"):
+            #print(item)  #一个测试点
+            data = []  #将每一部电影的具体信息装入data列表
+            item = str(item)
+            link = re.findall(findLink,item)[0]  #详情链接
+            data.append(link)
+            imgSrc = re.findall(findImgSrc,item)[0]  #图片链接
+            data.append(imgSrc)
+            titles = re.findall(findTitle,item)  #标题
+            if(len(titles) == 2 ):
+                ctitle = titles[0]
+                data.append(ctitle)
+                otitle = titles[1].replace("/","")
+                data.append(otitle)
+            else:
+                data.append(titles[0])
+                data.append(" ")
+            rating = re.findall(findRating,item)[0]  #评分
+            data.append(rating)
+            judgeNum = re.findall(findJudge,item)[0]  #评分人数
+            data.append(judgeNum)
+            bd = re.findall(findBd,item)[0]  #概要
+            bd = re.sub('\s',"",bd)
+            bd = re.sub(r'/',"",bd)
+            bd = re.sub('<br>',"",bd)
+            data.append(bd.strip())
+            datalist.append(data)  #将data列表装入datalist列表
+    return datalist
+
+
+#用来得到一个指定URL的网页内容(豆瓣top250)
+def askURL1(url):
+    #伪装为浏览器
+    head = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55"
+    }
+    resquest = urllib.request.Request(url,headers=head)
+    html = ""
+    try:
+        response = urllib.request.urlopen(resquest)
+        html = response.read().decode("utf-8")
+        #print(html)
+    except urllib.error.URLError as e:
+        if hasattr(e,"code"):
+            print(e.code)
+        if hasattr(e,"reason"):
+            print(e.reason)
+    return html
+
+
+def saveData1(datalist,savepath):
+    print("save...")
+    book = xlwt.Workbook(encoding="utf-8",style_compression=0)
+    sheet = book.add_sheet("豆瓣电影Top250",cell_overwrite_ok=True)
+    col = ("电影详情链接","图片链接","影片中文名字","影片外国名字","评分","评价数","相关信息")
+    for i in range(0,7):
+        sheet.write(0,i,col[i])
+    for i in range(0,250):
+        print("第%d条"%(i+1))
+        data = datalist[i]
+        for j in range(0,7):
+            sheet.write(i,j,data[j])
+    book.save("student.xls")
+
+
+def saveData2DB1(datalist,dbpath):
+    init_db1(dbpath)
+    conn = sqlite3.connect(dbpath)
+    cur = conn.cursor()
+    for data in datalist:
+        for index in range(len(data)):
+            if index == 4 or index == 5:
+                continue
+            data[index] = '"'+data[index].strip()+'"'
+        sql = '''
+              insert into movie250(info_link,pic_link,cname,ename,score,rated,info)
+              values(%s)'''%",".join(data)
+
+        cur.execute(sql)
+        conn.commit()
+    cur.close()
+    conn.close()
+
+
+def init_db1(dbpath):
+    sql = '''
+          create table movie250
+          (
+          id integer primary key autoincrement,
+          info_link text,
+          pic_link text,
+          cname varchar ,
+          ename varchar ,
+          score numeric ,
+          rated numeric ,
+          info text
+          )
+    '''
+    conn = sqlite3.connect(dbpath)
+    cursor = conn.cursor()
+    cursor.execute(sql)
+    conn.commit()
+    conn.close()
+
+
+
+def askURL2(url):
+        # 伪装为浏览器
+        head = {
+            "User-Agent":" Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36"
+        }
+        resquest = urllib.request.Request(url, headers=head)
+        html = ""
+        try:
+            response = urllib.request.urlopen(resquest)
+            html = response.read().decode("utf-8")
+            print(html)
+        except urllib.error.URLError as e:
+            if hasattr(e, "code"):
+                print(e.code)
+            if hasattr(e, "reason"):
+                print(e.reason)
+        return html
+
+
+def getData2(baseurl):
+    datalist = []  #将获得的数据装于datalist这个列表
+    html = askURL1(baseurl)
+    soup = BeautifulSoup(html,"html.parser")  #解析
+    for item in soup.find_all('li',class_= "list-item"):
+            print(item)  #一个测试点
+            data = []  #将每一部电影的具体信息装入data列表
+            item = str(item)
+            link = re.findall(findnowmovielink,item)[0]
+            data.append(link)
+            img = re.findall(findnowmovieimg,item)[0]
+            data.append(img)
+            title = re.findall(findnowmovietitle,item)[0]
+            data.append(title)
+            director = re.findall(findnowmoviedirector, item)[0]
+            data.append(director)
+            actors = re.findall(findnowmovieactors,item)[0]
+            data.append(actors)
+            time = re.findall(findnowmovietime, item)[0]
+            data.append(time)
+            region = re.findall(findnowmovieregion, item)[0]
+            data.append(region)
+            id = re.findall(findnowmovieid,item)[0]
+            data.append(id)
+            datalist.append(data)  #将data列表装入datalist列表
+    print(datalist)
+    return datalist
+
+
+
+def saveData2(datalist,savepath):
+    print("save...")
+    book = xlwt.Workbook(encoding="utf-8",style_compression=0)
+    sheet = book.add_sheet("豆瓣电影Top250",cell_overwrite_ok=True)
+    col = ("电影详情链接","图片链接","影片中文名字","导演","演员","时长","制片地区","电影id")
+    for i in range(0,8):
+        sheet.write(0,i,col[i])
+    for i in range(len(datalist)):
+        print("第%d条"%(i+1))
+        data = datalist[i]
+        for j in range(0,8):
+            sheet.write(i,j,data[j])
+    book.save("citymovie.xls")
+
+
+
+
+if __name__ == "__main__":
+    main()
+    print("爬取完成!")