|
|
import random
|
|
|
import re
|
|
|
|
|
|
import jieba as jieba
|
|
|
import pymysql
|
|
|
from django.http import HttpResponse
|
|
|
from django.shortcuts import render
|
|
|
from django.views.decorators.clickjacking import xframe_options_sameorigin
|
|
|
from jinja2 import Environment, FileSystemLoader
|
|
|
from pyecharts.faker import Faker
|
|
|
from pyecharts.globals import CurrentConfig
|
|
|
from django.http import HttpResponse
|
|
|
from pyecharts import options as opts
|
|
|
from pyecharts.charts import Bar, Pie
|
|
|
from pyecharts.charts import WordCloud
|
|
|
|
|
|
|
|
|
CurrentConfig.GLOBAL_ENV = Environment(loader=FileSystemLoader("./demo/templates"))
|
|
|
|
|
|
def home(request):
|
|
|
return render(request, 'home.html')
|
|
|
|
|
|
|
|
|
@xframe_options_sameorigin
|
|
|
def page_views(request, page):
|
|
|
# 解析路由，找到返回的页面
|
|
|
# current = request.path.split('/')[-1]
|
|
|
# return render(request, current + '.html')
|
|
|
return render(request, 'page' + str(page) + '.html')
|
|
|
|
|
|
|
|
|
def table_one(request):
|
|
|
x = []
|
|
|
xaxis = []
|
|
|
yaxis = []
|
|
|
|
|
|
conn = pymysql.connect(host='120.79.165.140',
|
|
|
user='kino',
|
|
|
passwd="student",
|
|
|
db='python_web_spider_DB')
|
|
|
cur = conn.cursor()
|
|
|
|
|
|
sql1 = "SELECT distinct 规格 from wh_xinfadi"
|
|
|
sql2 = "SELECT 规格,count(distinct 蔬菜种类) from wh_xinfadi group BY 规格"
|
|
|
cur.execute(sql1)
|
|
|
|
|
|
for sp in cur:
|
|
|
init = str(sp)
|
|
|
if '\'' in init:
|
|
|
this_kind = init.split('\'')[1]
|
|
|
else:
|
|
|
this_kind = init
|
|
|
if re.match(r'^([\u4e00-\u9fa5]+类)$', this_kind):
|
|
|
x.append(this_kind)
|
|
|
|
|
|
cur.execute(sql2)
|
|
|
for it in cur:
|
|
|
if it[0] in x:
|
|
|
xaxis.append(it[0])
|
|
|
yaxis.append(it[1])
|
|
|
c = (
|
|
|
Bar()
|
|
|
.extend_axis(xaxis)
|
|
|
.add_xaxis(xaxis)
|
|
|
.add_yaxis('北京新发地菜市', yaxis,)
|
|
|
.set_global_opts(
|
|
|
title_opts=opts.TitleOpts(
|
|
|
title="北京新发地菜市场",
|
|
|
subtitle="产品种类分析"
|
|
|
),
|
|
|
datazoom_opts=opts.DataZoomOpts(),
|
|
|
)
|
|
|
)
|
|
|
# conn.commit() # 记得提交
|
|
|
cur.close()
|
|
|
conn.close()
|
|
|
return HttpResponse(c.render_embed('北京新发地菜市场产品种类分析.html'))
|
|
|
|
|
|
|
|
|
def pie_views(requset):
|
|
|
data = [
|
|
|
['1990年及以前', 0],
|
|
|
['1991到1995年', 0],
|
|
|
['1996到2000年', 0],
|
|
|
['2001到2005年', 0],
|
|
|
['2006到2010年', 0],
|
|
|
['2011到2015年', 0],
|
|
|
['2016年至今', 0],
|
|
|
]
|
|
|
colors = []
|
|
|
while True:
|
|
|
color = random.choice(
|
|
|
[
|
|
|
"#c23531",
|
|
|
"#2f4554",
|
|
|
"#61a0a8",
|
|
|
"#d48265",
|
|
|
"#749f83",
|
|
|
"#ca8622",
|
|
|
"#bda29a",
|
|
|
"#6e7074",
|
|
|
"#546570",
|
|
|
"#c4ccd3",
|
|
|
"#f05b72",
|
|
|
"#444693",
|
|
|
"#726930",
|
|
|
"#b2d235",
|
|
|
"#6d8346",
|
|
|
"#ac6767",
|
|
|
"#1d953f",
|
|
|
"#6950a1",
|
|
|
]
|
|
|
)
|
|
|
if color not in colors:
|
|
|
colors.append(color)
|
|
|
if len(colors) == len(data):
|
|
|
break
|
|
|
conn = pymysql.connect(host='120.79.165.140',
|
|
|
user='kino',
|
|
|
passwd="student",
|
|
|
db='python_web_spider_DB')
|
|
|
cur = conn.cursor()
|
|
|
|
|
|
sql1 = "SELECT 上映年份 from wh_doubanmovie"
|
|
|
|
|
|
cur.execute(sql1)
|
|
|
for i in cur:
|
|
|
year = int(str(i).split('\'')[1])
|
|
|
if year <= 1990:
|
|
|
data[0][1] += 1
|
|
|
elif 1990 < year <= 1995:
|
|
|
data[1][1] += 1
|
|
|
elif 1995 < year <= 2000:
|
|
|
data[2][1] += 1
|
|
|
elif 2000 < year <= 2005:
|
|
|
data[3][1] += 1
|
|
|
elif 2005 < year <= 2010:
|
|
|
data[4][1] += 1
|
|
|
elif 2010 < year <= 2015:
|
|
|
data[5][1] += 1
|
|
|
elif 2015 < year:
|
|
|
data[6][1] += 1
|
|
|
|
|
|
c = (
|
|
|
Pie()
|
|
|
.add("", data)
|
|
|
.set_colors(colors)
|
|
|
# .add("", [list(z) for z in zip(Faker.choose(), Faker.values())])
|
|
|
# .set_colors(["blue", "green", "yellow", "red", "pink", "orange", "purple"])
|
|
|
.set_global_opts(title_opts=opts.TitleOpts(title=""))
|
|
|
.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
|
|
|
)
|
|
|
cur.close()
|
|
|
conn.close()
|
|
|
return HttpResponse(c.render_embed('豆瓣电影排行top250年份分布.html'))
|
|
|
|
|
|
|
|
|
def word_cloud(request):
|
|
|
x = []
|
|
|
data = []
|
|
|
delete = ['的','你','我','他','说','了','话','这','是','吗','吧','都',
|
|
|
'也','吗','吧','都','不','为','他们','啊','则','和','在',]
|
|
|
conn = pymysql.connect(host='120.79.165.140',
|
|
|
user='kino',
|
|
|
passwd="student",
|
|
|
db='python_web_spider_DB')
|
|
|
cur = conn.cursor()
|
|
|
|
|
|
sql1 = "SELECT comments from xjh_wangyiyun"
|
|
|
|
|
|
cur.execute(sql1)
|
|
|
txt = ''
|
|
|
for i in cur:
|
|
|
ss = str(i)
|
|
|
if '\'' in ss:
|
|
|
txt += ss.split('\'')[1]
|
|
|
ls = jieba.lcut(txt)
|
|
|
for item in ls:
|
|
|
if re.match(r'^([\u4e00-\u9fa5]+)$', item):
|
|
|
x.append(item)
|
|
|
for item in x[::]:
|
|
|
t = (item, x.count(item)*3)
|
|
|
data.append(t)
|
|
|
while item in x:
|
|
|
x.remove(item)
|
|
|
for item in data:
|
|
|
if item[1] == 0 or item[0] in delete:
|
|
|
data.remove(item)
|
|
|
data.sort(key=lambda x: x in data)
|
|
|
|
|
|
c = (
|
|
|
WordCloud()
|
|
|
.add(series_name="歌曲:Mood 的评论高频词", data_pair=data, word_size_range=[10, 70])
|
|
|
.set_global_opts(
|
|
|
title_opts=opts.TitleOpts(
|
|
|
title="网易云歌曲评论词云", title_textstyle_opts=opts.TextStyleOpts(font_size=23)
|
|
|
),
|
|
|
tooltip_opts=opts.TooltipOpts(is_show=True),
|
|
|
)
|
|
|
)
|
|
|
|
|
|
cur.close()
|
|
|
conn.close()
|
|
|
|
|
|
return HttpResponse(c.render_embed('网易云歌曲评论词云.html'))
|
|
|
|
|
|
# 从这里写你们的爬虫函数,例：
|
|
|
# def spider_fun(url, web_name):
|
|
|
# pass
|
|
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
import urllib.request,urllib.error #这里用urllib库实现requests库功能
|
|
|
import os
|
|
|
import re
|
|
|
import pandas as pd
|
|
|
|
|
|
#正则提取信息
|
|
|
#findLink = re.compile(r'href="(.*?)"') #提取网址
|
|
|
findTitle = re.compile(r'target="_blank">(.*?)</a>') #提取标题
|
|
|
findPrice = re.compile(r'<span class="highlight">(.*?)</span>') #提取价格
|
|
|
findTag = re.compile(r'/" target="_blank">(.*?)</a></div>') #提取商品类型
|
|
|
findPlace = re.compile(r'<div class="ad-item-detail">(.*?)</div>') #提取地址
|
|
|
|
|
|
def askURL(url): #访问网站获取信息
|
|
|
head = {
|
|
|
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'
|
|
|
}
|
|
|
request = urllib.request.Request(url,headers=head)
|
|
|
html = ""
|
|
|
try:
|
|
|
response = urllib.request.urlopen(request)
|
|
|
html = response.read().decode('utf-8')
|
|
|
#print(html) #test
|
|
|
except urllib.error.URLError as e:
|
|
|
if hasattr(e,'code'):
|
|
|
print(e.code)
|
|
|
if hasattr(e,'reason'):
|
|
|
print(e.reason)
|
|
|
return html
|
|
|
|
|
|
def getData(baseurl): #提取需要的信息
|
|
|
datalist = []
|
|
|
for i in range(1,4): #调用访问网站函数，访问每一页的信息,这里只访问了几页
|
|
|
url = baseurl + str(i)
|
|
|
html = askURL(url)
|
|
|
|
|
|
soup = BeautifulSoup(html,"html.parser")
|
|
|
for item in soup.find_all('div',class_="media-body"): #提取信息
|
|
|
#print(item) #test
|
|
|
data = []
|
|
|
|
|
|
item = str(item)
|
|
|
title = re.findall(findTitle,item)[0]
|
|
|
link = re.findall(findPlace,item)[0]
|
|
|
price = re.findall(findPrice,item)[0]
|
|
|
tag = re.findall(findTag,item)[0]
|
|
|
|
|
|
data.append(title)
|
|
|
data.append(link)
|
|
|
data.append(price)
|
|
|
data.append(tag)
|
|
|
datalist.append(data)
|
|
|
|
|
|
return datalist
|
|
|
|
|
|
def saveData(savepath,datalist,web_name): #保存文件
|
|
|
name = ["标题","地址","价格","类型"]
|
|
|
file = pd.DataFrame(columns=name,data=datalist) #整合表头和数据
|
|
|
file.to_csv(savepath+'/lyh_tiaozaomarket.csv') #保存至当前路径，命名为xxx.csv
|
|
|
print('已保存%s信息' % web_name)
|
|
|
|
|
|
|
|
|
'''---------代码开始的地方---------'''
|
|
|
|
|
|
def begin_spider(url, web_name):
|
|
|
url='https://guilin.baixing.com/ershou/?page='
|
|
|
web_name='桂林百姓网二手市场'
|
|
|
savepath = os.getcwd() #获取当前路径作为保存路径
|
|
|
datalist = getData(url)
|
|
|
saveData(savepath,datalist,web_name)
|
|
|
|
|
|
import csv
|
|
|
from lxml import etree
|
|
|
import requests
|
|
|
|
|
|
|
|
|
def begin_spider(url, web_name):
|
|
|
url = "https://guilin.zbj.com/search/f/?type=new&kw=saas"
|
|
|
# 设置headers，防止UA验证，Host为要爬取的域名,通过浏览器F12获取User-Agent
|
|
|
headers = {
|
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.51"
|
|
|
}
|
|
|
response = requests.get(url=url, headers=headers)
|
|
|
html = etree.HTML(response.text) #通过etree解析文本内容
|
|
|
divs = html.xpath("/html/body/div[6]/div/div/div[2]/div[6]/div[1]/div") #通过浏览器的F12获取对应页面的xpath，
|
|
|
f = open("data.csv", 'w', encoding='utf-8')
|
|
|
csv_writer = csv.writer(f)
|
|
|
for div in divs:
|
|
|
|
|
|
price_temp = div.xpath("./div/div/a[1]/div[2]/div[1]/span[1]/text()") #找到价格位置的Xpath
|
|
|
title_temp = div.xpath("./div/div/a[1]/div[2]/div[2]/p/text()")#找到项目名称位置的Xpath
|
|
|
Company_temp = div.xpath("./div/div/a[2]/div[1]/p/text()")#找到公司名字位置的Xpath
|
|
|
Address_temp = div.xpath("./div/div/a[2]/div[1]/div/span/text()")#找到公司地点位置的Xpath
|
|
|
# 以下均为对数据的处理
|
|
|
if len(price_temp) != 0 and len(title_temp) != 0 and len(Company_temp) != 0 and len(Address_temp) != 0:#为了处理一些较为特殊的数据，所以加次判断
|
|
|
price = price_temp[0].strip("¥") #去除价格的符号
|
|
|
title = "SAAS".join(title_temp) #将项目名称做美化
|
|
|
Company = Company_temp[0] #去除括号
|
|
|
Address = Address_temp[0] #去除括号
|
|
|
csv_writer.writerow([price, title, Company, Address]) #写入文件
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
begin_spider("https://guilin.zbj.com/search/f/?type=new&kw=saas", "猪八戒")
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
main()
|
|
|
|
|
|
|
|
|
|
|
|
import json
|
|
|
import time
|
|
|
import requests
|
|
|
|
|
|
# 设置headers，防止UA验证，Host为要爬取的域名,通过浏览器F12获取User-Agent
|
|
|
headers = {
|
|
|
'Host': 'music.163.com',
|
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.51"
|
|
|
}
|
|
|
|
|
|
|
|
|
def begin_spider(page, url, web_name):
|
|
|
"""
|
|
|
获取评论信息
|
|
|
"""
|
|
|
|
|
|
url = 'https://music.163.com/api/v1/resource/comments/R_SO_4_483671599?limit=10&offset=' + str(page) #调用网易云音乐评论区的api
|
|
|
response = requests.get(url=url, headers=headers)#获取请求
|
|
|
print(response.status_code)
|
|
|
# 将字符串转为json格式
|
|
|
result = json.loads(response.text)
|
|
|
items = result['comments'] #从一大堆的文字中提取评论区的内容，这个地方不懂得话，可以debug看一下
|
|
|
for item in items: #开始提取评论区内容
|
|
|
# 用户名
|
|
|
user_name = item['user']['nickname'].replace(',', '，') #从json串获取到用户名
|
|
|
# 评论内容
|
|
|
comment = item['content'].strip().replace('', '').replace(',', '，') #从json中获取评论内容
|
|
|
# 评论点赞数
|
|
|
praise = str(item['likedCount']) #从json中获取评论点赞数
|
|
|
# 评论时间
|
|
|
date = time.localtime(int(str(item['time'])[:10])) #从json中获取评论时间
|
|
|
date = time.strftime("%Y-%m-%d %H:%M:%S", date)
|
|
|
|
|
|
with open('test.csv', 'a', encoding='utf-8-sig') as f: #保存到文件中
|
|
|
f.write(user_name + ',' + comment + ',' + praise + ',' + date + '\n')
|
|
|
f.close()
|
|
|
|
|
|
|
|
|
def main():
|
|
|
for i in range(0, 100, 20):
|
|
|
begin_spider(i,"https://music.163.com/","网易云")
|
|
|
time.sleep(1)
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
main()
|
