4.3 KiB
educoder
#爬虫作为获取网络信息的重要手段，应该是每个计算机学习者的必备技能。我们小组在跟随头歌教程了解了爬虫的基本原理和流程之后，完成了课内的任务，并进行了简单的课外任务选取和实施。 #另外，笔者（组长）在CSDN社区了解了一些关于爬虫的相关文章之后，认识到，爬虫是一项强大而有广泛应用场景的计算机技术，真正技术高超的爬虫工程师可以完成对许多难以获取的信息的爬取，短期的学习，只能浅入这项技术的表层，实现对结构简单的网页的爬取。故本小组将在实操方面只实现对简单网页的文本爬取。 import requests url = "https://www.bitpush.news/covid19/" headers = { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36" } response = requests.get(url, headers=headers) html = response.text from lxml import etree parse = etree.HTMLParser(encoding='utf-8') doc = etree.HTML(html) country = doc.xpath('//div[@class="table_container"]//tbody/tr/td/span/text()')
person = doc.xpath('//div[@class="table_container"]//tbody/tr/td[2]/text()')
person = [x.replace(",", "") for x in person]
death = doc.xpath('//div[@class="table_container"]//tbody/tr/td[3]/text()')
death = [x.replace(",", "") for x in death] message = list(zip(country, person, death)) import csv with open("全球以及美国各州疫情数据爬取.csv", "w") as f: w = csv.writer(f) w.writerows(message) import pandas as pd
df = pd.read_csv("C://Users/86184/Desktop/Computer Desk/期末用/全球疫情数据爬取.csv",names=['area','person','death'],engine= "python",encoding='gbk') import matplotlib.pyplot as plt plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['figure.figsize'] = (100,50) x = df["area"].values y = df["death"].values plt.bar(x, y) plt.xlabel("国家/地区",fontsize=14) plt.ylabel("死亡人数",fontsize=14) plt.show() #在上述代码中，我们本质上实现了对网页html特定并列标签下内容的获取，但在一个复杂的网页html语言中寻找特定标签本就是一件繁琐的事情，所以在下一段代码中，我们利用列表实现了对并列标签下部分内容的截取，即获取美国各州疫情数据的获取。 import requests url = "https://www.bitpush.news/covid19/" headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"} response = requests.get(url, headers=headers) html = response.text from lxml import etree parse = etree.HTMLParser(encoding='utf-8') doc = etree.HTML(html) area = doc.xpath('//div[@class="table_container"]//tbody/tr/td/span/text()') person = doc.xpath('//div[@class="table_container"]//tbody/tr/td[2]/text()') person = [x.replace(",", "") for x in person] death = doc.xpath('//div[@class="table_container"]//tbody/tr/td[3]/text()') death = [x.replace(",", "") for x in death] message = list(zip(area, person, death)) import csv with open("美国数据.csv", "w") as f: w = csv.writer(f) w.writerows(message[-58:]) import pandas as pd
df = pd.read_csv("C://Users/86184/Desktop/Computer Desk/期末用/美国数据.csv",names=['area','person','death'],engine= "python",encoding='gbk') import matplotlib.pyplot as plt plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['figure.figsize'] = (100,50) x = df["area"].values y = df["death"].values plt.bar(x, y) plt.xlabel("州",fontsize=14) plt.ylabel("死亡人数",fontsize=14) plt.show() #最后，我们利用爬虫完成了对于菜鸟网站上列表的教程信息获取。 import requests url = "https://www.runoob.com/python3/python3-list.html" headers = { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36" } response = requests.get(url, headers=headers) html = response.text from lxml import etree parse = etree.HTMLParser(encoding='utf-8') doc = etree.HTML(html) tx1=doc.xpath('/html/body/div[4]/div/div[2]/div/div[3]/div/h1/text()') tx2=doc.xpath('/html/body/div[4]/div/div[2]/div/div[3]/div/p/text()') tx3=[] for i in range(len(tx2)): if i!=len(tx2)-1: tx3.append(tx2[i]) tx3.append('\n')
with open('菜鸟网页列表爬取.txt','w',encoding='utf-8') as f: f.writelines(tx3) #另外，数据可视化图片附文档内