first commit

master
planet 6 years ago
commit 1b97c3bb70

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="educoder_01" />
</profile>
</annotationProcessing>
</component>
</project>

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" defaultCharsetForPropertiesFiles="UTF-8" addBOMForNewFiles="with NO BOM">
<file url="PROJECT" charset="UTF-8" />
</component>
</project>

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

@ -0,0 +1,486 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="de56d332-3e9a-4691-80b0-2045792b3ea0" name="Default Changelist" comment="" />
<ignored path="$PROJECT_DIR$/out/" />
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/entity/positionInfo.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="913">
<caret line="94" column="13" lean-forward="true" selection-start-line="94" selection-start-column="13" selection-end-line="94" selection-end-column="13" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1617#1618#0" expanded="true" />
<element signature="e#1642#1643#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/demo.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="23" column="17" selection-start-line="23" selection-start-column="8" selection-end-line="23" selection-end-column="17" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Class" />
</list>
</option>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/java/net/educoder/App.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/demo2.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/dom4j.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/entity/PositionInfo.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/mr.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/entity/positionInfo.java" />
<option value="$PROJECT_DIR$/src/main/java/net/educoder/demo.java" />
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="userSettingsFile" value="D:\java_Tool\apache-maven-3.3.9\conf\settings.xml" />
</MavenGeneralSettings>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="7">
<option name="x" value="-10" />
<option name="y" value="-10" />
<option name="width" value="1385" />
<option name="height" value="745" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="educoder_01" type="b2602c69:ProjectViewProjectNode" />
<item name="educoder_01" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="educoder_01" type="b2602c69:ProjectViewProjectNode" />
<item name="educoder_01" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="educoder_01" type="b2602c69:ProjectViewProjectNode" />
<item name="educoder_01" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="educoder_01" type="b2602c69:ProjectViewProjectNode" />
<item name="educoder_01" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="educoder_01" type="b2602c69:ProjectViewProjectNode" />
<item name="educoder_01" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="educoder" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="com.android.tools.idea.instantapp.provision.ProvisionBeforeRunTaskProvider.myTimeStamp" value="1551942685955" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../jobs" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="nodejs_package_manager_path" value="npm" />
<property name="project.structure.last.edited" value="Project" />
<property name="project.structure.proportion" value="0.0" />
<property name="project.structure.side.proportion" value="0.0" />
<property name="settings.editor.selected.configurable" value="preferences.pluginManager" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Application.mr">
<configuration default="true" type="Applet">
<option name="POLICY_FILE" value="$APPLICATION_HOME_DIR$/bin/appletviewer.policy" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration name="mr" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="net.educoder.mr" />
<module name="educoder_01" />
<option name="PROGRAM_PARAMETERS" value="F:\java_work\educoder\content.txt F:\java_work\educoder\out" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration name="dom4j.testGetRoot" type="JUnit" factoryName="JUnit" temporary="true" nameIsGenerated="true">
<module name="educoder_01" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="net.educoder.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<option name="PACKAGE_NAME" value="net.educoder" />
<option name="MAIN_CLASS_NAME" value="net.educoder.dom4j" />
<option name="METHOD_NAME" value="testGetRoot" />
<option name="TEST_OBJECT" value="method" />
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="TestNG">
<option name="WORKING_DIRECTORY" value="$MODULE_DIR$" />
<properties />
<listeners />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="#org.jetbrains.idea.devkit.run.PluginConfigurationType">
<module name="" />
<option name="VM_PARAMETERS" value="-Xmx512m -Xms256m -XX:MaxPermSize=250m -ea" />
<option name="PROGRAM_PARAMETERS" />
<predefined_log_file enabled="true" id="idea.log" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<list>
<item itemvalue="Application.mr" />
<item itemvalue="JUnit.dom4j.testGetRoot" />
</list>
<recent_temporary>
<list>
<item itemvalue="JUnit.dom4j.testGetRoot" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="de56d332-3e9a-4691-80b0-2045792b3ea0" name="Default Changelist" comment="" />
<created>1551836077537</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1551836077537</updated>
<workItem from="1551836082985" duration="25652000" />
</task>
<servers />
</component>
<component name="TestHistory">
<history-entry file="dom4j_testGetRoot - 2019.03.06 at 15h 03m 41s.xml">
<configuration name="dom4j.testGetRoot" configurationId="JUnit" />
</history-entry>
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="25652000" />
</component>
<component name="ToolWindowManager">
<frame x="-7" y="-7" width="1550" height="838" extended-state="7" />
<editor active="true" />
<layout>
<window_info id="Designer" />
<window_info id="Image Layers" />
<window_info id="Capture Tool" />
<window_info id="UI Designer" />
<window_info id="Favorites" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Duplicate detector" weight="0.32921812" />
<window_info anchor="bottom" id="Terminal" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="bottom" id="Event Log" side_tool="true" />
<window_info anchor="bottom" id="Java Enterprise" />
<window_info anchor="bottom" id="Database Changes" />
<window_info anchor="bottom" id="Version Control" />
<window_info anchor="bottom" id="Messages" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.25240055" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.39917696" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="right" id="Maven" />
<window_info anchor="right" id="Palette" />
<window_info anchor="right" id="Capture Analysis" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Palette&#9;" />
<window_info anchor="right" id="Theme Preview" />
<window_info anchor="right" id="Commander" order="0" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="debuggerHistoryManager">
<expressions id="evaluateExpression">
<expression>
<expression-string>e.attributeValue(&quot;href&quot;)</expression-string>
<language-id>JAVA</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>spanList.get(i).asXml()</expression-string>
<language-id>JAVA</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
<expression>
<expression-string>spanList.get(i)</expression-string>
<language-id>JAVA</language-id>
<evaluation-mode>EXPRESSION</evaluation-mode>
</expression>
</expressions>
</component>
<component name="editorHistoryManager">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/httpcomponents/httpcore/4.4.10/httpcore-4.4.10.jar!/org/apache/http/util/EntityUtils.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-57">
<caret line="22" column="19" selection-start-line="22" selection-start-column="19" selection-end-line="22" selection-end-column="19" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/httpcomponents/httpclient/4.5.6/httpclient-4.5.6.jar!/org/apache/http/client/HttpClient.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="57">
<caret line="16" column="17" selection-start-line="16" selection-start-column="17" selection-end-line="16" selection-end-column="17" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/demo2.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="309">
<caret line="77" column="5" lean-forward="true" selection-start-line="77" selection-start-column="5" selection-end-line="77" selection-end-column="5" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/App.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="298">
<caret line="52" column="80" selection-start-line="52" selection-start-column="51" selection-end-line="52" selection-end-column="80" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.5/hadoop-common-2.6.5.jar!/org/apache/hadoop/io/IntWritable.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-190">
<caret line="21" column="11" selection-start-line="21" selection-start-column="11" selection-end-line="21" selection-end-column="11" />
<folding>
<element signature="e#530#531#0" expanded="true" />
<element signature="e#561#562#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/net/sourceforge/htmlunit/htmlunit/2.29/htmlunit-2.29.jar!/com/gargoylesoftware/htmlunit/html/DomNode.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="137">
<caret line="468" column="18" selection-start-line="468" selection-start-column="18" selection-end-line="468" selection-end-column="18" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar!/org/dom4j/DocumentHelper.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="137">
<caret line="120" column="27" selection-start-line="120" selection-start-column="27" selection-end-line="120" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/net/sourceforge/htmlunit/htmlunit/2.29/htmlunit-2.29.jar!/com/gargoylesoftware/htmlunit/html/HtmlElement.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="114">
<caret line="41" column="22" selection-start-line="41" selection-start-column="22" selection-end-line="41" selection-end-column="22" />
</state>
</provider>
</entry>
<entry file="jar://C:/Program Files/Java/jdk1.8.0_102/jre/lib/rt.jar!/sun/net/www/http/HttpClient.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="138">
<caret line="115" column="12" selection-start-line="115" selection-start-column="12" selection-end-line="115" selection-end-column="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/dom4j.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="304">
<caret line="58" column="9" lean-forward="true" selection-start-line="53" selection-start-column="7" selection-end-line="58" selection-end-column="9" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/hadoop/hadoop-common/2.6.5/hadoop-common-2.6.5.jar!/org/apache/hadoop/io/Text.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="494">
<caret line="49" column="11" selection-start-line="49" selection-start-column="11" selection-end-line="49" selection-end-column="11" />
<folding>
<element signature="e#1961#1962#0" expanded="true" />
<element signature="e#2001#2002#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/mr.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1444">
<caret line="76" column="53" selection-start-line="76" selection-start-column="53" selection-end-line="76" selection-end-column="53" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="247">
<caret line="26" column="25" lean-forward="true" selection-start-line="26" selection-start-column="25" selection-end-line="26" selection-end-column="25" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/entity/positionInfo.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="913">
<caret line="94" column="13" lean-forward="true" selection-start-line="94" selection-start-column="13" selection-end-line="94" selection-end-column="13" />
<folding>
<element signature="imports" expanded="true" />
<element signature="e#1617#1618#0" expanded="true" />
<element signature="e#1642#1643#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/java/net/educoder/demo.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120">
<caret line="23" column="17" selection-start-line="23" selection-start-column="8" selection-end-line="23" selection-end-column="17" />
<folding>
<element signature="imports" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="GlobalLibrariesConfigurable.UI">
<settings>
<last-edited>scala-sdk-2.10.6</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="JdkListConfigurable.UI">
<settings>
<last-edited>1.6</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.6</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
<state key="ProjectLibrariesConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>

@ -0,0 +1,2 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.planet</groupId>
<artifactId>educoder_01</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>2.29</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.5</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
</dependency>
</dependencies>
</project>

@ -0,0 +1,87 @@
package net.educoder;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.dom4j.Document;
import org.dom4j.DocumentException;
import org.dom4j.DocumentHelper;
import org.dom4j.Element;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class demo {
public static void main(String[] args) {
WebClient webClient = new WebClient();
webClient.getOptions().setCssEnabled(false); // 取消 CSS 支持 ✔
webClient.getOptions().setJavaScriptEnabled(false); // 取消 JavaScript支持 ✔
try {
String content = "";
for (int j = 1; j <= 100; j++) {
HtmlPage page = webClient.getPage("https://hr.tencent.com/position.php?keywords=%E8%AF%B7%E8%BE%93%E5%85%A5%E5%85%B3%E9%94%AE%E8%AF%8D&lid=0&tid=0&start=" + (j * 10));
List<HtmlElement> spanList = page.getByXPath("//tr[@class='even'] | //tr[@class='odd']");
for (int i = 0; i < spanList.size(); i++) {
String xmlStr = spanList.get(i).asXml();
Document document = DocumentHelper.parseText(xmlStr);
Element rootElement = document.getRootElement();
List nodes = rootElement.elements("td");
for (Iterator it = nodes.iterator(); it.hasNext(); ) {
Element elm = (Element) it.next();
Element a = elm.element("a");
if (a != null) {
String href = "https://hr.tencent.com/" + a.attributeValue("href").trim();
HtmlPage InfoPage = webClient.getPage(href);
List<HtmlElement> byXPath = InfoPage.getByXPath("//tr[4]");
String require = byXPath.get(0).asText().trim();
String position = a.getText().trim();
String formattingRequire = replaceBlank(require);
String s = position + "\00" + formattingRequire;
content += s;
continue;
}
String text = "\00" + elm.getText().trim();
content += text;
}
content += "\n";
}
}
File file = new File(args[0]);
PrintStream ps = new PrintStream(new FileOutputStream(file));
ps.println(content);
} catch (FailingHttpStatusCodeException e) {
e.printStackTrace();
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (DocumentException e) {
e.printStackTrace();
} finally {
webClient.close(); // 关闭客户端,释放内存
}
}
public static String replaceBlank(String str) {
String dest = "";
if (str != null) {
Pattern p = Pattern.compile("\\s*|\t|\r|\n");
Matcher m = p.matcher(str);
dest = m.replaceAll("");
}
return dest;
}
}

@ -0,0 +1,130 @@
package net.educoder.entity;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class positionInfo implements WritableComparable<positionInfo> {
private String position;
private String type;
private String num;
private String address;
private String time;
private String require;
private String label;
public positionInfo() {
}
public positionInfo(String position, String type, String num, String address, String time, String require,
String label) {
this.position = position;
this.type = type;
this.num = num;
this.address = address;
this.time = time;
this.require = require;
this.label = label;
}
public void setBean(String position, String type, String num, String address, String time, String require,
String label) {
this.position = position;
this.type = type;
this.num = num;
this.address = address;
this.time = time;
this.require = require;
this.label = label;
}
public String getPosition() {
return position;
}
public void setPosition(String position) {
this.position = position;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getNum() {
return num;
}
public void setNum(String num) {
this.num = num;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getRequire() {
return require;
}
public void setRequire(String require) {
this.require = require;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
@Override
public String toString() {
return position + "\00" + type + "\00" + num + "\00" + address + "\00" + time + "\00" + require + "\00" + label;
}
@Override
public int compareTo(positionInfo o) {
return -1;
}
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeUTF(position);
dataOutput.writeUTF(type);
dataOutput.writeUTF(num);
dataOutput.writeUTF(address);
dataOutput.writeUTF(time);
dataOutput.writeUTF(require);
dataOutput.writeUTF(label);
}
@Override
public void readFields(DataInput dataInput) throws IOException {
position = dataInput.readUTF();
type = dataInput.readUTF();
num = dataInput.readUTF();
address = dataInput.readUTF();
time = dataInput.readUTF();
require = dataInput.readUTF();
label = dataInput.readUTF();
}
}

@ -0,0 +1,80 @@
package net.educoder;
import net.educoder.entity.positionInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class mr {
static class mapper extends Mapper<LongWritable, Text, positionInfo, NullWritable> {
private positionInfo positionInfo = new positionInfo();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String str = value.toString();
String[] split = str.split("\00");
if (split.length == 6) {
String position = split[0];
String require = split[1];
String type = split[2];
String num = split[3];
String address = split[4];
String time = split[5];
positionInfo.setPosition(position);
positionInfo.setRequire(require);
positionInfo.setAddress(address);
positionInfo.setNum(num);
positionInfo.setTime(time);
positionInfo.setType(type);
boolean contains = position.contains("大数据");
if (contains) {
positionInfo.setLabel("大数据");
} else {
positionInfo.setLabel("其他");
}
context.write(positionInfo, NullWritable.get());
}
}
}
static class reducer extends Reducer<positionInfo, NullWritable, positionInfo, NullWritable> {
@Override
protected void reduce(positionInfo key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
context.write(key, NullWritable.get());
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setJarByClass(mr.class);
job.setMapperClass(mapper.class);
job.setReducerClass(reducer.class);
job.setMapOutputKeyClass(positionInfo.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(positionInfo.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
boolean result = job.waitForCompletion(true);
System.exit(result ? 0 : 1);
}
}

@ -0,0 +1,26 @@
HELP.md
/target/
!.mvn/wrapper/maven-wrapper.jar
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
/build/

@ -0,0 +1,114 @@
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.Properties;
public class MavenWrapperDownloader {
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL =
"https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if (mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if (mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: : " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if (!outputFile.getParentFile().exists()) {
if (!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

Binary file not shown.

@ -0,0 +1 @@
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.0/apache-maven-3.6.0-bin.zip

286
educoder_02/mvnw vendored

@ -0,0 +1,286 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven2 Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
# TODO classpath?
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
##########################################################################################
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if command -v wget > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
wget "$jarUrl" -O "$wrapperJarPath"
elif command -v curl > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
curl -o "$wrapperJarPath" "$jarUrl"
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
##########################################################################################
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

161
educoder_02/mvnw.cmd vendored

@ -0,0 +1,161 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven2 Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM set title of command window
title %0
@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"
FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO (
IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
)
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
if exist %WRAPPER_JAR% (
echo Found %WRAPPER_JAR%
) else (
echo Couldn't find %WRAPPER_JAR%, downloading it ...
echo Downloading from: %DOWNLOAD_URL%
powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"
echo Finished downloading %WRAPPER_JAR%
)
@REM End of extension
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
exit /B %ERROR_CODE%

@ -0,0 +1,54 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.3.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>net.educoder</groupId>
<artifactId>bi</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>demo</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,15 @@
package net.educoder.bi;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
@MapperScan("net.educoder.bi.mapper")
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

@ -0,0 +1,82 @@
package net.educoder.bi.controller;
import net.educoder.bi.entity.NameValue;
import net.educoder.bi.entity.PositionInfo;
import net.educoder.bi.mapper.PositionInfoMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@RestController
public class EchartController {
@Autowired
PositionInfoMapper mapper;
@RequestMapping("/getHotPosition")
public Map<String, Object> getData() {
Map<String, Object> map = new HashMap<>();
List<PositionInfo> hotPositionList = mapper.getHotPosition();
ArrayList<String> positionList = new ArrayList<>();
ArrayList<String> numList = new ArrayList<>();
for (PositionInfo pi : hotPositionList) {
String s = pi.getPosition().split("-")[1];
positionList.add(s);
numList.add(pi.getNum());
}
map.put("position", positionList);
map.put("num", numList);
return map;
}
@RequestMapping("/bigDataPositionByAddress")
public Map<String, Object> getPositionByLabel() {
Map<String, Object> map = new HashMap<>();
List<NameValue> bigDataPositionByAddress = mapper.getBigDataPositionByAddress();
map.put("bigDataPositionByAddress", bigDataPositionByAddress);
ArrayList<String> addressList = new ArrayList<>();
for (NameValue n : bigDataPositionByAddress) {
addressList.add(n.getName());
}
map.put("address", addressList);
return map;
}
@RequestMapping("/bigDataPositionTendency")
public Map<String, Object> getBigDataPositionTendency() {
Map<String, Object> map = new HashMap<>();
ArrayList<String> list = new ArrayList<>();
list.add("2");
list.add("4");
list.add("4");
list.add("6");
list.add("8");
list.add("11");
String nowNum = mapper.getBigDataPositionTendency();
list.add(nowNum);
map.put("num", list);
return map;
}
@RequestMapping("/getBigDataPositionRequireBySkill")
public Map<String, Object> getBigDataPositionRequireBySkill() {
Map<String, Object> map = new HashMap<>();
String[] strArr = {"scala", "spark", "hive", "java", "storm", "hadoop"};
ArrayList<String> numList = new ArrayList<>();
ArrayList<String> SkillList = new ArrayList<>();
for (String arr : strArr) {
String str = mapper.getBigDataPositionRequireBySkill("%"+arr+"%");
SkillList.add(arr);
numList.add(str);
}
map.put("SkillList", SkillList);
map.put("numList", numList);
return map;
}
}

@ -0,0 +1,30 @@
package net.educoder.bi.entity;
public class NameValue {
private String name;
private String value;
public NameValue() {
}
public NameValue(String name, String value) {
this.name = name;
this.value = value;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
}

@ -0,0 +1,68 @@
package net.educoder.bi.entity;
public class PositionInfo {
private String position;
private String type;
private String num;
private String address;
private String time;
private String require;
private String label;
public String getPosition() {
return position;
}
public void setPosition(String position) {
this.position = position;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getNum() {
return num;
}
public void setNum(String num) {
this.num = num;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getRequire() {
return require;
}
public void setRequire(String require) {
this.require = require;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
}

@ -0,0 +1,28 @@
package net.educoder.bi.mapper;
import net.educoder.bi.entity.NameValue;
import net.educoder.bi.entity.PositionInfo;
import org.apache.ibatis.annotations.Select;
import java.util.List;
public interface PositionInfoMapper {
@Select("select * from PositionInfo ")
List<PositionInfo> getAll();
@Select("select * from PositionInfo ORDER BY num DESC LIMIT 10")
List<PositionInfo> getHotPosition();
@Select("select * from PositionInfo where label = '大数据'")
List<PositionInfo> getBigDataPostion();
@Select("select sum(num)as 'value',address as 'name' from PositionInfo WHERE label = '大数据' GROUP BY address ")
List<NameValue> getBigDataPositionByAddress();
@Select("select sum(num) from PositionInfo WHERE label = '大数据'")
String getBigDataPositionTendency();
@Select("select count(*) from PositionInfo WHERE label = '大数据' AND require_ LIKE #{require};")
String getBigDataPositionRequireBySkill(String require);
}

@ -0,0 +1,4 @@
spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://localhost:3306/python_scrapy?useUnicode=true&characterEncoding=utf-8
spring.datasource.username=root
spring.datasource.password=root

@ -0,0 +1,172 @@
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="utf-8">
<title>ECharts</title>
<script src="echarts.js"></script>
<script src="jquery-3.2.1.js"></script>
</head>
<body>
<div id="main" style="width: 1200px;height:400px;"></div>
<div id="main2" style="width: 1200px;height:400px;"></div>
<div id="main3" style="width: 1200px;height:400px;"></div>
<div id="main4" style="width: 1200px;height:400px;"></div>
<script type="text/javascript">
var myChart = echarts.init(document.getElementById('main'));
var myChart2 = echarts.init(document.getElementById('main2'));
var myChart3 = echarts.init(document.getElementById('main3'));
var myChart4 = echarts.init(document.getElementById('main4'));
$.ajax({
url: "/getHotPosition",
success: function (data) {
// 指定图表的配置项和数据
var option = {
title: {
text: '热门职位top10'
},
tooltip: {},
legend: {
data: ['需求量']
},
xAxis: {
data: data.position
},
yAxis: {},
series: [{
name: '需求量',
type: 'bar',
data: data.num
}]
};
myChart.setOption(option);
},
dataType: "json",
type: "post"
})
$.ajax({
url: "/bigDataPositionByAddress",
success: function (data) {
// 指定图表的配置项和数据
option = {
tooltip: {
trigger: 'item',
formatter: "{a} <br/>{b}: {c} ({d}%)"
},
legend: {
orient: 'vertical',
x: 'left',
data: data.address
},
series: [
{
name: '大数据职位需求量',
type: 'pie',
radius: ['50%', '70%'],
avoidLabelOverlap: false,
label: {
normal: {
show: false,
position: 'center'
},
emphasis: {
show: true,
textStyle: {
fontSize: '30',
fontWeight: 'bold'
}
}
},
labelLine: {
normal: {
show: false
}
},
data: data.bigDataPositionByAddress
}
]
};
myChart2.setOption(option);
},
dataType: "json",
type: "post"
})
$.ajax({
url: "/bigDataPositionTendency",
success: function (data) {
// 指定图表的配置项和数据
option = {
title: {
text: '大数据职位趋势'
},
xAxis: {
type: 'category',
data: ['2013', '2014', '2015', '2016', '2017', '2018', '2019']
},
yAxis: {
type: 'value'
},
series: [{
data: data.num,
type: 'line'
}]
};
myChart3.setOption(option);
},
dataType: "json",
type: "post"
})
$.ajax({
url: "/getBigDataPositionRequireBySkill",
success: function (data) {
// 指定图表的配置项和数据
option = {
color: ['#3398DB'],
tooltip: {
trigger: 'axis',
axisPointer: { // 坐标轴指示器,坐标轴触发有效
type: 'shadow' // 默认为直线,可选为:'line' | 'shadow'
}
},
grid: {
left: '3%',
right: '4%',
bottom: '3%',
containLabel: true
},
xAxis: [
{
type: 'category',
data: data.SkillList,
axisTick: {
alignWithLabel: true
}
}
],
yAxis: [
{
type: 'value'
}
],
series: [
{
name: '技能要求',
type: 'bar',
barWidth: '60%',
data: data.numList
}
]
};
myChart4.setOption(option);
},
dataType: "json",
type: "post"
})
</script>
</body>
</html>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

@ -0,0 +1,16 @@
package net.educoder.bi;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class ApplicationTests {
@Test
public void contextLoads() {
}
}
Loading…
Cancel
Save