导入考试

tamguo-crawler/pom.xml

@@ -20,14 +20,6 @@
 	</properties>
 
 	<dependencies>
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-starter-web</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>org.springframework.boot</groupId>
-			<artifactId>spring-boot-starter-thymeleaf</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-jdbc</artifactId>
@@ -45,10 +37,6 @@
 			</exclusions>
 		</dependency>
 		<!-- mybatis-plus end -->
-		<dependency>
-			<groupId>net.sourceforge.nekohtml</groupId>
-			<artifactId>nekohtml</artifactId>
-		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-test</artifactId>
@@ -59,50 +47,10 @@
 			<artifactId>fastjson</artifactId>
 			<version>1.2.32</version>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.shiro</groupId>
-			<artifactId>shiro-spring</artifactId>
-			<version>1.2.5</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.shiro</groupId>
-			<artifactId>shiro-ehcache</artifactId>
-			<version>1.2.5</version>
-		</dependency>
-		<dependency>
-			<groupId>com.github.theborakompanioni</groupId>
-			<artifactId>thymeleaf-extras-shiro</artifactId>
-			<version>1.2.1</version>
-		</dependency>
-		<dependency>
-			<groupId>cn.songxinqiang</groupId>
-			<artifactId>com.baidu.ueditor</artifactId>
-			<version>1.1.2-edit-1.0</version>
-		</dependency>
-		<dependency>
-			<groupId>commons-codec</groupId>
-			<artifactId>commons-codec</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>commons-fileupload</groupId>
-			<artifactId>commons-fileupload</artifactId>
-			<version>1.3.1</version>
-		</dependency>
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>com.github.penggle</groupId>
-			<artifactId>kaptcha</artifactId>
-			<version>2.3.2</version>
-			<exclusions>
-				<exclusion>
-					<artifactId>javax.servlet-api</artifactId>
-					<groupId>javax.servlet</groupId>
-				</exclusion>
-			</exclusions>
-		</dependency>
 		<dependency>
 			<groupId>com.alibaba</groupId>
 			<artifactId>druid</artifactId>
@@ -118,19 +66,9 @@
 			<version>3.6</version>
 		</dependency>
 		<dependency>
-			<groupId>com.aliyun</groupId>
+		    <groupId>com.xuxueli</groupId>
-			<artifactId>aliyun-java-sdk-dysmsapi</artifactId>
+		    <artifactId>xxl-crawler</artifactId>
-			<version>1.0.0</version>
+		    <version>1.2.1</version>
-		</dependency>
-		<dependency>
-			<groupId>com.aliyun</groupId>
-			<artifactId>aliyun-java-sdk-core</artifactId>
-			<version>3.2.8</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-email</artifactId>
-			<version>1.5</version>
 		</dependency>
 	</dependencies>
 

tamguo-crawler/src/main/java/com/tamguo/TamguoCrawlerApplication.java

@@ -1,15 +1,8 @@
 package com.tamguo;
 
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.web.HttpMessageConverters;
 import org.springframework.boot.builder.SpringApplicationBuilder;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
-import org.springframework.http.converter.HttpMessageConverter;
-
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.fastjson.support.config.FastJsonConfig;
-import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter;
 
 @SpringBootApplication
 @ComponentScan("com.tamguo")
@@ -19,19 +12,4 @@ public class TamguoCrawlerApplication {
 		new SpringApplicationBuilder(TamguoCrawlerApplication.class).web(true).run(args);
 	}
 	
-	/**
-	 * FastJson替代Jackson
-	 * @return
-	 */
-	@Bean
-	public HttpMessageConverters fastJsonHttpMessageConverters() {
-		FastJsonHttpMessageConverter fastConverter = new FastJsonHttpMessageConverter();
-		FastJsonConfig fastJsonConfig = new FastJsonConfig();
-		fastJsonConfig.setDateFormat("yyyy-MM-dd HH:mm:ss");
-		fastJsonConfig.setSerializerFeatures(SerializerFeature.DisableCircularReferenceDetect);
-		fastConverter.setFastJsonConfig(fastJsonConfig);
-		HttpMessageConverter<?> converter = fastConverter;
-		return new HttpMessageConverters(converter);
-	}
-	
 }

tamguo-crawler/src/main/java/com/tamguo/config/dao/MyMetaObjectHandler.java

@@ -1,7 +1,6 @@
 package com.tamguo.config.dao;
 
 import com.baomidou.mybatisplus.mapper.MetaObjectHandler;
-import com.tamguo.TamguoCrawlerApplication;
 
 import org.apache.ibatis.reflection.MetaObject;
 import org.slf4j.Logger;
@@ -13,7 +12,7 @@ import org.slf4j.LoggerFactory;
 //@Component
 public class MyMetaObjectHandler extends MetaObjectHandler {
 
-    protected final static Logger logger = LoggerFactory.getLogger(TamguoCrawlerApplication.class);
+    protected final static Logger logger = LoggerFactory.getLogger(MyMetaObjectHandler.class);
 
     @Override
     public void insertFill(MetaObject metaObject) {

tamguo-crawler/src/main/java/com/tamguo/dao/SubjectMapper.java

@@ -0,0 +1,12 @@
+package com.tamguo.dao;
+
+import org.apache.ibatis.annotations.Param;
+
+import com.tamguo.config.dao.SuperMapper;
+import com.tamguo.model.SubjectEntity;
+
+public interface SubjectMapper extends SuperMapper<SubjectEntity>{
+
+	SubjectEntity findByName(@Param(value="name")String name);
+
+}

tamguo-crawler/src/main/java/com/tamguo/model/SubjectEntity.java

@@ -0,0 +1,42 @@
+package com.tamguo.model;
+
+import java.io.Serializable;
+import com.baomidou.mybatisplus.annotations.TableName;
+import com.tamguo.config.dao.SuperEntity;
+
+@TableName(value="tiku_subject")
+public class SubjectEntity extends SuperEntity<SubjectEntity> implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	private String name;
+	
+	private String courseId;
+	
+	private String courseName;
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getCourseId() {
+		return courseId;
+	}
+
+	public void setCourseId(String courseId) {
+		this.courseId = courseId;
+	}
+
+	public String getCourseName() {
+		return courseName;
+	}
+
+	public void setCourseName(String courseName) {
+		this.courseName = courseName;
+	}
+
+}

tamguo-crawler/src/main/java/com/tamguo/model/vo/SubjectVo.java

@@ -0,0 +1,23 @@
+package com.tamguo.model.vo;
+
+import java.util.List;
+
+import com.xuxueli.crawler.annotation.PageFieldSelect;
+import com.xuxueli.crawler.annotation.PageSelect;
+
+@PageSelect(cssQuery = "body")
+public class SubjectVo {
+
+	@PageFieldSelect(cssQuery = ".all-list-li")
+	private List<String> name;
+
+	public List<String> getName() {
+		return name;
+	}
+
+	public void setName(List<String> name) {
+		this.name = name;
+	}
+
+	
+}

tamguo-crawler/src/main/java/com/tamguo/service/IChapterService.java

@@ -0,0 +1,9 @@
+package com.tamguo.service;
+
+public interface IChapterService {
+
+	/**
+	 * 爬取章节数据
+	 */
+	void crawlerChapter();
+}

tamguo-crawler/src/main/java/com/tamguo/service/ICourseService.java

@@ -0,0 +1,10 @@
+package com.tamguo.service;
+
+public interface ICourseService {
+
+	/**
+	 * 爬取科目数据
+	 */
+	void crawlerCourse();
+	
+}

tamguo-crawler/src/main/java/com/tamguo/service/ISubjectService.java

@@ -0,0 +1,10 @@
+package com.tamguo.service;
+
+public interface ISubjectService {
+
+	/**
+	 *  爬取考试数据
+	 */
+	void crawlerSubject();
+	
+}

tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java

@@ -0,0 +1,51 @@
+package com.tamguo.service.impl;
+
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.tamguo.dao.SubjectMapper;
+import com.tamguo.model.SubjectEntity;
+import com.tamguo.model.vo.SubjectVo;
+import com.tamguo.service.ISubjectService;
+import com.xuxueli.crawler.XxlCrawler;
+import com.xuxueli.crawler.parser.PageParser;
+
+@Service
+public class SubjectService implements ISubjectService{
+	
+	@Autowired
+	SubjectMapper subjectMapper;
+	
+	@Override
+	public void crawlerSubject() {
+		XxlCrawler crawler = new XxlCrawler.Builder()
+            .setUrls("https://tiku.baidu.com/")
+            .setWhiteUrlRegexs("https://tiku\\.baidu\\.com/")
+            .setPageParser(new PageParser<SubjectVo>() {
+                @Override
+                public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
+                    // 解析封装 PageVo 对象
+                    String pageUrl = html.baseUri();
+                    System.out.println(pageUrl + "：" + subjectVo.toString());
+                    
+                    for(int i=0 ; i<subjectVo.getName().size() ; i++) {
+                    	String name = subjectVo.getName().get(i);
+                    		
+                    	SubjectEntity subject = subjectMapper.findByName(name);
+                    	if(subject != null) {
+                    		continue;
+                    	}
+                    	SubjectEntity entity = new SubjectEntity();
+                    	entity.setName(name);
+                    	subjectMapper.insert(entity);
+                    }
+                }
+            })
+            .build();
+		
+		crawler.start(true);
+	}
+
+}

tamguo-crawler/src/main/resources/application.properties

@@ -14,7 +14,7 @@ spring.datasource.testOnReturn=false
 spring.datasource.testWhileIdle=true
 spring.datasource.timeBetweenEvictionRunsMillis=60000
 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
-spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tiku?useUnicode=true&characterEncoding=UTF-8&useSSL=false
+spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tamguo?useUnicode=true&characterEncoding=UTF-8&useSSL=false
 spring.datasource.username=root
 spring.datasource.validationQuery=SELECT 1 FROM DUAL
 

tamguo-crawler/src/main/resources/mappers/SubjectMapper.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.tamguo.dao.SubjectMapper">
+
+	<select id="findByName" resultType="SubjectEntity">
+		select * from tiku_subject where name = #{name}
+	</select>
+	
+</mapper>

tamguo-crawler/src/test/java/com/tamguo/SubjectCrawler.java

@@ -0,0 +1,23 @@
+package com.tamguo;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.tamguo.service.ISubjectService;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class SubjectCrawler {
+
+	@Autowired
+	ISubjectService iSubjectService;
+	
+    @Test
+    public void crawlerSubject() throws Exception {
+        iSubjectService.crawlerSubject();
+    }
+	
+}