pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

探果题库爬虫

Browse Source
main
tamguo 7 years ago
parent 5b43a32c0f
commit 1ddb324183
14 changed files with 517 additions and 0 deletions
Show Stats Download Patch File Download Diff File

36
tamguo-crawler/.classpath
Unescape View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

1
tamguo-crawler/.gitignore vendored
Unescape View File

@ -0,0 +1 @@
/target/

23
tamguo-crawler/.project
Unescape View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>tamguo-crawler</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

6
tamguo-crawler/.settings/org.eclipse.core.resources.prefs
Unescape View File

@ -0,0 +1,6 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8

5
tamguo-crawler/.settings/org.eclipse.jdt.core.prefs
Unescape View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.8

4
tamguo-crawler/.settings/org.eclipse.m2e.core.prefs
Unescape View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

158
tamguo-crawler/pom.xml
Unescape View File

@ -0,0 +1,158 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.tamguo</groupId>
<artifactId>tamguo-crawler</artifactId>
<version>0.0.1-SNAPSHOT</version>
<description>探果题库爬虫</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.3.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
<mybatis-plus-boot-starter.version>2.1.9</mybatis-plus-boot-starter.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<!-- mybatis-plus begin -->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis-plus-boot-starter.version}</version>
<exclusions>
<exclusion>
<artifactId>tomcat-jdbc</artifactId>
<groupId>org.apache.tomcat</groupId>
</exclusion>
</exclusions>
</dependency>
<!-- mybatis-plus end -->
<dependency>
<groupId>net.sourceforge.nekohtml</groupId>
<artifactId>nekohtml</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.32</version>
</dependency>
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-spring</artifactId>
<version>1.2.5</version>
</dependency>
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-ehcache</artifactId>
<version>1.2.5</version>
</dependency>
<dependency>
<groupId>com.github.theborakompanioni</groupId>
<artifactId>thymeleaf-extras-shiro</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>cn.songxinqiang</groupId>
<artifactId>com.baidu.ueditor</artifactId>
<version>1.1.2-edit-1.0</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>com.github.penggle</groupId>
<artifactId>kaptcha</artifactId>
<version>2.3.2</version>
<exclusions>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.18</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.6</version>
</dependency>
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>aliyun-java-sdk-dysmsapi</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.aliyun</groupId>
<artifactId>aliyun-java-sdk-core</artifactId>
<version>3.2.8</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
<version>1.5</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>Camden.SR6</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<finalName>tamguo</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

37
tamguo-crawler/src/main/java/com/tamguo/TamguoCrawlerApplication.java
Unescape View File

@ -0,0 +1,37 @@
package com.tamguo;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.web.HttpMessageConverters;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.http.converter.HttpMessageConverter;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.alibaba.fastjson.support.config.FastJsonConfig;
import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter;
@SpringBootApplication
@ComponentScan("com.tamguo")
public class TamguoCrawlerApplication {
public static void main(String[] args) {
new SpringApplicationBuilder(TamguoCrawlerApplication.class).web(true).run(args);
}
/**
* FastJson替代Jackson
* @return
*/
@Bean
public HttpMessageConverters fastJsonHttpMessageConverters() {
FastJsonHttpMessageConverter fastConverter = new FastJsonHttpMessageConverter();
FastJsonConfig fastJsonConfig = new FastJsonConfig();
fastJsonConfig.setDateFormat("yyyy-MM-dd HH:mm:ss");
fastJsonConfig.setSerializerFeatures(SerializerFeature.DisableCircularReferenceDetect);
fastConverter.setFastJsonConfig(fastJsonConfig);
HttpMessageConverter<?> converter = fastConverter;
return new HttpMessageConverters(converter);
}
}

27
tamguo-crawler/src/main/java/com/tamguo/config/dao/MyMetaObjectHandler.java
Unescape View File

@ -0,0 +1,27 @@
package com.tamguo.config.dao;
import com.baomidou.mybatisplus.mapper.MetaObjectHandler;
import com.tamguo.TamguoCrawlerApplication;
import org.apache.ibatis.reflection.MetaObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* 注入公共字段自动填充,任选注入方式即可
*/
//@Component
public class MyMetaObjectHandler extends MetaObjectHandler {
protected final static Logger logger = LoggerFactory.getLogger(TamguoCrawlerApplication.class);
@Override
public void insertFill(MetaObject metaObject) {
logger.info("新增的时候干点不可描述的事情");
}
@Override
public void updateFill(MetaObject metaObject) {
logger.info("更新的时候干点不可描述的事情");
}
}

92
tamguo-crawler/src/main/java/com/tamguo/config/dao/MybatisPlusConfig.java
Unescape View File

@ -0,0 +1,92 @@
package com.tamguo.config.dao;
import java.util.ArrayList;
import java.util.List;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.baomidou.mybatisplus.mapper.ISqlInjector;
import com.baomidou.mybatisplus.mapper.LogicSqlInjector;
import com.baomidou.mybatisplus.mapper.MetaObjectHandler;
import com.baomidou.mybatisplus.plugins.PaginationInterceptor;
import com.baomidou.mybatisplus.plugins.PerformanceInterceptor;
import com.baomidou.mybatisplus.plugins.parser.ISqlParser;
import com.baomidou.mybatisplus.plugins.parser.tenant.TenantHandler;
import com.baomidou.mybatisplus.plugins.parser.tenant.TenantSqlParser;
import net.sf.jsqlparser.expression.Expression;
@Configuration
@MapperScan("com.tamguo.dao*")
public class MybatisPlusConfig {
@Bean
public PerformanceInterceptor performanceInterceptor() {
return new PerformanceInterceptor();
}
/**
* mybatis-plus分页插件<br>
* 文档：http://mp.baomidou.com<br>
*/
@Bean
public PaginationInterceptor paginationInterceptor() {
PaginationInterceptor paginationInterceptor = new PaginationInterceptor();
paginationInterceptor.setLocalPage(true);// 开启 PageHelper 的支持
/*
* 【测试多租户】 SQL 解析处理拦截器<br>
* 这里固定写成住户 1 实际情况你可以从cookie读取，因此数据看不到 【 麻花藤 】 这条记录（ 注意观察 SQL ）<br>
*/
List<ISqlParser> sqlParserList = new ArrayList<>();
TenantSqlParser tenantSqlParser = new TenantSqlParser();
tenantSqlParser.setTenantHandler(new TenantHandler() {
@Override
public Expression getTenantId() {
return null;
}
@Override
public String getTenantIdColumn() {
return "company_id";
}
@Override
public boolean doTableFilter(String tableName) {
// 这里可以判断是否过滤表
return true;
}
});
sqlParserList.add(tenantSqlParser);
paginationInterceptor.setSqlParserList(sqlParserList);
// 以下过滤方式与 @SqlParser(filter = true) 注解等效
// paginationInterceptor.setSqlParserFilter(new ISqlParserFilter() {
// @Override
// public boolean doFilter(MetaObject metaObject) {
// MappedStatement ms = PluginUtils.getMappedStatement(metaObject);
// // 过滤自定义查询此时无租户信息约束【 麻花藤 】出现
// if ("com.baomidou.springboot.mapper.UserMapper.selectListBySQL".equals(ms.getId())) {
// return true;
// }
// return false;
// }
// });
return paginationInterceptor;
}
@Bean
public MetaObjectHandler metaObjectHandler(){
return new MyMetaObjectHandler();
}
/**
* 注入sql注入器
*/
@Bean
public ISqlInjector sqlInjector(){
return new LogicSqlInjector();
}
}

29
tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java
Unescape View File

@ -0,0 +1,29 @@
package com.tamguo.config.dao;
import java.io.Serializable;
import com.baomidou.mybatisplus.activerecord.Model;
import com.baomidou.mybatisplus.annotations.TableId;
/**
* 实体父类
*/
public class SuperEntity<T extends Model<?>> extends Model<T> {
private static final long serialVersionUID = 1L;
@TableId("uid")
private String uid;
@Override
protected Serializable pkVal() {
return this.getUid();
}
public String getUid() {
return uid;
}
public void setUid(String uid) {
this.uid = uid;
}
}

11
tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperMapper.java
Unescape View File

@ -0,0 +1,11 @@
package com.tamguo.config.dao;
import com.baomidou.mybatisplus.mapper.BaseMapper;
/**
* 演示 mapper 父类，注意这个类不要让 mp 扫描到！！
*/
public interface SuperMapper<T> extends BaseMapper<T> {
// 这里可以放一些公共的方法
}

53
tamguo-crawler/src/main/java/com/tamguo/model/SchoolEntity.java
Unescape View File

@ -0,0 +1,53 @@
package com.tamguo.model;
import java.io.Serializable;
import com.baomidou.mybatisplus.annotations.TableName;
import com.tamguo.config.dao.SuperEntity;
/**
* The persistent class for the tiku_chapter database table.
*
*/
@TableName(value="tiku_school")
public class SchoolEntity extends SuperEntity<SchoolEntity> implements Serializable {
private static final long serialVersionUID = 1L;
private String areaId;
private String name;
private String image;
public SchoolEntity() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getImage() {
return image;
}
public void setImage(String image) {
this.image = image;
}
public static long getSerialversionuid() {
return serialVersionUID;
}
public String getAreaId() {
return areaId;
}
public void setAreaId(String areaId) {
this.areaId = areaId;
}
}

35
tamguo-crawler/src/main/resources/application.properties
Unescape View File

@ -0,0 +1,35 @@
spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.filters=stat,wall,log4j
spring.datasource.initialSize=5
spring.datasource.maxActive=20
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
spring.datasource.maxWait=60000
spring.datasource.minEvictableIdleTimeMillis=300000
spring.datasource.minIdle=5
spring.datasource.password=
spring.datasource.poolPreparedStatements=true
spring.datasource.testOnBorrow=false
spring.datasource.testOnReturn=false
spring.datasource.testWhileIdle=true
spring.datasource.timeBetweenEvictionRunsMillis=60000
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tiku?useUnicode=true&characterEncoding=UTF-8&useSSL=false
spring.datasource.username=root
spring.datasource.validationQuery=SELECT 1 FROM DUAL
mybatis-plus.mapper-locations=classpath:/mappers/*Mapper.xml
mybatis-plus.typeAliasesPackage=com.tamguo.model
#mybatis-plus.typeEnumsPackage=com.tamguo.model.enums
mybatis-plus.global-config.id-type=5
mybatis-plus.global-config.field-strategy=2
mybatis-plus.global-config.db-column-underline=true
mybatis-plus.global-config.refresh-mapper=true
mybatis-plus.global-config.key-generator=com.baomidou.mybatisplus.incrementer.H2KeyGenerator
mybatis-plus.global-config.logic-delete-value=0
mybatis-plus.global-config.logic-not-delete-value=1
mybatis-plus.global-config.sql-injector=com.baomidou.mybatisplus.mapper.LogicSqlInjector
mybatis-plus.global-config.meta-object-handler=com.tamguo.config.dao.MyMetaObjectHandler
mybatis-plus.global-config.sql-parser-cache=true
mybatis-plus.configuration.map-underscore-to-camel-case=true
mybatis-plus.configuration.cache-enabled=false
Write Preview
Loading…
Cancel
Save