diff --git a/tamguo-crawler/.classpath b/tamguo-crawler/.classpath new file mode 100644 index 0000000..16c89cc --- /dev/null +++ b/tamguo-crawler/.classpath @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tamguo-crawler/.gitignore b/tamguo-crawler/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/tamguo-crawler/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/tamguo-crawler/.project b/tamguo-crawler/.project new file mode 100644 index 0000000..b487433 --- /dev/null +++ b/tamguo-crawler/.project @@ -0,0 +1,23 @@ + + + tamguo-crawler + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/tamguo-crawler/.settings/org.eclipse.core.resources.prefs b/tamguo-crawler/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..04cfa2c --- /dev/null +++ b/tamguo-crawler/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/tamguo-crawler/.settings/org.eclipse.jdt.core.prefs b/tamguo-crawler/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..d59e09c --- /dev/null +++ b/tamguo-crawler/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/tamguo-crawler/.settings/org.eclipse.m2e.core.prefs b/tamguo-crawler/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..14b697b --- /dev/null +++ b/tamguo-crawler/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/tamguo-crawler/pom.xml b/tamguo-crawler/pom.xml new file mode 100644 index 0000000..7eefd94 --- /dev/null +++ b/tamguo-crawler/pom.xml @@ -0,0 +1,158 @@ + + 4.0.0 + com.tamguo + tamguo-crawler + 0.0.1-SNAPSHOT + 探果题库爬虫 + + org.springframework.boot + spring-boot-starter-parent + 1.5.3.RELEASE + + + + + UTF-8 + 1.8 + 2.1.9 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + org.springframework.boot + spring-boot-starter-jdbc + + + + com.baomidou + mybatis-plus-boot-starter + ${mybatis-plus-boot-starter.version} + + + tomcat-jdbc + org.apache.tomcat + + + + + + net.sourceforge.nekohtml + nekohtml + + + org.springframework.boot + spring-boot-starter-test + test + + + com.alibaba + fastjson + 1.2.32 + + + org.apache.shiro + shiro-spring + 1.2.5 + + + org.apache.shiro + shiro-ehcache + 1.2.5 + + + com.github.theborakompanioni + thymeleaf-extras-shiro + 1.2.1 + + + cn.songxinqiang + com.baidu.ueditor + 1.1.2-edit-1.0 + + + commons-codec + commons-codec + + + commons-fileupload + commons-fileupload + 1.3.1 + + + commons-io + commons-io + + + com.github.penggle + kaptcha + 2.3.2 + + + javax.servlet-api + javax.servlet + + + + + com.alibaba + druid + 1.0.18 + + + mysql + mysql-connector-java + + + org.apache.commons + commons-lang3 + 3.6 + + + com.aliyun + aliyun-java-sdk-dysmsapi + 1.0.0 + + + com.aliyun + aliyun-java-sdk-core + 3.2.8 + + + org.apache.commons + commons-email + 1.5 + + + + + + + org.springframework.cloud + spring-cloud-dependencies + Camden.SR6 + pom + import + + + + + + tamguo + + + org.springframework.boot + spring-boot-maven-plugin + + + + \ No newline at end of file diff --git a/tamguo-crawler/src/main/java/com/tamguo/TamguoCrawlerApplication.java b/tamguo-crawler/src/main/java/com/tamguo/TamguoCrawlerApplication.java new file mode 100644 index 0000000..40e17a0 --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/TamguoCrawlerApplication.java @@ -0,0 +1,37 @@ +package com.tamguo; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.web.HttpMessageConverters; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.converter.HttpMessageConverter; + +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.fastjson.support.config.FastJsonConfig; +import com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter; + +@SpringBootApplication +@ComponentScan("com.tamguo") +public class TamguoCrawlerApplication { + + public static void main(String[] args) { + new SpringApplicationBuilder(TamguoCrawlerApplication.class).web(true).run(args); + } + + /** + * FastJson替代Jackson + * @return + */ + @Bean + public HttpMessageConverters fastJsonHttpMessageConverters() { + FastJsonHttpMessageConverter fastConverter = new FastJsonHttpMessageConverter(); + FastJsonConfig fastJsonConfig = new FastJsonConfig(); + fastJsonConfig.setDateFormat("yyyy-MM-dd HH:mm:ss"); + fastJsonConfig.setSerializerFeatures(SerializerFeature.DisableCircularReferenceDetect); + fastConverter.setFastJsonConfig(fastJsonConfig); + HttpMessageConverter converter = fastConverter; + return new HttpMessageConverters(converter); + } + +} diff --git a/tamguo-crawler/src/main/java/com/tamguo/config/dao/MyMetaObjectHandler.java b/tamguo-crawler/src/main/java/com/tamguo/config/dao/MyMetaObjectHandler.java new file mode 100644 index 0000000..3c4417d --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/config/dao/MyMetaObjectHandler.java @@ -0,0 +1,27 @@ +package com.tamguo.config.dao; + +import com.baomidou.mybatisplus.mapper.MetaObjectHandler; +import com.tamguo.TamguoCrawlerApplication; + +import org.apache.ibatis.reflection.MetaObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 注入公共字段自动填充,任选注入方式即可 + */ +//@Component +public class MyMetaObjectHandler extends MetaObjectHandler { + + protected final static Logger logger = LoggerFactory.getLogger(TamguoCrawlerApplication.class); + + @Override + public void insertFill(MetaObject metaObject) { + logger.info("新增的时候干点不可描述的事情"); + } + + @Override + public void updateFill(MetaObject metaObject) { + logger.info("更新的时候干点不可描述的事情"); + } +} diff --git a/tamguo-crawler/src/main/java/com/tamguo/config/dao/MybatisPlusConfig.java b/tamguo-crawler/src/main/java/com/tamguo/config/dao/MybatisPlusConfig.java new file mode 100644 index 0000000..ab5ec42 --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/config/dao/MybatisPlusConfig.java @@ -0,0 +1,92 @@ +package com.tamguo.config.dao; + +import java.util.ArrayList; +import java.util.List; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.baomidou.mybatisplus.mapper.ISqlInjector; +import com.baomidou.mybatisplus.mapper.LogicSqlInjector; +import com.baomidou.mybatisplus.mapper.MetaObjectHandler; +import com.baomidou.mybatisplus.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.plugins.PerformanceInterceptor; +import com.baomidou.mybatisplus.plugins.parser.ISqlParser; +import com.baomidou.mybatisplus.plugins.parser.tenant.TenantHandler; +import com.baomidou.mybatisplus.plugins.parser.tenant.TenantSqlParser; + +import net.sf.jsqlparser.expression.Expression; + +@Configuration +@MapperScan("com.tamguo.dao*") +public class MybatisPlusConfig { + + @Bean + public PerformanceInterceptor performanceInterceptor() { + return new PerformanceInterceptor(); + } + + /** + * mybatis-plus分页插件
+ * 文档：http://mp.baomidou.com
+ */ + @Bean + public PaginationInterceptor paginationInterceptor() { + PaginationInterceptor paginationInterceptor = new PaginationInterceptor(); + paginationInterceptor.setLocalPage(true);// 开启 PageHelper 的支持 + /* + * 【测试多租户】 SQL 解析处理拦截器
+ * 这里固定写成住户 1 实际情况你可以从cookie读取，因此数据看不到 【 麻花藤 】 这条记录（ 注意观察 SQL ）
+ */ + List sqlParserList = new ArrayList<>(); + TenantSqlParser tenantSqlParser = new TenantSqlParser(); + tenantSqlParser.setTenantHandler(new TenantHandler() { + @Override + public Expression getTenantId() { + return null; + } + + @Override + public String getTenantIdColumn() { + return "company_id"; + } + + @Override + public boolean doTableFilter(String tableName) { + // 这里可以判断是否过滤表 + return true; + } + }); + + + sqlParserList.add(tenantSqlParser); + paginationInterceptor.setSqlParserList(sqlParserList); + // 以下过滤方式与 @SqlParser(filter = true) 注解等效 +// paginationInterceptor.setSqlParserFilter(new ISqlParserFilter() { +// @Override +// public boolean doFilter(MetaObject metaObject) { +// MappedStatement ms = PluginUtils.getMappedStatement(metaObject); +// // 过滤自定义查询此时无租户信息约束【 麻花藤 】出现 +// if ("com.baomidou.springboot.mapper.UserMapper.selectListBySQL".equals(ms.getId())) { +// return true; +// } +// return false; +// } +// }); + return paginationInterceptor; + } + + @Bean + public MetaObjectHandler metaObjectHandler(){ + return new MyMetaObjectHandler(); + } + + /** + * 注入sql注入器 + */ + @Bean + public ISqlInjector sqlInjector(){ + return new LogicSqlInjector(); + } +} diff --git a/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java b/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java new file mode 100644 index 0000000..8906d69 --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java @@ -0,0 +1,29 @@ +package com.tamguo.config.dao; + +import java.io.Serializable; +import com.baomidou.mybatisplus.activerecord.Model; +import com.baomidou.mybatisplus.annotations.TableId; + +/** + * 实体父类 + */ +public class SuperEntity> extends Model { + + private static final long serialVersionUID = 1L; + + @TableId("uid") + private String uid; + + @Override + protected Serializable pkVal() { + return this.getUid(); + } + + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } +} diff --git a/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperMapper.java b/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperMapper.java new file mode 100644 index 0000000..6b8b4cf --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperMapper.java @@ -0,0 +1,11 @@ +package com.tamguo.config.dao; + +import com.baomidou.mybatisplus.mapper.BaseMapper; + +/** + * 演示 mapper 父类，注意这个类不要让 mp 扫描到！！ + */ +public interface SuperMapper extends BaseMapper { + + // 这里可以放一些公共的方法 +} diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/SchoolEntity.java b/tamguo-crawler/src/main/java/com/tamguo/model/SchoolEntity.java new file mode 100644 index 0000000..f155fb1 --- /dev/null +++ b/tamguo-crawler/src/main/java/com/tamguo/model/SchoolEntity.java @@ -0,0 +1,53 @@ +package com.tamguo.model; + +import java.io.Serializable; +import com.baomidou.mybatisplus.annotations.TableName; +import com.tamguo.config.dao.SuperEntity; + + +/** + * The persistent class for the tiku_chapter database table. + * + */ +@TableName(value="tiku_school") +public class SchoolEntity extends SuperEntity implements Serializable { + private static final long serialVersionUID = 1L; + + private String areaId; + + private String name; + + private String image; + + public SchoolEntity() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public static long getSerialversionuid() { + return serialVersionUID; + } + + public String getAreaId() { + return areaId; + } + + public void setAreaId(String areaId) { + this.areaId = areaId; + } + +} \ No newline at end of file diff --git a/tamguo-crawler/src/main/resources/application.properties b/tamguo-crawler/src/main/resources/application.properties new file mode 100644 index 0000000..d84d17c --- /dev/null +++ b/tamguo-crawler/src/main/resources/application.properties @@ -0,0 +1,35 @@ +spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 +spring.datasource.driver-class-name=com.mysql.jdbc.Driver +spring.datasource.filters=stat,wall,log4j +spring.datasource.initialSize=5 +spring.datasource.maxActive=20 +spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 +spring.datasource.maxWait=60000 +spring.datasource.minEvictableIdleTimeMillis=300000 +spring.datasource.minIdle=5 +spring.datasource.password= +spring.datasource.poolPreparedStatements=true +spring.datasource.testOnBorrow=false +spring.datasource.testOnReturn=false +spring.datasource.testWhileIdle=true +spring.datasource.timeBetweenEvictionRunsMillis=60000 +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tiku?useUnicode=true&characterEncoding=UTF-8&useSSL=false +spring.datasource.username=root +spring.datasource.validationQuery=SELECT 1 FROM DUAL + +mybatis-plus.mapper-locations=classpath:/mappers/*Mapper.xml +mybatis-plus.typeAliasesPackage=com.tamguo.model +#mybatis-plus.typeEnumsPackage=com.tamguo.model.enums +mybatis-plus.global-config.id-type=5 +mybatis-plus.global-config.field-strategy=2 +mybatis-plus.global-config.db-column-underline=true +mybatis-plus.global-config.refresh-mapper=true +mybatis-plus.global-config.key-generator=com.baomidou.mybatisplus.incrementer.H2KeyGenerator +mybatis-plus.global-config.logic-delete-value=0 +mybatis-plus.global-config.logic-not-delete-value=1 +mybatis-plus.global-config.sql-injector=com.baomidou.mybatisplus.mapper.LogicSqlInjector +mybatis-plus.global-config.meta-object-handler=com.tamguo.config.dao.MyMetaObjectHandler +mybatis-plus.global-config.sql-parser-cache=true +mybatis-plus.configuration.map-underscore-to-camel-case=true +mybatis-plus.configuration.cache-enabled=false \ No newline at end of file