diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/QuestionEntity.java b/tamguo-crawler/src/main/java/com/tamguo/model/QuestionEntity.java index 82ff201..080560d 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/model/QuestionEntity.java +++ b/tamguo-crawler/src/main/java/com/tamguo/model/QuestionEntity.java @@ -36,6 +36,10 @@ public class QuestionEntity extends SuperEntity implements Seria private String score; private String auditStatus; + + private String sourceType; + + private String sourceUrl; public QuestionEntity() { } @@ -136,4 +140,20 @@ public class QuestionEntity extends SuperEntity implements Seria this.auditStatus = auditStatus; } + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public String getSourceUrl() { + return sourceUrl; + } + + public void setSourceUrl(String sourceUrl) { + this.sourceUrl = sourceUrl; + } + } \ No newline at end of file diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/enums/QuestionType.java b/tamguo-crawler/src/main/java/com/tamguo/model/enums/QuestionType.java index df9e3a1..3939e6b 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/model/enums/QuestionType.java +++ b/tamguo-crawler/src/main/java/com/tamguo/model/enums/QuestionType.java @@ -34,8 +34,10 @@ public enum QuestionType { return PANDUANTI; }else if("问答题".equals(value)) { return WENDATI; + }else if("选择题".equals(value)) { + return DANXUANTI; } - return DANXUANTI; + return WENDATI; } public Serializable getValue() { diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/vo/QuestionVo.java b/tamguo-crawler/src/main/java/com/tamguo/model/vo/QuestionVo.java index 302ba58..adbbb38 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/model/vo/QuestionVo.java +++ b/tamguo-crawler/src/main/java/com/tamguo/model/vo/QuestionVo.java @@ -14,7 +14,7 @@ public class QuestionVo { @PageFieldSelect(cssQuery = ".question-box-inner .questem-inner img", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:src") private List contentImages; - @PageFieldSelect(cssQuery=".queoptions-inner", selectType = XxlCrawlerConf.SelectType.HTML) + @PageFieldSelect(cssQuery=".que-options", selectType = XxlCrawlerConf.SelectType.HTML) private String queoptions; @PageFieldSelect(cssQuery=".exam-answer-content", selectType = XxlCrawlerConf.SelectType.HTML) @@ -38,7 +38,7 @@ public class QuestionVo { @PageFieldSelect(cssQuery=".que-title span:eq(2)",selectType = XxlCrawlerConf.SelectType.TEXT) private String year; - @PageFieldSelect(cssQuery=".kpoint-contain point point-item",selectType = XxlCrawlerConf.SelectType.TEXT) + @PageFieldSelect(cssQuery=".kpoint-contain .point .point-item",selectType = XxlCrawlerConf.SelectType.TEXT) private List reviewPoint; diff --git a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java index 1481845..341842f 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java +++ b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java @@ -26,6 +26,7 @@ import com.tamguo.model.CourseEntity; import com.tamguo.model.CrawlerQuestionEntity; import com.tamguo.model.QuestionEntity; import com.tamguo.model.SubjectEntity; +import com.tamguo.model.enums.QuestionType; import com.tamguo.model.vo.QuestionVo; import com.tamguo.service.IQuestionService; import com.xuxueli.crawler.XxlCrawler; @@ -50,9 +51,9 @@ public class QuestionService implements IQuestionService{ SubjectMapper subjectMapper; @Autowired CacheService cacheService; - private static final String FILES_NO_FORMAT = "00000"; - private static final String FILES_PREFIX = "FP"; - private static final String DOMAIN = "http://static.tamguo.com"; + private static final String FILES_NO_FORMAT = "0000000000"; + private static final String FILES_PREFIX = "FPIMAGE"; + private static final String DOMAIN = "http://www.tamguo.com"; private RunData runData; @@ -61,12 +62,17 @@ public class QuestionService implements IQuestionService{ XxlCrawler crawler = new XxlCrawler.Builder() .setAllowSpread(false) - .setThreadCount(10) + .setThreadCount(20) + .setFailRetryCount(5) .setPageLoader(new HtmlUnitPageLoader()) .setPageParser(new PageParser() { @Override public void parse(Document html, Element pageVoElement, QuestionVo questionVo) { + if(StringUtils.isEmpty(questionVo.getContent())) { + runData.addUrl(html.baseUri()); + return; + } CrawlerQuestionEntity condition = new CrawlerQuestionEntity(); condition.setQuestionUrl(html.baseUri()); CrawlerQuestionEntity crawlerQuestion = crawlerQuestionMapper.selectOne(condition); @@ -74,21 +80,47 @@ public class QuestionService implements IQuestionService{ CourseEntity course = courseMapper.selectById(chapter.getCourseId()); SubjectEntity subject = subjectMapper.selectById(course.getSubjectId()); + QuestionType questionType = QuestionType.getQuestionType(questionVo.getQuestionType()); + + QuestionEntity question = new QuestionEntity(); + if(questionType == QuestionType.DANXUANTI) { + if(!StringUtils.isEmpty(questionVo.getQueoptions())) { + question.setContent(questionVo.getContent() + questionVo.getQueoptions()); + }else { + question.setContent(questionVo.getContent()); + } + }else { + question.setContent(questionVo.getContent()); + } question.setAnalysis(questionVo.getAnalysis()); + if(StringUtils.isEmpty(question.getAnswer())) { + question.setAnalysis("


"); + } question.setAnswer(questionVo.getAnswer()); question.setAuditStatus("1"); question.setChapterId(chapter.getUid()); - question.setContent(questionVo.getContent()); question.setCourseId(course.getUid()); question.setPaperId(null); - question.setQuestionType("1"); + question.setQuestionType(questionType.getValue().toString()); if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) { question.setReviewPoint(StringUtils.join(questionVo.getReviewPoint().toArray(), ",")); } - question.setScore(questionVo.getScore()); + // 处理分数 + if(questionVo.getScore() != null) { + if(questionVo.getScore().contains("分")) { + question.setScore(questionVo.getScore()); + } + if(questionVo.getScore().contains("年")) { + question.setYear(questionVo.getScore()); + } + } + if(questionVo.getYear() != null) { + if(questionVo.getYear().contains("年")) { + question.setYear(questionVo.getYear()); + } + } question.setSubjectId(subject.getUid()); - question.setYear(questionVo.getYear()); if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) { Set imagesSet = new HashSet<>(questionVo.getAnswerImages()); @@ -124,8 +156,9 @@ public class QuestionService implements IQuestionService{ } } // 处理图片 + question.setSourceType("baidu"); + question.setSourceUrl(html.baseUri()); questionMapper.insert(question); - } public String getFileName(String img) { diff --git a/tamguo/src/main/java/com/tamguo/model/enums/QuestionType.java b/tamguo/src/main/java/com/tamguo/model/enums/QuestionType.java index 710bdcf..7ebd639 100644 --- a/tamguo/src/main/java/com/tamguo/model/enums/QuestionType.java +++ b/tamguo/src/main/java/com/tamguo/model/enums/QuestionType.java @@ -35,7 +35,7 @@ public enum QuestionType { }else if("5".equals(value)) { return WENDATI; } - return null; + return WENDATI; } public Serializable getValue() { diff --git a/tamguo/src/main/java/com/tamguo/web/QuestionContrller.java b/tamguo/src/main/java/com/tamguo/web/QuestionContrller.java index 980cc38..a0776a8 100644 --- a/tamguo/src/main/java/com/tamguo/web/QuestionContrller.java +++ b/tamguo/src/main/java/com/tamguo/web/QuestionContrller.java @@ -13,6 +13,7 @@ import com.tamguo.model.ChapterEntity; import com.tamguo.model.CourseEntity; import com.tamguo.model.QuestionEntity; import com.tamguo.model.SubjectEntity; +import com.tamguo.model.enums.QuestionType; import com.tamguo.service.IChapterService; import com.tamguo.service.ICourseService; import com.tamguo.service.IQuestionService; @@ -67,6 +68,7 @@ public class QuestionContrller { public ModelAndView question(@PathVariable String uid , ModelAndView model){ model.setViewName("question"); QuestionEntity question = iQuestionService.findNormalQuestion(uid); + question.setQuestionType(QuestionType.getQuestionType(question.getQuestionType()).getDesc()); model.addObject("question", question); // 推荐试题 diff --git a/tamguo/src/main/resources/application.properties b/tamguo/src/main/resources/application.properties index d77206e..9cb104b 100644 --- a/tamguo/src/main/resources/application.properties +++ b/tamguo/src/main/resources/application.properties @@ -1,7 +1,7 @@ sitenum=2 -domain.name=http://www.tanguoguo.com/ +domain.name=http://localhost/ admin.domain.name=http://admin.tanguoguo.com -server.port=8081 +server.port=80 jasypt.encryptor.password=tamguo spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 @@ -20,7 +20,7 @@ spring.datasource.testOnReturn=false spring.datasource.testWhileIdle=true spring.datasource.timeBetweenEvictionRunsMillis=60000 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.url=jdbc:mysql://127.0.0.1:3306/tamguo?useUnicode=true&characterEncoding=UTF-8&useSSL=false +spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tiku?useUnicode=true&characterEncoding=UTF-8&useSSL=false spring.datasource.username=root spring.datasource.validationQuery=SELECT 1 FROM DUAL @@ -47,7 +47,7 @@ spring.thymeleaf.encoding=UTF-8 spring.thymeleaf.content-type=text/html spring.thymeleaf.cache=false -redis.hostname=127.0.0.1 +redis.hostname=47.100.175.14 redis.port=6379 redis.password= diff --git a/tamguo/src/main/resources/redis.xml b/tamguo/src/main/resources/redis.xml index ed04f74..4055e0d 100644 --- a/tamguo/src/main/resources/redis.xml +++ b/tamguo/src/main/resources/redis.xml @@ -4,6 +4,6 @@ - + \ No newline at end of file diff --git a/tamguo/src/main/resources/templates/index.html b/tamguo/src/main/resources/templates/index.html index 4c127f2..00bd20f 100644 --- a/tamguo/src/main/resources/templates/index.html +++ b/tamguo/src/main/resources/templates/index.html @@ -5,7 +5,7 @@ 探果题库为老师创造价值_探果题库 - + diff --git a/tamguo/src/main/resources/templates/question.html b/tamguo/src/main/resources/templates/question.html index a850384..982108b 100644 --- a/tamguo/src/main/resources/templates/question.html +++ b/tamguo/src/main/resources/templates/question.html @@ -27,7 +27,7 @@
-
填空题 5 2016
+
填空题 5 2016