From 689af76638cfa02d204460b051a434cd9ace9c3e Mon Sep 17 00:00:00 2001 From: tamguo Date: Fri, 10 Aug 2018 15:46:03 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/tamguo/model/ChapterEntity.java | 10 ++++++++++ .../com/tamguo/service/impl/QuestionService.java | 11 ++++++----- .../com/tamguo/service/impl/SubjectService.java | 13 +++++++++---- .../src/main/java/com/tamguo/Application.java | 1 + 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java b/tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java index fa8c8be..5039668 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java +++ b/tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java @@ -13,6 +13,8 @@ import com.tamguo.config.dao.SuperEntity; public class ChapterEntity extends SuperEntity implements Serializable { private static final long serialVersionUID = 1L; + private String subjectId; + private String courseId; private String name; @@ -116,4 +118,12 @@ public class ChapterEntity extends SuperEntity implements Seriali this.parentCodes = parentCodes; } + public String getSubjectId() { + return subjectId; + } + + public void setSubjectId(String subjectId) { + this.subjectId = subjectId; + } + } \ No newline at end of file diff --git a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java index d7e1e28..757fa7f 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java +++ b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java @@ -52,7 +52,8 @@ public class QuestionService implements IQuestionService{ @Autowired CacheService cacheService; private static final String FILES_NO_FORMAT = "000000000"; - private static final String FILES_PREFIX = "likeshuxue"; + private static final String FILES_PREFIX = "shengwu"; + private static final String COURSE_ID = "shengwu"; private static final String DOMAIN = "http://www.tamguo.com"; private RunData runData; @@ -139,7 +140,7 @@ public class QuestionService implements IQuestionService{ System.out.println("down images " + (ret?"success":"fail") + "：" + img); // 替换URL - question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName)); + question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); } question.setAnswer(question.getAnswer()); } @@ -160,7 +161,7 @@ public class QuestionService implements IQuestionService{ System.out.println("down images " + (ret?"success":"fail") + "：" + img); // 替换URL - question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName)); + question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); } question.setAnalysis(question.getAnalysis()); } @@ -181,7 +182,7 @@ public class QuestionService implements IQuestionService{ System.out.println("down images " + (ret?"success":"fail") + "：" + img); // 替换URL - question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName)); + question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); } question.setContent(question.getContent()); } @@ -198,7 +199,7 @@ public class QuestionService implements IQuestionService{ } private String getFilePath() { - return "/home/webdata/files/question/"; + return "/home/webdata/files/question/" + COURSE_ID + "/"; } private String getFileDatePath() { diff --git a/tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java b/tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java index 7d0cd5b..561925c 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java +++ b/tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java @@ -30,8 +30,9 @@ import com.xuxueli.crawler.rundata.RunData; @Service public class SubjectService implements ISubjectService{ - private final static String COURSE_ID = "likeshuxue"; - private final static String BOOK_ID = "1025976567395184642"; + private final static String COURSE_ID = "shengwu"; + private final static String BOOK_ID = "1025976567395184645"; + private final static String SUBJECT_ID = "gaokao"; @Autowired SubjectMapper subjectMapper; @Autowired @@ -52,7 +53,7 @@ public class SubjectService implements ISubjectService{ @Override public void crawlerSubject() { XxlCrawler crawler = new XxlCrawler.Builder() - .setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-16-knowpoint-11") + .setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-20-knowpoint-11") .setAllowSpread(false) .setFailRetryCount(5) .setThreadCount(20) @@ -62,7 +63,7 @@ public class SubjectService implements ISubjectService{ public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) { // 解析封装 PageVo 对象 String pageUrl = html.baseUri(); - if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/")) { + if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-20-knowpoint-11")) { logger.info("开始解析书籍：{}" , pageUrl); ChapterEntity chapterCondition = new ChapterEntity(); chapterCondition.setName(subjectVo.getChapterCurrName()); @@ -82,6 +83,7 @@ public class SubjectService implements ISubjectService{ rootChapter.setTreeLevel(0); rootChapter.setParentCodes("-1,"); rootChapter.setBookId(BOOK_ID); + rootChapter.setSubjectId(SUBJECT_ID); chapterMapper.insert(rootChapter); Elements elements = pageVoElement.getElementsByClass("detail-chapter"); @@ -99,6 +101,7 @@ public class SubjectService implements ISubjectService{ chapter.setBookId(BOOK_ID); chapter.setTreeLeaf(false); chapter.setTreeLevel(1); + chapter.setSubjectId(SUBJECT_ID); chapterMapper.insert(chapter); chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getId() + ","); chapterMapper.updateById(chapter); @@ -119,6 +122,7 @@ public class SubjectService implements ISubjectService{ chapter1.setQuestionNum(0); chapter1.setPointNum(0); chapter1.setOrders(i+1); + chapter1.setSubjectId(SUBJECT_ID); chapter1.setTreeLeaf(false); chapter1.setTreeLevel(2); @@ -143,6 +147,7 @@ public class SubjectService implements ISubjectService{ chapter2.setOrders(k+1); chapter2.setTreeLeaf(true); chapter2.setTreeLevel(3); + chapter2.setSubjectId(SUBJECT_ID); chapterMapper.insert(chapter2); chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getId() + ","); chapterMapper.updateById(chapter2); diff --git a/tamguo-tms/src/main/java/com/tamguo/Application.java b/tamguo-tms/src/main/java/com/tamguo/Application.java index d1d90b8..f013cb3 100644 --- a/tamguo-tms/src/main/java/com/tamguo/Application.java +++ b/tamguo-tms/src/main/java/com/tamguo/Application.java @@ -1,3 +1,4 @@ + package com.tamguo; import org.springframework.boot.autoconfigure.SpringBootApplication;