From ee067a21c1e1c55c00748058b97fbe3932bb59b8 Mon Sep 17 00:00:00 2001 From: tamguo Date: Fri, 13 Jul 2018 13:33:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/tamguo/model/SubjectEntity.java | 20 ------------- .../tamguo/service/impl/QuestionService.java | 30 ++++++++++++++++--- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/tamguo-crawler/src/main/java/com/tamguo/model/SubjectEntity.java b/tamguo-crawler/src/main/java/com/tamguo/model/SubjectEntity.java index 7bc9cf9..7ac8592 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/model/SubjectEntity.java +++ b/tamguo-crawler/src/main/java/com/tamguo/model/SubjectEntity.java @@ -10,10 +10,6 @@ public class SubjectEntity extends SuperEntity implements Seriali private static final long serialVersionUID = 1L; private String name; - - private String courseId; - - private String courseName; public String getName() { return name; @@ -23,20 +19,4 @@ public class SubjectEntity extends SuperEntity implements Seriali this.name = name; } - public String getCourseId() { - return courseId; - } - - public void setCourseId(String courseId) { - this.courseId = courseId; - } - - public String getCourseName() { - return courseName; - } - - public void setCourseName(String courseName) { - this.courseName = courseName; - } - } diff --git a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java index 341842f..dc46dab 100644 --- a/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java +++ b/tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java @@ -51,7 +51,7 @@ public class QuestionService implements IQuestionService{ SubjectMapper subjectMapper; @Autowired CacheService cacheService; - private static final String FILES_NO_FORMAT = "0000000000"; + private static final String FILES_NO_FORMAT = "000000"; private static final String FILES_PREFIX = "FPIMAGE"; private static final String DOMAIN = "http://www.tamguo.com"; @@ -137,6 +137,7 @@ public class QuestionService implements IQuestionService{ // 替换URL questionVo.setAnswer(questionVo.getAnswer().replace(img, DOMAIN + getFilePath() + fileName)); } + question.setAnswer(questionVo.getAnswer()); } if (questionVo.getAnalysisImages()!=null && questionVo.getAnalysisImages().size() > 0) { @@ -154,7 +155,28 @@ public class QuestionService implements IQuestionService{ // 替换URL questionVo.setAnalysis(questionVo.getAnalysis().replace(img, DOMAIN + getFilePath() + fileName)); } + question.setAnalysis(questionVo.getAnalysis()); } + + if (questionVo.getContentImages()!=null && questionVo.getContentImages().size() > 0) { + Set imagesSet = new HashSet<>(questionVo.getContentImages()); + for (String img: imagesSet) { + + // 下载图片文件 + String fileName = getFileName(img); + File dir = new File(getFilePath()); + if (!dir.exists()) + dir.mkdirs(); + boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName); + System.out.println("down images " + (ret?"success":"fail") + ":" + img); + + // 替换URL + questionVo.setContent(questionVo.getContent().replace(img, DOMAIN + getFilePath() + fileName)); + } + question.setContent(questionVo.getContent()); + } + + // 处理图片 question.setSourceType("baidu"); question.setSourceUrl(html.baseUri()); @@ -166,13 +188,13 @@ public class QuestionService implements IQuestionService{ } private String getFilePath() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + SimpleDateFormat sdf = new SimpleDateFormat("ddHHmm"); String format = sdf.format(new Date()); return "/images/question/" + format + "/"; } private String getFileNo() { - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMM"); + SimpleDateFormat sdf = new SimpleDateFormat("ddHHmm"); String format = sdf.format(new Date()); DecimalFormat df = new DecimalFormat(FILES_NO_FORMAT); String key = FILES_PREFIX + format; @@ -184,7 +206,7 @@ public class QuestionService implements IQuestionService{ runData = crawler.getRunData(); int page = 1; - int pageSize = 100; + int pageSize = 1000; while(true) { Page questionPage = new Page(page , pageSize); List questionList = crawlerQuestionMapper.queryPageOrderUid(questionPage);