pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

优化

Browse Source
main
tamguo 7 years ago
parent c7d809b09d
commit 689af76638
4 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File

10
tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java
Unescape View File

@ -13,6 +13,8 @@ import com.tamguo.config.dao.SuperEntity;
public class ChapterEntity extends SuperEntity<ChapterEntity> implements Serializable {
private static final long serialVersionUID = 1L;
private String subjectId;
private String courseId;
private String name;
@ -116,4 +118,12 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
this.parentCodes = parentCodes;
}
public String getSubjectId() {
return subjectId;
}
public void setSubjectId(String subjectId) {
this.subjectId = subjectId;
}
}

11
tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java
Unescape View File

@ -52,7 +52,8 @@ public class QuestionService implements IQuestionService{
@Autowired
CacheService cacheService;
private static final String FILES_NO_FORMAT = "000000000";
private static final String FILES_PREFIX = "likeshuxue";
private static final String FILES_PREFIX = "shengwu";
private static final String COURSE_ID = "shengwu";
private static final String DOMAIN = "http://www.tamguo.com";
private RunData runData;
@ -139,7 +140,7 @@ public class QuestionService implements IQuestionService{
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
}
question.setAnswer(question.getAnswer());
}
@ -160,7 +161,7 @@ public class QuestionService implements IQuestionService{
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
}
question.setAnalysis(question.getAnalysis());
}
@ -181,7 +182,7 @@ public class QuestionService implements IQuestionService{
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
}
question.setContent(question.getContent());
}
@ -198,7 +199,7 @@ public class QuestionService implements IQuestionService{
}
private String getFilePath() {
return "/home/webdata/files/question/";
return "/home/webdata/files/question/" + COURSE_ID + "/";
}
private String getFileDatePath() {

13
tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java
Unescape View File

@ -30,8 +30,9 @@ import com.xuxueli.crawler.rundata.RunData;
@Service
public class SubjectService implements ISubjectService{
private final static String COURSE_ID = "likeshuxue";
private final static String BOOK_ID = "1025976567395184642";
private final static String COURSE_ID = "shengwu";
private final static String BOOK_ID = "1025976567395184645";
private final static String SUBJECT_ID = "gaokao";
@Autowired
SubjectMapper subjectMapper;
@Autowired
@ -52,7 +53,7 @@ public class SubjectService implements ISubjectService{
@Override
public void crawlerSubject() {
XxlCrawler crawler = new XxlCrawler.Builder()
.setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-16-knowpoint-11")
.setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-20-knowpoint-11")
.setAllowSpread(false)
.setFailRetryCount(5)
.setThreadCount(20)
@ -62,7 +63,7 @@ public class SubjectService implements ISubjectService{
public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
// 解析封装 PageVo 对象
String pageUrl = html.baseUri();
if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/")) {
if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-20-knowpoint-11")) {
logger.info("开始解析书籍：{}" , pageUrl);
ChapterEntity chapterCondition = new ChapterEntity();
chapterCondition.setName(subjectVo.getChapterCurrName());
@ -82,6 +83,7 @@ public class SubjectService implements ISubjectService{
rootChapter.setTreeLevel(0);
rootChapter.setParentCodes("-1,");
rootChapter.setBookId(BOOK_ID);
rootChapter.setSubjectId(SUBJECT_ID);
chapterMapper.insert(rootChapter);
Elements elements = pageVoElement.getElementsByClass("detail-chapter");
@ -99,6 +101,7 @@ public class SubjectService implements ISubjectService{
chapter.setBookId(BOOK_ID);
chapter.setTreeLeaf(false);
chapter.setTreeLevel(1);
chapter.setSubjectId(SUBJECT_ID);
chapterMapper.insert(chapter);
chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getId() + ",");
chapterMapper.updateById(chapter);
@ -119,6 +122,7 @@ public class SubjectService implements ISubjectService{
chapter1.setQuestionNum(0);
chapter1.setPointNum(0);
chapter1.setOrders(i+1);
chapter1.setSubjectId(SUBJECT_ID);
chapter1.setTreeLeaf(false);
chapter1.setTreeLevel(2);
@ -143,6 +147,7 @@ public class SubjectService implements ISubjectService{
chapter2.setOrders(k+1);
chapter2.setTreeLeaf(true);
chapter2.setTreeLevel(3);
chapter2.setSubjectId(SUBJECT_ID);
chapterMapper.insert(chapter2);
chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getId() + ",");
chapterMapper.updateById(chapter2);

1
tamguo-tms/src/main/java/com/tamguo/Application.java
Unescape View File

@ -1,3 +1,4 @@
package com.tamguo;
import org.springframework.boot.autoconfigure.SpringBootApplication;

Write Preview
Loading…
Cancel
Save