爬虫优化

main
tamguo 7 years ago
parent 2715688990
commit 992184638b

@ -17,9 +17,9 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
private String name;
private String parentId;
private String parentCode;
private String parentIds;
private String parentCodes;
private Integer treeLevel;
@ -52,14 +52,6 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
this.name = name;
}
public String getParentId() {
return this.parentId;
}
public void setParentId(String parentId) {
this.parentId = parentId;
}
public Integer getQuestionNum() {
return questionNum;
}
@ -92,14 +84,6 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
this.bookId = bookId;
}
public String getParentIds() {
return parentIds;
}
public void setParentIds(String parentIds) {
this.parentIds = parentIds;
}
public Integer getTreeLevel() {
return treeLevel;
}
@ -116,4 +100,20 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
this.treeLeaf = treeLeaf;
}
public String getParentCode() {
return parentCode;
}
public void setParentCode(String parentCode) {
this.parentCode = parentCode;
}
public String getParentCodes() {
return parentCodes;
}
public void setParentCodes(String parentCodes) {
this.parentCodes = parentCodes;
}
}

@ -28,10 +28,12 @@ public class CrawlerBookService implements ICrawlerBookService {
@Autowired
BookMapper bookMapper;
private Logger logger = LoggerFactory.getLogger(getClass());
@SuppressWarnings("unused")
private Logger logger = LoggerFactory.getLogger(getClass());
@Override
@SuppressWarnings("unchecked")
@Override
public void crawlerBook() {
List<BookEntity> bookEntities = bookMapper.selectList(Condition.EMPTY);
for (BookEntity bookEntity : bookEntities) {
@ -48,7 +50,7 @@ public class CrawlerBookService implements ICrawlerBookService {
.setPageParser(new PageParser<CrawlerBookVo>() {
@Override
public void parse(Document html, Element pageVoElement, CrawlerBookVo crawlerBookVo) {
String pageUrl = html.baseUri();
// String pageUrl = html.baseUri();
// 解析封装 PageVo 对象
String img = crawlerBookVo.getBookImage();
if (StringUtils.isNoneBlank(img)) {

@ -73,14 +73,14 @@ public class SubjectService implements ISubjectService{
ChapterEntity rootChapter = new ChapterEntity();
rootChapter.setCourseId(COURSE_ID);
rootChapter.setParentId("-1");
rootChapter.setParentCode("-1");
rootChapter.setName(subjectVo.getChapterCurrName());
rootChapter.setQuestionNum(0);
rootChapter.setPointNum(0);
rootChapter.setOrders(0);
rootChapter.setTreeLeaf(false);
rootChapter.setTreeLevel(0);
rootChapter.setParentIds("-1,");
rootChapter.setParentCodes("-1,");
rootChapter.setBookId(BOOK_ID);
chapterMapper.insert(rootChapter);
@ -91,7 +91,7 @@ public class SubjectService implements ISubjectService{
logger.info(chapterName);
ChapterEntity chapter = new ChapterEntity();
chapter.setCourseId(COURSE_ID);
chapter.setParentId(rootChapter.getUid());
chapter.setParentCode(rootChapter.getUid());
chapter.setName(chapterName);
chapter.setQuestionNum(0);
chapter.setPointNum(0);
@ -100,7 +100,7 @@ public class SubjectService implements ISubjectService{
chapter.setTreeLeaf(false);
chapter.setTreeLevel(1);
chapterMapper.insert(chapter);
chapter.setParentIds(rootChapter.getParentIds() + chapter.getUid() + ",");
chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getUid() + ",");
chapterMapper.updateById(chapter);
Elements detailKpoint1s = element.getElementsByClass("detail-kpoint-1");
@ -114,7 +114,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter1 = new ChapterEntity();
chapter1.setCourseId(COURSE_ID);
chapter1.setBookId(BOOK_ID);
chapter1.setParentId(chapter.getUid());
chapter1.setParentCode(chapter.getUid());
chapter1.setName(chapterName1);
chapter1.setQuestionNum(0);
chapter1.setPointNum(0);
@ -123,7 +123,7 @@ public class SubjectService implements ISubjectService{
chapter1.setTreeLeaf(false);
chapter1.setTreeLevel(2);
chapterMapper.insert(chapter1);
chapter1.setParentIds(chapter.getParentIds() + chapter1.getUid() + ",");
chapter1.setParentCodes(chapter.getParentCodes() + chapter1.getUid() + ",");
chapterMapper.updateById(chapter1);
@ -136,7 +136,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter2 = new ChapterEntity();
chapter2.setCourseId(COURSE_ID);
chapter2.setBookId(BOOK_ID);
chapter2.setParentId(chapter1.getUid());
chapter2.setParentCode(chapter1.getUid());
chapter2.setName(chapterName2);
chapter2.setQuestionNum(0);
chapter2.setPointNum(0);
@ -144,7 +144,7 @@ public class SubjectService implements ISubjectService{
chapter2.setTreeLeaf(true);
chapter2.setTreeLevel(3);
chapterMapper.insert(chapter2);
chapter2.setParentIds(chapter1.getParentIds() + chapter2.getUid() + ",");
chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getUid() + ",");
chapterMapper.updateById(chapter2);
Elements maskList = detailKpoint.getElementsByClass("mask");

Loading…
Cancel
Save