pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

章节优化

Browse Source
main
tamguo 7 years ago
parent e6f32c6b24
commit de7789fdba
8 changed files with 30 additions and 29 deletions
Show Stats Download Patch File Download Diff File

13
tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java
Unescape View File

@ -12,18 +12,19 @@ public class SuperEntity<T extends Model<?>> extends Model<T> {
private static final long serialVersionUID = 1L;
@TableId("id")
private String uid;
private String id;
@Override
protected Serializable pkVal() {
return this.getUid();
return this.getId();
}
public String getUid() {
return uid;
public String getId() {
return id;
}
public void setUid(String uid) {
this.uid = uid;
public void setId(String id) {
this.id = id;
}
}

6
tamguo-crawler/src/main/java/com/tamguo/service/impl/BookService.java
Unescape View File

@ -98,8 +98,8 @@ public class BookService implements IBookService {
}
question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1");
question.setChapterId(chapter.getUid());
question.setCourseId(course.getUid());
question.setChapterId(chapter.getId());
question.setCourseId(course.getId());
question.setPaperId(null);
question.setQuestionType(questionType.getValue().toString());
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
@ -119,7 +119,7 @@ public class BookService implements IBookService {
question.setYear(questionVo.getYear());
}
}
question.setSubjectId(subject.getUid());
question.setSubjectId(subject.getId());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {
Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages());

10
tamguo-crawler/src/main/java/com/tamguo/service/impl/ChapterService.java
Unescape View File

@ -51,19 +51,19 @@ public class ChapterService implements IChapterService{
@Override
public void modifyQuestionNum() {
int page = 1;
int pageSize = 1000;
int pageSize = 100;
while(true) {
Page<ChapterEntity> chapterPage = new Page<ChapterEntity>(page , pageSize);
List<ChapterEntity> chapterList = chapterMapper.queryList(chapterPage);
for(int i=0 ;i<chapterList.size() ; i++) {
// 处理数据
ChapterEntity chapter = chapterList.get(i);
Integer count = chapterMapper.queryCount(chapter.getUid());
Integer count = chapterMapper.queryCount(chapter.getId());
chapter.setQuestionNum(count);
chapterMapper.updateById(chapter);
}
page++;
if(chapterList.size() < 1000) {
if(chapterList.size() < 100) {
break;
}
}
@ -105,7 +105,7 @@ public class ChapterService implements IChapterService{
CourseEntity course = courseMapper.selectById(chapterEntity.getCourseId());
CrawlerChapterEntity crawlerChapter = new CrawlerChapterEntity();
crawlerChapter.setChapterUid(chapterEntity.getUid());
crawlerChapter.setChapterUid(chapterEntity.getId());
crawlerChapter.setChapterUrl(pageUrl);
crawlerChapter.setCourseUid(chapterEntity.getCourseId());
crawlerChapter.setSubjectUid(course.getSubjectId());
@ -120,7 +120,7 @@ public class ChapterService implements IChapterService{
}
if(!urls.contains(url)) {
crawlerChapter = new CrawlerChapterEntity();
crawlerChapter.setChapterUid(chapterEntity.getUid());
crawlerChapter.setChapterUid(chapterEntity.getId());
crawlerChapter.setChapterUrl(url);
crawlerChapter.setCourseUid(chapterEntity.getCourseId());
crawlerChapter.setSubjectUid(course.getSubjectId());

2
tamguo-crawler/src/main/java/com/tamguo/service/impl/CrawlerBookService.java
Unescape View File

@ -38,7 +38,7 @@ public class CrawlerBookService implements ICrawlerBookService {
List<BookEntity> bookEntities = bookMapper.selectList(Condition.EMPTY);
for (BookEntity bookEntity : bookEntities) {
String url = bookEntity.getReserveField1();
String bookId = bookEntity.getUid();
String bookId = bookEntity.getId();
String regexs = url.replaceAll("\\d+", "\\\\d+").replaceAll("\\.","\\\\.");
XxlCrawler crawler = new XxlCrawler.Builder()

6
tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java
Unescape View File

@ -100,8 +100,8 @@ public class QuestionService implements IQuestionService{
}
question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1");
question.setChapterId(chapter.getUid());
question.setCourseId(course.getUid());
question.setChapterId(chapter.getId());
question.setCourseId(course.getId());
question.setPaperId(null);
question.setQuestionType(questionType.getValue().toString());
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
@ -121,7 +121,7 @@ public class QuestionService implements IQuestionService{
question.setYear(questionVo.getYear());
}
}
question.setSubjectId(subject.getUid());
question.setSubjectId(subject.getId());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {
Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages());

14
tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java
Unescape View File

@ -91,7 +91,7 @@ public class SubjectService implements ISubjectService{
logger.info(chapterName);
ChapterEntity chapter = new ChapterEntity();
chapter.setCourseId(COURSE_ID);
chapter.setParentCode(rootChapter.getUid());
chapter.setParentCode(rootChapter.getId());
chapter.setName(chapterName);
chapter.setQuestionNum(0);
chapter.setPointNum(0);
@ -100,7 +100,7 @@ public class SubjectService implements ISubjectService{
chapter.setTreeLeaf(false);
chapter.setTreeLevel(1);
chapterMapper.insert(chapter);
chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getUid() + ",");
chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getId() + ",");
chapterMapper.updateById(chapter);
Elements detailKpoint1s = element.getElementsByClass("detail-kpoint-1");
@ -114,7 +114,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter1 = new ChapterEntity();
chapter1.setCourseId(COURSE_ID);
chapter1.setBookId(BOOK_ID);
chapter1.setParentCode(chapter.getUid());
chapter1.setParentCode(chapter.getId());
chapter1.setName(chapterName1);
chapter1.setQuestionNum(0);
chapter1.setPointNum(0);
@ -123,7 +123,7 @@ public class SubjectService implements ISubjectService{
chapter1.setTreeLeaf(false);
chapter1.setTreeLevel(2);
chapterMapper.insert(chapter1);
chapter1.setParentCodes(chapter.getParentCodes() + chapter1.getUid() + ",");
chapter1.setParentCodes(chapter.getParentCodes() + chapter1.getId() + ",");
chapterMapper.updateById(chapter1);
@ -136,7 +136,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter2 = new ChapterEntity();
chapter2.setCourseId(COURSE_ID);
chapter2.setBookId(BOOK_ID);
chapter2.setParentCode(chapter1.getUid());
chapter2.setParentCode(chapter1.getId());
chapter2.setName(chapterName2);
chapter2.setQuestionNum(0);
chapter2.setPointNum(0);
@ -144,7 +144,7 @@ public class SubjectService implements ISubjectService{
chapter2.setTreeLeaf(true);
chapter2.setTreeLevel(3);
chapterMapper.insert(chapter2);
chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getUid() + ",");
chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getId() + ",");
chapterMapper.updateById(chapter2);
Elements maskList = detailKpoint.getElementsByClass("mask");
@ -178,7 +178,7 @@ public class SubjectService implements ISubjectService{
if(crawlerQuestionMapper.selectOne(condition) == null) {
CrawlerQuestionEntity crawlerQuestion = new CrawlerQuestionEntity();
crawlerQuestion.setQuestionUrl(questionUrl);
crawlerQuestion.setChapterId(chapterEntity.getUid());
crawlerQuestion.setChapterId(chapterEntity.getId());
crawlerQuestion.setStatus("0");
crawlerQuestionMapper.insert(crawlerQuestion);
}else {

2
tamguo-crawler/src/main/resources/mappers/ChapterMapper.xml
Unescape View File

@ -3,7 +3,7 @@
<mapper namespace="com.tamguo.dao.ChapterMapper">
<select id="queryList" resultType="ChapterEntity">
select * from tiku_chapter order by uid desc
select * from tiku_chapter order by id desc
</select>
<select id="queryCount" resultType="java.lang.Integer">

6
tamguo-crawler/src/test/java/com/tamguo/SingleQuestionCrawler.java
Unescape View File

@ -81,16 +81,16 @@ public class SingleQuestionCrawler {
question.setAnalysis(questionVo.getAnalysis());
question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1");
question.setChapterId(chapter.getUid());
question.setChapterId(chapter.getId());
question.setContent(questionVo.getContent());
question.setCourseId(course.getUid());
question.setCourseId(course.getId());
question.setPaperId(null);
question.setQuestionType("1");
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
question.setReviewPoint(StringUtils.join(questionVo.getReviewPoint().toArray(), ","));
}
question.setScore(questionVo.getScore());
question.setSubjectId(subject.getUid());
question.setSubjectId(subject.getId());
question.setYear(questionVo.getYear());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {

Write Preview
Loading…
Cancel
Save