pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

章节优化

Browse Source
main
tamguo 7 years ago
parent e6f32c6b24
commit de7789fdba
8 changed files with 30 additions and 29 deletions
Show Stats Download Patch File Download Diff File

13
tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java
Unescape View File

@ -12,18 +12,19 @@ public class SuperEntity<T extends Model<?>> extends Model<T> {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
@TableId("id") @TableId("id")
private String uid; private String id;
@Override @Override
protected Serializable pkVal() { protected Serializable pkVal() {
return this.getUid(); return this.getId();
} }
public String getUid() { public String getId() {
return uid; return id;
} }
public void setUid(String uid) { public void setId(String id) {
this.uid = uid; this.id = id;
} }
} }

6
tamguo-crawler/src/main/java/com/tamguo/service/impl/BookService.java
Unescape View File

@ -98,8 +98,8 @@ public class BookService implements IBookService {
} }
question.setAnswer(questionVo.getAnswer()); question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1"); question.setAuditStatus("1");
question.setChapterId(chapter.getUid()); question.setChapterId(chapter.getId());
question.setCourseId(course.getUid()); question.setCourseId(course.getId());
question.setPaperId(null); question.setPaperId(null);
question.setQuestionType(questionType.getValue().toString()); question.setQuestionType(questionType.getValue().toString());
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) { if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
@ -119,7 +119,7 @@ public class BookService implements IBookService {
question.setYear(questionVo.getYear()); question.setYear(questionVo.getYear());
} }
} }
question.setSubjectId(subject.getUid()); question.setSubjectId(subject.getId());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) { if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {
Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages()); Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages());

10
tamguo-crawler/src/main/java/com/tamguo/service/impl/ChapterService.java
Unescape View File

@ -51,19 +51,19 @@ public class ChapterService implements IChapterService{
@Override @Override
public void modifyQuestionNum() { public void modifyQuestionNum() {
int page = 1; int page = 1;
int pageSize = 1000; int pageSize = 100;
while(true) { while(true) {
Page<ChapterEntity> chapterPage = new Page<ChapterEntity>(page , pageSize); Page<ChapterEntity> chapterPage = new Page<ChapterEntity>(page , pageSize);
List<ChapterEntity> chapterList = chapterMapper.queryList(chapterPage); List<ChapterEntity> chapterList = chapterMapper.queryList(chapterPage);
for(int i=0 ;i<chapterList.size() ; i++) { for(int i=0 ;i<chapterList.size() ; i++) {
// 处理数据 // 处理数据
ChapterEntity chapter = chapterList.get(i); ChapterEntity chapter = chapterList.get(i);
Integer count = chapterMapper.queryCount(chapter.getUid()); Integer count = chapterMapper.queryCount(chapter.getId());
chapter.setQuestionNum(count); chapter.setQuestionNum(count);
chapterMapper.updateById(chapter); chapterMapper.updateById(chapter);
} }
page++; page++;
if(chapterList.size() < 1000) { if(chapterList.size() < 100) {
break; break;
} }
} }
@ -105,7 +105,7 @@ public class ChapterService implements IChapterService{
CourseEntity course = courseMapper.selectById(chapterEntity.getCourseId()); CourseEntity course = courseMapper.selectById(chapterEntity.getCourseId());
CrawlerChapterEntity crawlerChapter = new CrawlerChapterEntity(); CrawlerChapterEntity crawlerChapter = new CrawlerChapterEntity();
crawlerChapter.setChapterUid(chapterEntity.getUid()); crawlerChapter.setChapterUid(chapterEntity.getId());
crawlerChapter.setChapterUrl(pageUrl); crawlerChapter.setChapterUrl(pageUrl);
crawlerChapter.setCourseUid(chapterEntity.getCourseId()); crawlerChapter.setCourseUid(chapterEntity.getCourseId());
crawlerChapter.setSubjectUid(course.getSubjectId()); crawlerChapter.setSubjectUid(course.getSubjectId());
@ -120,7 +120,7 @@ public class ChapterService implements IChapterService{
} }
if(!urls.contains(url)) { if(!urls.contains(url)) {
crawlerChapter = new CrawlerChapterEntity(); crawlerChapter = new CrawlerChapterEntity();
crawlerChapter.setChapterUid(chapterEntity.getUid()); crawlerChapter.setChapterUid(chapterEntity.getId());
crawlerChapter.setChapterUrl(url); crawlerChapter.setChapterUrl(url);
crawlerChapter.setCourseUid(chapterEntity.getCourseId()); crawlerChapter.setCourseUid(chapterEntity.getCourseId());
crawlerChapter.setSubjectUid(course.getSubjectId()); crawlerChapter.setSubjectUid(course.getSubjectId());

2
tamguo-crawler/src/main/java/com/tamguo/service/impl/CrawlerBookService.java
Unescape View File

@ -38,7 +38,7 @@ public class CrawlerBookService implements ICrawlerBookService {
List<BookEntity> bookEntities = bookMapper.selectList(Condition.EMPTY); List<BookEntity> bookEntities = bookMapper.selectList(Condition.EMPTY);
for (BookEntity bookEntity : bookEntities) { for (BookEntity bookEntity : bookEntities) {
String url = bookEntity.getReserveField1(); String url = bookEntity.getReserveField1();
String bookId = bookEntity.getUid(); String bookId = bookEntity.getId();
String regexs = url.replaceAll("\\d+", "\\\\d+").replaceAll("\\.","\\\\."); String regexs = url.replaceAll("\\d+", "\\\\d+").replaceAll("\\.","\\\\.");
XxlCrawler crawler = new XxlCrawler.Builder() XxlCrawler crawler = new XxlCrawler.Builder()

6
tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java
Unescape View File

@ -100,8 +100,8 @@ public class QuestionService implements IQuestionService{
} }
question.setAnswer(questionVo.getAnswer()); question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1"); question.setAuditStatus("1");
question.setChapterId(chapter.getUid()); question.setChapterId(chapter.getId());
question.setCourseId(course.getUid()); question.setCourseId(course.getId());
question.setPaperId(null); question.setPaperId(null);
question.setQuestionType(questionType.getValue().toString()); question.setQuestionType(questionType.getValue().toString());
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) { if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
@ -121,7 +121,7 @@ public class QuestionService implements IQuestionService{
question.setYear(questionVo.getYear()); question.setYear(questionVo.getYear());
} }
} }
question.setSubjectId(subject.getUid()); question.setSubjectId(subject.getId());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) { if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {
Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages()); Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages());

14
tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java
Unescape View File

@ -91,7 +91,7 @@ public class SubjectService implements ISubjectService{
logger.info(chapterName); logger.info(chapterName);
ChapterEntity chapter = new ChapterEntity(); ChapterEntity chapter = new ChapterEntity();
chapter.setCourseId(COURSE_ID); chapter.setCourseId(COURSE_ID);
chapter.setParentCode(rootChapter.getUid()); chapter.setParentCode(rootChapter.getId());
chapter.setName(chapterName); chapter.setName(chapterName);
chapter.setQuestionNum(0); chapter.setQuestionNum(0);
chapter.setPointNum(0); chapter.setPointNum(0);
@ -100,7 +100,7 @@ public class SubjectService implements ISubjectService{
chapter.setTreeLeaf(false); chapter.setTreeLeaf(false);
chapter.setTreeLevel(1); chapter.setTreeLevel(1);
chapterMapper.insert(chapter); chapterMapper.insert(chapter);
chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getUid() + ","); chapter.setParentCodes(rootChapter.getParentCodes() + chapter.getId() + ",");
chapterMapper.updateById(chapter); chapterMapper.updateById(chapter);
Elements detailKpoint1s = element.getElementsByClass("detail-kpoint-1"); Elements detailKpoint1s = element.getElementsByClass("detail-kpoint-1");
@ -114,7 +114,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter1 = new ChapterEntity(); ChapterEntity chapter1 = new ChapterEntity();
chapter1.setCourseId(COURSE_ID); chapter1.setCourseId(COURSE_ID);
chapter1.setBookId(BOOK_ID); chapter1.setBookId(BOOK_ID);
chapter1.setParentCode(chapter.getUid()); chapter1.setParentCode(chapter.getId());
chapter1.setName(chapterName1); chapter1.setName(chapterName1);
chapter1.setQuestionNum(0); chapter1.setQuestionNum(0);
chapter1.setPointNum(0); chapter1.setPointNum(0);
@ -123,7 +123,7 @@ public class SubjectService implements ISubjectService{
chapter1.setTreeLeaf(false); chapter1.setTreeLeaf(false);
chapter1.setTreeLevel(2); chapter1.setTreeLevel(2);
chapterMapper.insert(chapter1); chapterMapper.insert(chapter1);
chapter1.setParentCodes(chapter.getParentCodes() + chapter1.getUid() + ","); chapter1.setParentCodes(chapter.getParentCodes() + chapter1.getId() + ",");
chapterMapper.updateById(chapter1); chapterMapper.updateById(chapter1);
@ -136,7 +136,7 @@ public class SubjectService implements ISubjectService{
ChapterEntity chapter2 = new ChapterEntity(); ChapterEntity chapter2 = new ChapterEntity();
chapter2.setCourseId(COURSE_ID); chapter2.setCourseId(COURSE_ID);
chapter2.setBookId(BOOK_ID); chapter2.setBookId(BOOK_ID);
chapter2.setParentCode(chapter1.getUid()); chapter2.setParentCode(chapter1.getId());
chapter2.setName(chapterName2); chapter2.setName(chapterName2);
chapter2.setQuestionNum(0); chapter2.setQuestionNum(0);
chapter2.setPointNum(0); chapter2.setPointNum(0);
@ -144,7 +144,7 @@ public class SubjectService implements ISubjectService{
chapter2.setTreeLeaf(true); chapter2.setTreeLeaf(true);
chapter2.setTreeLevel(3); chapter2.setTreeLevel(3);
chapterMapper.insert(chapter2); chapterMapper.insert(chapter2);
chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getUid() + ","); chapter2.setParentCodes(chapter1.getParentCodes() + chapter2.getId() + ",");
chapterMapper.updateById(chapter2); chapterMapper.updateById(chapter2);
Elements maskList = detailKpoint.getElementsByClass("mask"); Elements maskList = detailKpoint.getElementsByClass("mask");
@ -178,7 +178,7 @@ public class SubjectService implements ISubjectService{
if(crawlerQuestionMapper.selectOne(condition) == null) { if(crawlerQuestionMapper.selectOne(condition) == null) {
CrawlerQuestionEntity crawlerQuestion = new CrawlerQuestionEntity(); CrawlerQuestionEntity crawlerQuestion = new CrawlerQuestionEntity();
crawlerQuestion.setQuestionUrl(questionUrl); crawlerQuestion.setQuestionUrl(questionUrl);
crawlerQuestion.setChapterId(chapterEntity.getUid()); crawlerQuestion.setChapterId(chapterEntity.getId());
crawlerQuestion.setStatus("0"); crawlerQuestion.setStatus("0");
crawlerQuestionMapper.insert(crawlerQuestion); crawlerQuestionMapper.insert(crawlerQuestion);
}else { }else {

2
tamguo-crawler/src/main/resources/mappers/ChapterMapper.xml
Unescape View File

@ -3,7 +3,7 @@
<mapper namespace="com.tamguo.dao.ChapterMapper"> <mapper namespace="com.tamguo.dao.ChapterMapper">
<select id="queryList" resultType="ChapterEntity"> <select id="queryList" resultType="ChapterEntity">
select * from tiku_chapter order by uid desc select * from tiku_chapter order by id desc
</select> </select>
<select id="queryCount" resultType="java.lang.Integer"> <select id="queryCount" resultType="java.lang.Integer">

6
tamguo-crawler/src/test/java/com/tamguo/SingleQuestionCrawler.java
Unescape View File

@ -81,16 +81,16 @@ public class SingleQuestionCrawler {
question.setAnalysis(questionVo.getAnalysis()); question.setAnalysis(questionVo.getAnalysis());
question.setAnswer(questionVo.getAnswer()); question.setAnswer(questionVo.getAnswer());
question.setAuditStatus("1"); question.setAuditStatus("1");
question.setChapterId(chapter.getUid()); question.setChapterId(chapter.getId());
question.setContent(questionVo.getContent()); question.setContent(questionVo.getContent());
question.setCourseId(course.getUid()); question.setCourseId(course.getId());
question.setPaperId(null); question.setPaperId(null);
question.setQuestionType("1"); question.setQuestionType("1");
if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) { if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
question.setReviewPoint(StringUtils.join(questionVo.getReviewPoint().toArray(), ",")); question.setReviewPoint(StringUtils.join(questionVo.getReviewPoint().toArray(), ","));
} }
question.setScore(questionVo.getScore()); question.setScore(questionVo.getScore());
question.setSubjectId(subject.getUid()); question.setSubjectId(subject.getId());
question.setYear(questionVo.getYear()); question.setYear(questionVo.getYear());
if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) { if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {

Write Preview
Loading…
Cancel
Save