爬蟲

tamguo-common/src/main/java/com/tamguo/common/utils/SystemConstant.java

@@ -79,4 +79,7 @@ public class SystemConstant {
 	
 	/** ALIYUN*/
 	public static final String ALIYUN_ACCESS_KEY_SECRET = "ONUKuCz85kU4In07y4dvpM28mfWOGa";
+	
+	/** 默认的章节根目录*/
+	public static final String CHAPTER_DEFAULT_ROOT_UID = "-1";
 }

tamguo-crawler/src/main/java/com/tamguo/config/dao/SuperEntity.java

@@ -11,7 +11,7 @@ public class SuperEntity<T extends Model<?>> extends Model<T> {
 
 	private static final long serialVersionUID = 1L;
 
-    @TableId("uid")
+    @TableId("id")
     private String uid;
 
     @Override

tamguo-crawler/src/main/java/com/tamguo/model/ChapterEntity.java

@@ -19,6 +19,14 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
 
 	private String parentId;
 	
+	private String parentIds;
+	
+	private Integer treeLevel;
+	
+	private Boolean treeLeaf;
+	
+	private String bookId;
+	
 	private Integer questionNum;
 	
 	private Integer pointNum;
@@ -76,4 +84,36 @@ public class ChapterEntity extends SuperEntity<ChapterEntity> implements Seriali
 		this.orders = orders;
 	}
 
+	public String getBookId() {
+		return bookId;
+	}
+
+	public void setBookId(String bookId) {
+		this.bookId = bookId;
+	}
+
+	public String getParentIds() {
+		return parentIds;
+	}
+
+	public void setParentIds(String parentIds) {
+		this.parentIds = parentIds;
+	}
+
+	public Integer getTreeLevel() {
+		return treeLevel;
+	}
+
+	public void setTreeLevel(Integer treeLevel) {
+		this.treeLevel = treeLevel;
+	}
+
+	public Boolean getTreeLeaf() {
+		return treeLeaf;
+	}
+
+	public void setTreeLeaf(Boolean treeLeaf) {
+		this.treeLeaf = treeLeaf;
+	}
+
 }

tamguo-crawler/src/main/java/com/tamguo/model/vo/SubjectVo.java

@@ -47,6 +47,9 @@ public class SubjectVo {
 	@PageFieldSelect(cssQuery = ".bd-content .question-box .question-box-inner .view-analyse a", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:href")
 	private List<String> questionUrls;
 	
+	@PageFieldSelect(cssQuery = ".nexttolearn .next-inner .tolearn", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:href")
+	private String nextQuestionPage;
+	
 	// 单个题目数据
 	@PageFieldSelect(cssQuery=".question-box-inner .questem-inner", selectType = XxlCrawlerConf.SelectType.HTML)
 	private String content;
@@ -183,4 +186,12 @@ public class SubjectVo {
 		this.answer = answer;
 	}
 
+	public String getNextQuestionPage() {
+		return nextQuestionPage;
+	}
+
+	public void setNextQuestionPage(String nextQuestionPage) {
+		this.nextQuestionPage = nextQuestionPage;
+	}
+
 }

tamguo-crawler/src/main/java/com/tamguo/service/impl/BookService.java

@@ -1,72 +1,222 @@
 package com.tamguo.service.impl;
 
+import com.baomidou.mybatisplus.plugins.Page;
+import com.tamguo.config.redis.CacheService;
 import com.tamguo.dao.ChapterMapper;
+import com.tamguo.dao.CourseMapper;
+import com.tamguo.dao.CrawlerQuestionMapper;
+import com.tamguo.dao.QuestionMapper;
+import com.tamguo.dao.SubjectMapper;
 import com.tamguo.model.ChapterEntity;
-import com.tamguo.model.vo.ChapterVo;
+import com.tamguo.model.CourseEntity;
+import com.tamguo.model.CrawlerQuestionEntity;
+import com.tamguo.model.QuestionEntity;
+import com.tamguo.model.SubjectEntity;
+import com.tamguo.model.enums.QuestionType;
+import com.tamguo.model.vo.QuestionVo;
 import com.tamguo.service.IBookService;
 import com.xuxueli.crawler.XxlCrawler;
+import com.xuxueli.crawler.conf.XxlCrawlerConf;
 import com.xuxueli.crawler.parser.PageParser;
+import com.xuxueli.crawler.parser.strategy.HtmlUnitPageLoader;
+import com.xuxueli.crawler.rundata.RunData;
+import com.xuxueli.crawler.util.FileUtil;
+
+import java.io.File;
+import java.text.DecimalFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.StringUtils;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.List;
-import java.util.UUID;
-
 @Service
 public class BookService implements IBookService {
 
-
+	private RunData runData;
+	@Autowired
+	QuestionMapper questionMapper;
+	@Autowired
+	CrawlerQuestionMapper crawlerQuestionMapper;
 	@Autowired
 	ChapterMapper chapterMapper;
-
-    private Logger logger = LoggerFactory.getLogger(getClass());
+	@Autowired
+	CourseMapper courseMapper;
+	@Autowired
+	SubjectMapper subjectMapper;
+	@Autowired
+	CacheService cacheService;
+	private static final String FILES_NO_FORMAT = "0000000";
+	private static final String FILES_PREFIX = "LIKESHUXUE";
+	private static final String DOMAIN = "http://www.tamguo.com";
 
 
     @Override
     public void crawlerBook() {
 		XxlCrawler crawler = new XxlCrawler.Builder()
-                .setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-27-jiaocai-11")
 	            .setAllowSpread(false)
-                .setFailRetryCount(5)
 	            .setThreadCount(20)
-                .setPageParser(new PageParser<ChapterVo>() {
+	            .setFailRetryCount(5)
+	            .setPageLoader(new HtmlUnitPageLoader())
+	            .setPageParser(new PageParser<QuestionVo>() {
+	            	
 	                @Override
-                    public void parse(Document html, Element pageVoElement, ChapterVo chapterVo) {
-                        // 解析封装 PageVo 对象
-                        String parentName = chapterVo.getName();
-                        ChapterEntity chapterEntity = new ChapterEntity();
-                        String uid = UUID.randomUUID().toString().replace("-", "");
-                        chapterEntity.setUid(uid);
-                        chapterEntity.setName(parentName);
-                        chapterEntity.setCourseId("0");
-                        chapterEntity.setCourseId("0");
-                        chapterEntity.setParentId("-1");
-                        chapterEntity.setQuestionNum(0);
-                        chapterEntity.setPointNum(0);
-                        chapterMapper.insert(chapterEntity);
-
-                        List<String> sonChapters = chapterVo.getSonChapters();
-                        sonChapters.forEach(s -> {
-                            ChapterEntity sonChapterEntity = new ChapterEntity();
-                            sonChapterEntity.setName(s);
-                            sonChapterEntity.setCourseId("0");
-                            sonChapterEntity.setCourseId("0");
-                            sonChapterEntity.setParentId(uid);
-                            sonChapterEntity.setQuestionNum(0);
-                            sonChapterEntity.setPointNum(0);
-                            chapterMapper.insert(sonChapterEntity);
-                        });
-
-                    }
-
-
-//                    }
+	                public void parse(Document html, Element pageVoElement, QuestionVo questionVo) {
+	                	if(StringUtils.isEmpty(questionVo.getContent())) {
+	                		runData.addUrl(html.baseUri());
+	                		return;
+	                	}
+                		CrawlerQuestionEntity condition = new CrawlerQuestionEntity();
+	                	condition.setQuestionUrl(html.baseUri());
+	                	CrawlerQuestionEntity crawlerQuestion = crawlerQuestionMapper.selectOne(condition);
+	                	ChapterEntity chapter = chapterMapper.selectById(crawlerQuestion.getChapterId());
+	                	CourseEntity course = courseMapper.selectById(chapter.getCourseId());
+	                	SubjectEntity subject = subjectMapper.selectById(course.getSubjectId());
+	                	
+	                	QuestionType questionType = QuestionType.getQuestionType(questionVo.getQuestionType());
+	                	
+
+	                	QuestionEntity question = new QuestionEntity();
+	                	if(questionType == QuestionType.DANXUANTI) {
+	                		if(!StringUtils.isEmpty(questionVo.getQueoptions())) {
+	                			question.setContent(questionVo.getContent() + questionVo.getQueoptions());	
+	                		}else {
+	                			question.setContent(questionVo.getContent());
+	                		}
+	                	}else {
+	                		question.setContent(questionVo.getContent());
+	                	}
+	                	question.setAnalysis(questionVo.getAnalysis());
+	                	if(StringUtils.isEmpty(question.getAnswer())) {
+	                		question.setAnalysis("<p> <span> 略 </span> <br> </p>");
+	                	}
+	                	question.setAnswer(questionVo.getAnswer());
+	                	question.setAuditStatus("1");
+	                	question.setChapterId(chapter.getUid());
+	                	question.setCourseId(course.getUid());
+	                	question.setPaperId(null);
+	                	question.setQuestionType(questionType.getValue().toString());
+	                	if(questionVo.getReviewPoint() != null && questionVo.getReviewPoint().size() > 0) {
+		                	question.setReviewPoint(StringUtils.join(questionVo.getReviewPoint().toArray(), ","));
+	                	}
+	                	// 处理分数
+	                	if(questionVo.getScore() != null) {
+	                		if(questionVo.getScore().contains("分")) {
+		                		question.setScore(questionVo.getScore());
+		                	}
+		                	if(questionVo.getScore().contains("年")) {
+		                		question.setYear(questionVo.getScore());
+		                	}
+	                	}
+	                	if(questionVo.getYear() != null) {
+	                		if(questionVo.getYear().contains("年")) {
+	                			question.setYear(questionVo.getYear());
+	                		}
+	                	}
+	                	question.setSubjectId(subject.getUid());
+	                	
+	                	if (questionVo.getAnswerImages()!=null && questionVo.getAnswerImages().size() > 0) {
+                            Set<String> imagesSet = new HashSet<>(questionVo.getAnswerImages());
+                            for (String img: imagesSet) {
+
+                                // 下载图片文件
+                            	String fileName = getFileName(img);
+                                File dir = new File(getFilePath());
+                				if (!dir.exists())
+                					dir.mkdirs();
+                                boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
+                                System.out.println("down images " + (ret?"success":"fail") + "：" + img);
+                                
+                                // 替换URL
+                                questionVo.setAnswer(questionVo.getAnswer().replace(img, DOMAIN + getFilePath() + fileName));
+                            }
+                            question.setAnswer(questionVo.getAnswer());
+                        }
+	                	
+	                	if (questionVo.getAnalysisImages()!=null && questionVo.getAnalysisImages().size() > 0) {
+                            Set<String> imagesSet = new HashSet<>(questionVo.getAnalysisImages());
+                            for (String img: imagesSet) {
+
+                                // 下载图片文件
+                                String fileName = getFileName(img);
+                                File dir = new File(getFilePath());
+                				if (!dir.exists())
+                					dir.mkdirs();
+                                boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
+                                System.out.println("down images " + (ret?"success":"fail") + "：" + img);
+                                
+                                // 替换URL
+                                questionVo.setAnalysis(questionVo.getAnalysis().replace(img, DOMAIN + getFilePath() + fileName));
+                            }
+                            question.setAnalysis(questionVo.getAnalysis());
+                        }
+	                	
+	                	if (questionVo.getContentImages()!=null && questionVo.getContentImages().size() > 0) {
+                            Set<String> imagesSet = new HashSet<>(questionVo.getContentImages());
+                            for (String img: imagesSet) {
+
+                                // 下载图片文件
+                                String fileName = getFileName(img);
+                                File dir = new File(getFilePath());
+                				if (!dir.exists())
+                					dir.mkdirs();
+                                boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
+                                System.out.println("down images " + (ret?"success":"fail") + "：" + img);
+                                
+                                // 替换URL
+                                questionVo.setContent(questionVo.getContent().replace(img, DOMAIN + getFilePath() + fileName));
+                            }
+                            question.setContent(questionVo.getContent());
+                        }
+	                	
+	                	
+	                	// 处理图片
+	                	question.setSourceType("baidu");
+	                	question.setSourceUrl(html.baseUri());
+	                	questionMapper.insert(question);
+	                }
+	                
+	                public String getFileName(String img) {
+	            		return getFileNo() + img.substring(img.lastIndexOf("."));
+	            	}
+	                
+	                private String getFilePath() {
+	                	SimpleDateFormat sdf = new SimpleDateFormat("ddHHmm");
+	            		String format = sdf.format(new Date());
+	            		return "/images/question/" + format + "/";
+	                }
+
+	            	private String getFileNo() {
+	            		SimpleDateFormat sdf = new SimpleDateFormat("ddHHmm");
+	            		String format = sdf.format(new Date());
+	            		DecimalFormat df = new DecimalFormat(FILES_NO_FORMAT);
+	            		String key = FILES_PREFIX + format;
+	            		Long incr = cacheService.incr(key);
+	            		String avatorNo = FILES_PREFIX + df.format(incr);
+	            		return avatorNo;
+	            	}
 	        }).build();
 			
+			runData = crawler.getRunData();
+			int page = 1;
+			int pageSize = 1000;
+			while(true) {
+				Page<CrawlerQuestionEntity> questionPage = new Page<CrawlerQuestionEntity>(page , pageSize);
+				List<CrawlerQuestionEntity> questionList = crawlerQuestionMapper.queryPageOrderUid(questionPage);
+				for(int i=0 ;i<questionList.size() ; i++) {
+					runData.addUrl(questionList.get(i).getQuestionUrl());
+				}
+				page++;
+				if(questionList.size() < 100) {
+					break;
+				}
+			}
 			// 获取科目
 			crawler.start(true);
 	}

tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java

@@ -5,6 +5,7 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.StringUtils;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
@@ -19,9 +20,7 @@ import com.tamguo.dao.CourseMapper;
 import com.tamguo.dao.CrawlerQuestionMapper;
 import com.tamguo.dao.SubjectMapper;
 import com.tamguo.model.ChapterEntity;
-import com.tamguo.model.CourseEntity;
 import com.tamguo.model.CrawlerQuestionEntity;
-import com.tamguo.model.SubjectEntity;
 import com.tamguo.model.vo.SubjectVo;
 import com.tamguo.service.ISubjectService;
 import com.xuxueli.crawler.XxlCrawler;
@@ -31,6 +30,8 @@ import com.xuxueli.crawler.rundata.RunData;
 @Service
 public class SubjectService implements ISubjectService{
 	
+	private final static String COURSE_ID = "likeshuxue";
+	private final static String BOOK_ID = "1025976567395184642";
 	@Autowired
 	SubjectMapper subjectMapper;
 	@Autowired
@@ -42,8 +43,6 @@ public class SubjectService implements ISubjectService{
 	
 	private Logger logger = LoggerFactory.getLogger(getClass());
 	
-	private Set<String> urls = new HashSet<>();
-	
 	private Set<String> questionUrls = new HashSet<String>();
 	
 	private Map<String, Object> chapterQuestionListMap = new HashMap<>();
@@ -53,7 +52,7 @@ public class SubjectService implements ISubjectService{
 	@Override
 	public void crawlerSubject() {
 		XxlCrawler crawler = new XxlCrawler.Builder()
-            .setUrls("https://tiku.baidu.com/")
+            .setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-16-knowpoint-11")
             .setAllowSpread(false)
             .setFailRetryCount(5)
             .setThreadCount(20)
@@ -63,68 +62,8 @@ public class SubjectService implements ISubjectService{
                 public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
                     // 解析封装 PageVo 对象
                     String pageUrl = html.baseUri();
-                    if(pageUrl.equals("https://tiku.baidu.com/")) {
-                    	logger.info("开始解析考试分类：{}" , pageUrl);
-                    	for(int i=0 ; i<subjectVo.getName().size() ; i++) {
-                        	String name = subjectVo.getName().get(i);
-                        		
-                        	SubjectEntity subject = subjectMapper.findByName(name);
-                        	if(subject != null) {
-                        		continue;
-                        	}
-                        	SubjectEntity entity = new SubjectEntity();
-                        	if(name.equals("国考")) {
-                        		name = "公务员（国考）";
-                        	}
-                        	entity.setName(name);
-                        	subjectMapper.insert(entity);
-                        }
-                    	
-                	 	// 加入科目爬取数据
-                    	for(String url : subjectVo.getCourseUrls()) {
-                    		runData.addUrl(url);
-                    	}
-                    }
-                    if(pageUrl.contains("https://tiku.baidu.com/tikupc/homepage/")) {
-                    	logger.info("开始解析科目分类：{}" , pageUrl);
-                    	for(int i=0 ; i<subjectVo.getCourseName().size() ; i++) {
-                        	logger.info("科目名称:{}" , subjectVo.getCourseName().get(i));
-                        	SubjectEntity subject = subjectMapper.findByName(subjectVo.getSubjectName());
-                        	if(subject == null) {
-                        		continue;
-                        	}
-                        	CourseEntity course = new CourseEntity();
-                        	course.setName(subjectVo.getCourseName().get(i));
-                        	CourseEntity courseEntity = courseMapper.selectOne(course);
-                        	if(courseEntity != null) {
-                        		continue;
-                        	}
-                        	course.setOrders(i+1);
-                        	course.setPointNum(0);
-                        	course.setQuestionNum(0);
-                        	course.setSeoDescription(subjectVo.getCourseName().get(i));
-                        	course.setSeoKeywords(subjectVo.getCourseName().get(i));
-                        	course.setSeoTitle(subjectVo.getCourseName().get(i));
-                        	course.setSubjectId(subject.getUid());
-                        	
-                        	courseMapper.insert(course);
-                        }
-                    	
-                    	// 加入科目爬取数据
-                    	for(String url : subjectVo.getChapterUrlsTemp()) {
-                    		runData.addUrl(url);
-                    	}
-                    }
-                    
                     if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/")) {
-                    	logger.info("开始解析章节：{}" , pageUrl);
-                    	CourseEntity courseCondition = new CourseEntity();
-                    	courseCondition.setName(subjectVo.getChapterPageCourseName());
-                    	CourseEntity c = courseMapper.selectOne(courseCondition);
-                    	if(c == null) {
-                    		runData.addUrl(pageUrl);
-                    		return;
-                    	}
+                    	logger.info("开始解析书籍：{}" , pageUrl);
                     	ChapterEntity chapterCondition = new ChapterEntity();
                     	chapterCondition.setName(subjectVo.getChapterCurrName());
                     	ChapterEntity chapterEntity = chapterMapper.selectOne(chapterCondition);
@@ -133,12 +72,16 @@ public class SubjectService implements ISubjectService{
                     	}
                     	
                     	ChapterEntity rootChapter = new ChapterEntity();
-                    	rootChapter.setCourseId(c.getUid());
+                    	rootChapter.setCourseId(COURSE_ID);
                     	rootChapter.setParentId("-1");
                     	rootChapter.setName(subjectVo.getChapterCurrName());
                     	rootChapter.setQuestionNum(0);
                     	rootChapter.setPointNum(0);
                     	rootChapter.setOrders(0);
+                    	rootChapter.setTreeLeaf(false);
+                    	rootChapter.setTreeLevel(0);
+                    	rootChapter.setParentIds("-1,");
+                    	rootChapter.setBookId(BOOK_ID);
                 		chapterMapper.insert(rootChapter);
                 		
                     	Elements elements = pageVoElement.getElementsByClass("detail-chapter");
@@ -147,13 +90,18 @@ public class SubjectService implements ISubjectService{
                     		String chapterName = element.getElementsByClass("detail-chapter-title").get(0).getElementsByTag("h3").text();
                     		logger.info(chapterName);
                     		ChapterEntity chapter = new ChapterEntity();
-                    		chapter.setCourseId(c.getUid());
+                    		chapter.setCourseId(COURSE_ID);
                     		chapter.setParentId(rootChapter.getUid());
                     		chapter.setName(chapterName);
                     		chapter.setQuestionNum(0);
                     		chapter.setPointNum(0);
                     		chapter.setOrders(n+1);
+                    		chapter.setBookId(BOOK_ID);
+                    		chapter.setTreeLeaf(false);
+                    		chapter.setTreeLevel(1);
                     		chapterMapper.insert(chapter);
+                    		chapter.setParentIds(rootChapter.getParentIds() + chapter.getUid() + ",");
+                    		chapterMapper.updateById(chapter);
                     		
                     		Elements detailKpoint1s = element.getElementsByClass("detail-kpoint-1");
                     		for(Element detailKpoint1 : detailKpoint1s) {
@@ -164,13 +112,19 @@ public class SubjectService implements ISubjectService{
                         			logger.info(chapterName1);
                         			
                         			ChapterEntity chapter1 = new ChapterEntity();
-                        			chapter1.setCourseId(c.getUid());
+                        			chapter1.setCourseId(COURSE_ID);
+                        			chapter1.setBookId(BOOK_ID);
                         			chapter1.setParentId(chapter.getUid());
                         			chapter1.setName(chapterName1);
                         			chapter1.setQuestionNum(0);
                         			chapter1.setPointNum(0);
                         			chapter1.setOrders(i+1);
+                        			
+                        			chapter1.setTreeLeaf(false);
+                        			chapter1.setTreeLevel(2);
                             		chapterMapper.insert(chapter1);
+                        			chapter1.setParentIds(chapter.getParentIds() + chapter1.getUid() + ",");
+                        			chapterMapper.updateById(chapter1);
                         			
                         			
                         			Elements detailKpoint2s =  detailKpoint1.getElementsByClass("detail-kpoint-2");
@@ -180,17 +134,23 @@ public class SubjectService implements ISubjectService{
                         				logger.info(chapterName2);
                         				
                         				ChapterEntity chapter2 = new ChapterEntity();
-                        				chapter2.setCourseId(c.getUid());
+                        				chapter2.setCourseId(COURSE_ID);
+                        				chapter2.setBookId(BOOK_ID);
                         				chapter2.setParentId(chapter1.getUid());
                         				chapter2.setName(chapterName2);
                         				chapter2.setQuestionNum(0);
                         				chapter2.setPointNum(0);
                         				chapter2.setOrders(k+1);
+                        				chapter2.setTreeLeaf(true);
+                        				chapter2.setTreeLevel(3);
                                 		chapterMapper.insert(chapter2);
+                        				chapter2.setParentIds(chapter1.getParentIds() + chapter2.getUid() + ",");
+                                		chapterMapper.updateById(chapter2);
+                                		
                                 		Elements maskList = detailKpoint.getElementsByClass("mask");
                                 		if(maskList.size() > 0) {
                                 			String questionUrl = maskList.get(0).getElementsByTag("a").attr("abs:href");
-                                    		questionUrl = questionUrl.replace("1-5", "1-1000");
+                                    	//	questionUrl = questionUrl.replace("1-5", "1-20");
                                     		chapterQuestionListMap.put(questionUrl, chapter2);
                                     		
                                     		runData.addUrl(questionUrl);
@@ -200,34 +160,19 @@ public class SubjectService implements ISubjectService{
                     		}
                     	}
                     	
-                    	// 剔除已经爬取的数据
-                    	urls.add(pageUrl);
-                    	logger.info("url:{}" ,pageUrl );
-                    	logger.info("subjectVo:{}" ,JSONObject.toJSON(subjectVo) );
-                    	if(subjectVo.getChapterUrls() != null) {
-                    		for(String url : subjectVo.getChapterUrls()) {
-                        		if(url.equals(pageVoElement.getElementsByClass("main-inner").get(0).getElementsByClass("selected").get(0).getElementsByTag("a").attr("abs:href"))) {
-                        			continue;
-                        		}
-                        		if(!urls.contains(url)) {
-                            		runData.addUrl(url);
-                        		}
-                        	}
-                    	}
                     }
                     
                     if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterdetail")) {
                     	// 加入待解析题目列表
                     	logger.info("url : {}" , pageUrl);
                     	logger.info("subjectVo : {}" , JSONObject.toJSON(subjectVo));
+                		ChapterEntity chapterEntity = (ChapterEntity) chapterQuestionListMap.get(pageUrl);
                         for(String questionUrl : subjectVo.getQuestionUrls()) {
                         	if(!questionUrls.contains(questionUrl)) {
                         		//  处理URL 
                         	//	runData.addUrl(questionUrl);
                         		questionUrls.add(questionUrl);
                         		
-                        		ChapterEntity chapterEntity = (ChapterEntity) chapterQuestionListMap.get(pageUrl);
-                        		
                         		CrawlerQuestionEntity condition = new CrawlerQuestionEntity();
                         		condition.setQuestionUrl(questionUrl);
                         		if(crawlerQuestionMapper.selectOne(condition) == null) {
@@ -241,12 +186,12 @@ public class SubjectService implements ISubjectService{
                         		}
                         	}
                         }
+                        if(!StringUtils.isEmpty(subjectVo.getNextQuestionPage())) {
+                        	runData.addUrl(subjectVo.getNextQuestionPage());
+                        	chapterQuestionListMap.put(subjectVo.getNextQuestionPage(), chapterEntity);
+                        }
                     }
                     
-                    /*if(pageUrl.contains("https://tiku.baidu.com/tikupc/singledetail")) {
-                    	ChapterEntity chapterEntity = (ChapterEntity) chapterQuestionMap.get(pageUrl);
-                    	System.out.println(chapterEntity);
-                    }*/
                 }
         }).build();
 		

tamguo-crawler/src/main/resources/application.properties

@@ -15,7 +15,7 @@ spring.datasource.testOnReturn=false
 spring.datasource.testWhileIdle=true
 spring.datasource.timeBetweenEvictionRunsMillis=60000
 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
-spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tiku?useUnicode=true&characterEncoding=UTF-8&useSSL=false
+spring.datasource.url=jdbc:mysql://47.100.175.14:3306/tamguo?useUnicode=true&characterEncoding=UTF-8&useSSL=false
 spring.datasource.username=root
 spring.datasource.validationQuery=SELECT 1 FROM DUAL
 

tamguo-crawler/src/main/resources/redis.xml

@@ -4,6 +4,6 @@
 	<pool maxActive="50" maxIdle="20" maxWait="1000" />
 	<servers>
 		<!-- test -->
-		<server ip="47.100.175.14" port="6379"/>
+		<server ip="127.0.0.1" port="6379"/>
 	</servers>
 </redis>

tamguo-modules-core/src/main/java/com/tamguo/modules/tiku/service/IChapterService.java

@@ -1,8 +1,13 @@
 package com.tamguo.modules.tiku.service;
 
+import java.util.List;
+
 import com.baomidou.mybatisplus.service.IService;
 import com.tamguo.modules.tiku.model.ChapterEntity;
 
 public interface IChapterService extends IService<ChapterEntity>{
 	
+	// 获取科目章节
+	public List<ChapterEntity> findChapterTree(String bookId);
+
 }

tamguo-modules-core/src/main/java/com/tamguo/modules/tiku/service/impl/ChapterServiceImpl.java

@@ -1,7 +1,15 @@
 package com.tamguo.modules.tiku.service.impl;
 
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import com.baomidou.mybatisplus.mapper.Condition;
 import com.baomidou.mybatisplus.service.impl.ServiceImpl;
+import com.tamguo.common.utils.SystemConstant;
 import com.tamguo.modules.tiku.dao.ChapterMapper;
 import com.tamguo.modules.tiku.model.ChapterEntity;
 import com.tamguo.modules.tiku.service.IChapterService;
@@ -9,4 +17,54 @@ import com.tamguo.modules.tiku.service.IChapterService;
 @Service
 public class ChapterServiceImpl extends ServiceImpl<ChapterMapper, ChapterEntity> implements IChapterService{
 
+	@Transactional(readOnly=false)
+	@SuppressWarnings("unchecked")
+	@Override
+	public List<ChapterEntity> findChapterTree(String bookId) {
+		List<ChapterEntity> chapterList = baseMapper.selectList(Condition.create().eq("book_id", bookId));
+		
+		// 获取根chapter UID
+		String rootUid = StringUtils.EMPTY;
+		for(int i=0 ; i<chapterList.size() ; i++){
+			ChapterEntity chapter = chapterList.get(i);
+			if(chapter.getParentId().equals(SystemConstant.CHAPTER_DEFAULT_ROOT_UID)){
+				rootUid = chapter.getId();
+			}
+		}
+		// 获取第一层结构
+		List<ChapterEntity> entitys = new ArrayList<>();
+		for(int i=0 ; i<chapterList.size() ; i++){
+			ChapterEntity chapter = chapterList.get(i);
+			if(rootUid.equals(chapter.getParentId())){
+				entitys.add(chapter);
+			}
+		}
+		for(int i=0 ; i<entitys.size() ; i++){
+			ChapterEntity entity = entitys.get(i);
+			List<ChapterEntity> childs = new ArrayList<>();
+			for(int k=0 ; k<chapterList.size() ; k++){
+				ChapterEntity chapter = chapterList.get(k);
+				if(entity.getId().equals(chapter.getParentId())){
+					childs.add(chapter);
+				}
+			}
+			entity.setChildChapterList(childs);
+		}
+		for(int i=0 ; i<entitys.size() ; i++){
+			List<ChapterEntity> childs = entitys.get(i).getChildChapterList();
+			for(int k=0 ; k<childs.size() ; k++){
+				ChapterEntity child = childs.get(k);
+				List<ChapterEntity> tmpChilds = new ArrayList<>();
+				for(int n=0 ; n<chapterList.size() ; n++){
+					ChapterEntity chapter = chapterList.get(n);
+					if(child.getId().equals(chapter.getParentId())){
+						tmpChilds.add(chapter);
+					}
+				}
+				child.setChildChapterList(tmpChilds);
+			}
+		}
+		return entitys;
+	}
+	
 }

tamguo-tms/src/main/java/com/tamguo/web/tiku/CourseController.java

@@ -45,7 +45,7 @@ public class CourseController {
 			BookEntity book = null;
 			if(bookList.size() > 0) {
 				book = bookList.get(0);
-				chapterList = iChapterService.selectList(Condition.create().eq("book_id", book.getId()));
+				chapterList = iChapterService.findChapterTree(book.getId());
 			}
 			SubjectEntity subject = iSubjectService.selectById(course.getSubjectId());
 			List<CourseEntity> courseList = iCourseService.selectList(Condition.create().eq("subject_id", course.getSubjectId()));