爬数据

tamguo-crawler/src/main/java/com/tamguo/model/vo/CourseVo.java

@@ -0,0 +1,22 @@
+package com.tamguo.model.vo;
+
+import java.util.List;
+
+import com.xuxueli.crawler.annotation.PageFieldSelect;
+import com.xuxueli.crawler.annotation.PageSelect;
+
+@PageSelect(cssQuery = "body")
+public class CourseVo {
+
+	@PageFieldSelect(cssQuery = ".course-item")
+	private List<String> name;
+
+	public List<String> getName() {
+		return name;
+	}
+
+	public void setName(List<String> name) {
+		this.name = name;
+	}
+	
+}

tamguo-crawler/src/main/java/com/tamguo/model/vo/SubjectVo.java

@@ -10,7 +10,10 @@ public class SubjectVo {
 
 	@PageFieldSelect(cssQuery = ".all-list-li")
 	private List<String> name;
-
+	
+	@PageFieldSelect(cssQuery=".course-list-container .course-list .course-item")
+	private List<String> courseName;
+	
 	public List<String> getName() {
 		return name;
 	}
@@ -19,5 +22,4 @@ public class SubjectVo {
 		this.name = name;
 	}
 
-	
 }

tamguo-crawler/src/main/java/com/tamguo/service/ICourseService.java

@@ -1,10 +0,0 @@
-package com.tamguo.service;
-
-public interface ICourseService {
-
-	/**
-	 * 爬取科目数据
-	 */
-	void crawlerCourse();
-	
-}

tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java

@@ -2,6 +2,7 @@ package com.tamguo.service.impl;
 
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -11,6 +12,7 @@ import com.tamguo.model.vo.SubjectVo;
 import com.tamguo.service.ISubjectService;
 import com.xuxueli.crawler.XxlCrawler;
 import com.xuxueli.crawler.parser.PageParser;
+import com.xuxueli.crawler.rundata.RunData;
 
 @Service
 public class SubjectService implements ISubjectService{
@@ -18,12 +20,15 @@ public class SubjectService implements ISubjectService{
 	@Autowired
 	SubjectMapper subjectMapper;
 	
+	private RunData runData;
+	
 	@Override
 	public void crawlerSubject() {
 		XxlCrawler crawler = new XxlCrawler.Builder()
             .setUrls("https://tiku.baidu.com/")
             .setWhiteUrlRegexs("https://tiku\\.baidu\\.com/")
             .setPageParser(new PageParser<SubjectVo>() {
+            	
                 @Override
                 public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
                     // 解析封装 PageVo 对象
@@ -40,11 +45,24 @@ public class SubjectService implements ISubjectService{
                     	SubjectEntity entity = new SubjectEntity();
                     	entity.setName(name);
                     	subjectMapper.insert(entity);
+                    	
+                    	// 获取Course
+                    	Elements elements = pageVoElement.getElementsByClass("all-list-li");
+                    	for(int k=0 ; k<elements.size() ; k++) {
+                    		Element element = elements.get(k);
+                    	 	String url = element.child(0).attr("href");
+                    	 	runData.addUrl(url);
+                    	}
                     }
                 }
-            })
+                
-            .build();
+        }).build();
+		
+		runData = crawler.getRunData();
+		
+		
 		
+		// 获取科目
 		crawler.start(true);
 	}