请求地址优化

tamguo-crawler/src/test/java/com/tamguo/PaperCrawler.java

@@ -30,14 +30,14 @@ public class PaperCrawler {
 	private final String SUBJECT_ID = "gaokao";
 	// 科目
 	private final String COURSE_ID = "likeshuxue";
-	// 110000 北京 | 310000 上海
-	private final String AREA_ID = "310000";
+	// 110000 北京 | 310000 上海 | 500000 重庆 | 120100 天津
+	private final String AREA_ID = "120100";
 	// 年份
-	private final String YEAR = "2016";
+	private final String YEAR = "2013";
 	// 真题试卷             类型(1:真题试卷,2:模拟试卷,3:押题预测,4:名校精品)
-	private final String PAPER_TYPE = "4";
+	private final String PAPER_TYPE = "2";
 	// 开始采集的URL
-	private final String START_URL = "https://tiku.baidu.com/tikupc/paperlist/1bfd700abb68a98271fefa04-16-7-2016-93-1-download";
+	private final String START_URL = "https://tiku.baidu.com/tikupc/paperlist/1bfd700abb68a98271fefa04-16-4-2013-211-1-download";
 	
 	private RunData runData;
 	

tamguo-tms/src/main/java/com/tamguo/web/tiku/BookController.java

@@ -2,6 +2,11 @@ package com.tamguo.web.tiku;
 
 import java.util.Arrays;
 import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -22,6 +27,7 @@ import com.tamguo.modules.tiku.service.ISubjectService;
 @Controller
 public class BookController {
 
+	private Logger logger = LoggerFactory.getLogger(getClass());
 	@Autowired
 	IBookService iBookService;
 	@Autowired
@@ -33,8 +39,10 @@ public class BookController {
 	
 	@SuppressWarnings("unchecked")
 	@RequestMapping(value = {"book/{uid}.html"}, method = RequestMethod.GET)
-	public ModelAndView index(@PathVariable String uid , ModelAndView model) {
+	public ModelAndView index(@PathVariable String uid , ModelAndView model , HttpServletRequest request) {
 		try {
+			// request url 
+    		logger.info("request url :{}" , request.getRequestURI());
 			BookEntity book = iBookService.selectById(uid);
 			SubjectEntity subject = iSubjectService.selectById(book.getSubjectId());
 			List<CourseEntity> courseList = iCourseService.selectList(Condition.create().eq("subject_id", subject.getId()).orderAsc(Arrays.asList("sort")));

tamguo-tms/src/main/java/com/tamguo/web/tiku/CourseController.java

@@ -5,6 +5,8 @@ import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -31,6 +33,7 @@ import com.tamguo.modules.tiku.service.ISubjectService;
 @Controller
 public class CourseController {
 	
+	private Logger logger = LoggerFactory.getLogger(getClass());
 	@Autowired
 	IChapterService iChapterService;
 	@Autowired
@@ -44,6 +47,8 @@ public class CourseController {
 	@RequestMapping(value = {"course/{uid}.html"}, method = RequestMethod.GET)
 	public ModelAndView index(HttpServletRequest request , @PathVariable String uid , ModelAndView model) {
 		try {
+    		// request url 
+    		logger.info("request url :{}" , request.getRequestURI());
 			CourseEntity course = iCourseService.selectById(uid);
 			List<BookEntity> bookList = iBookService.selectList(Condition.create().eq("course_id", uid));
 			List<ChapterEntity> chapterList = null;

tamguo-tms/src/main/java/com/tamguo/web/tiku/IndexController.java

@@ -1,5 +1,9 @@
 package com.tamguo.web.tiku;
 
+import javax.servlet.http.HttpServletRequest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -8,14 +12,20 @@ import org.springframework.web.servlet.ModelAndView;
 @Controller
 public class IndexController {
 	
+	private Logger logger = LoggerFactory.getLogger(getClass());
+	
 	@RequestMapping(value = "/", method = RequestMethod.GET)
-    public ModelAndView indexAction(ModelAndView model) {
+    public ModelAndView indexAction(ModelAndView model , HttpServletRequest request) {
+    	// request url 
+		logger.info("request url :{}" , request.getRequestURI());
     	model.setViewName("index");
         return model;
     }
 	
     @RequestMapping(value = "/index", method = RequestMethod.GET)
-    public ModelAndView mainAction(ModelAndView model) {
+    public ModelAndView mainAction(ModelAndView model , HttpServletRequest request) {
+    	// request url 
+		logger.info("request url :{}" , request.getRequestURI());
     	model.setViewName("index");
         return model;
     }

tamguo-tms/src/main/java/com/tamguo/web/tiku/PaperController.java

@@ -5,6 +5,8 @@ import java.util.List;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -36,6 +38,8 @@ import com.tamguo.utils.PageUtils;
 @Controller
 public class PaperController {
 	
+	private Logger logger = LoggerFactory.getLogger(getClass());
+	
 	@Autowired
 	private ICourseService iCourseService;
 	@Autowired
@@ -52,6 +56,8 @@ public class PaperController {
     public ModelAndView indexAction(HttpServletRequest request , @PathVariable String subjectId , @PathVariable String courseId , @PathVariable String paperType,
     		@PathVariable String year , @PathVariable String area , @PathVariable Integer pageNum, ModelAndView model) {
     	try {
+    		// request url 
+    		logger.info("request url :{}" , request.getRequestURI());
         	CourseEntity course = iCourseService.selectById(courseId);
 			List<CourseEntity> courseList = iCourseService.selectList(Condition.create().eq("subject_id", subjectId));
         	SubjectEntity subject = iSubjectService.selectById(subjectId);
@@ -105,6 +111,8 @@ public class PaperController {
 	@RequestMapping(value = {"/paper/{paperId}.html"}, method = RequestMethod.GET)
 	public ModelAndView indexAction(HttpServletRequest request , @PathVariable String paperId , ModelAndView model){
 		try {
+    		// request url 
+    		logger.info("request url :{}" , request.getRequestURI());
 			model.setViewName("paper");
 			PaperEntity paper = iPaperService.selectById(paperId);
 			model.addObject("paper", paper);

tamguo-tms/src/main/java/com/tamguo/web/tiku/QuestionContrller.java

@@ -2,6 +2,10 @@ package com.tamguo.web.tiku;
 
 import java.util.Arrays;
 
+import javax.servlet.http.HttpServletRequest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -25,6 +29,8 @@ import com.tamguo.modules.tiku.service.ISubjectService;
 @Controller
 public class QuestionContrller {
 	
+	private Logger logger = LoggerFactory.getLogger(getClass());
+
 	@Autowired
 	private IQuestionService iQuestionService;
 	@Autowired
@@ -37,8 +43,10 @@ public class QuestionContrller {
 	@SuppressWarnings("unchecked")
 	@RequestMapping(value = {"questionlist/{chapterId}-{current}-{size}.html"}, method = RequestMethod.GET)
 	public ModelAndView questionList(@PathVariable String chapterId , @PathVariable Integer current , 
-			@PathVariable Integer size , ModelAndView model){
+			@PathVariable Integer size , ModelAndView model , HttpServletRequest request){
 		try {
+			// request url 
+			logger.info("request url :{}" , request.getRequestURI());
 			model.setViewName("questionList");
 
 			ChapterEntity chapter = iChapterService.selectById(chapterId);

tamguo-tms/src/main/java/com/tamguo/web/tiku/SubjectController.java

@@ -55,6 +55,8 @@ public class SubjectController {
 	@RequestMapping(value = {"subject/{subjectId}.html"}, method = RequestMethod.GET)
     public ModelAndView indexAction(@PathVariable String subjectId , HttpServletRequest request , ModelAndView model) {
 		try {
+			// request url 
+			logger.info("request url :{}" , request.getRequestURI());
 			SubjectEntity subject = iSubjectService.selectById(subjectId);
 			List<CourseEntity> courseList = iCourseService.selectList(Condition.create().eq("subject_id", subjectId).orderAsc(Arrays.asList("sort")));
 			// 获取第一个科目