diff --git a/tamguo-admin/pom.xml b/tamguo-admin/pom.xml index 585dcc1..35c6488 100644 --- a/tamguo-admin/pom.xml +++ b/tamguo-admin/pom.xml @@ -142,11 +142,6 @@ commons-email 1.5 - - com.xuxueli - xxl-crawler - 1.2.1 - diff --git a/tamguo-admin/src/main/java/com/tamguo/admin/crawler/XxlCrawlerTask.java b/tamguo-admin/src/main/java/com/tamguo/admin/crawler/XxlCrawlerTask.java deleted file mode 100644 index 3248dbe..0000000 --- a/tamguo-admin/src/main/java/com/tamguo/admin/crawler/XxlCrawlerTask.java +++ /dev/null @@ -1,245 +0,0 @@ -package com.tamguo.admin.crawler; - -import com.tamguo.admin.model.QuestionEntity; -import com.tamguo.admin.service.IQuestionService; -import com.xuxueli.crawler.XxlCrawler; -import com.xuxueli.crawler.annotation.PageFieldSelect; -import com.xuxueli.crawler.annotation.PageSelect; -import com.xuxueli.crawler.parser.PageParser; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.EnableScheduling; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import java.math.BigInteger; - -/** - * 爬虫示例01:爬取页面数据并封装VO对象 - * - * @author xuxueli 2017-10-09 19:48:48 - */ -@Component -@EnableScheduling -public class XxlCrawlerTask { - Logger logger = LoggerFactory.getLogger(this.getClass()); - - - @Autowired - private IQuestionService iQuestionService; - - - @PageSelect(cssQuery = ".que-multi") - public static class PageVo { - - @PageFieldSelect(cssQuery = ".que-multi") - private String question; - - - public String getQuestion() { - return question; - } - - public void setQuestion(String question) { - this.question = question; - } - - - - @Override - public String toString() { - return "PageVo{" + - "question='" + question + '\'' + - '}'; - } - } - - @PageSelect(cssQuery = ".question-box-inner") - public static class PageLiZongVo { - - @PageFieldSelect(cssQuery = ".question-box-inner") - private String question; - - - public String getQuestion() { - return question; - } - - public void setQuestion(String question) { - this.question = question; - } - - - - @Override - public String toString() { - return "PageLiZongVo{" + - "question='" + question + '\'' + - '}'; - } - } - - @PageSelect(cssQuery = ".question-box-inner") - public static class PageWenZongVo { - - @PageFieldSelect(cssQuery = ".question-box-inner") - private String question; - - - public String getQuestion() { - return question; - } - - public void setQuestion(String question) { - this.question = question; - } - - - - @Override - public String toString() { - return "PageWenZongVo{" + - "question='" + question + '\'' + - '}'; - } - } - - /** - * @description 2018北京语文 - * @author sh00859 - * @date 2018/6/29 - */ -// @Scheduled(cron = "0 0/1 * * * ?") - public void crawlerYWData() { - XxlCrawler crawler = new XxlCrawler.Builder() - .setUrls("https://tiku.baidu.com/tikupc/paperdetail/4baa90f5f61fb7360b4c656b") - .setWhiteUrlRegexs("https://tiku\\.baidu\\.com/tikupc/paperdetail/4baa90f5f61fb7360b4c656b") - .setPageParser(new PageParser() { - @Override - public void parse(Document html, Element pageVoElement, PageVo pageVo) { - // 解析封装 PageVo 对象 - String pageUrl = html.baseUri(); - System.out.println(pageUrl + ":" + pageVo.toString()); - QuestionEntity question = new QuestionEntity(); - question.setChapterId(new BigInteger("1")); - question.setCourseId("1012550050327625730"); - question.setPaperId(new BigInteger("1012550408013676545")); - question.setContent(pageVo.getQuestion() == null ? "无" : pageVo.getQuestion());//问题 -// question.setAnswer(pageVo.getAnswer() == null ? "无" : pageVo.getAnswer());//回答 - question.setAnswer("无");//回答 - question.setAnalysis("暂无解释"); - question.setQuestionType("5"); - question.setReviewPoint("语文"); - question.setSubjectId("13"); - question.setScore(10); - question.setYear("2018"); - try { - iQuestionService.save(question); - } catch (Exception e) { - logger.error("错误信息[{}]", e); - } - - } - }) - .build(); - - System.out.println("start"); - crawler.start(true); - System.out.println("end"); - } - - - /** - * @description 2018北京理综 - * @author sh00859 - * @date 2018/6/29 - */ -// @Scheduled(cron = "0 0 19 * * ?") - public void crawlerSXData() { - XxlCrawler crawler = new XxlCrawler.Builder() - .setUrls("https://tiku.baidu.com/tikupc/paperdetail/de62bec66137ee06eff91868") - .setWhiteUrlRegexs("https://tiku\\.baidu\\.com/tikupc/paperdetail/de62bec66137ee06eff91868") - .setPageParser(new PageParser() { - @Override - public void parse(Document html, Element pageVoElement, PageLiZongVo pageVo) { - // 解析封装 PageVo 对象 - String pageUrl = html.baseUri(); - System.out.println(pageUrl + ":" + pageVo.toString()); - QuestionEntity question = new QuestionEntity(); - question.setChapterId(new BigInteger("1")); - question.setCourseId("1012652550204428289"); - question.setPaperId(new BigInteger("1012652716789600257")); - question.setContent(pageVo.getQuestion() == null ? "无" : pageVo.getQuestion());//问题 -// question.setAnswer(pageVo.getAnswer() == null ? "无" : pageVo.getAnswer());//回答 - question.setAnswer("无");//回答 - question.setAnalysis("暂无解释"); - question.setQuestionType("5"); - question.setReviewPoint("理综"); - question.setSubjectId("13"); - question.setScore(10); - question.setYear("2018"); - try { - iQuestionService.save(question); - } catch (Exception e) { - logger.error("错误信息[{}]", e); - } - - } - }) - .build(); - - System.out.println("start"); - crawler.start(true); - System.out.println("end"); - } - - - /** - * @description 2018北京文综 - * @author sh00859 - * @date 2018/6/29 - */ - @Scheduled(cron = "0 27 19 * * ?") - public void crawlerWZData() { - XxlCrawler crawler = new XxlCrawler.Builder() - .setUrls("https://tiku.baidu.com/tikupc/paperdetail/acf80b22bcd126fff7050b72") - .setWhiteUrlRegexs("https://tiku\\.baidu\\.com/tikupc/paperdetail/acf80b22bcd126fff7050b72") - .setPageParser(new PageParser() { - @Override - public void parse(Document html, Element pageVoElement, PageWenZongVo pageVo) { - // 解析封装 PageVo 对象 - String pageUrl = html.baseUri(); - System.out.println(pageUrl + ":" + pageVo.toString()); - QuestionEntity question = new QuestionEntity(); - question.setChapterId(new BigInteger("1")); - question.setCourseId("1012658027151851521"); - question.setPaperId(new BigInteger("1012658169615581186")); - question.setContent(pageVo.getQuestion() == null ? "无" : pageVo.getQuestion());//问题 -// question.setAnswer(pageVo.getAnswer() == null ? "无" : pageVo.getAnswer());//回答 - question.setAnswer("无");//回答 - question.setAnalysis("暂无解释"); - question.setQuestionType("5"); - question.setReviewPoint("理综"); - question.setSubjectId("13"); - question.setScore(10); - question.setYear("2018"); - try { - iQuestionService.save(question); - } catch (Exception e) { - logger.error("错误信息[{}]", e); - } - - } - }) - .build(); - - System.out.println("start"); - crawler.start(true); - System.out.println("end"); - } - - -} diff --git a/tamguo-admin/src/main/java/com/tamguo/admin/model/CourseEntity.java b/tamguo-admin/src/main/java/com/tamguo/admin/model/CourseEntity.java index 10ac294..0403b1f 100644 --- a/tamguo-admin/src/main/java/com/tamguo/admin/model/CourseEntity.java +++ b/tamguo-admin/src/main/java/com/tamguo/admin/model/CourseEntity.java @@ -25,8 +25,6 @@ public class CourseEntity extends SuperEntity implements Serializa private BigInteger questionNum; - private String icon; - private Integer orders; private String seoTitle; @@ -76,14 +74,6 @@ public class CourseEntity extends SuperEntity implements Serializa this.pointNum = pointNum; } - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - public Integer getOrders() { return orders; } diff --git a/tamguo-admin/src/main/resources/mappers/CourseMapper.xml b/tamguo-admin/src/main/resources/mappers/CourseMapper.xml index b99a568..945435b 100644 --- a/tamguo-admin/src/main/resources/mappers/CourseMapper.xml +++ b/tamguo-admin/src/main/resources/mappers/CourseMapper.xml @@ -8,7 +8,6 @@ - @@ -24,7 +23,6 @@ c.point_num, c.question_num, c.subject_id, - c.icon FROM tiku_course c WHERE @@ -34,7 +32,6 @@