|
|
@ -31,7 +31,7 @@ import com.xuxueli.crawler.rundata.RunData;
|
|
|
|
public class SubjectService implements ISubjectService{
|
|
|
|
public class SubjectService implements ISubjectService{
|
|
|
|
|
|
|
|
|
|
|
|
private final static String COURSE_ID = "wuli";
|
|
|
|
private final static String COURSE_ID = "wuli";
|
|
|
|
private final static String BOOK_ID = "1060742388004642890";
|
|
|
|
private final static String BOOK_ID = "1060742388004642891";
|
|
|
|
private final static String SUBJECT_ID = "gaokao";
|
|
|
|
private final static String SUBJECT_ID = "gaokao";
|
|
|
|
@Autowired
|
|
|
|
@Autowired
|
|
|
|
SubjectMapper subjectMapper;
|
|
|
|
SubjectMapper subjectMapper;
|
|
|
@ -53,7 +53,7 @@ public class SubjectService implements ISubjectService{
|
|
|
|
@Override
|
|
|
|
@Override
|
|
|
|
public void crawlerSubject() {
|
|
|
|
public void crawlerSubject() {
|
|
|
|
XxlCrawler crawler = new XxlCrawler.Builder()
|
|
|
|
XxlCrawler crawler = new XxlCrawler.Builder()
|
|
|
|
.setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-18-jiaocai-11")
|
|
|
|
.setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-18-jiaocai-12")
|
|
|
|
.setAllowSpread(false)
|
|
|
|
.setAllowSpread(false)
|
|
|
|
.setFailRetryCount(5)
|
|
|
|
.setFailRetryCount(5)
|
|
|
|
.setThreadCount(20)
|
|
|
|
.setThreadCount(20)
|
|
|
@ -63,7 +63,7 @@ public class SubjectService implements ISubjectService{
|
|
|
|
public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
|
|
|
|
public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
|
|
|
|
// 解析封装 PageVo 对象
|
|
|
|
// 解析封装 PageVo 对象
|
|
|
|
String pageUrl = html.baseUri();
|
|
|
|
String pageUrl = html.baseUri();
|
|
|
|
if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-18-jiaocai-11")) {
|
|
|
|
if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-18-jiaocai-12")) {
|
|
|
|
logger.info("开始解析书籍:{}" , pageUrl);
|
|
|
|
logger.info("开始解析书籍:{}" , pageUrl);
|
|
|
|
ChapterEntity chapterCondition = new ChapterEntity();
|
|
|
|
ChapterEntity chapterCondition = new ChapterEntity();
|
|
|
|
chapterCondition.setName(subjectVo.getChapterCurrName());
|
|
|
|
chapterCondition.setName(subjectVo.getChapterCurrName());
|
|
|
|