pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bug

Browse Source
main
tamguo 7 years ago
parent a48491397c
commit f14923fb78
7 changed files with 35 additions and 90 deletions
Show Stats Download Patch File Download Diff File

2
tamguo-crawler/src/main/java/com/tamguo/service/impl/BookService.java
Unescape View File

@ -53,7 +53,7 @@ public class BookService implements IBookService {
@Autowired @Autowired
CacheService cacheService; CacheService cacheService;
private static final String FILES_NO_FORMAT = "0000000"; private static final String FILES_NO_FORMAT = "0000000";
private static final String FILES_PREFIX = "LIKESHUXUE"; private static final String FILES_PREFIX = "LINCHZHIYEYISHI";
private static final String DOMAIN = "http://www.tamguo.com"; private static final String DOMAIN = "http://www.tamguo.com";

11
tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java
Unescape View File

@ -53,9 +53,8 @@ public class QuestionService extends ServiceImpl<QuestionMapper, QuestionEntity>
@Autowired @Autowired
CacheService cacheService; CacheService cacheService;
private static final String FILES_NO_FORMAT = "000000000"; private static final String FILES_NO_FORMAT = "000000000";
private static final String FILES_PREFIX = "zz"; private static final String FILES_PREFIX = "kyzz";
private static final String COURSE_ID = "zhengzhi"; private static final String COURSE_ID = "kaoyanzhengzhi";
private static final String DOMAIN = "http://www.tamguo.com";
private RunData runData; private RunData runData;
@ -141,7 +140,7 @@ public class QuestionService extends ServiceImpl<QuestionMapper, QuestionEntity>
System.out.println("down images " + (ret?"success":"fail") + "：" + img); System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL // 替换URL
question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); question.setAnswer(question.getAnswer().replace(img, "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
} }
question.setAnswer(question.getAnswer()); question.setAnswer(question.getAnswer());
} }
@ -162,7 +161,7 @@ public class QuestionService extends ServiceImpl<QuestionMapper, QuestionEntity>
System.out.println("down images " + (ret?"success":"fail") + "：" + img); System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL // 替换URL
question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); question.setAnalysis(question.getAnalysis().replace(img, "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
} }
question.setAnalysis(question.getAnalysis()); question.setAnalysis(question.getAnalysis());
} }
@ -183,7 +182,7 @@ public class QuestionService extends ServiceImpl<QuestionMapper, QuestionEntity>
System.out.println("down images " + (ret?"success":"fail") + "：" + img); System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL // 替换URL
question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName)); question.setContent(question.getContent().replace(img, "/files/question/" + COURSE_ID + '/' + fileDatePath + "/" + fileName));
} }
question.setContent(question.getContent()); question.setContent(question.getContent());
} }

10
tamguo-crawler/src/main/java/com/tamguo/service/impl/SubjectService.java
Unescape View File

@ -30,9 +30,9 @@ import com.xuxueli.crawler.rundata.RunData;
@Service @Service
public class SubjectService implements ISubjectService{ public class SubjectService implements ISubjectService{
private final static String COURSE_ID = "zhengzhi"; private final static String COURSE_ID = "kaoyanzhengzhi";
private final static String BOOK_ID = "1025976567395184649"; private final static String BOOK_ID = "1025976567395184651";
private final static String SUBJECT_ID = "gaokao"; private final static String SUBJECT_ID = "kaoyan";
@Autowired @Autowired
SubjectMapper subjectMapper; SubjectMapper subjectMapper;
@Autowired @Autowired
@ -53,7 +53,7 @@ public class SubjectService implements ISubjectService{
@Override @Override
public void crawlerSubject() { public void crawlerSubject() {
XxlCrawler crawler = new XxlCrawler.Builder() XxlCrawler crawler = new XxlCrawler.Builder()
.setUrls("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-26-knowpoint-11") .setUrls("https://tiku.baidu.com/tikupc/chapterlist/f3eb19e8b8f67c1cfad6b805-342-knowpoint-11")
.setAllowSpread(false) .setAllowSpread(false)
.setFailRetryCount(5) .setFailRetryCount(5)
.setThreadCount(20) .setThreadCount(20)
@ -63,7 +63,7 @@ public class SubjectService implements ISubjectService{
public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) { public void parse(Document html, Element pageVoElement, SubjectVo subjectVo) {
// 解析封装 PageVo 对象 // 解析封装 PageVo 对象
String pageUrl = html.baseUri(); String pageUrl = html.baseUri();
if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/1bfd700abb68a98271fefa04-26-knowpoint-11")) { if(pageUrl.contains("https://tiku.baidu.com/tikupc/chapterlist/f3eb19e8b8f67c1cfad6b805-342-knowpoint-11")) {
logger.info("开始解析书籍：{}" , pageUrl); logger.info("开始解析书籍：{}" , pageUrl);
ChapterEntity chapterCondition = new ChapterEntity(); ChapterEntity chapterCondition = new ChapterEntity();
chapterCondition.setName(subjectVo.getChapterCurrName()); chapterCondition.setName(subjectVo.getChapterCurrName());

12
tamguo-crawler/src/test/java/com/tamguo/PaperCrawler.java
Unescape View File

@ -29,18 +29,18 @@ public class PaperCrawler {
// 高考 // 高考
private final String SUBJECT_ID = "gaokao"; private final String SUBJECT_ID = "gaokao";
// 科目 // 科目
private final String COURSE_ID = "wenkeshuxue"; private final String COURSE_ID = "wuli";
// 110000 北京 | 310000 上海 | 500000 重庆 | 120100 天津 | 370000 山东 | 410000 河南 | 420000 湖北 | 320000 江苏 | 330000 浙江 // 110000 北京 | 310000 上海 | 500000 重庆 | 120000 天津 | 370000 山东 | 410000 河南 | 420000 湖北 | 320000 江苏 | 330000 浙江
// 140000 山西 | 350000 福建 | 340000 安徽 | 220000 吉林 | 150000 内蒙古 | 640000 宁夏 | 650000 新疆 | 广西 450000 | 210000 辽宁 // 140000 山西 | 350000 福建 | 340000 安徽 | 220000 吉林 | 150000 内蒙古 | 640000 宁夏 | 650000 新疆 | 广西 450000 | 210000 辽宁
// 230000 黑龙江 | 610000 陕西 | 360000 江西 | 440000 广东 | 430000 湖南 | 460000 海南 | 530000 云南 | 510000 四川 | 630000 青海 // 230000 黑龙江 | 610000 陕西 | 360000 江西 | 440000 广东 | 430000 湖南 | 460000 海南 | 530000 云南 | 510000 四川 | 630000 青海
// 620000 甘肃 | 130000 河北 | 540000 西藏 | 贵州 520000 // 620000 甘肃 | 130000 河北 | 540000 西藏 | 贵州 520000
private final String AREA_ID = "140000"; private final String AREA_ID = "350000";
// 年份 // 年份
private final String YEAR = "2015"; private final String YEAR = "2016";
// 真题试卷 类型(1:真题试卷,2:模拟试卷,3:押题预测,4:名校精品) // 真题试卷 类型(1:真题试卷,2:模拟试卷,3:押题预测,4:名校精品)
private final String PAPER_TYPE = "3"; private final String PAPER_TYPE = "2";
// 开始采集的URL // 开始采集的URL
private final String START_URL = "https://tiku.baidu.com/tikupc/paperlist/1bfd700abb68a98271fefa04-17-2-2015-741-1-download"; private final String START_URL = "https://tiku.baidu.com/tikupc/paperlist/1bfd700abb68a98271fefa04-18-4-2016-799-1-download";
private RunData runData; private RunData runData;

4
tamguo-crawler/src/test/java/com/tamguo/PaperQuestionCrawler.java
Unescape View File

@ -63,8 +63,8 @@ public class PaperQuestionCrawler {
@Autowired @Autowired
CacheService cacheService; CacheService cacheService;
private static final String FILES_NO_FORMAT = "000000000"; private static final String FILES_NO_FORMAT = "000000000";
private static final String FILES_PREFIX = "wenkeshuxue"; private static final String FILES_PREFIX = "wuli";
private static final String COURSE_ID = "wenkeshuxue"; private static final String COURSE_ID = "wuli";
private RunData runData; private RunData runData;

8
tamguo-tms/src/main/java/com/tamguo/web/tiku/SubjectController.java
Unescape View File

@ -17,15 +17,18 @@ import org.springframework.web.servlet.ModelAndView;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.baomidou.mybatisplus.mapper.Condition; import com.baomidou.mybatisplus.mapper.Condition;
import com.baomidou.mybatisplus.plugins.Page;
import com.tamguo.common.utils.Result; import com.tamguo.common.utils.Result;
import com.tamguo.modules.sys.service.ISysAreaService; import com.tamguo.modules.sys.service.ISysAreaService;
import com.tamguo.modules.tiku.model.BookEntity; import com.tamguo.modules.tiku.model.BookEntity;
import com.tamguo.modules.tiku.model.ChapterEntity; import com.tamguo.modules.tiku.model.ChapterEntity;
import com.tamguo.modules.tiku.model.CourseEntity; import com.tamguo.modules.tiku.model.CourseEntity;
import com.tamguo.modules.tiku.model.PaperEntity;
import com.tamguo.modules.tiku.model.SubjectEntity; import com.tamguo.modules.tiku.model.SubjectEntity;
import com.tamguo.modules.tiku.service.IBookService; import com.tamguo.modules.tiku.service.IBookService;
import com.tamguo.modules.tiku.service.IChapterService; import com.tamguo.modules.tiku.service.IChapterService;
import com.tamguo.modules.tiku.service.ICourseService; import com.tamguo.modules.tiku.service.ICourseService;
import com.tamguo.modules.tiku.service.IPaperService;
import com.tamguo.modules.tiku.service.ISubjectService; import com.tamguo.modules.tiku.service.ISubjectService;
import com.tamguo.utils.BrowserUtils; import com.tamguo.utils.BrowserUtils;
@ -50,6 +53,8 @@ public class SubjectController {
private ICourseService iCourseService; private ICourseService iCourseService;
@Autowired @Autowired
private IBookService iBookService; private IBookService iBookService;
@Autowired
private IPaperService iPaperService;
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@RequestMapping(value = {"subject/{subjectId}.html"}, method = RequestMethod.GET) @RequestMapping(value = {"subject/{subjectId}.html"}, method = RequestMethod.GET)
@ -67,11 +72,14 @@ public class SubjectController {
BookEntity book = bookList.get(0); BookEntity book = bookList.get(0);
chapterList = iChapterService.selectList(Condition.create().eq("book_id", book.getId())); chapterList = iChapterService.selectList(Condition.create().eq("book_id", book.getId()));
} }
// 获取最新的试卷
Page<PaperEntity> paperPage = iPaperService.selectPage(new Page<>(1, 15) , Condition.create().eq("subject_id", subjectId).orderDesc(Arrays.asList("id")));
model.addObject("subject", subject); model.addObject("subject", subject);
model.addObject("course" , course); model.addObject("course" , course);
model.addObject("courseList", courseList); model.addObject("courseList", courseList);
model.addObject("chapterList" , chapterList); model.addObject("chapterList" , chapterList);
model.addObject("areaList", iSysAreaService.selectList(Condition.create().eq("tree_level", "0"))); model.addObject("areaList", iSysAreaService.selectList(Condition.create().eq("tree_level", "0")));
model.addObject("paperList", paperPage.getRecords());
if(BrowserUtils.isMobile(request.getHeader("user-agent"))) { if(BrowserUtils.isMobile(request.getHeader("user-agent"))) {
model.setViewName("mobile/subject"); model.setViewName("mobile/subject");
}else { }else {

78
tamguo-tms/src/main/resources/templates/mobile/subject.html
Unescape View File

@ -45,6 +45,10 @@
<!--科目导航--> <!--科目导航-->
<div class="mod_box channel_nav"> <div class="mod_box channel_nav">
<h3 class="mod_tit">
<a class="post_ulog_search lazyload_ulog_action"
th:href="${domainName + 'paperlist/'+subject.id+'-0-1-0-0-1.html'}" th:text="${subject.name + '科目'}">考研科目</a>
</h3>
<div class="gridbox col_4"> <div class="gridbox col_4">
<h2 class="box_col" th:title="${c.name}" th:each="c,cStat:${courseList}"> <h2 class="box_col" th:title="${c.name}" th:each="c,cStat:${courseList}">
<a th:href="${domainName + 'course/' + c.id + '.html'}" class="inner post_ulog"> <a th:href="${domainName + 'course/' + c.id + '.html'}" class="inner post_ulog">
@ -56,18 +60,15 @@
</div> </div>
<!--/频道导航--> <!--/频道导航-->
<!--真題列表--> <!--最新试卷列表-->
<div class="mod_box house_lists"> <div class="mod_box house_lists">
<h3 class="mod_tit arrow"> <h3 class="mod_tit arrow">
<a class="post_ulog_search lazyload_ulog_action" <a class="post_ulog_search lazyload_ulog_action"
data-ulog_search="ljweb_id=30001&amp;ljweb_mod=m_pages_homepage_recommend-ershoufang&amp;ljweb_bl=more&amp;ljweb_strategy_id=default" th:href="${domainName + 'paperlist/'+subject.id+'-0-1-0-0-1.html'}">为您推荐最新试卷<span class="gray right">更多</span></a>
data-evtid_lazyaction="10889"
data-ulog_lazyaction="recommend_module=m_ershou_shouye&amp;fb_service_id=100001&amp;fb_context_id={&quot;req_source_type&quot;:&quot;zhiying_m&quot;}&amp;fb_expo_id=r87474238291230720&amp;fb_strategy_id=10000076&amp;fb_item_type=ershou_house&amp;fb_item_detail=107000203369&amp;fb_item_location=1&amp;fb_ab_test_flag=reco_rule-100001-default-default&amp;fb_response_ts=1535591483&amp;fb_query_id=188971149"
th:href="${domainName + 'paperlist/gaokao-0-1-0-0-1.html'}">为您推荐真题试卷<span class="gray right">更多</span></a>
</h3> </h3>
<div class="mod_cont"> <div class="mod_cont">
<ul class="lists"> <ul class="lists">
<li class="pictext" th:each="p,pStat:${historyPaperList}"> <li class="pictext arrow" th:each="p,pStat:${paperList}">
<a th:href="${domainName + 'paper/' + p.id + '.html'}" class="a_mask post_ulog_search lazyload_ulog_search post_ulog_action lazyload_ulog_action"></a> <a th:href="${domainName + 'paper/' + p.id + '.html'}" class="a_mask post_ulog_search lazyload_ulog_search post_ulog_action lazyload_ulog_action"></a>
<div class="flexbox"> <div class="flexbox">
<div class="item_list"> <div class="item_list">
@ -87,70 +88,7 @@
</div> </div>
</div> </div>
<!--/真題列表--> <!--/真題列表-->
<!--模拟列表-->
<div class="mod_box house_lists">
<h3 class="mod_tit arrow">
<a class="post_ulog_search lazyload_ulog_action"
data-ulog_search="ljweb_id=30001&amp;ljweb_mod=m_pages_homepage_recommend-ershoufang&amp;ljweb_bl=more&amp;ljweb_strategy_id=default"
data-evtid_lazyaction="10889"
data-ulog_lazyaction="recommend_module=m_ershou_shouye&amp;fb_service_id=100001&amp;fb_context_id={&quot;req_source_type&quot;:&quot;zhiying_m&quot;}&amp;fb_expo_id=r87474238291230720&amp;fb_strategy_id=10000076&amp;fb_item_type=ershou_house&amp;fb_item_detail=107000203369&amp;fb_item_location=1&amp;fb_ab_test_flag=reco_rule-100001-default-default&amp;fb_response_ts=1535591483&amp;fb_query_id=188971149"
th:href="${domainName + 'paperlist/gaokao-0-2-0-0-1.html'}">为您推荐模拟试卷<span class="gray right">更多</span></a>
</h3>
<div class="mod_cont">
<ul class="lists">
<li class="pictext" th:each="p,pStat:${simulationPaperList}">
<a th:href="${domainName + 'paper/' + p.id + '.html'}" class="a_mask post_ulog_search lazyload_ulog_search post_ulog_action lazyload_ulog_action"></a>
<div class="flexbox">
<div class="item_list">
<div class="item_main" th:text="${p.name}">理科数学 东城区2017年高三上学期期末考试</div>
<div class="tag_box">
<span class="tag" style="color: rgb(242, 161, 47); background-color: rgba(242, 161, 47, 0.15);"
th:title="${p.areaName}" th:text="${p.areaName}">北京</span> <span class="tag"
style="color: rgb(51, 190, 133); background-color: rgba(51, 190, 133, 0.15);"
title="真题">模拟</span> <span class="tag"
style="color: rgb(123, 189, 255); background-color: rgba(123, 189, 255, 0.15);"
th:title="${p.courseName}" th:text="${p.courseName}">理科数学</span>
</div>
</div>
</div>
</li>
</ul>
</div>
</div>
<!--/模拟列表-->
<!--热门列表-->
<div class="mod_box house_lists">
<h3 class="mod_tit arrow">
<a class="post_ulog_search lazyload_ulog_action"
data-ulog_search="ljweb_id=30001&amp;ljweb_mod=m_pages_homepage_recommend-ershoufang&amp;ljweb_bl=more&amp;ljweb_strategy_id=default"
data-evtid_lazyaction="10889"
data-ulog_lazyaction="recommend_module=m_ershou_shouye&amp;fb_service_id=100001&amp;fb_context_id={&quot;req_source_type&quot;:&quot;zhiying_m&quot;}&amp;fb_expo_id=r87474238291230720&amp;fb_strategy_id=10000076&amp;fb_item_type=ershou_house&amp;fb_item_detail=107000203369&amp;fb_item_location=1&amp;fb_ab_test_flag=reco_rule-100001-default-default&amp;fb_response_ts=1535591483&amp;fb_query_id=188971149"
th:href="${domainName + 'paperlist/gaokao-0-0-0-0-1.html'}">为您推荐热门试卷<span class="gray right">更多</span></a>
</h3>
<div class="mod_cont">
<ul class="lists">
<li class="pictext" th:each="p,pStat:${hotPaperList}">
<a th:href="${domainName + 'paper/' + p.id + '.html'}" class="a_mask post_ulog_search lazyload_ulog_search post_ulog_action lazyload_ulog_action"></a>
<div class="flexbox">
<div class="item_list">
<div class="item_main" th:text="${p.name}">理科数学 东城区2017年高三上学期期末考试</div>
<div class="tag_box">
<span class="tag" style="color: rgb(242, 161, 47); background-color: rgba(242, 161, 47, 0.15);"
th:title="${p.areaName}" th:text="${p.areaName}">北京</span> <span class="tag"
style="color: rgb(51, 190, 133); background-color: rgba(51, 190, 133, 0.15);"
title="热门" th:text>热门</span> <span class="tag"
style="color: rgb(123, 189, 255); background-color: rgba(123, 189, 255, 0.15);"
th:title="${p.courseName}" th:text="${p.courseName}">理科数学</span>
</div>
</div>
</div>
</li>
</ul>
</div>
</div>
<!--/模拟列表-->
</div> </div>
<!--底部:导航当前页用h1着重强调--> <!--底部:导航当前页用h1着重强调-->

Write Preview
Loading…
Cancel
Save