pbn32vpxk
/
tamguo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix bug

Browse Source
main
tamguo 7 years ago
parent d3a88157b5
commit 5ab44111fc
6 changed files with 43 additions and 59 deletions
Show Stats Download Patch File Download Diff File

44
tamguo-crawler/src/main/java/com/tamguo/model/CourseEntity.java
Unescape View File

@ -20,13 +20,8 @@ public class CourseEntity extends SuperEntity<CourseEntity> implements Serializa
private Integer questionNum;
private Integer orders;
private Integer sort;
private String seoTitle;
private String seoKeywords;
private String seoDescription;
public CourseEntity() {
}
@ -63,40 +58,15 @@ public class CourseEntity extends SuperEntity<CourseEntity> implements Serializa
this.pointNum = pointNum;
}
public Integer getOrders() {
return orders;
}
public void setOrders(Integer orders) {
this.orders = orders;
}
public String getSeoTitle() {
return seoTitle;
}
public void setSeoTitle(String seoTitle) {
this.seoTitle = seoTitle;
}
public String getSeoKeywords() {
return seoKeywords;
}
public void setSeoKeywords(String seoKeywords) {
this.seoKeywords = seoKeywords;
}
public String getSeoDescription() {
return seoDescription;
public static long getSerialversionuid() {
return serialVersionUID;
}
public void setSeoDescription(String seoDescription) {
this.seoDescription = seoDescription;
public Integer getSort() {
return sort;
}
public static long getSerialversionuid() {
return serialVersionUID;
public void setSort(Integer sort) {
this.sort = sort;
}
}

2
tamguo-crawler/src/main/java/com/tamguo/model/vo/QuestionVo.java
Unescape View File

@ -11,7 +11,7 @@ public class QuestionVo {
@PageFieldSelect(cssQuery=".question-box-inner .questem-inner", selectType = XxlCrawlerConf.SelectType.HTML)
private String content;
@PageFieldSelect(cssQuery = ".question-box-inner .questem-inner img", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:src")
@PageFieldSelect(cssQuery = ".question-box-inner img", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "abs:src")
private List<String> contentImages;
@PageFieldSelect(cssQuery=".que-options", selectType = XxlCrawlerConf.SelectType.HTML)

50
tamguo-crawler/src/main/java/com/tamguo/service/impl/QuestionService.java
Unescape View File

@ -51,8 +51,8 @@ public class QuestionService implements IQuestionService{
SubjectMapper subjectMapper;
@Autowired
CacheService cacheService;
private static final String FILES_NO_FORMAT = "000000";
private static final String FILES_PREFIX = "FPIMAGE";
private static final String FILES_NO_FORMAT = "000000000";
private static final String FILES_PREFIX = "likeshuxue";
private static final String DOMAIN = "http://www.tamguo.com";
private RunData runData;
@ -75,6 +75,7 @@ public class QuestionService implements IQuestionService{
}
CrawlerQuestionEntity condition = new CrawlerQuestionEntity();
condition.setQuestionUrl(html.baseUri());
System.out.println(html.baseUri());
CrawlerQuestionEntity crawlerQuestion = crawlerQuestionMapper.selectOne(condition);
ChapterEntity chapter = chapterMapper.selectById(crawlerQuestion.getChapterId());
CourseEntity course = courseMapper.selectById(chapter.getCourseId());
@ -94,7 +95,7 @@ public class QuestionService implements IQuestionService{
question.setContent(questionVo.getContent());
}
question.setAnalysis(questionVo.getAnalysis());
if(StringUtils.isEmpty(question.getAnswer())) {
if(StringUtils.isEmpty(question.getAnalysis())) {
question.setAnalysis("<p> <span> 略 </span> <br> </p>");
}
question.setAnswer(questionVo.getAnswer());
@ -128,16 +129,19 @@ public class QuestionService implements IQuestionService{
// 下载图片文件
String fileName = getFileName(img);
File dir = new File(getFilePath());
String filePath = getFilePath();
String fileDatePath = getFileDatePath();
File dir = new File(filePath +fileDatePath+ "/");
if (!dir.exists())
dir.mkdirs();
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, filePath +fileDatePath+ "/", fileName);
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
questionVo.setAnswer(questionVo.getAnswer().replace(img, DOMAIN + getFilePath() + fileName));
question.setAnswer(question.getAnswer().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
}
question.setAnswer(questionVo.getAnswer());
question.setAnswer(question.getAnswer());
}
if (questionVo.getAnalysisImages()!=null && questionVo.getAnalysisImages().size() > 0) {
@ -146,16 +150,19 @@ public class QuestionService implements IQuestionService{
// 下载图片文件
String fileName = getFileName(img);
File dir = new File(getFilePath());
String filePath = getFilePath();
String fileDatePath = getFileDatePath();
File dir = new File(filePath +fileDatePath+ "/");
if (!dir.exists())
dir.mkdirs();
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, filePath +fileDatePath+ "/", fileName);
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
questionVo.setAnalysis(questionVo.getAnalysis().replace(img, DOMAIN + getFilePath() + fileName));
question.setAnalysis(question.getAnalysis().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
}
question.setAnalysis(questionVo.getAnalysis());
question.setAnalysis(question.getAnalysis());
}
if (questionVo.getContentImages()!=null && questionVo.getContentImages().size() > 0) {
@ -164,16 +171,19 @@ public class QuestionService implements IQuestionService{
// 下载图片文件
String fileName = getFileName(img);
File dir = new File(getFilePath());
if (!dir.exists())
String filePath = getFilePath();
String fileDatePath = getFileDatePath();
File dir = new File(filePath +fileDatePath+ "/");
if (!dir.exists()) {
dir.mkdirs();
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, getFilePath(), fileName);
}
boolean ret = FileUtil.downFile(img, XxlCrawlerConf.TIMEOUT_MILLIS_DEFAULT, filePath +fileDatePath+ "/", fileName);
System.out.println("down images " + (ret?"success":"fail") + "：" + img);
// 替换URL
questionVo.setContent(questionVo.getContent().replace(img, DOMAIN + getFilePath() + fileName));
question.setContent(question.getContent().replace(img, DOMAIN + "/files/question/" + fileDatePath + "/" + fileName));
}
question.setContent(questionVo.getContent());
question.setContent(question.getContent());
}
@ -188,9 +198,13 @@ public class QuestionService implements IQuestionService{
}
private String getFilePath() {
return "/home/webdata/files/question/";
}
private String getFileDatePath() {
SimpleDateFormat sdf = new SimpleDateFormat("ddHHmm");
String format = sdf.format(new Date());
return "/images/question/" + format + "/";
return format;
}
private String getFileNo() {
@ -214,7 +228,7 @@ public class QuestionService implements IQuestionService{
runData.addUrl(questionList.get(i).getQuestionUrl());
}
page++;
if(questionList.size() < 100) {
if(questionList.size() < 1000) {
break;
}
}

2
tamguo-crawler/src/main/resources/application.properties
Unescape View File

@ -1,4 +1,4 @@
domain.name=http://www.tamguo.com/
domain.name=http://localhost:8081/
spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.filters=stat,wall,log4j

2
tamguo-crawler/src/main/resources/mappers/CrawlerQuestionMapper.xml
Unescape View File

@ -3,6 +3,6 @@
<mapper namespace="com.tamguo.dao.CrawlerQuestionMapper">
<select id="queryPageOrderUid" resultType="CrawlerQuestionEntity">
select * from crawler_question order by uid desc
select * from crawler_question order by id desc
</select>
</mapper>

2
tamguo-tms/src/main/java/com/tamguo/web/tiku/QuestionContrller.java
Unescape View File

@ -81,7 +81,7 @@ public class QuestionContrller {
model.addObject("question", question);
// 推荐试题
model.addObject("featuredQuestionList", iQuestionService.selectList(Condition.create().eq("subject_id", question.getSubjectId()).eq("course_id", question.getCourseId())));
model.addObject("featuredQuestionList", iQuestionService.selectPage(new Page<>(1, 3), Condition.create().eq("subject_id", question.getSubjectId()).eq("course_id", question.getCourseId())).getRecords());
return model;
} catch (Exception e) {
model.setViewName("404");

Write Preview
Loading…
Cancel
Save