|
|
|
|
@ -19,7 +19,9 @@ const collectionNameMap = {
|
|
|
|
|
|
|
|
|
|
const PagePathMap = {
|
|
|
|
|
shixun:"markdown/shixun/shixun/shixun",
|
|
|
|
|
path:"markdown/path/path/path"
|
|
|
|
|
path:"markdown/path/path/path",
|
|
|
|
|
mooc_case:"markdown/mooc_case/mooc_case/mooc_case",
|
|
|
|
|
competition:"markdown/competition/competition/competition"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -40,75 +42,48 @@ exports.main = async (event, context) => {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function submitPages({Time}){
|
|
|
|
|
console.log("start submitPages");
|
|
|
|
|
async function submitPage({route, timeStart, timeEnd}){
|
|
|
|
|
const db = cloud.database();
|
|
|
|
|
const _ = db.command;
|
|
|
|
|
const pageHistoryCollection = db.collection(collectionNameMap.pageHistory);
|
|
|
|
|
|
|
|
|
|
let shixunPath = PagePathMap.shixun;
|
|
|
|
|
let pathPath = PagePathMap.path;
|
|
|
|
|
let timeEnd = new Date(Time);
|
|
|
|
|
let timeStart = new Date(timeEnd.getTime() - config.submitPagesGapDay*milliSecondsPerDay);
|
|
|
|
|
|
|
|
|
|
let shixunResult = await pageHistoryCollection.where({
|
|
|
|
|
page: shixunPath,
|
|
|
|
|
let data = await pageHistoryCollection.aggregate()
|
|
|
|
|
.match({
|
|
|
|
|
page: route,
|
|
|
|
|
status: 200,
|
|
|
|
|
isCrawl: false,
|
|
|
|
|
time: _.lt(timeEnd).and(_.gte(timeStart))
|
|
|
|
|
}).get();
|
|
|
|
|
})
|
|
|
|
|
.group({
|
|
|
|
|
_id:"$options"
|
|
|
|
|
})
|
|
|
|
|
.end();
|
|
|
|
|
|
|
|
|
|
if(data.list.length==0)
|
|
|
|
|
return "empty history: " + route;
|
|
|
|
|
let pages = data.list.map(i=>{
|
|
|
|
|
let query = Object.keys(i._id).map(k=>`${k}=${i._id[k]}`).join("&");
|
|
|
|
|
return {
|
|
|
|
|
path: route,
|
|
|
|
|
query
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
let res = await cloud.openapi.search.submitPages({pages});
|
|
|
|
|
console.log("submit pages success", route, pages.length, res, pages);
|
|
|
|
|
return "submit success: " + route + " " + pages.length;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let pathResult = await pageHistoryCollection.where({
|
|
|
|
|
page: pathPath,
|
|
|
|
|
status: 200,
|
|
|
|
|
isCrawl: false,
|
|
|
|
|
time:_.lt(timeEnd).and(_.gte(timeStart))
|
|
|
|
|
}).get();
|
|
|
|
|
|
|
|
|
|
let shixunQueries = shixunResult.data.reduce(function(arr, v){
|
|
|
|
|
var query = Object.keys(v.options).map(k=>`${k}=${v.options[k]}`).join("&");
|
|
|
|
|
if(arr.indexOf(query)==-1)
|
|
|
|
|
arr.push(query);
|
|
|
|
|
return arr
|
|
|
|
|
},[]);
|
|
|
|
|
|
|
|
|
|
let pathQueries = pathResult.data.reduce(function(arr, v){
|
|
|
|
|
var query = Object.keys(v.options).map(k=>`${k}=${v.options[k]}`).join("&");
|
|
|
|
|
if(arr.indexOf(query)==-1)
|
|
|
|
|
arr.push(query);
|
|
|
|
|
return arr
|
|
|
|
|
},[]);
|
|
|
|
|
|
|
|
|
|
console.info("shixunQueries", shixunQueries.length, shixunQueries);
|
|
|
|
|
console.info("pathQueries", pathQueries.length, pathQueries);
|
|
|
|
|
|
|
|
|
|
if(shixunQueries.length>0){
|
|
|
|
|
var pages = shixunQueries.map(i=>{
|
|
|
|
|
return {
|
|
|
|
|
path: shixunPath,
|
|
|
|
|
query: i
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
var res = await cloud.openapi.search.submitPages({pages});
|
|
|
|
|
console.log("submit shixun pages",pages.length, res);
|
|
|
|
|
console.info("arguments", pages);
|
|
|
|
|
}else{
|
|
|
|
|
console.warn("no shixun history");
|
|
|
|
|
}
|
|
|
|
|
if(pathQueries.length>0){
|
|
|
|
|
var pages = pathQueries.map(i=>{
|
|
|
|
|
return {
|
|
|
|
|
path: pathPath,
|
|
|
|
|
query: i
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
var res = await cloud.openapi.search.submitPages({pages});
|
|
|
|
|
console.log("submit path pages success", pages.length, res);
|
|
|
|
|
console.info("arguments", pages);
|
|
|
|
|
}else{
|
|
|
|
|
console.warn("no path history");
|
|
|
|
|
}
|
|
|
|
|
return {code:0, message:"success"};
|
|
|
|
|
async function submitPages({Time}){
|
|
|
|
|
let timeEnd = new Date(Time);
|
|
|
|
|
let timeStart = new Date(timeEnd.getTime() - config.submitPagesGapDay*milliSecondsPerDay);
|
|
|
|
|
// pages to be submitted
|
|
|
|
|
let routes = [
|
|
|
|
|
"shixun", "path", "mooc_case", "competition"
|
|
|
|
|
]
|
|
|
|
|
let promises = routes.map(i=>{
|
|
|
|
|
return submitPage({route: PagePathMap[i], timeStart, timeEnd})
|
|
|
|
|
})
|
|
|
|
|
return Promise.all(promises)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function clearPageHistory({Time}){
|
|
|
|
|
|
