|
|
/**
|
|
|
* @class ActiveDevices 活跃设备模型 - 每日跑批合并，仅添加本周/本月首次访问的设备。
|
|
|
*/
|
|
|
const BaseMod = require('./base')
|
|
|
const Platform = require('./platform')
|
|
|
const Channel = require('./channel')
|
|
|
const Version = require('./version')
|
|
|
const SessionLog = require('./sessionLog')
|
|
|
const {
|
|
|
DateTime,
|
|
|
UniCrypto
|
|
|
} = require('../lib')
|
|
|
module.exports = class ActiveDevices extends BaseMod {
|
|
|
constructor() {
|
|
|
super()
|
|
|
this.tableName = 'active-devices'
|
|
|
this.platforms = []
|
|
|
this.channels = []
|
|
|
this.versions = []
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* @desc 活跃设备统计 - 为周统计/月统计提供周活/月活数据
|
|
|
* @param {date|time} date
|
|
|
* @param {bool} reset
|
|
|
*/
|
|
|
async stat(date, reset) {
|
|
|
const dateTime = new DateTime()
|
|
|
const dateDimension = dateTime.getTimeDimensionByType('day', -1, date)
|
|
|
this.startTime = dateDimension.startTime
|
|
|
// 查看当前时间段数据是否已存在,防止重复生成
|
|
|
if (!reset) {
|
|
|
const checkRes = await this.getCollection(this.tableName).where({
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
}
|
|
|
}).get()
|
|
|
if (checkRes.data.length > 0) {
|
|
|
console.log('data have exists')
|
|
|
return {
|
|
|
code: 1003,
|
|
|
msg: 'Devices data in this time have already existed'
|
|
|
}
|
|
|
}
|
|
|
} else {
|
|
|
const delRes = await this.delete(this.tableName, {
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
}
|
|
|
})
|
|
|
console.log('Delete old data result:', JSON.stringify(delRes))
|
|
|
}
|
|
|
|
|
|
const sessionLog = new SessionLog()
|
|
|
const statRes = await this.aggregate(sessionLog.tableName, {
|
|
|
project: {
|
|
|
appid: 1,
|
|
|
version: 1,
|
|
|
platform: 1,
|
|
|
channel: 1,
|
|
|
is_first_visit: 1,
|
|
|
create_time: 1,
|
|
|
device_id: 1
|
|
|
},
|
|
|
match: {
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
}
|
|
|
},
|
|
|
group: {
|
|
|
_id: {
|
|
|
appid: '$appid',
|
|
|
version: '$version',
|
|
|
platform: '$platform',
|
|
|
channel: '$channel',
|
|
|
device_id: '$device_id'
|
|
|
},
|
|
|
is_new: {
|
|
|
$max: '$is_first_visit'
|
|
|
},
|
|
|
create_time: {
|
|
|
$min: '$create_time'
|
|
|
}
|
|
|
},
|
|
|
sort: {
|
|
|
create_time: 1
|
|
|
},
|
|
|
getAll: true
|
|
|
})
|
|
|
|
|
|
let res = {
|
|
|
code: 0,
|
|
|
msg: 'success'
|
|
|
}
|
|
|
// if (this.debug) {
|
|
|
// console.log('statRes', JSON.stringify(statRes))
|
|
|
// }
|
|
|
if (statRes.data.length > 0) {
|
|
|
const uniCrypto = new UniCrypto()
|
|
|
// 同应用、平台、渠道、版本的数据合并
|
|
|
const statData = [];
|
|
|
let statKey;
|
|
|
let data
|
|
|
|
|
|
const statOldRes = await this.aggregate(sessionLog.tableName, {
|
|
|
project: {
|
|
|
appid: 1,
|
|
|
version: 1,
|
|
|
platform: 1,
|
|
|
channel: 1,
|
|
|
is_first_visit: 1,
|
|
|
create_time: 1,
|
|
|
old_device_id: 1
|
|
|
},
|
|
|
match: {
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
},
|
|
|
old_device_id: {$exists: true}
|
|
|
},
|
|
|
group: {
|
|
|
_id: {
|
|
|
appid: '$appid',
|
|
|
version: '$version',
|
|
|
platform: '$platform',
|
|
|
channel: '$channel',
|
|
|
old_device_id: '$old_device_id'
|
|
|
},
|
|
|
create_time: {
|
|
|
$min: '$create_time'
|
|
|
}
|
|
|
},
|
|
|
sort: {
|
|
|
create_time: 1
|
|
|
},
|
|
|
getAll: true
|
|
|
})
|
|
|
if (this.debug) {
|
|
|
console.log('statOldRes', JSON.stringify(statOldRes))
|
|
|
}
|
|
|
for (const sti in statRes.data) {
|
|
|
data = statRes.data[sti]
|
|
|
statKey = uniCrypto.md5(data._id.appid + data._id.platform + data._id.version + data._id
|
|
|
.channel)
|
|
|
if (!statData[statKey]) {
|
|
|
statData[statKey] = {
|
|
|
appid: data._id.appid,
|
|
|
platform: data._id.platform,
|
|
|
version: data._id.version,
|
|
|
channel: data._id.channel,
|
|
|
device_ids: [],
|
|
|
old_device_ids: [],
|
|
|
info: [],
|
|
|
old_info: []
|
|
|
}
|
|
|
statData[statKey].device_ids.push(data._id.device_id)
|
|
|
statData[statKey].info[data._id.device_id] = {
|
|
|
is_new: data.is_new,
|
|
|
create_time: data.create_time
|
|
|
}
|
|
|
} else {
|
|
|
statData[statKey].device_ids.push(data._id.device_id)
|
|
|
statData[statKey].info[data._id.device_id] = {
|
|
|
is_new: data.is_new,
|
|
|
create_time: data.create_time
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
if(statOldRes.data.length) {
|
|
|
const oldDeviceIds = []
|
|
|
for(const osti in statOldRes.data) {
|
|
|
if(!statOldRes.data[osti]._id.old_device_id) {
|
|
|
continue
|
|
|
}
|
|
|
oldDeviceIds.push(statOldRes.data[osti]._id.old_device_id)
|
|
|
}
|
|
|
if(oldDeviceIds.length) {
|
|
|
const statOldDidRes = await this.aggregate(sessionLog.tableName, {
|
|
|
project: {
|
|
|
appid: 1,
|
|
|
version: 1,
|
|
|
platform: 1,
|
|
|
channel: 1,
|
|
|
is_first_visit: 1,
|
|
|
create_time: 1,
|
|
|
device_id: 1
|
|
|
},
|
|
|
match: {
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
},
|
|
|
device_id: {$in: oldDeviceIds}
|
|
|
},
|
|
|
group: {
|
|
|
_id: {
|
|
|
appid: '$appid',
|
|
|
version: '$version',
|
|
|
platform: '$platform',
|
|
|
channel: '$channel',
|
|
|
old_device_id: '$device_id'
|
|
|
},
|
|
|
create_time: {
|
|
|
$min: '$create_time'
|
|
|
}
|
|
|
},
|
|
|
sort: {
|
|
|
create_time: 1
|
|
|
},
|
|
|
getAll: true
|
|
|
})
|
|
|
|
|
|
if(statOldDidRes.data.length){
|
|
|
for(const osti in statOldDidRes.data) {
|
|
|
data = statOldDidRes.data[osti]
|
|
|
statKey = uniCrypto.md5(data._id.appid + data._id.platform + data._id.version + data._id
|
|
|
.channel)
|
|
|
if(!data._id.old_device_id) {
|
|
|
continue
|
|
|
}
|
|
|
|
|
|
if (!statData[statKey]) {
|
|
|
statData[statKey] = {
|
|
|
appid: data._id.appid,
|
|
|
platform: data._id.platform,
|
|
|
version: data._id.version,
|
|
|
channel: data._id.channel,
|
|
|
device_ids: [],
|
|
|
old_device_ids: [],
|
|
|
old_info: []
|
|
|
}
|
|
|
statData[statKey].old_device_ids.push(data._id.old_device_id)
|
|
|
} else {
|
|
|
statData[statKey].old_device_ids.push(data._id.old_device_id)
|
|
|
}
|
|
|
if(!statData[statKey].old_info[data._id.old_device_id]) {
|
|
|
statData[statKey].old_info[data._id.old_device_id] = {
|
|
|
create_time: data.create_time
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
this.fillData = []
|
|
|
for (const sk in statData) {
|
|
|
await this.getFillData(statData[sk])
|
|
|
}
|
|
|
|
|
|
if (this.fillData.length > 0) {
|
|
|
res = await this.batchInsert(this.tableName, this.fillData)
|
|
|
}
|
|
|
}
|
|
|
return res
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 获取填充数据
|
|
|
* @param {Object} data
|
|
|
*/
|
|
|
async getFillData(data) {
|
|
|
// 平台信息
|
|
|
let platformInfo = null
|
|
|
if (this.platforms && this.platforms[data.platform]) {
|
|
|
platformInfo = this.platforms[data.platform]
|
|
|
} else {
|
|
|
const platform = new Platform()
|
|
|
platformInfo = await platform.getPlatformAndCreate(data.platform, null)
|
|
|
if (!platformInfo || platformInfo.length === 0) {
|
|
|
platformInfo._id = ''
|
|
|
}
|
|
|
this.platforms[data.platform] = platformInfo
|
|
|
if (this.debug) {
|
|
|
console.log('platformInfo', JSON.stringify(platformInfo))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// 渠道信息
|
|
|
let channelInfo = null
|
|
|
const channelKey = data.appid + '_' + platformInfo._id + '_' + data.channel
|
|
|
if (this.channels && this.channels[channelKey]) {
|
|
|
channelInfo = this.channels[channelKey]
|
|
|
} else {
|
|
|
const channel = new Channel()
|
|
|
channelInfo = await channel.getChannelAndCreate(data.appid, platformInfo._id, data.channel)
|
|
|
if (!channelInfo || channelInfo.length === 0) {
|
|
|
channelInfo._id = ''
|
|
|
}
|
|
|
this.channels[channelKey] = channelInfo
|
|
|
if (this.debug) {
|
|
|
console.log('channelInfo', JSON.stringify(channelInfo))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
// 版本信息
|
|
|
let versionInfo = null
|
|
|
const versionKey = data.appid + '_' + data.platform + '_' + data.version
|
|
|
if (this.versions && this.versions[versionKey]) {
|
|
|
versionInfo = this.versions[versionKey]
|
|
|
} else {
|
|
|
const version = new Version()
|
|
|
versionInfo = await version.getVersionAndCreate(data.appid, data.platform, data.version)
|
|
|
if (!versionInfo || versionInfo.length === 0) {
|
|
|
versionInfo._id = ''
|
|
|
}
|
|
|
this.versions[versionKey] = versionInfo
|
|
|
if (this.debug) {
|
|
|
console.log('versionInfo', JSON.stringify(versionInfo))
|
|
|
}
|
|
|
}
|
|
|
|
|
|
const datetime = new DateTime()
|
|
|
const dateDimension = datetime.getTimeDimensionByType('week', 0, this.startTime)
|
|
|
const dateMonthDimension = datetime.getTimeDimensionByType('month', 0, this.startTime)
|
|
|
|
|
|
if(data.device_ids) {
|
|
|
// 取出本周已经存储的device_id
|
|
|
const weekHaveDeviceList = []
|
|
|
const haveWeekList = await this.selectAll(this.tableName, {
|
|
|
appid: data.appid,
|
|
|
version_id: versionInfo._id,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
device_id: {
|
|
|
$in: data.device_ids
|
|
|
},
|
|
|
dimension: 'week',
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
}
|
|
|
}, {
|
|
|
device_id: 1
|
|
|
})
|
|
|
if (haveWeekList.data.length > 0) {
|
|
|
for (const hui in haveWeekList.data) {
|
|
|
weekHaveDeviceList.push(haveWeekList.data[hui].device_id)
|
|
|
}
|
|
|
}
|
|
|
if (this.debug) {
|
|
|
console.log('weekHaveDeviceList', JSON.stringify(weekHaveDeviceList))
|
|
|
}
|
|
|
|
|
|
// 取出本月已经存储的device_id
|
|
|
const monthHaveDeviceList = []
|
|
|
const haveMonthList = await this.selectAll(this.tableName, {
|
|
|
appid: data.appid,
|
|
|
version_id: versionInfo._id,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
device_id: {
|
|
|
$in: data.device_ids
|
|
|
},
|
|
|
dimension: 'month',
|
|
|
create_time: {
|
|
|
$gte: dateMonthDimension.startTime,
|
|
|
$lte: dateMonthDimension.endTime
|
|
|
}
|
|
|
}, {
|
|
|
device_id: 1
|
|
|
})
|
|
|
if (haveMonthList.data.length > 0) {
|
|
|
for (const hui in haveMonthList.data) {
|
|
|
monthHaveDeviceList.push(haveMonthList.data[hui].device_id)
|
|
|
}
|
|
|
}
|
|
|
if (this.debug) {
|
|
|
console.log('monthHaveDeviceList', JSON.stringify(monthHaveDeviceList))
|
|
|
}
|
|
|
//数据填充
|
|
|
for (const ui in data.device_ids) {
|
|
|
//周活跃数据填充
|
|
|
if (!weekHaveDeviceList.includes(data.device_ids[ui])) {
|
|
|
this.fillData.push({
|
|
|
appid: data.appid,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
version_id: versionInfo._id,
|
|
|
is_new: data.info[data.device_ids[ui]].is_new,
|
|
|
device_id: data.device_ids[ui],
|
|
|
dimension: 'week',
|
|
|
create_time: data.info[data.device_ids[ui]].create_time
|
|
|
})
|
|
|
}
|
|
|
//月活跃数据填充
|
|
|
if (!monthHaveDeviceList.includes(data.device_ids[ui])) {
|
|
|
this.fillData.push({
|
|
|
appid: data.appid,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
version_id: versionInfo._id,
|
|
|
is_new: data.info[data.device_ids[ui]].is_new,
|
|
|
device_id: data.device_ids[ui],
|
|
|
dimension: 'month',
|
|
|
create_time: data.info[data.device_ids[ui]].create_time
|
|
|
})
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if(data.old_device_ids) {
|
|
|
// 取出本周已经存储的old_device_id
|
|
|
const weekHaveOldDeviceList = []
|
|
|
const haveOldWeekList = await this.selectAll(this.tableName, {
|
|
|
appid: data.appid,
|
|
|
version_id: versionInfo._id,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
device_id: {
|
|
|
$in: data.old_device_ids
|
|
|
},
|
|
|
dimension: 'week-old',
|
|
|
create_time: {
|
|
|
$gte: dateDimension.startTime,
|
|
|
$lte: dateDimension.endTime
|
|
|
}
|
|
|
}, {
|
|
|
device_id: 1
|
|
|
})
|
|
|
if (haveOldWeekList.data.length > 0) {
|
|
|
for (const hui in haveOldWeekList.data) {
|
|
|
weekHaveOldDeviceList.push(haveOldWeekList.data[hui].device_id)
|
|
|
}
|
|
|
}
|
|
|
if (this.debug) {
|
|
|
console.log('weekHaveOldDeviceList', JSON.stringify(weekHaveOldDeviceList))
|
|
|
}
|
|
|
|
|
|
// 取出本月已经存储的old_device_id
|
|
|
const monthHaveOldDeviceList = []
|
|
|
const haveOldMonthList = await this.selectAll(this.tableName, {
|
|
|
appid: data.appid,
|
|
|
version_id: versionInfo._id,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
device_id: {
|
|
|
$in: data.old_device_ids
|
|
|
},
|
|
|
dimension: 'month-old',
|
|
|
create_time: {
|
|
|
$gte: dateMonthDimension.startTime,
|
|
|
$lte: dateMonthDimension.endTime
|
|
|
}
|
|
|
}, {
|
|
|
device_id: 1
|
|
|
})
|
|
|
if (haveOldMonthList.data.length > 0) {
|
|
|
for (const hui in haveOldMonthList.data) {
|
|
|
monthHaveOldDeviceList.push(haveOldMonthList.data[hui].device_id)
|
|
|
}
|
|
|
}
|
|
|
if (this.debug) {
|
|
|
console.log('monthHaveOldDeviceList', JSON.stringify(monthHaveOldDeviceList))
|
|
|
}
|
|
|
//数据填充
|
|
|
for (const ui in data.old_device_ids) {
|
|
|
//周活跃数据填充
|
|
|
if (!weekHaveOldDeviceList.includes(data.old_device_ids[ui])) {
|
|
|
this.fillData.push({
|
|
|
appid: data.appid,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
version_id: versionInfo._id,
|
|
|
is_new: 0,
|
|
|
device_id: data.old_device_ids[ui],
|
|
|
dimension: 'week-old',
|
|
|
create_time: data.old_info[data.old_device_ids[ui]].create_time
|
|
|
})
|
|
|
}
|
|
|
//月活跃数据填充
|
|
|
if (!monthHaveOldDeviceList.includes(data.old_device_ids[ui])) {
|
|
|
this.fillData.push({
|
|
|
appid: data.appid,
|
|
|
platform_id: platformInfo._id,
|
|
|
channel_id: channelInfo._id,
|
|
|
version_id: versionInfo._id,
|
|
|
is_new: 0,
|
|
|
device_id: data.old_device_ids[ui],
|
|
|
dimension: 'month-old',
|
|
|
create_time: data.old_info[data.old_device_ids[ui]].create_time
|
|
|
})
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
return true
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 日志清理，此处日志为临时数据并不需要自定义清理，默认为固定值即可
|
|
|
*/
|
|
|
async clean() {
|
|
|
// 清除周数据，周留存统计最高需要10周数据，多余的为无用数据
|
|
|
const weeks = 10
|
|
|
console.log('Clean device\'s weekly logs - week:', weeks)
|
|
|
|
|
|
const dateTime = new DateTime()
|
|
|
|
|
|
const res = await this.delete(this.tableName, {
|
|
|
dimension: 'week',
|
|
|
create_time: {
|
|
|
$lt: dateTime.getTimeBySetWeek(0 - weeks)
|
|
|
}
|
|
|
})
|
|
|
|
|
|
if (!res.code) {
|
|
|
console.log('Clean device\'s weekly logs - res:', res)
|
|
|
}
|
|
|
|
|
|
// 清除月数据，月留存统计最高需要10个月数据，多余的为无用数据
|
|
|
const monthes = 10
|
|
|
console.log('Clean device\'s monthly logs - month:', monthes)
|
|
|
const monthRes = await this.delete(this.tableName, {
|
|
|
dimension: 'month',
|
|
|
create_time: {
|
|
|
$lt: dateTime.getTimeBySetMonth(0 - monthes)
|
|
|
}
|
|
|
})
|
|
|
if (!monthRes.code) {
|
|
|
console.log('Clean device\'s monthly logs - res:', res)
|
|
|
}
|
|
|
return monthRes
|
|
|
}
|
|
|
}
|
