|
|
import os
|
|
|
import sys
|
|
|
sys.path.append(os.getcwd())
|
|
|
import numpy as np
|
|
|
import pandas as pd
|
|
|
from tqdm import tqdm
|
|
|
import warnings
|
|
|
from datetime import datetime
|
|
|
from config import logger
|
|
|
from config import test_user_id
|
|
|
from config import shixun_features_save_path
|
|
|
from config import shixun_all_user_item_feats
|
|
|
from matching.shixun.recall_comm import get_item_info_df
|
|
|
from matching.shixun.recall_comm import get_all_select_df
|
|
|
from matching.shixun.recall_comm import get_select_item_info
|
|
|
from matching.shixun.recall_comm import get_rank_hist_and_last_select
|
|
|
from matching.shixun.multi_recall_predict import multi_recall_predict
|
|
|
from ranking.shixun.rank_features_engineering import get_embedding, get_recall_list
|
|
|
from ranking.shixun.rank_comm import fill_is_trainee_hab
|
|
|
from ranking.shixun.rank_comm import get_rank_item_info_dict
|
|
|
|
|
|
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
|
|
|
|
|
|
tqdm.pandas()
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
all_select_df, recall_list_dict = None, None
|
|
|
item_content_emb_dict, item_w2v_emb_dict = None, None
|
|
|
item_youtube_emb_dict, user_youtube_emb_dict = None, None
|
|
|
shixun_info_df, user_features = None, None
|
|
|
all_user_item_feats_df = None
|
|
|
|
|
|
|
|
|
def create_rank_behavior_feature(user_id,
|
|
|
recall_list,
|
|
|
select_hist_df,
|
|
|
shixuns_info,
|
|
|
shixun_info_dict,
|
|
|
shixuns_emb,
|
|
|
user_emb=None,
|
|
|
N=1):
|
|
|
"""
|
|
|
基于用户历史行为生成排序模型特征
|
|
|
:param users_id: 用户id
|
|
|
:param recall_list: 用户召回的候选物品列表
|
|
|
:param select_hist_df: 用户历史选择的物品
|
|
|
:param shixuns_info: 物品信息
|
|
|
:param shixuns_emb: 物品的embedding向量,可以用item_content_emb, item_w2v_emb, youtube_item_emb
|
|
|
:param user_emb: 用户的embedding向量，可以是youtube_user_emb, 也可以不用，
|
|
|
如果要传的话，shixuns_emb就要用youtube_item_emb，保持维度一样
|
|
|
:param N: 最近的N次选择，由于行为日志里面很多用户只存在一次历史选择，为了不产生空值，默认为1
|
|
|
"""
|
|
|
|
|
|
shixuns_info['shixun_id'] = shixuns_info['shixun_id'].astype(int)
|
|
|
select_hist_df['user_id'] = select_hist_df['user_id'].astype(int)
|
|
|
|
|
|
# 建立一个二维列表保存结果， 后面要转成DataFrame
|
|
|
all_user_feas = []
|
|
|
shixun_id_list = shixuns_info['shixun_id'].values.tolist()
|
|
|
|
|
|
# 该用户的最后N次选择
|
|
|
hist_user_items = select_hist_df[select_hist_df['user_id']==user_id]['shixun_id'][-N:]
|
|
|
|
|
|
# 遍历该用户的召回列表
|
|
|
for rank, (shixun_id, score) in enumerate(recall_list):
|
|
|
# 不在物品信息中的跳过，以免报错
|
|
|
if shixun_id not in shixun_id_list:
|
|
|
continue
|
|
|
|
|
|
shixun_id = int(shixun_id)
|
|
|
cur_shixuns_info = shixun_info_dict[shixun_id]
|
|
|
|
|
|
# 该物品建立时间, 访问次数，选择人数，关卡数量
|
|
|
|
|
|
a_create_time = cur_shixuns_info[0][0]
|
|
|
a_trainee = cur_shixuns_info[0][1]
|
|
|
a_visits_count = cur_shixuns_info[0][2]
|
|
|
a_myshixuns_count = cur_shixuns_info[0][3]
|
|
|
a_challenges_count = cur_shixuns_info[0][4]
|
|
|
a_averge_star = cur_shixuns_info[0][5]
|
|
|
a_task_pass = cur_shixuns_info[0][6]
|
|
|
single_user_fea = [user_id, shixun_id]
|
|
|
|
|
|
# 计算与最后选择的物品的相似度的和，最大值、最小值、均值
|
|
|
sim_fea = []
|
|
|
time_fea = []
|
|
|
visits_fea = []
|
|
|
myshixuns_fea = []
|
|
|
challenges_fea = []
|
|
|
trainee_fea = []
|
|
|
averge_star_fea = []
|
|
|
task_pass_fea = []
|
|
|
|
|
|
# 遍历用户的最后N次选择物品
|
|
|
for hist_item in hist_user_items:
|
|
|
if (hist_item not in shixun_id_list):
|
|
|
continue
|
|
|
|
|
|
hist_item = int(hist_item)
|
|
|
hist_shixuns_info = shixun_info_dict[hist_item]
|
|
|
|
|
|
b_create_time = hist_shixuns_info[0][0]
|
|
|
b_trainee = hist_shixuns_info[0][1]
|
|
|
b_visits_count = hist_shixuns_info[0][2]
|
|
|
b_myshixuns_count = hist_shixuns_info[0][3]
|
|
|
b_challenges_count = hist_shixuns_info[0][4]
|
|
|
b_averge_star = hist_shixuns_info[0][5]
|
|
|
b_task_pass = hist_shixuns_info[0][6]
|
|
|
|
|
|
if (hist_item not in shixuns_emb) or (shixun_id not in shixuns_emb):
|
|
|
sim_fea.append(0.0)
|
|
|
else:
|
|
|
sim_fea.append(np.dot(shixuns_emb[hist_item], shixuns_emb[shixun_id]))
|
|
|
|
|
|
time_fea.append(abs(a_create_time - b_create_time))
|
|
|
visits_fea.append(abs(a_visits_count - b_visits_count))
|
|
|
myshixuns_fea.append(abs(a_myshixuns_count - b_myshixuns_count))
|
|
|
challenges_fea.append(abs(a_challenges_count - b_challenges_count))
|
|
|
trainee_fea.append(abs(a_trainee - b_trainee))
|
|
|
averge_star_fea.append(abs(a_averge_star - b_averge_star))
|
|
|
task_pass_fea.append(abs(a_task_pass-b_task_pass))
|
|
|
|
|
|
|
|
|
if (len(sim_fea) != 0) and (len(time_fea) != 0) and (len(visits_fea) != 0) and \
|
|
|
(len(myshixuns_fea) != 0) and (len(challenges_fea) != 0) and \
|
|
|
(len(trainee_fea) != 0) and (len(averge_star_fea) != 0)and (len(task_pass_fea) != 0):
|
|
|
|
|
|
# 相似性特征
|
|
|
single_user_fea.extend(sim_fea)
|
|
|
|
|
|
# 时间差特征
|
|
|
single_user_fea.extend(time_fea)
|
|
|
|
|
|
# 访问次数差特征
|
|
|
single_user_fea.extend(visits_fea)
|
|
|
|
|
|
# 选课人数差特征
|
|
|
single_user_fea.extend(myshixuns_fea)
|
|
|
|
|
|
# 关卡数量差特征
|
|
|
single_user_fea.extend(challenges_fea)
|
|
|
|
|
|
# 难易程度差特征
|
|
|
single_user_fea.extend(trainee_fea)
|
|
|
|
|
|
# 平均星数差特征
|
|
|
single_user_fea.extend(averge_star_fea)
|
|
|
|
|
|
#通过人数差特征
|
|
|
single_user_fea.extend(task_pass_fea)
|
|
|
|
|
|
# 相似性的统计特征
|
|
|
single_user_fea.extend([max(sim_fea), min(sim_fea), sum(sim_fea), sum(sim_fea) / len(sim_fea)])
|
|
|
|
|
|
if user_emb: # 如果用户向量有的话，这里计算该召回物品与用户的相似性特征
|
|
|
if (user_id not in user_emb) or (shixun_id not in shixuns_emb):
|
|
|
single_user_fea.append(0.0)
|
|
|
else:
|
|
|
single_user_fea.append(np.dot(user_emb[user_id], shixuns_emb[shixun_id]))
|
|
|
|
|
|
single_user_fea.extend([score, rank])
|
|
|
|
|
|
# 加入到总的表中
|
|
|
all_user_feas.append(single_user_fea)
|
|
|
|
|
|
# 定义交叉特征
|
|
|
id_cols = ['user_id', 'shixun_id']
|
|
|
sim_cols = ['sim' + str(i) for i in range(N)]
|
|
|
time_cols = ['time_diff' + str(i) for i in range(N)]
|
|
|
vists_cols = ['visit_diff' + str(i) for i in range(N)]
|
|
|
myshixuns_cols = ['myshixuns_diff' + str(i) for i in range(N)]
|
|
|
challenge_cols = ['challenges_diff' + str(i) for i in range(N)]
|
|
|
trainee_cols = ['trainee_diff' + str(i) for i in range(N)]
|
|
|
averge_star_cols = ['averge_star_diff' + str(i) for i in range(N)]
|
|
|
task_pass_cols = ['task_pass_diff' + str(i) for i in range(N)]
|
|
|
|
|
|
sat_cols = ['sim_max', 'sim_min', 'sim_sum', 'sim_mean']
|
|
|
user_item_sim_cols = ['user_item_sim'] if user_emb else []
|
|
|
user_score_rank_label = ['score', 'rank']
|
|
|
|
|
|
# 交叉特征列表
|
|
|
cols = id_cols + sim_cols + time_cols + vists_cols + myshixuns_cols + challenge_cols \
|
|
|
+ trainee_cols + averge_star_cols + task_pass_cols + sat_cols + user_item_sim_cols + user_score_rank_label
|
|
|
|
|
|
# 转成DataFrame
|
|
|
features_df = pd.DataFrame(all_user_feas, columns=cols)
|
|
|
|
|
|
return features_df
|
|
|
|
|
|
def init_rank_features():
|
|
|
global all_select_df
|
|
|
global recall_list_dict
|
|
|
global item_bert_emb_dict
|
|
|
global item_word2vec_emb_dict
|
|
|
global item_youtube_emb_dict
|
|
|
global user_youtube_emb_dict
|
|
|
global item_dssm_emb_dict
|
|
|
global user_dssm_emb_dict
|
|
|
global shixun_info_df
|
|
|
global user_features
|
|
|
global all_user_item_feats_df
|
|
|
global shixun_info_dict
|
|
|
global shixun_hot
|
|
|
|
|
|
logger.info("加载物品行为数据")
|
|
|
all_select_df = get_all_select_df(offline=False)
|
|
|
|
|
|
logger.info('获取物品召回数据')
|
|
|
recall_list_dict = get_recall_list(single_recall_model='youtubednn', multi_recall=True)
|
|
|
|
|
|
logger.info('获取物品向量化特征')
|
|
|
# item_content_emb_dict, item_w2v_emb_dict, \
|
|
|
# item_youtube_emb_dict, user_youtube_emb_dict = get_embedding()
|
|
|
item_bert_emb_dict, item_word2vec_emb_dict,\
|
|
|
item_youtube_emb_dict, user_youtube_emb_dict,item_dssm_emb_dict,user_dssm_emb_dict=get_embedding()
|
|
|
|
|
|
|
|
|
|
|
|
logger.info('获取物品信息')
|
|
|
shixun_info_df = get_item_info_df()
|
|
|
|
|
|
# 用到的物品信息特征
|
|
|
shixun_info_df = shixun_info_df[['shixun_id', 'visits', 'trainee',
|
|
|
'myshixuns_count', 'challenges_count', 'averge_star', 'created_at_ts','task_pass']]
|
|
|
|
|
|
logger.info('生成物品信息字典')
|
|
|
shixun_info_dict = get_rank_item_info_dict(shixun_info_df)
|
|
|
|
|
|
logger.info('获取用户特征')
|
|
|
user_features = pd.read_csv(shixun_features_save_path + 'user_features_df.csv', sep='\t', encoding='utf-8')
|
|
|
|
|
|
logger.info('获取物品热度特征')
|
|
|
shixun_hot = pd.read_csv(shixun_features_save_path + 'shixun_hot_level.csv', sep='\t', encoding='utf-8')
|
|
|
|
|
|
logger.info('获取排序模型特征')
|
|
|
all_user_item_feats_df = pd.read_csv(shixun_all_user_item_feats, sep='\t', encoding='utf-8')
|
|
|
|
|
|
|
|
|
def build_rank_features_online(user_id, user_recall_item_dict):
|
|
|
"""
|
|
|
根据用户召回列表构建排序模型特征
|
|
|
"""
|
|
|
# 没有召回数据返回空DataFrame
|
|
|
if user_id not in user_recall_item_dict:
|
|
|
return pd.DataFrame()
|
|
|
|
|
|
start_time = datetime.now()
|
|
|
|
|
|
# 获取用户的召回列表
|
|
|
recall_list = user_recall_item_dict[user_id]
|
|
|
|
|
|
select_info = get_select_item_info(all_select_df, user_id)
|
|
|
select_hist, select_last = get_rank_hist_and_last_select(select_info)
|
|
|
|
|
|
user_item_feats_df = create_rank_behavior_feature(user_id,
|
|
|
recall_list,
|
|
|
select_hist,
|
|
|
shixun_info_df,
|
|
|
shixun_info_dict,
|
|
|
item_youtube_emb_dict,
|
|
|
user_youtube_emb_dict)
|
|
|
|
|
|
if not user_item_feats_df.empty:
|
|
|
# 拼接用户特征
|
|
|
user_item_feats_df = user_item_feats_df.merge(user_features, on='user_id', how='left')
|
|
|
|
|
|
# 接接物品特征
|
|
|
user_item_feats_df = user_item_feats_df.merge(shixun_info_df, on='shixun_id', how='left')
|
|
|
|
|
|
# 接接物品热度特征
|
|
|
user_item_feats_df = user_item_feats_df.merge(shixun_hot, on='shixun_id', how='left')
|
|
|
|
|
|
# 是否在用户选择的实训难度中
|
|
|
user_item_feats_df['is_trainee_hab'] = user_item_feats_df.progress_apply(
|
|
|
lambda x: fill_is_trainee_hab(x), axis=1)
|
|
|
|
|
|
del user_item_feats_df['trainee_list']
|
|
|
|
|
|
user_item_feats_df = user_item_feats_df.reset_index()
|
|
|
|
|
|
# 计算耗时毫秒
|
|
|
end_time = datetime.utcnow()
|
|
|
cost_time_millisecond = round(float((end_time - start_time).microseconds / 1000.0), 3)
|
|
|
logger.info(f"在线生成排序模型特征耗时: {cost_time_millisecond} 毫秒")
|
|
|
|
|
|
return user_item_feats_df
|
|
|
|
|
|
def build_rank_features_offline(user_id):
|
|
|
"""
|
|
|
直接读取离线的排序模型特征
|
|
|
"""
|
|
|
user_item_feats_df = all_user_item_feats_df[all_user_item_feats_df['user_id'] == user_id]
|
|
|
return user_item_feats_df
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
init_rank_features()
|
|
|
user_recall_item_dict, only_cold_start_recall = multi_recall_predict(test_user_id, topk=100)
|
|
|
|
|
|
user_item_feats_df = build_rank_features_online(test_user_id, user_recall_item_dict)
|
|
|
user_item_feats_df.to_csv(shixun_features_save_path + 'user_item_feats_df.csv', sep='\t', index=False)
|
|
|
|
|
|
|
