import os import sys sys.path.append(os.getcwd()) import numpy as np import pandas as pd import pickle from tqdm import tqdm from sklearn.preprocessing import MinMaxScaler import warnings from utils import reduce_mem from config import logger from config import samples_mode from config import shixun_features_save_path from config import myshixuns_train_data, myshixuns_test_data from config import shixun_itemcf_recall_dict from config import shixun_item_embedding_recall_dict from config import shixun_youtubednn_recall_dict from config import shixun_youtubednn_usercf_recall_dict from config import shixun_dssm_recall_dict from config import shixun_pinsage_recall_dict from config import shixun_final_recall_items_dict # from config import shixuns_bert_emb_dict from config import shixun_item_w2v_emb_dict from config import shixun_train_user_item_feats from config import shixun_val_user_item_feats from config import shixun_test_user_item_feats from config import shixun_youtube_user_emb_dict from config import shixun_youtube_item_emb_dict from config import shixun_all_user_item_feats from config import shixun_dssm_item_emb_dict from config import shixun_dssm_user_emb_dict from config import offline_mode from matching.shixun.recall_comm import get_item_info_df from matching.shixun.recall_comm import get_hist_and_last_select from matching.shixun.recall_comm import get_user_info_df from ranking.shixun.rank_comm import fill_is_trainee_hab from ranking.shixun.rank_comm import get_rank_item_info_dict # from ranking.shixun.rank_comm import get_item_bert_emb_dict tqdm.pandas() warnings.filterwarnings('ignore') def train_val_split(all_select_df, sample_rate=0.3): """ 划分训练集和验证集 :param all_select_df: 指的是训练集 :param sample_rate: 采样作为验证集的用户比率 """ all_select = all_select_df all_user_ids = all_select.user_id.unique() # replace = True表示可以重复抽样，反之不可以 sample_user_ids = np.random.choice(all_user_ids, size=int(len(all_user_ids) * sample_rate), replace=False) select_val = all_select[all_select['user_id'].isin(sample_user_ids)] select_train = all_select[~all_select['user_id'].isin(sample_user_ids)] # 将验证集中的最后一次选择给抽取出来作为答案 select_val = select_val.sort_values(['user_id', 'created_timestamp']) val_ans = select_val.groupby('user_id').tail(1) select_val = select_val.groupby('user_id').progress_apply(lambda x: x[:-1]).reset_index(drop=True) # 如果该用户只有一个选择数据，又被分到ans中 # 止方法保证答案中出现的用户再验证集中还有 val_ans = val_ans[val_ans.user_id.isin(select_val.user_id.unique())] select_val = select_val[select_val.user_id.isin(val_ans.user_id.unique())] return select_train, select_val, val_ans def get_train_val_test_data(offline=True): """ 读取训练、验证、测试集 """ if offline: select_train_data = pd.read_csv(myshixuns_train_data, sep='\t', encoding='utf-8') select_train, select_val, val_ans=train_val_split(select_train_data, sample_rate=0.3) else: select_train = pd.read_csv(myshixuns_train_data, sep='\t', encoding='utf-8') select_val = None val_ans = None select_test = pd.read_csv(myshixuns_test_data, sep='\t', encoding='utf-8') return select_train, select_val, select_test, val_ans def get_recall_list(single_recall_model=None, multi_recall=False): """ 返回多路召回列表或者部分单路召回列表 """ if multi_recall: return pickle.load(open(shixun_final_recall_items_dict, 'rb')) if single_recall_model == 'i2i_itemcf': return pickle.load(open(shixun_itemcf_recall_dict, 'rb')) elif single_recall_model == 'i2i_emb_itemcf': return pickle.load(open(shixun_item_embedding_recall_dict, 'rb')) elif single_recall_model == 'user_cf': return pickle.load(open(shixun_youtubednn_usercf_recall_dict, 'rb')) elif single_recall_model == 'youtubednn': return pickle.load(open(shixun_youtubednn_recall_dict, 'rb')) elif single_recall_model == 'dssm': return pickle.load(open(shixun_dssm_recall_dict, 'rb')) elif single_recall_model == 'pinsage': return pickle.load(open(shixun_pinsage_recall_dict, 'rb')) def get_embedding(): """ 通过字典查询对应的item的Embedding """ #获取shixun的bert embedding字典 # if os.path.exists(shixuns_bert_emb_dict): # item_bert_emb_dict = pickle.load(open(shixuns_bert_emb_dict, 'rb')) # else: # item_bert_emb_dict = get_item_bert_emb_dict() # w2v Embedding是需要提前训练好的 if os.path.exists(shixun_item_w2v_emb_dict): item_word2vec_emb_dict = pickle.load(open(shixun_item_w2v_emb_dict, 'rb')) else: print(os.path.basename(shixun_item_w2v_emb_dict) + ' file not exist.') if os.path.exists(shixun_youtube_item_emb_dict): item_youtube_emb_dict = pickle.load(open(shixun_youtube_item_emb_dict, 'rb')) else: print(os.path.basename(shixun_youtube_item_emb_dict) + 'file not exist.') if os.path.exists(shixun_youtube_user_emb_dict): user_youtube_emb_dict = pickle.load(open(shixun_youtube_user_emb_dict, 'rb')) else: print(os.path.basename(shixun_youtube_user_emb_dict) + 'file not exist.') if os.path.exists(shixun_dssm_item_emb_dict): item_dssm_emb_dict = pickle.load(open(shixun_dssm_item_emb_dict, 'rb')) else: print(os.path.basename(shixun_dssm_item_emb_dict) + 'file not exist.') if os.path.exists(shixun_dssm_user_emb_dict): user_dssm_emb_dict = pickle.load(open(shixun_dssm_user_emb_dict, 'rb')) else: print(os.path.basename(shixun_dssm_user_emb_dict) + 'file not exist.') return item_word2vec_emb_dict, item_youtube_emb_dict, user_youtube_emb_dict,item_dssm_emb_dict,user_dssm_emb_dict def recall_dict_2_df(recall_list_dict): """ 召回列表转换成DataFrame形式, [user, item, score] """ df_row_list = [] for user, recall_list in tqdm(recall_list_dict.items()): for item, score in recall_list: df_row_list.append([user, item, score]) col_names = ['user_id', 'sim_item', 'score'] recall_list_df = pd.DataFrame(df_row_list, columns=col_names) return recall_list_df def neg_sample_recall_data(recall_items_df, sample_rate=0.05): """ 负采样函数，可以控制负采样时的比例, 这里给了一个默认的值 """ logger.info('采样之前数据') pos_data = recall_items_df[recall_items_df['label'] == 1] neg_data = recall_items_df[recall_items_df['label'] == 0] print('正样本数量：', len(pos_data), '负样本数量：', len(neg_data), '正样本比率：', round(len(pos_data)/(len(pos_data) + len(neg_data)), 6)) # 分组采样函数 def neg_sample_func(group_df): neg_num = len(group_df) # 保证最少有一个 sample_num = max(int(neg_num * sample_rate), 1) # 保证最多不超过20个，这里可以根据实际情况进行选择 sample_num = min(sample_num, 20) return group_df.sample(n=sample_num, replace=True) # 对用户进行负采样，保证所有用户都在采样后的数据中 neg_data_user_sample = neg_data.groupby('user_id', group_keys=False). \ progress_apply(neg_sample_func) # 对物品进行负采样，保证所有物品都在采样后的数据中 neg_data_item_sample = neg_data.groupby('sim_item', group_keys=False). \ progress_apply(neg_sample_func) # 将上述两种情况下的采样数据合并 neg_data_new = neg_data_user_sample.append(neg_data_item_sample) # 由于上述两个操作是分开的，可能将两个相同的数据给重复选择了，所以需要对合并后的数据进行去重 neg_data_new = neg_data_new.sort_values(['user_id', 'score']).drop_duplicates( ['user_id', 'sim_item'], keep='last') # 将正样本数据合并 data_new = pd.concat([pos_data, neg_data_new], ignore_index=True) logger.info('采样之后数据') pos_data = data_new[data_new['label'] == 1] neg_data = data_new[data_new['label'] == 0] print('正样本数量：', len(pos_data), '负样本数量：', len(neg_data), '正样本比率：', round(len(pos_data)/(len(pos_data) + len(neg_data)), 4)) return data_new def sample_test_recall_data(recall_items_df, sample_rate=0.05): """ 测试样采样函数，可以控制采样的比例, 这里给了一个默认的值 """ logger.info('采样之前样本数量：' + str(len(recall_items_df))) # 分组采样函数 def neg_sample_func(group_df): neg_num = len(group_df) # 保证最少有一个 sample_num = max(int(neg_num * sample_rate), 1) # 保证最多不超过20个，这里可以根据实际情况进行选择 sample_num = min(sample_num, 20) return group_df.sample(n=sample_num, replace=True) # 对用户进行负采样，保证所有用户都在采样后的数据中 data_user_sample = recall_items_df.groupby('user_id', group_keys=False). \ progress_apply(neg_sample_func) # 对物品进行负采样，保证所有物品都在采样后的数据中 data_item_sample = recall_items_df.groupby('sim_item', group_keys=False). \ progress_apply(neg_sample_func) # 将上述两种情况下的采样数据合并 data_new = data_user_sample.append(data_item_sample) # 由于上述两个操作是分开的，可能将两个相同的数据给重复选择了，所以需要对合并后的数据进行去重 data_new = data_new.sort_values(['user_id', 'score']).drop_duplicates( ['user_id', 'sim_item'], keep='last') logger.info('采样之后样本数量：' + str(len(data_new))) return data_new def get_rank_label_df(recall_list_df, label_df, is_test=False): """ 召回数据打标签 """ # 测试集是没有标签了，为了后面代码统一一些，这里直接给一个负数替代 if is_test: recall_list_df['label'] = -1 return recall_list_df label_df = label_df.rename(columns={'shixun_id': 'sim_item'}) recall_list_df_ = recall_list_df.merge(label_df[['user_id', 'sim_item', 'created_timestamp']], \ how='left', on=['user_id', 'sim_item']) recall_list_df_['label'] = recall_list_df_['created_timestamp'].progress_apply( lambda x: 0 if np.isnan(x) else 1) del recall_list_df_['created_timestamp'] return recall_list_df_ def get_user_recall_item_label_df(select_train_hist, select_val_hist, select_test_hist, select_train_last, select_val_last, recall_list_df): """ 获取用户召回列表训练，验证，测试集的标签 """ # 获取训练数据的召回列表 train_user_items_df = recall_list_df[recall_list_df['user_id'].isin(select_train_hist['user_id'].unique())] logger.info('训练集数据打标签') train_user_item_label_df = get_rank_label_df(train_user_items_df, select_train_last, is_test=False) logger.info('训练集数据负采样') train_user_item_label_df = neg_sample_recall_data(train_user_item_label_df) if select_val_hist is not None: val_user_items_df = recall_list_df[recall_list_df['user_id'].isin(select_val_hist['user_id'].unique())] logger.info('验证集数据打标签') val_user_item_label_df = get_rank_label_df(val_user_items_df, select_val_last, is_test=False) logger.info('验证集数据负采样') val_user_item_label_df = neg_sample_recall_data(val_user_item_label_df) else: val_user_item_label_df = None # 测试集数据进行随机采样，减少生成特征的时间 test_user_items_df = recall_list_df[recall_list_df['user_id'].isin(select_test_hist['user_id'].unique())] logger.info('测试集数据打标签') test_user_item_label_df = get_rank_label_df(test_user_items_df, None, is_test=True) logger.info('测试集数据随机采样') test_user_item_label_df = sample_test_recall_data(test_user_item_label_df) return train_user_item_label_df, val_user_item_label_df, test_user_item_label_df def make_tuple_func(group_df): """ 将最终的召回的df数据转换成字典的形式做排序特征 """ row_data = [] for name, row_df in group_df.iterrows(): row_data.append((row_df['sim_item'], row_df['score'], row_df['label'])) return row_data def get_cos_similar_matrix(v1, v2): #获取两个向量的余弦相似度 num = np.dot(v1, v2) # 向量点乘 denom = np.linalg.norm(v1).reshape(-1) * np.linalg.norm(v2).reshape(-1) # 求模长的乘积 res = num / denom res[np.isneginf(res)] = 0.0 #负无穷大的赋值0 # return num return float(0.5 + 0.5 * res) def create_behavior_feature(users_id, recall_list, select_hist_df, shixuns_info, shixun_info_dict, shixuns_emb, user_emb=None, N=1): """ 基于用户历史行为生成相关特征 :param users_id: 用户id :param recall_list: 对于每个用户召回的候选物品列表 :param select_hist_df: 用户历史选择的物品 :param shixuns_info: 物品信息 :param shixuns_emb: 物品的embedding向量,可以用item_bert_emb, item_w2v_emb, youtube_item_emb,dssm_item_emb :param user_emb: 用户的embedding向量,可以是youtube_user_emb, 也可以不用, 如果要传的话shixuns_emb就要用youtube_item_emb,保持维度一样 :param N: 最近的N次选择,由于行为日志里面很多用户只存在一次历史选择,为了不产生空值,默认为1 """ shixuns_info['shixun_id'] = shixuns_info['shixun_id'].astype(int) select_hist_df['user_id'] = select_hist_df['user_id'].astype(int) # 建立一个二维列表保存结果， 后面要转成DataFrame all_user_feas = [] shixun_id_list = shixuns_info['shixun_id'].values.tolist() for user_id in tqdm(users_id): # 该用户的最后N次选择 hist_user_items = select_hist_df[select_hist_df['user_id']==user_id]['shixun_id'][-N:] # 遍历该用户的召回列表 for rank, (shixun_id, score, label) in enumerate(recall_list[user_id]): # 不在物品信息中的跳过，以免报错 if shixun_id not in shixun_id_list: continue shixun_id = int(shixun_id) cur_shixuns_info = shixun_info_dict[shixun_id] # 实训建立时间, 访问次数，选择人数，关卡数量, 难易程度，平均星数,通过人数 a_create_time = cur_shixuns_info[0][0] a_trainee = cur_shixuns_info[0][1] a_visits_count = cur_shixuns_info[0][2] a_myshixuns_count = cur_shixuns_info[0][3] a_challenges_count = cur_shixuns_info[0][4] a_averge_star = cur_shixuns_info[0][5] a_task_pass = cur_shixuns_info[0][6] single_user_fea = [user_id, shixun_id] # 计算与最后选择的物品的相似度的和，最大值、最小值、均值 sim_fea = [] time_fea = [] visits_fea = [] myshixuns_fea = [] challenges_fea = [] trainee_fea = [] averge_star_fea = [] task_pass_fea = [] # 遍历用户的最后N次选择物品 for hist_item in hist_user_items: if (hist_item not in shixun_id_list): continue hist_item = int(hist_item) hist_shixuns_info = shixun_info_dict[hist_item] b_create_time = hist_shixuns_info[0][0] b_trainee = hist_shixuns_info[0][1] b_visits_count = hist_shixuns_info[0][2] b_myshixuns_count = hist_shixuns_info[0][3] b_challenges_count = hist_shixuns_info[0][4] b_averge_star = hist_shixuns_info[0][5] b_task_pass = hist_shixuns_info[0][6] if (hist_item not in shixuns_emb) or (shixun_id not in shixuns_emb): sim_fea.append(0.0) else: # sim_fea.append(np.dot(shixuns_emb[hist_item], shixuns_emb[shixun_id])) sim_fea.append(get_cos_similar_matrix(shixuns_emb[hist_item], shixuns_emb[shixun_id]))#余弦相似度 time_fea.append(abs(a_create_time - b_create_time)) visits_fea.append(abs(a_visits_count - b_visits_count)) myshixuns_fea.append(abs(a_myshixuns_count - b_myshixuns_count)) challenges_fea.append(abs(a_challenges_count - b_challenges_count)) trainee_fea.append(abs(a_trainee - b_trainee)) averge_star_fea.append(abs(a_averge_star - b_averge_star)) task_pass_fea.append(abs(a_task_pass-b_task_pass)) if (len(sim_fea) != 0) and (len(time_fea) != 0) and (len(visits_fea) != 0) and \ (len(myshixuns_fea) != 0) and (len(challenges_fea) != 0) and \ (len(trainee_fea) != 0) and (len(averge_star_fea) != 0)and (len(task_pass_fea) != 0): # 相似性特征 single_user_fea.extend(sim_fea) # 时间差特征 single_user_fea.extend(time_fea) # 访问次数差特征 single_user_fea.extend(visits_fea) # 选课人数差特征 single_user_fea.extend(myshixuns_fea) # 关卡数量差特征 single_user_fea.extend(challenges_fea) # 难易程度差特征 single_user_fea.extend(trainee_fea) # 平均星数差特征 single_user_fea.extend(averge_star_fea) #通过人数差特征 single_user_fea.extend(task_pass_fea) # 相似性的统计特征 single_user_fea.extend([max(sim_fea), min(sim_fea), sum(sim_fea), sum(sim_fea) / len(sim_fea)]) if user_emb: # 如果用户向量有的话，这里计算该召回物品与用户的相似性特征 if (user_id not in user_emb) or (shixun_id not in shixuns_emb): single_user_fea.append(0.0) else: single_user_fea.append(np.dot(user_emb[user_id], shixuns_emb[shixun_id])) single_user_fea.extend([score, rank, label]) # 加入到总的表中 all_user_feas.append(single_user_fea) # 定义交叉特征 id_cols = ['user_id', 'shixun_id'] sim_cols = ['sim' + str(i) for i in range(N)] time_cols = ['time_diff' + str(i) for i in range(N)] vists_cols = ['visit_diff' + str(i) for i in range(N)] myshixuns_cols = ['myshixuns_diff' + str(i) for i in range(N)] challenge_cols = ['challenges_diff' + str(i) for i in range(N)] trainee_cols = ['trainee_diff' + str(i) for i in range(N)] averge_star_cols = ['averge_star_diff' + str(i) for i in range(N)] task_pass_cols = ['task_pass_diff' + str(i) for i in range(N)] sat_cols = ['sim_max', 'sim_min', 'sim_sum', 'sim_mean'] user_item_sim_cols = ['user_item_sim'] if user_emb else [] user_score_rank_label = ['score', 'rank', 'label'] # 交叉特征列表 cols = id_cols + sim_cols + time_cols + vists_cols + myshixuns_cols + challenge_cols \ + trainee_cols + averge_star_cols + task_pass_cols + sat_cols + user_item_sim_cols + user_score_rank_label # 转成DataFrame features_df = pd.DataFrame(all_user_feas, columns=cols) return features_df def active_level(all_data, cols): """ 生成用户活跃度的特征 根据用户选择物品时间和选择物品的次数生成用户活跃度 如果用户选择物品之间的时间间隔比较小，同时选择的物品次数很多，就认为此用户是活跃用户 1. 首先根据user_id分组, 对于每个用户计算选择物品的次数，两两选择物品时间间隔的均值 2. 把选择次数取倒数和时间间隔的均值统一归一化，然后两者相加合并，该值越小说明用户越活跃 3. 注意：上面两两选择物品的时间间隔均值，会出现如果用户只选择了一次的情况， 这时候时间间隔均值那里会出现空值，对于这种情况最后特征那里给个大数进行区分 """ if os.path.exists(shixun_features_save_path + 'user_active_level.csv'): user_act = pd.read_csv(shixun_features_save_path + 'user_active_level.csv', sep='\t', encoding='utf-8') return user_act data = all_data[cols] data.sort_values(['user_id', 'created_timestamp'], inplace=True) user_act = pd.DataFrame(data.groupby('user_id', as_index=False)[['shixun_id', 'created_timestamp']].\ agg({'shixun_id':np.size, 'created_timestamp': {list}}).values, \ columns=['user_id', 'select_size', 'created_timestamp']) # 计算时间间隔的均值 def time_diff_mean(l): if len(l) == 1: return 1 else: return np.mean([j-i for i, j in list(zip(l[:-1], l[1:]))]) user_act['user_time_diff_mean'] = user_act['created_timestamp'].progress_apply(lambda x: time_diff_mean(x)) # 选择次数取倒数 user_act['select_size'] = 1 / user_act['select_size'] # 两者归一化 user_act['select_size'] = (user_act['select_size'] - user_act['select_size'].min()) / \ (user_act['select_size'].max() - user_act['select_size'].min()) user_act['user_time_diff_mean'] = (user_act['user_time_diff_mean'] - user_act['user_time_diff_mean'].min()) / \ (user_act['user_time_diff_mean'].max() - user_act['user_time_diff_mean'].min()) user_act['active_level'] = user_act['select_size'] + user_act['user_time_diff_mean'] user_act['user_id'] = user_act['user_id'].astype('int') del user_act['created_timestamp'] user_act.to_csv(shixun_features_save_path + 'user_active_level.csv', index=False, header=True, sep='\t') return user_act def hot_level(all_data, cols): """ 生成物品热度的特征 根据物品选择时间和被选择物品的次数来衡量物品热度特征 如果物品在很短的时间间隔之内被选择了很多次，说明物品比较热门 1. 根据物品进行分组，对于每个物品的用户，计算选择的时间间隔 2. 将用户的数量取倒数，然后用户的数量和时间间隔归一化，相加得到热度特征 该值越小说明被选择的次数越大且时间间隔越短，物品比较热门 """ if os.path.exists(shixun_features_save_path + 'shixun_hot_level.csv'): shixun_hot = pd.read_csv(shixun_features_save_path + 'shixun_hot_level.csv', sep='\t', encoding='utf-8') return shixun_hot data = all_data[cols] data.sort_values(['shixun_id', 'created_timestamp'], inplace=True) shixun_hot = pd.DataFrame(data.groupby('shixun_id', as_index=False) \ [['user_id', 'created_timestamp']]. \ agg({'user_id': np.size, 'created_timestamp': {list}}).values, \ columns=['shixun_id', 'user_num', 'created_timestamp']) # 计算被选择时间间隔的均值 def time_diff_mean(l): if len(l) == 1: return 1 else: return np.mean([j-i for i, j in list(zip(l[:-1], l[1:]))]) shixun_hot['item_time_diff_mean'] = shixun_hot['created_timestamp']. \ progress_apply(lambda x: time_diff_mean(x)) # 选择次数取倒数 shixun_hot['user_num'] = 1 / shixun_hot['user_num'] # 两者归一化 shixun_hot['user_num'] = (shixun_hot['user_num'] - shixun_hot['user_num'].min()) /\ (shixun_hot['user_num'].max() - shixun_hot['user_num'].min()) shixun_hot['item_time_diff_mean'] = (shixun_hot['item_time_diff_mean'] - shixun_hot['item_time_diff_mean'].min()) /\ (shixun_hot['item_time_diff_mean'].max() - shixun_hot['item_time_diff_mean'].min()) shixun_hot['hot_level'] = shixun_hot['user_num'] + shixun_hot['item_time_diff_mean'] shixun_hot['shixun_id'] = shixun_hot['shixun_id'].astype('int') del shixun_hot['created_timestamp'] shixun_hot.to_csv(shixun_features_save_path + 'shixun_hot_level.csv', index=False, header=True, sep='\t') return shixun_hot def user_time_hob_fea(all_data, cols): """ 生成用户的时间习惯特征 根据用户选择的历史物品的时间做统计求均值 可以看出用户习惯一天什么时候选择物品 """ user_time_hob_info = all_data[cols] # 先把时间戳进行归一化 mm = MinMaxScaler() user_time_hob_info['created_timestamp'] = mm.fit_transform(user_time_hob_info[['created_timestamp']]) user_time_hob_info['created_at_ts'] = mm.fit_transform(user_time_hob_info[['created_at_ts']]) user_time_hob_info = user_time_hob_info.groupby('user_id').agg('mean').reset_index() user_time_hob_info.rename(columns={'created_timestamp': 'user_time_hob1', 'created_at_ts': 'user_time_hob2'}, inplace=True) return user_time_hob_info def user_trainee_hob_fea(all_data, cols): """ 用户的选择的实训难易爱好 根据用户选择的实训难易度转成一个列表 后面汇总的时候再单独制作一个特征，如果难度在这里面为1否则为0 """ user_category_hob_info = all_data[cols] user_category_hob_info['trainee'] = user_category_hob_info['trainee'].astype(str) user_category_hob_info = user_category_hob_info.groupby('user_id').agg({set}).reset_index() user_trainee_hob_info = pd.DataFrame() user_trainee_hob_info['user_id'] = user_category_hob_info['user_id'] user_trainee_hob_info['trainee_list'] = user_category_hob_info['trainee'] return user_trainee_hob_info def build_rank_features_engineering(): """ 排序模型特征工程 """ logger.info('获取训练验证测试数据集') # offline和online的区别就是验证集是否为空 # online时select_val, val_ans为空 select_train, select_val, select_test, val_ans = get_train_val_test_data(offline=offline_mode) logger.info('获取用户历史选择和最后一次选择') select_train_hist, select_train_last = get_hist_and_last_select(select_train) if select_val is not None: select_val_hist, select_val_last = select_val, val_ans else: select_val_hist, select_val_last = None, None select_test_hist = select_test # 读取离线召回数据 # 全量数据时只选择pinsage召回的结果 # 增量数据时选择多路召回合并的结果 logger.info('获取召回列表数据') recall_list_dict = get_recall_list(single_recall_model='pinsage', multi_recall=samples_mode) logger.info('召回数据转换成DataFrame...') recall_list_df = recall_dict_2_df(recall_list_dict) logger.info('给训练验证测试数据集打标签，负采样...') train_user_item_label_df, val_user_item_label_df, test_user_item_label_df=\ get_user_recall_item_label_df(select_train_hist, select_val_hist, select_test_hist, select_train_last, select_val_last, recall_list_df) logger.info('召回数据转换成字典') train_user_item_label_tuples = train_user_item_label_df.groupby('user_id'). \ progress_apply(make_tuple_func).reset_index() train_user_item_label_tuples_dict = dict(zip(train_user_item_label_tuples['user_id'], train_user_item_label_tuples[0])) if val_user_item_label_df is not None: val_user_item_label_tuples = val_user_item_label_df.groupby('user_id'). \ progress_apply(make_tuple_func).reset_index() val_user_item_label_tuples_dict = dict(zip(val_user_item_label_tuples['user_id'], val_user_item_label_tuples[0])) else: val_user_item_label_tuples_dict = None test_user_item_label_tuples = test_user_item_label_df.groupby('user_id'). \ progress_apply(make_tuple_func).reset_index() test_user_item_label_tuples_dict = dict(zip(test_user_item_label_tuples['user_id'], test_user_item_label_tuples[0])) logger.info("获取用户信息") users_info = get_user_info_df() # 用到的用户信息特征 users_info = users_info[['user_id', 'gender', 'school_id', 'identity','edu_background', 'logins', 'grade', 'experience']] logger.info('获取物品信息') shixun_info_df = get_item_info_df() # 用到的物品信息特征 shixun_info_df = shixun_info_df[['shixun_id', 'visits', 'trainee', 'myshixuns_count', 'challenges_count', 'averge_star','task_pass', 'created_at_ts']] logger.info('生成物品信息字典') shixun_info_dict = get_rank_item_info_dict(shixun_info_df) logger.info('获取物品向量化特征') item_word2vec_emb_dict, item_youtube_emb_dict,\ user_youtube_emb_dict,item_dssm_emb_dict,user_dssm_emb_dict = get_embedding() if os.path.exists(shixun_features_save_path + 'train_user_item_behavior_feats_df.csv'): train_user_item_feats_df = pd.read_csv(shixun_features_save_path + 'train_user_item_behavior_feats_df.csv', sep='\t', encoding='utf-8') reduce_mem(train_user_item_feats_df) else: logger.info('生成训练数据集中物品交叉特征') train_user_item_feats_df = create_behavior_feature(train_user_item_label_tuples_dict.keys(), train_user_item_label_tuples_dict, select_train_hist, shixun_info_df, shixun_info_dict, item_word2vec_emb_dict) train_user_item_feats_df.to_csv(shixun_features_save_path + 'train_user_item_behavior_feats_df.csv', sep='\t', index=False, header=True) reduce_mem(train_user_item_feats_df) if os.path.exists(shixun_features_save_path + 'val_user_item_behavior_feats_df.csv'): val_user_item_feats_df = pd.read_csv(shixun_features_save_path + 'val_user_item_behavior_feats_df.csv', sep='\t', encoding='utf-8') reduce_mem(val_user_item_feats_df) else: if val_user_item_label_tuples_dict is not None: logger.info('生成验证数据集中物品交叉特征') val_user_item_feats_df = create_behavior_feature(val_user_item_label_tuples_dict.keys(), val_user_item_label_tuples_dict, select_val_hist, shixun_info_df, shixun_info_dict, item_word2vec_emb_dict) else: val_user_item_feats_df = None if val_user_item_feats_df is not None: val_user_item_feats_df.to_csv(shixun_features_save_path + 'val_user_item_behavior_feats_df.csv', sep='\t', index=False, header=True) reduce_mem(val_user_item_feats_df) if os.path.exists(shixun_features_save_path + 'test_user_item_behavior_feats_df.csv'): test_user_item_feats_df = pd.read_csv(shixun_features_save_path + 'test_user_item_behavior_feats_df.csv', sep='\t', encoding='utf-8') reduce_mem(test_user_item_feats_df) else: logger.info('生成测试数据集中物品交叉特征') test_user_item_feats_df = create_behavior_feature(test_user_item_label_tuples_dict.keys(), test_user_item_label_tuples_dict, select_test_hist, shixun_info_df, shixun_info_dict, item_word2vec_emb_dict) test_user_item_feats_df.to_csv(shixun_features_save_path + 'test_user_item_behavior_feats_df.csv', sep='\t', index=False, header=True) reduce_mem(test_user_item_feats_df) # 物品行为数据，就是前面的所有数据 if select_val is not None: all_data = select_train.append(select_val) all_data = select_train.append(select_test) # 拼上物品信息 all_data = all_data.merge(shixun_info_df, on='shixun_id', how='left') logger.info('生成用户活跃度特征') user_act_fea = active_level(all_data, ['user_id', 'shixun_id', 'created_timestamp']) logger.info('生成物品热度特征') shixun_hot_fea = hot_level(all_data, ['user_id', 'shixun_id', 'created_timestamp']) # 用户时间特征 user_time_hob_cols = ['user_id', 'created_timestamp', 'created_at_ts'] user_time_hob_info = user_time_hob_fea(all_data, user_time_hob_cols) # 用户选择的实训难度特征 user_category_hob_cols = ['user_id', 'trainee'] user_trainee_hob_info = user_trainee_hob_fea(all_data, user_category_hob_cols) # 用户选择的实训访问次数特征 user_visits_count_info = all_data.groupby('user_id')['visits'].agg('mean').reset_index() user_visits_count_info.rename(columns={'visits': 'visits_hbo'}, inplace=True) # 用户选择的实训选用次数特征 user_myshixuns_count_info = all_data.groupby('user_id')['myshixuns_count'].agg('mean').reset_index() user_myshixuns_count_info.rename(columns={'myshixuns_count': 'myshixuns_hbo'}, inplace=True) # 用户选择的实训关卡数量特征 user_challenges_count_info = all_data.groupby('user_id')['challenges_count'].agg('mean').reset_index() user_challenges_count_info.rename(columns={'challenges_count': 'challenges_hbo'}, inplace=True) # 用户选择的实训评价星数特征 user_averge_star_info = all_data.groupby('user_id')['averge_star'].agg('mean').reset_index() user_averge_star_info.rename(columns={'averge_star': 'averge_star_hbo'}, inplace=True) # 用户选择的实训通过人数特征 user_task_pass_info = all_data.groupby('user_id')['task_pass'].agg('mean').reset_index() user_task_pass_info.rename(columns={'task_pass': 'task_pass_hbo'}, inplace=True) # 用户选择的实训数量特征 user_shixun_num_info = all_data.groupby('user_id')['shixun_id'].agg([('shixun_num', 'count')]).reset_index() user_shixun_num_info.rename(columns={'shixun_num': 'seq_length'}, inplace=True) logger.info('合并用户特征') user_features = pd.merge(user_act_fea, user_time_hob_info, on='user_id') user_features = user_features.merge(user_trainee_hob_info, on='user_id') user_features = user_features.merge(user_visits_count_info, on='user_id') user_features = user_features.merge(user_myshixuns_count_info, on='user_id') user_features = user_features.merge(user_challenges_count_info, on='user_id') user_features = user_features.merge(user_averge_star_info, on='user_id') user_features = user_features.merge(user_task_pass_info, on='user_id') user_features = user_features.merge(user_shixun_num_info, on='user_id') # 合并用户人口学统计特征 user_features = user_features.merge(users_info, on='user_id', how='left') logger.info('保存用户特征') user_features.to_csv(shixun_features_save_path + 'user_features_df.csv', sep='\t', header=True, index=False) logger.info('拼接用户特征') train_user_item_feats_df = train_user_item_feats_df.merge(user_features, on='user_id', how='left') if val_user_item_feats_df is not None: val_user_item_feats_df = val_user_item_feats_df.merge(user_features, on='user_id', how='left') else: val_user_item_feats_df = None test_user_item_feats_df = test_user_item_feats_df.merge(user_features, on='user_id',how='left') logger.info('拼接物品特征') train_user_item_feats_df = train_user_item_feats_df.merge(shixun_info_df, on='shixun_id', how='left') train_user_item_feats_df = train_user_item_feats_df.merge(shixun_hot_fea, on='shixun_id', how='left') if val_user_item_feats_df is not None: val_user_item_feats_df = val_user_item_feats_df.merge(shixun_info_df, on='shixun_id', how='left') val_user_item_feats_df = val_user_item_feats_df.merge(shixun_hot_fea, on='shixun_id', how='left') else: val_user_item_feats_df = None test_user_item_feats_df = test_user_item_feats_df.merge(shixun_info_df, on='shixun_id', how='left') test_user_item_feats_df = test_user_item_feats_df.merge(shixun_hot_fea, on='shixun_id', how='left') # 是否在用户选择的实训难度中 train_user_item_feats_df['is_trainee_hab'] = train_user_item_feats_df.progress_apply( lambda x: fill_is_trainee_hab(x), axis=1) if val_user_item_feats_df is not None: val_user_item_feats_df['is_trainee_hab'] = val_user_item_feats_df.progress_apply( lambda x: fill_is_trainee_hab(x), axis=1) else: val_user_item_feats_df = None test_user_item_feats_df['is_trainee_hab'] = test_user_item_feats_df.progress_apply( lambda x: fill_is_trainee_hab(x), axis=1) # 删除排序模型用不到的特征 del train_user_item_feats_df['trainee_list'] if val_user_item_feats_df is not None: del val_user_item_feats_df['trainee_list'] else: val_user_item_feats_df = None del test_user_item_feats_df['trainee_list'] logger.info('保存所有特征') train_user_item_feats_df.to_csv(shixun_train_user_item_feats, sep='\t', index=False, header=True) if val_user_item_feats_df is not None: val_user_item_feats_df.to_csv(shixun_val_user_item_feats, sep='\t', index=False, header=True) test_user_item_feats_df.to_csv(shixun_test_user_item_feats, sep='\t', index=False, header=True) all_user_item_feats_df = train_user_item_feats_df.append(test_user_item_feats_df) if val_user_item_feats_df is not None: all_user_item_feats_df = all_user_item_feats_df.append(val_user_item_feats_df) all_user_item_feats_df.to_csv(shixun_all_user_item_feats, sep='\t', index=False, header=True) if __name__ == '__main__': build_rank_features_engineering()