import os import sys sys.path.append(os.getcwd()) import pandas as pd import numpy as np from collections import defaultdict from tqdm import tqdm import math import warnings import pickle import collections from datetime import datetime from utils import get_file_size from config import logger from config import subject_itemcf_i2i_sim_data, need_metric_recall from config import subject_itemcf_recall_dict from config import subjects_embed_path, offline_mode from config import samples_mode from config import test_user_id from matching.subject.recall_comm import get_all_select_df from matching.subject.recall_comm import get_user_info_df, get_item_info_df from matching.subject.recall_comm import get_user_item_time_dict from matching.subject.recall_comm import get_hist_and_last_select, get_recall_item_info_dict from matching.subject.recall_comm import get_item_topk_select, metrics_recall from matching.subject.item_embedding_recall import embdding_i2i_sim tqdm.pandas() warnings.filterwarnings('ignore') def itemcf_sim(df, item_info_dict): """ 物品相似性矩阵计算，基于物品的协同过滤+关联规则 :param df: 物品行为数据 :param item_created_time_dict: 物品创建时间的字典 :return: 物品与物品的相似性矩阵 """ if os.path.exists(subject_itemcf_i2i_sim_data) and (get_file_size(subject_itemcf_i2i_sim_data) > 1): i2i_sim_ = pickle.load(open(subject_itemcf_i2i_sim_data, 'rb')) return i2i_sim_ # 获取用户选择的item字典 user_item_time_dict = get_user_item_time_dict(df) # 计算物品相似度 i2i_sim = {} item_cnt = defaultdict(int) for user_id, item_time_list in tqdm(user_item_time_dict.items()): # 在基于物品的协同过滤优化的时候考虑时间因素 for loc1, (item_i, i_select_time) in enumerate(item_time_list): item_cnt[item_i] += 1 i2i_sim.setdefault(item_i, {}) for loc2, (item_j, j_select_time) in enumerate(item_time_list): if (item_i == item_j) or \ (item_i not in item_info_dict) or \ (item_j not in item_info_dict): continue # 考虑物品的正向顺序选择和反向顺序选择 loc_alpha = 1.0 if loc2 > loc1 else 0.7 # 位置信息权重，其中的参数可以调节 loc_weight = loc_alpha * (0.9 ** (np.abs(loc2 - loc1) - 1)) # 选择时间权重，其中的参数可以调节 select_time_weight = np.exp(0.7 ** np.abs(i_select_time - j_select_time)) # 两个物品创建时间差的权重，其中的参数可以调节 item_i_created_time = item_info_dict[item_i]['created_at_ts'] item_j_created_time = item_info_dict[item_j]['created_at_ts'] created_time_weight = np.exp(0.8 ** np.abs(item_i_created_time - item_j_created_time)) i2i_sim[item_i].setdefault(item_j, 0) # 考虑多种因素的权重计算最终的物品之间的相似度 i2i_sim[item_i][item_j] += loc_weight * select_time_weight * created_time_weight / math.log(len(item_time_list) + 1) i2i_sim_ = i2i_sim.copy() # 余弦相似度分母部分计算 for item_i, related_items in i2i_sim.items(): for item_j, wij in related_items.items(): i2i_sim_[item_i][item_j] = wij / math.sqrt(item_cnt[item_i] * item_cnt[item_j]) # 将得到的相似性矩阵保存到本地 pickle.dump(i2i_sim_, open(subject_itemcf_i2i_sim_data, 'wb')) return i2i_sim_ # 基于物品协同过滤进行召回 def item_based_recommend(user_id, user_item_time_dict, i2i_sim, sim_item_topk, recall_item_num, item_topk_select, item_info_dict, emb_i2i_sim): """ 基于物品协同过滤+关联规则的召回 :param user_id: 用户id :param user_item_time_dict: 字典, 根据选择时间获取用户的选择物品序列 {user1: [(item1, time1), (item2, time2)..]...} :param i2i_sim: 字典，物品相似度矩阵 :param sim_item_topk: 整数，选择与当前物品最相似的k个物品 :param recall_item_num: 整数，需要召回的物品数量 :param item_topk_select: 列表，选择次数最多的物品列表，用于召回补全 :param item_info_dict: 字典，物品信息字典 :param emb_i2i_sim: 字典，物品embedding相似度矩阵 :return: 召回的物品列表 [(item1, score1), (item2, score2)...] """ # 获取用户历史选择的物品, 包括物品ID, 选择时间 user_hist_items = user_item_time_dict[user_id] # 获取用户历史选择的物品, 只包括物品ID user_hist_items_ = {item_id for item_id, _ in user_hist_items} filtered_item_name_list = [] filtered_item_name_list.clear() items_rank = {} for loc, (hist_item, select_time) in enumerate(user_hist_items): # 过滤物品信息字典中没有的物品 if hist_item not in item_info_dict: continue hist_item_created_time = item_info_dict[hist_item]['created_at_ts'] for sim_item, wij in sorted(i2i_sim[hist_item].items(), key=lambda x: x[1], reverse=True)[:sim_item_topk]: # 过滤历史选择的物品 if sim_item in user_hist_items_: continue # 过滤物品信息字典中没有的物品 if sim_item not in item_info_dict: continue sim_item_created_time = item_info_dict[sim_item]['created_at_ts'] sim_item_name = item_info_dict[sim_item]['subject_name'] # 过滤物品名称重复的 if sim_item_name in filtered_item_name_list: continue filtered_item_name_list.append(sim_item_name) # 物品创建时间差权重 created_time_weight = np.exp(0.8 ** np.abs(hist_item_created_time - sim_item_created_time)) # 相似物品和历史选择物品序列中历史物品所在的位置权重 loc_weight = (0.9 ** (len(user_hist_items) - loc)) content_weight = 1.0 if emb_i2i_sim.get(hist_item, {}).get(sim_item, None) is not None: content_weight += emb_i2i_sim[hist_item][sim_item] if emb_i2i_sim.get(sim_item, {}).get(hist_item, None) is not None: content_weight += emb_i2i_sim[sim_item][hist_item] items_rank.setdefault(sim_item, 0) items_rank[sim_item] += created_time_weight * loc_weight * content_weight * wij # 提前结束计算，没必要计算全部，否则在线召回将耗时很长 # 只计算最近选择的物品相似的物品。等于是捕捉用户最近的兴趣 if len(items_rank) >= (recall_item_num * 3): items_rank = sorted(items_rank.items(), key=lambda x: x[1], reverse=True)[:recall_item_num] return items_rank # 不足的用热门物品补全 if len(items_rank) < recall_item_num: for index, item in enumerate(item_topk_select): # 填充的item应该不在原来的列表中 if item in items_rank.items(): continue # 随便给个负数就行 items_rank[item] = - index - 100 # 达到召回的数量 if len(items_rank) == recall_item_num: break items_rank = sorted(items_rank.items(), key=lambda x: x[1], reverse=True)[:recall_item_num] return items_rank def init_itemcf_recall(): """ 初始化召回用到的一些数据 """ global train_hist_select_df global user_item_time_dict global i2i_sim, sim_item_topk global recall_item_num global item_topk_select global item_info_dict global emb_i2i_sim global train_last_select_df logger.info("加载物品行为数据") all_select_df = get_all_select_df(offline=False) logger.info("加载物品信息数据") item_info = get_item_info_df() logger.info("获取用户信息数据") users_info = get_user_info_df() all_select_df = all_select_df.merge(users_info, how='left', on='user_id') all_select_df = all_select_df.merge(item_info, on='subject_id') sim_item_topk = 120 recall_item_num = 100 logger.info('获取物品信息') item_info_df = get_item_info_df() logger.info('生成物品信息字典') item_info_dict = get_recall_item_info_dict(item_info_df) logger.info('生成itemcf相似度矩阵') i2i_sim = itemcf_sim(all_select_df, item_info_dict) logger.info('生成物品embedding相似度矩阵') item_emb_df = pd.read_csv(subjects_embed_path, sep='\t', encoding='utf-8') emb_i2i_sim = embdding_i2i_sim(item_emb_df, topk=recall_item_num) # 为了召回评估，提取最后一次选择作为召回评估 # 如果不需要做召回评估直接使用全量的训练集进行召 if need_metric_recall: logger.info('获取物品行为数据历史和最后一次选择') train_hist_select_df, train_last_select_df = get_hist_and_last_select(all_select_df) else: train_hist_select_df = all_select_df logger.info('获取用户选择的物品列表') user_item_time_dict = get_user_item_time_dict(train_hist_select_df) logger.info('获取选择次数最多的物品') item_topk_select = get_item_topk_select(train_hist_select_df, k=recall_item_num) def itemcf_recall(user_id, topk): """ itemcf召回调用接口 """ start_time = datetime.now() logger.info(f"本次需要进行itemcf召回的用户ID: {user_id}") recall_results = {} recall_results.clear() if user_id not in user_item_time_dict: return recall_results recall_results = item_based_recommend(user_id, user_item_time_dict, i2i_sim, topk + (topk // 3), topk, item_topk_select, item_info_dict, emb_i2i_sim) # 计算耗时毫秒 end_time = datetime.utcnow() cost_time_millisecond = round(float((end_time - start_time).microseconds / 1000.0), 3) logger.info(f"本次召回耗时: {cost_time_millisecond} 毫秒") return recall_results def itemcf_recall_train(): """ itemcf召回训练和评估 """ # 调用初始化召回用到的一些数据 init_itemcf_recall() # 只在采样模式下计算所有用户的召回数据并进行召回效果评估 # 如果用全量数据计算所有用户的召回数据会非常耗时 if samples_mode == True and need_metric_recall: # 定义召回物品的字典 user_recall_items_dict = collections.defaultdict(dict) logger.info('生成itemcf所有用户的召回列表') for user_id in tqdm(train_hist_select_df['user_id'].unique()): user_recall_items_dict[user_id] = item_based_recommend(user_id, user_item_time_dict, i2i_sim, sim_item_topk, recall_item_num, item_topk_select, item_info_dict, emb_i2i_sim) logger.info('保存itemcf召回结果') pickle.dump(user_recall_items_dict, open(subject_itemcf_recall_dict, 'wb')) logger.info('itemcf召回效果评估') metrics_recall(user_recall_items_dict, train_last_select_df, topk=recall_item_num) if __name__ == '__main__': itemcf_recall_train() recall_results = itemcf_recall(user_id=test_user_id, topk=20) print(recall_results)