|
|
import os
|
|
|
import sys
|
|
|
sys.path.append(os.getcwd())
|
|
|
import pandas as pd
|
|
|
import numpy as np
|
|
|
from tqdm import tqdm
|
|
|
import faiss
|
|
|
import warnings
|
|
|
import pickle
|
|
|
import collections
|
|
|
from datetime import datetime
|
|
|
from config import logger,offline_mode
|
|
|
from config import need_metric_recall
|
|
|
from config import shixun_emb_i2i_sim_data
|
|
|
from config import shixun_item_embedding_recall_dict
|
|
|
from config import shixun_merge_emb_path
|
|
|
from config import samples_mode
|
|
|
from config import test_user_id
|
|
|
from matching.shixun.recall_comm import get_all_select_df
|
|
|
from matching.shixun.recall_comm import get_item_info_df
|
|
|
from matching.shixun.recall_comm import get_user_item_time_dict
|
|
|
from matching.shixun.recall_comm import get_hist_and_last_select, get_item_info_dict
|
|
|
from matching.shixun.recall_comm import get_item_topk_select,get_user_info_df
|
|
|
from matching.shixun.recall_comm import metrics_recall
|
|
|
from utils import get_file_size
|
|
|
|
|
|
|
|
|
tqdm.pandas()
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
|
|
|
def embdding_i2i_sim(item_emb_df, topk):
|
|
|
"""
|
|
|
基于物品embedding计算相似性矩阵，返回topk个与其最相似的物品
|
|
|
"""
|
|
|
# 加载之前保存的
|
|
|
if os.path.exists(shixun_emb_i2i_sim_data) and (get_file_size(shixun_emb_i2i_sim_data) > 1):
|
|
|
item_sim_dict = pickle.load(open(shixun_emb_i2i_sim_data, 'rb'))
|
|
|
return item_sim_dict
|
|
|
|
|
|
# 物品索引与物品id的字典映射
|
|
|
item_idx_2_rawid_dict = dict(zip(item_emb_df.index, item_emb_df['shixun_id']))
|
|
|
|
|
|
item_emb_cols = [x for x in item_emb_df.columns if 'shixun_id' not in x]
|
|
|
item_emb_np = np.ascontiguousarray(item_emb_df[item_emb_cols].values, dtype=np.float32)
|
|
|
|
|
|
# 向量进行单位化
|
|
|
item_emb_np = item_emb_np / np.linalg.norm(item_emb_np, axis=1, keepdims=True)
|
|
|
|
|
|
# 建立faiss索引
|
|
|
item_index = faiss.IndexFlatIP(item_emb_np.shape[1])
|
|
|
item_index.add(item_emb_np)
|
|
|
|
|
|
# 相似度查询，给每个索引位置上的向量返回topk个item以及相似度
|
|
|
sim, idx = item_index.search(item_emb_np, topk) # 返回的是列表
|
|
|
|
|
|
# 将向量检索的结果保存成原始id的对应关系
|
|
|
item_sim_dict = collections.defaultdict(dict)
|
|
|
for target_idx, sim_value_list, rele_idx_list in tqdm(zip(range(len(item_emb_np)), sim, idx)):
|
|
|
target_raw_id = item_idx_2_rawid_dict[target_idx]
|
|
|
|
|
|
# 从1开始是为了去掉物品本身, 所以最终获得的相似物品只有topk-1
|
|
|
for rele_idx, sim_value in zip(rele_idx_list[1:], sim_value_list[1:]):
|
|
|
rele_raw_id = item_idx_2_rawid_dict[rele_idx]
|
|
|
item_sim_dict[target_raw_id][rele_raw_id] = item_sim_dict.get(target_raw_id, {}).get(rele_raw_id, 0) + sim_value
|
|
|
|
|
|
# 保存embedding i2i相似度矩阵
|
|
|
logger.info("保存embedding i2i相似度矩阵")
|
|
|
pickle.dump(item_sim_dict, open(shixun_emb_i2i_sim_data, 'wb'))
|
|
|
|
|
|
|
|
|
return item_sim_dict
|
|
|
|
|
|
# 基于物品的召回i2i
|
|
|
def item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
sim_item_topk,
|
|
|
recall_item_num,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim):
|
|
|
"""
|
|
|
基于物品协同过滤的召回
|
|
|
:param user_id: 用户id
|
|
|
:param user_item_time_dict: 字典, 根据选择时间获取用户的选择物品序列 {user1: [(item1, time1), (item2, time2)..]...}
|
|
|
:param i2i_sim: 字典，物品相似性矩阵
|
|
|
:param sim_item_topk: 整数，选择与当前物品最相似的k个物品
|
|
|
:param recall_item_num: 整数，最后召回的物品数量
|
|
|
:param item_topk_select: 列表，选择次数最多的物品列表，用于召回补全
|
|
|
:param item_created_time_dict: 字典，物品创建的时间
|
|
|
:param emb_i2i_sim: 物品embedding相似度矩阵
|
|
|
|
|
|
:return: 召回的物品列表 [(item1, score1), (item2, score2)...]
|
|
|
"""
|
|
|
# 获取用户历史交互的物品
|
|
|
user_hist_items = user_item_time_dict[user_id]
|
|
|
user_hist_items_ = {user_id for user_id, _ in user_hist_items}
|
|
|
|
|
|
item_rank = {}
|
|
|
for loc, (i, select_time) in enumerate(user_hist_items):
|
|
|
for j, wij in sorted(i2i_sim[i].items(), key=lambda x: x[1], reverse=True)[:sim_item_topk]:
|
|
|
if j in user_hist_items_:
|
|
|
continue
|
|
|
|
|
|
# 物品创建时间差权重
|
|
|
created_time_weight = np.exp(0.8 ** np.abs(item_created_time_dict[i] - item_created_time_dict[j]))
|
|
|
|
|
|
# 相似物品和历史选择物品序列中历史物品所在的位置权重
|
|
|
loc_weight = (0.9 ** (len(user_hist_items) - loc))
|
|
|
|
|
|
content_weight = 1.0
|
|
|
|
|
|
if emb_i2i_sim.get(i, {}).get(j, None) is not None:
|
|
|
content_weight += emb_i2i_sim[i][j]
|
|
|
|
|
|
if emb_i2i_sim.get(j, {}).get(i, None) is not None:
|
|
|
content_weight += emb_i2i_sim[j][i]
|
|
|
|
|
|
item_rank.setdefault(j, 0)
|
|
|
item_rank[j] += created_time_weight * loc_weight * content_weight * wij
|
|
|
|
|
|
# 不足的，用热门物品补全
|
|
|
if len(item_rank) < recall_item_num:
|
|
|
for i, item in enumerate(item_topk_select):
|
|
|
|
|
|
# 填充的item应该不在原来的列表中
|
|
|
if item in item_rank.items():
|
|
|
continue
|
|
|
|
|
|
# 随便给个负数就行
|
|
|
item_rank[item] = - i - 100
|
|
|
if len(item_rank) == recall_item_num:
|
|
|
break
|
|
|
|
|
|
item_rank = sorted(item_rank.items(), key=lambda x: x[1], reverse=True)[:recall_item_num]
|
|
|
|
|
|
return item_rank
|
|
|
|
|
|
|
|
|
def init_item_embedding_recall():
|
|
|
"""
|
|
|
初始化召回用到的一些数据
|
|
|
"""
|
|
|
global train_hist_select_df
|
|
|
global user_item_time_dict
|
|
|
global i2i_sim, sim_item_topk
|
|
|
global recall_item_num
|
|
|
global item_topk_select
|
|
|
global item_created_time_dict
|
|
|
global emb_i2i_sim
|
|
|
global train_last_select_df
|
|
|
|
|
|
|
|
|
logger.info("加载物品行为数据")
|
|
|
all_select_df = get_all_select_df(offline=offline_mode)
|
|
|
|
|
|
logger.info("加载物品属性数据")
|
|
|
item_info_df = get_item_info_df()
|
|
|
|
|
|
logger.info("获取用户信息数据")
|
|
|
users_info_df = get_user_info_df()
|
|
|
|
|
|
|
|
|
all_select_df = all_select_df.merge(users_info_df, on='user_id')
|
|
|
all_select_df = all_select_df.merge(item_info_df, on='shixun_id')
|
|
|
|
|
|
sim_item_topk = 120
|
|
|
recall_item_num = 100
|
|
|
|
|
|
logger.info('获取物品基本信息')
|
|
|
item_info_df = get_item_info_df()
|
|
|
|
|
|
logger.info('生成物品信息字典')
|
|
|
item_visists_dict, item_trainee_dict, item_created_time_dict, \
|
|
|
item_averge_star_dict, item_myshixuns_count_dict, item_challenges_count_dict = \
|
|
|
get_item_info_dict(item_info_df)
|
|
|
|
|
|
logger.info('生成物品embedding相似度矩阵')
|
|
|
item_emb_df = pd.read_csv(shixun_merge_emb_path, sep='\t', encoding='utf-8')
|
|
|
emb_i2i_sim = embdding_i2i_sim(item_emb_df, topk=recall_item_num)
|
|
|
|
|
|
# 为了召回评估，提取最后一次选择作为召回评估
|
|
|
# 如果不需要做召回评估直接使用全量的训练集进行召回
|
|
|
if samples_mode and need_metric_recall:
|
|
|
logger.info('获取物品行为数据历史和最后一次选择')
|
|
|
train_hist_select_df, train_last_select_df = get_hist_and_last_select(all_select_df)
|
|
|
else:
|
|
|
train_hist_select_df = all_select_df
|
|
|
|
|
|
logger.info('获取用户选择物品列表')
|
|
|
user_item_time_dict = get_user_item_time_dict(train_hist_select_df)
|
|
|
|
|
|
# 加载物品embedding相似度矩阵
|
|
|
i2i_sim = pickle.load(open(shixun_emb_i2i_sim_data, 'rb'))
|
|
|
|
|
|
logger.info('获取选择次数最多的物品')
|
|
|
item_topk_select = get_item_topk_select(train_hist_select_df, k=recall_item_num)
|
|
|
|
|
|
|
|
|
def item_embedding_recall(user_id, topk):
|
|
|
"""
|
|
|
item embedding召回调用接口
|
|
|
"""
|
|
|
start_time = datetime.now()
|
|
|
|
|
|
logger.info(f"本次需要item embedding召回的用户ID: {user_id}")
|
|
|
|
|
|
recall_results = {}
|
|
|
recall_results.clear()
|
|
|
|
|
|
if user_id not in user_item_time_dict:
|
|
|
return recall_results
|
|
|
|
|
|
recall_results = item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
topk + (topk // 2),
|
|
|
topk,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim)
|
|
|
|
|
|
# 计算耗时毫秒
|
|
|
end_time = datetime.utcnow()
|
|
|
cost_time_millisecond = round(float((end_time - start_time).microseconds / 1000.0), 3)
|
|
|
|
|
|
logger.info(f"本次召回耗时: {cost_time_millisecond} 毫秒")
|
|
|
|
|
|
return recall_results
|
|
|
|
|
|
|
|
|
def item_embedding_recall_train():
|
|
|
# 调用初始化召回用到的一些数据
|
|
|
init_item_embedding_recall()
|
|
|
|
|
|
# 只在采样模式下计算所有用户的召回数据并进行召回效果评估
|
|
|
# 如果用全量数据计算所有用户的召回数据会非常耗时
|
|
|
|
|
|
if samples_mode == True and need_metric_recall:
|
|
|
|
|
|
# 定义召回物品的字典
|
|
|
user_recall_items_dict = collections.defaultdict(dict)
|
|
|
|
|
|
logger.info('生成item embedding所有用户的召回列表')
|
|
|
for user_id in tqdm(train_hist_select_df['user_id'].unique()):
|
|
|
user_recall_items_dict[user_id] = item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
sim_item_topk,
|
|
|
recall_item_num,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim)
|
|
|
|
|
|
logger.info('保存item embedding召回结果')
|
|
|
pickle.dump(user_recall_items_dict, open(shixun_item_embedding_recall_dict, 'wb'))
|
|
|
|
|
|
logger.info('item embedding召回效果评估')
|
|
|
metrics_recall(user_recall_items_dict, train_last_select_df, topk=recall_item_num)
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
item_embedding_recall_train()
|
|
|
|
|
|
recall_results = item_embedding_recall(user_id=test_user_id, topk=20)
|
|
|
print(recall_results) |
