|
|
import os
|
|
|
import sys
|
|
|
sys.path.append(os.getcwd())
|
|
|
import pandas as pd
|
|
|
import numpy as np
|
|
|
from collections import defaultdict
|
|
|
from tqdm import tqdm
|
|
|
import math
|
|
|
import warnings
|
|
|
import pickle
|
|
|
import collections
|
|
|
from datetime import datetime
|
|
|
from utils import get_file_size
|
|
|
from config import logger,offline_mode
|
|
|
from config import shixun_itemcf_i2i_sim_data, need_metric_recall
|
|
|
from config import shixun_itemcf_recall_dict
|
|
|
from config import shixun_merge_emb_path
|
|
|
from config import samples_mode
|
|
|
from config import test_user_id
|
|
|
from matching.shixun.recall_comm import get_all_select_df
|
|
|
from matching.shixun.recall_comm import get_user_info_df, get_item_info_df
|
|
|
from matching.shixun.recall_comm import get_user_item_time_dict,get_user_item_dict
|
|
|
from matching.shixun.recall_comm import get_cf_hist_and_last_select, get_item_info_dict
|
|
|
from matching.shixun.recall_comm import get_item_topk_select, metrics_recall
|
|
|
from matching.shixun.item_wordem_i2i_sim import embedding_i2i_sim
|
|
|
|
|
|
tqdm.pandas()
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
def itemcf_sim(df, item_created_time_dict):
|
|
|
"""
|
|
|
物品相似性矩阵计算，基于物品的协同过滤+关联规则
|
|
|
:param df: 物品行为数据
|
|
|
:param item_created_time_dict: 物品创建时间的字典
|
|
|
:return: 物品与物品的相似性矩阵
|
|
|
"""
|
|
|
if os.path.exists(shixun_itemcf_i2i_sim_data) and (get_file_size(shixun_itemcf_i2i_sim_data) > 1):
|
|
|
i2i_sim_ = pickle.load(open(shixun_itemcf_i2i_sim_data, 'rb'))
|
|
|
return i2i_sim_
|
|
|
|
|
|
print("获取用户选择的item字典属性")
|
|
|
user_item_dict = get_user_item_dict(df)
|
|
|
|
|
|
# 计算物品相似度
|
|
|
i2i_sim = {}
|
|
|
item_cnt = defaultdict(int)
|
|
|
|
|
|
for user_id, item_list in tqdm(user_item_dict.items()):
|
|
|
# 在基于物品的协同过滤优化的时候考虑时间因素
|
|
|
for loc1, (i, i_visits,i_challenges_count,i_avergestar,i_task,i_select_time) in enumerate(item_list):
|
|
|
item_cnt[i] += 1
|
|
|
i2i_sim.setdefault(i, {})
|
|
|
|
|
|
for loc2, (j, j_visits,j_challenges_count,j_avergestar,j_task,j_select_time) in enumerate(item_list):
|
|
|
if(i == j) or (i not in item_created_time_dict) or (j not in item_created_time_dict):
|
|
|
continue
|
|
|
|
|
|
# 考虑物品的正向顺序选择和反向顺序选择
|
|
|
loc_alpha = 1.0 if loc2 > loc1 else 0.7
|
|
|
|
|
|
# 位置信息权重，其中的参数可以调节
|
|
|
loc_weight = loc_alpha * (0.9 ** (np.abs(loc2 - loc1) - 1))
|
|
|
|
|
|
# 选择时间权重，其中的参数可以调节
|
|
|
select_time_weight = loc_alpha*(np.exp(0.7 ** np.abs(i_select_time - j_select_time)))
|
|
|
|
|
|
# 两个物品创建时间差的权重，其中的参数可以调节
|
|
|
created_time_weight = loc_alpha*(np.exp(0.8 ** np.abs(item_created_time_dict[i] - item_created_time_dict[j])))
|
|
|
|
|
|
#两个物品访问量差的权重，其中参数可以调节
|
|
|
visit_weight = loc_alpha*(np.exp(0.9 ** np.abs(i_visits - j_visits)))
|
|
|
|
|
|
#两个物品挑战数量差的权重，其中参数可以调节
|
|
|
challenges_weight = loc_alpha*(np.exp(0.5 ** np.abs(i_challenges_count - j_challenges_count)))
|
|
|
|
|
|
#两个物品平均星数量差的权重，其中参数可以调节
|
|
|
avergestar_weight = loc_alpha*(np.exp(0.8 ** np.abs(i_avergestar - j_avergestar)))
|
|
|
|
|
|
#两个物品通过数量差的权重，其中参数可以调节
|
|
|
task_weight = loc_alpha*(np.exp(0.5 ** np.abs(i_task - j_task)))
|
|
|
|
|
|
i2i_sim[i].setdefault(j, 0)
|
|
|
|
|
|
# 考虑多种因素的权重计算最终的物品之间的相似度
|
|
|
i2i_sim[i][j] += loc_weight * select_time_weight * created_time_weight*visit_weight*challenges_weight*avergestar_weight*task_weight/ math.log(len(item_list) + 1)
|
|
|
|
|
|
i2i_sim_ = i2i_sim.copy()
|
|
|
|
|
|
# 余弦相似度分母部分计算
|
|
|
for i, related_items in i2i_sim.items():
|
|
|
for j, wij in related_items.items():
|
|
|
i2i_sim_[i][j] = wij / math.sqrt(item_cnt[i] * item_cnt[j])
|
|
|
|
|
|
# 将得到的相似性矩阵保存到本地
|
|
|
pickle.dump(i2i_sim_, open(shixun_itemcf_i2i_sim_data, 'wb'))
|
|
|
|
|
|
return i2i_sim_
|
|
|
|
|
|
|
|
|
# 基于物品协同过滤进行召回
|
|
|
def item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
sim_item_topk,
|
|
|
recall_item_num,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim):
|
|
|
"""
|
|
|
基于物品协同过滤+关联规则的召回
|
|
|
:param user_id: 用户id
|
|
|
:param user_item_time_dict: 字典, 根据选择时间获取用户的选择物品序列 {user1: [(item1, time1), (item2, time2)..]...}
|
|
|
:param i2i_sim: 字典，物品相似度矩阵
|
|
|
:param sim_item_topk: 整数，选择与当前物品最相似的k个物品
|
|
|
:param recall_item_num: 整数，需要召回的物品数量
|
|
|
:param item_topk_select: 列表，选择次数最多的物品列表，用于召回补全
|
|
|
:param item_created_time_dic: 字典，物品创建时间
|
|
|
:param emb_i2i_sim: 字典，物品embedding相似度矩阵
|
|
|
|
|
|
:return: 召回的物品列表 [(item1, score1), (item2, score2)...]
|
|
|
"""
|
|
|
# 获取用户历史选择的物品
|
|
|
user_hist_items = user_item_time_dict[user_id]
|
|
|
user_hist_items_ = {item_id for item_id, _ in user_hist_items}
|
|
|
|
|
|
item_rank = {}
|
|
|
for loc, (i, select_time) in enumerate(user_hist_items):
|
|
|
for j, wij in sorted(i2i_sim[i].items(), key=lambda x: x[1], reverse=True)[:sim_item_topk]:
|
|
|
if j in user_hist_items_:
|
|
|
continue
|
|
|
|
|
|
# 物品创建时间差权重
|
|
|
created_time_weight = np.exp(0.8 ** np.abs(item_created_time_dict[i] - item_created_time_dict[j]))
|
|
|
|
|
|
# 相似物品和历史选择物品序列中历史物品所在的位置权重
|
|
|
loc_weight = (0.9 ** (len(user_hist_items) - loc))
|
|
|
|
|
|
content_weight = 1.0
|
|
|
if emb_i2i_sim.get(i, {}).get(j, None) is not None:
|
|
|
content_weight += emb_i2i_sim[i][j]
|
|
|
|
|
|
if emb_i2i_sim.get(j, {}).get(i, None) is not None:
|
|
|
content_weight += emb_i2i_sim[j][i]
|
|
|
|
|
|
item_rank.setdefault(j, 0)
|
|
|
item_rank[j] += created_time_weight * loc_weight * content_weight * wij
|
|
|
|
|
|
# 不足的用热门物品补全
|
|
|
if len(item_rank) < recall_item_num:
|
|
|
for i, item in enumerate(item_topk_select):
|
|
|
|
|
|
# 填充的item应该不在原来的列表中
|
|
|
if item in item_rank.items():
|
|
|
continue
|
|
|
|
|
|
# 随便给个负数就行
|
|
|
item_rank[item] = - i - 100
|
|
|
|
|
|
# 达到召回的数量
|
|
|
if len(item_rank) == recall_item_num:
|
|
|
break
|
|
|
|
|
|
item_rank = sorted(item_rank.items(), key=lambda x: x[1], reverse=True)[:recall_item_num]
|
|
|
|
|
|
return item_rank
|
|
|
|
|
|
|
|
|
def init_itemcf_recall():
|
|
|
"""
|
|
|
初始化召回用到的一些数据
|
|
|
"""
|
|
|
global train_hist_select_df
|
|
|
global user_item_time_dict
|
|
|
global i2i_sim, sim_item_topk
|
|
|
global recall_item_num
|
|
|
global item_topk_select
|
|
|
global item_created_time_dict
|
|
|
global emb_i2i_sim
|
|
|
global train_last_select_df
|
|
|
|
|
|
logger.info("加载物品行为数据")
|
|
|
all_select_df = get_all_select_df(offline=offline_mode)
|
|
|
|
|
|
logger.info("加载物品属性数据")
|
|
|
item_info_df = get_item_info_df()
|
|
|
|
|
|
logger.info("获取用户信息数据")
|
|
|
users_info_df = get_user_info_df()
|
|
|
|
|
|
|
|
|
all_select_df = all_select_df.merge(users_info_df, on='user_id')
|
|
|
all_select_df = all_select_df.merge(item_info_df, on='shixun_id')
|
|
|
|
|
|
sim_item_topk = 120
|
|
|
recall_item_num = 100
|
|
|
|
|
|
logger.info('生成物品信息字典')
|
|
|
item_visists_dict, item_trainee_dict, item_created_time_dict, \
|
|
|
item_averge_star_dict, item_myshixuns_count_dict, item_challenges_count_dict = \
|
|
|
get_item_info_dict(item_info_df)
|
|
|
|
|
|
logger.info('生成itemcf相似度矩阵')
|
|
|
i2i_sim = itemcf_sim(all_select_df, item_created_time_dict)
|
|
|
|
|
|
logger.info('生成物品embedding相似度矩阵')
|
|
|
item_emb_df = pd.read_csv(shixun_merge_emb_path, sep='\t', encoding='utf-8')
|
|
|
emb_i2i_sim = embedding_i2i_sim(item_emb_df, topk=recall_item_num)
|
|
|
|
|
|
# 为了召回评估，提取最后一次选择作为召回评估
|
|
|
# 如果不需要做召回评估直接使用全量的训练集进行召
|
|
|
|
|
|
if need_metric_recall:
|
|
|
logger.info('获取物品行为数据历史和最后一次选择')
|
|
|
train_hist_select_df, train_last_select_df = get_cf_hist_and_last_select(all_select_df)
|
|
|
else:
|
|
|
train_hist_select_df = all_select_df
|
|
|
|
|
|
logger.info('获取用户选择的物品列表')
|
|
|
user_item_time_dict = get_user_item_time_dict(train_hist_select_df)
|
|
|
|
|
|
logger.info('获取选择次数最多的物品')
|
|
|
item_topk_select = get_item_topk_select(train_hist_select_df, k=recall_item_num)
|
|
|
|
|
|
|
|
|
def itemcf_recall(user_id, topk):
|
|
|
"""
|
|
|
itemcf召回调用接口
|
|
|
"""
|
|
|
start_time = datetime.now()
|
|
|
|
|
|
logger.info(f"本次需要进行itemcf召回的用户ID: {user_id}")
|
|
|
|
|
|
recall_results = {}
|
|
|
recall_results.clear()
|
|
|
|
|
|
if user_id not in user_item_time_dict:
|
|
|
return recall_results
|
|
|
|
|
|
recall_results = item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
topk + (topk // 2),
|
|
|
topk,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim)
|
|
|
|
|
|
# 计算耗时毫秒
|
|
|
end_time = datetime.utcnow()
|
|
|
cost_time_millisecond = round(float((end_time - start_time).microseconds / 1000.0), 3)
|
|
|
|
|
|
logger.info(f"本次召回耗时: {cost_time_millisecond} 毫秒")
|
|
|
|
|
|
return recall_results
|
|
|
|
|
|
|
|
|
def itemcf_recall_train():
|
|
|
"""
|
|
|
itemcf召回训练和评估
|
|
|
"""
|
|
|
# 调用初始化召回用到的一些数据
|
|
|
init_itemcf_recall()
|
|
|
|
|
|
# 只在采样模式下计算所有用户的召回数据并进行召回效果评估
|
|
|
# 如果用全量数据计算所有用户的召回数据会非常耗时
|
|
|
|
|
|
if samples_mode == True and need_metric_recall:
|
|
|
|
|
|
# 定义召回物品的字典
|
|
|
user_recall_items_dict = collections.defaultdict(dict)
|
|
|
|
|
|
logger.info('生成itemcf所有用户的召回列表')
|
|
|
for user_id in tqdm(train_hist_select_df['user_id'].unique()):
|
|
|
|
|
|
try:
|
|
|
user_recall_items_dict[user_id] = item_based_recommend(user_id,
|
|
|
user_item_time_dict,
|
|
|
i2i_sim,
|
|
|
sim_item_topk,
|
|
|
recall_item_num,
|
|
|
item_topk_select,
|
|
|
item_created_time_dict,
|
|
|
emb_i2i_sim)
|
|
|
except:
|
|
|
continue
|
|
|
logger.info('保存itemcf召回结果')
|
|
|
pickle.dump(user_recall_items_dict, open(shixun_itemcf_recall_dict, 'wb'))
|
|
|
|
|
|
logger.info('itemcf召回效果评估')
|
|
|
metrics_recall(user_recall_items_dict, train_last_select_df, topk=recall_item_num)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
itemcf_recall_train()
|
|
|
recall_results = itemcf_recall(user_id=test_user_id, topk=20)
|
|
|
print(recall_results) |
