|
|
import pandas as pd
|
|
|
import numpy as np
|
|
|
from tqdm import tqdm
|
|
|
import os
|
|
|
import sys
|
|
|
sys.path.append(os.getcwd())
|
|
|
import warnings
|
|
|
import pickle
|
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
|
from config import mysubjects_train_data, mysubjects_test_data
|
|
|
from config import subjects_data_path, subjects_embed_path, subject_save_path
|
|
|
from config import subject_user_item_time_dict_data
|
|
|
from config import subject_item_user_time_dict, data_parent_path
|
|
|
from config import subjects_emb_dict
|
|
|
from config import users_data_path
|
|
|
from utils import reduce_mem
|
|
|
from config import logger
|
|
|
from joblib import Parallel, delayed
|
|
|
from data_process import process_users_data_null
|
|
|
|
|
|
|
|
|
tqdm.pandas()
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
def get_all_select_sample(sample_nums=10000):
|
|
|
"""
|
|
|
从训练集中划出一部分数据来调试代码
|
|
|
"""
|
|
|
if os.path.exists(data_parent_path + 'mysubjects_train_sample.csv'):
|
|
|
all_select = pd.read_csv(data_parent_path + 'mysubjects_train_sample.csv', sep='\t', encoding='utf-8')
|
|
|
reduce_mem(all_select)
|
|
|
return all_select
|
|
|
|
|
|
all_select = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')
|
|
|
|
|
|
# user_id过滤重复
|
|
|
all_user_ids = all_select.user_id.unique()
|
|
|
|
|
|
# 只采集指定数量的user_id
|
|
|
sample_user_ids = np.random.choice(all_user_ids, size=sample_nums, replace=False)
|
|
|
|
|
|
# 取出这些user_id选择的课程
|
|
|
all_select = all_select[all_select['user_id'].isin(sample_user_ids)]
|
|
|
|
|
|
# 根据user_id, subject_id去重
|
|
|
all_select = all_select.drop_duplicates((['user_id', 'subject_id']))
|
|
|
|
|
|
all_select.to_csv(data_parent_path + 'mysubjects_train_sample.csv', sep='\t', index=False, header=True)
|
|
|
reduce_mem(all_select)
|
|
|
return all_select
|
|
|
|
|
|
|
|
|
def get_all_select_df(offline = False):
|
|
|
"""
|
|
|
读取课程数据时分成线上和线下
|
|
|
线上预测时将测试集中的课程数据合并到总的数据中
|
|
|
线下验证模型的有效性或者特征的有效性时只使用训练集
|
|
|
"""
|
|
|
if offline:
|
|
|
all_select = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')
|
|
|
else:
|
|
|
train_mysubjects = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')
|
|
|
|
|
|
test_mysubjects = pd.read_csv(mysubjects_test_data, sep='\t', encoding='utf-8')
|
|
|
|
|
|
all_select = train_mysubjects.append(test_mysubjects)
|
|
|
|
|
|
all_select = all_select.drop_duplicates((['user_id', 'subject_id']))
|
|
|
reduce_mem(all_select)
|
|
|
return all_select
|
|
|
|
|
|
|
|
|
def get_user_info_df():
|
|
|
"""
|
|
|
读取用户的基本属性
|
|
|
"""
|
|
|
user_info_df = pd.read_csv(users_data_path, sep='\t', encoding='utf-8', quoting=3)
|
|
|
user_info_df = process_users_data_null(user_info_df)
|
|
|
reduce_mem(user_info_df)
|
|
|
return user_info_df
|
|
|
|
|
|
|
|
|
def get_item_info_df():
|
|
|
"""
|
|
|
读取课程的基本属性
|
|
|
"""
|
|
|
item_info_df = pd.read_csv(subjects_data_path, sep='\t', encoding='utf-8')
|
|
|
reduce_mem(item_info_df)
|
|
|
|
|
|
return item_info_df
|
|
|
|
|
|
|
|
|
def get_recall_item_info_df(item_info_df, subject_id_list):
|
|
|
"""
|
|
|
读取课程的基本属性
|
|
|
"""
|
|
|
recall_item_info_df = item_info_df[item_info_df['subject_id'].isin(subject_id_list)]
|
|
|
return recall_item_info_df
|
|
|
|
|
|
|
|
|
def get_select_item_info(all_select_df, user_id):
|
|
|
"""
|
|
|
获取某个用户选择的课程行为数据
|
|
|
"""
|
|
|
# select_item_df = all_select_df[all_select_df['user_id'] == user_id]
|
|
|
|
|
|
# 用loc会快一些
|
|
|
select_item_df = all_select_df.loc[all_select_df['user_id'] == user_id, :].reset_index()
|
|
|
|
|
|
return select_item_df
|
|
|
|
|
|
|
|
|
def get_item_emb_dict():
|
|
|
"""
|
|
|
生成和读取课程的Embedding数据
|
|
|
"""
|
|
|
# 加载已经保存的Embedding数据
|
|
|
if os.path.exists(subjects_emb_dict):
|
|
|
item_emb_dict = pickle.load(open(subjects_emb_dict, 'rb'))
|
|
|
return item_emb_dict
|
|
|
|
|
|
# 生成课程的Embedding数据
|
|
|
item_emb_df = pd.read_csv(subjects_embed_path, sep='\t', encoding='utf-8')
|
|
|
|
|
|
item_emb_cols = [x for x in item_emb_df.columns if 'emb' in x]
|
|
|
item_emb_np = np.ascontiguousarray(item_emb_df[item_emb_cols])
|
|
|
# 进行归一化
|
|
|
item_emb_np = item_emb_np / np.linalg.norm(item_emb_np, axis=1, keepdims=True)
|
|
|
|
|
|
item_emb_dict = dict(zip(item_emb_df['subject_id'], item_emb_np))
|
|
|
pickle.dump(item_emb_dict, open(subjects_emb_dict, 'wb'))
|
|
|
|
|
|
return item_emb_dict
|
|
|
|
|
|
|
|
|
def get_user_item_time_dict(select_df):
|
|
|
"""
|
|
|
根据时间获取用户选择的课程序列
|
|
|
{user1: [(item1, time1), (item2, time2)..]...}
|
|
|
"""
|
|
|
def make_item_time_pair(df):
|
|
|
"""
|
|
|
构造选课程，选择时间列表
|
|
|
"""
|
|
|
return list(zip(df['subject_id'], df['created_timestamp']))
|
|
|
|
|
|
# 加载之前pickel保存的
|
|
|
if os.path.exists(subject_user_item_time_dict_data):
|
|
|
user_item_time_dict = pickle.load(open(subject_user_item_time_dict_data, 'rb'))
|
|
|
return user_item_time_dict
|
|
|
|
|
|
# 按选择时间排序
|
|
|
select_df = select_df.sort_values('created_timestamp')
|
|
|
|
|
|
# 按用户分组生成用户选择的课程
|
|
|
user_item_time_df = select_df.groupby('user_id')['subject_id', 'created_timestamp'].progress_apply(lambda x: make_item_time_pair(x)).reset_index(). \
|
|
|
rename(columns = {0: 'item_time_list'})
|
|
|
|
|
|
# 生成用户选择的课程字典
|
|
|
user_item_time_dict = dict(zip(user_item_time_df['user_id'], user_item_time_df['item_time_list']))
|
|
|
|
|
|
pickle.dump(user_item_time_dict, open(subject_user_item_time_dict_data, 'wb'))
|
|
|
|
|
|
return user_item_time_dict
|
|
|
|
|
|
|
|
|
def get_item_user_time_dict(select_df):
|
|
|
"""
|
|
|
根据选择时间获取课程被选择的用户序列
|
|
|
{item1: [(user1, time1), (user2, time2)...]...}
|
|
|
"""
|
|
|
def make_user_time_pair(df):
|
|
|
return list(zip(df['user_id'], df['created_timestamp']))
|
|
|
|
|
|
# 加载之前pickel保存的
|
|
|
if os.path.exists(subject_item_user_time_dict):
|
|
|
item_user_time_dict = pickle.load(open(subject_item_user_time_dict, 'rb'))
|
|
|
return item_user_time_dict
|
|
|
|
|
|
select_df = select_df.sort_values('created_timestamp')
|
|
|
|
|
|
item_user_time_df = select_df.groupby('subject_id')['user_id', 'created_timestamp']. \
|
|
|
progress_apply(lambda x: make_user_time_pair(x)).reset_index(). \
|
|
|
rename(columns = {0: 'user_time_list'})
|
|
|
|
|
|
item_user_time_dict = dict(zip(item_user_time_df['subject_id'], item_user_time_df['user_time_list']))
|
|
|
pickle.dump(item_user_time_dict, open(subject_item_user_time_dict, 'wb'))
|
|
|
|
|
|
return item_user_time_dict
|
|
|
|
|
|
|
|
|
def get_hist_and_last_select(all_select):
|
|
|
"""
|
|
|
获取课程行为数据的历史选择和最后一次选择
|
|
|
"""
|
|
|
# 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
|
|
|
def hist_func(user_df):
|
|
|
if len(user_df) == 1:
|
|
|
return user_df
|
|
|
else:
|
|
|
return user_df[:-1]
|
|
|
|
|
|
def apply_parallel(df_grouped, func):
|
|
|
results = Parallel(n_jobs=20)(delayed(func)(group) for name, group in df_grouped)
|
|
|
return pd.concat(results)
|
|
|
|
|
|
if os.path.exists(subject_save_path + 'select_last_df.csv') and \
|
|
|
os.path.exists(subject_save_path + 'select_hist_df.csv'):
|
|
|
select_last_df = pd.read_csv(subject_save_path + 'select_last_df.csv', sep='\t', encoding='utf-8')
|
|
|
select_hist_df = pd.read_csv(subject_save_path + 'select_hist_df.csv', sep='\t', encoding='utf-8')
|
|
|
else:
|
|
|
all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
|
|
|
select_last_df = all_select.groupby('user_id').tail(1)
|
|
|
|
|
|
# 多进程并行提升处理速度
|
|
|
df_grouped = all_select.groupby('user_id')
|
|
|
select_hist_df = apply_parallel(df_grouped, hist_func)
|
|
|
select_hist_df = select_hist_df.reset_index(drop=True)
|
|
|
|
|
|
#select_hist_df = all_select.groupby('user_id').progress_apply(hist_func).reset_index(drop = True)
|
|
|
|
|
|
select_last_df.to_csv(subject_save_path + 'select_last_df.csv', sep='\t', index=False, header=True)
|
|
|
select_hist_df.to_csv(subject_save_path + 'select_hist_df.csv', sep='\t', index=False, header=True)
|
|
|
|
|
|
reduce_mem(select_last_df)
|
|
|
reduce_mem(select_hist_df)
|
|
|
return select_hist_df, select_last_df
|
|
|
|
|
|
def get_all_hist_and_last_select(all_select):
|
|
|
"""
|
|
|
获取课程行为数据的历史选择和最后一次选择
|
|
|
"""
|
|
|
# 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
|
|
|
def hist_func(user_df):
|
|
|
if len(user_df) == 1:
|
|
|
return user_df
|
|
|
else:
|
|
|
return user_df[:-1]
|
|
|
|
|
|
def apply_parallel(df_grouped, func):
|
|
|
results = Parallel(n_jobs=20)(delayed(func)(group) for name, group in df_grouped)
|
|
|
return pd.concat(results)
|
|
|
|
|
|
if os.path.exists(subject_save_path + 'select_all_last_df.csv') and \
|
|
|
os.path.exists(subject_save_path + 'select_all_hist_df.csv'):
|
|
|
select_last_df = pd.read_csv(subject_save_path + 'select_all_last_df.csv', sep='\t', encoding='utf-8')
|
|
|
select_hist_df = pd.read_csv(subject_save_path + 'select_all_hist_df.csv', sep='\t', encoding='utf-8')
|
|
|
else:
|
|
|
all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
|
|
|
select_last_df = all_select.groupby('user_id').tail(1)
|
|
|
|
|
|
# 多进程并行提升处理速度
|
|
|
df_grouped = all_select.groupby('user_id')
|
|
|
select_hist_df = apply_parallel(df_grouped, hist_func)
|
|
|
select_hist_df = select_hist_df.reset_index(drop=True)
|
|
|
|
|
|
#select_hist_df = all_select.groupby('user_id').progress_apply(hist_func).reset_index(drop = True)
|
|
|
|
|
|
select_last_df.to_csv(subject_save_path + 'select_all_last_df.csv', sep='\t', index=False, header=True)
|
|
|
select_hist_df.to_csv(subject_save_path + 'select_all_hist_df.csv', sep='\t', index=False, header=True)
|
|
|
|
|
|
reduce_mem(select_last_df)
|
|
|
reduce_mem(select_hist_df)
|
|
|
return select_hist_df, select_last_df
|
|
|
|
|
|
def get_rank_hist_and_last_select(all_select):
|
|
|
"""
|
|
|
获取排序课程行为数据的历史选择和最后一次选择
|
|
|
"""
|
|
|
all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
|
|
|
select_last_df = all_select.groupby('user_id').tail(1)
|
|
|
|
|
|
# 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
|
|
|
def hist_func(user_df):
|
|
|
if len(user_df) == 1:
|
|
|
return user_df
|
|
|
else:
|
|
|
return user_df[:-1]
|
|
|
|
|
|
select_hist_df = all_select.groupby('user_id').apply(hist_func).reset_index(drop=True)
|
|
|
|
|
|
return select_hist_df, select_last_df
|
|
|
|
|
|
|
|
|
def get_recall_item_info_dict(item_info_df):
|
|
|
"""
|
|
|
获取课程id对应的基本属性，保存成字典的形式
|
|
|
方便后面召回阶段，冷启动阶段直接使用
|
|
|
"""
|
|
|
if os.path.exists(subject_save_path + 'subjects_info_recall_dict.pkl'):
|
|
|
item_info_dict = pickle.load(open(subject_save_path + 'subjects_info_recall_dict.pkl', 'rb'))
|
|
|
return item_info_dict
|
|
|
|
|
|
item_info_df['visits'] = item_info_df['visits'].fillna(0)
|
|
|
item_info_df['visits'] = item_info_df['visits'].astype(int)
|
|
|
|
|
|
item_info_df['study_count'] = item_info_df['study_count'].fillna(0)
|
|
|
item_info_df['study_count'] = item_info_df['visits'].astype(int)
|
|
|
|
|
|
item_info_df['course_study_count'] = item_info_df['course_study_count'].fillna(0)
|
|
|
item_info_df['course_study_count'] = item_info_df['course_study_count'].astype(int)
|
|
|
|
|
|
item_info_df['passed_count'] = item_info_df['passed_count'].fillna(0)
|
|
|
item_info_df['passed_count'] = item_info_df['passed_count'].astype(int)
|
|
|
|
|
|
item_info_df['course_used_count'] = item_info_df['course_used_count'].fillna(0)
|
|
|
item_info_df['course_used_count'] = item_info_df['course_used_count'].astype(int)
|
|
|
|
|
|
item_info_df['school_used_count'] = item_info_df['school_used_count'].fillna(0)
|
|
|
item_info_df['school_used_count'] = item_info_df['school_used_count'].astype(int)
|
|
|
|
|
|
item_info_df['challenge_count'] = item_info_df['challenge_count'].fillna(0)
|
|
|
item_info_df['challenge_count'] = item_info_df['challenge_count'].astype(int)
|
|
|
|
|
|
item_info_df['evaluate_count'] = item_info_df['evaluate_count'].fillna(0)
|
|
|
item_info_df['evaluate_count'] = item_info_df['evaluate_count'].astype(int)
|
|
|
|
|
|
item_info_df['video_study_time'] = item_info_df['video_study_time'].fillna(0)
|
|
|
item_info_df['video_study_time'] = item_info_df['video_study_time'].astype(int)
|
|
|
|
|
|
item_info_df['study_pdf_attachment_count'] = item_info_df['study_pdf_attachment_count'].fillna(0)
|
|
|
item_info_df['study_pdf_attachment_count'] = item_info_df['study_pdf_attachment_count'].astype(int)
|
|
|
|
|
|
item_info_df['averge_star'] = item_info_df['averge_star'].fillna(0.0)
|
|
|
item_info_df['averge_star'] = item_info_df['averge_star'].astype(float)
|
|
|
|
|
|
# 创建时间进行归一化
|
|
|
max_min_scaler = lambda x : (x-np.min(x))/(np.max(x)-np.min(x))
|
|
|
item_info_df['created_at_ts'] = item_info_df[['created_at_ts']].progress_apply(max_min_scaler)
|
|
|
|
|
|
# 课程名称
|
|
|
item_name_dict = dict(zip(item_info_df['subject_id'], item_info_df['subject_name']))
|
|
|
|
|
|
# 课程访问次数
|
|
|
item_visists_dict = dict(zip(item_info_df['subject_id'], item_info_df['visits']))
|
|
|
|
|
|
# 课程创建时间
|
|
|
item_created_time_dict = dict(zip(item_info_df['subject_id'], item_info_df['created_at_ts']))
|
|
|
|
|
|
# 课程评价星数
|
|
|
item_averge_star_dict = dict(zip(item_info_df['subject_id'], item_info_df['averge_star']))
|
|
|
|
|
|
# 课程学生人数
|
|
|
item_study_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['study_count']))
|
|
|
|
|
|
# 课程课堂学习人数
|
|
|
item_course_study_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['course_study_count']))
|
|
|
|
|
|
# 课程通过人数
|
|
|
item_passed_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['passed_count']))
|
|
|
|
|
|
# 课程课堂使用数量
|
|
|
item_course_used_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['course_used_count']))
|
|
|
|
|
|
# 课程学校使用数量
|
|
|
item_school_used_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['school_used_count']))
|
|
|
|
|
|
# 课程关卡数量
|
|
|
item_challenge_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['challenge_count']))
|
|
|
|
|
|
# 课程关卡评测次数
|
|
|
item_evaluate_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['evaluate_count']))
|
|
|
|
|
|
# 课程视频学习时长
|
|
|
item_video_study_time_dict = dict(zip(item_info_df['subject_id'], item_info_df['video_study_time']))
|
|
|
|
|
|
# 课程PDF附件数量
|
|
|
item_study_pdf_attachment_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['study_pdf_attachment_count']))
|
|
|
|
|
|
item_info_dict = {}
|
|
|
item_info_dict.clear()
|
|
|
|
|
|
for subject_id, \
|
|
|
subject_name, \
|
|
|
item_visists, \
|
|
|
item_created_time, \
|
|
|
item_averge_star, \
|
|
|
item_study_count, \
|
|
|
item_course_study_count, \
|
|
|
item_passed_count, \
|
|
|
item_course_used_count, \
|
|
|
item_school_used_count, \
|
|
|
item_challenge_count, \
|
|
|
item_evaluate_count, \
|
|
|
item_video_study_time, \
|
|
|
item_study_pdf_attachment_count in tqdm(zip(item_visists_dict.keys(),
|
|
|
item_name_dict.values(),
|
|
|
item_visists_dict.values(),
|
|
|
item_created_time_dict.values(),
|
|
|
item_averge_star_dict.values(),
|
|
|
item_study_count_dict.values(),
|
|
|
item_course_study_count_dict.values(),
|
|
|
item_passed_count_dict.values(),
|
|
|
item_course_used_count_dict.values(),
|
|
|
item_school_used_count_dict.values(),
|
|
|
item_challenge_count_dict.values(),
|
|
|
item_evaluate_count_dict.values(),
|
|
|
item_video_study_time_dict.values(),
|
|
|
item_study_pdf_attachment_count_dict.values())):
|
|
|
|
|
|
item_info_dict.setdefault(subject_id, {})
|
|
|
item_info_dict[subject_id]['subject_name'] = subject_name
|
|
|
item_info_dict[subject_id]['visists'] = item_visists
|
|
|
item_info_dict[subject_id]['created_at_ts'] = item_created_time
|
|
|
item_info_dict[subject_id]['averge_star'] = item_averge_star
|
|
|
item_info_dict[subject_id]['study_count'] = item_study_count
|
|
|
item_info_dict[subject_id]['course_study_count'] = item_course_study_count
|
|
|
item_info_dict[subject_id]['passed_count'] = item_passed_count
|
|
|
item_info_dict[subject_id]['course_used_count'] = item_course_used_count
|
|
|
item_info_dict[subject_id]['school_used_count'] = item_school_used_count
|
|
|
item_info_dict[subject_id]['challenge_count'] = item_challenge_count
|
|
|
item_info_dict[subject_id]['evaluate_count'] = item_evaluate_count
|
|
|
item_info_dict[subject_id]['video_study_time'] = item_video_study_time
|
|
|
item_info_dict[subject_id]['study_pdf_attachment_count'] = item_study_pdf_attachment_count
|
|
|
|
|
|
pickle.dump(item_info_dict, open(subject_save_path + 'subjects_info_recall_dict.pkl', 'wb'))
|
|
|
return item_info_dict
|
|
|
|
|
|
|
|
|
def get_user_hist_item_info_dict(all_select):
|
|
|
"""
|
|
|
获取用户历史选择的课程信息
|
|
|
"""
|
|
|
logger.info("获取用户历史选择的课程信息字典")
|
|
|
|
|
|
if os.path.exists(subject_save_path + 'user_hist_item_info_dict.pkl'):
|
|
|
user_hist_item_info_dict = pickle.load(open(subject_save_path + 'user_hist_item_info_dict.pkl', 'rb'))
|
|
|
return user_hist_item_info_dict
|
|
|
|
|
|
def get_last_item_created_time(x):
|
|
|
return max(x.values.tolist())
|
|
|
|
|
|
all_select['visits'] = all_select['visits'].fillna(0)
|
|
|
all_select['study_count'] = all_select['study_count'].fillna(0)
|
|
|
all_select['course_study_count'] = all_select['course_study_count'].fillna(0)
|
|
|
all_select['passed_count'] = all_select['passed_count'].fillna(0)
|
|
|
all_select['course_used_count'] = all_select['course_used_count'].fillna(0)
|
|
|
all_select['school_used_count'] = all_select['school_used_count'].fillna(0)
|
|
|
all_select['challenge_count'] = all_select['challenge_count'].fillna(0)
|
|
|
all_select['evaluate_count'] = all_select['evaluate_count'].fillna(0)
|
|
|
all_select['video_study_time'] = all_select['video_study_time'].fillna(0)
|
|
|
all_select['study_pdf_attachment_count'] = all_select['study_pdf_attachment_count'].fillna(0)
|
|
|
all_select['averge_star'] = all_select['averge_star'].fillna(0.0)
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均访问量的集合字典
|
|
|
user_hist_item_visits = all_select.groupby('user_id')['visits'].agg('mean').reset_index()
|
|
|
user_hist_item_visits_dict = dict(zip(user_hist_item_visits['user_id'], user_hist_item_visits['visits']))
|
|
|
|
|
|
# 获取user_id对应的用户选择课程的集合
|
|
|
user_hist_item_ids = all_select.groupby('user_id')['subject_id'].agg(set).reset_index()
|
|
|
user_hist_item_ids_dict = dict(zip(user_hist_item_ids['user_id'], user_hist_item_ids['subject_id']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择的课程的评价的星数
|
|
|
user_hist_item_averge_star = all_select.groupby('user_id')['averge_star'].agg(set).reset_index()
|
|
|
user_hist_item_averge_star_dict = dict(zip(user_hist_item_averge_star['user_id'], user_hist_item_averge_star['averge_star']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均学习人数的集合字典
|
|
|
user_hist_item_study_count = all_select.groupby('user_id')['study_count'].agg('mean').reset_index()
|
|
|
user_hist_item_study_count_dict = dict(zip(user_hist_item_study_count['user_id'], user_hist_item_study_count['study_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均课堂学习数量的集合字典
|
|
|
user_hist_item_course_study_count = all_select.groupby('user_id')['course_study_count'].agg('mean').reset_index()
|
|
|
user_hist_item_course_study_count_dict = dict(zip(user_hist_item_course_study_count['user_id'], user_hist_item_course_study_count['course_study_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均课堂学习数量的集合字典
|
|
|
user_hist_item_passed_count = all_select.groupby('user_id')['passed_count'].agg('mean').reset_index()
|
|
|
user_hist_item_passed_count_dict = dict(zip(user_hist_item_passed_count['user_id'], user_hist_item_passed_count['passed_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均课堂使用次数的集合字典
|
|
|
user_hist_item_course_used_count = all_select.groupby('user_id')['course_used_count'].agg('mean').reset_index()
|
|
|
user_hist_item_course_used_count_dict = dict(zip(user_hist_item_course_used_count['user_id'], user_hist_item_course_used_count['course_used_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均学校使用次数的集合字典
|
|
|
user_hist_item_school_used_count = all_select.groupby('user_id')['school_used_count'].agg('mean').reset_index()
|
|
|
user_hist_item_school_used_count_dict = dict(zip(user_hist_item_school_used_count['user_id'], user_hist_item_school_used_count['school_used_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
|
|
|
user_hist_item_challenge_count = all_select.groupby('user_id')['challenge_count'].agg('mean').reset_index()
|
|
|
user_hist_item_challenge_count_dict = dict(zip(user_hist_item_challenge_count['user_id'], user_hist_item_challenge_count['challenge_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
|
|
|
user_hist_item_evaluate_count = all_select.groupby('user_id')['evaluate_count'].agg('mean').reset_index()
|
|
|
user_hist_item_evaluate_count_dict = dict(zip(user_hist_item_evaluate_count['user_id'], user_hist_item_evaluate_count['evaluate_count']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
|
|
|
user_hist_item_video_study_time = all_select.groupby('user_id')['video_study_time'].agg('mean').reset_index()
|
|
|
user_hist_item_video_study_time_dict = dict(zip(user_hist_item_video_study_time['user_id'], user_hist_item_video_study_time['video_study_time']))
|
|
|
|
|
|
# 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
|
|
|
user_hist_item_study_pdf_attachment_count = all_select.groupby('user_id')['study_pdf_attachment_count'].agg('mean').reset_index()
|
|
|
user_hist_item_study_pdf_attachment_count_dict = dict(zip(user_hist_item_study_pdf_attachment_count['user_id'], user_hist_item_study_pdf_attachment_count['study_pdf_attachment_count']))
|
|
|
|
|
|
# 获取user_id对应的用户最后一次选择的课程的创建时间
|
|
|
all_select_ = all_select.sort_values('created_at_ts')
|
|
|
|
|
|
user_last_item_created_time = all_select_.groupby('user_id')['created_at_ts']. \
|
|
|
progress_apply(get_last_item_created_time).reset_index()
|
|
|
|
|
|
max_min_scaler = lambda x : (x-np.min(x))/(np.max(x)-np.min(x))
|
|
|
user_last_item_created_time['created_at_ts'] = user_last_item_created_time[['created_at_ts']]. \
|
|
|
progress_apply(max_min_scaler)
|
|
|
|
|
|
user_last_item_created_time_dict = dict(zip(user_last_item_created_time['user_id'], \
|
|
|
user_last_item_created_time['created_at_ts']))
|
|
|
|
|
|
user_hist_item_info_dict = {}
|
|
|
user_hist_item_info_dict.clear()
|
|
|
|
|
|
for user_id, \
|
|
|
hist_item_ids, \
|
|
|
hist_item_visits, \
|
|
|
hist_item_averge_star, \
|
|
|
hist_item_study_count, \
|
|
|
hist_item_course_study_count, \
|
|
|
hist_item_passed_count, \
|
|
|
hist_item_course_used_count, \
|
|
|
hist_item_school_used_count, \
|
|
|
hist_item_challenge_count, \
|
|
|
hist_item_evaluate_count, \
|
|
|
hist_item_video_study_time, \
|
|
|
hist_item_study_pdf_attachment_count, \
|
|
|
last_item_created_time in tqdm(zip(user_hist_item_ids_dict.keys(),
|
|
|
user_hist_item_ids_dict.values(),
|
|
|
user_hist_item_visits_dict.values(),
|
|
|
user_hist_item_averge_star_dict.values(),
|
|
|
user_hist_item_study_count_dict.values(),
|
|
|
user_hist_item_course_study_count_dict.values(),
|
|
|
user_hist_item_passed_count_dict.values(),
|
|
|
user_hist_item_course_used_count_dict.values(),
|
|
|
user_hist_item_school_used_count_dict.values(),
|
|
|
user_hist_item_challenge_count_dict.values(),
|
|
|
user_hist_item_evaluate_count_dict.values(),
|
|
|
user_hist_item_video_study_time_dict.values(),
|
|
|
user_hist_item_study_pdf_attachment_count_dict.values(),
|
|
|
user_last_item_created_time_dict.values())):
|
|
|
|
|
|
user_hist_item_info_dict.setdefault(user_id, {})
|
|
|
user_hist_item_info_dict[user_id]['hist_item_ids'] = hist_item_ids
|
|
|
user_hist_item_info_dict[user_id]['hist_item_visits'] = hist_item_visits
|
|
|
user_hist_item_info_dict[user_id]['hist_item_averge_star'] = hist_item_averge_star
|
|
|
user_hist_item_info_dict[user_id]['hist_item_study_count'] = hist_item_study_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_course_study_count'] = hist_item_course_study_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_passed_count'] = hist_item_passed_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_course_used_count'] = hist_item_course_used_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_school_used_count'] = hist_item_school_used_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_challenge_count'] = hist_item_challenge_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_evaluate_count'] = hist_item_evaluate_count
|
|
|
user_hist_item_info_dict[user_id]['hist_item_video_study_time'] = hist_item_video_study_time
|
|
|
user_hist_item_info_dict[user_id]['hist_item_study_pdf_attachment_count'] = hist_item_study_pdf_attachment_count
|
|
|
user_hist_item_info_dict[user_id]['last_item_created_time'] = last_item_created_time
|
|
|
|
|
|
|
|
|
pickle.dump(user_hist_item_info_dict, open(subject_save_path + 'user_hist_item_info_dict.pkl', 'wb'))
|
|
|
return user_hist_item_info_dict
|
|
|
|
|
|
|
|
|
def get_item_topk_select(select_df, k):
|
|
|
"""
|
|
|
获取被选择次数最多的课程，用来做召回补全
|
|
|
"""
|
|
|
topk_select = select_df['subject_id'].value_counts().index[:k]
|
|
|
return topk_select
|
|
|
|
|
|
|
|
|
def get_user_activate_degree_dict(all_select_df):
|
|
|
"""
|
|
|
将用户的选择次数作为获取用户活跃度的指标
|
|
|
"""
|
|
|
# 加载保存好的用户活跃度字典
|
|
|
if os.path.exists(subject_save_path + 'user_activate_degree_dict.pkl'):
|
|
|
user_activate_degree_dict = pickle.load(open(subject_save_path + 'user_activate_degree_dict.pkl', 'rb'))
|
|
|
return user_activate_degree_dict
|
|
|
|
|
|
all_select_df_ = all_select_df.groupby('user_id')['subject_id'].count().reset_index()
|
|
|
|
|
|
# 用户活跃度归一化
|
|
|
mm = MinMaxScaler()
|
|
|
all_select_df_['subject_id'] = mm.fit_transform(all_select_df_[['subject_id']])
|
|
|
user_activate_degree_dict = dict(zip(all_select_df_['user_id'], all_select_df_['subject_id']))
|
|
|
|
|
|
# 保存生成好的用户活跃度字典
|
|
|
pickle.dump(user_activate_degree_dict, open(subject_save_path + 'user_activate_degree_dict.pkl', 'wb'))
|
|
|
return user_activate_degree_dict
|
|
|
|
|
|
|
|
|
def metrics_recall(user_recall_items_dict, train_last_select_df, topk=10):
|
|
|
"""
|
|
|
召回评估，依次评估召回的前 10, 20, 30，...topk/10 个课程的击中率
|
|
|
"""
|
|
|
# 生成用户最后选择课程的字典
|
|
|
last_select_item_dict = dict(zip(train_last_select_df['user_id'], train_last_select_df['subject_id']))
|
|
|
|
|
|
# 用户数量
|
|
|
user_num = len(user_recall_items_dict)
|
|
|
|
|
|
for k in range(10, topk + 1, 10):
|
|
|
hit_num = 0
|
|
|
for user_id, item_list in user_recall_items_dict.items():
|
|
|
# 获取前k个召回的结果
|
|
|
tmp_recall_items = [x[0] for x in user_recall_items_dict[user_id][:k]]
|
|
|
|
|
|
if last_select_item_dict[user_id] in set(tmp_recall_items):
|
|
|
hit_num += 1
|
|
|
|
|
|
hit_rate = round(hit_num * 1.0 / user_num, 5)
|
|
|
print(' topk: ', k, ' : ', 'hit_num: ', hit_num, 'hit_rate: ', hit_rate, 'user_num : ', user_num)
|
|
|
|
|
|
def metrics_pinsage_recall(user_recall_items_dict, train_last_select_df, topk=10):
|
|
|
"""
|
|
|
召回评估，依次评估召回的前 10, 20, 30，...topk/10 个物品的击中率
|
|
|
"""
|
|
|
# 生成用户最后选择物品的字典
|
|
|
last_select_item_dict = dict(zip(train_last_select_df['user'], train_last_select_df['item']))
|
|
|
|
|
|
# 用户数量
|
|
|
user_num = len(user_recall_items_dict)
|
|
|
print(user_num)
|
|
|
|
|
|
for k in range(10, topk + 1, 10):
|
|
|
hit_num = 0
|
|
|
for user_id, item_list in user_recall_items_dict.items():
|
|
|
# 获取前k个召回的结果
|
|
|
tmp_recall_items = [x[0] for x in user_recall_items_dict[user_id][:k]]
|
|
|
|
|
|
if last_select_item_dict[user_id] in set(tmp_recall_items):
|
|
|
hit_num += 1
|
|
|
|
|
|
hit_rate = round(hit_num * 1.0 / user_num, 5)
|
|
|
print(' topk: ', k, ' : ', 'hit_num: ', hit_num, 'hit_rate: ', hit_rate, 'user_num : ', user_num)
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
logger.info('获取课程信息')
|
|
|
item_info_df = get_item_info_df()
|
|
|
|
|
|
logger.info('生成课程信息字典')
|
|
|
item_info_dict = get_recall_item_info_dict(item_info_df) |
