EduCoder_Study_RS/matching/subject/recall_comm.py

import pandas as pd
import numpy as np
from tqdm import tqdm
import os
import sys
sys.path.append(os.getcwd())
import warnings
import pickle
from sklearn.preprocessing import MinMaxScaler
from config import mysubjects_train_data, mysubjects_test_data
from config import subjects_data_path, subjects_embed_path, subject_save_path
from config import subject_user_item_time_dict_data
from config import subject_item_user_time_dict, data_parent_path
from config import subjects_emb_dict
from config import users_data_path
from utils import reduce_mem
from config import logger
from joblib import Parallel, delayed
from data_process import process_users_data_null


tqdm.pandas()
warnings.filterwarnings('ignore')

def get_all_select_sample(sample_nums=10000):
    """
    从训练集中划出一部分数据来调试代码
    """
    if os.path.exists(data_parent_path + 'mysubjects_train_sample.csv'):
        all_select = pd.read_csv(data_parent_path + 'mysubjects_train_sample.csv', sep='\t', encoding='utf-8')
        reduce_mem(all_select)
        return all_select

    all_select = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')

    # user_id过滤重复
    all_user_ids = all_select.user_id.unique()

    # 只采集指定数量的user_id
    sample_user_ids = np.random.choice(all_user_ids, size=sample_nums, replace=False) 

    # 取出这些user_id选择的课程
    all_select = all_select[all_select['user_id'].isin(sample_user_ids)]
    
    # 根据user_id, subject_id去重
    all_select = all_select.drop_duplicates((['user_id', 'subject_id']))

    all_select.to_csv(data_parent_path + 'mysubjects_train_sample.csv', sep='\t', index=False, header=True)
    reduce_mem(all_select)
    return all_select


def get_all_select_df(offline = False):
    """
    读取课程数据时分成线上和线下
    线上预测时将测试集中的课程数据合并到总的数据中
    线下验证模型的有效性或者特征的有效性时只使用训练集
    """    
    if offline:
        all_select = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')
    else:
        train_mysubjects = pd.read_csv(mysubjects_train_data, sep='\t', encoding='utf-8')
        
        test_mysubjects = pd.read_csv(mysubjects_test_data, sep='\t', encoding='utf-8')

        all_select = train_mysubjects.append(test_mysubjects)
    
    all_select = all_select.drop_duplicates((['user_id', 'subject_id']))
    reduce_mem(all_select)
    return all_select


def get_user_info_df():
    """
    读取用户的基本属性
    """
    user_info_df = pd.read_csv(users_data_path, sep='\t', encoding='utf-8', quoting=3) 
    user_info_df = process_users_data_null(user_info_df)
    reduce_mem(user_info_df)
    return user_info_df  


def get_item_info_df():
    """
    读取课程的基本属性
    """
    item_info_df = pd.read_csv(subjects_data_path, sep='\t', encoding='utf-8')   
    reduce_mem(item_info_df)

    return item_info_df
    

def get_recall_item_info_df(item_info_df, subject_id_list):
    """
    读取课程的基本属性
    """
    recall_item_info_df = item_info_df[item_info_df['subject_id'].isin(subject_id_list)]
    return recall_item_info_df


def get_select_item_info(all_select_df, user_id):
    """
    获取某个用户选择的课程行为数据
    """    
    # select_item_df = all_select_df[all_select_df['user_id'] == user_id]

    # 用loc会快一些
    select_item_df = all_select_df.loc[all_select_df['user_id'] == user_id, :].reset_index()

    return select_item_df


def get_item_emb_dict():
    """
    生成和读取课程的Embedding数据
    """
    # 加载已经保存的Embedding数据
    if os.path.exists(subjects_emb_dict):
        item_emb_dict = pickle.load(open(subjects_emb_dict, 'rb'))
        return item_emb_dict

    # 生成课程的Embedding数据
    item_emb_df = pd.read_csv(subjects_embed_path, sep='\t', encoding='utf-8')
    
    item_emb_cols = [x for x in item_emb_df.columns if 'emb' in x]
    item_emb_np = np.ascontiguousarray(item_emb_df[item_emb_cols])
    # 进行归一化
    item_emb_np = item_emb_np / np.linalg.norm(item_emb_np, axis=1, keepdims=True)

    item_emb_dict = dict(zip(item_emb_df['subject_id'], item_emb_np))
    pickle.dump(item_emb_dict, open(subjects_emb_dict, 'wb'))
    
    return item_emb_dict


def get_user_item_time_dict(select_df):
    """
    根据时间获取用户选择的课程序列   
    {user1: [(item1, time1), (item2, time2)..]...}
    """    
    def make_item_time_pair(df):
        """
        构造选课程，选择时间列表
        """        
        return list(zip(df['subject_id'], df['created_timestamp']))

    # 加载之前pickel保存的
    if os.path.exists(subject_user_item_time_dict_data):
        user_item_time_dict = pickle.load(open(subject_user_item_time_dict_data, 'rb'))
        return user_item_time_dict

    # 按选择时间排序
    select_df = select_df.sort_values('created_timestamp')   
    
    # 按用户分组生成用户选择的课程
    user_item_time_df = select_df.groupby('user_id')['subject_id', 'created_timestamp'].progress_apply(lambda x: make_item_time_pair(x)).reset_index(). \
        rename(columns = {0: 'item_time_list'})
    
    # 生成用户选择的课程字典
    user_item_time_dict = dict(zip(user_item_time_df['user_id'], user_item_time_df['item_time_list']))
    
    pickle.dump(user_item_time_dict, open(subject_user_item_time_dict_data, 'wb'))
    
    return user_item_time_dict


def get_item_user_time_dict(select_df):
    """
    根据选择时间获取课程被选择的用户序列  
    {item1: [(user1, time1), (user2, time2)...]...}
    """
    def make_user_time_pair(df):
        return list(zip(df['user_id'], df['created_timestamp']))

    # 加载之前pickel保存的
    if os.path.exists(subject_item_user_time_dict):
        item_user_time_dict = pickle.load(open(subject_item_user_time_dict, 'rb'))
        return item_user_time_dict
  
    select_df = select_df.sort_values('created_timestamp')

    item_user_time_df = select_df.groupby('subject_id')['user_id', 'created_timestamp']. \
        progress_apply(lambda x: make_user_time_pair(x)).reset_index(). \
        rename(columns = {0: 'user_time_list'})
    
    item_user_time_dict = dict(zip(item_user_time_df['subject_id'], item_user_time_df['user_time_list']))
    pickle.dump(item_user_time_dict, open(subject_item_user_time_dict, 'wb'))

    return item_user_time_dict       
    

def get_hist_and_last_select(all_select):
    """
    获取课程行为数据的历史选择和最后一次选择
    """    
    # 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
    def hist_func(user_df):
        if len(user_df) == 1:
            return user_df
        else:
            return user_df[:-1]

    def apply_parallel(df_grouped, func):
        results = Parallel(n_jobs=20)(delayed(func)(group) for name, group in df_grouped)
        return pd.concat(results)

    if os.path.exists(subject_save_path + 'select_last_df.csv') and \
       os.path.exists(subject_save_path + 'select_hist_df.csv'):
        select_last_df = pd.read_csv(subject_save_path + 'select_last_df.csv', sep='\t', encoding='utf-8')
        select_hist_df = pd.read_csv(subject_save_path + 'select_hist_df.csv', sep='\t', encoding='utf-8')
    else:
        all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
        select_last_df = all_select.groupby('user_id').tail(1)

        # 多进程并行提升处理速度
        df_grouped = all_select.groupby('user_id')
        select_hist_df = apply_parallel(df_grouped, hist_func)
        select_hist_df = select_hist_df.reset_index(drop=True)

        #select_hist_df = all_select.groupby('user_id').progress_apply(hist_func).reset_index(drop = True)

        select_last_df.to_csv(subject_save_path + 'select_last_df.csv', sep='\t', index=False, header=True)
        select_hist_df.to_csv(subject_save_path + 'select_hist_df.csv', sep='\t', index=False, header=True)

    reduce_mem(select_last_df)
    reduce_mem(select_hist_df)
    return select_hist_df, select_last_df
    
def get_all_hist_and_last_select(all_select):
    """
    获取课程行为数据的历史选择和最后一次选择
    """    
    # 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
    def hist_func(user_df):
        if len(user_df) == 1:
            return user_df
        else:
            return user_df[:-1]

    def apply_parallel(df_grouped, func):
        results = Parallel(n_jobs=20)(delayed(func)(group) for name, group in df_grouped)
        return pd.concat(results)

    if os.path.exists(subject_save_path + 'select_all_last_df.csv') and \
       os.path.exists(subject_save_path + 'select_all_hist_df.csv'):
        select_last_df = pd.read_csv(subject_save_path + 'select_all_last_df.csv', sep='\t', encoding='utf-8')
        select_hist_df = pd.read_csv(subject_save_path + 'select_all_hist_df.csv', sep='\t', encoding='utf-8')
    else:
        all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
        select_last_df = all_select.groupby('user_id').tail(1)

        # 多进程并行提升处理速度
        df_grouped = all_select.groupby('user_id')
        select_hist_df = apply_parallel(df_grouped, hist_func)
        select_hist_df = select_hist_df.reset_index(drop=True)

        #select_hist_df = all_select.groupby('user_id').progress_apply(hist_func).reset_index(drop = True)

        select_last_df.to_csv(subject_save_path + 'select_all_last_df.csv', sep='\t', index=False, header=True)
        select_hist_df.to_csv(subject_save_path + 'select_all_hist_df.csv', sep='\t', index=False, header=True)

    reduce_mem(select_last_df)
    reduce_mem(select_hist_df)
    return select_hist_df, select_last_df

def get_rank_hist_and_last_select(all_select):
    """
    获取排序课程行为数据的历史选择和最后一次选择
    """    
    all_select = all_select.sort_values(by=['user_id', 'created_timestamp'])
    select_last_df = all_select.groupby('user_id').tail(1)

    # 如果用户只有一个选择，hist为空，会导致训练的时候这个用户不可见，此时默认泄露一下
    def hist_func(user_df):
        if len(user_df) == 1:
            return user_df
        else:
            return user_df[:-1]

    select_hist_df = all_select.groupby('user_id').apply(hist_func).reset_index(drop=True)

    return select_hist_df, select_last_df


def get_recall_item_info_dict(item_info_df):
    """
    获取课程id对应的基本属性，保存成字典的形式
    方便后面召回阶段，冷启动阶段直接使用
    """
    if os.path.exists(subject_save_path + 'subjects_info_recall_dict.pkl'):
        item_info_dict = pickle.load(open(subject_save_path + 'subjects_info_recall_dict.pkl', 'rb'))
        return item_info_dict

    item_info_df['visits'] = item_info_df['visits'].fillna(0)   
    item_info_df['visits'] = item_info_df['visits'].astype(int)      

    item_info_df['study_count'] = item_info_df['study_count'].fillna(0)
    item_info_df['study_count'] = item_info_df['visits'].astype(int)    

    item_info_df['course_study_count'] = item_info_df['course_study_count'].fillna(0)  
    item_info_df['course_study_count'] = item_info_df['course_study_count'].astype(int)     

    item_info_df['passed_count'] = item_info_df['passed_count'].fillna(0) 
    item_info_df['passed_count'] = item_info_df['passed_count'].astype(int)        

    item_info_df['course_used_count'] = item_info_df['course_used_count'].fillna(0)
    item_info_df['course_used_count'] = item_info_df['course_used_count'].astype(int)
       
    item_info_df['school_used_count'] = item_info_df['school_used_count'].fillna(0)  
    item_info_df['school_used_count'] = item_info_df['school_used_count'].astype(int)      

    item_info_df['challenge_count'] = item_info_df['challenge_count'].fillna(0)  
    item_info_df['challenge_count'] = item_info_df['challenge_count'].astype(int)          

    item_info_df['evaluate_count'] = item_info_df['evaluate_count'].fillna(0) 
    item_info_df['evaluate_count'] = item_info_df['evaluate_count'].astype(int)      

    item_info_df['video_study_time'] = item_info_df['video_study_time'].fillna(0)  
    item_info_df['video_study_time'] = item_info_df['video_study_time'].astype(int)     

    item_info_df['study_pdf_attachment_count'] = item_info_df['study_pdf_attachment_count'].fillna(0) 
    item_info_df['study_pdf_attachment_count'] = item_info_df['study_pdf_attachment_count'].astype(int)      

    item_info_df['averge_star'] = item_info_df['averge_star'].fillna(0.0) 
    item_info_df['averge_star'] = item_info_df['averge_star'].astype(float)           

    # 创建时间进行归一化
    max_min_scaler = lambda x : (x-np.min(x))/(np.max(x)-np.min(x))
    item_info_df['created_at_ts'] = item_info_df[['created_at_ts']].progress_apply(max_min_scaler)

    # 课程名称
    item_name_dict = dict(zip(item_info_df['subject_id'], item_info_df['subject_name']))
    
    # 课程访问次数
    item_visists_dict = dict(zip(item_info_df['subject_id'], item_info_df['visits']))

    # 课程创建时间
    item_created_time_dict = dict(zip(item_info_df['subject_id'], item_info_df['created_at_ts']))

    # 课程评价星数
    item_averge_star_dict = dict(zip(item_info_df['subject_id'], item_info_df['averge_star']))

    # 课程学生人数
    item_study_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['study_count']))

    # 课程课堂学习人数
    item_course_study_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['course_study_count']))

    # 课程通过人数
    item_passed_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['passed_count']))

    # 课程课堂使用数量
    item_course_used_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['course_used_count']))

    # 课程学校使用数量
    item_school_used_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['school_used_count']))

    # 课程关卡数量
    item_challenge_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['challenge_count']))

    # 课程关卡评测次数
    item_evaluate_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['evaluate_count']))

    # 课程视频学习时长
    item_video_study_time_dict = dict(zip(item_info_df['subject_id'], item_info_df['video_study_time']))

    # 课程PDF附件数量
    item_study_pdf_attachment_count_dict = dict(zip(item_info_df['subject_id'], item_info_df['study_pdf_attachment_count']))

    item_info_dict = {}
    item_info_dict.clear()

    for subject_id, \
        subject_name, \
        item_visists, \
        item_created_time, \
        item_averge_star, \
        item_study_count, \
        item_course_study_count, \
        item_passed_count, \
        item_course_used_count, \
        item_school_used_count, \
        item_challenge_count, \
        item_evaluate_count, \
        item_video_study_time, \
        item_study_pdf_attachment_count in tqdm(zip(item_visists_dict.keys(), 
                                        item_name_dict.values(),
                                        item_visists_dict.values(), 
                                        item_created_time_dict.values(),
                                        item_averge_star_dict.values(),
                                        item_study_count_dict.values(),
                                        item_course_study_count_dict.values(),
                                        item_passed_count_dict.values(),
                                        item_course_used_count_dict.values(),
                                        item_school_used_count_dict.values(),
                                        item_challenge_count_dict.values(),
                                        item_evaluate_count_dict.values(),
                                        item_video_study_time_dict.values(),
                                        item_study_pdf_attachment_count_dict.values())):

        item_info_dict.setdefault(subject_id, {})
        item_info_dict[subject_id]['subject_name'] = subject_name
        item_info_dict[subject_id]['visists'] = item_visists
        item_info_dict[subject_id]['created_at_ts'] = item_created_time
        item_info_dict[subject_id]['averge_star'] = item_averge_star
        item_info_dict[subject_id]['study_count'] = item_study_count
        item_info_dict[subject_id]['course_study_count'] = item_course_study_count
        item_info_dict[subject_id]['passed_count'] = item_passed_count
        item_info_dict[subject_id]['course_used_count'] = item_course_used_count
        item_info_dict[subject_id]['school_used_count'] = item_school_used_count
        item_info_dict[subject_id]['challenge_count'] = item_challenge_count
        item_info_dict[subject_id]['evaluate_count'] = item_evaluate_count
        item_info_dict[subject_id]['video_study_time'] = item_video_study_time
        item_info_dict[subject_id]['study_pdf_attachment_count'] = item_study_pdf_attachment_count

    pickle.dump(item_info_dict, open(subject_save_path + 'subjects_info_recall_dict.pkl', 'wb'))
    return item_info_dict


def get_user_hist_item_info_dict(all_select):
    """
    获取用户历史选择的课程信息
    """    
    logger.info("获取用户历史选择的课程信息字典")

    if os.path.exists(subject_save_path + 'user_hist_item_info_dict.pkl'):
        user_hist_item_info_dict = pickle.load(open(subject_save_path + 'user_hist_item_info_dict.pkl', 'rb'))
        return user_hist_item_info_dict

    def get_last_item_created_time(x):
        return max(x.values.tolist())     

    all_select['visits'] =  all_select['visits'].fillna(0)
    all_select['study_count'] = all_select['study_count'].fillna(0)
    all_select['course_study_count'] = all_select['course_study_count'].fillna(0)
    all_select['passed_count'] =  all_select['passed_count'].fillna(0)
    all_select['course_used_count'] = all_select['course_used_count'].fillna(0)
    all_select['school_used_count'] = all_select['school_used_count'].fillna(0)
    all_select['challenge_count'] =  all_select['challenge_count'].fillna(0)
    all_select['evaluate_count'] = all_select['evaluate_count'].fillna(0)
    all_select['video_study_time'] = all_select['video_study_time'].fillna(0)
    all_select['study_pdf_attachment_count'] = all_select['study_pdf_attachment_count'].fillna(0)
    all_select['averge_star'] =  all_select['averge_star'].fillna(0.0)
    
    # 获取user_id对应的用户历史选择课程平均访问量的集合字典
    user_hist_item_visits = all_select.groupby('user_id')['visits'].agg('mean').reset_index()
    user_hist_item_visits_dict = dict(zip(user_hist_item_visits['user_id'], user_hist_item_visits['visits']))
    
    # 获取user_id对应的用户选择课程的集合
    user_hist_item_ids = all_select.groupby('user_id')['subject_id'].agg(set).reset_index()
    user_hist_item_ids_dict = dict(zip(user_hist_item_ids['user_id'], user_hist_item_ids['subject_id']))    
 
    # 获取user_id对应的用户历史选择的课程的评价的星数
    user_hist_item_averge_star = all_select.groupby('user_id')['averge_star'].agg(set).reset_index()
    user_hist_item_averge_star_dict = dict(zip(user_hist_item_averge_star['user_id'], user_hist_item_averge_star['averge_star']))

    # 获取user_id对应的用户历史选择课程平均学习人数的集合字典
    user_hist_item_study_count = all_select.groupby('user_id')['study_count'].agg('mean').reset_index()
    user_hist_item_study_count_dict = dict(zip(user_hist_item_study_count['user_id'], user_hist_item_study_count['study_count']))          

    # 获取user_id对应的用户历史选择课程平均课堂学习数量的集合字典
    user_hist_item_course_study_count = all_select.groupby('user_id')['course_study_count'].agg('mean').reset_index()
    user_hist_item_course_study_count_dict = dict(zip(user_hist_item_course_study_count['user_id'], user_hist_item_course_study_count['course_study_count']))          

    # 获取user_id对应的用户历史选择课程平均课堂学习数量的集合字典
    user_hist_item_passed_count = all_select.groupby('user_id')['passed_count'].agg('mean').reset_index()
    user_hist_item_passed_count_dict = dict(zip(user_hist_item_passed_count['user_id'], user_hist_item_passed_count['passed_count']))          

    # 获取user_id对应的用户历史选择课程平均课堂使用次数的集合字典
    user_hist_item_course_used_count = all_select.groupby('user_id')['course_used_count'].agg('mean').reset_index()
    user_hist_item_course_used_count_dict = dict(zip(user_hist_item_course_used_count['user_id'], user_hist_item_course_used_count['course_used_count']))          

    # 获取user_id对应的用户历史选择课程平均学校使用次数的集合字典
    user_hist_item_school_used_count = all_select.groupby('user_id')['school_used_count'].agg('mean').reset_index()
    user_hist_item_school_used_count_dict = dict(zip(user_hist_item_school_used_count['user_id'], user_hist_item_school_used_count['school_used_count']))          

    # 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
    user_hist_item_challenge_count = all_select.groupby('user_id')['challenge_count'].agg('mean').reset_index()
    user_hist_item_challenge_count_dict = dict(zip(user_hist_item_challenge_count['user_id'], user_hist_item_challenge_count['challenge_count']))          

    # 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
    user_hist_item_evaluate_count = all_select.groupby('user_id')['evaluate_count'].agg('mean').reset_index()
    user_hist_item_evaluate_count_dict = dict(zip(user_hist_item_evaluate_count['user_id'], user_hist_item_evaluate_count['evaluate_count']))          

    # 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
    user_hist_item_video_study_time = all_select.groupby('user_id')['video_study_time'].agg('mean').reset_index()
    user_hist_item_video_study_time_dict = dict(zip(user_hist_item_video_study_time['user_id'], user_hist_item_video_study_time['video_study_time']))          

    # 获取user_id对应的用户历史选择课程平均关卡数量的集合字典
    user_hist_item_study_pdf_attachment_count = all_select.groupby('user_id')['study_pdf_attachment_count'].agg('mean').reset_index()
    user_hist_item_study_pdf_attachment_count_dict = dict(zip(user_hist_item_study_pdf_attachment_count['user_id'], user_hist_item_study_pdf_attachment_count['study_pdf_attachment_count']))          

    # 获取user_id对应的用户最后一次选择的课程的创建时间
    all_select_ = all_select.sort_values('created_at_ts')
    
    user_last_item_created_time = all_select_.groupby('user_id')['created_at_ts']. \
        progress_apply(get_last_item_created_time).reset_index()
    
    max_min_scaler = lambda x : (x-np.min(x))/(np.max(x)-np.min(x))
    user_last_item_created_time['created_at_ts'] = user_last_item_created_time[['created_at_ts']]. \
        progress_apply(max_min_scaler)
    
    user_last_item_created_time_dict = dict(zip(user_last_item_created_time['user_id'], \
                                                user_last_item_created_time['created_at_ts']))

    user_hist_item_info_dict = {}
    user_hist_item_info_dict.clear()

    for user_id, \
        hist_item_ids, \
        hist_item_visits, \
        hist_item_averge_star, \
        hist_item_study_count, \
        hist_item_course_study_count, \
        hist_item_passed_count, \
        hist_item_course_used_count, \
        hist_item_school_used_count, \
        hist_item_challenge_count, \
        hist_item_evaluate_count, \
        hist_item_video_study_time, \
        hist_item_study_pdf_attachment_count, \
        last_item_created_time in tqdm(zip(user_hist_item_ids_dict.keys(), 
                                            user_hist_item_ids_dict.values(), 
                                            user_hist_item_visits_dict.values(),
                                            user_hist_item_averge_star_dict.values(),
                                            user_hist_item_study_count_dict.values(),
                                            user_hist_item_course_study_count_dict.values(),
                                            user_hist_item_passed_count_dict.values(),
                                            user_hist_item_course_used_count_dict.values(),
                                            user_hist_item_school_used_count_dict.values(),
                                            user_hist_item_challenge_count_dict.values(),
                                            user_hist_item_evaluate_count_dict.values(),
                                            user_hist_item_video_study_time_dict.values(),
                                            user_hist_item_study_pdf_attachment_count_dict.values(),
                                            user_last_item_created_time_dict.values())):

        user_hist_item_info_dict.setdefault(user_id, {})
        user_hist_item_info_dict[user_id]['hist_item_ids'] = hist_item_ids
        user_hist_item_info_dict[user_id]['hist_item_visits'] = hist_item_visits
        user_hist_item_info_dict[user_id]['hist_item_averge_star'] = hist_item_averge_star
        user_hist_item_info_dict[user_id]['hist_item_study_count'] = hist_item_study_count
        user_hist_item_info_dict[user_id]['hist_item_course_study_count'] = hist_item_course_study_count
        user_hist_item_info_dict[user_id]['hist_item_passed_count'] = hist_item_passed_count
        user_hist_item_info_dict[user_id]['hist_item_course_used_count'] = hist_item_course_used_count
        user_hist_item_info_dict[user_id]['hist_item_school_used_count'] = hist_item_school_used_count
        user_hist_item_info_dict[user_id]['hist_item_challenge_count'] = hist_item_challenge_count
        user_hist_item_info_dict[user_id]['hist_item_evaluate_count'] = hist_item_evaluate_count
        user_hist_item_info_dict[user_id]['hist_item_video_study_time'] = hist_item_video_study_time
        user_hist_item_info_dict[user_id]['hist_item_study_pdf_attachment_count'] = hist_item_study_pdf_attachment_count
        user_hist_item_info_dict[user_id]['last_item_created_time'] = last_item_created_time


    pickle.dump(user_hist_item_info_dict, open(subject_save_path + 'user_hist_item_info_dict.pkl', 'wb'))    
    return user_hist_item_info_dict


def get_item_topk_select(select_df, k):
    """
    获取被选择次数最多的课程，用来做召回补全
    """
    topk_select = select_df['subject_id'].value_counts().index[:k]
    return topk_select


def get_user_activate_degree_dict(all_select_df):
    """
    将用户的选择次数作为获取用户活跃度的指标
    """
    # 加载保存好的用户活跃度字典
    if os.path.exists(subject_save_path + 'user_activate_degree_dict.pkl'):
        user_activate_degree_dict = pickle.load(open(subject_save_path + 'user_activate_degree_dict.pkl', 'rb'))
        return user_activate_degree_dict

    all_select_df_ = all_select_df.groupby('user_id')['subject_id'].count().reset_index()
    
    # 用户活跃度归一化
    mm = MinMaxScaler()
    all_select_df_['subject_id'] = mm.fit_transform(all_select_df_[['subject_id']])
    user_activate_degree_dict = dict(zip(all_select_df_['user_id'], all_select_df_['subject_id']))

    # 保存生成好的用户活跃度字典
    pickle.dump(user_activate_degree_dict, open(subject_save_path + 'user_activate_degree_dict.pkl', 'wb'))
    return user_activate_degree_dict
    

def metrics_recall(user_recall_items_dict, train_last_select_df, topk=10):
    """
    召回评估，依次评估召回的前 10, 20, 30，...topk/10 个课程的击中率
    """
    # 生成用户最后选择课程的字典
    last_select_item_dict = dict(zip(train_last_select_df['user_id'], train_last_select_df['subject_id']))
    
    # 用户数量
    user_num = len(user_recall_items_dict)
    
    for k in range(10, topk + 1, 10):
        hit_num = 0
        for user_id, item_list in user_recall_items_dict.items():
            # 获取前k个召回的结果
            tmp_recall_items = [x[0] for x in user_recall_items_dict[user_id][:k]]

            if last_select_item_dict[user_id] in set(tmp_recall_items):
                hit_num += 1
        
        hit_rate = round(hit_num * 1.0 / user_num, 5)
        print(' topk: ', k, ' : ', 'hit_num: ', hit_num, 'hit_rate: ', hit_rate, 'user_num : ', user_num)

def metrics_pinsage_recall(user_recall_items_dict, train_last_select_df, topk=10):
    """
    召回评估，依次评估召回的前 10, 20, 30，...topk/10 个物品的击中率
    """
    # 生成用户最后选择物品的字典
    last_select_item_dict = dict(zip(train_last_select_df['user'], train_last_select_df['item']))
    
    # 用户数量
    user_num = len(user_recall_items_dict)
    print(user_num)
    
    for k in range(10, topk + 1, 10):
        hit_num = 0
        for user_id, item_list in user_recall_items_dict.items():
            # 获取前k个召回的结果
            tmp_recall_items = [x[0] for x in user_recall_items_dict[user_id][:k]]

            if last_select_item_dict[user_id] in set(tmp_recall_items):
                hit_num += 1
        
        hit_rate = round(hit_num * 1.0 / user_num, 5)
        print(' topk: ', k, ' : ', 'hit_num: ', hit_num, 'hit_rate: ', hit_rate, 'user_num : ', user_num)


if __name__ == '__main__':

    logger.info('获取课程信息')
    item_info_df = get_item_info_df()

    logger.info('生成课程信息字典')
    item_info_dict = get_recall_item_info_dict(item_info_df)