|
|
import os
|
|
|
import sys
|
|
|
sys.path.append(os.getcwd())
|
|
|
import pickle
|
|
|
import numpy as np
|
|
|
import pandas as pd
|
|
|
from tqdm import tqdm
|
|
|
from sklearn.preprocessing import LabelEncoder
|
|
|
from sklearn.preprocessing import MinMaxScaler
|
|
|
import warnings
|
|
|
from deepctr.models import BST
|
|
|
from deepctr.feature_column import SparseFeat, VarLenSparseFeat, DenseFeat, get_feature_names
|
|
|
from tensorflow.keras.preprocessing.sequence import pad_sequences
|
|
|
from tensorflow.keras.preprocessing import text
|
|
|
from tensorflow.keras import backend as K
|
|
|
from tensorflow.keras.layers import *
|
|
|
from tensorflow.keras.models import *
|
|
|
from tensorflow.keras.callbacks import *
|
|
|
import tensorflow as tf
|
|
|
from config import offline_mode, shixun_features_save_path
|
|
|
from config import shixun_train_user_item_feats
|
|
|
from config import shixun_val_user_item_feats
|
|
|
from config import shixun_test_user_item_feats
|
|
|
from config import shixun_model_save_path
|
|
|
from config import shixun_all_user_item_feats
|
|
|
from config import logger
|
|
|
from config import shixun_rank_dense_fea
|
|
|
from config import shixun_rank_sparse_fea
|
|
|
from config import shixun_max_seq_len
|
|
|
from config import shixun_rank_feats_columns
|
|
|
from matching.shixun.recall_comm import get_all_select_df
|
|
|
from matching.shixun.recall_comm import get_item_info_df
|
|
|
|
|
|
|
|
|
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
|
|
|
|
|
|
K.set_learning_phase(True)
|
|
|
if tf.__version__ >= '2.0.0':
|
|
|
tf.compat.v1.disable_eager_execution()
|
|
|
|
|
|
tqdm.pandas()
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
# 数据准备函数
|
|
|
def get_bst_feats_columns(df,
|
|
|
dense_fea,
|
|
|
sparse_fea,
|
|
|
behavior_fea,
|
|
|
his_behavior_fea,
|
|
|
emb_dim=32,
|
|
|
max_len=100):
|
|
|
"""
|
|
|
数据准备函数:
|
|
|
df: 数据集
|
|
|
dense_fea: 数值型特征列
|
|
|
sparse_fea: 离散型特征列
|
|
|
behavior_fea: 用户的候选行为特征列
|
|
|
his_behavior_fea: 用户的历史行为特征列
|
|
|
embedding_dim: embedding的维度，这里为了简单，统一把离散型特征列采用一样的隐向量维度
|
|
|
max_len: 用户序列的最大长度
|
|
|
"""
|
|
|
sparse_feature_columns = [SparseFeat(feat, vocabulary_size=df[feat].nunique() + 1,
|
|
|
embedding_dim = emb_dim) for feat in sparse_fea]
|
|
|
|
|
|
dense_feature_columns = [DenseFeat(feat, 1, ) for feat in dense_fea]
|
|
|
|
|
|
var_feature_columns = [VarLenSparseFeat(SparseFeat(feat, vocabulary_size=df['shixun_id'].nunique() + 1,
|
|
|
embedding_dim=emb_dim, embedding_name='shixun_id'), maxlen=max_len)
|
|
|
for feat in his_behavior_fea]
|
|
|
|
|
|
dnn_feature_columns = sparse_feature_columns + dense_feature_columns + var_feature_columns
|
|
|
|
|
|
# 建立x, x是一个字典的形式
|
|
|
x = {}
|
|
|
|
|
|
for name in get_feature_names(dnn_feature_columns):
|
|
|
if name in his_behavior_fea:
|
|
|
# 这是历史行为序列
|
|
|
his_list = [l for l in df[name]]
|
|
|
|
|
|
# 二维数组
|
|
|
x[name] = pad_sequences(his_list, maxlen=max_len, padding='post')
|
|
|
else:
|
|
|
x[name] = df[name].values
|
|
|
|
|
|
return x, dnn_feature_columns
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
logger.info('加载物品行为数据')
|
|
|
all_data = get_all_select_df()
|
|
|
|
|
|
logger.info('生成用户历史选择物品数据')
|
|
|
hist_select = all_data[['user_id', 'shixun_id']].groupby('user_id').agg({list}).reset_index()
|
|
|
his_behavior_df = pd.DataFrame()
|
|
|
|
|
|
his_behavior_df['user_id'] = hist_select['user_id']
|
|
|
his_behavior_df['hist_shixun_id'] = hist_select['shixun_id']
|
|
|
|
|
|
logger.info('获取物品信息')
|
|
|
shixun_info_df = get_item_info_df()
|
|
|
|
|
|
# 物品总数量
|
|
|
shixuns_count = len(shixun_info_df['shixun_id'].unique().tolist())
|
|
|
|
|
|
logger.info('加载用户行为特征')
|
|
|
|
|
|
# 所有用户物品特征
|
|
|
all_user_item_feats_df = pd.read_csv(shixun_all_user_item_feats, sep='\t', encoding='utf-8')
|
|
|
|
|
|
# shixun_id转换成int
|
|
|
train_user_item_feats_df = pd.read_csv(shixun_train_user_item_feats, sep='\t', encoding='utf-8')
|
|
|
train_user_item_feats_df['shixun_id'] = train_user_item_feats_df['shixun_id'].astype(int)
|
|
|
|
|
|
if offline_mode:
|
|
|
val_user_item_feats_df = pd.read_csv(shixun_val_user_item_feats, sep='\t', encoding='utf-8')
|
|
|
val_user_item_feats_df['shixun_id'] = val_user_item_feats_df['shixun_id'].astype(int)
|
|
|
else:
|
|
|
val_user_item_feats_df = None
|
|
|
|
|
|
test_user_item_feats_df = pd.read_csv(shixun_test_user_item_feats, sep='\t', encoding='utf-8')
|
|
|
test_user_item_feats_df['shixun_id'] = test_user_item_feats_df['shixun_id'].astype(int)
|
|
|
|
|
|
# 做特征的时候为了方便，给测试集也打上了一个无效的标签，这里直接删掉
|
|
|
del test_user_item_feats_df['label']
|
|
|
|
|
|
train_user_item_feats_df = train_user_item_feats_df.merge(his_behavior_df, on='user_id')
|
|
|
if offline_mode:
|
|
|
val_user_item_feats_df = val_user_item_feats_df.merge(his_behavior_df, on='user_id')
|
|
|
else:
|
|
|
val_user_item_feats_df = None
|
|
|
|
|
|
test_user_item_feats_df = test_user_item_feats_df.merge(his_behavior_df, on='user_id')
|
|
|
|
|
|
# 把特征分开
|
|
|
sparse_fea = shixun_rank_sparse_fea
|
|
|
|
|
|
behavior_fea = ['shixun_id']
|
|
|
|
|
|
hist_behavior_fea = ['hist_shixun_id']
|
|
|
|
|
|
dense_fea = shixun_rank_dense_fea
|
|
|
|
|
|
train_user_item_feats_df[dense_fea] = train_user_item_feats_df[dense_fea].fillna(0, )
|
|
|
if val_user_item_feats_df is not None:
|
|
|
val_user_item_feats_df[dense_fea] = val_user_item_feats_df[dense_fea].fillna(0, )
|
|
|
test_user_item_feats_df[dense_fea] = test_user_item_feats_df[dense_fea].fillna(0, )
|
|
|
|
|
|
# 处理inf的值
|
|
|
train_user_item_feats_df.replace([np.inf, -np.inf], 0, inplace=True)
|
|
|
test_user_item_feats_df.replace([np.inf, -np.inf], 0, inplace=True)
|
|
|
|
|
|
# dense特征进行归一化
|
|
|
for feat in dense_fea:
|
|
|
min_max_scaler = MinMaxScaler()
|
|
|
min_max_scaler.fit((all_user_item_feats_df[[feat]]))
|
|
|
pickle.dump(min_max_scaler, open(shixun_model_save_path + 'min_max_scaler_' + feat + '.model', 'wb'))
|
|
|
|
|
|
train_user_item_feats_df[feat] = min_max_scaler.transform(train_user_item_feats_df[[feat]])
|
|
|
if val_user_item_feats_df is not None:
|
|
|
val_user_item_feats_df[feat] = min_max_scaler.transform(val_user_item_feats_df[[feat]])
|
|
|
test_user_item_feats_df[feat] = min_max_scaler.transform(test_user_item_feats_df[[feat]])
|
|
|
|
|
|
# sparse特征进行LabelEncoder
|
|
|
for feat in sparse_fea:
|
|
|
label_encoder = LabelEncoder()
|
|
|
if feat == 'shixun_id':
|
|
|
label_encoder.fit(shixun_info_df[[feat]])
|
|
|
shixun_id_lable_encoder = label_encoder
|
|
|
else:
|
|
|
label_encoder.fit(all_user_item_feats_df[[feat]])
|
|
|
if feat == 'user_id':
|
|
|
user_id_label_encoder = label_encoder
|
|
|
pickle.dump(label_encoder, open(shixun_model_save_path + feat + '_label_encoder.model', 'wb'))
|
|
|
|
|
|
train_user_item_feats_df[feat] = label_encoder.transform(train_user_item_feats_df[[feat]])
|
|
|
if val_user_item_feats_df is not None:
|
|
|
val_user_item_feats_df[feat] = label_encoder.transform(val_user_item_feats_df[[feat]])
|
|
|
test_user_item_feats_df[feat] = label_encoder.transform(test_user_item_feats_df[[feat]])
|
|
|
|
|
|
# 准备训练数据
|
|
|
x_train, dnn_feature_columns = get_bst_feats_columns(train_user_item_feats_df,
|
|
|
dense_fea,
|
|
|
sparse_fea,
|
|
|
behavior_fea,
|
|
|
hist_behavior_fea,
|
|
|
max_len=shixun_max_seq_len)
|
|
|
y_train = train_user_item_feats_df['label'].values
|
|
|
|
|
|
if val_user_item_feats_df is not None:
|
|
|
# 准备验证数据
|
|
|
x_val, dnn_feature_columns = get_bst_feats_columns(val_user_item_feats_df,
|
|
|
dense_fea,
|
|
|
sparse_fea,
|
|
|
behavior_fea,
|
|
|
hist_behavior_fea,
|
|
|
max_len=shixun_max_seq_len)
|
|
|
y_val = val_user_item_feats_df['label'].values
|
|
|
|
|
|
dense_fea = [x for x in dense_fea if x != 'label']
|
|
|
x_test, dnn_feature_columns = get_bst_feats_columns(test_user_item_feats_df,
|
|
|
dense_fea,
|
|
|
sparse_fea,
|
|
|
behavior_fea,
|
|
|
hist_behavior_fea,
|
|
|
max_len=shixun_max_seq_len)
|
|
|
|
|
|
# 建立模型
|
|
|
model = BST(dnn_feature_columns, behavior_fea)
|
|
|
|
|
|
# 查看模型结构
|
|
|
model.summary()
|
|
|
|
|
|
# 模型编译
|
|
|
# model.compile('adam', 'binary_crossentropy', metrics=['binary_crossentropy', tf.keras.metrics.AUC()])
|
|
|
model.compile(optimizer="Adam",
|
|
|
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
|
|
|
metrics=['AUC']#评价指标：AUC,精确度，均方误差
|
|
|
)
|
|
|
# 模型训练
|
|
|
if val_user_item_feats_df is not None:
|
|
|
history = model.fit(x_train,
|
|
|
y_train,
|
|
|
verbose=1,
|
|
|
epochs=5,
|
|
|
validation_data=(x_val, y_val),
|
|
|
batch_size=128)
|
|
|
else:
|
|
|
# 也可以使用下面的语句用自己采样出来的验证集
|
|
|
history = model.fit(x_train,
|
|
|
y_train,
|
|
|
verbose=1,
|
|
|
epochs=5,
|
|
|
validation_split=0.3,
|
|
|
batch_size=128)
|
|
|
|
|
|
# 保存训练好的BST模型
|
|
|
model.save(shixun_model_save_path + 'bst_model.h5')
|
|
|
|
|
|
|
|
|
# 模型预测
|
|
|
test_user_item_feats_df['pred_score'] = model.predict(x_test, verbose=1, batch_size=256)
|
|
|
|
|
|
# 还原user_id和shixun_id
|
|
|
test_user_item_feats_df['user_id'] = user_id_label_encoder.inverse_transform(test_user_item_feats_df[['user_id']])
|
|
|
test_user_item_feats_df['shixun_id'] = shixun_id_lable_encoder.inverse_transform(test_user_item_feats_df[['shixun_id']])
|
|
|
|
|
|
item_info_df = get_item_info_df()
|
|
|
item_info_df = item_info_df[['shixun_id', 'shixun_name']]
|
|
|
test_user_item_feats_df = test_user_item_feats_df.merge(item_info_df, how='left', on='shixun_id')
|
|
|
|
|
|
test_user_item_feats_df.sort_values(by=['user_id', 'pred_score'], ascending=[True, False], inplace=True)
|
|
|
test_user_item_feats_df['pred_rank'] = test_user_item_feats_df.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first').astype(int)
|
|
|
|
|
|
for feat in shixun_rank_feats_columns:
|
|
|
min_max_scaler = pickle.load(open(shixun_model_save_path + 'min_max_scaler_' + feat + '.model', 'rb'))
|
|
|
|
|
|
train_user_item_feats_df[feat] = min_max_scaler.inverse_transform(train_user_item_feats_df[[feat]])
|
|
|
if val_user_item_feats_df is not None:
|
|
|
val_user_item_feats_df[feat] = min_max_scaler.inverse_transform(val_user_item_feats_df[[feat]])
|
|
|
test_user_item_feats_df[feat] = min_max_scaler.inverse_transform(test_user_item_feats_df[[feat]])
|
|
|
|
|
|
test_user_item_feats_df[['user_id', 'shixun_id', 'shixun_name'] + shixun_rank_feats_columns + ['pred_score', 'pred_rank']].to_csv(
|
|
|
shixun_features_save_path + 'bst_rank_score.csv', sep='\t', index=False, header=True)
|
