|
|
import random
|
|
|
import pandas as pd
|
|
|
import numpy as np
|
|
|
from copy import deepcopy
|
|
|
|
|
|
random.seed(0)
|
|
|
|
|
|
class DataProcess(object):
|
|
|
def __init__(self, filename):
|
|
|
self._filename = filename
|
|
|
self._loadData()
|
|
|
self._preProcess()
|
|
|
self._binarize(self._originalRatings)
|
|
|
# 对'userId'这一列的数据，先去重，然后构成一个用户列表
|
|
|
self._userPool = set(self._originalRatings['userId'].unique())
|
|
|
self._itemPool = set(self._originalRatings['itemId'].unique())
|
|
|
# print("user_pool size: ", len(self._userPool))
|
|
|
# print("item_pool size: ", len(self._itemPool))
|
|
|
|
|
|
self._select_Negatives(self._originalRatings)
|
|
|
self._split_pool(self._preprocessRatings)
|
|
|
|
|
|
def _loadData(self):
|
|
|
self._originalRatings = pd.read_csv(self._filename, sep='::', header=None, names=['uid', 'mid', 'rating', 'timestamp'],
|
|
|
engine='python')
|
|
|
return self._originalRatings
|
|
|
|
|
|
def _preProcess(self):
|
|
|
"""
|
|
|
对user和item都重新编号，这里这么做的原因是因为，模型的输入是one-hot向量，需要把user和item都限制在Embedding的长度之内，
|
|
|
模型的两个输入的长度分别是user和item的数量，所以要重新从0编号。
|
|
|
"""
|
|
|
# 1. 新建名为"userId"的列，这列对用户从0开始编号
|
|
|
user_id = self._originalRatings[['uid']].drop_duplicates().reindex()
|
|
|
user_id['userId'] = np.arange(len(user_id)) #根据user的长度创建一个数组
|
|
|
# 将原先的DataFrame与user_id按照"uid"这一列进行合并
|
|
|
self._originalRatings = pd.merge(self._originalRatings, user_id, on=['uid'], how='left')
|
|
|
|
|
|
# 2. 对物品进行重新排列
|
|
|
item_id = self._originalRatings[['mid']].drop_duplicates()
|
|
|
item_id['itemId'] = np.arange(len(item_id))
|
|
|
self._originalRatings = pd.merge(self._originalRatings, item_id, on=['mid'], how='left')
|
|
|
|
|
|
# 按照['userId', 'itemId', 'rating', 'timestamp']的顺序重新排列
|
|
|
self._originalRatings = self._originalRatings[['userId', 'itemId', 'rating', 'timestamp']]
|
|
|
# print(self._originalRatings)
|
|
|
# print('Range of userId is [{}, {}]'.format(self._originalRatings.userId.min(), self._originalRatings.userId.max()))
|
|
|
# print('Range of itemId is [{}, {}]'.format(self._originalRatings.itemId.min(), self._originalRatings.itemId.max()))
|
|
|
|
|
|
def _binarize(self, ratings):
|
|
|
"""
|
|
|
binarize data into 0 or 1 for implicit feedback
|
|
|
"""
|
|
|
ratings = deepcopy(ratings)
|
|
|
ratings['rating'][ratings['rating'] > 0] = 1.0
|
|
|
self._preprocessRatings = ratings
|
|
|
# print("binary: \n", self._preprocessRatings)
|
|
|
|
|
|
def _select_Negatives(self, ratings):
|
|
|
"""
|
|
|
Select al;l negative samples and 100 sampled negative items for each user.
|
|
|
"""
|
|
|
# 构造user-item表
|
|
|
interact_status = ratings.groupby('userId')['itemId'].apply(set).reset_index().rename(
|
|
|
columns={'itemId': 'interacted_items'})
|
|
|
# print("interact_status: \n", interact_status)
|
|
|
|
|
|
# 把与用户没有产生过交互的样本都当做是负样本
|
|
|
interact_status['negative_items'] = interact_status['interacted_items'].apply(lambda x: self._itemPool - x)
|
|
|
|
|
|
# 从上面的全部负样本中随机选99个出来
|
|
|
interact_status['negative_samples'] = interact_status['negative_items'].apply(lambda x: random.sample(x, 99))
|
|
|
# print("after sampling interact_status: \n", interact_status)
|
|
|
#
|
|
|
# print("select and rearrange columns")
|
|
|
self._negatives = interact_status[['userId', 'negative_items', 'negative_samples']]
|
|
|
|
|
|
def _split_pool(self, ratings):
|
|
|
"""leave one out train/test split """
|
|
|
# print("sort by timestamp descend")
|
|
|
# 先按照'userID'进行分组，然后根据时间戳降序排列
|
|
|
ratings['rank_latest'] = ratings.groupby(['userId'])['timestamp'].rank(method='first', ascending=False)
|
|
|
# print(ratings)
|
|
|
|
|
|
# 选取排名第一的数据作为测试集，也就是最新的那个数据
|
|
|
test = ratings[ratings['rank_latest'] == 1]
|
|
|
# 选取所有排名靠后的，也就是历史数据当做训练集
|
|
|
train = ratings[ratings['rank_latest'] > 1]
|
|
|
# print("test: \n", test)
|
|
|
# print("train: \n", train)
|
|
|
|
|
|
# print("size of test {0}, size of train {1}".format(len(test), len(train)))
|
|
|
|
|
|
# 确保训练集和测试集的userId是一样的
|
|
|
assert train['userId'].nunique() == test['userId'].nunique()
|
|
|
|
|
|
self.train_ratings = train[['userId', 'itemId', 'rating']]
|
|
|
self.test_ratings = test[['userId', 'itemId', 'rating']]
|
|
|
|
|
|
def sample_generator(self, num_negatives):
|
|
|
# 合并之后的train_ratings的列包括['userId','itemId'，'rating','negative_items']
|
|
|
train_ratings = pd.merge(self.train_ratings, self._negatives[['userId', 'negative_items']], on='userId')
|
|
|
# 从用户的全部负样本集合中随机选择num_negatives个样本当做负样本，并产生一个新的名为"negatives"的列
|
|
|
train_ratings['negatives'] = train_ratings['negative_items'].apply(lambda x: random.sample(x, num_negatives))
|
|
|
# print(train_ratings)
|
|
|
|
|
|
# 构造模型所需要的数据，分别是输入user、items以及目标分值ratings。
|
|
|
users, items, ratings = [], [], []
|
|
|
for row in train_ratings.itertuples():
|
|
|
# 构造正样本，分别是userId， itemId以及目标分值1
|
|
|
users.append(int(row.userId))
|
|
|
items.append(int(row.itemId))
|
|
|
ratings.append(float(row.rating))
|
|
|
# 为每个用户构造num_negatives个负样本，分别是userId， itemId以及目标分值0
|
|
|
for i in range(num_negatives):
|
|
|
users.append(int(row.userId))
|
|
|
items.append(int(row.negatives[i]))
|
|
|
ratings.append(float(0)) # 负样本的ratings为0，直接强行设置为0
|
|
|
|
|
|
return users, items, ratings
|
|
|
|
|
|
def test_generator(self, num_negatives):
|
|
|
# 合并之后的train_ratings的列包括['userId','itemId'，'rating','negative_items']
|
|
|
test_ratings = pd.merge(self.test_ratings, self._negatives[['userId', 'negative_items']], on='userId')
|
|
|
# 从用户的全部负样本集合中随机选择num_negatives个样本当做负样本，并产生一个新的名为"negatives"的列
|
|
|
test_ratings['negatives'] = test_ratings['negative_items'].apply(lambda x: random.sample(x, num_negatives))
|
|
|
# print(test_ratings)
|
|
|
|
|
|
# 构造模型所需要的数据，分别是输入user、items以及目标分值ratings。
|
|
|
users, items, ratings = [], [], []
|
|
|
for row in test_ratings.itertuples():
|
|
|
# 构造正样本，分别是userId， itemId以及目标分值1
|
|
|
users.append(int(row.userId))
|
|
|
items.append(int(row.itemId))
|
|
|
ratings.append(float(row.rating))
|
|
|
# 为每个用户构造num_negatives个负样本，分别是userId， itemId以及目标分值0
|
|
|
for i in range(num_negatives):
|
|
|
users.append(int(row.userId))
|
|
|
items.append(int(row.negatives[i]))
|
|
|
ratings.append(float(0)) # 负样本的ratings为0，直接强行设置为0
|
|
|
|
|
|
return users, items, ratings |
