diff --git a/hpo/er_model_hpo.py b/hpo/er_model_hpo.py index 36ee954..af8b14c 100644 --- a/hpo/er_model_hpo.py +++ b/hpo/er_model_hpo.py @@ -13,16 +13,12 @@ class Classifier: @property def configspace(self) -> ConfigurationSpace: cs = ConfigurationSpace(seed=0) - ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1') - selected_attrs = ltable.columns.values.tolist() - block_attr_items = selected_attrs[:] - block_attr_items.remove(ltable_id) - - block_attr = Categorical("block_attr", block_attr_items) ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf") - ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap") + # todo 每个分类器的超参数 + tree_criterion = Categorical("dt_criterion", ["gini", "entropy", "log_loss"], default="gini") + - cs.add_hyperparameters([block_attr, ml_matcher, ml_blocker]) + cs.add_hyperparameters([ml_matcher]) return cs def train(self, config: Configuration, seed: int = 0) -> float: diff --git a/md_discovery/md_discover.py b/md_discovery/md_discover.py index 0617990..9b74450 100644 --- a/md_discovery/md_discover.py +++ b/md_discovery/md_discover.py @@ -19,16 +19,9 @@ from settings import * def md_discover(config: Configuration, source_path, target_path): - # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 - # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值 - # mds_list = pairs_inference(source_path, target_attr) mds_list = discover(source_path, target_attr) with open(target_path, 'w') as f: for _ in mds_list: f.write('Target:'+str(target_attr) + '\t') f.write(str(_)) f.write('\n') - - -# if __name__ == '__main__': -# md_discover() diff --git a/md_discovery/md_mining.py b/md_discovery/md_mining.py new file mode 100644 index 0000000..8405be4 --- /dev/null +++ b/md_discovery/md_mining.py @@ -0,0 +1,157 @@ +import pandas as pd +import torch +import matplotlib.pyplot as plt +from torch import LongTensor +from tqdm import tqdm + +from settings import * + + +# note 对表进行嵌入时定位了有空值的cell, 计算相似度时有空值则置为-1.0000 +def mining(train: pd.DataFrame): + # data is train set, in which each row represents a tuple pair + train = train.astype(str) + + # 尝试不将左右表key手动调整相同，而是只看gold属性是否为1 + # 故将左右表key直接去除 + data = train.drop(columns=['_id', 'ltable_' + ltable_id, 'rtable_' + rtable_id], inplace=False) + # data中现存属性：除key以外左右表属性和gold, 不含_id + columns = data.columns.values.tolist() + + # 列表, 每个元素为二元组, 包含对应列的索引 + col_tuple_list = build_col_tuple_list(columns) + + length = data.shape[0] + width = data.shape[1] + + # 嵌入data每一个cell, 纵向遍历 + # note 此处已重设索引 + data = data.reset_index(drop=True) + sentences = data.values.flatten(order='F').tolist() + embedding = model.encode(sentences, convert_to_tensor=True, device="cuda", batch_size=256, show_progress_bar=True) + + split_embedding = torch.split(embedding, length, dim=0) + table_tensor = torch.stack(split_embedding, dim=0, out=None) + norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2) + + # sim_tensor_dict = {} + sim_tensor_list = [] + for col_tuple in col_tuple_list: + mask = ((data[columns[col_tuple[0]]].isin([''])) | (data[columns[col_tuple[1]]].isin(['']))) + empty_string_indices = data[mask].index.tolist() + + lattr_tensor = norm_table_tensor[col_tuple[0]] + rattr_tensor = norm_table_tensor[col_tuple[1]] + mul_tensor = lattr_tensor * rattr_tensor + sim_tensor = torch.sum(mul_tensor, 1) + # 将有空字符串的位置强制置为-1.0000 + sim_tensor = sim_tensor.scatter(0, torch.tensor(empty_string_indices, device='cuda').long(), -1.0000) + sim_tensor = torch.round(sim_tensor, decimals=2) + sim_tensor_list.append(sim_tensor.unsqueeze(1)) + # sim_tensor_dict[columns[col_tuple[0]].replace('ltable_', '')] = sim_tensor + sim_table_tensor = torch.cat(sim_tensor_list, dim=1) + + # 创建一个1列的tensor，长度与相似度张量相同，先初始化为全0 + label_tensor = torch.zeros((sim_table_tensor.size(0), 1), device='cuda') + # 生成带标签的相似度张量 + sim_table_tensor_labeled = torch.cat((sim_table_tensor, label_tensor), 1) + # 找到匹配元组对的行索引 + mask = (data['gold'].isin(['1'])) + match_pair_indices = data[mask].index.tolist() + # 根据索引将匹配的行标签置为1 + sim_table_tensor_labeled[match_pair_indices, -1] = 1.00 + + md_list = init_md_list(len(col_tuple_list)) + result_md_list = [] + + sorted_unique_value_tensor_list = [] + for _ in range(len(col_tuple_list)): + # 将sim_table_tensor每一列的值从小到大排列，加入列表 + sorted_unique_value_tensor = torch.sort(sim_table_tensor[:, _].unique()).values + # 将每一列可能的相似度取值中小于0的都删掉 + sorted_unique_value_tensor = sorted_unique_value_tensor[sorted_unique_value_tensor >= 0] + sorted_unique_value_tensor_list.append(sorted_unique_value_tensor) + + result_list = [] + # 遍历MD列表, 将满足的直接加入结果列表, 不满足的看能否收紧, 不能收紧直接跳过 + # 若能收紧则将收紧后的一个个加入暂存列表, 并在该轮遍历结束后替换MD列表, 直到MD列表为空 + while len(md_list) > 0: + tmp_list = [] + for md_tensor in tqdm(md_list): + md_tensor_labeled = torch.cat((md_tensor, torch.tensor([0.5], device='cuda')), 0) + abs_support, confidence = get_metrics(md_tensor_labeled, sim_table_tensor_labeled) + # 如果support小于1, 没必要收紧阈值, 跳过 + if abs_support >= 1: + # 如果support满足但confidence不满足, 需要收紧阈值 + if confidence < confidence_threshold: + for _ in range(len(md_tensor)): + new_md_tensor = md_tensor.clone() + if new_md_tensor[_] == -1.00: + new_md_tensor[_] = sorted_unique_value_tensor_list[_][0] + if len(tmp_list) == 0: + tmp_list.append(new_md_tensor) + else: + stacked_tmp_tensors = torch.stack(tmp_list) + is_contained = (stacked_tmp_tensors == new_md_tensor) .all(dim=1).any() + if not is_contained: + tmp_list.append(new_md_tensor) + else: + a_tensor = sorted_unique_value_tensor_list[_] + b_value = new_md_tensor[_] + next_index = torch.where(a_tensor == b_value)[0].item() + 1 + if next_index < len(a_tensor): + new_md_tensor[_] = a_tensor[next_index] + tmp_list.append(new_md_tensor) + # torch.where(sorted_unique_value_tensor_list[2] == 0.16)[0].item() + # 如果都满足, 直接加进结果列表 + else: + result_list.append(md_tensor) + md_list = tmp_list + + print(1) + # sim_tensor = torch.matmul(norm_table_tensor, norm_table_tensor.transpose(1, 2)) + # sim_tensor = sim_tensor.float() + # sim_tensor = torch.round(sim_tensor, decimals=4) + + +def build_col_tuple_list(columns_): + col_tuple_list_ = [] + for _ in columns_: + if _.startswith('ltable'): + left_index = columns_.index(_) + right_index = columns_.index(_.replace('ltable_', 'rtable_')) + col_tuple_list_.append((left_index, right_index)) + return col_tuple_list_ + + +def init_md_list(md_dimension: int): + md_list_ = [] + # 创建全为-1的初始MD, 保留两位小数 + init_md_tensor = torch.full((md_dimension, ), -1.0, device='cuda') + init_md_tensor = torch.round(init_md_tensor, decimals=2) + md_list_.append(init_md_tensor) + return md_list_ + + +def get_metrics(md_tensor_labeled_, sim_table_tensor_labeled_): + table_tensor_length = sim_table_tensor_labeled_.size()[0] + # MD原本为列向量, 转置为行向量 + md_tensor_labeled_2d = md_tensor_labeled_.unsqueeze(1).transpose(0, 1) + # 沿行扩展1倍(不扩展), 沿列扩展至与相似度表同样长 + md_tensor_labeled_2d = md_tensor_labeled_2d.repeat(table_tensor_length, 1) + # 去掉标签列, 判断每一行相似度是否大于等于MD要求, 该张量行数与sim_table_tensor_labeled_相同, 少一列标签列 + support_tensor = torch.ge(sim_table_tensor_labeled_[:, :-1], md_tensor_labeled_2d[:, :-1]) + # 沿行方向判断support_tensor每一行是否都为True, 行数不变, 压缩为1列 + support_tensor = torch.all(support_tensor, dim=1, keepdim=True) + # 统计这个tensor中True的个数, 即为absolute support + abs_support_ = torch.sum(support_tensor).item() + + # 保留标签列, 判断每一行相似度是否大于等于MD要求 + support_tensor = torch.ge(sim_table_tensor_labeled_, md_tensor_labeled_2d) + # 统计既满足相似度要求也匹配的, abs_strict_support表示左右都满足的个数 + support_tensor = torch.all(support_tensor, dim=1, keepdim=True) + abs_strict_support_ = torch.sum(support_tensor).item() + # 计算confidence + confidence_ = abs_strict_support_ / abs_support_ if abs_support_ > 0 else 0 + return abs_support_, confidence_ + diff --git a/ml_er/magellan_new.py b/ml_er/magellan_new.py new file mode 100644 index 0000000..14d27d0 --- /dev/null +++ b/ml_er/magellan_new.py @@ -0,0 +1,75 @@ +import time +import pandas as pd +import py_entitymatching as em +import py_entitymatching.catalog.catalog_manager as cm +from tqdm import tqdm + +from md_discovery.md_mining import mining +from settings import * + + +def blocking_mining(): + start = time.time() + ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1') + cm.set_key(ltable, ltable_id) + rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1') + cm.set_key(rtable, rtable_id) + mappings = pd.read_csv(mapping_path, encoding='ISO-8859-1') + matching_number = len(mappings) + if ltable_id == rtable_id: + tables_id = rtable_id + attributes = ltable.columns.values.tolist() + lattributes = ['ltable_' + i for i in attributes] + rattributes = ['rtable_' + i for i in attributes] + cm.set_key(ltable, ltable_id) + cm.set_key(rtable, rtable_id) + + blocker = em.OverlapBlocker() + candidate = blocker.block_tables(ltable, rtable, ltable_block_attr, rtable_block_attr, allow_missing=True, + l_output_attrs=attributes, r_output_attrs=attributes, n_jobs=1, + overlap_size=1, show_progress=False) + candidate['gold'] = 0 + candidate = candidate.reset_index(drop=True) + + # 根据mapping表标注数据 + candidate_match_rows = [] + for t in tqdm(mappings.itertuples()): + mask = ((candidate['ltable_' + ltable_id].isin([getattr(t, 'ltable_id')])) & + (candidate['rtable_' + rtable_id].isin([getattr(t, 'rtable_id')]))) + matching_indices = candidate[mask].index + candidate_match_rows.extend(matching_indices.tolist()) + match_rows_mask = candidate.index.isin(candidate_match_rows) + candidate.loc[match_rows_mask, 'gold'] = 1 + candidate.fillna(value="", inplace=True) + + candidate_mismatch = candidate[candidate['gold'] == 0] + candidate_match = candidate[candidate['gold'] == 1] + candidate_mismatch = candidate_mismatch.sample(n=3*len(candidate_match)) + candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) + # 如果拼接后不重设索引可能导致索引重复 + candidate_for_train_test = candidate_for_train_test.reset_index(drop=True) + cm.set_key(candidate_for_train_test, '_id') + cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id) + cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id) + cm.set_ltable(candidate_for_train_test, ltable) + cm.set_rtable(candidate_for_train_test, rtable) + block_recall = len(candidate_match) / matching_number + + # 分为训练测试集 + train_proportion = 0.5 + sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0) + train_set = sets['train'] + test_set = sets['test'] + end_blocking = time.time() + print(end_blocking - start) + + mining(train_set) + return 1 + + +def matching(): + return 1 + + +if __name__ == '__main__': + blocking_mining() diff --git a/ml_er/ml_entity_resolver.py b/ml_er/ml_entity_resolver.py index 2721ce7..c65847e 100644 --- a/ml_er/ml_entity_resolver.py +++ b/ml_er/ml_entity_resolver.py @@ -170,8 +170,6 @@ def er_process(config: Configuration): rid_mapping_list.append(row[mapping_rid]) selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)] - # if len(lr_attrs_map) > 0: - # selected_ltable = selected_ltable.rename(columns=lr_attrs_map) # 参照右表，修改左表中与右表对应但不同名的字段 tables_id = rtable_id selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)] selected_attrs = selected_ltable.columns.values.tolist() # 两张表中的字段名 @@ -180,17 +178,11 @@ def er_process(config: Configuration): cm.set_key(selected_ltable, tables_id) cm.set_key(selected_rtable, tables_id) - if config["ml_blocker"] == "over_lap": - blocker = em.OverlapBlocker() - candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], - config["block_attr"], allow_missing=True, - l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, - overlap_size=1, show_progress=False) - elif config["ml_blocker"] == "attr_equiv": - blocker = em.AttrEquivalenceBlocker() - candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], - config["block_attr"], allow_missing=True, - l_output_attrs=selected_attrs, r_output_attrs=selected_attrs) + blocker = em.OverlapBlocker() + candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], + config["block_attr"], allow_missing=True, + l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, + overlap_size=1, show_progress=False) candidate['gold'] = 0 candidate = candidate.reset_index(drop=True) @@ -228,6 +220,7 @@ def er_process(config: Configuration): cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + tables_id) cm.set_ltable(candidate_for_train_test, selected_ltable) cm.set_rtable(candidate_for_train_test, selected_rtable) + block_recall = len(candidate_match) / matching_number # 分为训练测试集 train_proportion = 0.7 @@ -295,6 +288,7 @@ def er_process(config: Configuration): predictions_attrs.extend(attrs_with_r_prefix) predictions_attrs.extend(['gold', 'predicted']) predictions = predictions[predictions_attrs] + # 必须从训练集内挖MD train_attrs = predictions_attrs[:] train_attrs.remove('predicted') train_set = train_set[train_attrs] diff --git a/settings.py b/settings.py index ea06d98..7ebfeb8 100644 --- a/settings.py +++ b/settings.py @@ -1,20 +1,22 @@ from sentence_transformers import SentenceTransformer -ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\tableA.csv' -rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\tableB.csv' -mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\matches.csv' +ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableA.csv' +rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableB.csv' +mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\matches.csv' mapping_lid = 'ltable_id' # mapping表中左表id名 mapping_rid = 'rtable_id' # mapping表中右表id名 +ltable_block_attr = 'name' +rtable_block_attr = 'name' ltable_id = 'id' # 左表id字段名称 rtable_id = 'id' # 右表id字段名称 target_attr = 'id' # 进行md挖掘时的目标字段 # lr_attrs_map = {} # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致 -model = SentenceTransformer('E:\\Data\\Research\\Models\\roberta-large-nli-stsb-mean-tokens') +model = SentenceTransformer('E:\\Data\\Research\\Models\\all-MiniLM-L6-v2') interpre_weight = 1 # 可解释性权重 similarity_threshold = 0.1 support_threshold = 1 -confidence_threshold = 0.25 +confidence_threshold = 0.6 er_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\ml_er\\output\\' md_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\md_discovery\\output\\'