将blocking阶段单拎出来;

重写了MD挖掘代码(后续要改成随机生成加过滤)

hpo/er_model_hpo.py

@@ -13,16 +13,12 @@ class Classifier:
     @property
     def configspace(self) -> ConfigurationSpace:
         cs = ConfigurationSpace(seed=0)
-        ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
-        selected_attrs = ltable.columns.values.tolist()
-        block_attr_items = selected_attrs[:]
-        block_attr_items.remove(ltable_id)
-
-        block_attr = Categorical("block_attr", block_attr_items)
         ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf")
-        ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap")
+        # todo 每个分类器的超参数
+        tree_criterion = Categorical("dt_criterion", ["gini", "entropy", "log_loss"], default="gini")
+
 
-        cs.add_hyperparameters([block_attr, ml_matcher, ml_blocker])
+        cs.add_hyperparameters([ml_matcher])
         return cs
 
     def train(self, config: Configuration, seed: int = 0) -> float:

md_discovery/md_discover.py

@@ -19,16 +19,9 @@ from settings import *
 
 
 def md_discover(config: Configuration, source_path, target_path):
-    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
-    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值
-    # mds_list = pairs_inference(source_path, target_attr)
     mds_list = discover(source_path, target_attr)
     with open(target_path, 'w') as f:
         for _ in mds_list:
             f.write('Target:'+str(target_attr) + '\t')
             f.write(str(_))
             f.write('\n')
-
-
-# if __name__ == '__main__':
-#     md_discover()

md_discovery/md_mining.py

@@ -0,0 +1,157 @@
+import pandas as pd
+import torch
+import matplotlib.pyplot as plt
+from torch import LongTensor
+from tqdm import tqdm
+
+from settings import *
+
+
+# note 对表进行嵌入时定位了有空值的cell, 计算相似度时有空值则置为-1.0000
+def mining(train: pd.DataFrame):
+    # data is train set, in which each row represents a tuple pair
+    train = train.astype(str)
+
+    # 尝试不将左右表key手动调整相同，而是只看gold属性是否为1
+    # 故将左右表key直接去除
+    data = train.drop(columns=['_id', 'ltable_' + ltable_id, 'rtable_' + rtable_id], inplace=False)
+    # data中现存属性：除key以外左右表属性和gold, 不含_id
+    columns = data.columns.values.tolist()
+
+    # 列表, 每个元素为二元组, 包含对应列的索引
+    col_tuple_list = build_col_tuple_list(columns)
+
+    length = data.shape[0]
+    width = data.shape[1]
+
+    # 嵌入data每一个cell, 纵向遍历
+    # note 此处已重设索引
+    data = data.reset_index(drop=True)
+    sentences = data.values.flatten(order='F').tolist()
+    embedding = model.encode(sentences, convert_to_tensor=True, device="cuda", batch_size=256, show_progress_bar=True)
+
+    split_embedding = torch.split(embedding, length, dim=0)
+    table_tensor = torch.stack(split_embedding, dim=0, out=None)
+    norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2)
+
+    # sim_tensor_dict = {}
+    sim_tensor_list = []
+    for col_tuple in col_tuple_list:
+        mask = ((data[columns[col_tuple[0]]].isin([''])) | (data[columns[col_tuple[1]]].isin([''])))
+        empty_string_indices = data[mask].index.tolist()
+
+        lattr_tensor = norm_table_tensor[col_tuple[0]]
+        rattr_tensor = norm_table_tensor[col_tuple[1]]
+        mul_tensor = lattr_tensor * rattr_tensor
+        sim_tensor = torch.sum(mul_tensor, 1)
+        # 将有空字符串的位置强制置为-1.0000
+        sim_tensor = sim_tensor.scatter(0, torch.tensor(empty_string_indices, device='cuda').long(), -1.0000)
+        sim_tensor = torch.round(sim_tensor, decimals=2)
+        sim_tensor_list.append(sim_tensor.unsqueeze(1))
+        # sim_tensor_dict[columns[col_tuple[0]].replace('ltable_', '')] = sim_tensor
+    sim_table_tensor = torch.cat(sim_tensor_list, dim=1)
+
+    # 创建一个1列的tensor，长度与相似度张量相同，先初始化为全0
+    label_tensor = torch.zeros((sim_table_tensor.size(0), 1), device='cuda')
+    # 生成带标签的相似度张量
+    sim_table_tensor_labeled = torch.cat((sim_table_tensor, label_tensor), 1)
+    # 找到匹配元组对的行索引
+    mask = (data['gold'].isin(['1']))
+    match_pair_indices = data[mask].index.tolist()
+    # 根据索引将匹配的行标签置为1
+    sim_table_tensor_labeled[match_pair_indices, -1] = 1.00
+
+    md_list = init_md_list(len(col_tuple_list))
+    result_md_list = []
+
+    sorted_unique_value_tensor_list = []
+    for _ in range(len(col_tuple_list)):
+        # 将sim_table_tensor每一列的值从小到大排列，加入列表
+        sorted_unique_value_tensor = torch.sort(sim_table_tensor[:, _].unique()).values
+        # 将每一列可能的相似度取值中小于0的都删掉
+        sorted_unique_value_tensor = sorted_unique_value_tensor[sorted_unique_value_tensor >= 0]
+        sorted_unique_value_tensor_list.append(sorted_unique_value_tensor)
+
+    result_list = []
+    # 遍历MD列表, 将满足的直接加入结果列表, 不满足的看能否收紧, 不能收紧直接跳过
+    # 若能收紧则将收紧后的一个个加入暂存列表, 并在该轮遍历结束后替换MD列表, 直到MD列表为空
+    while len(md_list) > 0:
+        tmp_list = []
+        for md_tensor in tqdm(md_list):
+            md_tensor_labeled = torch.cat((md_tensor, torch.tensor([0.5], device='cuda')), 0)
+            abs_support, confidence = get_metrics(md_tensor_labeled, sim_table_tensor_labeled)
+            # 如果support小于1, 没必要收紧阈值, 跳过
+            if abs_support >= 1:
+                # 如果support满足但confidence不满足, 需要收紧阈值
+                if confidence < confidence_threshold:
+                    for _ in range(len(md_tensor)):
+                        new_md_tensor = md_tensor.clone()
+                        if new_md_tensor[_] == -1.00:
+                            new_md_tensor[_] = sorted_unique_value_tensor_list[_][0]
+                            if len(tmp_list) == 0:
+                                tmp_list.append(new_md_tensor)
+                            else:
+                                stacked_tmp_tensors = torch.stack(tmp_list)
+                                is_contained = (stacked_tmp_tensors == new_md_tensor) .all(dim=1).any()
+                                if not is_contained:
+                                    tmp_list.append(new_md_tensor)
+                        else:
+                            a_tensor = sorted_unique_value_tensor_list[_]
+                            b_value = new_md_tensor[_]
+                            next_index = torch.where(a_tensor == b_value)[0].item() + 1
+                            if next_index < len(a_tensor):
+                                new_md_tensor[_] = a_tensor[next_index]
+                                tmp_list.append(new_md_tensor)
+                    # torch.where(sorted_unique_value_tensor_list[2] == 0.16)[0].item()
+                # 如果都满足, 直接加进结果列表
+                else:
+                    result_list.append(md_tensor)
+        md_list = tmp_list
+
+    print(1)
+    # sim_tensor = torch.matmul(norm_table_tensor, norm_table_tensor.transpose(1, 2))
+    # sim_tensor = sim_tensor.float()
+    # sim_tensor = torch.round(sim_tensor, decimals=4)
+
+
+def build_col_tuple_list(columns_):
+    col_tuple_list_ = []
+    for _ in columns_:
+        if _.startswith('ltable'):
+            left_index = columns_.index(_)
+            right_index = columns_.index(_.replace('ltable_', 'rtable_'))
+            col_tuple_list_.append((left_index, right_index))
+    return col_tuple_list_
+
+
+def init_md_list(md_dimension: int):
+    md_list_ = []
+    # 创建全为-1的初始MD, 保留两位小数
+    init_md_tensor = torch.full((md_dimension, ), -1.0, device='cuda')
+    init_md_tensor = torch.round(init_md_tensor, decimals=2)
+    md_list_.append(init_md_tensor)
+    return md_list_
+
+
+def get_metrics(md_tensor_labeled_, sim_table_tensor_labeled_):
+    table_tensor_length = sim_table_tensor_labeled_.size()[0]
+    # MD原本为列向量, 转置为行向量
+    md_tensor_labeled_2d = md_tensor_labeled_.unsqueeze(1).transpose(0, 1)
+    # 沿行扩展1倍(不扩展), 沿列扩展至与相似度表同样长
+    md_tensor_labeled_2d = md_tensor_labeled_2d.repeat(table_tensor_length, 1)
+    # 去掉标签列, 判断每一行相似度是否大于等于MD要求, 该张量行数与sim_table_tensor_labeled_相同, 少一列标签列
+    support_tensor = torch.ge(sim_table_tensor_labeled_[:, :-1], md_tensor_labeled_2d[:, :-1])
+    # 沿行方向判断support_tensor每一行是否都为True, 行数不变, 压缩为1列
+    support_tensor = torch.all(support_tensor, dim=1, keepdim=True)
+    # 统计这个tensor中True的个数, 即为absolute support
+    abs_support_ = torch.sum(support_tensor).item()
+
+    # 保留标签列, 判断每一行相似度是否大于等于MD要求
+    support_tensor = torch.ge(sim_table_tensor_labeled_, md_tensor_labeled_2d)
+    # 统计既满足相似度要求也匹配的, abs_strict_support表示左右都满足的个数
+    support_tensor = torch.all(support_tensor, dim=1, keepdim=True)
+    abs_strict_support_ = torch.sum(support_tensor).item()
+    # 计算confidence
+    confidence_ = abs_strict_support_ / abs_support_ if abs_support_ > 0 else 0
+    return abs_support_, confidence_
+

ml_er/magellan_new.py

@@ -0,0 +1,75 @@
+import time
+import pandas as pd
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+from tqdm import tqdm
+
+from md_discovery.md_mining import mining
+from settings import *
+
+
+def blocking_mining():
+    start = time.time()
+    ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
+    cm.set_key(ltable, ltable_id)
+    rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
+    cm.set_key(rtable, rtable_id)
+    mappings = pd.read_csv(mapping_path, encoding='ISO-8859-1')
+    matching_number = len(mappings)
+    if ltable_id == rtable_id:
+        tables_id = rtable_id
+    attributes = ltable.columns.values.tolist()
+    lattributes = ['ltable_' + i for i in attributes]
+    rattributes = ['rtable_' + i for i in attributes]
+    cm.set_key(ltable, ltable_id)
+    cm.set_key(rtable, rtable_id)
+
+    blocker = em.OverlapBlocker()
+    candidate = blocker.block_tables(ltable, rtable, ltable_block_attr, rtable_block_attr, allow_missing=True,
+                                     l_output_attrs=attributes, r_output_attrs=attributes, n_jobs=1,
+                                     overlap_size=1, show_progress=False)
+    candidate['gold'] = 0
+    candidate = candidate.reset_index(drop=True)
+
+    # 根据mapping表标注数据
+    candidate_match_rows = []
+    for t in tqdm(mappings.itertuples()):
+        mask = ((candidate['ltable_' + ltable_id].isin([getattr(t, 'ltable_id')])) &
+                (candidate['rtable_' + rtable_id].isin([getattr(t, 'rtable_id')])))
+        matching_indices = candidate[mask].index
+        candidate_match_rows.extend(matching_indices.tolist())
+    match_rows_mask = candidate.index.isin(candidate_match_rows)
+    candidate.loc[match_rows_mask, 'gold'] = 1
+    candidate.fillna(value="", inplace=True)
+
+    candidate_mismatch = candidate[candidate['gold'] == 0]
+    candidate_match = candidate[candidate['gold'] == 1]
+    candidate_mismatch = candidate_mismatch.sample(n=3*len(candidate_match))
+    candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+    # 如果拼接后不重设索引可能导致索引重复
+    candidate_for_train_test = candidate_for_train_test.reset_index(drop=True)
+    cm.set_key(candidate_for_train_test, '_id')
+    cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id)
+    cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id)
+    cm.set_ltable(candidate_for_train_test, ltable)
+    cm.set_rtable(candidate_for_train_test, rtable)
+    block_recall = len(candidate_match) / matching_number
+
+    # 分为训练测试集
+    train_proportion = 0.5
+    sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0)
+    train_set = sets['train']
+    test_set = sets['test']
+    end_blocking = time.time()
+    print(end_blocking - start)
+
+    mining(train_set)
+    return 1
+
+
+def matching():
+    return 1
+
+
+if __name__ == '__main__':
+    blocking_mining()

ml_er/ml_entity_resolver.py

@@ -170,8 +170,6 @@ def er_process(config: Configuration):
         rid_mapping_list.append(row[mapping_rid])
 
     selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
-    # if len(lr_attrs_map) > 0:
-    #     selected_ltable = selected_ltable.rename(columns=lr_attrs_map)  # 参照右表，修改左表中与右表对应但不同名的字段
     tables_id = rtable_id
     selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
     selected_attrs = selected_ltable.columns.values.tolist()  # 两张表中的字段名
@@ -180,17 +178,11 @@ def er_process(config: Configuration):
     cm.set_key(selected_ltable, tables_id)
     cm.set_key(selected_rtable, tables_id)
 
-    if config["ml_blocker"] == "over_lap":
     blocker = em.OverlapBlocker()
     candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"],
                                      config["block_attr"], allow_missing=True,
                                      l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
                                      overlap_size=1, show_progress=False)
-    elif config["ml_blocker"] == "attr_equiv":
-        blocker = em.AttrEquivalenceBlocker()
-        candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"],
-                                         config["block_attr"], allow_missing=True,
-                                         l_output_attrs=selected_attrs, r_output_attrs=selected_attrs)
 
     candidate['gold'] = 0
     candidate = candidate.reset_index(drop=True)
@@ -228,6 +220,7 @@ def er_process(config: Configuration):
     cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + tables_id)
     cm.set_ltable(candidate_for_train_test, selected_ltable)
     cm.set_rtable(candidate_for_train_test, selected_rtable)
+    block_recall = len(candidate_match) / matching_number
 
     # 分为训练测试集
     train_proportion = 0.7
@@ -295,6 +288,7 @@ def er_process(config: Configuration):
     predictions_attrs.extend(attrs_with_r_prefix)
     predictions_attrs.extend(['gold', 'predicted'])
     predictions = predictions[predictions_attrs]
+    # 必须从训练集内挖MD
     train_attrs = predictions_attrs[:]
     train_attrs.remove('predicted')
     train_set = train_set[train_attrs]

settings.py

@@ -1,20 +1,22 @@
 from sentence_transformers import SentenceTransformer
 
-ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\tableA.csv'
-rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\tableB.csv'
-mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Walmart-Amazon_dirty\matches.csv'
+ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableA.csv'
+rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableB.csv'
+mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\matches.csv'
 mapping_lid = 'ltable_id'  # mapping表中左表id名
 mapping_rid = 'rtable_id'  # mapping表中右表id名
+ltable_block_attr = 'name'
+rtable_block_attr = 'name'
 ltable_id = 'id'  # 左表id字段名称
 rtable_id = 'id'  # 右表id字段名称
 target_attr = 'id'  # 进行md挖掘时的目标字段
 # lr_attrs_map = {}  # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致
 
-model = SentenceTransformer('E:\\Data\\Research\\Models\\roberta-large-nli-stsb-mean-tokens')
+model = SentenceTransformer('E:\\Data\\Research\\Models\\all-MiniLM-L6-v2')
 interpre_weight = 1  # 可解释性权重
 similarity_threshold = 0.1
 support_threshold = 1
-confidence_threshold = 0.25
+confidence_threshold = 0.6
 
 er_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\ml_er\\output\\'
 md_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\md_discovery\\output\\'