9.16

.gitignore

@@ -0,0 +1 @@
+/deprecated/

deprecated/inference_from_record_pairs.py

@@ -1,152 +0,0 @@
-import pandas as pd
-import time
-import Levenshtein
-import copy
-
-def my_Levenshtein_ratio(str1, str2):
-    return 1 - Levenshtein.distance(str1, str2) / max(len(str1), len(str2))
-
-
-def if_minimal(md, md_list, target_col):
-    # 假设这个md是minimal
-    minimal = True
-    for _ in md_list:
-        if _ != md:
-            # 假设列表中每一个md都使当前md不minimal
-            exist = True
-            # 如果左边任何一个大于，则假设不成立
-            for col in list(set(_.keys()) - {target_col}):
-                if _[col] > md[col]:
-                    exist = False
-            # 如果右边小于，假设也不成立
-            if _[target_col] < md[target_col]:
-                exist = False
-            # 任何一次假设成立，当前md不minimal
-            if exist:
-                minimal = False
-                break
-    return minimal
-
-
-def satisfy_confidence(md, df, conf_thresh, target_col):
-    support = 0
-    support_plus = 0
-    for row1 in df.itertuples():
-        i = row1[0]
-        df_slice = df[i + 1:]
-        for row2 in df_slice.itertuples():
-            left_satisfy = True
-            both_satisfy = True
-            for col in df.columns.values.tolist():
-                sim = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                if col == target_col:
-                    if sim < 1:
-                        both_satisfy = False
-                else:
-                    if sim < md[col]:
-                        left_satisfy = False
-                        both_satisfy = False
-            if left_satisfy:
-                support += 1
-            if both_satisfy:
-                support_plus += 1
-    if support == 0:
-        return False, 0.0
-    confidence = support_plus / support
-    return confidence >= conf_thresh, confidence
-
-
-def inference_from_record_pairs(path, threshold, target_col):
-    data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1')
-    data = data.astype(str)
-    columns = data.columns.values.tolist()
-
-    md_list = []
-    minimal_vio = []
-    init_md = {}
-    for col in columns:
-        init_md[col] = 1 if col == target_col else 0
-    md_list.append(init_md)
-
-    for row1 in data.itertuples():
-        # 获取当前行的索引，从后一行开始切片
-        i = row1[0]
-        data1 = data[i + 1:]
-        for row2 in data1.itertuples():
-            violated_mds = []
-            # sims是两行的相似度
-            sims = {}
-            for col in columns:
-                similarity = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                sims[col] = similarity
-
-            # 寻找violated md,从md列表中删除并加入vio列表
-            for md in md_list:
-                lhs_satis = True
-                rhs_satis = True
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] < md[col]:
-                        lhs_satis = False
-                if sims[target_col] < md[target_col]:
-                    rhs_satis = False
-                if lhs_satis == True and rhs_satis == False:
-                    md_list.remove(md)
-                    violated_mds.append(md)
-            minimal_vio.extend(violated_mds)
-
-            for vio_md in violated_mds:
-                # 特殊化右侧,我们需要右侧百分百相似，其实不需要降低右侧阈值
-                # if sims[target_col] >= threshold:
-                #     new_rhs = sims[target_col]
-                #     spec_r_md = copy.deepcopy(vio_md)
-                #     spec_r_md[target_col] = new_rhs
-                #     if if_minimal(spec_r_md, md_list, target_col):
-                #         md_list.append(spec_r_md)
-
-                # 特殊化左侧
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] + 0.001 <= 1:
-                        spec_l_md = copy.deepcopy(vio_md)
-                        spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.001
-                        if if_minimal(spec_l_md, md_list, target_col):
-                            md_list.append(spec_l_md)
-
-            for vio in minimal_vio:
-                if not if_minimal(vio, md_list, target_col):
-                    minimal_vio.remove(vio)
-
-    tmp = copy.deepcopy(minimal_vio)
-    for _ in tmp:
-        satis, conf = satisfy_confidence(_, data, 0.8, target_col)
-        if not satis:
-            minimal_vio.remove(_)
-
-    for _ in tmp:
-        if not if_minimal(_, minimal_vio, target_col):
-            minimal_vio.remove(_)
-
-    return md_list, minimal_vio
-
-
-if __name__ == '__main__':
-    # 目前可以仿照这个main函数写
-    path = "input/T_positive_with_id_concat_single_tuple.csv"
-    start = time.time()
-    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
-    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
-    # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
-    mds, mds_vio = inference_from_record_pairs(path, 0.7, 'id_concat')
-
-    # 将列表1写入本地，路径需自己修改
-    md_path = 'output/md.txt'
-    with open(md_path, 'w') as f:
-        for _ in mds:
-            f.write(str(_) + '\n')
-
-    # 将列表2写入本地，路径需自己修改
-    vio_path = 'output/vio.txt'
-    with open(vio_path, 'w') as f:
-        for _ in mds_vio:
-            f.write(str(_) + '\n')
-
-    print(time.time() - start)

hpo/magellan_hpo.py

@@ -0,0 +1,249 @@
+from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+import pandas as pd
+
+from smac import HyperparameterOptimizationFacade, Scenario
+from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
+# todo 距离度量用户可调
+
+# 全局变量，每次迭代后清空列表，加入新的md路径
+# todo:
+# 默认路径为 "../md_discovery/output/xxx.txt"
+# 真阳/假阴  mds/vio  共4个md文件
+md_paths = []
+
+
+def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int,
+                        test_proportion: float) -> dict:
+    new_df = df.reset_index(drop=False, inplace=False)
+    gold = new_df[labeled_attr]
+    predicted = new_df[predicted_attr]
+    gold_negative = gold[gold == 0].index.values
+    gold_positive = gold[gold == 1].index.values
+    predicted_negative = predicted[predicted == 0].index.values
+    predicted_positive = predicted[predicted == 1].index.values
+
+    false_positive_indices = list(set(gold_negative).intersection(predicted_positive))
+    true_positive_indices = list(set(gold_positive).intersection(predicted_positive))
+    false_negative_indices = list(set(gold_positive).intersection(predicted_negative))
+
+    num_true_positives = float(len(true_positive_indices))
+    num_false_positives = float(len(false_positive_indices))
+    num_false_negatives = float(len(false_negative_indices))
+
+    precision_denominator = num_true_positives + num_false_positives
+    recall_denominator = num_true_positives + num_false_negatives
+
+    precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator
+    recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator
+    F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall)
+    my_recall = num_true_positives / (matching_number * test_proportion)
+
+    return {"precision": precision, "recall": recall, "F1": F1, "my_recall": my_recall}
+
+
+def load_mds(paths: list) -> list:
+    if len(paths) == 0:
+        return []
+    all_mds = []
+    # 传入md路径列表
+    for md_path in paths:
+        mds = []
+        # 打开每一个md文件
+        with open(md_path, 'r') as f:
+            # 读取每一行的md，加入该文件的md列表
+            for line in f.readlines():
+                md_metadata = line.strip().split('\t')
+                md = eval(md_metadata[0].replace('md:', ''))
+                mds.append(md)
+        all_mds.extend(mds)
+    return all_mds
+
+
+def is_explicable(row, all_mds: list) -> bool:
+    attrs = all_mds[0].keys()  # 从第一条md中读取所有字段
+    for md in all_mds:
+        explicable = True  # 假设这条md能解释当前元组
+        for a in attrs:
+            threshold = md[a]
+            if my_Levenshtein_ratio(str(getattr(row, 'ltable_'+a)), str(getattr(row, 'rtable_'+a))) < threshold:
+                explicable = False  # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组
+                break  # 不再与当前md的其他相似度阈值比较，跳转到下一条md
+        if explicable:
+            return True  # 任意一条md能解释，直接返回
+    return False  # 遍历结束，不能解释
+
+
+class SVM:
+    @property
+    def configspace(self) -> ConfigurationSpace:
+        # Build Configuration Space which defines all parameters and their ranges
+        cs = ConfigurationSpace(seed=0)
+
+        l_overlap_attr = Categorical("l_overlap_attr", ["title", "description", "manufacturer", "price"], default="title")
+        overlap_size = Integer("overlap_size", (1, 3), default=1)
+
+        cs.add_hyperparameters([l_overlap_attr, overlap_size])
+
+        return cs
+
+    # train 就是整个函数 只需将返回结果由预测变成预测结果的评估
+    def train(self, config: Configuration, seed: int = 0) -> float:
+        path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
+        path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
+        path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
+        Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1')
+        cm.set_key(Amazon, 'id')
+        Amazon.fillna("", inplace=True)
+        Google = pd.read_csv(path_Google, encoding='ISO-8859-1')
+        cm.set_key(Google, 'id')
+        Google.fillna("", inplace=True)
+        Mappings = pd.read_csv(path_Mappings)
+
+        # 仅保留两表中出现在映射表中的行，增大正样本比例
+        l_id_list = []
+        r_id_list = []
+        # 全部转为字符串
+        Amazon = Amazon.astype(str)
+        Google = Google.astype(str)
+        Mappings = Mappings.astype(str)
+        matching_number = len(Mappings)  # 所有阳性样本数，商品数据集应为1300
+
+        for index, row in Mappings.iterrows():
+            l_id_list.append(row["idAmazon"])
+            r_id_list.append(row["idGoogleBase"])
+        selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
+        selected_Google = Google[Google['id'].isin(r_id_list)]
+        cm.set_key(selected_Amazon, 'id')
+        cm.set_key(selected_Google, 'id')
+
+        # todo blocker可调
+        # 1.blocker类型（商品数据集可能只适合overlap）
+        # 2.overlap字段（对应关系）
+        # 3.overlap_size
+        blocker = em.OverlapBlocker()
+        overlap_attr = 'name' if config["l_overlap_attr"] == 'title' else config["l_overlap_attr"]
+        candidate = blocker.block_tables(selected_Amazon, selected_Google, config["l_overlap_attr"], overlap_attr,
+                                         l_output_attrs=['id', 'title', 'description', 'manufacturer', 'price'],
+                                         r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+                                         overlap_size=config["overlap_size"], show_progress=False)
+        candidate['gold'] = 0
+
+        candidate_match_rows = []
+        for index, row in candidate.iterrows():
+            l_id = row["ltable_id"]
+            map_row = Mappings[Mappings['idAmazon'] == l_id]
+
+            if map_row is not None:
+                r_id = map_row["idGoogleBase"]
+                for value in r_id:
+                    if value == row["rtable_id"]:
+                        candidate_match_rows.append(row["_id"])
+            else:
+                continue
+        for row in candidate_match_rows:
+            candidate.loc[row, 'gold'] = 1
+
+        # 裁剪负样本，保持正负样本数量一致
+        candidate_mismatch = candidate[candidate['gold'] == 0]
+        candidate_match = candidate[candidate['gold'] == 1]
+        candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
+        # 拼接正负样本
+        candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+        cm.set_key(candidate_for_train_test, '_id')
+        cm.set_fk_ltable(candidate_for_train_test, 'ltable_id')
+        cm.set_fk_rtable(candidate_for_train_test, 'rtable_id')
+        cm.set_ltable(candidate_for_train_test, selected_Amazon)
+        cm.set_rtable(candidate_for_train_test, selected_Google)
+
+        # 分为训练测试集
+        train_proportion = 0.7
+        test_proportion = 0.3
+        sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0)
+        train_set = sets['train']
+        test_set = sets['test']
+
+        rf = em.RFMatcher(name='RF', random_state=0)
+        feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
+
+        train_feature_vecs = em.extract_feature_vecs(train_set,
+                                                     feature_table=feature_table,
+                                                     attrs_after=['gold'],
+                                                     show_progress=False)
+
+        test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
+                                                    attrs_after=['ltable_title', 'ltable_description', 'ltable_manufacturer',
+                                                                 'ltable_price', 'rtable_name', 'rtable_description',
+                                                                 'rtable_manufacturer', 'rtable_price', 'gold'], show_progress=False)
+
+        # todo 参数可调 用drop删除特征向量中的列？
+        # 1.exclude_attrs
+        # 去掉id相关的相似度
+        rf.fit(table=train_feature_vecs,
+               exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
+               target_attr='gold')
+
+        # 1.exclude_attrs
+        predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title',
+                                                                         'ltable_description', 'ltable_manufacturer',
+                                                                         'ltable_price', 'rtable_name', 'rtable_description',
+                                                                         'rtable_manufacturer', 'rtable_price', 'gold'],
+                                 append=True, target_attr='predicted', inplace=False)
+        eval_result = em.eval_matches(predictions, 'gold', 'predicted')
+        em.print_eval_summary(eval_result)
+        indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, test_proportion)
+        print(indicators)
+
+        # 计算可解释性
+        predictions = predictions[
+            ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price',
+             'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']]
+        epl_match = 0  # 可解释，预测match
+        nepl_mismatch = 0  # 不可解释，预测mismatch
+        md_list = load_mds(md_paths)  # 从全局变量中读取所有的md
+        for row in predictions.itertuples():
+            if is_explicable(row, md_list):
+                if getattr(row, 'predicted') == 1:
+                    epl_match += 1
+            else:
+                if getattr(row, 'predicted') == 0:
+                    nepl_mismatch += 1
+        epl_ability = (epl_match + nepl_mismatch) / len(predictions)  # 可解释性
+        f1 = indicators['F1']
+        performance = 0.5 * epl_ability + 0.5 * f1  # 可解释性与F1的权重暂时定为0.5
+        # todo 权重用户可调
+        return 1 - performance
+
+
+if __name__ == "__main__":
+    classifier = SVM()
+
+    # Next, we create an object, holding general information about the run
+    scenario = Scenario(
+        classifier.configspace,
+        n_trials=12,  # We want to run max 50 trials (combination of config and seed)
+    )
+
+    initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=3)
+
+    # Now we use SMAC to find the best hyperparameters
+    smac = HyperparameterOptimizationFacade(
+        scenario,
+        classifier.train,
+        initial_design=initial_design,
+        overwrite=True,  # If the run exists, we overwrite it; alternatively, we can continue from last state
+    )
+
+    incumbent = smac.optimize()
+
+    # Get cost of default configuration
+    default_cost = smac.validate(classifier.configspace.get_default_configuration())
+    print(f"Default cost: {default_cost}")
+
+    # Let's calculate the cost of the incumbent
+    incumbent_cost = smac.validate(incumbent)
+    print(f"Incumbent cost: {incumbent_cost}")
+
+    print(f"Configuration:{incumbent.values()}")
+    print(f"MAX_F1:{1-classifier.train(incumbent)}")

md_discovery/functions/multi_process_infer_by_pairs.py

@@ -38,7 +38,7 @@ def if_minimal(md, md_list, target_col):
 
 def remove_by_confidence(md, l, relation, target_col, lock):
     support, confidence = get_one_md_metadata(md, relation, target_col)
-    # todo: replace constant 0.8
+    # todo confidence可调
     if confidence < 0.8:
         with lock:
             l.remove(md)

md_discovery/script/get_support_and_confidence.py

@@ -0,0 +1,67 @@
+import time
+from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs
+from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata
+
+
+# # 若不输出support和confidence，使用以下两块代码
+# # 将列表1写入本地，路径需自己修改
+# md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt'
+# with open(md_path, 'w') as f:
+#     for _ in mds:
+#         f.write(str(_) + '\n')
+#
+# # 将列表2写入本地，路径需自己修改
+# vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt'
+# with open(vio_path, 'w') as f:
+#     for _ in vio:
+#         f.write(str(_) + '\n')
+
+
+if __name__ == '__main__':
+    # 目前可以仿照这个main函数写
+    tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv"
+    fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv"
+    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
+    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
+    # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
+    tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, 0.7, 'id')
+    fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, 0.7, 'id')
+
+    # 如果不需要输出support和confidence，去掉下面两行
+    tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, 'id')
+    tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, 'id')
+
+    fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, 'id')
+    fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, 'id')
+
+    # 若输出support和confidence，使用以下两块代码
+    # 将列表1写入本地，路径需自己修改
+    tp_mds_path = "../output/tp_mds.txt"
+    tp_vio_path = "../output/tp_vio.txt"
+    
+    with open(tp_mds_path, 'w') as f:
+        for _ in tp_mds_meta:
+            for i in _.keys():
+                f.write(i + ':' + str(_[i]) + '\t')
+            f.write('\n')
+
+    with open(tp_vio_path, 'w') as f:
+        for _ in tp_vio_meta:
+            for i in _.keys():
+                f.write(i + ':' + str(_[i]) + '\t')
+            f.write('\n')
+
+    fn_mds_path = "../output/fn_mds.txt"
+    fn_vio_path = "../output/fn_vio.txt"
+
+    with open(fn_mds_path, 'w') as f:
+        for _ in fn_mds_meta:
+            for i in _.keys():
+                f.write(i + ':' + str(_[i]) + '\t')
+            f.write('\n')
+
+    with open(fn_vio_path, 'w') as f:
+        for _ in fn_vio_meta:
+            for i in _.keys():
+                f.write(i + ':' + str(_[i]) + '\t')
+            f.write('\n')

ml_er/Goods Dataset-8.14.py

@@ -0,0 +1,153 @@
+import sys
+
+from py_entitymatching.debugmatcher.debug_gui_utils import _get_metric
+
+sys.path.append('/home/w/PycharmProjects/py_entitymatching/py_entitymatching')
+
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+import pandas as pd
+import time
+import six
+
+
+def load_data(left_path: str, right_path: str, mapping_path: str):
+    left = pd.read_csv(left_path, encoding='ISO-8859-1')
+    cm.set_key(left, left.columns.values.tolist()[0])
+    left.fillna("", inplace=True)
+    left = left.astype(str)
+
+    right = pd.read_csv(right_path, encoding='ISO-8859-1')
+    cm.set_key(right, right.columns.values.tolist()[0])
+    right.fillna("", inplace=True)
+    right = right.astype(str)
+
+    mapping = pd.read_csv(mapping_path)
+    mapping = mapping.astype(str)
+    return left, right, mapping
+
+
+if __name__ == '__main__':
+    # 读入公开数据，注册并填充空值
+    path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
+    path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
+    path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
+    Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1')
+    cm.set_key(Amazon, 'id')
+    Amazon.fillna("", inplace=True)
+    Google = pd.read_csv(path_Google, encoding='ISO-8859-1')
+    cm.set_key(Google, 'id')
+    Google.fillna("", inplace=True)
+    Mappings = pd.read_csv(path_Mappings)
+
+    # 仅保留两表中出现在映射表中的行，增大正样本比例
+    l_id_list = []
+    r_id_list = []
+    # 全部转为字符串
+    Amazon = Amazon.astype(str)
+    Google = Google.astype(str)
+    Mappings = Mappings.astype(str)
+    for index, row in Mappings.iterrows():
+        l_id_list.append(row["idAmazon"])
+        r_id_list.append(row["idGoogleBase"])
+    selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
+    selected_Amazon = selected_Amazon.rename(columns={'title': 'name'})
+    selected_Google = Google[Google['id'].isin(r_id_list)]
+    cm.set_key(selected_Amazon, 'id')
+    cm.set_key(selected_Google, 'id')
+
+    #########################################################################
+    # False-retain True-remove
+    def match_last_name(ltuple, rtuple):
+        l_last_name = ltuple['name']
+        r_last_name = rtuple['name']
+        if l_last_name != r_last_name:
+            return True
+        else:
+            return False
+
+    bb = em.BlackBoxBlocker()
+    bb.set_black_box_function(match_last_name)
+
+    Candidate = bb.block_tables(selected_Amazon, selected_Google, l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'])
+    #########################################################################
+    # block 并将gold标记为0
+    blocker = em.OverlapBlocker()
+    candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name',
+                                     l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+                                     r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+                                     overlap_size=0, show_progress=False)
+    candidate['gold'] = 0
+
+    start = time.time()
+    candidate_match_rows = []
+    for index, row in candidate.iterrows():
+        l_id = row["ltable_id"]
+        map_row = Mappings[Mappings['idAmazon'] == l_id]
+
+        if map_row is not None:
+            r_id = map_row["idGoogleBase"]
+            for value in r_id:
+                if value == row["rtable_id"]:
+                    candidate_match_rows.append(row["_id"])
+        else:
+            continue
+    for row in candidate_match_rows:
+        candidate.loc[row, 'gold'] = 1
+
+    # 裁剪负样本，保持正负样本数量一致
+    candidate_mismatch = candidate[candidate['gold'] == 0]
+    candidate_match = candidate[candidate['gold'] == 1]
+    candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
+    # 拼接正负样本
+    candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+    cm.set_key(candidate_for_train_test, '_id')
+    cm.set_fk_ltable(candidate_for_train_test, 'ltable_id')
+    cm.set_fk_rtable(candidate_for_train_test, 'rtable_id')
+    cm.set_ltable(candidate_for_train_test, selected_Amazon)
+    cm.set_rtable(candidate_for_train_test, selected_Google)
+
+    # 分为训练测试集
+    sets = em.split_train_test(candidate_for_train_test, train_proportion=0.7, random_state=0)
+    train_set = sets['train']
+    test_set = sets['test']
+
+    dt = em.DTMatcher(name='DecisionTree', random_state=0)
+    svm = em.SVMMatcher(name='SVM', random_state=0)
+    rf = em.RFMatcher(name='RF', random_state=0)
+    lg = em.LogRegMatcher(name='LogReg', random_state=0)
+    ln = em.LinRegMatcher(name='LinReg')
+    nb = em.NBMatcher(name='NaiveBayes')
+    feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
+
+    train_feature_vecs = em.extract_feature_vecs(train_set,
+                                                 feature_table=feature_table,
+                                                 attrs_after='gold',
+                                                 show_progress=False)
+
+    result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs,
+                               exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
+                               k=5,
+                               target_attr='gold', metric_to_select_matcher='f1', random_state=0)
+
+    test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
+                                                attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer',
+                                                             'ltable_price', 'rtable_name', 'rtable_description',
+                                                             'rtable_manufacturer', 'rtable_price', 'gold'], show_progress=False)
+
+    rf.fit(table=train_feature_vecs,
+           exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
+           target_attr='gold')
+    predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_name',
+                                                                     'ltable_description', 'ltable_manufacturer',
+                                                                     'ltable_price', 'rtable_name', 'rtable_description',
+                                                                     'rtable_manufacturer', 'rtable_price', 'gold'],
+                             append=True, target_attr='predicted', inplace=False)
+    eval_result = em.eval_matches(predictions, 'gold', 'predicted')
+    em.print_eval_summary(eval_result)
+
+    output_path = "output/eval_result" + str(time.time()) + ".txt"
+    with open(output_path, 'w') as f:
+        for key, value in six.iteritems(_get_metric(eval_result)):
+            f.write(key + " : " + value)
+            f.write('\n')

ml_er/Goods Dataset.py

@@ -0,0 +1,273 @@
+import sys
+
+from py_entitymatching.debugmatcher.debug_gui_utils import _get_metric
+
+sys.path.append('/home/w/PycharmProjects/py_entitymatching/py_entitymatching')
+
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+import pandas as pd
+import time
+import six
+from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
+
+
+def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"):
+    tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)]
+    fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)]
+    # 将真阳/假阴表中左右ID调整一致
+    for index, row in tp.iterrows():
+        tp.loc[index, "rtable_id"] = row["ltable_id"]
+    for index, row in fn.iterrows():
+        fn.loc[index, "rtable_id"] = row["ltable_id"]
+
+    pred_columns = pred.columns.values.tolist()
+    l_columns = []
+    r_columns = []
+    columns = []  # todo 前提是两张表内对应字段名调整一致
+    for _ in pred_columns:
+        if _.startswith('ltable'):
+            l_columns.append(_)
+        elif _.startswith('rtable'):
+            r_columns.append(_)
+    for _ in l_columns:
+        columns.append(_.replace('ltable_', ''))
+
+    tpl = tp[l_columns]
+    tpr = tp[r_columns]
+    tpl.columns = columns
+    tpr.columns = columns
+
+    fnl = fn[l_columns]
+    fnr = fn[r_columns]
+    fnl.columns = columns
+    fnr.columns = columns
+
+    tp_single_tuple = pd.concat([tpl, tpr])
+    fn_single_tuple = pd.concat([fnl, fnr])
+
+    tp_single_tuple.to_csv(tp_single_tuple_path, sep=',', index=False, header=True)
+    fn_single_tuple.to_csv(fn_single_tuple_path, sep=',', index=False, header=True)
+
+
+def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int,
+                        test_proportion: float) -> dict:
+    new_df = df.reset_index(drop=False, inplace=False)
+    gold = new_df[labeled_attr]
+    predicted = new_df[predicted_attr]
+    gold_negative = gold[gold == 0].index.values
+    gold_positive = gold[gold == 1].index.values
+    predicted_negative = predicted[predicted == 0].index.values
+    predicted_positive = predicted[predicted == 1].index.values
+
+    false_positive_indices = list(set(gold_negative).intersection(predicted_positive))
+    true_positive_indices = list(set(gold_positive).intersection(predicted_positive))
+    false_negative_indices = list(set(gold_positive).intersection(predicted_negative))
+
+    num_true_positives = float(len(true_positive_indices))
+    num_false_positives = float(len(false_positive_indices))
+    num_false_negatives = float(len(false_negative_indices))
+
+    precision_denominator = num_true_positives + num_false_positives
+    recall_denominator = num_true_positives + num_false_negatives
+
+    precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator
+    recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator
+    F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall)
+    my_recall = num_true_positives / (matching_number * test_proportion)
+
+    return {"precision": precision, "recall": recall, "F1": F1, "my_recall": my_recall}
+
+
+def load_mds(paths: list) -> list:
+    if len(paths) == 0:
+        return []
+    all_mds = []
+    # 传入md路径列表
+    for md_path in paths:
+        mds = []
+        # 打开每一个md文件
+        with open(md_path, 'r') as f:
+            # 读取每一行的md，加入该文件的md列表
+            for line in f.readlines():
+                md_metadata = line.strip().split('\t')
+                md = eval(md_metadata[0].replace('md:', ''))
+                mds.append(md)
+        all_mds.extend(mds)
+    return all_mds
+
+
+def is_explicable(row, all_mds: list) -> bool:
+    attrs = all_mds[0].keys()  # 从第一条md中读取所有字段
+    for md in all_mds:
+        explicable = True  # 假设这条md能解释当前元组
+        for a in attrs:
+            threshold = md[a]
+            if my_Levenshtein_ratio(str(getattr(row, 'ltable_'+a)), str(getattr(row, 'rtable_'+a))) < threshold:
+                explicable = False  # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组
+                break  # 不再与当前md的其他相似度阈值比较，跳转到下一条md
+        if explicable:
+            return True  # 任意一条md能解释，直接返回
+    return False  # 遍历结束，不能解释
+
+
+def load_data(left_path: str, right_path: str, mapping_path: str):
+    left = pd.read_csv(left_path, encoding='ISO-8859-1')
+    cm.set_key(left, left.columns.values.tolist()[0])
+    left.fillna("", inplace=True)
+    left = left.astype(str)
+
+    right = pd.read_csv(right_path, encoding='ISO-8859-1')
+    cm.set_key(right, right.columns.values.tolist()[0])
+    right.fillna("", inplace=True)
+    right = right.astype(str)
+
+    mapping = pd.read_csv(mapping_path)
+    mapping = mapping.astype(str)
+    return left, right, mapping
+
+
+if __name__ == '__main__':
+    # 读入公开数据，注册并填充空值
+    path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
+    path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
+    path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
+    Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1')
+    cm.set_key(Amazon, 'id')
+    Amazon.fillna("", inplace=True)
+    Google = pd.read_csv(path_Google, encoding='ISO-8859-1')
+    cm.set_key(Google, 'id')
+    Google.fillna("", inplace=True)
+    Mappings = pd.read_csv(path_Mappings)
+
+    # 仅保留两表中出现在映射表中的行，增大正样本比例
+    l_id_list = []
+    r_id_list = []
+    # 全部转为字符串
+    Amazon = Amazon.astype(str)
+    Google = Google.astype(str)
+    Mappings = Mappings.astype(str)
+    matching_number = len(Mappings)  # 所有阳性样本数，商品数据集应为1300
+
+    for index, row in Mappings.iterrows():
+        l_id_list.append(row["idAmazon"])
+        r_id_list.append(row["idGoogleBase"])
+    selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
+    selected_Amazon = selected_Amazon.rename(columns={'title': 'name'})
+    selected_Google = Google[Google['id'].isin(r_id_list)]
+    cm.set_key(selected_Amazon, 'id')
+    cm.set_key(selected_Google, 'id')
+
+    # block 并将gold标记为0
+    blocker = em.OverlapBlocker()
+    candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name',
+                                     l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+                                     r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+                                     overlap_size=1, show_progress=False)
+    candidate['gold'] = 0
+
+    start = time.time()
+    candidate_match_rows = []
+    for index, row in candidate.iterrows():
+        l_id = row["ltable_id"]
+        map_row = Mappings[Mappings['idAmazon'] == l_id]
+
+        if map_row is not None:
+            r_id = map_row["idGoogleBase"]
+            for value in r_id:
+                if value == row["rtable_id"]:
+                    candidate_match_rows.append(row["_id"])
+        else:
+            continue
+    for row in candidate_match_rows:
+        candidate.loc[row, 'gold'] = 1
+
+    # 裁剪负样本，保持正负样本数量一致
+    candidate_mismatch = candidate[candidate['gold'] == 0]
+    candidate_match = candidate[candidate['gold'] == 1]
+    candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
+    # 拼接正负样本
+    candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+    cm.set_key(candidate_for_train_test, '_id')
+    cm.set_fk_ltable(candidate_for_train_test, 'ltable_id')
+    cm.set_fk_rtable(candidate_for_train_test, 'rtable_id')
+    cm.set_ltable(candidate_for_train_test, selected_Amazon)
+    cm.set_rtable(candidate_for_train_test, selected_Google)
+
+    # 分为训练测试集
+    train_proportion = 0.7
+    test_proportion = 0.3
+    sets = em.split_train_test(candidate_for_train_test, train_proportion=0.7, random_state=0)
+    train_set = sets['train']
+    test_set = sets['test']
+
+    dt = em.DTMatcher(name='DecisionTree', random_state=0)
+    svm = em.SVMMatcher(name='SVM', random_state=0)
+    rf = em.RFMatcher(name='RF', random_state=0)
+    lg = em.LogRegMatcher(name='LogReg', random_state=0)
+    ln = em.LinRegMatcher(name='LinReg')
+    nb = em.NBMatcher(name='NaiveBayes')
+    feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
+
+    train_feature_vecs = em.extract_feature_vecs(train_set,
+                                                 feature_table=feature_table,
+                                                 attrs_after=['gold'],
+                                                 show_progress=False)
+
+    result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs,
+                               exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
+                               k=5,
+                               target_attr='gold', metric_to_select_matcher='f1', random_state=0)
+
+    test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
+                                                attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer',
+                                                             'ltable_price', 'rtable_name', 'rtable_description',
+                                                             'rtable_manufacturer', 'rtable_price', 'gold'],
+                                                show_progress=False)
+
+    rf.fit(table=train_feature_vecs,
+           exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
+           target_attr='gold')
+    predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_name',
+                                                                     'ltable_description', 'ltable_manufacturer',
+                                                                     'ltable_price', 'rtable_name',
+                                                                     'rtable_description',
+                                                                     'rtable_manufacturer', 'rtable_price', 'gold'],
+                             append=True, target_attr='predicted', inplace=False)
+    eval_result = em.eval_matches(predictions, 'gold', 'predicted')
+    em.print_eval_summary(eval_result)
+    indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, test_proportion)
+    print(indicators)
+
+    # 计算可解释性
+    ################################################################################################################
+    predictions = predictions[
+        ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price',
+         'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']]
+    epl_match = 0  # 可解释，预测match
+    nepl_mismatch = 0  # 不可解释，预测mismatch
+    p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt"
+    p_vio = "/home/w/A-New Folder/8.14/Goods Dataset/TP_vio_list.txt"
+    md_paths: list = [p_md, p_vio]
+    md_list = load_mds(md_paths)  # 从全局变量中读取所有的md
+    for row in predictions.itertuples():
+        if is_explicable(row, md_list):
+            if getattr(row, 'predicted') == 1:
+                epl_match += 1
+        else:
+            if getattr(row, 'predicted') == 0:
+                nepl_mismatch += 1
+
+    epl_ability = (epl_match + nepl_mismatch) / len(predictions)
+    ################################################################################################################
+    process_prediction_for_md_discovery(predictions)
+    # todo 将prediction表处理成真阳/假阴表提供给挖掘算法
+
+
+    output_path = "output/eval_result" + str(time.time()) + ".txt"
+    with open(output_path, 'w') as f:
+        for key, value in six.iteritems(_get_metric(eval_result)):
+            f.write(key + " : " + value)
+            f.write('\n')
+        f.write('my_recall:' + str(indicators["my_recall"]))
+        f.write('\n')

script/get_support_and_confidence.py

@@ -1,48 +0,0 @@
-import time
-from functions.multi_process_infer_by_pairs import inference_from_record_pairs
-from functions.multi_process_infer_by_pairs import get_mds_metadata
-
-if __name__ == '__main__':
-    # 目前可以仿照这个main函数写
-    path = "/home/w/PycharmProjects/matching_dependency/input/T_positive_with_id_concat_single_tuple.csv"
-    start = time.time()
-    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
-    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
-    # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
-    mds, mds_vio = inference_from_record_pairs(path, 0.1, 'id_concat')
-
-    # 如果不需要输出support和confidence，去掉下面两行
-    mds_meta = get_mds_metadata(mds, path, 'id_concat')
-    mds_vio_meta = get_mds_metadata(mds_vio, path, 'id_concat')
-
-    # # 若不输出support和confidence，使用以下两块代码
-    # # 将列表1写入本地，路径需自己修改
-    # md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt'
-    # with open(md_path, 'w') as f:
-    #     for _ in mds:
-    #         f.write(str(_) + '\n')
-    #
-    # # 将列表2写入本地，路径需自己修改
-    # vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt'
-    # with open(vio_path, 'w') as f:
-    #     for _ in mds_vio:
-    #         f.write(str(_) + '\n')
-
-    # 若输出support和confidence，使用以下两块代码
-    # 将列表1写入本地，路径需自己修改
-    md_path = "output/md.txt"
-    with open(md_path, 'w') as f:
-        for _ in mds_meta:
-            for i in _.keys():
-                f.write(i + ':' + str(_[i]) + '\t')
-            f.write('\n')
-
-    # 将列表2写入本地，路径需自己修改
-    vio_path = "output/vio.txt"
-    with open(vio_path, 'w') as f:
-        for _ in mds_vio_meta:
-            for i in _.keys():
-                f.write(i + ':' + str(_[i]) + '\t')
-            f.write('\n')
-
-    print(time.time() - start)