9.19

entrance.py

@@ -0,0 +1,34 @@
+# this is the entrance of the auto-ER procedure
+from md_discovery.script.md_discover import md_discover
+
+
+# todo: magellan ER模块读入初始化配置或hpo配置
+# todo: 模块间的自动化调用
+#  入口到ER/HPO到ER
+
+
+def run(l_table_path, r_table_path, mapping_path):
+    # while The termination condition is not met:
+    while True:
+    #     er()
+        md_discover()
+    #     hpo()
+    return
+
+
+if __name__ == '__main__':
+    # todo：使用input函数输入变量值（不方便就不用input）
+    #  7. 距离度量方式    ？
+    ltable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
+    rtable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
+    mapping_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
+    mapping_lid = 'idAmazon'  # mapping表中左表id名
+    mapping_rid = 'idGoogleBase'  # mapping表中右表id名
+    ltable_id = 'id'  # 左表id字段名称
+    rtable_id = 'id'  # 右表id字段名称
+    target_attr = 'id'  # 进行md挖掘时的目标字段
+    lr_attrs_map = {'title': 'name'}  # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致
+    similarity_threshold = 0.7
+    confidence_threshold = 0.8
+    interpretability_weight = 0.3
+    print(ltable_path)

hpo/magellan_hpo.py

@@ -1,17 +1,20 @@
-from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer
+from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer, Float
 import py_entitymatching as em
 import py_entitymatching.catalog.catalog_manager as cm
 import pandas as pd
+from py_entitymatching.blocker.blocker import Blocker
+from py_entitymatching.matcher.mlmatcher import MLMatcher
 
 from smac import HyperparameterOptimizationFacade, Scenario
 from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
-# todo 距离度量用户可调
+from entrance import *
+# todo 距离度量用户可设置
 
 # 全局变量，每次迭代后清空列表，加入新的md路径
 # todo:
 #  默认路径为 "../md_discovery/output/xxx.txt"
 #  真阳/假阴  mds/vio  共4个md文件
-md_paths = []
+
 
 
 def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int,
@@ -56,6 +59,8 @@ def load_mds(paths: list) -> list:
             for line in f.readlines():
                 md_metadata = line.strip().split('\t')
                 md = eval(md_metadata[0].replace('md:', ''))
+                confidence = eval(md_metadata[2].replace('confidence:', ''))
+                if confidence > 0:
                     mds.append(md)
         all_mds.extend(mds)
     return all_mds
@@ -80,65 +85,70 @@ class SVM:
     def configspace(self) -> ConfigurationSpace:
         # Build Configuration Space which defines all parameters and their ranges
         cs = ConfigurationSpace(seed=0)
-
-        l_overlap_attr = Categorical("l_overlap_attr", ["title", "description", "manufacturer", "price"], default="title")
+        # todo
+        #  block_attr 取消打桩
+        block_attr = Categorical("block_attr", ["name", "description", "manufacturer", "price"], default="title")
         overlap_size = Integer("overlap_size", (1, 3), default=1)
+        ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf")
+        ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap")
 
-        cs.add_hyperparameters([l_overlap_attr, overlap_size])
-
+        cs.add_hyperparameters([block_attr, overlap_size, ml_matcher, ml_blocker])
         return cs
 
     # train 就是整个函数 只需将返回结果由预测变成预测结果的评估
-    def train(self, config: Configuration, seed: int = 0) -> float:
-        path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
-        path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
-        path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
-        Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1')
-        cm.set_key(Amazon, 'id')
-        Amazon.fillna("", inplace=True)
-        Google = pd.read_csv(path_Google, encoding='ISO-8859-1')
-        cm.set_key(Google, 'id')
-        Google.fillna("", inplace=True)
-        Mappings = pd.read_csv(path_Mappings)
+    def train(self, config: Configuration) -> float:
+        ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
+        cm.set_key(ltable, ltable_id)
+        ltable.fillna("", inplace=True)
+        rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
+        cm.set_key(rtable, rtable_id)
+        rtable.fillna("", inplace=True)
+        mappings = pd.read_csv(mapping_path)
 
         # 仅保留两表中出现在映射表中的行，增大正样本比例
-        l_id_list = []
-        r_id_list = []
+        lid_mapping_list = []
+        rid_mapping_list = []
         # 全部转为字符串
-        Amazon = Amazon.astype(str)
-        Google = Google.astype(str)
-        Mappings = Mappings.astype(str)
-        matching_number = len(Mappings)  # 所有阳性样本数，商品数据集应为1300
-
-        for index, row in Mappings.iterrows():
-            l_id_list.append(row["idAmazon"])
-            r_id_list.append(row["idGoogleBase"])
-        selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
-        selected_Google = Google[Google['id'].isin(r_id_list)]
-        cm.set_key(selected_Amazon, 'id')
-        cm.set_key(selected_Google, 'id')
-
-        # todo blocker可调
-        # 1.blocker类型（商品数据集可能只适合overlap）
-        # 2.overlap字段（对应关系）
-        # 3.overlap_size
+        ltable = ltable.astype(str)
+        rtable = rtable.astype(str)
+        mappings = mappings.astype(str)
+        matching_number = len(mappings)  # 所有阳性样本数，商品数据集应为1300
+
+        for index, row in mappings.iterrows():
+            lid_mapping_list.append(row[mapping_lid])
+            rid_mapping_list.append(row[mapping_rid])
+
+        selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
+        selected_ltable = selected_ltable.rename(columns=lr_attrs_map)  # 参照右表，修改左表中与右表对应但不同名的字段
+        selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
+        selected_attrs = selected_ltable.columns.values.tolist()  # 两张表中的字段名
+        attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs]
+        attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs]
+        cm.set_key(selected_ltable, ltable_id)
+        cm.set_key(selected_rtable, rtable_id)
+
+        blocker = None
+        if config["ml_blocker"] == "over_lap":
             blocker = em.OverlapBlocker()
-        overlap_attr = 'name' if config["l_overlap_attr"] == 'title' else config["l_overlap_attr"]
-        candidate = blocker.block_tables(selected_Amazon, selected_Google, config["l_overlap_attr"], overlap_attr,
-                                         l_output_attrs=['id', 'title', 'description', 'manufacturer', 'price'],
-                                         r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
-                                         overlap_size=config["overlap_size"], show_progress=False)
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
+                                             overlap_size=config["overlap_size"], show_progress=False, n_jobs=-1)
+        elif config["ml_blocker"] == "attr_equiv":
+            blocker = em.AttrEquivalenceBlocker()
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, n_jobs=-1)
+
         candidate['gold'] = 0
 
         candidate_match_rows = []
         for index, row in candidate.iterrows():
-            l_id = row["ltable_id"]
-            map_row = Mappings[Mappings['idAmazon'] == l_id]
+            l_id = row['ltable_' + ltable_id]
+            map_row = mappings[mappings[mapping_lid] == l_id]
 
             if map_row is not None:
-                r_id = map_row["idGoogleBase"]
+                r_id = map_row[mapping_rid]
                 for value in r_id:
-                    if value == row["rtable_id"]:
+                    if value == row['rtable_' + rtable_id]:
                         candidate_match_rows.append(row["_id"])
             else:
                 continue
@@ -148,14 +158,15 @@ class SVM:
         # 裁剪负样本，保持正负样本数量一致
         candidate_mismatch = candidate[candidate['gold'] == 0]
         candidate_match = candidate[candidate['gold'] == 1]
+        if len(candidate_mismatch) > len(candidate_match):
             candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
         # 拼接正负样本
         candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
         cm.set_key(candidate_for_train_test, '_id')
-        cm.set_fk_ltable(candidate_for_train_test, 'ltable_id')
-        cm.set_fk_rtable(candidate_for_train_test, 'rtable_id')
-        cm.set_ltable(candidate_for_train_test, selected_Amazon)
-        cm.set_rtable(candidate_for_train_test, selected_Google)
+        cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id)
+        cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id)
+        cm.set_ltable(candidate_for_train_test, selected_ltable)
+        cm.set_rtable(candidate_for_train_test, selected_rtable)
 
         # 分为训练测试集
         train_proportion = 0.7
@@ -164,8 +175,20 @@ class SVM:
         train_set = sets['train']
         test_set = sets['test']
 
-        rf = em.RFMatcher(name='RF', random_state=0)
-        feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
+        matcher = None
+        if config["ml_matcher"] == "dt":
+            matcher = em.DTMatcher(name='DecisionTree', random_state=0)
+        elif config["ml_matcher"] == "svm":
+            matcher = em.SVMMatcher(name='SVM', random_state=0)
+        elif config["ml_matcher"] == "rf":
+            matcher = em.RFMatcher(name='RF', random_state=0)
+        elif config["ml_matcher"] == "lg":
+            matcher = em.LogRegMatcher(name='LogReg', random_state=0)
+        elif config["ml_matcher"] == "ln":
+            matcher = em.LinRegMatcher(name='LinReg')
+        elif config["ml_matcher"] == "nb":
+            matcher = em.NBMatcher(name='NaiveBayes')
+        feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
 
         train_feature_vecs = em.extract_feature_vecs(train_set,
                                                      feature_table=feature_table,
@@ -180,12 +203,12 @@ class SVM:
         # todo 参数可调 用drop删除特征向量中的列？
         # 1.exclude_attrs
         # 去掉id相关的相似度
-        rf.fit(table=train_feature_vecs,
+        matcher.fit(table=train_feature_vecs,
                exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
                target_attr='gold')
 
         # 1.exclude_attrs
-        predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title',
+        predictions = matcher.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title',
                                                                          'ltable_description', 'ltable_manufacturer',
                                                                          'ltable_price', 'rtable_name', 'rtable_description',
                                                                          'rtable_manufacturer', 'rtable_price', 'gold'],
@@ -196,9 +219,14 @@ class SVM:
         print(indicators)
 
         # 计算可解释性
-        predictions = predictions[
-            ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price',
-             'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']]
+        predictions_attrs = []
+        predictions_attrs.extend(attrs_with_l_prefix)
+        predictions_attrs.extend(attrs_with_r_prefix)
+        predictions_attrs.extend(['gold', 'predicted'])
+        predictions = predictions[predictions_attrs]
+
+        md_paths = ['../md_discovery/output/tp_mds.txt', '../md_discovery/output/tp_vio.txt',
+                    '../md_discovery/output/fn_mds.txt', '../md_discovery/output/fn_vio.txt']
         epl_match = 0  # 可解释，预测match
         nepl_mismatch = 0  # 不可解释，预测mismatch
         md_list = load_mds(md_paths)  # 从全局变量中读取所有的md
@@ -211,8 +239,7 @@ class SVM:
                     nepl_mismatch += 1
         epl_ability = (epl_match + nepl_mismatch) / len(predictions)  # 可解释性
         f1 = indicators['F1']
-        performance = 0.5 * epl_ability + 0.5 * f1  # 可解释性与F1的权重暂时定为0.5
-        # todo 权重用户可调
+        performance = interpretability_weight * epl_ability + (1 - interpretability_weight) * f1
         return 1 - performance
 
 
@@ -235,6 +262,9 @@ if __name__ == "__main__":
         overwrite=True,  # If the run exists, we overwrite it; alternatively, we can continue from last state
     )
 
+    # todo
+    #  如果new_recall过低则避免其成为最优解
+    #  将损失函数置为1/用new_recall降低F1从而提高损失函数
     incumbent = smac.optimize()
 
     # Get cost of default configuration

md_discovery/functions/multi_process_infer_by_pairs.py

@@ -38,7 +38,6 @@ def if_minimal(md, md_list, target_col):
 
 def remove_by_confidence(md, l, relation, target_col, lock):
     support, confidence = get_one_md_metadata(md, relation, target_col)
-    # todo confidence可调
     if confidence < 0.8:
         with lock:
             l.remove(md)

md_discovery/script/md_discover.py

@@ -1,7 +1,7 @@
 import time
 from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs
 from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata
-
+from entrance import *
 
 # # 若不输出support和confidence，使用以下两块代码
 # # 将列表1写入本地，路径需自己修改
@@ -17,22 +17,22 @@ from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata
 #         f.write(str(_) + '\n')
 
 
-if __name__ == '__main__':
+def md_discover():
     # 目前可以仿照这个main函数写
     tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv"
     fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv"
     # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
     # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
     # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
-    tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, 0.7, 'id')
-    fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, 0.7, 'id')
+    tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, similarity_threshold, target_attr)
+    fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, similarity_threshold, target_attr)
 
     # 如果不需要输出support和confidence，去掉下面两行
-    tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, 'id')
-    tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, 'id')
+    tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, target_attr)
+    tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, target_attr)
 
-    fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, 'id')
-    fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, 'id')
+    fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, target_attr)
+    fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, target_attr)
 
     # 若输出support和confidence，使用以下两块代码
     # 将列表1写入本地，路径需自己修改

ml_er/Goods Dataset.py

@@ -9,10 +9,14 @@ import py_entitymatching.catalog.catalog_manager as cm
 import pandas as pd
 import time
 import six
+from ConfigSpace import Configuration
 from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
+from entrance import *
+from hpo.magellan_hpo import incumbent
 
 
 def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"):
+    # 提取预测表中真阳和假阴部分
     tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)]
     fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)]
     # 将真阳/假阴表中左右ID调整一致
@@ -24,17 +28,21 @@ def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path
     pred_columns = pred.columns.values.tolist()
     l_columns = []
     r_columns = []
-    columns = []  # todo 前提是两张表内对应字段名调整一致
+    columns = []
+    # 将预测表中左表和右表字段名分别加入两个列表
     for _ in pred_columns:
         if _.startswith('ltable'):
             l_columns.append(_)
         elif _.startswith('rtable'):
             r_columns.append(_)
+    # 将左表中字段名去掉前缀，作为统一的字段名列表(前提是两张表内对应字段名调整一致)
     for _ in l_columns:
         columns.append(_.replace('ltable_', ''))
 
+    # 将表拆分成左右两部分
     tpl = tp[l_columns]
     tpr = tp[r_columns]
+    # 将左右两部分字段名统一
     tpl.columns = columns
     tpr.columns = columns
 
@@ -92,6 +100,8 @@ def load_mds(paths: list) -> list:
             for line in f.readlines():
                 md_metadata = line.strip().split('\t')
                 md = eval(md_metadata[0].replace('md:', ''))
+                confidence = eval(md_metadata[2].replace('confidence:', ''))
+                if confidence > 0:
                     mds.append(md)
         all_mds.extend(mds)
     return all_mds
@@ -127,55 +137,58 @@ def load_data(left_path: str, right_path: str, mapping_path: str):
     return left, right, mapping
 
 
-if __name__ == '__main__':
-    # 读入公开数据，注册并填充空值
-    path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
-    path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
-    path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
-    Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1')
-    cm.set_key(Amazon, 'id')
-    Amazon.fillna("", inplace=True)
-    Google = pd.read_csv(path_Google, encoding='ISO-8859-1')
-    cm.set_key(Google, 'id')
-    Google.fillna("", inplace=True)
-    Mappings = pd.read_csv(path_Mappings)
+def ml_er(config: Configuration):
+    # todo:
+    #  if config is not None -> load configs
+    #  else use default configs
+    #  1. overlap_attr
+    ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
+    cm.set_key(ltable, ltable_id)
+    ltable.fillna("", inplace=True)
+    rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
+    cm.set_key(rtable, rtable_id)
+    rtable.fillna("", inplace=True)
+    mappings = pd.read_csv(mapping_path)
 
     # 仅保留两表中出现在映射表中的行，增大正样本比例
-    l_id_list = []
-    r_id_list = []
+    lid_mapping_list = []
+    rid_mapping_list = []
     # 全部转为字符串
-    Amazon = Amazon.astype(str)
-    Google = Google.astype(str)
-    Mappings = Mappings.astype(str)
-    matching_number = len(Mappings)  # 所有阳性样本数，商品数据集应为1300
-
-    for index, row in Mappings.iterrows():
-        l_id_list.append(row["idAmazon"])
-        r_id_list.append(row["idGoogleBase"])
-    selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
-    selected_Amazon = selected_Amazon.rename(columns={'title': 'name'})
-    selected_Google = Google[Google['id'].isin(r_id_list)]
-    cm.set_key(selected_Amazon, 'id')
-    cm.set_key(selected_Google, 'id')
-
+    ltable = ltable.astype(str)
+    rtable = rtable.astype(str)
+    mappings = mappings.astype(str)
+    matching_number = len(mappings)  # 所有阳性样本数，商品数据集应为1300
+
+    for index, row in mappings.iterrows():
+        lid_mapping_list.append(row[mapping_lid])
+        rid_mapping_list.append(row[mapping_rid])
+
+    selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
+    selected_ltable = selected_ltable.rename(columns=lr_attrs_map)  # 参照右表，修改左表中与右表对应但不同名的字段
+    selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
+    selected_attrs = selected_ltable.columns.values.tolist()  # 两张表中的字段名
+    attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs]
+    attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs]
+    cm.set_key(selected_ltable, ltable_id)
+    cm.set_key(selected_rtable, rtable_id)
+
+    # todo 所有可调参数需读入配置并有默认值
     # block 并将gold标记为0
     blocker = em.OverlapBlocker()
-    candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name',
-                                     l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
-                                     r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
+    candidate = blocker.block_tables(selected_ltable, selected_rtable, 'name', 'name',
+                                     l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
                                      overlap_size=1, show_progress=False)
     candidate['gold'] = 0
 
-    start = time.time()
     candidate_match_rows = []
     for index, row in candidate.iterrows():
-        l_id = row["ltable_id"]
-        map_row = Mappings[Mappings['idAmazon'] == l_id]
+        l_id = row['ltable_' + ltable_id]
+        map_row = mappings[mappings[mapping_lid] == l_id]
 
         if map_row is not None:
-            r_id = map_row["idGoogleBase"]
+            r_id = map_row[mapping_rid]
             for value in r_id:
-                if value == row["rtable_id"]:
+                if value == row['rtable_' + rtable_id]:
                     candidate_match_rows.append(row["_id"])
         else:
             continue
@@ -185,14 +198,15 @@ if __name__ == '__main__':
     # 裁剪负样本，保持正负样本数量一致
     candidate_mismatch = candidate[candidate['gold'] == 0]
     candidate_match = candidate[candidate['gold'] == 1]
+    if len(candidate_mismatch) > len(candidate_match):
         candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
     # 拼接正负样本
     candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
     cm.set_key(candidate_for_train_test, '_id')
-    cm.set_fk_ltable(candidate_for_train_test, 'ltable_id')
-    cm.set_fk_rtable(candidate_for_train_test, 'rtable_id')
-    cm.set_ltable(candidate_for_train_test, selected_Amazon)
-    cm.set_rtable(candidate_for_train_test, selected_Google)
+    cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id)
+    cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id)
+    cm.set_ltable(candidate_for_train_test, selected_ltable)
+    cm.set_rtable(candidate_for_train_test, selected_rtable)
 
     # 分为训练测试集
     train_proportion = 0.7
@@ -201,24 +215,14 @@ if __name__ == '__main__':
     train_set = sets['train']
     test_set = sets['test']
 
-    dt = em.DTMatcher(name='DecisionTree', random_state=0)
-    svm = em.SVMMatcher(name='SVM', random_state=0)
     rf = em.RFMatcher(name='RF', random_state=0)
-    lg = em.LogRegMatcher(name='LogReg', random_state=0)
-    ln = em.LinRegMatcher(name='LinReg')
-    nb = em.NBMatcher(name='NaiveBayes')
-    feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
+    feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
 
     train_feature_vecs = em.extract_feature_vecs(train_set,
                                                  feature_table=feature_table,
                                                  attrs_after=['gold'],
                                                  show_progress=False)
 
-    result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs,
-                               exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
-                               k=5,
-                               target_attr='gold', metric_to_select_matcher='f1', random_state=0)
-
     test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
                                                 attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer',
                                                              'ltable_price', 'rtable_name', 'rtable_description',
@@ -241,9 +245,12 @@ if __name__ == '__main__':
 
     # 计算可解释性
     ################################################################################################################
-    predictions = predictions[
-        ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price',
-         'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']]
+    predictions_attrs = []
+    predictions_attrs.extend(attrs_with_l_prefix)
+    predictions_attrs.extend(attrs_with_r_prefix)
+    predictions_attrs.extend(['gold', 'predicted'])
+    predictions = predictions[predictions_attrs]
+
     epl_match = 0  # 可解释，预测match
     nepl_mismatch = 0  # 不可解释，预测mismatch
     p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt"
@@ -261,8 +268,6 @@ if __name__ == '__main__':
     epl_ability = (epl_match + nepl_mismatch) / len(predictions)
     ################################################################################################################
     process_prediction_for_md_discovery(predictions)
-    # todo 将prediction表处理成真阳/假阴表提供给挖掘算法
-
 
     output_path = "output/eval_result" + str(time.time()) + ".txt"
     with open(output_path, 'w') as f: