From 27795aac1dde5b086deef547b6e7c526cc0170a1 Mon Sep 17 00:00:00 2001 From: HuangJintao <1447537163@qq.com> Date: Tue, 19 Sep 2023 22:29:08 +0800 Subject: [PATCH] 9.19 --- entrance.py | 34 ++++ hpo/__init__.py | 0 hpo/magellan_hpo.py | 158 +++++++++++------- .../functions/multi_process_infer_by_pairs.py | 1 - ...pport_and_confidence.py => md_discover.py} | 16 +- ml_er/Goods Dataset.py | 123 +++++++------- 6 files changed, 200 insertions(+), 132 deletions(-) create mode 100644 entrance.py create mode 100644 hpo/__init__.py rename md_discovery/script/{get_support_and_confidence.py => md_discover.py} (93%) diff --git a/entrance.py b/entrance.py new file mode 100644 index 0000000..6ec509b --- /dev/null +++ b/entrance.py @@ -0,0 +1,34 @@ +# this is the entrance of the auto-ER procedure +from md_discovery.script.md_discover import md_discover + + +# todo: magellan ER模块读入初始化配置或hpo配置 +# todo: 模块间的自动化调用 +# 入口到ER/HPO到ER + + +def run(l_table_path, r_table_path, mapping_path): + # while The termination condition is not met: + while True: + # er() + md_discover() + # hpo() + return + + +if __name__ == '__main__': + # todo：使用input函数输入变量值（不方便就不用input） + # 7. 距离度量方式 ？ + ltable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' + rtable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' + mapping_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' + mapping_lid = 'idAmazon' # mapping表中左表id名 + mapping_rid = 'idGoogleBase' # mapping表中右表id名 + ltable_id = 'id' # 左表id字段名称 + rtable_id = 'id' # 右表id字段名称 + target_attr = 'id' # 进行md挖掘时的目标字段 + lr_attrs_map = {'title': 'name'} # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致 + similarity_threshold = 0.7 + confidence_threshold = 0.8 + interpretability_weight = 0.3 + print(ltable_path) diff --git a/hpo/__init__.py b/hpo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hpo/magellan_hpo.py b/hpo/magellan_hpo.py index be4bcbd..8376e99 100644 --- a/hpo/magellan_hpo.py +++ b/hpo/magellan_hpo.py @@ -1,17 +1,20 @@ -from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer +from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer, Float import py_entitymatching as em import py_entitymatching.catalog.catalog_manager as cm import pandas as pd +from py_entitymatching.blocker.blocker import Blocker +from py_entitymatching.matcher.mlmatcher import MLMatcher from smac import HyperparameterOptimizationFacade, Scenario from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio -# todo 距离度量用户可调 +from entrance import * +# todo 距离度量用户可设置 # 全局变量，每次迭代后清空列表，加入新的md路径 # todo: -# 默认路径为 "../md_discovery/output/xxx.txt" -# 真阳/假阴 mds/vio 共4个md文件 -md_paths = [] +# 默认路径为 "../md_discovery/output/xxx.txt" +# 真阳/假阴 mds/vio 共4个md文件 + def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int, @@ -56,7 +59,9 @@ def load_mds(paths: list) -> list: for line in f.readlines(): md_metadata = line.strip().split('\t') md = eval(md_metadata[0].replace('md:', '')) - mds.append(md) + confidence = eval(md_metadata[2].replace('confidence:', '')) + if confidence > 0: + mds.append(md) all_mds.extend(mds) return all_mds @@ -80,65 +85,70 @@ class SVM: def configspace(self) -> ConfigurationSpace: # Build Configuration Space which defines all parameters and their ranges cs = ConfigurationSpace(seed=0) - - l_overlap_attr = Categorical("l_overlap_attr", ["title", "description", "manufacturer", "price"], default="title") + # todo + # block_attr 取消打桩 + block_attr = Categorical("block_attr", ["name", "description", "manufacturer", "price"], default="title") overlap_size = Integer("overlap_size", (1, 3), default=1) + ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf") + ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap") - cs.add_hyperparameters([l_overlap_attr, overlap_size]) - + cs.add_hyperparameters([block_attr, overlap_size, ml_matcher, ml_blocker]) return cs # train 就是整个函数 只需将返回结果由预测变成预测结果的评估 - def train(self, config: Configuration, seed: int = 0) -> float: - path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' - path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' - path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' - Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') - cm.set_key(Amazon, 'id') - Amazon.fillna("", inplace=True) - Google = pd.read_csv(path_Google, encoding='ISO-8859-1') - cm.set_key(Google, 'id') - Google.fillna("", inplace=True) - Mappings = pd.read_csv(path_Mappings) + def train(self, config: Configuration) -> float: + ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1') + cm.set_key(ltable, ltable_id) + ltable.fillna("", inplace=True) + rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1') + cm.set_key(rtable, rtable_id) + rtable.fillna("", inplace=True) + mappings = pd.read_csv(mapping_path) # 仅保留两表中出现在映射表中的行，增大正样本比例 - l_id_list = [] - r_id_list = [] + lid_mapping_list = [] + rid_mapping_list = [] # 全部转为字符串 - Amazon = Amazon.astype(str) - Google = Google.astype(str) - Mappings = Mappings.astype(str) - matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 - - for index, row in Mappings.iterrows(): - l_id_list.append(row["idAmazon"]) - r_id_list.append(row["idGoogleBase"]) - selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)] - selected_Google = Google[Google['id'].isin(r_id_list)] - cm.set_key(selected_Amazon, 'id') - cm.set_key(selected_Google, 'id') - - # todo blocker可调 - # 1.blocker类型（商品数据集可能只适合overlap） - # 2.overlap字段（对应关系） - # 3.overlap_size - blocker = em.OverlapBlocker() - overlap_attr = 'name' if config["l_overlap_attr"] == 'title' else config["l_overlap_attr"] - candidate = blocker.block_tables(selected_Amazon, selected_Google, config["l_overlap_attr"], overlap_attr, - l_output_attrs=['id', 'title', 'description', 'manufacturer', 'price'], - r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], - overlap_size=config["overlap_size"], show_progress=False) + ltable = ltable.astype(str) + rtable = rtable.astype(str) + mappings = mappings.astype(str) + matching_number = len(mappings) # 所有阳性样本数，商品数据集应为1300 + + for index, row in mappings.iterrows(): + lid_mapping_list.append(row[mapping_lid]) + rid_mapping_list.append(row[mapping_rid]) + + selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)] + selected_ltable = selected_ltable.rename(columns=lr_attrs_map) # 参照右表，修改左表中与右表对应但不同名的字段 + selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)] + selected_attrs = selected_ltable.columns.values.tolist() # 两张表中的字段名 + attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs] + attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs] + cm.set_key(selected_ltable, ltable_id) + cm.set_key(selected_rtable, rtable_id) + + blocker = None + if config["ml_blocker"] == "over_lap": + blocker = em.OverlapBlocker() + candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"], + l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, + overlap_size=config["overlap_size"], show_progress=False, n_jobs=-1) + elif config["ml_blocker"] == "attr_equiv": + blocker = em.AttrEquivalenceBlocker() + candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"], + l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, n_jobs=-1) + candidate['gold'] = 0 candidate_match_rows = [] for index, row in candidate.iterrows(): - l_id = row["ltable_id"] - map_row = Mappings[Mappings['idAmazon'] == l_id] + l_id = row['ltable_' + ltable_id] + map_row = mappings[mappings[mapping_lid] == l_id] if map_row is not None: - r_id = map_row["idGoogleBase"] + r_id = map_row[mapping_rid] for value in r_id: - if value == row["rtable_id"]: + if value == row['rtable_' + rtable_id]: candidate_match_rows.append(row["_id"]) else: continue @@ -148,14 +158,15 @@ class SVM: # 裁剪负样本，保持正负样本数量一致 candidate_mismatch = candidate[candidate['gold'] == 0] candidate_match = candidate[candidate['gold'] == 1] - candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) + if len(candidate_mismatch) > len(candidate_match): + candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) # 拼接正负样本 candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) cm.set_key(candidate_for_train_test, '_id') - cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') - cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') - cm.set_ltable(candidate_for_train_test, selected_Amazon) - cm.set_rtable(candidate_for_train_test, selected_Google) + cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id) + cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id) + cm.set_ltable(candidate_for_train_test, selected_ltable) + cm.set_rtable(candidate_for_train_test, selected_rtable) # 分为训练测试集 train_proportion = 0.7 @@ -164,8 +175,20 @@ class SVM: train_set = sets['train'] test_set = sets['test'] - rf = em.RFMatcher(name='RF', random_state=0) - feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) + matcher = None + if config["ml_matcher"] == "dt": + matcher = em.DTMatcher(name='DecisionTree', random_state=0) + elif config["ml_matcher"] == "svm": + matcher = em.SVMMatcher(name='SVM', random_state=0) + elif config["ml_matcher"] == "rf": + matcher = em.RFMatcher(name='RF', random_state=0) + elif config["ml_matcher"] == "lg": + matcher = em.LogRegMatcher(name='LogReg', random_state=0) + elif config["ml_matcher"] == "ln": + matcher = em.LinRegMatcher(name='LinReg') + elif config["ml_matcher"] == "nb": + matcher = em.NBMatcher(name='NaiveBayes') + feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False) train_feature_vecs = em.extract_feature_vecs(train_set, feature_table=feature_table, @@ -180,12 +203,12 @@ class SVM: # todo 参数可调 用drop删除特征向量中的列？ # 1.exclude_attrs # 去掉id相关的相似度 - rf.fit(table=train_feature_vecs, + matcher.fit(table=train_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], target_attr='gold') # 1.exclude_attrs - predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title', + predictions = matcher.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title', 'ltable_description', 'ltable_manufacturer', 'ltable_price', 'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold'], @@ -196,9 +219,14 @@ class SVM: print(indicators) # 计算可解释性 - predictions = predictions[ - ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', - 'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] + predictions_attrs = [] + predictions_attrs.extend(attrs_with_l_prefix) + predictions_attrs.extend(attrs_with_r_prefix) + predictions_attrs.extend(['gold', 'predicted']) + predictions = predictions[predictions_attrs] + + md_paths = ['../md_discovery/output/tp_mds.txt', '../md_discovery/output/tp_vio.txt', + '../md_discovery/output/fn_mds.txt', '../md_discovery/output/fn_vio.txt'] epl_match = 0 # 可解释，预测match nepl_mismatch = 0 # 不可解释，预测mismatch md_list = load_mds(md_paths) # 从全局变量中读取所有的md @@ -211,8 +239,7 @@ class SVM: nepl_mismatch += 1 epl_ability = (epl_match + nepl_mismatch) / len(predictions) # 可解释性 f1 = indicators['F1'] - performance = 0.5 * epl_ability + 0.5 * f1 # 可解释性与F1的权重暂时定为0.5 - # todo 权重用户可调 + performance = interpretability_weight * epl_ability + (1 - interpretability_weight) * f1 return 1 - performance @@ -235,6 +262,9 @@ if __name__ == "__main__": overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state ) + # todo + # 如果new_recall过低则避免其成为最优解 + # 将损失函数置为1/用new_recall降低F1从而提高损失函数 incumbent = smac.optimize() # Get cost of default configuration diff --git a/md_discovery/functions/multi_process_infer_by_pairs.py b/md_discovery/functions/multi_process_infer_by_pairs.py index 46e8f5e..8f6b867 100644 --- a/md_discovery/functions/multi_process_infer_by_pairs.py +++ b/md_discovery/functions/multi_process_infer_by_pairs.py @@ -38,7 +38,6 @@ def if_minimal(md, md_list, target_col): def remove_by_confidence(md, l, relation, target_col, lock): support, confidence = get_one_md_metadata(md, relation, target_col) - # todo confidence可调 if confidence < 0.8: with lock: l.remove(md) diff --git a/md_discovery/script/get_support_and_confidence.py b/md_discovery/script/md_discover.py similarity index 93% rename from md_discovery/script/get_support_and_confidence.py rename to md_discovery/script/md_discover.py index 9f7eff8..6a87c9b 100644 --- a/md_discovery/script/get_support_and_confidence.py +++ b/md_discovery/script/md_discover.py @@ -1,7 +1,7 @@ import time from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata - +from entrance import * # # 若不输出support和confidence，使用以下两块代码 # # 将列表1写入本地，路径需自己修改 @@ -17,22 +17,22 @@ from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata # f.write(str(_) + '\n') -if __name__ == '__main__': +def md_discover(): # 目前可以仿照这个main函数写 tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv" fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv" # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8) # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段 - tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, 0.7, 'id') - fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, 0.7, 'id') + tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, similarity_threshold, target_attr) + fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, similarity_threshold, target_attr) # 如果不需要输出support和confidence，去掉下面两行 - tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, 'id') - tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, 'id') + tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, target_attr) + tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, target_attr) - fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, 'id') - fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, 'id') + fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, target_attr) + fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, target_attr) # 若输出support和confidence，使用以下两块代码 # 将列表1写入本地，路径需自己修改 diff --git a/ml_er/Goods Dataset.py b/ml_er/Goods Dataset.py index 4a30649..c77dd03 100644 --- a/ml_er/Goods Dataset.py +++ b/ml_er/Goods Dataset.py @@ -9,10 +9,14 @@ import py_entitymatching.catalog.catalog_manager as cm import pandas as pd import time import six +from ConfigSpace import Configuration from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio +from entrance import * +from hpo.magellan_hpo import incumbent def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"): + # 提取预测表中真阳和假阴部分 tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)] fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)] # 将真阳/假阴表中左右ID调整一致 @@ -24,17 +28,21 @@ def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path pred_columns = pred.columns.values.tolist() l_columns = [] r_columns = [] - columns = [] # todo 前提是两张表内对应字段名调整一致 + columns = [] + # 将预测表中左表和右表字段名分别加入两个列表 for _ in pred_columns: if _.startswith('ltable'): l_columns.append(_) elif _.startswith('rtable'): r_columns.append(_) + # 将左表中字段名去掉前缀，作为统一的字段名列表(前提是两张表内对应字段名调整一致) for _ in l_columns: columns.append(_.replace('ltable_', '')) + # 将表拆分成左右两部分 tpl = tp[l_columns] tpr = tp[r_columns] + # 将左右两部分字段名统一 tpl.columns = columns tpr.columns = columns @@ -92,7 +100,9 @@ def load_mds(paths: list) -> list: for line in f.readlines(): md_metadata = line.strip().split('\t') md = eval(md_metadata[0].replace('md:', '')) - mds.append(md) + confidence = eval(md_metadata[2].replace('confidence:', '')) + if confidence > 0: + mds.append(md) all_mds.extend(mds) return all_mds @@ -127,55 +137,58 @@ def load_data(left_path: str, right_path: str, mapping_path: str): return left, right, mapping -if __name__ == '__main__': - # 读入公开数据，注册并填充空值 - path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' - path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' - path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' - Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') - cm.set_key(Amazon, 'id') - Amazon.fillna("", inplace=True) - Google = pd.read_csv(path_Google, encoding='ISO-8859-1') - cm.set_key(Google, 'id') - Google.fillna("", inplace=True) - Mappings = pd.read_csv(path_Mappings) +def ml_er(config: Configuration): + # todo: + # if config is not None -> load configs + # else use default configs + # 1. overlap_attr + ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1') + cm.set_key(ltable, ltable_id) + ltable.fillna("", inplace=True) + rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1') + cm.set_key(rtable, rtable_id) + rtable.fillna("", inplace=True) + mappings = pd.read_csv(mapping_path) # 仅保留两表中出现在映射表中的行，增大正样本比例 - l_id_list = [] - r_id_list = [] + lid_mapping_list = [] + rid_mapping_list = [] # 全部转为字符串 - Amazon = Amazon.astype(str) - Google = Google.astype(str) - Mappings = Mappings.astype(str) - matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 - - for index, row in Mappings.iterrows(): - l_id_list.append(row["idAmazon"]) - r_id_list.append(row["idGoogleBase"]) - selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)] - selected_Amazon = selected_Amazon.rename(columns={'title': 'name'}) - selected_Google = Google[Google['id'].isin(r_id_list)] - cm.set_key(selected_Amazon, 'id') - cm.set_key(selected_Google, 'id') - + ltable = ltable.astype(str) + rtable = rtable.astype(str) + mappings = mappings.astype(str) + matching_number = len(mappings) # 所有阳性样本数，商品数据集应为1300 + + for index, row in mappings.iterrows(): + lid_mapping_list.append(row[mapping_lid]) + rid_mapping_list.append(row[mapping_rid]) + + selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)] + selected_ltable = selected_ltable.rename(columns=lr_attrs_map) # 参照右表，修改左表中与右表对应但不同名的字段 + selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)] + selected_attrs = selected_ltable.columns.values.tolist() # 两张表中的字段名 + attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs] + attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs] + cm.set_key(selected_ltable, ltable_id) + cm.set_key(selected_rtable, rtable_id) + + # todo 所有可调参数需读入配置并有默认值 # block 并将gold标记为0 blocker = em.OverlapBlocker() - candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name', - l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], - r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + candidate = blocker.block_tables(selected_ltable, selected_rtable, 'name', 'name', + l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, overlap_size=1, show_progress=False) candidate['gold'] = 0 - start = time.time() candidate_match_rows = [] for index, row in candidate.iterrows(): - l_id = row["ltable_id"] - map_row = Mappings[Mappings['idAmazon'] == l_id] + l_id = row['ltable_' + ltable_id] + map_row = mappings[mappings[mapping_lid] == l_id] if map_row is not None: - r_id = map_row["idGoogleBase"] + r_id = map_row[mapping_rid] for value in r_id: - if value == row["rtable_id"]: + if value == row['rtable_' + rtable_id]: candidate_match_rows.append(row["_id"]) else: continue @@ -185,14 +198,15 @@ if __name__ == '__main__': # 裁剪负样本，保持正负样本数量一致 candidate_mismatch = candidate[candidate['gold'] == 0] candidate_match = candidate[candidate['gold'] == 1] - candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) + if len(candidate_mismatch) > len(candidate_match): + candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) # 拼接正负样本 candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) cm.set_key(candidate_for_train_test, '_id') - cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') - cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') - cm.set_ltable(candidate_for_train_test, selected_Amazon) - cm.set_rtable(candidate_for_train_test, selected_Google) + cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id) + cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id) + cm.set_ltable(candidate_for_train_test, selected_ltable) + cm.set_rtable(candidate_for_train_test, selected_rtable) # 分为训练测试集 train_proportion = 0.7 @@ -201,24 +215,14 @@ if __name__ == '__main__': train_set = sets['train'] test_set = sets['test'] - dt = em.DTMatcher(name='DecisionTree', random_state=0) - svm = em.SVMMatcher(name='SVM', random_state=0) rf = em.RFMatcher(name='RF', random_state=0) - lg = em.LogRegMatcher(name='LogReg', random_state=0) - ln = em.LinRegMatcher(name='LinReg') - nb = em.NBMatcher(name='NaiveBayes') - feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) + feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False) train_feature_vecs = em.extract_feature_vecs(train_set, feature_table=feature_table, attrs_after=['gold'], show_progress=False) - result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs, - exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], - k=5, - target_attr='gold', metric_to_select_matcher='f1', random_state=0) - test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table, attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', 'rtable_name', 'rtable_description', @@ -241,9 +245,12 @@ if __name__ == '__main__': # 计算可解释性 ################################################################################################################ - predictions = predictions[ - ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', - 'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] + predictions_attrs = [] + predictions_attrs.extend(attrs_with_l_prefix) + predictions_attrs.extend(attrs_with_r_prefix) + predictions_attrs.extend(['gold', 'predicted']) + predictions = predictions[predictions_attrs] + epl_match = 0 # 可解释，预测match nepl_mismatch = 0 # 不可解释，预测mismatch p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt" @@ -261,8 +268,6 @@ if __name__ == '__main__': epl_ability = (epl_match + nepl_mismatch) / len(predictions) ################################################################################################################ process_prediction_for_md_discovery(predictions) - # todo 将prediction表处理成真阳/假阴表提供给挖掘算法 - output_path = "output/eval_result" + str(time.time()) + ".txt" with open(output_path, 'w') as f: