diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..52d424e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/deprecated/ diff --git a/deprecated/inference_from_record_pairs.py b/deprecated/inference_from_record_pairs.py deleted file mode 100644 index 4d1b96a..0000000 --- a/deprecated/inference_from_record_pairs.py +++ /dev/null @@ -1,152 +0,0 @@ -import pandas as pd -import time -import Levenshtein -import copy - -def my_Levenshtein_ratio(str1, str2): - return 1 - Levenshtein.distance(str1, str2) / max(len(str1), len(str2)) - - -def if_minimal(md, md_list, target_col): - # 假设这个md是minimal - minimal = True - for _ in md_list: - if _ != md: - # 假设列表中每一个md都使当前md不minimal - exist = True - # 如果左边任何一个大于，则假设不成立 - for col in list(set(_.keys()) - {target_col}): - if _[col] > md[col]: - exist = False - # 如果右边小于，假设也不成立 - if _[target_col] < md[target_col]: - exist = False - # 任何一次假设成立，当前md不minimal - if exist: - minimal = False - break - return minimal - - -def satisfy_confidence(md, df, conf_thresh, target_col): - support = 0 - support_plus = 0 - for row1 in df.itertuples(): - i = row1[0] - df_slice = df[i + 1:] - for row2 in df_slice.itertuples(): - left_satisfy = True - both_satisfy = True - for col in df.columns.values.tolist(): - sim = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col)) - if col == target_col: - if sim < 1: - both_satisfy = False - else: - if sim < md[col]: - left_satisfy = False - both_satisfy = False - if left_satisfy: - support += 1 - if both_satisfy: - support_plus += 1 - if support == 0: - return False, 0.0 - confidence = support_plus / support - return confidence >= conf_thresh, confidence - - -def inference_from_record_pairs(path, threshold, target_col): - data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1') - data = data.astype(str) - columns = data.columns.values.tolist() - - md_list = [] - minimal_vio = [] - init_md = {} - for col in columns: - init_md[col] = 1 if col == target_col else 0 - md_list.append(init_md) - - for row1 in data.itertuples(): - # 获取当前行的索引，从后一行开始切片 - i = row1[0] - data1 = data[i + 1:] - for row2 in data1.itertuples(): - violated_mds = [] - # sims是两行的相似度 - sims = {} - for col in columns: - similarity = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col)) - sims[col] = similarity - - # 寻找violated md,从md列表中删除并加入vio列表 - for md in md_list: - lhs_satis = True - rhs_satis = True - for col in list(set(columns) - {target_col}): - if sims[col] < md[col]: - lhs_satis = False - if sims[target_col] < md[target_col]: - rhs_satis = False - if lhs_satis == True and rhs_satis == False: - md_list.remove(md) - violated_mds.append(md) - minimal_vio.extend(violated_mds) - - for vio_md in violated_mds: - # 特殊化右侧,我们需要右侧百分百相似，其实不需要降低右侧阈值 - # if sims[target_col] >= threshold: - # new_rhs = sims[target_col] - # spec_r_md = copy.deepcopy(vio_md) - # spec_r_md[target_col] = new_rhs - # if if_minimal(spec_r_md, md_list, target_col): - # md_list.append(spec_r_md) - - # 特殊化左侧 - for col in list(set(columns) - {target_col}): - if sims[col] + 0.001 <= 1: - spec_l_md = copy.deepcopy(vio_md) - spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.001 - if if_minimal(spec_l_md, md_list, target_col): - md_list.append(spec_l_md) - - for vio in minimal_vio: - if not if_minimal(vio, md_list, target_col): - minimal_vio.remove(vio) - - tmp = copy.deepcopy(minimal_vio) - for _ in tmp: - satis, conf = satisfy_confidence(_, data, 0.8, target_col) - if not satis: - minimal_vio.remove(_) - - for _ in tmp: - if not if_minimal(_, minimal_vio, target_col): - minimal_vio.remove(_) - - return md_list, minimal_vio - - -if __name__ == '__main__': - # 目前可以仿照这个main函数写 - path = "input/T_positive_with_id_concat_single_tuple.csv" - start = time.time() - # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 - # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8) - # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段 - mds, mds_vio = inference_from_record_pairs(path, 0.7, 'id_concat') - - # 将列表1写入本地，路径需自己修改 - md_path = 'output/md.txt' - with open(md_path, 'w') as f: - for _ in mds: - f.write(str(_) + '\n') - - # 将列表2写入本地，路径需自己修改 - vio_path = 'output/vio.txt' - with open(vio_path, 'w') as f: - for _ in mds_vio: - f.write(str(_) + '\n') - - print(time.time() - start) diff --git a/hpo/magellan_hpo.py b/hpo/magellan_hpo.py new file mode 100644 index 0000000..be4bcbd --- /dev/null +++ b/hpo/magellan_hpo.py @@ -0,0 +1,249 @@ +from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer +import py_entitymatching as em +import py_entitymatching.catalog.catalog_manager as cm +import pandas as pd + +from smac import HyperparameterOptimizationFacade, Scenario +from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio +# todo 距离度量用户可调 + +# 全局变量，每次迭代后清空列表，加入新的md路径 +# todo: +# 默认路径为 "../md_discovery/output/xxx.txt" +# 真阳/假阴 mds/vio 共4个md文件 +md_paths = [] + + +def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int, + test_proportion: float) -> dict: + new_df = df.reset_index(drop=False, inplace=False) + gold = new_df[labeled_attr] + predicted = new_df[predicted_attr] + gold_negative = gold[gold == 0].index.values + gold_positive = gold[gold == 1].index.values + predicted_negative = predicted[predicted == 0].index.values + predicted_positive = predicted[predicted == 1].index.values + + false_positive_indices = list(set(gold_negative).intersection(predicted_positive)) + true_positive_indices = list(set(gold_positive).intersection(predicted_positive)) + false_negative_indices = list(set(gold_positive).intersection(predicted_negative)) + + num_true_positives = float(len(true_positive_indices)) + num_false_positives = float(len(false_positive_indices)) + num_false_negatives = float(len(false_negative_indices)) + + precision_denominator = num_true_positives + num_false_positives + recall_denominator = num_true_positives + num_false_negatives + + precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator + recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator + F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall) + my_recall = num_true_positives / (matching_number * test_proportion) + + return {"precision": precision, "recall": recall, "F1": F1, "my_recall": my_recall} + + +def load_mds(paths: list) -> list: + if len(paths) == 0: + return [] + all_mds = [] + # 传入md路径列表 + for md_path in paths: + mds = [] + # 打开每一个md文件 + with open(md_path, 'r') as f: + # 读取每一行的md，加入该文件的md列表 + for line in f.readlines(): + md_metadata = line.strip().split('\t') + md = eval(md_metadata[0].replace('md:', '')) + mds.append(md) + all_mds.extend(mds) + return all_mds + + +def is_explicable(row, all_mds: list) -> bool: + attrs = all_mds[0].keys() # 从第一条md中读取所有字段 + for md in all_mds: + explicable = True # 假设这条md能解释当前元组 + for a in attrs: + threshold = md[a] + if my_Levenshtein_ratio(str(getattr(row, 'ltable_'+a)), str(getattr(row, 'rtable_'+a))) < threshold: + explicable = False # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组 + break # 不再与当前md的其他相似度阈值比较，跳转到下一条md + if explicable: + return True # 任意一条md能解释，直接返回 + return False # 遍历结束，不能解释 + + +class SVM: + @property + def configspace(self) -> ConfigurationSpace: + # Build Configuration Space which defines all parameters and their ranges + cs = ConfigurationSpace(seed=0) + + l_overlap_attr = Categorical("l_overlap_attr", ["title", "description", "manufacturer", "price"], default="title") + overlap_size = Integer("overlap_size", (1, 3), default=1) + + cs.add_hyperparameters([l_overlap_attr, overlap_size]) + + return cs + + # train 就是整个函数 只需将返回结果由预测变成预测结果的评估 + def train(self, config: Configuration, seed: int = 0) -> float: + path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' + path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' + path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' + Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') + cm.set_key(Amazon, 'id') + Amazon.fillna("", inplace=True) + Google = pd.read_csv(path_Google, encoding='ISO-8859-1') + cm.set_key(Google, 'id') + Google.fillna("", inplace=True) + Mappings = pd.read_csv(path_Mappings) + + # 仅保留两表中出现在映射表中的行，增大正样本比例 + l_id_list = [] + r_id_list = [] + # 全部转为字符串 + Amazon = Amazon.astype(str) + Google = Google.astype(str) + Mappings = Mappings.astype(str) + matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 + + for index, row in Mappings.iterrows(): + l_id_list.append(row["idAmazon"]) + r_id_list.append(row["idGoogleBase"]) + selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)] + selected_Google = Google[Google['id'].isin(r_id_list)] + cm.set_key(selected_Amazon, 'id') + cm.set_key(selected_Google, 'id') + + # todo blocker可调 + # 1.blocker类型（商品数据集可能只适合overlap） + # 2.overlap字段（对应关系） + # 3.overlap_size + blocker = em.OverlapBlocker() + overlap_attr = 'name' if config["l_overlap_attr"] == 'title' else config["l_overlap_attr"] + candidate = blocker.block_tables(selected_Amazon, selected_Google, config["l_overlap_attr"], overlap_attr, + l_output_attrs=['id', 'title', 'description', 'manufacturer', 'price'], + r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + overlap_size=config["overlap_size"], show_progress=False) + candidate['gold'] = 0 + + candidate_match_rows = [] + for index, row in candidate.iterrows(): + l_id = row["ltable_id"] + map_row = Mappings[Mappings['idAmazon'] == l_id] + + if map_row is not None: + r_id = map_row["idGoogleBase"] + for value in r_id: + if value == row["rtable_id"]: + candidate_match_rows.append(row["_id"]) + else: + continue + for row in candidate_match_rows: + candidate.loc[row, 'gold'] = 1 + + # 裁剪负样本，保持正负样本数量一致 + candidate_mismatch = candidate[candidate['gold'] == 0] + candidate_match = candidate[candidate['gold'] == 1] + candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) + # 拼接正负样本 + candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) + cm.set_key(candidate_for_train_test, '_id') + cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') + cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') + cm.set_ltable(candidate_for_train_test, selected_Amazon) + cm.set_rtable(candidate_for_train_test, selected_Google) + + # 分为训练测试集 + train_proportion = 0.7 + test_proportion = 0.3 + sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0) + train_set = sets['train'] + test_set = sets['test'] + + rf = em.RFMatcher(name='RF', random_state=0) + feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) + + train_feature_vecs = em.extract_feature_vecs(train_set, + feature_table=feature_table, + attrs_after=['gold'], + show_progress=False) + + test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table, + attrs_after=['ltable_title', 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], show_progress=False) + + # todo 参数可调 用drop删除特征向量中的列？ + # 1.exclude_attrs + # 去掉id相关的相似度 + rf.fit(table=train_feature_vecs, + exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], + target_attr='gold') + + # 1.exclude_attrs + predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title', + 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], + append=True, target_attr='predicted', inplace=False) + eval_result = em.eval_matches(predictions, 'gold', 'predicted') + em.print_eval_summary(eval_result) + indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, test_proportion) + print(indicators) + + # 计算可解释性 + predictions = predictions[ + ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', + 'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] + epl_match = 0 # 可解释，预测match + nepl_mismatch = 0 # 不可解释，预测mismatch + md_list = load_mds(md_paths) # 从全局变量中读取所有的md + for row in predictions.itertuples(): + if is_explicable(row, md_list): + if getattr(row, 'predicted') == 1: + epl_match += 1 + else: + if getattr(row, 'predicted') == 0: + nepl_mismatch += 1 + epl_ability = (epl_match + nepl_mismatch) / len(predictions) # 可解释性 + f1 = indicators['F1'] + performance = 0.5 * epl_ability + 0.5 * f1 # 可解释性与F1的权重暂时定为0.5 + # todo 权重用户可调 + return 1 - performance + + +if __name__ == "__main__": + classifier = SVM() + + # Next, we create an object, holding general information about the run + scenario = Scenario( + classifier.configspace, + n_trials=12, # We want to run max 50 trials (combination of config and seed) + ) + + initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=3) + + # Now we use SMAC to find the best hyperparameters + smac = HyperparameterOptimizationFacade( + scenario, + classifier.train, + initial_design=initial_design, + overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state + ) + + incumbent = smac.optimize() + + # Get cost of default configuration + default_cost = smac.validate(classifier.configspace.get_default_configuration()) + print(f"Default cost: {default_cost}") + + # Let's calculate the cost of the incumbent + incumbent_cost = smac.validate(incumbent) + print(f"Incumbent cost: {incumbent_cost}") + + print(f"Configuration:{incumbent.values()}") + print(f"MAX_F1:{1-classifier.train(incumbent)}") diff --git a/md_discovery/__init__.py b/md_discovery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/md_discovery/functions/__init__.py b/md_discovery/functions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functions/multi_process_infer_by_pairs.py b/md_discovery/functions/multi_process_infer_by_pairs.py similarity index 99% rename from functions/multi_process_infer_by_pairs.py rename to md_discovery/functions/multi_process_infer_by_pairs.py index 199d4ef..46e8f5e 100644 --- a/functions/multi_process_infer_by_pairs.py +++ b/md_discovery/functions/multi_process_infer_by_pairs.py @@ -38,7 +38,7 @@ def if_minimal(md, md_list, target_col): def remove_by_confidence(md, l, relation, target_col, lock): support, confidence = get_one_md_metadata(md, relation, target_col) - # todo: replace constant 0.8 + # todo confidence可调 if confidence < 0.8: with lock: l.remove(md) diff --git a/md_discovery/script/__init__.py b/md_discovery/script/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/md_discovery/script/get_support_and_confidence.py b/md_discovery/script/get_support_and_confidence.py new file mode 100644 index 0000000..9f7eff8 --- /dev/null +++ b/md_discovery/script/get_support_and_confidence.py @@ -0,0 +1,67 @@ +import time +from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs +from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata + + +# # 若不输出support和confidence，使用以下两块代码 +# # 将列表1写入本地，路径需自己修改 +# md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt' +# with open(md_path, 'w') as f: +# for _ in mds: +# f.write(str(_) + '\n') +# +# # 将列表2写入本地，路径需自己修改 +# vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt' +# with open(vio_path, 'w') as f: +# for _ in vio: +# f.write(str(_) + '\n') + + +if __name__ == '__main__': + # 目前可以仿照这个main函数写 + tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv" + fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv" + # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 + # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8) + # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段 + tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, 0.7, 'id') + fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, 0.7, 'id') + + # 如果不需要输出support和confidence，去掉下面两行 + tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, 'id') + tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, 'id') + + fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, 'id') + fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, 'id') + + # 若输出support和confidence，使用以下两块代码 + # 将列表1写入本地，路径需自己修改 + tp_mds_path = "../output/tp_mds.txt" + tp_vio_path = "../output/tp_vio.txt" + + with open(tp_mds_path, 'w') as f: + for _ in tp_mds_meta: + for i in _.keys(): + f.write(i + ':' + str(_[i]) + '\t') + f.write('\n') + + with open(tp_vio_path, 'w') as f: + for _ in tp_vio_meta: + for i in _.keys(): + f.write(i + ':' + str(_[i]) + '\t') + f.write('\n') + + fn_mds_path = "../output/fn_mds.txt" + fn_vio_path = "../output/fn_vio.txt" + + with open(fn_mds_path, 'w') as f: + for _ in fn_mds_meta: + for i in _.keys(): + f.write(i + ':' + str(_[i]) + '\t') + f.write('\n') + + with open(fn_vio_path, 'w') as f: + for _ in fn_vio_meta: + for i in _.keys(): + f.write(i + ':' + str(_[i]) + '\t') + f.write('\n') diff --git a/ml_er/Goods Dataset-8.14.py b/ml_er/Goods Dataset-8.14.py new file mode 100644 index 0000000..be42eb3 --- /dev/null +++ b/ml_er/Goods Dataset-8.14.py @@ -0,0 +1,153 @@ +import sys + +from py_entitymatching.debugmatcher.debug_gui_utils import _get_metric + +sys.path.append('/home/w/PycharmProjects/py_entitymatching/py_entitymatching') + +import py_entitymatching as em +import py_entitymatching.catalog.catalog_manager as cm +import pandas as pd +import time +import six + + +def load_data(left_path: str, right_path: str, mapping_path: str): + left = pd.read_csv(left_path, encoding='ISO-8859-1') + cm.set_key(left, left.columns.values.tolist()[0]) + left.fillna("", inplace=True) + left = left.astype(str) + + right = pd.read_csv(right_path, encoding='ISO-8859-1') + cm.set_key(right, right.columns.values.tolist()[0]) + right.fillna("", inplace=True) + right = right.astype(str) + + mapping = pd.read_csv(mapping_path) + mapping = mapping.astype(str) + return left, right, mapping + + +if __name__ == '__main__': + # 读入公开数据，注册并填充空值 + path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' + path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' + path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' + Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') + cm.set_key(Amazon, 'id') + Amazon.fillna("", inplace=True) + Google = pd.read_csv(path_Google, encoding='ISO-8859-1') + cm.set_key(Google, 'id') + Google.fillna("", inplace=True) + Mappings = pd.read_csv(path_Mappings) + + # 仅保留两表中出现在映射表中的行，增大正样本比例 + l_id_list = [] + r_id_list = [] + # 全部转为字符串 + Amazon = Amazon.astype(str) + Google = Google.astype(str) + Mappings = Mappings.astype(str) + for index, row in Mappings.iterrows(): + l_id_list.append(row["idAmazon"]) + r_id_list.append(row["idGoogleBase"]) + selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)] + selected_Amazon = selected_Amazon.rename(columns={'title': 'name'}) + selected_Google = Google[Google['id'].isin(r_id_list)] + cm.set_key(selected_Amazon, 'id') + cm.set_key(selected_Google, 'id') + + ######################################################################### + # False-retain True-remove + def match_last_name(ltuple, rtuple): + l_last_name = ltuple['name'] + r_last_name = rtuple['name'] + if l_last_name != r_last_name: + return True + else: + return False + + bb = em.BlackBoxBlocker() + bb.set_black_box_function(match_last_name) + + Candidate = bb.block_tables(selected_Amazon, selected_Google, l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price']) + ######################################################################### + # block 并将gold标记为0 + blocker = em.OverlapBlocker() + candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name', + l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + overlap_size=0, show_progress=False) + candidate['gold'] = 0 + + start = time.time() + candidate_match_rows = [] + for index, row in candidate.iterrows(): + l_id = row["ltable_id"] + map_row = Mappings[Mappings['idAmazon'] == l_id] + + if map_row is not None: + r_id = map_row["idGoogleBase"] + for value in r_id: + if value == row["rtable_id"]: + candidate_match_rows.append(row["_id"]) + else: + continue + for row in candidate_match_rows: + candidate.loc[row, 'gold'] = 1 + + # 裁剪负样本，保持正负样本数量一致 + candidate_mismatch = candidate[candidate['gold'] == 0] + candidate_match = candidate[candidate['gold'] == 1] + candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) + # 拼接正负样本 + candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) + cm.set_key(candidate_for_train_test, '_id') + cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') + cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') + cm.set_ltable(candidate_for_train_test, selected_Amazon) + cm.set_rtable(candidate_for_train_test, selected_Google) + + # 分为训练测试集 + sets = em.split_train_test(candidate_for_train_test, train_proportion=0.7, random_state=0) + train_set = sets['train'] + test_set = sets['test'] + + dt = em.DTMatcher(name='DecisionTree', random_state=0) + svm = em.SVMMatcher(name='SVM', random_state=0) + rf = em.RFMatcher(name='RF', random_state=0) + lg = em.LogRegMatcher(name='LogReg', random_state=0) + ln = em.LinRegMatcher(name='LinReg') + nb = em.NBMatcher(name='NaiveBayes') + feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) + + train_feature_vecs = em.extract_feature_vecs(train_set, + feature_table=feature_table, + attrs_after='gold', + show_progress=False) + + result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs, + exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], + k=5, + target_attr='gold', metric_to_select_matcher='f1', random_state=0) + + test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table, + attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], show_progress=False) + + rf.fit(table=train_feature_vecs, + exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], + target_attr='gold') + predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_name', + 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], + append=True, target_attr='predicted', inplace=False) + eval_result = em.eval_matches(predictions, 'gold', 'predicted') + em.print_eval_summary(eval_result) + + output_path = "output/eval_result" + str(time.time()) + ".txt" + with open(output_path, 'w') as f: + for key, value in six.iteritems(_get_metric(eval_result)): + f.write(key + " : " + value) + f.write('\n') diff --git a/ml_er/Goods Dataset.py b/ml_er/Goods Dataset.py new file mode 100644 index 0000000..4a30649 --- /dev/null +++ b/ml_er/Goods Dataset.py @@ -0,0 +1,273 @@ +import sys + +from py_entitymatching.debugmatcher.debug_gui_utils import _get_metric + +sys.path.append('/home/w/PycharmProjects/py_entitymatching/py_entitymatching') + +import py_entitymatching as em +import py_entitymatching.catalog.catalog_manager as cm +import pandas as pd +import time +import six +from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio + + +def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"): + tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)] + fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)] + # 将真阳/假阴表中左右ID调整一致 + for index, row in tp.iterrows(): + tp.loc[index, "rtable_id"] = row["ltable_id"] + for index, row in fn.iterrows(): + fn.loc[index, "rtable_id"] = row["ltable_id"] + + pred_columns = pred.columns.values.tolist() + l_columns = [] + r_columns = [] + columns = [] # todo 前提是两张表内对应字段名调整一致 + for _ in pred_columns: + if _.startswith('ltable'): + l_columns.append(_) + elif _.startswith('rtable'): + r_columns.append(_) + for _ in l_columns: + columns.append(_.replace('ltable_', '')) + + tpl = tp[l_columns] + tpr = tp[r_columns] + tpl.columns = columns + tpr.columns = columns + + fnl = fn[l_columns] + fnr = fn[r_columns] + fnl.columns = columns + fnr.columns = columns + + tp_single_tuple = pd.concat([tpl, tpr]) + fn_single_tuple = pd.concat([fnl, fnr]) + + tp_single_tuple.to_csv(tp_single_tuple_path, sep=',', index=False, header=True) + fn_single_tuple.to_csv(fn_single_tuple_path, sep=',', index=False, header=True) + + +def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int, + test_proportion: float) -> dict: + new_df = df.reset_index(drop=False, inplace=False) + gold = new_df[labeled_attr] + predicted = new_df[predicted_attr] + gold_negative = gold[gold == 0].index.values + gold_positive = gold[gold == 1].index.values + predicted_negative = predicted[predicted == 0].index.values + predicted_positive = predicted[predicted == 1].index.values + + false_positive_indices = list(set(gold_negative).intersection(predicted_positive)) + true_positive_indices = list(set(gold_positive).intersection(predicted_positive)) + false_negative_indices = list(set(gold_positive).intersection(predicted_negative)) + + num_true_positives = float(len(true_positive_indices)) + num_false_positives = float(len(false_positive_indices)) + num_false_negatives = float(len(false_negative_indices)) + + precision_denominator = num_true_positives + num_false_positives + recall_denominator = num_true_positives + num_false_negatives + + precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator + recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator + F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall) + my_recall = num_true_positives / (matching_number * test_proportion) + + return {"precision": precision, "recall": recall, "F1": F1, "my_recall": my_recall} + + +def load_mds(paths: list) -> list: + if len(paths) == 0: + return [] + all_mds = [] + # 传入md路径列表 + for md_path in paths: + mds = [] + # 打开每一个md文件 + with open(md_path, 'r') as f: + # 读取每一行的md，加入该文件的md列表 + for line in f.readlines(): + md_metadata = line.strip().split('\t') + md = eval(md_metadata[0].replace('md:', '')) + mds.append(md) + all_mds.extend(mds) + return all_mds + + +def is_explicable(row, all_mds: list) -> bool: + attrs = all_mds[0].keys() # 从第一条md中读取所有字段 + for md in all_mds: + explicable = True # 假设这条md能解释当前元组 + for a in attrs: + threshold = md[a] + if my_Levenshtein_ratio(str(getattr(row, 'ltable_'+a)), str(getattr(row, 'rtable_'+a))) < threshold: + explicable = False # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组 + break # 不再与当前md的其他相似度阈值比较，跳转到下一条md + if explicable: + return True # 任意一条md能解释，直接返回 + return False # 遍历结束，不能解释 + + +def load_data(left_path: str, right_path: str, mapping_path: str): + left = pd.read_csv(left_path, encoding='ISO-8859-1') + cm.set_key(left, left.columns.values.tolist()[0]) + left.fillna("", inplace=True) + left = left.astype(str) + + right = pd.read_csv(right_path, encoding='ISO-8859-1') + cm.set_key(right, right.columns.values.tolist()[0]) + right.fillna("", inplace=True) + right = right.astype(str) + + mapping = pd.read_csv(mapping_path) + mapping = mapping.astype(str) + return left, right, mapping + + +if __name__ == '__main__': + # 读入公开数据，注册并填充空值 + path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' + path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' + path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' + Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') + cm.set_key(Amazon, 'id') + Amazon.fillna("", inplace=True) + Google = pd.read_csv(path_Google, encoding='ISO-8859-1') + cm.set_key(Google, 'id') + Google.fillna("", inplace=True) + Mappings = pd.read_csv(path_Mappings) + + # 仅保留两表中出现在映射表中的行，增大正样本比例 + l_id_list = [] + r_id_list = [] + # 全部转为字符串 + Amazon = Amazon.astype(str) + Google = Google.astype(str) + Mappings = Mappings.astype(str) + matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 + + for index, row in Mappings.iterrows(): + l_id_list.append(row["idAmazon"]) + r_id_list.append(row["idGoogleBase"]) + selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)] + selected_Amazon = selected_Amazon.rename(columns={'title': 'name'}) + selected_Google = Google[Google['id'].isin(r_id_list)] + cm.set_key(selected_Amazon, 'id') + cm.set_key(selected_Google, 'id') + + # block 并将gold标记为0 + blocker = em.OverlapBlocker() + candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name', + l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], + overlap_size=1, show_progress=False) + candidate['gold'] = 0 + + start = time.time() + candidate_match_rows = [] + for index, row in candidate.iterrows(): + l_id = row["ltable_id"] + map_row = Mappings[Mappings['idAmazon'] == l_id] + + if map_row is not None: + r_id = map_row["idGoogleBase"] + for value in r_id: + if value == row["rtable_id"]: + candidate_match_rows.append(row["_id"]) + else: + continue + for row in candidate_match_rows: + candidate.loc[row, 'gold'] = 1 + + # 裁剪负样本，保持正负样本数量一致 + candidate_mismatch = candidate[candidate['gold'] == 0] + candidate_match = candidate[candidate['gold'] == 1] + candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) + # 拼接正负样本 + candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) + cm.set_key(candidate_for_train_test, '_id') + cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') + cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') + cm.set_ltable(candidate_for_train_test, selected_Amazon) + cm.set_rtable(candidate_for_train_test, selected_Google) + + # 分为训练测试集 + train_proportion = 0.7 + test_proportion = 0.3 + sets = em.split_train_test(candidate_for_train_test, train_proportion=0.7, random_state=0) + train_set = sets['train'] + test_set = sets['test'] + + dt = em.DTMatcher(name='DecisionTree', random_state=0) + svm = em.SVMMatcher(name='SVM', random_state=0) + rf = em.RFMatcher(name='RF', random_state=0) + lg = em.LogRegMatcher(name='LogReg', random_state=0) + ln = em.LinRegMatcher(name='LinReg') + nb = em.NBMatcher(name='NaiveBayes') + feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) + + train_feature_vecs = em.extract_feature_vecs(train_set, + feature_table=feature_table, + attrs_after=['gold'], + show_progress=False) + + result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs, + exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], + k=5, + target_attr='gold', metric_to_select_matcher='f1', random_state=0) + + test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table, + attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], + show_progress=False) + + rf.fit(table=train_feature_vecs, + exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], + target_attr='gold') + predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_name', + 'ltable_description', 'ltable_manufacturer', + 'ltable_price', 'rtable_name', + 'rtable_description', + 'rtable_manufacturer', 'rtable_price', 'gold'], + append=True, target_attr='predicted', inplace=False) + eval_result = em.eval_matches(predictions, 'gold', 'predicted') + em.print_eval_summary(eval_result) + indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, test_proportion) + print(indicators) + + # 计算可解释性 + ################################################################################################################ + predictions = predictions[ + ['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', + 'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] + epl_match = 0 # 可解释，预测match + nepl_mismatch = 0 # 不可解释，预测mismatch + p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt" + p_vio = "/home/w/A-New Folder/8.14/Goods Dataset/TP_vio_list.txt" + md_paths: list = [p_md, p_vio] + md_list = load_mds(md_paths) # 从全局变量中读取所有的md + for row in predictions.itertuples(): + if is_explicable(row, md_list): + if getattr(row, 'predicted') == 1: + epl_match += 1 + else: + if getattr(row, 'predicted') == 0: + nepl_mismatch += 1 + + epl_ability = (epl_match + nepl_mismatch) / len(predictions) + ################################################################################################################ + process_prediction_for_md_discovery(predictions) + # todo 将prediction表处理成真阳/假阴表提供给挖掘算法 + + + output_path = "output/eval_result" + str(time.time()) + ".txt" + with open(output_path, 'w') as f: + for key, value in six.iteritems(_get_metric(eval_result)): + f.write(key + " : " + value) + f.write('\n') + f.write('my_recall:' + str(indicators["my_recall"])) + f.write('\n') diff --git a/ml_er/__init__.py b/ml_er/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/script/get_support_and_confidence.py b/script/get_support_and_confidence.py deleted file mode 100644 index 8a16007..0000000 --- a/script/get_support_and_confidence.py +++ /dev/null @@ -1,48 +0,0 @@ -import time -from functions.multi_process_infer_by_pairs import inference_from_record_pairs -from functions.multi_process_infer_by_pairs import get_mds_metadata - -if __name__ == '__main__': - # 目前可以仿照这个main函数写 - path = "/home/w/PycharmProjects/matching_dependency/input/T_positive_with_id_concat_single_tuple.csv" - start = time.time() - # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 - # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8) - # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段 - mds, mds_vio = inference_from_record_pairs(path, 0.1, 'id_concat') - - # 如果不需要输出support和confidence，去掉下面两行 - mds_meta = get_mds_metadata(mds, path, 'id_concat') - mds_vio_meta = get_mds_metadata(mds_vio, path, 'id_concat') - - # # 若不输出support和confidence，使用以下两块代码 - # # 将列表1写入本地，路径需自己修改 - # md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt' - # with open(md_path, 'w') as f: - # for _ in mds: - # f.write(str(_) + '\n') - # - # # 将列表2写入本地，路径需自己修改 - # vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt' - # with open(vio_path, 'w') as f: - # for _ in mds_vio: - # f.write(str(_) + '\n') - - # 若输出support和confidence，使用以下两块代码 - # 将列表1写入本地，路径需自己修改 - md_path = "output/md.txt" - with open(md_path, 'w') as f: - for _ in mds_meta: - for i in _.keys(): - f.write(i + ':' + str(_[i]) + '\t') - f.write('\n') - - # 将列表2写入本地，路径需自己修改 - vio_path = "output/vio.txt" - with open(vio_path, 'w') as f: - for _ in mds_vio_meta: - for i in _.keys(): - f.write(i + ':' + str(_[i]) + '\t') - f.write('\n') - - print(time.time() - start)