p04316279
/
matching_dependency
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

9.19

Browse Source
HuangJintao
HuangJintao 1 year ago
parent 838dba05c4
commit 27795aac1d
6 changed files with 200 additions and 132 deletions
Show Stats Download Patch File Download Diff File

34
entrance.py
Unescape View File

@ -0,0 +1,34 @@
# this is the entrance of the auto-ER procedure
from md_discovery.script.md_discover import md_discover
# todo: magellan ER模块读入初始化配置或hpo配置
# todo: 模块间的自动化调用
# 入口到ER/HPO到ER
def run(l_table_path, r_table_path, mapping_path):
# while The termination condition is not met:
while True:
# er()
md_discover()
# hpo()
return
if __name__ == '__main__':
# todo：使用input函数输入变量值（不方便就不用input）
# 7. 距离度量方式 ？
ltable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv'
rtable_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv'
mapping_path = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv'
mapping_lid = 'idAmazon' # mapping表中左表id名
mapping_rid = 'idGoogleBase' # mapping表中右表id名
ltable_id = 'id' # 左表id字段名称
rtable_id = 'id' # 右表id字段名称
target_attr = 'id' # 进行md挖掘时的目标字段
lr_attrs_map = {'title': 'name'} # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致
similarity_threshold = 0.7
confidence_threshold = 0.8
interpretability_weight = 0.3
print(ltable_path)

0
hpo/__init__.py
Unescape View File

148
hpo/magellan_hpo.py
Unescape View File

@ -1,17 +1,20 @@
from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer, Float
import py_entitymatching as em import py_entitymatching as em
import py_entitymatching.catalog.catalog_manager as cm import py_entitymatching.catalog.catalog_manager as cm
import pandas as pd import pandas as pd
from py_entitymatching.blocker.blocker import Blocker
from py_entitymatching.matcher.mlmatcher import MLMatcher
from smac import HyperparameterOptimizationFacade, Scenario from smac import HyperparameterOptimizationFacade, Scenario
from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
# todo 距离度量用户可调 from entrance import *
# todo 距离度量用户可设置
# 全局变量，每次迭代后清空列表，加入新的md路径 # 全局变量，每次迭代后清空列表，加入新的md路径
# todo: # todo:
# 默认路径为 "../md_discovery/output/xxx.txt" # 默认路径为 "../md_discovery/output/xxx.txt"
# 真阳/假阴 mds/vio 共4个md文件 # 真阳/假阴 mds/vio 共4个md文件
md_paths = []
def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int, def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int,
@ -56,6 +59,8 @@ def load_mds(paths: list) -> list:
for line in f.readlines(): for line in f.readlines():
md_metadata = line.strip().split('\t') md_metadata = line.strip().split('\t')
md = eval(md_metadata[0].replace('md:', '')) md = eval(md_metadata[0].replace('md:', ''))
confidence = eval(md_metadata[2].replace('confidence:', ''))
if confidence > 0:
mds.append(md) mds.append(md)
all_mds.extend(mds) all_mds.extend(mds)
return all_mds return all_mds
@ -80,65 +85,70 @@ class SVM:
def configspace(self) -> ConfigurationSpace: def configspace(self) -> ConfigurationSpace:
# Build Configuration Space which defines all parameters and their ranges # Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace(seed=0) cs = ConfigurationSpace(seed=0)
# todo
l_overlap_attr = Categorical("l_overlap_attr", ["title", "description", "manufacturer", "price"], default="title") # block_attr 取消打桩
block_attr = Categorical("block_attr", ["name", "description", "manufacturer", "price"], default="title")
overlap_size = Integer("overlap_size", (1, 3), default=1) overlap_size = Integer("overlap_size", (1, 3), default=1)
ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf")
ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap")
cs.add_hyperparameters([l_overlap_attr, overlap_size]) cs.add_hyperparameters([block_attr, overlap_size, ml_matcher, ml_blocker])
return cs return cs
# train 就是整个函数 只需将返回结果由预测变成预测结果的评估 # train 就是整个函数 只需将返回结果由预测变成预测结果的评估
def train(self, config: Configuration, seed: int = 0) -> float: def train(self, config: Configuration) -> float:
path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' cm.set_key(ltable, ltable_id)
path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' ltable.fillna("", inplace=True)
Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
cm.set_key(Amazon, 'id') cm.set_key(rtable, rtable_id)
Amazon.fillna("", inplace=True) rtable.fillna("", inplace=True)
Google = pd.read_csv(path_Google, encoding='ISO-8859-1') mappings = pd.read_csv(mapping_path)
cm.set_key(Google, 'id')
Google.fillna("", inplace=True)
Mappings = pd.read_csv(path_Mappings)
# 仅保留两表中出现在映射表中的行，增大正样本比例 # 仅保留两表中出现在映射表中的行，增大正样本比例
l_id_list = [] lid_mapping_list = []
r_id_list = [] rid_mapping_list = []
# 全部转为字符串 # 全部转为字符串
Amazon = Amazon.astype(str) ltable = ltable.astype(str)
Google = Google.astype(str) rtable = rtable.astype(str)
Mappings = Mappings.astype(str) mappings = mappings.astype(str)
matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 matching_number = len(mappings) # 所有阳性样本数，商品数据集应为1300
for index, row in Mappings.iterrows(): for index, row in mappings.iterrows():
l_id_list.append(row["idAmazon"]) lid_mapping_list.append(row[mapping_lid])
r_id_list.append(row["idGoogleBase"]) rid_mapping_list.append(row[mapping_rid])
selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
selected_Google = Google[Google['id'].isin(r_id_list)] selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
cm.set_key(selected_Amazon, 'id') selected_ltable = selected_ltable.rename(columns=lr_attrs_map) # 参照右表，修改左表中与右表对应但不同名的字段
cm.set_key(selected_Google, 'id') selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
selected_attrs = selected_ltable.columns.values.tolist() # 两张表中的字段名
# todo blocker可调 attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs]
# 1.blocker类型（商品数据集可能只适合overlap） attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs]
# 2.overlap字段（对应关系） cm.set_key(selected_ltable, ltable_id)
# 3.overlap_size cm.set_key(selected_rtable, rtable_id)
blocker = None
if config["ml_blocker"] == "over_lap":
blocker = em.OverlapBlocker() blocker = em.OverlapBlocker()
overlap_attr = 'name' if config["l_overlap_attr"] == 'title' else config["l_overlap_attr"] candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
candidate = blocker.block_tables(selected_Amazon, selected_Google, config["l_overlap_attr"], overlap_attr, l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
l_output_attrs=['id', 'title', 'description', 'manufacturer', 'price'], overlap_size=config["overlap_size"], show_progress=False, n_jobs=-1)
r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], elif config["ml_blocker"] == "attr_equiv":
overlap_size=config["overlap_size"], show_progress=False) blocker = em.AttrEquivalenceBlocker()
candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
l_output_attrs=selected_attrs, r_output_attrs=selected_attrs, n_jobs=-1)
candidate['gold'] = 0 candidate['gold'] = 0
candidate_match_rows = [] candidate_match_rows = []
for index, row in candidate.iterrows(): for index, row in candidate.iterrows():
l_id = row["ltable_id"] l_id = row['ltable_' + ltable_id]
map_row = Mappings[Mappings['idAmazon'] == l_id] map_row = mappings[mappings[mapping_lid] == l_id]
if map_row is not None: if map_row is not None:
r_id = map_row["idGoogleBase"] r_id = map_row[mapping_rid]
for value in r_id: for value in r_id:
if value == row["rtable_id"]: if value == row['rtable_' + rtable_id]:
candidate_match_rows.append(row["_id"]) candidate_match_rows.append(row["_id"])
else: else:
continue continue
@ -148,14 +158,15 @@ class SVM:
# 裁剪负样本，保持正负样本数量一致 # 裁剪负样本，保持正负样本数量一致
candidate_mismatch = candidate[candidate['gold'] == 0] candidate_mismatch = candidate[candidate['gold'] == 0]
candidate_match = candidate[candidate['gold'] == 1] candidate_match = candidate[candidate['gold'] == 1]
if len(candidate_mismatch) > len(candidate_match):
candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
# 拼接正负样本 # 拼接正负样本
candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
cm.set_key(candidate_for_train_test, '_id') cm.set_key(candidate_for_train_test, '_id')
cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id)
cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id)
cm.set_ltable(candidate_for_train_test, selected_Amazon) cm.set_ltable(candidate_for_train_test, selected_ltable)
cm.set_rtable(candidate_for_train_test, selected_Google) cm.set_rtable(candidate_for_train_test, selected_rtable)
# 分为训练测试集 # 分为训练测试集
train_proportion = 0.7 train_proportion = 0.7
@ -164,8 +175,20 @@ class SVM:
train_set = sets['train'] train_set = sets['train']
test_set = sets['test'] test_set = sets['test']
rf = em.RFMatcher(name='RF', random_state=0) matcher = None
feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False) if config["ml_matcher"] == "dt":
matcher = em.DTMatcher(name='DecisionTree', random_state=0)
elif config["ml_matcher"] == "svm":
matcher = em.SVMMatcher(name='SVM', random_state=0)
elif config["ml_matcher"] == "rf":
matcher = em.RFMatcher(name='RF', random_state=0)
elif config["ml_matcher"] == "lg":
matcher = em.LogRegMatcher(name='LogReg', random_state=0)
elif config["ml_matcher"] == "ln":
matcher = em.LinRegMatcher(name='LinReg')
elif config["ml_matcher"] == "nb":
matcher = em.NBMatcher(name='NaiveBayes')
feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
train_feature_vecs = em.extract_feature_vecs(train_set, train_feature_vecs = em.extract_feature_vecs(train_set,
feature_table=feature_table, feature_table=feature_table,
@ -180,12 +203,12 @@ class SVM:
# todo 参数可调 用drop删除特征向量中的列？ # todo 参数可调 用drop删除特征向量中的列？
# 1.exclude_attrs # 1.exclude_attrs
# 去掉id相关的相似度 # 去掉id相关的相似度
rf.fit(table=train_feature_vecs, matcher.fit(table=train_feature_vecs,
exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'], exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
target_attr='gold') target_attr='gold')
# 1.exclude_attrs # 1.exclude_attrs
predictions = rf.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title', predictions = matcher.predict(table=test_feature_vecs, exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'ltable_title',
'ltable_description', 'ltable_manufacturer', 'ltable_description', 'ltable_manufacturer',
'ltable_price', 'rtable_name', 'rtable_description', 'ltable_price', 'rtable_name', 'rtable_description',
'rtable_manufacturer', 'rtable_price', 'gold'], 'rtable_manufacturer', 'rtable_price', 'gold'],
@ -196,9 +219,14 @@ class SVM:
print(indicators) print(indicators)
# 计算可解释性 # 计算可解释性
predictions = predictions[ predictions_attrs = []
['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', predictions_attrs.extend(attrs_with_l_prefix)
'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] predictions_attrs.extend(attrs_with_r_prefix)
predictions_attrs.extend(['gold', 'predicted'])
predictions = predictions[predictions_attrs]
md_paths = ['../md_discovery/output/tp_mds.txt', '../md_discovery/output/tp_vio.txt',
'../md_discovery/output/fn_mds.txt', '../md_discovery/output/fn_vio.txt']
epl_match = 0 # 可解释，预测match epl_match = 0 # 可解释，预测match
nepl_mismatch = 0 # 不可解释，预测mismatch nepl_mismatch = 0 # 不可解释，预测mismatch
md_list = load_mds(md_paths) # 从全局变量中读取所有的md md_list = load_mds(md_paths) # 从全局变量中读取所有的md
@ -211,8 +239,7 @@ class SVM:
nepl_mismatch += 1 nepl_mismatch += 1
epl_ability = (epl_match + nepl_mismatch) / len(predictions) # 可解释性 epl_ability = (epl_match + nepl_mismatch) / len(predictions) # 可解释性
f1 = indicators['F1'] f1 = indicators['F1']
performance = 0.5 * epl_ability + 0.5 * f1 # 可解释性与F1的权重暂时定为0.5 performance = interpretability_weight * epl_ability + (1 - interpretability_weight) * f1
# todo 权重用户可调
return 1 - performance return 1 - performance
@ -235,6 +262,9 @@ if __name__ == "__main__":
overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state overwrite=True, # If the run exists, we overwrite it; alternatively, we can continue from last state
) )
# todo
# 如果new_recall过低则避免其成为最优解
# 将损失函数置为1/用new_recall降低F1从而提高损失函数
incumbent = smac.optimize() incumbent = smac.optimize()
# Get cost of default configuration # Get cost of default configuration

1
md_discovery/functions/multi_process_infer_by_pairs.py
Unescape View File

@ -38,7 +38,6 @@ def if_minimal(md, md_list, target_col):
def remove_by_confidence(md, l, relation, target_col, lock): def remove_by_confidence(md, l, relation, target_col, lock):
support, confidence = get_one_md_metadata(md, relation, target_col) support, confidence = get_one_md_metadata(md, relation, target_col)
# todo confidence可调
if confidence < 0.8: if confidence < 0.8:
with lock: with lock:
l.remove(md) l.remove(md)

16
md_discovery/script/get_support_and_confidence.py → md_discovery/script/md_discover.py
Unescape View File

@ -1,7 +1,7 @@
import time import time
from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs from md_discovery.functions.multi_process_infer_by_pairs import inference_from_record_pairs
from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata
from entrance import *
# # 若不输出support和confidence，使用以下两块代码 # # 若不输出support和confidence，使用以下两块代码
# # 将列表1写入本地，路径需自己修改 # # 将列表1写入本地，路径需自己修改
@ -17,22 +17,22 @@ from md_discovery.functions.multi_process_infer_by_pairs import get_mds_metadata
# f.write(str(_) + '\n') # f.write(str(_) + '\n')
if __name__ == '__main__': def md_discover():
# 目前可以仿照这个main函数写 # 目前可以仿照这个main函数写
tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv" tp_single_tuple_path = "../../ml_er/output/tp_single_tuple.csv"
fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv" fn_single_tuple_path = "../../ml_er/output/fn_single_tuple.csv"
# 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段 # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
# 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8) # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
# 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段 # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, 0.7, 'id') tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, similarity_threshold, target_attr)
fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, 0.7, 'id') fn_mds, fn_vio = inference_from_record_pairs(fn_single_tuple_path, similarity_threshold, target_attr)
# 如果不需要输出support和confidence，去掉下面两行 # 如果不需要输出support和confidence，去掉下面两行
tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, 'id') tp_mds_meta = get_mds_metadata(tp_mds, tp_single_tuple_path, target_attr)
tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, 'id') tp_vio_meta = get_mds_metadata(tp_vio, tp_single_tuple_path, target_attr)
fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, 'id') fn_mds_meta = get_mds_metadata(fn_mds, fn_single_tuple_path, target_attr)
fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, 'id') fn_vio_meta = get_mds_metadata(fn_vio, fn_single_tuple_path, target_attr)
# 若输出support和confidence，使用以下两块代码 # 若输出support和confidence，使用以下两块代码
# 将列表1写入本地，路径需自己修改 # 将列表1写入本地，路径需自己修改

119
ml_er/Goods Dataset.py
Unescape View File

@ -9,10 +9,14 @@ import py_entitymatching.catalog.catalog_manager as cm
import pandas as pd import pandas as pd
import time import time
import six import six
from ConfigSpace import Configuration
from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio from md_discovery.functions.multi_process_infer_by_pairs import my_Levenshtein_ratio
from entrance import *
from hpo.magellan_hpo import incumbent
def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"): def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path: str = "output/tp_single_tuple.csv", fn_single_tuple_path: str = "output/fn_single_tuple.csv"):
# 提取预测表中真阳和假阴部分
tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)] tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)]
fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)] fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)]
# 将真阳/假阴表中左右ID调整一致 # 将真阳/假阴表中左右ID调整一致
@ -24,17 +28,21 @@ def process_prediction_for_md_discovery(pred: pd.DataFrame, tp_single_tuple_path
pred_columns = pred.columns.values.tolist() pred_columns = pred.columns.values.tolist()
l_columns = [] l_columns = []
r_columns = [] r_columns = []
columns = [] # todo 前提是两张表内对应字段名调整一致 columns = []
# 将预测表中左表和右表字段名分别加入两个列表
for _ in pred_columns: for _ in pred_columns:
if _.startswith('ltable'): if _.startswith('ltable'):
l_columns.append(_) l_columns.append(_)
elif _.startswith('rtable'): elif _.startswith('rtable'):
r_columns.append(_) r_columns.append(_)
# 将左表中字段名去掉前缀，作为统一的字段名列表(前提是两张表内对应字段名调整一致)
for _ in l_columns: for _ in l_columns:
columns.append(_.replace('ltable_', '')) columns.append(_.replace('ltable_', ''))
# 将表拆分成左右两部分
tpl = tp[l_columns] tpl = tp[l_columns]
tpr = tp[r_columns] tpr = tp[r_columns]
# 将左右两部分字段名统一
tpl.columns = columns tpl.columns = columns
tpr.columns = columns tpr.columns = columns
@ -92,6 +100,8 @@ def load_mds(paths: list) -> list:
for line in f.readlines(): for line in f.readlines():
md_metadata = line.strip().split('\t') md_metadata = line.strip().split('\t')
md = eval(md_metadata[0].replace('md:', '')) md = eval(md_metadata[0].replace('md:', ''))
confidence = eval(md_metadata[2].replace('confidence:', ''))
if confidence > 0:
mds.append(md) mds.append(md)
all_mds.extend(mds) all_mds.extend(mds)
return all_mds return all_mds
@ -127,55 +137,58 @@ def load_data(left_path: str, right_path: str, mapping_path: str):
return left, right, mapping return left, right, mapping
if __name__ == '__main__': def ml_er(config: Configuration):
# 读入公开数据，注册并填充空值 # todo:
path_Amazon = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amazon.csv' # if config is not None -> load configs
path_Google = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/GoogleProducts.csv' # else use default configs
path_Mappings = '/home/w/PycharmProjects/py_entitymatching/py_entitymatching/datasets/end-to-end/Amazon-GoogleProducts/Amzon_GoogleProducts_perfectMapping.csv' # 1. overlap_attr
Amazon = pd.read_csv(path_Amazon, encoding='ISO-8859-1') ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
cm.set_key(Amazon, 'id') cm.set_key(ltable, ltable_id)
Amazon.fillna("", inplace=True) ltable.fillna("", inplace=True)
Google = pd.read_csv(path_Google, encoding='ISO-8859-1') rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
cm.set_key(Google, 'id') cm.set_key(rtable, rtable_id)
Google.fillna("", inplace=True) rtable.fillna("", inplace=True)
Mappings = pd.read_csv(path_Mappings) mappings = pd.read_csv(mapping_path)
# 仅保留两表中出现在映射表中的行，增大正样本比例 # 仅保留两表中出现在映射表中的行，增大正样本比例
l_id_list = [] lid_mapping_list = []
r_id_list = [] rid_mapping_list = []
# 全部转为字符串 # 全部转为字符串
Amazon = Amazon.astype(str) ltable = ltable.astype(str)
Google = Google.astype(str) rtable = rtable.astype(str)
Mappings = Mappings.astype(str) mappings = mappings.astype(str)
matching_number = len(Mappings) # 所有阳性样本数，商品数据集应为1300 matching_number = len(mappings) # 所有阳性样本数，商品数据集应为1300
for index, row in Mappings.iterrows(): for index, row in mappings.iterrows():
l_id_list.append(row["idAmazon"]) lid_mapping_list.append(row[mapping_lid])
r_id_list.append(row["idGoogleBase"]) rid_mapping_list.append(row[mapping_rid])
selected_Amazon = Amazon[Amazon['id'].isin(l_id_list)]
selected_Amazon = selected_Amazon.rename(columns={'title': 'name'}) selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
selected_Google = Google[Google['id'].isin(r_id_list)] selected_ltable = selected_ltable.rename(columns=lr_attrs_map) # 参照右表，修改左表中与右表对应但不同名的字段
cm.set_key(selected_Amazon, 'id') selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
cm.set_key(selected_Google, 'id') selected_attrs = selected_ltable.columns.values.tolist() # 两张表中的字段名
attrs_with_l_prefix = ['ltable_'+i for i in selected_attrs]
attrs_with_r_prefix = ['rtable_'+i for i in selected_attrs]
cm.set_key(selected_ltable, ltable_id)
cm.set_key(selected_rtable, rtable_id)
# todo 所有可调参数需读入配置并有默认值
# block 并将gold标记为0 # block 并将gold标记为0
blocker = em.OverlapBlocker() blocker = em.OverlapBlocker()
candidate = blocker.block_tables(selected_Amazon, selected_Google, 'name', 'name', candidate = blocker.block_tables(selected_ltable, selected_rtable, 'name', 'name',
l_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'], l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
r_output_attrs=['id', 'name', 'description', 'manufacturer', 'price'],
overlap_size=1, show_progress=False) overlap_size=1, show_progress=False)
candidate['gold'] = 0 candidate['gold'] = 0
start = time.time()
candidate_match_rows = [] candidate_match_rows = []
for index, row in candidate.iterrows(): for index, row in candidate.iterrows():
l_id = row["ltable_id"] l_id = row['ltable_' + ltable_id]
map_row = Mappings[Mappings['idAmazon'] == l_id] map_row = mappings[mappings[mapping_lid] == l_id]
if map_row is not None: if map_row is not None:
r_id = map_row["idGoogleBase"] r_id = map_row[mapping_rid]
for value in r_id: for value in r_id:
if value == row["rtable_id"]: if value == row['rtable_' + rtable_id]:
candidate_match_rows.append(row["_id"]) candidate_match_rows.append(row["_id"])
else: else:
continue continue
@ -185,14 +198,15 @@ if __name__ == '__main__':
# 裁剪负样本，保持正负样本数量一致 # 裁剪负样本，保持正负样本数量一致
candidate_mismatch = candidate[candidate['gold'] == 0] candidate_mismatch = candidate[candidate['gold'] == 0]
candidate_match = candidate[candidate['gold'] == 1] candidate_match = candidate[candidate['gold'] == 1]
if len(candidate_mismatch) > len(candidate_match):
candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match)) candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
# 拼接正负样本 # 拼接正负样本
candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match]) candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
cm.set_key(candidate_for_train_test, '_id') cm.set_key(candidate_for_train_test, '_id')
cm.set_fk_ltable(candidate_for_train_test, 'ltable_id') cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + ltable_id)
cm.set_fk_rtable(candidate_for_train_test, 'rtable_id') cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + rtable_id)
cm.set_ltable(candidate_for_train_test, selected_Amazon) cm.set_ltable(candidate_for_train_test, selected_ltable)
cm.set_rtable(candidate_for_train_test, selected_Google) cm.set_rtable(candidate_for_train_test, selected_rtable)
# 分为训练测试集 # 分为训练测试集
train_proportion = 0.7 train_proportion = 0.7
@ -201,24 +215,14 @@ if __name__ == '__main__':
train_set = sets['train'] train_set = sets['train']
test_set = sets['test'] test_set = sets['test']
dt = em.DTMatcher(name='DecisionTree', random_state=0)
svm = em.SVMMatcher(name='SVM', random_state=0)
rf = em.RFMatcher(name='RF', random_state=0) rf = em.RFMatcher(name='RF', random_state=0)
lg = em.LogRegMatcher(name='LogReg', random_state=0) feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
ln = em.LinRegMatcher(name='LinReg')
nb = em.NBMatcher(name='NaiveBayes')
feature_table = em.get_features_for_matching(selected_Amazon, selected_Google, validate_inferred_attr_types=False)
train_feature_vecs = em.extract_feature_vecs(train_set, train_feature_vecs = em.extract_feature_vecs(train_set,
feature_table=feature_table, feature_table=feature_table,
attrs_after=['gold'], attrs_after=['gold'],
show_progress=False) show_progress=False)
result = em.select_matcher([dt, rf, svm, ln, lg, nb], table=train_feature_vecs,
exclude_attrs=['_id', 'ltable_id', 'rtable_id', 'gold'],
k=5,
target_attr='gold', metric_to_select_matcher='f1', random_state=0)
test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table, test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer', attrs_after=['ltable_name', 'ltable_description', 'ltable_manufacturer',
'ltable_price', 'rtable_name', 'rtable_description', 'ltable_price', 'rtable_name', 'rtable_description',
@ -241,9 +245,12 @@ if __name__ == '__main__':
# 计算可解释性 # 计算可解释性
################################################################################################################ ################################################################################################################
predictions = predictions[ predictions_attrs = []
['ltable_id', 'rtable_id', 'ltable_name', 'ltable_description', 'ltable_manufacturer', 'ltable_price', predictions_attrs.extend(attrs_with_l_prefix)
'rtable_name', 'rtable_description', 'rtable_manufacturer', 'rtable_price', 'gold', 'predicted']] predictions_attrs.extend(attrs_with_r_prefix)
predictions_attrs.extend(['gold', 'predicted'])
predictions = predictions[predictions_attrs]
epl_match = 0 # 可解释，预测match epl_match = 0 # 可解释，预测match
nepl_mismatch = 0 # 不可解释，预测mismatch nepl_mismatch = 0 # 不可解释，预测mismatch
p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt" p_md = "/home/w/A-New Folder/8.14/Goods Dataset/TP_md_list.txt"
@ -261,8 +268,6 @@ if __name__ == '__main__':
epl_ability = (epl_match + nepl_mismatch) / len(predictions) epl_ability = (epl_match + nepl_mismatch) / len(predictions)
################################################################################################################ ################################################################################################################
process_prediction_for_md_discovery(predictions) process_prediction_for_md_discovery(predictions)
# todo 将prediction表处理成真阳/假阴表提供给挖掘算法
output_path = "output/eval_result" + str(time.time()) + ".txt" output_path = "output/eval_result" + str(time.time()) + ".txt"
with open(output_path, 'w') as f: with open(output_path, 'w') as f:

Write Preview
Loading…
Cancel
Save