|
|
import os
|
|
|
import sys
|
|
|
import time
|
|
|
|
|
|
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
|
|
|
sys.path.append('/root/hjt/md_bayesian_er_ditto/')
|
|
|
import pdb
|
|
|
import pickle
|
|
|
import torch
|
|
|
import json
|
|
|
import numpy as np
|
|
|
import pandas as pd
|
|
|
import random
|
|
|
from tqdm import tqdm
|
|
|
from setting import *
|
|
|
from colorama import Fore
|
|
|
from argparse import Namespace
|
|
|
import ConfigSpace
|
|
|
from ConfigSpace import Configuration
|
|
|
from ditto.matcher import set_seed, predict, tune_threshold, load_model
|
|
|
from ConfigSpace.read_and_write import json as csj
|
|
|
from ditto.ditto_light.dataset import DittoDataset
|
|
|
from ditto.ditto_light.summarize import Summarizer
|
|
|
from ditto.ditto_light.knowledge import *
|
|
|
from ditto.ditto_light.ditto import train
|
|
|
|
|
|
|
|
|
def matching(hpo_config):
|
|
|
print(Fore.BLUE + f'Config: {hpo_config}')
|
|
|
|
|
|
with open(md_output_dir + "/mds.pickle", "rb") as file:
|
|
|
md_list = pickle.load(file)
|
|
|
|
|
|
hp = Namespace()
|
|
|
hp.task = directory_path.replace('/root/hjt/DeepMatcher Dataset/', '')
|
|
|
|
|
|
# only a single task for baseline
|
|
|
task = hp.task
|
|
|
# load task configuration
|
|
|
configs = json.load(open('../ditto/configs.json'))
|
|
|
configs = {conf['name']: conf for conf in configs}
|
|
|
config = configs[task]
|
|
|
|
|
|
config['trainset'] = '/root/hjt/md_bayesian_er_ditto/ditto/' + config['trainset']
|
|
|
config['validset'] = '/root/hjt/md_bayesian_er_ditto/ditto/' + config['validset']
|
|
|
config['testset'] = '/root/hjt/md_bayesian_er_ditto/ditto/' + config['testset']
|
|
|
|
|
|
trainset = config['trainset']
|
|
|
validset = config['validset']
|
|
|
testset = config['testset']
|
|
|
|
|
|
hp.run_id = 0
|
|
|
hp.batch_size = hpo_config['batch_size']
|
|
|
hp.max_len = hpo_config['max_len']
|
|
|
hp.lr = 3e-5
|
|
|
# hp.n_epochs = 20
|
|
|
hp.n_epochs = 2
|
|
|
# hp.finetuning
|
|
|
hp.save_model = True
|
|
|
hp.input_path = config['testset']
|
|
|
hp.output_path = '/root/autodl-tmp/output/predictions.jsonl'
|
|
|
|
|
|
hp.logdir = '/root/autodl-tmp/checkpoints/'
|
|
|
hp.checkpoint_path = '/root/autodl-tmp/checkpoints/'
|
|
|
hp.lm = hpo_config['language_model']
|
|
|
hp.fp16 = hpo_config['half_precision_float']
|
|
|
hp.da = hpo_config['data_augmentation']
|
|
|
hp.alpha_aug = 0.8
|
|
|
hp.dk = None
|
|
|
hp.summarize = hpo_config['summarize']
|
|
|
hp.size = None
|
|
|
hp.use_gpu = True
|
|
|
|
|
|
seed = hp.run_id
|
|
|
random.seed(seed)
|
|
|
np.random.seed(seed)
|
|
|
torch.manual_seed(seed)
|
|
|
if torch.cuda.is_available():
|
|
|
torch.cuda.manual_seed_all(seed)
|
|
|
|
|
|
# create the tag of the run
|
|
|
run_tag = '%s_lm=%s_da=%s_dk=%s_su=%s_size=%s_id=%d' % (task, hp.lm, hp.da,
|
|
|
hp.dk, hp.summarize, str(hp.size), hp.run_id)
|
|
|
run_tag = run_tag.replace('/', '_')
|
|
|
|
|
|
# summarize the sequences up to the max sequence length
|
|
|
if hp.summarize:
|
|
|
summarizer = Summarizer(config, lm=hp.lm)
|
|
|
trainset = summarizer.transform_file(trainset, max_len=hp.max_len)
|
|
|
validset = summarizer.transform_file(validset, max_len=hp.max_len)
|
|
|
testset = summarizer.transform_file(testset, max_len=hp.max_len)
|
|
|
|
|
|
# load train/dev/test sets
|
|
|
train_dataset = DittoDataset(trainset,
|
|
|
lm=hp.lm,
|
|
|
max_len=hp.max_len,
|
|
|
size=hp.size,
|
|
|
da=hp.da)
|
|
|
valid_dataset = DittoDataset(validset, lm=hp.lm)
|
|
|
test_dataset = DittoDataset(testset, lm=hp.lm)
|
|
|
|
|
|
# train and evaluate the model
|
|
|
train(train_dataset,
|
|
|
valid_dataset,
|
|
|
test_dataset,
|
|
|
run_tag, hp)
|
|
|
|
|
|
set_seed(123)
|
|
|
config, model = load_model(hp.task, hp.checkpoint_path,
|
|
|
hp.lm, hp.use_gpu, hp.fp16)
|
|
|
|
|
|
summarizer = dk_injector = None
|
|
|
|
|
|
if hp.summarize:
|
|
|
summarizer = Summarizer(config, hp.lm)
|
|
|
|
|
|
# tune threshold
|
|
|
threshold = tune_threshold(config, model, hp)
|
|
|
|
|
|
# run prediction
|
|
|
predict(hp.input_path, hp.output_path, config, model,
|
|
|
summarizer=summarizer,
|
|
|
max_len=hp.max_len,
|
|
|
lm=hp.lm,
|
|
|
dk_injector=dk_injector,
|
|
|
threshold=threshold)
|
|
|
|
|
|
predictions_raw = pd.read_json(hp.output_path, encoding='ISO-8859-1', lines=True)
|
|
|
predictions = pd.read_csv(directory_path + '/test_whole.csv', encoding='ISO-8859-1')
|
|
|
predictions['predicted'] = predictions_raw['match']
|
|
|
indicators = evaluate_prediction(predictions, 'label', 'predicted')
|
|
|
predictions.drop(columns='_id', inplace=True)
|
|
|
predictions = predictions.reset_index(drop=True)
|
|
|
predictions = predictions.astype(str)
|
|
|
|
|
|
sim_tensor_dict = build_col_pairs_sim_tensor_dict(predictions)
|
|
|
predictions['confidence'] = 0
|
|
|
predictions['md'] = ''
|
|
|
|
|
|
epl_match = 0 # 可解释，预测match
|
|
|
if len(md_list) > 0:
|
|
|
for row in tqdm(predictions.itertuples()):
|
|
|
if str(getattr(row, 'predicted')) == str(1):
|
|
|
conf, md_dict = is_explicable(row, md_list, sim_tensor_dict)
|
|
|
if conf > 0:
|
|
|
predictions.loc[row[0], 'confidence'] = conf
|
|
|
predictions.loc[row[0], 'md'] = str(md_dict)
|
|
|
epl_match += 1
|
|
|
|
|
|
df = predictions[predictions['predicted'] == str(1)]
|
|
|
interpretability = epl_match / len(df) # 可解释性
|
|
|
indicators['interpretability'] = interpretability
|
|
|
performance = interpre_weight * interpretability + (1 - interpre_weight) * indicators["F1"]
|
|
|
indicators['performance'] = performance
|
|
|
print(Fore.BLUE + f'ER Indicators: {indicators}')
|
|
|
predictions.to_csv(er_output_dir + '/predictions.csv', sep=',', index=False, header=True)
|
|
|
print(Fore.CYAN + f'Finish Time: {time.time()}')
|
|
|
return indicators
|
|
|
|
|
|
|
|
|
def evaluate_prediction(prediction_: pd.DataFrame, labeled_attr: str, predicted_attr: str) -> dict:
|
|
|
new_df = prediction_.reset_index(drop=False, inplace=False)
|
|
|
gold = new_df[labeled_attr]
|
|
|
predicted = new_df[predicted_attr]
|
|
|
gold_negative = gold[gold == 0].index.values
|
|
|
gold_positive = gold[gold == 1].index.values
|
|
|
predicted_negative = predicted[predicted == 0].index.values
|
|
|
predicted_positive = predicted[predicted == 1].index.values
|
|
|
|
|
|
false_positive_indices = list(set(gold_negative).intersection(predicted_positive))
|
|
|
true_positive_indices = list(set(gold_positive).intersection(predicted_positive))
|
|
|
false_negative_indices = list(set(gold_positive).intersection(predicted_negative))
|
|
|
|
|
|
num_true_positives = float(len(true_positive_indices))
|
|
|
num_false_positives = float(len(false_positive_indices))
|
|
|
num_false_negatives = float(len(false_negative_indices))
|
|
|
|
|
|
precision_denominator = num_true_positives + num_false_positives
|
|
|
recall_denominator = num_true_positives + num_false_negatives
|
|
|
|
|
|
precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator
|
|
|
recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator
|
|
|
F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall)
|
|
|
|
|
|
return {"precision": precision, "recall": recall, "F1": F1}
|
|
|
|
|
|
|
|
|
def build_col_pairs_sim_tensor_dict(predictions: pd.DataFrame):
|
|
|
predictions_attrs = predictions.columns.values.tolist()
|
|
|
col_tuple_list = []
|
|
|
for _ in predictions_attrs:
|
|
|
if _.startswith('ltable'):
|
|
|
left_index = predictions_attrs.index(_)
|
|
|
right_index = predictions_attrs.index(_.replace('ltable_', 'rtable_'))
|
|
|
col_tuple_list.append((left_index, right_index))
|
|
|
|
|
|
length = predictions.shape[0]
|
|
|
# width = predictions.shape[1]
|
|
|
predictions = predictions.reset_index(drop=True)
|
|
|
sentences = predictions.values.flatten(order='F').tolist()
|
|
|
|
|
|
embedding = model.encode(sentences, convert_to_tensor=True, device="cuda", batch_size=256, show_progress_bar=True)
|
|
|
split_embedding = torch.split(embedding, length, dim=0)
|
|
|
table_tensor = torch.stack(split_embedding, dim=0, out=None)
|
|
|
# prediction的归一化嵌入张量
|
|
|
norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2)
|
|
|
sim_tensor_dict = {}
|
|
|
for col_tuple in col_tuple_list:
|
|
|
lattr_tensor = norm_table_tensor[col_tuple[0]]
|
|
|
rattr_tensor = norm_table_tensor[col_tuple[1]]
|
|
|
mul_tensor = lattr_tensor * rattr_tensor
|
|
|
sim_tensor = torch.sum(mul_tensor, 1)
|
|
|
sim_tensor = torch.round(sim_tensor * 100) / 100
|
|
|
sim_tensor_dict[predictions_attrs[col_tuple[0]].replace('ltable_', '')] = sim_tensor
|
|
|
return sim_tensor_dict
|
|
|
|
|
|
|
|
|
def is_explicable(row, all_mds: list, st_dict):
|
|
|
attrs = all_mds[0][0].keys() # 从第一条md_tuple中的md字典中读取所有字段
|
|
|
for md_tuple in all_mds:
|
|
|
explicable = True # 假设这条md能解释当前元组
|
|
|
for a in attrs:
|
|
|
if st_dict[a][row[0]].item() < md_tuple[0][a]:
|
|
|
explicable = False # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组
|
|
|
break # 不再与当前md的其他相似度阈值比较，跳转到下一条md
|
|
|
if explicable:
|
|
|
return md_tuple[2], md_tuple[0] # 任意一条md能解释，直接返回
|
|
|
return -1.0, {} # 遍历结束，不能解释
|
|
|
|
|
|
|
|
|
def ml_er(config: Configuration):
|
|
|
indicators = matching(config)
|
|
|
output_path = er_output_dir + "/eval_result.txt"
|
|
|
with open(output_path, 'w') as _f:
|
|
|
_f.write('precision:' + str(indicators['precision']) + '\n')
|
|
|
_f.write('recall:' + str(indicators['recall']) + '\n')
|
|
|
_f.write('F1:' + str(indicators["F1"]) + '\n')
|
|
|
_f.write('interpretability:' + str(indicators['interpretability']) + '\n')
|
|
|
_f.write('performance:' + str(indicators['performance']) + '\n')
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
if os.path.isfile(hpo_output_dir + "/incumbent.json"):
|
|
|
with open(hpo_output_dir + "/configspace.json", 'r') as f:
|
|
|
dict_configspace = json.load(f)
|
|
|
str_configspace = json.dumps(dict_configspace)
|
|
|
configspace = csj.read(str_configspace)
|
|
|
with open(hpo_output_dir + "/incumbent.json", 'r') as f:
|
|
|
dic = json.load(f)
|
|
|
configuration = ConfigSpace.Configuration(configspace, values=dic)
|
|
|
ml_er(configuration)
|
