p04316279
/
matching_dependency
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

随机生成MD并过滤

Browse Source
MD-metrics-HPO
HuangJintao 8 months ago
parent a6a58e178f
commit b21b0aa496
3 changed files with 115 additions and 60 deletions
Show Stats Download Patch File Download Diff File

145
md_discovery/md_mining.py
Unescape View File

@ -1,3 +1,5 @@
import random
import operator
import pandas as pd import pandas as pd
import torch import torch
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@ -17,6 +19,7 @@ def mining(train: pd.DataFrame):
data = train.drop(columns=['_id', 'ltable_' + ltable_id, 'rtable_' + rtable_id], inplace=False) data = train.drop(columns=['_id', 'ltable_' + ltable_id, 'rtable_' + rtable_id], inplace=False)
# data中现存属性：除key以外左右表属性和gold, 不含_id # data中现存属性：除key以外左右表属性和gold, 不含_id
columns = data.columns.values.tolist() columns = data.columns.values.tolist()
columns_without_prefix = [_.replace('ltable_', '') for _ in columns if _.startswith('ltable_')]
# 列表, 每个元素为二元组, 包含对应列的索引 # 列表, 每个元素为二元组, 包含对应列的索引
col_tuple_list = build_col_tuple_list(columns) col_tuple_list = build_col_tuple_list(columns)
@ -61,9 +64,6 @@ def mining(train: pd.DataFrame):
# 根据索引将匹配的行标签置为1 # 根据索引将匹配的行标签置为1
sim_table_tensor_labeled[match_pair_indices, -1] = 1.00 sim_table_tensor_labeled[match_pair_indices, -1] = 1.00
md_list = init_md_list(len(col_tuple_list))
result_md_list = []
sorted_unique_value_tensor_list = [] sorted_unique_value_tensor_list = []
for _ in range(len(col_tuple_list)): for _ in range(len(col_tuple_list)):
# 将sim_table_tensor每一列的值从小到大排列，加入列表 # 将sim_table_tensor每一列的值从小到大排列，加入列表
@ -72,46 +72,58 @@ def mining(train: pd.DataFrame):
sorted_unique_value_tensor = sorted_unique_value_tensor[sorted_unique_value_tensor >= 0] sorted_unique_value_tensor = sorted_unique_value_tensor[sorted_unique_value_tensor >= 0]
sorted_unique_value_tensor_list.append(sorted_unique_value_tensor) sorted_unique_value_tensor_list.append(sorted_unique_value_tensor)
# 随机生成候选MD, 形成一个二维张量, 每一行代表一个候选MD
candidate_mds_tensor = build_candidate_md_matrix(sorted_unique_value_tensor_list)
result_list = [] result_list = []
# 遍历每一个MD
for _ in tqdm(range(candidate_mds_tensor.shape[0])):
# 对每一个MD加一个0.5的标记, 意为match
md_tensor_labeled = torch.cat((candidate_mds_tensor[_], torch.tensor([0.5], device='cuda')), 0)
abs_support, confidence = get_metrics(md_tensor_labeled, sim_table_tensor_labeled)
if abs_support >= support_threshold and confidence >= confidence_threshold:
md_list_format = [round(i, 2) for i in candidate_mds_tensor[_].tolist()]
md_dict_format = {}
for k in range(0, len(columns_without_prefix)):
md_dict_format[columns_without_prefix[k]] = md_list_format[k]
result_list.append((md_dict_format, abs_support, confidence))
result_list.sort(key=operator.itemgetter(2), reverse=True)
mds_to_txt(result_list)
return result_list
# 遍历MD列表, 将满足的直接加入结果列表, 不满足的看能否收紧, 不能收紧直接跳过 # 遍历MD列表, 将满足的直接加入结果列表, 不满足的看能否收紧, 不能收紧直接跳过
# 若能收紧则将收紧后的一个个加入暂存列表, 并在该轮遍历结束后替换MD列表, 直到MD列表为空 # 若能收紧则将收紧后的一个个加入暂存列表, 并在该轮遍历结束后替换MD列表, 直到MD列表为空
while len(md_list) > 0: # while len(md_list) > 0:
tmp_list = [] # tmp_list = []
for md_tensor in tqdm(md_list): # for md_tensor in tqdm(md_list):
md_tensor_labeled = torch.cat((md_tensor, torch.tensor([0.5], device='cuda')), 0) # md_tensor_labeled = torch.cat((md_tensor, torch.tensor([0.5], device='cuda')), 0)
abs_support, confidence = get_metrics(md_tensor_labeled, sim_table_tensor_labeled) # abs_support, confidence = get_metrics(md_tensor_labeled, sim_table_tensor_labeled)
# 如果support小于1, 没必要收紧阈值, 跳过 # # 如果support小于1, 没必要收紧阈值, 跳过
if abs_support >= 1: # if abs_support >= 1:
# 如果support满足但confidence不满足, 需要收紧阈值 # # 如果support满足但confidence不满足, 需要收紧阈值
if confidence < confidence_threshold: # if confidence < confidence_threshold:
for _ in range(len(md_tensor)): # for _ in range(len(md_tensor)):
new_md_tensor = md_tensor.clone() # new_md_tensor = md_tensor.clone()
if new_md_tensor[_] == -1.00: # if new_md_tensor[_] == -1.00:
new_md_tensor[_] = sorted_unique_value_tensor_list[_][0] # new_md_tensor[_] = sorted_unique_value_tensor_list[_][0]
if len(tmp_list) == 0: # if len(tmp_list) == 0:
tmp_list.append(new_md_tensor) # tmp_list.append(new_md_tensor)
else: # else:
stacked_tmp_tensors = torch.stack(tmp_list) # stacked_tmp_tensors = torch.stack(tmp_list)
is_contained = (stacked_tmp_tensors == new_md_tensor) .all(dim=1).any() # is_contained = (stacked_tmp_tensors == new_md_tensor) .all(dim=1).any()
if not is_contained: # if not is_contained:
tmp_list.append(new_md_tensor) # tmp_list.append(new_md_tensor)
else: # else:
a_tensor = sorted_unique_value_tensor_list[_] # a_tensor = sorted_unique_value_tensor_list[_]
b_value = new_md_tensor[_] # b_value = new_md_tensor[_]
next_index = torch.where(a_tensor == b_value)[0].item() + 1 # next_index = torch.where(a_tensor == b_value)[0].item() + 1
if next_index < len(a_tensor): # if next_index < len(a_tensor):
new_md_tensor[_] = a_tensor[next_index] # new_md_tensor[_] = a_tensor[next_index]
tmp_list.append(new_md_tensor) # tmp_list.append(new_md_tensor)
# torch.where(sorted_unique_value_tensor_list[2] == 0.16)[0].item() # # torch.where(sorted_unique_value_tensor_list[2] == 0.16)[0].item()
# 如果都满足, 直接加进结果列表 # # 如果都满足, 直接加进结果列表
else: # else:
result_list.append(md_tensor) # result_list.append(md_tensor)
md_list = tmp_list # md_list = tmp_list
print(1)
# sim_tensor = torch.matmul(norm_table_tensor, norm_table_tensor.transpose(1, 2))
# sim_tensor = sim_tensor.float()
# sim_tensor = torch.round(sim_tensor, decimals=4)
def build_col_tuple_list(columns_): def build_col_tuple_list(columns_):
@ -124,13 +136,13 @@ def build_col_tuple_list(columns_):
return col_tuple_list_ return col_tuple_list_
def init_md_list(md_dimension: int): # def init_md_list(md_dimension: int):
md_list_ = [] # md_list_ = []
# 创建全为-1的初始MD, 保留两位小数 # # 创建全为-1的初始MD, 保留两位小数
init_md_tensor = torch.full((md_dimension, ), -1.0, device='cuda') # init_md_tensor = torch.full((md_dimension, ), -1.0, device='cuda')
init_md_tensor = torch.round(init_md_tensor, decimals=2) # init_md_tensor = torch.round(init_md_tensor, decimals=2)
md_list_.append(init_md_tensor) # md_list_.append(init_md_tensor)
return md_list_ # return md_list_
def get_metrics(md_tensor_labeled_, sim_table_tensor_labeled_): def get_metrics(md_tensor_labeled_, sim_table_tensor_labeled_):
@ -155,3 +167,42 @@ def get_metrics(md_tensor_labeled_, sim_table_tensor_labeled_):
confidence_ = abs_strict_support_ / abs_support_ if abs_support_ > 0 else 0 confidence_ = abs_strict_support_ / abs_support_ if abs_support_ > 0 else 0
return abs_support_, confidence_ return abs_support_, confidence_
# 随机生成MD, 拼成一个矩阵, 每一行代表一条MD
def build_candidate_md_matrix(sorted_unique_value_tensor_list_: list):
# 假设先随机抽取20000条
length_ = len(sorted_unique_value_tensor_list_)
N = 20000
# 对于第一列所有相似度取值, 随机有放回地抽取N个, 生成行索引
indices = torch.randint(0, len(sorted_unique_value_tensor_list_[0]), (N, 1))
# 为每一列生成一个索引张量, 表示从相应列张量中随机选择的值的索引
for _ in range(1, length_):
indices = torch.cat((indices, torch.randint(0, len(sorted_unique_value_tensor_list_[_]), (N, 1))), dim=1)
# 使用生成的索引从每个列相似度张量中选取值, 构成新的张量
candidate_md_matrix_list = []
for _ in range(length_):
candidate_md_matrix_list.append(sorted_unique_value_tensor_list_[_][indices[:, _].long()].unsqueeze(1))
candidate_md_matrix_ = torch.cat(candidate_md_matrix_list, dim=1)
# 此tensor将与其他置为-1的tensor拼接
joint_candidate_md_matrix_ = candidate_md_matrix_.clone()
# 随机将1列, 2列......, M-1列置为-1
for i in range(length_ - 1):
index_list_format = []
for j in range(candidate_md_matrix_.shape[0]):
# 对每条MD，随机选择将要置为-1的列索引
index_list_format.append(random.sample([_ for _ in range(0, length_)], i + 1))
index = torch.tensor(index_list_format, device='cuda')
# 随机调整为-1后的MD集合
modified_candidate = candidate_md_matrix_.scatter(1, index, -1)
joint_candidate_md_matrix_ = torch.cat((joint_candidate_md_matrix_, modified_candidate), 0)
joint_candidate_md_matrix_ = joint_candidate_md_matrix_.unique(dim=0)
return joint_candidate_md_matrix_
def mds_to_txt(result_list_):
p = md_output_dir + "mds.txt"
with open(p, 'w') as f:
for _ in result_list_:
f.write(f'MD: {str(_[0])}\tAbsolute Support: {str(_[1])}\tConfidence: {str(_[2])}')
f.write('\n')

14
ml_er/magellan_new.py
Unescape View File

@ -26,16 +26,18 @@ def blocking_mining():
blocker = em.OverlapBlocker() blocker = em.OverlapBlocker()
candidate = blocker.block_tables(ltable, rtable, ltable_block_attr, rtable_block_attr, allow_missing=True, candidate = blocker.block_tables(ltable, rtable, ltable_block_attr, rtable_block_attr, allow_missing=True,
l_output_attrs=attributes, r_output_attrs=attributes, n_jobs=1, l_output_attrs=attributes, r_output_attrs=attributes, n_jobs=-1,
overlap_size=1, show_progress=False) overlap_size=1, show_progress=False)
candidate['gold'] = 0 candidate['gold'] = 0
candidate = candidate.reset_index(drop=True) candidate = candidate.reset_index(drop=True)
block_time = time.time()
print(f'Block Time: {block_time - start}')
# 根据mapping表标注数据 # 根据mapping表标注数据
candidate_match_rows = [] candidate_match_rows = []
for t in tqdm(mappings.itertuples()): for t in tqdm(mappings.itertuples()):
mask = ((candidate['ltable_' + ltable_id].isin([getattr(t, 'ltable_id')])) & mask = ((candidate['ltable_' + ltable_id].isin([getattr(t, mapping_lid)])) &
(candidate['rtable_' + rtable_id].isin([getattr(t, 'rtable_id')]))) (candidate['rtable_' + rtable_id].isin([getattr(t, mapping_rid)])))
matching_indices = candidate[mask].index matching_indices = candidate[mask].index
candidate_match_rows.extend(matching_indices.tolist()) candidate_match_rows.extend(matching_indices.tolist())
match_rows_mask = candidate.index.isin(candidate_match_rows) match_rows_mask = candidate.index.isin(candidate_match_rows)
@ -60,10 +62,12 @@ def blocking_mining():
sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0) sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0)
train_set = sets['train'] train_set = sets['train']
test_set = sets['test'] test_set = sets['test']
end_blocking = time.time() label_and_split_time = time.time()
print(end_blocking - start) print(f'Label and Split Time: {label_and_split_time - block_time}')
mining(train_set) mining(train_set)
mining_time = time.time()
print(f'Mining Time: {mining_time - label_and_split_time}')
return 1 return 1

16
settings.py
Unescape View File

@ -1,12 +1,12 @@
from sentence_transformers import SentenceTransformer from sentence_transformers import SentenceTransformer
ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableA.csv' ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableA.csv'
rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\tableB.csv' rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableB.csv'
mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\matches.csv' mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\matches.csv'
mapping_lid = 'ltable_id' # mapping表中左表id名 mapping_lid = 'idDBLP' # mapping表中左表id名
mapping_rid = 'rtable_id' # mapping表中右表id名 mapping_rid = 'idScholar' # mapping表中右表id名
ltable_block_attr = 'name' ltable_block_attr = 'title'
rtable_block_attr = 'name' rtable_block_attr = 'title'
ltable_id = 'id' # 左表id字段名称 ltable_id = 'id' # 左表id字段名称
rtable_id = 'id' # 右表id字段名称 rtable_id = 'id' # 右表id字段名称
target_attr = 'id' # 进行md挖掘时的目标字段 target_attr = 'id' # 进行md挖掘时的目标字段
@ -16,7 +16,7 @@ model = SentenceTransformer('E:\\Data\\Research\\Models\\all-MiniLM-L6-v2')
interpre_weight = 1 # 可解释性权重 interpre_weight = 1 # 可解释性权重
similarity_threshold = 0.1 similarity_threshold = 0.1
support_threshold = 1 support_threshold = 1
confidence_threshold = 0.6 confidence_threshold = 0.75
er_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\ml_er\\output\\' er_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\ml_er\\output\\'
md_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\md_discovery\\output\\' md_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\md_discovery\\output\\'

Write Preview
Loading…
Cancel
Save