import operator import numpy as np import pandas as pd import copy import torch from ConfigSpace import Configuration from tqdm import tqdm from settings import model, similarity_threshold, support_threshold, confidence_threshold def is_minimal(md, md_list, target_col): # 假设这个md是minimal if len(md_list) == 0: return True minimal = True for _ in md_list: if isinstance(_, tuple): _ = _[0] if _ != md: other_cols = list(set(_.keys()) - {target_col}) # 假设列表中每一个md都使当前md不minimal exist = True # 如果左边任何一个大于，则假设不成立 for col in other_cols: if _[col] > md[col]: exist = False break # 如果右边小于，假设也不成立 if _[target_col] < md[target_col]: exist = False # 任何一次假设成立，当前md不minimal if exist: minimal = False break return minimal def pairs_inference(path, target_col): data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1') data.fillna("", inplace=True) data = data.astype(str) columns = data.columns.values.tolist() target_index = columns.index(target_col) cols_but_target = list(set(columns) - {target_col}) length = data.shape[0] width = data.shape[1] sentences = [] for col in range(0, width): for row in range(0, length): cell_value = data.values[row, col] sentences.append(cell_value) embedding = model.encode(sentences, convert_to_tensor=True, device="cuda") split_embedding = torch.split(embedding, length, dim=0) table_tensor = torch.stack(split_embedding, dim=0, out=None) norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2) sim_tensor = torch.matmul(norm_table_tensor, norm_table_tensor.transpose(1, 2)) # sim_tensor = torch.round(sim_tensor, decimals=3) # torch.save(sim_tensor, md_output_dir + "tensor.pt") md_list = [] minimal_vio = [] init_md = {} for col in columns: init_md[col] = 1 if col == target_col else -1 md_list.append(init_md) for row1 in tqdm(range(0, length - 1)): terminate = False for row2 in range(row1 + 1, length): violated_mds = [] # sims是两行的相似度 sims = {} for col_index in range(0, width): col = columns[col_index] similarity = sim_tensor[col_index, row1, row2].item() sims[col] = similarity # 寻找violated md,从md列表中删除并加入vio列表 for md in md_list[:]: lhs_satis = True rhs_satis = True for col in cols_but_target: if sims[col] < md[col]: lhs_satis = False break if sims[target_col] < md[target_col]: rhs_satis = False if lhs_satis == True and rhs_satis == False: md_list.remove(md) violated_mds.append(md) # for vio_md in violated_mds: # # 特殊化左侧 # for col in cols_but_target: # if sims[col] + 0.01 <= 1: # spec_l_md = copy.deepcopy(vio_md) # spec_l_md[col] = simt if sims[col] < simt else sims[col] + 0.01 # if is_minimal(spec_l_md, md_list, target_col): # md_list.append(spec_l_md) # if vio_md not in minimal_vio: # minimal_vio.append(vio_md) for vio_md in violated_mds: vio_md_support, vio_md_confidence = get_metrics(vio_md, data, sim_tensor, target_col, target_index) if vio_md_support >= support_threshold: for col in cols_but_target: if sims[col] < 1.0: spec_l_md = copy.deepcopy(vio_md) if sims[col] < similarity_threshold: spec_l_md[col] = similarity_threshold else: if sims[col] + 0.01 <= 1.0: spec_l_md[col] = sims[col] + 0.01 else: spec_l_md[col] = 1.0 if is_minimal(spec_l_md, md_list, target_col): md_list.append(spec_l_md) if vio_md not in minimal_vio: minimal_vio.append(vio_md) if len(md_list) == 0: terminate = True break if terminate: break if len(minimal_vio) > 0: for md in minimal_vio[:]: support, confidence = get_metrics(md, data, sim_tensor, target_col, target_index) if support >= support_threshold and confidence >= confidence_threshold: minimal_vio.append((md, support, confidence)) minimal_vio.remove(md) if len(md_list) > 0: # 去除重复MD tmp = [] for _ in md_list: if _ not in tmp: tmp.append(_) md_list = tmp # 去除support小于阈值MD for _ in md_list[:]: support, confidence = get_metrics(_, data, sim_tensor, target_col, target_index) if support >= support_threshold and confidence >= confidence_threshold: md_list.append((_, support, confidence)) md_list.remove(_) # 去除不minimal的MD for md_tuple in md_list[:]: if not is_minimal(md_tuple[0], md_list, target_col) and md_tuple[2] < 0.5: md_list.remove(md_tuple) if len(minimal_vio) > 0: for vio_tuple in minimal_vio[:]: if not is_minimal(vio_tuple[0], md_list, target_col) and vio_tuple[2] < 0.5: minimal_vio.remove(vio_tuple) if len(minimal_vio) > 0: for vio_tuple in minimal_vio[:]: if not is_minimal(vio_tuple[0], minimal_vio, target_col) and vio_tuple[2] < 0.5: minimal_vio.remove(vio_tuple) result = [] result.extend(md_list) result.extend(minimal_vio) result.sort(key=operator.itemgetter(2), reverse=True) print(f'\033[33mList Length: {len(result)}\033[0m') return result def get_metrics(current_md, data, sim_tensor, target_col, target_index): columns = data.columns.values.tolist() length = data.shape[0] width = data.shape[1] md_tensor = list(current_md.values()) md_tensor = torch.tensor(md_tensor, device='cuda') md_tensor_2d = md_tensor.unsqueeze(1) md_tensor_3d = md_tensor_2d.unsqueeze(2) md_tensor_3d = md_tensor_3d.repeat(1, length, length) sim_tensor = torch.round(sim_tensor, decimals=4) sup_tensor = torch.ge(sim_tensor, md_tensor_3d) ini_slice = torch.ones((length, length), dtype=torch.bool, device='cuda') for i in range(0, width): if i != target_index: sup_tensor_slice = sup_tensor[i] ini_slice = torch.logical_and(ini_slice, sup_tensor_slice) sup_tensor_int = ini_slice.int() support_Naumann = torch.count_nonzero(sup_tensor_int).item() support_Naumann = (support_Naumann - length) / 2 ini_slice = torch.logical_and(ini_slice, sup_tensor[target_index]) conf_tensor_int = ini_slice.int() support_Fan = torch.count_nonzero(conf_tensor_int).item() support_Fan = (support_Fan - length) / 2 confidence = support_Fan / support_Naumann if support_Naumann > 0 else 0 return support_Fan, confidence