新增数据集

datasets/Walmart-Amazon/matches.csv

@@ -0,0 +1,963 @@
+ltable_id,rtable_id
+3,4378
+5,20932
+6,9015
+7,1303
+15,4282
+15,17364
+15,17810
+19,17429
+20,16837
+25,6984
+29,6031
+30,11088
+33,11683
+34,3067
+39,10872
+41,6816
+42,172
+49,8096
+53,17892
+54,6572
+56,9164
+56,21964
+57,13302
+64,8391
+65,7156
+65,11129
+66,3668
+69,14641
+78,12347
+80,6526
+81,17194
+86,11617
+86,7610
+88,18980
+88,3269
+90,21917
+91,17687
+93,8350
+94,9051
+95,10966
+97,15904
+99,4520
+101,2593
+102,11759
+107,12090
+108,21709
+108,12441
+114,9223
+120,14795
+128,2701
+128,10640
+130,9890
+131,5595
+134,3740
+136,2576
+137,21120
+138,11468
+144,7970
+145,15891
+147,21192
+148,16398
+148,16350
+148,18936
+151,691
+155,3446
+157,16205
+159,9709
+159,5664
+164,18937
+172,1152
+173,1249
+178,19149
+179,8545
+181,12012
+182,7399
+184,2959
+186,1928
+191,1243
+192,11131
+194,6876
+194,20939
+199,5511
+201,6828
+205,10517
+211,400
+215,6601
+217,5274
+218,5046
+219,8696
+221,16624
+222,1354
+222,17534
+225,12627
+229,13258
+231,10436
+235,21783
+239,10617
+246,13589
+246,19031
+258,10646
+261,5192
+264,16640
+266,2949
+272,3965
+276,15426
+281,7824
+284,2675
+286,18626
+294,19911
+297,15653
+302,16807
+302,16641
+303,19299
+309,19291
+310,4720
+312,13299
+313,8870
+315,7705
+317,1082
+319,21483
+322,9990
+323,1244
+324,8519
+330,1599
+334,10661
+336,20174
+337,6115
+345,8386
+346,8304
+351,10048
+356,21591
+360,19150
+360,11566
+362,16307
+370,15071
+376,9389
+378,21690
+382,12582
+387,3169
+388,1450
+395,99
+396,7149
+396,7144
+396,17848
+398,16514
+399,713
+413,284
+415,17243
+415,20385
+424,8534
+426,14868
+428,6990
+429,3273
+431,18464
+446,5774
+447,13174
+448,13173
+452,16645
+453,10899
+453,4892
+454,1891
+455,19347
+461,18389
+463,11440
+465,7304
+465,15278
+468,20645
+469,2720
+469,19257
+470,10874
+472,15367
+475,12746
+475,20392
+476,2717
+480,376
+481,14853
+483,9130
+491,19957
+496,4496
+498,5577
+500,12544
+502,11435
+503,10206
+507,11953
+512,9060
+514,14881
+518,2138
+522,14014
+523,3626
+523,7630
+527,485
+528,21347
+528,76
+529,17879
+530,18531
+530,6615
+531,19817
+537,790
+538,17174
+539,11542
+540,9383
+541,9189
+542,13159
+544,13664
+545,7624
+545,7622
+547,9295
+548,3414
+550,19539
+550,2555
+555,19009
+558,5883
+559,20713
+559,21310
+561,18152
+563,18915
+575,1128
+575,5075
+577,21736
+577,19367
+579,15015
+584,965
+585,14867
+588,20108
+589,16953
+589,15495
+591,5594
+594,20843
+595,14869
+597,22043
+598,3626
+599,556
+600,4691
+600,12881
+604,15829
+611,8429
+614,22045
+615,17519
+617,19540
+625,18443
+630,12817
+632,13149
+635,7145
+636,9151
+639,7371
+639,10128
+645,6760
+648,4759
+652,7445
+652,6708
+656,10558
+658,21942
+660,10273
+662,17258
+663,7135
+669,12490
+678,8962
+680,843
+681,20251
+684,17573
+687,20212
+688,4783
+692,18704
+693,2611
+693,104
+694,20006
+695,12569
+699,4283
+699,3856
+705,16536
+705,12895
+707,11829
+707,8042
+709,1096
+714,16478
+715,7850
+717,20106
+719,6972
+720,13398
+722,12995
+723,9675
+725,16800
+725,16277
+734,5991
+735,12273
+737,6759
+738,18197
+739,11687
+741,11537
+742,8235
+745,11810
+752,7075
+759,8381
+765,15415
+769,6362
+770,11223
+771,5517
+781,17483
+784,21196
+784,12733
+785,10845
+788,14901
+797,6508
+801,13029
+801,8931
+803,4121
+805,14294
+810,1480
+810,11428
+811,453
+816,11998
+818,979
+828,10887
+829,8897
+832,19556
+832,4597
+836,8739
+838,715
+841,7503
+848,3153
+848,602
+853,6747
+857,7347
+861,11942
+862,5782
+864,11894
+870,14930
+871,20288
+872,15268
+874,5644
+875,4226
+887,12783
+892,21096
+893,11620
+895,12463
+898,7383
+898,11343
+898,15335
+902,6661
+903,16989
+905,19636
+906,3469
+907,13994
+908,13023
+916,8910
+917,17317
+919,17371
+920,3246
+925,21757
+926,4426
+931,6451
+932,2813
+934,13148
+935,14800
+936,14004
+938,15252
+939,3615
+941,2805
+944,14177
+949,7346
+950,15942
+955,4702
+961,14317
+973,21568
+974,19753
+975,13422
+979,9553
+983,18364
+986,6669
+991,12312
+992,19592
+996,2750
+997,4379
+999,4923
+1007,18142
+1007,11116
+1009,780
+1010,15003
+1011,13413
+1014,7028
+1022,12116
+1024,12674
+1029,16791
+1030,10555
+1031,13438
+1037,12168
+1038,5010
+1044,3009
+1066,8297
+1070,18620
+1071,16126
+1073,2522
+1074,20740
+1077,11884
+1080,14454
+1081,21587
+1083,20644
+1090,16634
+1092,13747
+1093,5462
+1093,16869
+1102,7853
+1104,1397
+1107,12700
+1108,10672
+1109,8091
+1119,3471
+1121,14109
+1125,8267
+1134,8789
+1136,17383
+1137,4429
+1138,9085
+1140,9097
+1141,9253
+1145,4787
+1145,2867
+1148,16756
+1149,9481
+1151,18330
+1153,9237
+1153,13245
+1155,14873
+1159,13109
+1165,3695
+1177,20811
+1178,18153
+1183,11288
+1186,16092
+1187,5479
+1189,2759
+1190,10706
+1193,4493
+1193,5592
+1194,14916
+1195,5364
+1197,4814
+1200,159
+1207,6898
+1208,16935
+1208,3254
+1208,18170
+1209,9717
+1209,14109
+1210,1324
+1217,5249
+1218,5267
+1224,8070
+1229,5842
+1230,4667
+1233,18428
+1233,12407
+1236,7615
+1237,6974
+1242,2465
+1244,10288
+1246,21594
+1246,17191
+1248,16298
+1252,9635
+1256,20411
+1262,12734
+1263,22051
+1266,4232
+1268,4380
+1280,17949
+1280,7637
+1282,1240
+1286,16535
+1287,15033
+1291,17829
+1292,18602
+1295,6405
+1298,8455
+1299,14223
+1299,15462
+1301,6494
+1305,2558
+1311,14760
+1315,15122
+1315,7783
+1315,21112
+1316,4506
+1317,20506
+1320,884
+1324,912
+1325,10507
+1334,8618
+1338,17543
+1340,8496
+1345,20145
+1345,14257
+1349,4659
+1349,12191
+1353,6819
+1355,559
+1356,11269
+1359,1903
+1360,994
+1362,17388
+1365,18372
+1367,12146
+1370,5172
+1377,12161
+1380,20890
+1385,17536
+1391,2976
+1394,19580
+1396,2438
+1397,20805
+1398,4586
+1400,14943
+1400,18727
+1402,20183
+1405,14894
+1412,16818
+1413,2788
+1417,7248
+1418,14715
+1423,9727
+1424,8192
+1426,10692
+1437,11704
+1439,19654
+1441,15284
+1451,4719
+1454,10760
+1456,18231
+1457,8346
+1460,14498
+1466,15575
+1466,13592
+1466,11641
+1467,16059
+1473,3250
+1473,3248
+1474,3961
+1475,8650
+1475,12654
+1476,11083
+1477,2432
+1478,1766
+1481,4162
+1482,8566
+1484,15558
+1485,1375
+1489,2077
+1498,5234
+1499,4637
+1504,4817
+1506,16666
+1508,19514
+1509,9332
+1511,4519
+1512,20844
+1513,1592
+1522,1025
+1522,21029
+1525,12543
+1528,16916
+1528,811
+1529,1441
+1534,1090
+1535,18999
+1539,4756
+1542,10127
+1545,21374
+1549,18948
+1550,11326
+1552,9156
+1558,12852
+1559,14553
+1560,19669
+1562,12169
+1563,430
+1564,6914
+1571,6998
+1573,5415
+1577,13540
+1578,3323
+1579,11460
+1584,11475
+1589,2645
+1591,3205
+1592,16658
+1595,13390
+1602,10608
+1602,19807
+1606,18687
+1607,13172
+1609,2509
+1610,10450
+1613,17218
+1614,11350
+1618,6364
+1621,16058
+1624,5013
+1626,13619
+1628,6288
+1632,20070
+1637,1270
+1640,9564
+1641,15009
+1641,11044
+1643,2779
+1651,13804
+1652,14384
+1653,1649
+1656,14169
+1657,16332
+1660,11172
+1666,6194
+1671,20794
+1684,17948
+1686,20038
+1686,17665
+1694,11026
+1694,20397
+1694,3793
+1695,17274
+1697,16578
+1697,21667
+1699,17103
+1701,18549
+1703,17701
+1704,16231
+1705,14645
+1706,17026
+1708,16226
+1713,21093
+1714,18512
+1716,17396
+1717,17261
+1718,8845
+1720,20522
+1726,18463
+1727,14918
+1728,12125
+1732,5720
+1733,21812
+1738,20132
+1742,15527
+1751,20117
+1753,16220
+1758,13845
+1758,9789
+1761,12647
+1761,4576
+1766,5888
+1771,235
+1774,18402
+1776,16437
+1776,15489
+1776,13179
+1777,3750
+1784,11403
+1789,11778
+1790,19800
+1791,19538
+1792,8494
+1802,14924
+1803,6872
+1810,577
+1813,3253
+1813,1607
+1827,2578
+1829,19353
+1832,432
+1832,13396
+1833,9342
+1834,16331
+1837,13677
+1841,18623
+1841,2326
+1843,4900
+1846,15547
+1850,17488
+1851,13348
+1860,5560
+1861,18357
+1862,15434
+1863,21325
+1866,14359
+1868,4494
+1875,9386
+1877,2927
+1882,10854
+1883,19123
+1884,17578
+1888,6729
+1891,15304
+1893,11120
+1895,7660
+1899,5458
+1901,10882
+1903,10674
+1904,15684
+1905,7800
+1907,10481
+1908,20111
+1919,543
+1922,20240
+1927,19026
+1930,15375
+1931,11214
+1932,9095
+1932,9016
+1934,9578
+1936,18140
+1936,11295
+1937,8500
+1943,15780
+1944,828
+1946,1315
+1949,17302
+1951,20443
+1952,1769
+1953,11438
+1957,17570
+1960,2284
+1963,6961
+1964,14803
+1966,656
+1968,21511
+1972,3590
+1972,3593
+1973,3155
+1975,8180
+1976,12093
+1980,4278
+1983,6913
+1984,1120
+1986,6413
+1987,10905
+1987,16252
+1988,2476
+1988,9474
+1992,1274
+1997,12451
+1999,20189
+2000,14202
+2004,19520
+2004,1747
+2006,13921
+2007,20461
+2013,9226
+2014,11740
+2014,9456
+2017,16793
+2018,11227
+2018,5207
+2019,16543
+2021,17451
+2021,17374
+2024,18710
+2028,12891
+2029,14035
+2030,13167
+2033,18679
+2037,8519
+2037,21543
+2039,7443
+2040,9550
+2047,8524
+2050,21912
+2052,6286
+2054,7601
+2056,3067
+2061,352
+2063,6308
+2069,21448
+2071,8814
+2074,17803
+2076,18297
+2078,3894
+2080,16295
+2083,14539
+2087,11143
+2088,22025
+2088,17263
+2089,7048
+2092,13767
+2093,16311
+2094,15247
+2096,2974
+2100,20731
+2101,15843
+2103,2963
+2106,1253
+2107,8358
+2118,20621
+2119,20314
+2121,4432
+2123,1794
+2125,16072
+2132,7358
+2133,8990
+2133,10434
+2136,4705
+2136,19006
+2139,101
+2140,6965
+2141,11048
+2142,13934
+2147,2887
+2149,4784
+2150,21343
+2151,7338
+2154,11211
+2155,4509
+2155,17864
+2155,4318
+2156,14545
+2158,9026
+2159,17899
+2160,9858
+2163,8469
+2164,3694
+2165,21076
+2167,17733
+2179,18387
+2184,11689
+2186,13583
+2188,2671
+2190,2310
+2190,21661
+2191,16173
+2192,3144
+2193,9730
+2194,5101
+2196,19484
+2196,17366
+2199,20395
+2199,18138
+2201,17198
+2209,17162
+2210,16560
+2211,4646
+2213,21798
+2216,17494
+2216,20675
+2219,245
+2221,11340
+2224,14686
+2229,14755
+2230,12949
+2231,17839
+2236,1662
+2237,1696
+2239,326
+2244,594
+2245,3566
+2248,8368
+2252,16783
+2253,11265
+2254,345
+2255,14015
+2257,8356
+2259,14834
+2260,13722
+2264,19007
+2265,18698
+2266,1616
+2266,13318
+2267,2527
+2273,5255
+2274,356
+2275,19662
+2276,12760
+2293,6548
+2295,21434
+2296,12892
+2297,10869
+2305,10515
+2315,9548
+2317,15825
+2324,20693
+2326,19268
+2327,8544
+2328,16352
+2331,21976
+2332,18200
+2335,8279
+2339,2236
+2340,3758
+2341,7893
+2342,638
+2343,4784
+2344,4984
+2344,6970
+2344,14944
+2345,15061
+2349,18166
+2353,13476
+2356,5527
+2360,17905
+2360,7249
+2364,16366
+2364,17915
+2365,8618
+2368,15351
+2371,10923
+2374,3051
+2378,11830
+2379,10267
+2379,264
+2381,3482
+2387,20821
+2392,15893
+2394,13251
+2396,18101
+2397,10117
+2405,8668
+2407,12311
+2408,6808
+2409,11714
+2412,7233
+2418,7679
+2422,8663
+2435,20419
+2438,2861
+2440,6643
+2441,21373
+2461,13243
+2462,14117
+2462,8175
+2468,16832
+2468,10813
+2471,19283
+2474,14437
+2476,6955
+2477,4884
+2478,16862
+2479,10669
+2481,6885
+2482,19119
+2483,11841
+2488,5635
+2496,12024
+2497,19525
+2501,4362
+2502,16032
+2503,20051
+2507,5038
+2508,11582
+2510,11625
+2511,21503
+2513,20426
+2515,4424
+2516,20309
+2519,2668
+2522,3324
+2523,10365
+2527,13282
+2531,19976
+2536,12333
+2538,2279
+2539,10705
+2540,5175
+2542,6690
+2543,2533
+2548,19117

draw/draw_confidence_histogram.py

@@ -7,7 +7,7 @@ from pyecharts.globals import ThemeType
 
 if __name__ == '__main__':
     outcome_dir = r'E:\Data\Research\Outcome'
-    configs_dir = r'\Magellan+Smac+roberta-large-nli-stsb-mean-tokens'
+    configs_dir = r'\Magellan+Smac+roberta-large-nli-stsb-mean-tokens+inter-0.5'
     datasets_list = os.listdir(outcome_dir)
 
     for _ in datasets_list:

draw/draw_f1_inter_bars.py

@@ -8,7 +8,7 @@ from pyecharts.globals import ThemeType
 
 if __name__ == '__main__':
     outcome_dir = r'E:\Data\Research\Outcome'
-    configs_dir = r'\Magellan+Smac+roberta-large-nli-stsb-mean-tokens'
+    configs_dir = r'\Magellan+Smac+roberta-large-nli-stsb-mean-tokens+inter-0.5'
     datasets_list = os.listdir(outcome_dir)
     f1 = []
     interpretability = []

generate_matches.py

@@ -8,7 +8,7 @@ from sentence_transformers import SentenceTransformer
 from torch import nn
 
 if __name__ == '__main__':
-    directory = r'E:\Data\Research\Projects\matching_dependency\datasets\iTunes-Amazon'
+    directory = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-ACM_dirty'
     train = pd.read_csv(directory + r'\train.csv', encoding='ISO-8859-1')
     valid = pd.read_csv(directory + r'\valid.csv', encoding='ISO-8859-1')
     test = pd.read_csv(directory + r'\test.csv', encoding='ISO-8859-1')

md_discovery/discovery_executor_gpu.py

@@ -95,7 +95,7 @@ def discover(path, target_col):
     sim_tensor = torch.round(sim_tensor, decimals=4)
 
     # 小于6列的可以尝试做笛卡尔积，大于6列可能指数爆炸
-    if width <= 6:
+    if width < 6:
         # 列出除目标列以外所有列的所有取值，做笛卡尔积，结果为所有可能MD取值
         cartesian = build_cartesian(width, target_index)
         # 抽取sample_number / (width - 1)条MD，不含-1

settings.py

@@ -1,17 +1,17 @@
 from sentence_transformers import SentenceTransformer
 
-ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableA.csv'
-rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableB.csv'
-mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\matches.csv'
-mapping_lid = 'idDBLP'  # mapping表中左表id名
-mapping_rid = 'idScholar'  # mapping表中右表id名
+ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Abt-Buy\tableA.csv'
+rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Abt-Buy\tableB.csv'
+mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\Abt-Buy\matches.csv'
+mapping_lid = 'idAbt'  # mapping表中左表id名
+mapping_rid = 'idBuy'  # mapping表中右表id名
 ltable_id = 'id'  # 左表id字段名称
 rtable_id = 'id'  # 右表id字段名称
 target_attr = 'id'  # 进行md挖掘时的目标字段
 # lr_attrs_map = {}  # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致
 
 model = SentenceTransformer('E:\\Data\\Research\\Models\\roberta-large-nli-stsb-mean-tokens')
-interpre_weight = 0.5  # 可解释性权重
+interpre_weight = 0.9  # 可解释性权重
 similarity_threshold = 0.1
 support_threshold = 1
 confidence_threshold = 0.25

table_embedding.py

@@ -0,0 +1,7 @@
+from sentence_transformers import SentenceTransformer
+
+if __name__ == '__main__':
+    model = SentenceTransformer('E:\\Data\\Research\\Models\\tapex-base-finetuned-wikisql')
+    sentences = ['公积金转入深圳', '公积金转出深圳']
+    embedding = model.encode(sentences, convert_to_tensor=True, device="cuda")
+    print(1)