11.9

2.迭代轮数自动识别
3.超参数优化结果落盘

.gitignore

@@ -0,0 +1,4 @@
+/deprecated/
+/ml_er/output/*
+/md_discovery/output/*
+/hpo/output/*

README.md

@@ -1,2 +0,0 @@
-# matching_dependency
-

datasets/BeerAdvo-RateBeer/matches.csv

@@ -0,0 +1,69 @@
+ltable_id,rtable_id
+61,2710
+120,213
+190,805
+271,1410
+294,1586
+386,2204
+418,2441
+480,2576
+563,1177
+781,2835
+862,2906
+890,2580
+982,2999
+1004,20
+1030,43
+1042,55
+1157,155
+1394,358
+1456,409
+1460,412
+1474,427
+1522,467
+1621,548
+1648,571
+1672,590
+1748,652
+1780,685
+1842,737
+1850,742
+1914,793
+1931,810
+2007,203
+2163,985
+2175,996
+2242,1048
+2289,1085
+2362,1136
+2463,1206
+2491,1230
+2506,1242
+2558,1284
+2606,1320
+2689,1373
+2700,1380
+2722,1393
+2764,1423
+2867,1493
+2909,1529
+2992,1590
+3035,1620
+3073,1647
+3079,1650
+3210,1737
+3381,1859
+3468,1921
+3622,2020
+3758,2120
+3800,2146
+3802,2149
+3917,2224
+4003,2284
+4063,2331
+4088,2351
+4101,2360
+4157,2405
+4161,2409
+4237,2466
+4286,2362

datasets/Fodors-Zagats/matches.csv

@@ -0,0 +1,111 @@
+ltable_id,rtable_id
+0,218
+1,219
+2,220
+3,221
+4,222
+5,223
+6,224
+7,225
+8,226
+9,227
+10,228
+11,229
+12,230
+13,231
+14,232
+15,233
+16,234
+17,235
+18,236
+19,237
+21,239
+22,240
+23,241
+24,242
+25,243
+26,244
+27,245
+28,246
+29,247
+30,248
+31,249
+32,250
+33,251
+34,252
+35,253
+36,254
+37,255
+38,256
+39,257
+40,258
+41,259
+42,260
+43,261
+44,262
+45,263
+46,264
+47,265
+48,266
+49,267
+50,268
+51,269
+52,270
+53,271
+54,272
+55,273
+56,274
+57,275
+58,276
+59,277
+61,279
+62,280
+63,281
+64,282
+65,283
+66,284
+67,285
+68,286
+69,287
+70,288
+71,289
+72,290
+73,291
+74,292
+75,293
+76,294
+77,295
+78,296
+79,297
+80,298
+81,299
+82,300
+83,301
+84,302
+85,303
+86,304
+87,305
+88,306
+89,307
+90,308
+91,309
+92,310
+93,311
+94,312
+95,313
+96,314
+97,315
+98,316
+99,317
+100,318
+101,319
+102,320
+103,321
+104,322
+105,323
+106,324
+107,325
+108,326
+109,327
+110,328
+111,329

datasets/Fodors-Zagats/tableA.csv

@@ -0,0 +1,534 @@
+id,name,addr,city,phone,type,class
+0,` arnie morton \ 's of chicago ',' 435 s. la cienega blv . ',` los angeles ',310/246 -1501,american,0
+1,` art \ 's delicatessen ',' 12224 ventura blvd. ',` studio city ',818/762 -1221,american,1
+2,` hotel bel-air ',' 701 stone canyon rd. ',` bel air ',310/472 -1211,californian,2
+3,` cafe bizou ',' 14016 ventura blvd. ',` sherman oaks ',818/788 -3536,french,3
+4,campanile,' 624 s. la brea ave. ',` los angeles ',213/938 -1447,american,4
+5,` chinois on main ',' 2709 main st. ',` santa monica ',310/392 -9025,french,5
+6,citrus,' 6703 melrose ave. ',` los angeles ',213/857 -0034,californian,6
+7,fenix,' 8358 sunset blvd. west ',hollywood,213/848 -6677,american,7
+8,granita,' 23725 w. malibu rd. ',malibu,310/456 -0488,californian,8
+9,` grill on the alley ',' 9560 dayton way ',` los angeles ',310/276 -0615,american,9
+10,` restaurant katsu ',' 1972 n. hillhurst ave. ',` los angeles ',213/665 -1891,asian,10
+11,` l \ ` orangerie ',' 903 n. la cienega blvd. ',` los angeles ',310/652 -9770,french,11
+12,` le chardonnay ',' 8284 melrose ave. ',` los angeles ',213/655 -8880,french,12
+13,` locanda veneta ',' 3rd st. ',` los angeles ',310/274 -1893,italian,13
+14,matsuhisa,' 129 n. la cienega blvd. ',` beverly hills ',310/659 -9639,asian,14
+15,` the palm ',' 9001 santa monica blvd. ',` los angeles ',310/550 -8811,american,15
+16,patina,' 5955 melrose ave. ',` los angeles ',213/467 -1108,californian,16
+17,` philippe \ 's the original ',' 1001 n. alameda st. ',` los angeles ',213/628 -3781,american,17
+18,` pinot bistro ',' 12969 ventura blvd. ',` los angeles ',818/990 -0500,french,18
+19,` rex il ristorante ',' 617 s. olive st. ',` los angeles ',213/627 -2300,italian,19
+20,spago,' 1114 horn ave. ',` los angeles ',310/652 -4025,californian,20
+21,valentino,' 3115 pico blvd. ',` santa monica ',310/829 -4313,italian,21
+22,` yujean kang \ 's gourmet chinese cuisine ','67 n. raymond ave. ',` los angeles ',818/585 -0855,asian,22
+23,'21 club ','21 w. 52nd st. ',` new york ',212/582 -7200,american,23
+24,aquavit,'13 w. 54th st. ',` new york ',212/307 -7311,continental,24
+25,aureole,'34 e. 61st st. ',` new york ',' 212 / 319-1660 ',american,25
+26,` cafe lalo ',' 201 w. 83rd st. ',` new york ',212/496 -6031,` coffee bar ',26
+27,` cafe des artistes ',' 1 w. 67th st. ',` new york ',212/877 -3500,continental,27
+28,` carmine \ 's ',' 2450 broadway between 90th and 91st sts . ',` new york ',212/362 -2200,italian,28
+29,` carnegie deli ',' 854 7th ave. between 54th and 55th sts . ',` new york ',212/757 -2245,delicatessen,29
+30,chanterelle,' 2 harrison st. near hudson st. ',` new york ',212/966 -6960,american,30
+31,daniel,'20 e. 76th st. ',` new york ',212/288 -0033,french,31
+32,dawat,' 210 e. 58th st. ',` new york ',212/355 -7555,asian,32
+33,felidia,' 243 e. 58th st. ',` new york ',212/758 -1479,italian,33
+34,` four seasons grill room ','99 e. 52nd st. ',` new york ',212/754 -9494,american,34
+35,` gotham bar & grill ','12 e. 12th st. ',` new york ',212/620 -4020,american,35
+36,` gramercy tavern ','42 e. 20th st. between park ave. s and broadway ',` new york ',212/477 -0777,american,36
+37,` island spice ',' 402 w. 44th st. ',` new york ',212/765 -1737,` tel caribbean ',37
+38,` jo jo ',' 160 e. 64th st. ',` new york ',212/223 -5656,american,38
+39,` la caravelle ','33 w. 55th st. ',` new york ',212/586 -4252,french,39
+40,` la cote basque ','60 w. 55th st. between 5th and 6th ave. ',` new york ',212/688 -6525,french,40
+41,` le bernardin ',' 155 w. 51st st. ',` new york ',212/489 -1515,french,41
+42,` les celebrites ',' 160 central park s ',` new york ',212/484 -5113,french,42
+43,lespinasse,' 2 e. 55th st. ',` new york ',212/339 -6719,american,43
+44,lutece,' 249 e. 50th st. ',` new york ',212/752 -2225,french,44
+45,` manhattan ocean club ','57 w. 58th st. ',` new york ',' 212 / 371-7777 ',seafood,45
+46,march,' 405 e. 58th st. ',` new york ',212/754 -6272,american,46
+47,` mesa grill ',' 102 5th ave. between 15th and 16th sts . ',` new york ',212/807 -7400,american,47
+48,` mi cocina ','57 jane st. off hudson st. ',` new york ',212/627 -8273,mexican,48
+49,montrachet,' 239 w. broadway between walker and white sts . ',` new york ',' 212 / 219-2777 ',french,49
+50,oceana,'55 e. 54th st. ',` new york ',212/759 -5941,seafood,50
+51,` park avenue cafe ',' 100 e. 63rd st. ',` new york ',212/644 -1900,american,51
+52,petrossian,' 182 w. 58th st. ',` new york ',212/245 -2214,french,52
+53,picholine,'35 w. 64th st. ',` new york ',212/724 -8585,mediterranean,53
+54,pisces,'95 ave. a at 6th st. ',` new york ',212/260 -6660,seafood,54
+55,` rainbow room ','30 rockefeller plaza ',` new york ',212/632 -5000,` or 212/632 -5100 american ',55
+56,` river cafe ',' 1 water st. at the east river ',brooklyn,718/522 -5200,american,56
+57,` san domenico ',' 240 central park s ',` new york ',212/265 -5959,italian,57
+58,` second avenue deli ',' 156 2nd ave. at 10th st. ',` new york ',212/677 -0606,delicatessen,58
+59,seryna,'11 e. 53rd st. ',` new york ',212/980 -9393,asian,59
+60,` shun lee west ','43 w. 65th st. ',` new york ',212/371 -8844,asian,60
+61,` sign of the dove ',' 1110 3rd ave. at 65th st. ',` new york ',212/861 -8080,american,61
+62,` smith & wollensky ',' 201 e. 49th st. ',` new york ',212/753 -1530,american,62
+63,` tavern on the green ',` in central park at 67th st. ',` new york ',212/873 -3200,american,63
+64,` uncle nick \ 's ',' 747 9th ave. between 50th and 51st sts . ',` new york ',212/315 -1726,mediterranean,64
+65,` union square cafe ','21 e. 16th st. ',` new york ',212/243 -4020,american,65
+66,` virgil \ 's ',' 152 w. 44th st. ',` new york ',' 212 / 921-9494 ',american,66
+67,` chin \ 's ',' 3200 las vegas blvd. s ',` las vegas ',702/733 -8899,asian,67
+68,` coyote cafe ',' 3799 las vegas blvd. s ',` las vegas ',702/891 -7349,southwestern,68
+69,` le montrachet ',' 3000 w. paradise rd. ',` las vegas ',702/732 -5111,continental,69
+70,` palace court ',' 3570 las vegas blvd. s ',` las vegas ',702/731 -7547,continental,70
+71,` second street grille ',' 200 e. fremont st. ',` las vegas ',702/385 -3232,seafood,71
+72,` steak house ',' 2880 las vegas blvd. s ',` las vegas ',702/734 -0410,` steak houses ',72
+73,tillerman,' 2245 e. flamingo rd. ',` las vegas ',702/731 -4036,seafood,73
+74,abruzzi,' 2355 peachtree rd. . peachtree battle shopping center ',atlanta,404/261 -8186,italian,74
+75,bacchanalia,' 3125 piedmont rd. . near peachtree rd. ',atlanta,404/365 -0410,international,75
+76,` bone \ 's ',' 3130 piedmont road ',atlanta,404/237 -2663,american,76
+77,` brasserie le coze ',' 3393 peachtree rd. . lenox square mall near neiman marcus ',atlanta,404/266 -1440,french,77
+78,` buckhead diner ',' 3073 piedmont road ',atlanta,404/262 -3336,american,78
+79,ciboulette,' 1529 piedmont ave. ',atlanta,404/874 -7600,french,79
+80,delectables,' 1 margaret mitchell sq. ',atlanta,404/681 -2909,american,80
+81,` georgia grille ',' 2290 peachtree rd. . peachtree square shopping center ',atlanta,404/352 -3517,american,81
+82,` hedgerose heights inn ',' 490 e. paces ferry rd. ',atlanta,404/233 -7673,international,82
+83,` heera of india ',' 595 piedmont ave. rio shopping mall ',atlanta,404/876 -4408,asian,83
+84,` indigo coastal grill ',' 1397 n. highland ave. ',atlanta,404/876 -0676,caribbean,84
+85,` la grotta ',' 2637 peachtree rd. . peachtree house condominium ',atlanta,404/231 -1368,italian,85
+86,` mary mac \ 's tea room ',' 224 ponce de leon ave. ',atlanta,404/876 -1800,southern,86
+87,` nikolai \ 's roof ',' 255 courtland st. at harris st. ',atlanta,404/221 -6362,continental,87
+88,` pano \ 's and paul \ 's ',' 1232 w. paces ferry rd. ',atlanta,404/261 -3662,international,88
+89,` cafe ritz-carlton buckhead ',' 3434 peachtree rd. ',atlanta,404/237 -2700,` ext 6108 international ',89
+90,` dining room ritz-carlton buckhead ',' 3434 peachtree rd. ',atlanta,404/237 -2700,international,90
+91,` restaurant ritz-carlton atlanta ',' 181 peachtree st. ',atlanta,404/659 -0400,continental,91
+92,toulouse,' b peachtree rd. ',atlanta,404/351 -9533,french,92
+93,` veni vidi vici ','41 14th st. ',atlanta,404/875 -8424,italian,93
+94,` alain rondelli ',' 126 clement st. ',` san francisco ',415/387 -0408,french,94
+95,aqua,' 252 california st. ',` san francisco ',415/956 -9662,seafood,95
+96,boulevard,' 1 mission st. ',` san francisco ',415/543 -6084,american,96
+97,` cafe claude ',' 7 claude la . ',` san francisco ',415/392 -3505,french,97
+98,` campton place ',' 340 stockton st. ',` san francisco ',415/955 -5555,american,98
+99,` chez michel ',' 804 northpoint ',` san francisco ',415/775 -7036,french,99
+100,` fleur de lys ',' 777 sutter st. ',` san francisco ',415/673 -7779,french,100
+101,fringale,' 570 4th st. ',` san francisco ',415/543 -0573,french,101
+102,` hawthorne lane ','22 hawthorne st. ',` san francisco ',415/777 -9779,american,102
+103,` khan toke thai house ',' 5937 geary blvd. ',` san francisco ',415/668 -6654,asian,103
+104,` la folie ',' 2316 polk st. ',` san francisco ',415/776 -5577,french,104
+105,lulu,' 816 folsom st. ',` san francisco ',415/495 -5775,mediterranean,105
+106,` masa \ 's ',' 648 bush st. ',` san francisco ',415/989 -7154,french,106
+107,` mifune japan center kintetsu building ',' 1737 post st. ',` san francisco ',415/922 -0337,asian,107
+108,` plumpjack cafe ',' 3201 fillmore st. ',` san francisco ',415/563 -4755,mediterranean,108
+109,postrio,' 545 post st. ',` san francisco ',415/776 -7825,american,109
+110,` ritz-carlton restaurant and dining room ',' 600 stockton st. ',` san francisco ',415/296 -7465,american,110
+111,` rose pistola ',' 532 columbus ave. ',` san francisco ',415/399 -0499,italian,111
+112,bolo,'23 e. 22nd st. ',` new york ',212/228 -2200,mediterranean,191
+113,` il nido ',' 251 e. 53rd st. ',` new york ',212/753 -8450,italian,267
+114,remi,' 145 w. 53rd st. ',` new york ',212/581 -4242,italian,334
+115,` adriano \ 's ristorante ',' 2930 beverly glen circle ',` los angeles ',310/475 -9807,italian,112
+116,` barney greengrass ',' 9570 wilshire blvd. ',` beverly hills ',310/777 -5877,american,113
+117,beaurivage,' 26025 pacific coast hwy . ',malibu,310/456 -5733,french,114
+118,` bistro garden ',' 176 n. canon dr. ',` los angeles ',310/550 -3900,californian,115
+119,` border grill ',' 4th st. ',` los angeles ',310/451 -1655,mexican,116
+120,` broadway deli ',' 3rd st. promenade ',` santa monica ',310/451 -0616,american,117
+121,` ca \ ` brea ',' 346 s. la brea ave. ',` los angeles ',213/938 -2863,italian,118
+122,` ca \ ` del sol ',' 4100 cahuenga blvd. ',` los angeles ',818/985 -4669,italian,119
+123,` cafe pinot ',' 700 w. fifth st. ',` los angeles ',213/239 -6500,californian,120
+124,` california pizza kitchen ',' 207 s. beverly dr. ',` los angeles ',310/275 -1101,californian,121
+125,` canter \ 's ',' 419 n. fairfax ave. ',` los angeles ',213/651 -2030 .,american,122
+126,cava,' 3rd st. ',` los angeles ',213/658 -8898,mediterranean,123
+127,` cha cha cha ',' 656 n. virgil ave. ',` los angeles ',213/664 -7723,caribbean,124
+128,` chan dara ',' 310 n. larchmont blvd. ',` los angeles ',213/467 -1052,asian,125
+129,` clearwater cafe ',' 168 w. colorado blvd. ',` los angeles ',818/356 -0959,` health food ',126
+130,` dining room ',' 9500 wilshire blvd. ',` los angeles ',310/275 -5200,californian,127
+131,dive !,' 10250 santa monica blvd. ',` los angeles ',310/788 -,` dive american ',128
+132,drago,' 2628 wilshire blvd. ',` santa monica ',310/828 -1585,italian,129
+133,` drai \ 's ',' 730 n. la cienega blvd. ',` los angeles ',310/358 -8585,french,130
+134,` dynasty room ',' 930 hilgard ave. ',` los angeles ',310/208 -8765,continental,131
+135,eclipse,' 8800 melrose ave. ',` los angeles ',310/724 -5959,californian,132
+136,` ed debevic \ 's ',' 134 n. la cienega ',` los angeles ',310/659 -1952,american,133
+137,` el cholo ',' 1121 s. western ave. ',` los angeles ',213/734 -2773,mexican,134
+138,` gilliland \ 's ',' 2424 main st. ',` santa monica ',310/392 -3901,american,135
+139,` gladstone \ 's ',' 4 fish 17300 pacific coast hwy . at sunset blvd. ',` pacific palisades ',310/454 -3474,american,136
+140,` hard rock cafe ',' 8600 beverly blvd. ',` los angeles ',310/276 -7605,american,137
+141,` harry \ 's bar & american grill ',' 2020 ave. of the stars ',` los angeles ',310/277 -2333,italian,138
+142,` il fornaio cucina italiana ',' 301 n. beverly dr. ',` los angeles ',310/550 -8330,italian,139
+143,` jack sprat \ 's grill ',' 10668 w. pico blvd. ',` los angeles ',310/837 -6662,` health food ',140
+144,` jackson \ 's farm ',' 439 n. beverly drive ',` los angeles ',310/273 -5578,californian,141
+145,` jimmy \ 's ',' 201 moreno dr. ',` los angeles ',310/552 -2394,continental,142
+146,joss,' 9255 sunset blvd. ',` los angeles ',310/276 -1886,asian,143
+147,` le colonial ',' 8783 beverly blvd. ',` los angeles ',310/289 -0660,asian,144
+148,` le dome ',' 8720 sunset blvd. ',` los angeles ',310/659 -6919,french,145
+149,` louise \ 's trattoria ',' 4500 los feliz blvd. ',` los angeles ',213/667 -0777,italian,146
+150,` mon kee seafood restaurant ',' 679 n. spring st. ',` los angeles ',213/628 -6717,asian,147
+151,` morton \ 's ',' 8764 melrose ave. ',` los angeles ',310/276 -5205,american,148
+152,` nate \ ` n \ ' al \ 's ',' 414 n. beverly dr. ',` los angeles ',310/274 -0101,american,149
+153,nicola,' 601 s. figueroa st. ',` los angeles ',213/485 -0927,american,150
+154,` ocean avenue ',' 1401 ocean ave. ',` santa monica ',310/394 -5669,american,151
+155,orleans,' 11705 national blvd. ',` los angeles ',310/479 -4187,cajun,152
+156,` pacific dining car ',' 6th st. ',` los angeles ',213/483 -6000,american,153
+157,` paty \ 's ',' 10001 riverside dr. ',` toluca lake ',818/761 -9126,american,154
+158,` pinot hollywood ',' 1448 n. gower st. ',` los angeles ',213/461 -8800,californian,155
+159,posto,' 14928 ventura blvd. ',` sherman oaks ',818/784 -4400,italian,156
+160,prego,' 362 n. camden dr. ',` los angeles ',310/277 -7346,italian,157
+161,` rj \ 's the rib joint ',' 252 n. beverly dr. ',` los angeles ',310/274 -7427,american,158
+162,remi,' 3rd st. promenade ',` santa monica ',310/393 -6545,italian,159
+163,` restaurant horikawa ',' 111 s. san pedro st. ',` los angeles ',213/680 -9355,asian,160
+164,` roscoe \ 's house of chicken \ ` n \ ' waffles ',' 1514 n. gower st. ',` los angeles ',213/466 -9329,american,161
+165,` schatzi on main ',' 3110 main st. ',` los angeles ',310/399 -4800,continental,162
+166,sofi,' 3rd st. ',` los angeles ',213/651 -0346,mediterranean,163
+167,swingers,' 8020 beverly blvd. ',` los angeles ',213/653 -5858,american,164
+168,` tavola calda ',' 7371 melrose ave. ',` los angeles ',213/658 -6340,italian,165
+169,` the mandarin ',' 430 n. camden dr. ',` los angeles ',310/859 -0926,asian,166
+170,` tommy tang \ 's ',' 7313 melrose ave. ',` los angeles ',213/937 -5733,asian,167
+171,` tra di noi ',' 3835 cross creek rd. ',` los angeles ',310/456 -0169,italian,168
+172,` trader vic \ 's ',' 9876 wilshire blvd. ',` los angeles ',310/276 -6345,asian,169
+173,vida,' 1930 north hillhurst ave. ',` los feliz ',213/660 -4446,american,170
+174,` west beach cafe ','60 n. venice blvd. ',` los angeles ',310/823 -5396,american,171
+175,'20 mott ','20 mott st. between bowery and pell st. ',` new york ',212/964 -0380,asian,172
+176,' 9 jones street ',' 9 jones st. ',` new york ',212/989 -1220,american,173
+177,adrienne,' 700 5th ave. at 55th st. ',` new york ',212/903 -3918,french,174
+178,agrotikon,' 322 e. 14 st. between 1st and 2nd aves . ',` new york ',212/473 -2602,mediterranean,175
+179,aja,' 937 broadway at 22nd st. ',` new york ',212/473 -8388,american,176
+180,alamo,' 304 e. 48th st. ',` new york ',' 212 / 759-0590 ',mexican,177
+181,` alley \ 's end ',' 311 w. 17th st. ',` new york ',212/627 -8899,american,178
+182,` ambassador grill ',' 1 united nations plaza at 44th st. ',` new york ',212/702 -5014,american,179
+183,` american place ',' 2 park ave. at 32nd st. ',` new york ',212/684 -2122,american,180
+184,` anche vivolo ',' 222 e. 58th st. between 2nd and 3rd aves . ',` new york ',212/308 -0112,italian,181
+185,arizona,' 206 206 e. 60th st. ',` new york ',212/838 -0440,american,182
+186,` arturo \ 's ',' 106 w. houston st. off thompson st. ',` new york ',212/677 -3820,italian,183
+187,` au mandarin ',' 200-250 vesey st. world financial center ',` new york ',212/385 -0313,asian,184
+188,` bar anise ',' 1022 3rd ave. between 60th and 61st sts . ',` new york ',212/355 -1112,mediterranean,185
+189,barbetta,' 321 w. 46th st. ',` new york ',212/246 -9171,italian,186
+190,` ben benson \ 's ',' 123 w. 52nd st. ',` new york ',212/581 -8888,american,187
+191,` big cup ',' 228 8th ave. between 21st and 22nd sts . ',` new york ',212/206 -0059,` coffee bar ',188
+192,` billy \ 's ',' 948 1st ave. between 52nd and 53rd sts . ',` new york ',212/753 -1870,american,189
+193,` boca chica ','13 1st ave. near 1st st. ',` new york ',212/473 -0108,` latin american ',190
+194,boonthai,' 1393a 2nd ave. between 72nd and 73rd sts . ',` new york ',212/249 -8484,asian,192
+195,bouterin,' 420 e. 59th st. off 1st ave. ',` new york ',212/758 -0323,french,193
+196,` brothers bar-b-q ',' 225 varick st. at clarkston st. ',` new york ',212/727 -2775,american,194
+197,bruno,' 240 e. 58th st. ',` new york ',212/688 -4190,italian,195
+198,` bryant park grill roof restaurant and bp cafe ','25 w. 40th st. between 5th and 6th aves . ',` new york ',212/840 -6500,american,196
+199,c3,' 103 waverly pl . near washington sq. ',` new york ',212/254 -1200,american,197
+200,ct,' 111 e. 22nd st. between park ave. s and lexington ave. ',` new york ',212/995 -8500,french,198
+201,` cafe bianco ',' 1486 2nd ave. between 77th and 78th sts . ',` new york ',212/988 -2655,` coffee bar ',199
+202,` cafe botanica ',' 160 central park s ',` new york ',212/484 -5120,french,200
+203,` cafe la fortuna ','69 w. 71st st. ',` new york ',212/724 -5846,` coffee bar ',201
+204,` cafe luxembourg ',' 200 w. 70th st. ',` new york ',212/873 -7411,french,202
+205,` cafe pierre ',' 2 e. 61st st. ',` new york ',212/940 -8185,french,203
+206,` cafe centro ',' 200 park ave. between 45th st. and vanderbilt ave. ',` new york ',212/818 -1222,french,204
+207,` cafe fes ',' 246 w. 4th st. at charles st. ',` new york ',212/924 -7653,mediterranean,205
+208,` caffe dante ','81 macdougal st. between houston and bleeker sts . ',` new york ',212/982 -5275,` coffee bar ',206
+209,` caffe dell \ ` artista ','46 greenwich ave. ',` new york ',212/645 -4431,` coffee bar ',207
+210,` caffe lure ',' 169 sullivan st. between houston and bleecker sts . ',` new york ',212/473 -2642,french,208
+211,` caffe reggio ',' 119 macdougal st. between 3rd and bleecker sts . ',` new york ',212/475 -9557,` coffee bar ',209
+212,` caffe roma ',' 385 broome st. at mulberry ',` new york ',212/226 -8413,` coffee bar ',210
+213,` caffe vivaldi ','32 jones st. at bleecker st. ',` new york ',212/691 -7538,` coffee bar ',211
+214,` caffe bondi ristorante ',' 7 w. 20th st. ',` new york ',212/691 -8136,italian,212
+215,` capsouto freres ',' 451 washington st. near watts st. ',` new york ',212/966 -4900,french,213
+216,` captain \ 's table ',' 860 2nd ave. at 46th st. ',` new york ',212/697 -9538,seafood,214
+217,` casa la femme ',' 150 wooster st. between houston and prince sts . ',` new york ',212/505 -0005,` middle eastern ',215
+218,` cendrillon asian grill & marimba bar ','45 mercer st. between broome and grand sts . ',` new york ',212/343 -9012,asian,216
+219,` chez jacqueline ','72 macdougal st. between w. houston and bleecker sts . ',` new york ',212/505 -0727,french,217
+220,chiam,' 160 e. 48th st. ',` new york ',212/371 -2323,asian,218
+221,` china grill ','60 w. 53rd st. ',` new york ',212/333 -7788,american,219
+222,cite,' 120 w. 51st st. ',` new york ',212/956 -7100,french,220
+223,` coco pazzo ','23 e. 74th st. ',` new york ',212/794 -0205,italian,221
+224,` columbus bakery ',' 53rd sts . ',` new york ',212/421 -0334,` coffee bar ',222
+225,` corrado cafe ',' 1013 3rd ave. between 60th and 61st sts . ',` new york ',212/753 -5100,` coffee bar ',223
+226,` cupcake cafe ',' 522 9th ave. at 39th st. ',` new york ',212/465 -1530,` coffee bar ',224
+227,` da nico ',' 164 mulberry st. between grand and broome sts . ',` new york ',212/343 -1212,italian,225
+228,` dean & deluca ',' 121 prince st. ',` new york ',212/254 -8776,` coffee bar ',226
+229,diva,' 341 w. broadway near grand st. ',` new york ',212/941 -9024,italian,227
+230,` dix et sept ',' 181 w. 10th st. ',` new york ',212/645 -8023,french,228
+231,docks,' 633 3rd ave. at 40th st. ',` new york ',' 212 / 986-8080 ',seafood,229
+232,` duane park cafe ',' 157 duane st. between w. broadway and hudson st. ',` new york ',212/732 -5555,american,230
+233,` el teddy \ 's ',' 219 w. broadway between franklin and white sts . ',` new york ',212/941 -7070,mexican,231
+234,'em ily \ 's ',' 1325 5th ave. at 111th st. ',` new york ',212/996 -1212,american,232
+235,'em pire korea ',' 6 e. 32nd st. ',` new york ',212/725 -1333,asian,233
+236,` ernie \ 's ',' 2150 broadway between 75th and 76th sts . ',` new york ',212/496 -1588,american,234
+237,` evergreen cafe ',' 1288 1st ave. at 69th st. ',` new york ',212/744 -3266,asian,235
+238,` f. ille ponte ristorante ','39 desbrosses st. near west st. ',` new york ',212/226 -4621,italian,236
+239,felix,' 340 w. broadway at grand st. ',` new york ',212/431 -0021,french,237
+240,ferrier,'29 e. 65th st. ',` new york ',212/772 -9000,french,238
+241,` fifty seven fifty seven ','57 e. 57th st. ',` new york ',212/758 -5757,american,239
+242,` film center cafe ',' 635 9th ave. between 44th and 45th sts . ',` new york ',' 212 / 262-2525 ',american,240
+243,` fiorello \ 's roman cafe ',' 1900 broadway between 63rd and 64th sts . ',` new york ',212/595 -5330,italian,241
+244,firehouse,' 522 columbus ave. between 85th and 86th sts . ',` new york ',212/595 -3139,american,242
+245,first,'87 1st ave. between 5th and 6th sts . ',` new york ',212/674 -3823,american,243
+246,` fishin eddie ','73 w. 71st st. ',` new york ',212/874 -3474,seafood,244
+247,` fleur de jour ',' 348 e. 62nd st. ',` new york ',212/355 -2020,` coffee bar ',245
+248,flowers,'21 west 17th st. between 5th and 6th aves . ',` new york ',212/691 -8888,american,246
+249,follonico,' 6 w. 24th st. ',` new york ',212/691 -6359,italian,247
+250,` fraunces tavern ','54 pearl st. at broad st. ',` new york ',212/269 -0144,american,248
+251,` french roast ',' 458 6th ave. at 11th st. ',` new york ',212/533 -2233,french,249
+252,` french roast cafe ',' 2340 broadway at 85th st. ',` new york ',212/799 -1533,` coffee bar ',250
+253,` frico bar ',' 402 w. 43rd st. off 9th ave. ',` new york ',212/564 -7272,italian,251
+254,` fujiyama mama ',' 467 columbus ave. between 82nd and 83rd sts . ',` new york ',212/769 -1144,asian,252
+255,` gabriela \ 's ',' 685 amsterdam ave. at 93rd st. ',` new york ',212/961 -0574,mexican,253
+256,` gallagher \ 's ',' 228 w. 52nd st. ',` new york ',212/245 -5336,american,254
+257,` gianni \ 's ','15 fulton st. ',` new york ',212/608 -7300,seafood,255
+258,girafe,' 208 e. 58th st. between 2nd and 3rd aves . ',` new york ',212/752 -3054,italian,256
+259,global,'33 93 2nd ave. between 5th and 6th sts . ',` new york ',212/477 -8427,american,257
+260,` golden unicorn ','18 e. broadway at catherine st. ',` new york ',' 212 / 941-0911 ',asian,258
+261,` grand ticino ',' 228 thompson st. between w. 3rd and bleecker sts . ',` new york ',212/777 -5922,italian,259
+262,halcyon,' 151 w. 54th st. in the rihga royal hotel ',` new york ',212/468 -8888,american,260
+263,` hard rock cafe ',' 221 w. 57th st. ',` new york ',212/489 -6565,american,261
+264,` hi-life restaurant and lounge ',' 1340 1st ave. at 72nd st. ',` new york ',212/249 -3600,american,262
+265,home,'20 cornelia st. between bleecker and w. 4th st. ',` new york ',212/243 -9579,american,263
+266,` hudson river club ',' 4 world financial center ',` new york ',212/786 -1500,american,264
+267,' i trulli ',' 122 e. 27th st. between lexington and park aves . ',` new york ',212/481 -7372,italian,265
+268,` il cortile ',' 125 mulberry st. between canal and hester sts . ',` new york ',212/226 -6060,italian,266
+269,` inca grill ',' 492 broome st. near w. broadway ',` new york ',212/966 -3371,` latin american ',268
+270,indochine,' 430 lafayette st. between 4th st. and astor pl . ',` new york ',212/505 -5111,asian,269
+271,` internet cafe ','82 e. 3rd st. between 1st and 2nd aves . ',` new york ',' 212 / 614-0747 ',` coffee bar ',270
+272,ipanema,'13 w. 46th st. ',` new york ',212/730 -5848,` latin american ',271
+273,` jean lafitte ','68 w. 58th st. ',` new york ',212/751 -2323,french,272
+274,` jewel of india ','15 w. 44th st. ',` new york ',212/869 -5544,asian,273
+275,` jimmy sung \ 's ',' 219 e. 44th st. between 2nd and 3rd aves . ',` new york ',212/682 -5678,asian,274
+276,` joe allen ',' 326 w. 46th st. ',` new york ',212/581 -6464,american,275
+277,` judson grill ',' 152 w. 52nd st. ',` new york ',212/582 -5252,american,276
+278,` l \ ` absinthe ',' 227 e. 67th st. ',` new york ',212/794 -4950,french,277
+279,` l \ ` auberge ',' 1191 1st ave. between 64th and 65th sts . ',` new york ',212/288 -8791,` middle eastern ',278
+280,` l \ ` auberge du midi ',' 310 w. 4th st. between w. 12th and bank sts . ',` new york ',212/242 -4705,french,279
+281,` l \ ` udo ',' 432 lafayette st. near astor pl . ',` new york ',212/388 -0978,french,280
+282,` la reserve ',' 4 w. 49th st. ',` new york ',212/247 -2993,french,281
+283,` lanza restaurant ',' 168 1st ave. between 10th and 11th sts . ',` new york ',212/674 -7014,italian,282
+284,` lattanzi ristorante ',' 361 w. 46th st. ',` new york ',212/315 -0980,italian,283
+285,layla,' 211 w. broadway at franklin st. ',` new york ',212/431 -0700,` middle eastern ',284
+286,` le chantilly ',' 106 e. 57th st. ',` new york ',212/751 -2931,french,285
+287,` le colonial ',' 149 e. 57th st. ',` new york ',' 212 / 752-0808 ',asian,286
+288,` le gamin ','50 macdougal st. between houston and prince sts . ',` new york ',212/254 -4678,` coffee bar ',287
+289,` le jardin ','25 cleveland pl . near spring st. ',` new york ',212/343 -9599,french,288
+290,` le madri ',' 168 w. 18th st. ',` new york ',212/727 -8022,italian,289
+291,` le marais ',' 150 w. 46th st. ',` new york ',212/869 -0900,american,290
+292,` le perigord ',' 405 e. 52nd st. ',` new york ',212/755 -6244,french,291
+293,` le select ',' 507 columbus ave. between 84th and 85th sts . ',` new york ',212/875 -1993,american,292
+294,` les halles ',' 411 park ave. s between 28th and 29th sts . ',` new york ',212/679 -4111,french,293
+295,` lincoln tavern ','51 w. 64th st. ',` new york ',212/721 -8271,american,294
+296,lola,'30 west 22nd st. between 5th and 6th ave. ',` new york ',212/675 -6700,american,295
+297,` lucky strike ','59 grand st. between wooster st. and w. broadway ',` new york ',212/941 -0479,` or 212/941 -0772 american ',296
+298,` mad fish ',' 2182 broadway between 77th and 78th sts . ',` new york ',212/787 -0202,seafood,297
+299,` main street ',' 446 columbus ave. between 81st and 82nd sts . ',` new york ',212/873 -5025,american,298
+300,` mangia e bevi ',' 800 9th ave. at 53rd st. ',` new york ',212/956 -3976,italian,299
+301,` manhattan cafe ',' 1161 1st ave. between 63rd and 64th sts . ',` new york ',212/888 -6556,american,300
+302,` manila garden ',' 325 e. 14th st. between 1st and 2nd aves . ',` new york ',212/777 -6314,asian,301
+303,marichu,' 342 e. 46th st. between 1st and 2nd aves . ',` new york ',212/370 -1866,french,302
+304,` marquet patisserie ','15 e. 12th st. between 5th ave. and university pl . ',` new york ',212/229 -9313,` coffee bar ',303
+305,match,' 160 mercer st. between houston and prince sts . ',` new york ',212/906 -9173,american,304
+306,` matthew \ 's ',' 1030 3rd ave. at 61st st. ',` new york ',212/838 -4343,american,305
+307,` mavalli palace ','46 e. 29th st. ',` new york ',212/679 -5535,asian,306
+308,` milan cafe and coffee bar ',' 120 w. 23rd st. ',` new york ',212/807 -1801,` coffee bar ',307
+309,` monkey bar ','60 e. 54th st. ',` new york ',212/838 -2600,american,308
+310,montien,' 1134 1st ave. between 62nd and 63rd sts . ',` new york ',212/421 -4433,asian,309
+311,` morton \ 's ',' 551 5th ave. at 45th st. ',` new york ',212/972 -3315,american,310
+312,` motown cafe ',' 104 w. 57th st. near 6th ave. ',` new york ',212/581 -8030,american,311
+313,` new york kom tang soot bul house ','32 w. 32nd st. ',` new york ',' 212 / 947-8482 ',asian,312
+314,` new york noodletown ','28 1/2 bowery at bayard st. ',` new york ',212/349 -0923,asian,313
+315,newsbar,' 2 w. 19th st. ',` new york ',212/255 -3996,` coffee bar ',314
+316,odeon,' 145 w. broadway at thomas st. ',` new york ',212/233 -0507,american,315
+317,orso,' 322 w. 46th st. ',` new york ',212/489 -7212,italian,316
+318,` osteria al droge ',' 142 w. 44th st. ',` new york ',212/944 -3643,italian,317
+319,otabe,'68 e. 56th st. ',` new york ',212/223 -7575,asian,318
+320,pacifica,' 138 lafayette st. between canal and howard sts . ',` new york ',212/941 -4168,asian,319
+321,palio,' 151 w. 51st . st. ',` new york ',212/245 -4850,italian,320
+322,pamir,' 1065 1st ave. at 58th st. ',` new york ',212/644 -9258,` middle eastern ',321
+323,` parioli romanissimo ','24 e. 81st st. ',` new york ',212/288 -2391,italian,322
+324,patria,' 250 park ave. s at 20th st. ',` new york ',212/777 -6211,` latin american ',323
+325,` peacock alley ',' 301 park ave. between 49th and 50th sts . ',` new york ',212/872 -4895,french,324
+326,` pen & pencil ',' 205 e. 45th st. ',` new york ',212/682 -8660,american,325
+327,` penang soho ',' 109 spring st. between greene and mercer sts . ',` new york ',212/274 -8883,asian,326
+328,persepolis,' 1423 2nd ave. between 74th and 75th sts . ',` new york ',212/535 -1100,` middle eastern ',327
+329,` planet hollywood ',' 140 w. 57th st. ',` new york ',212/333 -7827,american,328
+330,pomaire,' 371 w. 46th st. off 9th ave. ',` new york ',' 212 / 956-3055 ',` latin american ',329
+331,` popover cafe ',' 551 amsterdam ave. between 86th and 87th sts . ',` new york ',212/595 -8555,american,330
+332,` post house ','28 e. 63rd st. ',` new york ',212/935 -2888,american,331
+333,rain,' 100 w. 82nd st. ',` new york ',212/501 -0776,asian,332
+334,` red tulip ',' 439 e. 75th st. ',` new york ',212/734 -4893,` eastern european ',333
+335,republic,' 37a union sq. w between 16th and 17th sts . ',` new york ',212/627 -7172,asian,335
+336,` roettelle a. g ',' 126 e. 7th st. between 1st ave. and ave. a ',` new york ',212/674 -4140,continental,336
+337,` rosa mexicano ',' 1063 1st ave. at 58th st. ',` new york ',212/753 -7407,mexican,337
+338,` ruth \ 's chris ',' 148 w. 51st st. ',` new york ',212/245 -9600,american,338
+339,s.p.q.r,' 133 mulberry st. between hester and grand sts . ',` new york ',212/925 -3120,italian,339
+340,` sal anthony \ 's ','55 irving pl . ',` new york ',212/982 -9030,italian,340
+341,` sammy \ 's roumanian steak house ',' 157 chrystie st. at delancey st. ',` new york ',212/673 -0330,` east european ',341
+342,` san pietro ','18 e. 54th st. ',` new york ',212/753 -9015,italian,342
+343,` sant ambroeus ',' 1000 madison ave. between 77th and 78th sts . ',` new york ',212/570 -2211,` coffee bar ',343
+344,` sarabeth \ 's kitchen ',' 423 amsterdam ave. between 80th and 81st sts . ',` new york ',212/496 -6280,american,344
+345,` sea grill ','19 w. 49th st. ',` new york ',212/332 -7610,seafood,345
+346,serendipity,' 3 225 e. 60th st. ',` new york ',212/838 -3531,american,346
+347,` seventh regiment mess and bar ',' 643 park ave. at 66th st. ',` new york ',212/744 -4107,american,347
+348,sfuzzi,'58 w. 65th st. ',` new york ',212/873 -3700,american,348
+349,shaan,'57 w. 48th st. ',` new york ',' 212 / 977-8400 ',asian,349
+350,` sofia fabulous pizza ',' 1022 madison ave. near 79th st. ',` new york ',212/734 -2676,italian,350
+351,` spring street natural restaurant & bar ','62 spring st. at lafayette st. ',` new york ',212/966 -0290,american,351
+352,` stage deli ',' 834 7th ave. between 53rd and 54th sts . ',` new york ',212/245 -7850,delicatessen,352
+353,stingray,' 428 amsterdam ave. between 80th and 81st sts . ',` new york ',212/501 -7515,seafood,353
+354,` sweet \ ` n \ ` tart cafe ','76 mott st. at canal st. ',` new york ',212/334 -8088,asian,354
+355,' t salon ',' 143 mercer st. at prince st. ',` new york ',212/925 -3700,` coffee bar ',355
+356,` tang pavillion ','65 w. 55th st. ',` new york ',212/956 -6888,asian,356
+357,tapika,' 950 8th ave. at 56th st. ',` new york ',' 212 / 397-3737 ',american,357
+358,` teresa \ 's ',' 103 1st ave. between 6th and 7th sts . ',` new york ',212/228 -0604,` east european ',358
+359,terrace,' 400 w. 119th st. between amsterdam and morningside aves . ',` new york ',212/666 -9490,continental,359
+360,` the coffee pot ',' 350 9th ave. at 49th st. ',` new york ',212/265 -3566,` coffee bar ',360
+361,` the savannah club ',' 2420 broadway at 89th st. ',` new york ',212/496 -1066,american,361
+362,` trattoria dell \ ` arte ',' 900 7th ave. between 56th and 57th sts . ',` new york ',212/245 -9800,italian,362
+363,triangolo,' 345 e. 83rd st. ',` new york ',212/472 -4488,italian,363
+364,` tribeca grill ',' 375 greenwich st. near franklin st. ',` new york ',212/941 -3900,american,364
+365,` trois jean ',' 154 e. 79th st. between lexington and 3rd aves . ',` new york ',212/988 -4858,` coffee bar ',365
+366,` tse yang ','34 e. 51st st. ',` new york ',212/688 -5447,asian,366
+367,` turkish kitchen ',' 386 3rd ave. between 27th and 28th sts . ',` new york ',212/679 -1810,` middle eastern ',367
+368,` two two two ',' 222 w. 79th st. ',` new york ',212/799 -0400,american,368
+369,` veniero \ 's pasticceria ',' 342 e. 11th st. near 1st ave. ',` new york ',212/674 -7264,` coffee bar ',369
+370,verbena,'54 irving pl . at 17th st. ',` new york ',212/260 -5454,american,370
+371,` victor \ 's cafe ','52 236 w. 52nd st. ',` new york ',212/586 -7714,` latin american ',371
+372,` vince & eddie \ 's ','70 w. 68th st. ',` new york ',212/721 -0068,american,372
+373,vong,' 200 e. 54th st. ',` new york ',212/486 -9592,american,373
+374,` water club ',' 500 e. 30th st. ',` new york ',212/683 -3333,american,374
+375,west,' 63rd street steakhouse 44 w. 63rd st. ',` new york ',212/246 -6363,american,375
+376,xunta,' 174 1st ave. between 10th and 11th sts . ',` new york ',212/614 -0620,mediterranean,376
+377,` zen palate ','34 union sq. e at 16th st. ',` new york ',212/614 -9291,` and 212/614 -9345 asian ',377
+378,zoe,'90 prince st. between broadway and mercer st. ',` new york ',212/966 -6722,american,378
+379,abbey,' 163 ponce de leon ave. ',atlanta,404/876 -8532,international,379
+380,` aleck \ 's barbecue heaven ',' 783 martin luther king jr. dr. ',atlanta,404/525 -2062,barbecue,380
+381,` annie \ 's thai castle ',' 3195 roswell rd. ',atlanta,404/264 -9546,asian,381
+382,anthonys,' 3109 piedmont rd. . just south of peachtree rd. ',atlanta,404/262 -7379,american,382
+383,` atlanta fish market ',' 265 pharr rd. ',atlanta,404/262 -3165,american,383
+384,` beesley \ 's of buckhead ',' 260 e. paces ferry road ',atlanta,404/264 -1334,continental,384
+385,` bertolini \ 's ',' 3500 peachtree rd. . phipps plaza ',atlanta,404/233 -2333,italian,385
+386,bistango,' 1100 peachtree st. ',atlanta,404/724 -0901,mediterranean,386
+387,` cafe renaissance ',' 7050 jimmy carter blvd. . norcross ',atlanta,770/441 -- 0291,american,387
+388,` camille \ 's ',' 1186 n. highland ave. ',atlanta,404/872 -7203,italian,388
+389,cassis,' 3300 peachtree rd. . grand hyatt ',atlanta,404/365 -8100,mediterranean,389
+390,` city grill ','50 hurt plaza ',atlanta,404/524 -2489,international,390
+391,` coco loco ','40 buckhead crossing mall on the sidney marcus blvd. ',atlanta,404/364 -0212,caribbean,391
+392,` colonnade restaurant ',' 1879 cheshire bridge rd. ',atlanta,404/874 -5642,southern,392
+393,` dante \ 's down the hatch buckhead ',' 3380 peachtree rd. ',atlanta,404/266 -1600,continental,393
+394,` dante \ 's down the hatch ',` underground underground mall underground atlanta ',atlanta,404/577 -1800,continental,394
+395,` fat matt \ 's rib shack ',' 1811 piedmont ave. near cheshire bridge rd. ',atlanta,404/607 -1622,barbecue,395
+396,` french quarter food shop ',' 923 peachtree st. at 8th st. ',atlanta,404/875 -2489,southern,396
+397,` holt bros. bar-b-q ',' 6359 jimmy carter blvd. . at buford hwy . norcross ',atlanta,770/242 -3984,barbecue,397
+398,` horseradish grill ',' 4320 powers ferry rd. ',atlanta,404/255 -7277,southern,398
+399,` hsu \ 's gourmet ',' 192 peachtree center ave. at international blvd. ',atlanta,404/659 -2788,asian,399
+400,` imperial fez ',' 2285 peachtree rd. . peachtree battle condominium ',atlanta,404/351 -0870,mediterranean,400
+401,kamogawa,' 3300 peachtree rd. . grand hyatt ',atlanta,404/841 -0314,asian,401
+402,` la grotta at ravinia dunwoody rd. ',` holiday inn/crowne plaza at ravinia dunwoody ',atlanta,770/395 -9925,italian,402
+403,` little szechuan ',' c buford hwy . northwoods plaza doraville ',atlanta,770/451 -0192,asian,403
+404,` lowcountry barbecue ',' 6301 roswell rd. . sandy springs plaza sandy springs ',atlanta,404/255 -5160,barbecue,404
+405,` luna si ',' 1931 peachtree rd. ',atlanta,404/355 -5993,continental,405
+406,` mambo restaurante cubano ',' 1402 n. highland ave. ',atlanta,404/874 -2626,caribbean,406
+407,` mckinnon \ 's louisiane ',' 3209 maple dr. ',atlanta,404/237 -1313,southern,407
+408,` mi spia dunwoody rd. ',` park place across from perimeter mall dunwoody ',atlanta,770/393 -1333,italian,408
+409,` nickiemoto \ 's : a sushi bar ',' 247 buckhead ave. east village sq. ',atlanta,404/842 -0334,fusion,409
+410,palisades,' 1829 peachtree rd. ',atlanta,404/350 -6755,continental,410
+411,` pleasant peasant ',' 555 peachtree st. at linden ave. ',atlanta,404/874 -3223,american,411
+412,pricci,' 500 pharr rd. ',atlanta,404/237 -2941,italian,412
+413,` r.j. \ 's uptown kitchen & wine bar ',' 870 n. highland ave. ',atlanta,404/875 -7775,american,413
+414,` rib ranch ','25 irby ave. ',atlanta,404/233 -7644,barbecue,414
+415,` sa tsu ki ',' 3043 buford hwy . ',atlanta,404/325 -5285,asian,415
+416,` sato sushi and thai ',' 6050 peachtree pkwy . norcross ',atlanta,770/449 -0033,asian,416
+417,` south city kitchen ',' 1144 crescent ave. ',atlanta,404/873 -7358,southern,417
+418,` south of france ',' 2345 cheshire bridge rd. ',atlanta,404/325 -6963,french,418
+419,` stringer \ 's fish camp and oyster bar ',' 3384 shallowford rd. . chamblee ',atlanta,770/458 -7145,southern,419
+420,` sundown cafe ',' 2165 cheshire bridge rd. ',atlanta,404/321 -1118,american,420
+421,` taste of new orleans ',' 889 w. peachtree st. ',atlanta,404/874 -5535,southern,421
+422,tomtom,' 3393 peachtree rd. ',atlanta,404/264 -1163,continental,422
+423,` antonio \ 's ',' 3700 w. flamingo ',` las vegas ',702/252 -7737,italian,423
+424,` bally \ 's big kitchen ',' 3645 las vegas blvd. s ',` las vegas ',702/739 -4111,buffets,424
+425,` bamboo garden ',' 4850 flamingo rd. ',` las vegas ',702/871 -3262,asian,425
+426,` battista \ 's hole in the wall ',' 4041 audrie st. at flamingo rd. ',` las vegas ',702/732 -1424,italian,426
+427,` bertolini \ 's ',' 3570 las vegas blvd. s ',` las vegas ',702/735 -4663,italian,427
+428,` binion \ 's coffee shop ',' 128 fremont st. ',` las vegas ',702/382 -1600,` coffee shops/diners ',428
+429,bistro,' 3400 las vegas blvd. s ',` las vegas ',702/791 -7111,continental,429
+430,broiler,' 4111 boulder hwy . ',` las vegas ',702/432 -7777,american,430
+431,` bugsy \ 's diner ',' 3555 las vegas blvd. s ',` las vegas ',702/733 -3111,` coffee shops/diners ',431
+432,` cafe michelle ',' 1350 e. flamingo rd. ',` las vegas ',702/735 -8686,american,432
+433,` cafe roma ',' 3570 las vegas blvd. s ',` las vegas ',702/731 -7547,` coffee shops/diners ',433
+434,` capozzoli \ 's ',' 3333 s. maryland pkwy . ',` las vegas ',702/731 -5311,italian,434
+435,` carnival world ',' 3700 w. flamingo rd. ',` las vegas ',702/252 -7777,buffets,435
+436,` center stage plaza hotel ',' 1 main st. ',` las vegas ',702/386 -2512,american,436
+437,` circus circus ',' 2880 las vegas blvd. s ',` las vegas ',702/734 -0410,buffets,437
+438,'em press court ',' 3570 las vegas blvd. s ',` las vegas ',702/731 -7888,asian,438
+439,feast,' 2411 w. sahara ave. ',` las vegas ',702/367 -2411,buffets,439
+440,` golden nugget hotel ',' 129 e. fremont st. ',` las vegas ',702/385 -7111,buffets,440
+441,` golden steer ',' 308 w. sahara ave. ',` las vegas ',702/384 -4470,` steak houses ',441
+442,` lillie langtry \ 's ',' 129 e. fremont st. ',` las vegas ',702/385 -7111,asian,442
+443,` mandarin court ',' 1510 e. flamingo rd. ',` las vegas ',702/737 -1234,asian,443
+444,` margarita \ 's mexican cantina ',' 3120 las vegas blvd. s ',` las vegas ',702/794 -8200,mexican,444
+445,` mary \ 's diner ',' 5111 w. boulder hwy . ',` las vegas ',702/454 -8073,` coffee shops/diners ',445
+446,mikado,' 3400 las vegas blvd. s ',` las vegas ',702/791 -7111,asian,446
+447,pamplemousse,' 400 e. sahara ave. ',` las vegas ',702/733 -2066,continental,447
+448,` ralph \ 's diner ',' 3000 las vegas blvd. s ',` las vegas ',702/732 -6330,` coffee shops/diners ',448
+449,` the bacchanal ',' 3570 las vegas blvd. s ',` las vegas ',702/731 -7525,` only in las vegas ',449
+450,venetian,' 3713 w. sahara ave. ',` las vegas ',702/876 -4190,italian,450
+451,` viva mercado \ 's ',' 6182 w. flamingo rd. ',` las vegas ',702/871 -8826,mexican,451
+452,` yolie \ 's ',' 3900 paradise rd. ',` las vegas ',702/794 -0700,` steak houses ',452
+453,2223,' 2223 market st. ',` san francisco ',415/431 -0692,american,453
+454,acquarello,' 1722 sacramento st. ',` san francisco ',415/567 -5432,italian,454
+455,` bardelli \ 's ',' 243 o \ ` farrell st. ',` san francisco ',415/982 -0243,` old san francisco ',455
+456,betelnut,' 2030 union st. ',` san francisco ',415/929 -8855,asian,456
+457,` bistro roti ',' 155 steuart st. ',` san francisco ',415/495 -6500,french,457
+458,bix,'56 gold st. ',` san francisco ',415/433 -6300,american,458
+459,bizou,' 598 fourth st. ',` san francisco ',415/543 -2222,french,459
+460,` buca giovanni ',' 800 greenwich st. ',` san francisco ',415/776 -7766,italian,460
+461,` cafe adriano ',' 3347 fillmore st. ',` san francisco ',415/474 -4180,italian,461
+462,` cafe marimba ',' 2317 chestnut st. ',` san francisco ',415/776 -1506,` mexican/latin american/spanish ',462
+463,` california culinary academy ',' 625 polk st. ',` san francisco ',415/771 -3500,french,463
+464,` capp \ 's corner ',' 1600 powell st. ',` san francisco ',415/989 -2589,italian,464
+465,carta,' 1772 market st. ',` san francisco ',415/863 -3516,american,465
+466,chevys,' 4th and howard sts . ',` san francisco ',415/543 -8060,` mexican/latin american/spanish ',466
+467,` cypress club ',' 500 jackson st. ',` san francisco ',415/296 -8555,american,467
+468,` des alpes ',' 732 broadway ',` san francisco ',415/788 -9900,french,468
+469,faz,' 161 sutter st. ',` san francisco ',415/362 -0404,` greek and middle eastern ',469
+470,` fog city diner ',' 1300 battery st. ',` san francisco ',415/982 -2000,american,470
+471,` garden court ',` market and new montgomery sts . ',` san francisco ',415/546 -5011,` old san francisco ',471
+472,` gaylord \ 's ',` ghirardelli sq. ',` san francisco ',415/771 -8822,asian,472
+473,` grand cafe hotel monaco ',' 501 geary st. ',` san francisco ',415/292 -0101,american,473
+474,greens,` bldg. a fort mason ',` san francisco ',415/771 -6222,vegetarian,474
+475,` harbor village ',' 4 embarcadero center ',` san francisco ',415/781 -8833,asian,475
+476,` harris \ '',' 2100 van ness ave. ',` san francisco ',415/673 -1888,` steak houses ',476
+477,` harry denton \ 's ',' 161 steuart st. ',` san francisco ',415/882 -1333,american,477
+478,` hayes street grill ',' 320 hayes st. ',` san francisco ',415/863 -5545,seafood,478
+479,helmand,' 430 broadway ',` san francisco ',415/362 -0641,` greek and middle eastern ',479
+480,` hong kong flower lounge ',' 5322 geary blvd. ',` san francisco ',415/668 -8998,asian,480
+481,` hong kong villa ',' 2332 clement st. ',` san francisco ',415/752 -8833,asian,481
+482,` hyde street bistro ',' 1521 hyde st. ',` san francisco ',415/441 -7778,italian,482
+483,` il fornaio levi \ 's plaza ',' 1265 battery st. ',` san francisco ',415/986 -0100,italian,483
+484,` izzy \ 's steak & chop house ',' 3345 steiner st. ',` san francisco ',415/563 -0487,` steak houses ',484
+485,` jack \ 's ',' 615 sacramento st. ',` san francisco ',415/986 -9854,` old san francisco ',485
+486,` kabuto sushi ',' 5116 geary blvd. ',` san francisco ',415/752 -5652,asian,486
+487,` katia \ 's ',' 600 5th ave. ',` san francisco ',415/668 -9292,'',487
+488,` kuleto \ 's ',' 221 powell st. ',` san francisco ',415/397 -7720,italian,488
+489,` kyo-ya . sheraton palace hotel ',' 2 new montgomery st. at market st. ',` san francisco ',415/546 -5000,asian,489
+490,` l \ ` osteria del forno ',' 519 columbus ave. ',` san francisco ',415/982 -1124,italian,490
+491,` le central ',' 453 bush st. ',` san francisco ',415/391 -2233,french,491
+492,` le soleil ',' 133 clement st. ',` san francisco ',415/668 -4848,asian,492
+493,` macarthur park ',' 607 front st. ',` san francisco ',415/398 -5700,american,493
+494,manora,' 3226 mission st. ',` san francisco ',415/861 -6224,asian,494
+495,maykadeh,' 470 green st. ',` san francisco ',415/362 -8286,` greek and middle eastern ',495
+496,` mccormick & kuleto \ 's ',` ghirardelli sq. ',` san francisco ',415/929 -1730,seafood,496
+497,millennium,' 246 mcallister st. ',` san francisco ',415/487 -9800,vegetarian,497
+498,` moose \ 's ',' 1652 stockton st. ',` san francisco ',415/989 -7800,mediterranean,498
+499,` north india ',' 3131 webster st. ',` san francisco ',415/931 -1556,asian,499
+500,` one market ',' 1 market st. ',` san francisco ',415/777 -5577,american,500
+501,oritalia,' 1915 fillmore st. ',` san francisco ',415/346 -1333,italian,501
+502,` pacific pan pacific hotel ',' 500 post st. ',` san francisco ',415/929 -2087,french,502
+503,` palio d \ ` asti ',' 640 sacramento st. ',` san francisco ',415/395 -9800,italian,503
+504,` pane e vino ',' 3011 steiner st. ',` san francisco ',415/346 -2111,italian,504
+505,pastis,' 1015 battery st. ',` san francisco ',415/391 -2555,french,505
+506,` perry \ 's ',' 1944 union st. ',` san francisco ',415/922 -9022,american,506
+507,` r & g lounge ',' 631 b kearny st. ',` san francisco ',415/982 -7877,` or 415/982 -3811 asian ',507
+508,rubicon,' 558 sacramento st. ',` san francisco ',415/434 -4100,american,508
+509,rumpus,' 1 tillman pl . ',` san francisco ',415/421 -2300,american,509
+510,sanppo,' 1702 post st. ',` san francisco ',415/346 -3486,asian,510
+511,` scala \ 's bistro ',' 432 powell st. ',` san francisco ',415/395 -8555,italian,511
+512,` south park cafe ',' 108 south park ',` san francisco ',415/495 -7275,french,512
+513,` splendido embarcadero ',4,` san francisco ',415/986 -3222,mediterranean,513
+514,stars,' 150 redwood alley ',` san francisco ',415/861 -7827,american,514
+515,` stars cafe ',' 500 van ness ave. ',` san francisco ',415/861 -4344,american,515
+516,` stoyanof \ 's cafe ',' 1240 9th ave. ',` san francisco ',415/664 -3664,` greek and middle eastern ',516
+517,` straits cafe ',' 3300 geary blvd. ',` san francisco ',415/668 -1783,asian,517
+518,suppenkuche,' 601 hayes st. ',` san francisco ',415/252 -9289,russian/german,518
+519,` tadich grill ',' 240 california st. ',` san francisco ',415/391 -2373,seafood,519
+520,` the heights ',' 3235 sacramento st. ',` san francisco ',415/474 -8890,french,520
+521,thepin,' 298 gough st. ',` san francisco ',415/863 -9335,asian,521
+522,` ton kiang ',' 3148 geary blvd. ',` san francisco ',415/752 -4440,asian,522
+523,vertigo,' 600 montgomery st. ',` san francisco ',415/433 -7250,mediterranean,523
+524,` vivande porta via ',' 2125 fillmore st. ',` san francisco ',415/346 -4430,italian,524
+525,` vivande ristorante ',' 670 golden gate ave. ',` san francisco ',415/673 -9245,italian,525
+526,` world wrapps ',' 2257 chestnut st. ',` san francisco ',415/563 -9727,american,526
+527,` wu kong ',' 101 spear st. ',` san francisco ',415/957 -9300,asian,527
+528,` yank sing ',' 427 battery st. ',` san francisco ',415/541 -4949,asian,528
+529,` yaya cuisine ',' 1220 9th ave. ',` san francisco ',415/566 -6966,` greek and middle eastern ',529
+530,` yoyo tsumami bistro ',' 1611 post st. ',` san francisco ',415/922 -7788,french,530
+531,zarzuela,' 2000 hyde st. ',` san francisco ',415/346 -0800,` mexican/latin american/spanish ',531
+532,` zuni cafe & grill ',' 1658 market st. ',` san francisco ',415/552 -2522,mediterranean,532

datasets/Fodors-Zagats/tableB.csv

@@ -0,0 +1,332 @@
+id,name,addr,city,phone,type,class
+0,` apple pan the ',' 10801 w. pico blvd. ',` west la ',310-475-3585,american,534
+1,` asahi ramen ',' 2027 sawtelle blvd. ',` west la ',310-479-2231,` noodle shops ',535
+2,` baja fresh ',' 3345 kimber dr. ',` westlake village ',805-498-4049,mexican,536
+3,` belvedere the ',' 9882 little santa monica blvd. ',` beverly hills ',310-788-2306,` pacific new wave ',537
+4,` benita \ 's frites ',' 1433 third st. promenade ',` santa monica ',310-458-2889,` fast food ',538
+5,` bernard \ 's ',' 515 s. olive st. ',` los angeles ',213-612-1580,continental,539
+6,` bistro 45 ','45 s. mentor ave. ',pasadena,818-795-2478,californian,540
+7,` brent \ 's deli ',' 19565 parthenia ave. ',northridge,818-886-5679,delis,541
+8,` brighton coffee shop ',' 9600 brighton way ',` beverly hills ',310-276-7732,` coffee shops ',542
+9,` bristol farms market cafe ',' 1570 rosecrans ave. s. ',pasadena,310-643-5229,californian,543
+10,` bruno \ 's ',' 3838 centinela ave. ',` mar vista ',310-397-5703,italian,544
+11,` cafe \ '50s ',' 838 lincoln blvd. ',venice,310-399-1955,american,545
+12,` cafe blanc ',' 9777 little santa monica blvd. ',` beverly hills ',310-888-0108,` pacific new wave ',546
+13,` cassell \ 's ',' 3266 w. sixth st. ',la,213-480-8668,hamburgers,547
+14,` chez melange ',' 1716 pch ',` redondo beach ',310-540-1222,eclectic,548
+15,diaghilev,' 1020 n. san vicente blvd. ',` w. hollywood ',310-854-1111,russian,549
+16,` don antonio \ 's ',' 1136 westwood blvd. ',westwood,310-209-1422,italian,550
+17,` duke \ 's ',' 8909 sunset blvd. ',` w. hollywood ',310-652-3100,` coffee shops ',551
+18,` falafel king ',' 1059 broxton ave. ',westwood,310-208-4444,` middle eastern ',552
+19,` feast from the east ',' 1949 westwood blvd. ',` west la ',310-475-0400,chinese,553
+20,` gumbo pot the ',' 6333 w. third st. ',la,213-933-0358,cajun/creole,554
+21,` hollywood hills coffee shop ',' 6145 franklin ave. ',hollywood,213-467-7678,` coffee shops ',555
+22,` indo cafe ',' 10428 1/2 national blvd. ',la,310-815-1290,indonesian,556
+23,` jan \ 's family restaurant ',' 8424 beverly blvd. ',la,213-651-2866,` coffee shops ',557
+24,jiraffe,' 502 santa monica blvd ',` santa monica ',310-917-6671,californian,558
+25,` jody maroni \ 's sausage kingdom ',' 2011 ocean front walk ',venice,310-306-1995,` hot dogs ',559
+26,` joe \ 's ',' 1023 abbot kinney blvd. ',venice,310-399-5811,` american ( new ) ',560
+27,` john o \ ` groats ',' 10516 w. pico blvd. ',` west la ',310-204-0692,` coffee shops ',561
+28,` johnnie \ 's pastrami ',' 4017 s. sepulveda blvd. ',` culver city ',310-397-6654,delis,562
+29,` johnny reb \ 's southern smokehouse ',' 4663 long beach blvd. ',` long beach ',310-423-7327,southern/soul,563
+30,` johnny rockets ( la ) ',' 7507 melrose ave. ',la,213-651-3361,american,564
+31,` killer shrimp ',' 4000 colfax ave. ',` studio city ',818-508-1570,seafood,565
+32,` kokomo cafe ',' 6333 w. third st. ',la,213-933-0773,american,566
+33,` koo koo roo ',' 8393 w. beverly blvd. ',la,213-655-9045,chicken,567
+34,` la cachette ',' 10506 little santa monica blvd. ',` century city ',310-470-4992,` french ( new ) ',568
+35,` la salsa ( la ) ',' 22800 pch ',malibu,310-456-6299,mexican,569
+36,` la serenata de garibaldi ',' 1842 e. first ',` st. boyle hts . ',213-265-2887,mexican/tex-mex,570
+37,` langer \ 's ',' 704 s. alvarado st. ',la,213-483-8050,delis,571
+38,` local nochol ',' 30869 thousand oaks blvd. ',` westlake village ',818-706-7706,` health food ',572
+39,` main course the ',' 10509 w. pico blvd. ',` rancho park ',310-475-7564,american,573
+40,` mani \ 's bakery & espresso bar ',' 519 s. fairfax ave. ',la,213-938-8800,desserts,574
+41,` martha \ 's ',' 22nd street grill 25 22nd ',` st. hermosa beach ',310-376-7786,american,575
+42,` maxwell \ 's cafe ',' 13329 washington blvd. ',` marina del rey ',310-306-7829,american,576
+43,` michael \ 's ( los angeles ) ',' 1147 third st. ',` santa monica ',310-451-0843,californian,577
+44,mishima,' 8474 w. third st. ',la,213-782-0181,` noodle shops ',578
+45,` mo better meatty meat ',' 7261 melrose ave. ',la,213-935-5280,hamburgers,579
+46,` mulberry st. ',' 17040 ventura blvd. ',encino,818-906-8881,pizza,580
+47,` ocean park cafe ',' 3117 ocean park blvd. ',` santa monica ',310-452-5728,american,581
+48,` ocean star ',' 145 n. atlantic blvd. ',` monterey park ',818-308-2128,seafood,582
+49,` original pantry bakery ',' 875 s. figueroa st. downtown ',la,213-627-6879,diners,583
+50,` parkway grill ',' 510 s. arroyo pkwy . ',pasadena,818-795-1001,californian,584
+51,` pho hoa ',' 642 broadway ',chinatown,213-626-5530,vietnamese,585
+52,` pink \ 's famous chili dogs ',' 709 n. la brea ave. ',la,213-931-4223,` hot dogs ',586
+53,` poquito mas ',' 2635 w. olive ave. ',burbank,818-563-2252,mexican,587
+54,r-23,' 923 e. third st. ',` los angeles ',213-687-7178,japanese,588
+55,` rae \ 's ',' 2901 pico blvd. ',` santa monica ',310-828-7937,diners,589
+56,` rubin \ 's red hots ',' 15322 ventura blvd. ',encino,818-905-6515,` hot dogs ',590
+57,` ruby \ 's ( la ) ','45 s. fair oaks ave. ',pasadena,818-796-7829,diners,591
+58,` russell \ 's burgers ',' 1198 pch ',` seal beach ',310-596-9556,hamburgers,592
+59,` ruth \ 's chris steak house ( los angeles ) ',' 224 s. beverly dr. ',` beverly hills ',310-859-8744,steakhouses,593
+60,shiro,' 1505 mission st. s. ',pasadena,818-799-4774,` pacific new wave ',594
+61,` sushi nozawa ',' 11288 ventura blvd. ',` studio city ',818-508-7017,japanese,595
+62,` sweet lady jane ',' 8360 melrose ave. ',la,213-653-7145,desserts,596
+63,taiko,' 11677 san vicente blvd. ',brentwood,310-207-7782,` noodle shops ',597
+64,` tommy \ 's ',' 2575 beverly blvd. ',la,213-389-9060,hamburgers,598
+65,` uncle bill \ 's pancake house ',' 1305 highland ave. ',` manhattan beach ',310-545-5177,diners,599
+66,` water grill ',' 544 s. grand ave. ',` los angeles ',213-891-0900,seafood,600
+67,` zankou chicken ',' 1415 e. colorado st. ',glendale,818-244-1937,` middle eastern ',601
+68,` afghan kebab house ',' 764 ninth ave. ',` new york city ',212-307-1612,afghan,602
+69,arcadia,'21 e. 62nd st. ',` new york city ',212-223-2900,` american ( new ) ',603
+70,` benny \ 's burritos ','93 ave. a ',` new york city ',212-254-2054,mexican,604
+71,` cafe con leche ',' 424 amsterdam ave. ',` new york city ',212-595-7000,cuban,605
+72,` corner bistro ',' 331 w. fourth st. ',` new york city ',212-242-9502,hamburgers,606
+73,` cucina della fontana ',' 368 bleecker st. ',` new york city ',212-242-0636,italian,607
+74,` cucina di pesce ','87 e. fourth st. ',` new york city ',212-260-6800,seafood,608
+75,darbar,'44 w. 56th st. ',` new york city ',212-432-7227,indian,609
+76,` ej \ 's luncheonette ',' 432 sixth ave. ',` new york city ',212-473-5555,diners,610
+77,` edison cafe ',' 228 w. 47th st. ',` new york city ',212-840-5000,diners,611
+78,` elias corner ',' 24-02 31st st. ',queens,718-932-1510,greek,612
+79,` good enough to eat ',' 483 amsterdam ave. ',` new york city ',212-496-0163,american,613
+80,` gray \ 's papaya ',' 2090 broadway ',` new york city ',212-799-0243,` hot dogs ',614
+81,` il mulino ','86 w. third st. ',` new york city ',212-673-3783,italian,615
+82,` jackson diner ',' 37-03 74th st. ',queens,718-672-1232,indian,616
+83,` joe \ 's shanghai ',' 9 pell st. ',queens,718-539-3838,chinese,617
+84,` john \ 's pizzeria ','48 w. 65th st. ',` new york city ',212-721-7001,pizza,618
+85,` kelley & ping ',' 127 greene st. ',` new york city ',212-228-1212,pan-asian,619
+86,kiev,' 117 second ave. ',` new york city ',212-674-4040,ukrainian,620
+87,` kuruma zushi ',' 2nd fl . ',` new york city ',212-317-2802,japanese,621
+88,` la caridad ',' 2199 broadway ',` new york city ',212-874-2780,cuban,622
+89,` la grenouille ',' 3 e. 52nd st. ',` new york city ',212-752-1495,` french ( classic ) ',623
+90,` lemongrass grill ',' 61a seventh ave. ',brooklyn,718-399-7100,thai,624
+91,` lombardi \ 's ','32 spring st. ',` new york city ',212-941-7994,pizza,625
+92,` marnie \ 's noodle shop ',' 466 hudson st. ',` new york city ',212-741-3214,asian,626
+93,menchanko-tei,'39 w. 55th st. ',` new york city ',212-247-1585,japanese,627
+94,` mitali east-west ',' 296 bleecker st. ',` new york city ',212-989-1367,indian,628
+95,` monsoon ( ny ) ',' 435 amsterdam ave. ',` new york city ',212-580-8686,thai,629
+96,moustache,' 405 atlantic ave. ',brooklyn,718-852-5555,` middle eastern ',630
+97,nobu,' 105 hudson st. ',` new york city ',212-219-0500,japanese,631
+98,` one if by land tibs ','17 barrow st. ',` new york city ',212-228-0822,continental,632
+99,` oyster bar ',` lower level ',` new york city ',212-490-6650,seafood,633
+100,palm,' 837 second ave. ',` new york city ',212-687-2953,steakhouses,634
+101,` palm too ',' 840 second ave. ',` new york city ',212-697-5198,steakhouses,635
+102,` patsy \ 's pizza ','19 old fulton st. ',brooklyn,718-858-4300,pizza,636
+103,` peter luger steak house ',' 178 broadway ',brooklyn,718-387-7400,steakhouses,637
+104,` rose of india ',' 308 e. sixth st. ',` new york city ',212-533-5011,indian,638
+105,` sam \ 's noodle shop ',' 411 third ave. ',` new york city ',212-213-2288,chinese,639
+106,` sarabeth \ 's ',' 1295 madison ave. ',` new york city ',212-410-7335,american,640
+107,` sparks steak house ',' 210 e. 46th st. ',` new york city ',212-687-4855,steakhouses,641
+108,` stick to your ribs ',' 5-16 51st ave. ',queens,718-937-3030,bbq,642
+109,sushisay,'38 e. 51st st. ',` new york city ',212-755-1780,japanese,643
+110,` sylvia \ 's ',' 328 lenox ave. ',` new york city ',212-996-0660,southern/soul,644
+111,` szechuan hunan cottage ',' 1588 york ave. ',` new york city ',212-535-5223,chinese,645
+112,` szechuan kitchen ',' 1460 first ave. ',` new york city ',212-249-4615,chinese,646
+113,` teresa \ 's ','80 montague st. ',queens,718-520-2910,polish,647
+114,` thai house cafe ',' 151 hudson st. ',` new york city ',212-334-1085,thai,648
+115,` thailand restaurant ',' 106 bayard st. ',` new york city ',212-349-3132,thai,649
+116,veselka,' 144 second ave. ',` new york city ',212-228-9682,ukrainian,650
+117,` westside cottage ',' 689 ninth ave. ',` new york city ',212-245-0800,chinese,651
+118,` windows on the world ',' 107th fl . ',` new york city ',212-524-7000,eclectic,652
+119,` wollensky \ 's grill ',' 205 e. 49th st. ',` new york city ',212-753-0444,steakhouses,653
+120,yama,' 122 e. 17th st. ',` new york city ',212-475-0969,japanese,654
+121,zarela,' 953 second ave. ',` new york city ',212-644-6740,mexican,655
+122,` andre \ 's french restaurant ',' 401 s. 6th st. ',` las vegas ',702-385-5016,` french ( classic ) ',656
+123,` buccaneer bay club ',' 3300 las vegas blvd. s. ',` las vegas ',702-894-7350,continental,657
+124,` buzio \ 's in the rio ',' 3700 w. flamingo rd. ',` las vegas ',702-252-7697,seafood,658
+125,'em eril \ 's new orleans fish house ',' 3799 las vegas blvd. s. ',` las vegas ',702-891-7374,seafood,659
+126,` fiore rotisserie & grille ',' 3700 w. flamingo rd. ',` las vegas ',702-252-7702,italian,660
+127,` hugo \ 's cellar ',' 202 e. fremont st. ',` las vegas ',702-385-4011,continental,661
+128,` madame ching \ 's ',' 3300 las vegas blvd. s. ',` las vegas ',702-894-7111,asian,662
+129,` mayflower cuisinier ',' 4750 w. sahara ave. ',` las vegas ',702-870-8432,chinese,663
+130,` michael \ 's ( las vegas ) ',' 3595 las vegas blvd. s. ',` las vegas ',702-737-7111,continental,664
+131,` monte carlo ',' 3145 las vegas blvd. s. ',` las vegas ',702-733-4524,` french ( new ) ',665
+132,moongate,' 3400 las vegas blvd. s. ',` las vegas ',702-791-7352,chinese,666
+133,` morton \ 's of chicago ( las vegas ) ',' 3200 las vegas blvd. s. ',` las vegas ',702-893-0703,steakhouses,667
+134,` nicky blair \ 's ',' 3925 paradise rd. ',` las vegas ',702-792-9900,italian,668
+135,` piero \ 's restaurant ',' 355 convention center dr. ',` las vegas ',702-369-2305,italian,669
+136,` spago ( las vegas ) ',' 3500 las vegas blvd. s. ',` las vegas ',702-369-6300,californian,670
+137,` steakhouse the ',' 128 e. fremont st. ',` las vegas ',702-382-1600,steakhouses,671
+138,` stefano \ 's ',' 129 fremont st. ',` las vegas ',702-385-7111,italian,672
+139,` sterling brunch ',' 3645 las vegas blvd. s. ',` las vegas ',702-739-4651,eclectic,673
+140,` tre visi ',' 3799 las vegas blvd. s. ',` las vegas ',702-891-7331,italian,674
+141,' 103 west ',' 103 w. paces ferry rd. ',atlanta,404-233-5993,continental,675
+142,` alon \ 's at the terrace ',' 659 peachtree st. ',atlanta,404-724-0444,sandwiches,676
+143,` baker \ 's cajun cafe ',' 1134 euclid ave. ',atlanta,404-223-5039,cajun/creole,677
+144,` barbecue kitchen ',' 1437 virginia ave. ',atlanta,404-766-9906,bbq,678
+145,` bistro the ','56 e. andrews dr. nw ',atlanta,404-231-5733,` french bistro ',679
+146,` bobby & june \ 's kountry kitchen ',' 375 14th st. ',atlanta,404-876-3872,southern/soul,680
+147,` bradshaw \ 's restaurant ',' 2911 s. pharr court ',atlanta,404-261-7015,southern/soul,681
+148,` brookhaven cafe ',' 4274 peachtree rd. ',atlanta,404-231-5907,vegetarian,682
+149,` cafe sunflower ',' 5975 roswell rd. ',atlanta,404-256-1675,` health food ',683
+150,canoe,' 4199 paces ferry rd. ',atlanta,770-432-2663,` american ( new ) ',684
+151,` carey \ 's ',' 1021 cobb pkwy . se ',marietta,770-422-8042,hamburgers,685
+152,` carey \ 's corner ',' 1215 powers ferry rd. ',marietta,770-933-0909,hamburgers,686
+153,chops,'70 w. paces ferry rd. ',atlanta,404-262-2675,steakhouses,687
+154,chopstix,' 4279 roswell rd. ',atlanta,404-255-4868,chinese,688
+155,` deacon burton \ 's soulfood restaurant ',' 1029 edgewood ave. se ',atlanta,404-523-1929,southern/soul,689
+156,eats,' 600 ponce de leon ave. ',atlanta,404-888-9149,italian,690
+157,` flying biscuit the ',' 1655 mclendon ave. ',atlanta,404-687-8888,eclectic,691
+158,frijoleros,' 1031 peachtree st. ne ',atlanta,404-892-8226,tex-mex,692
+159,` greenwood \ 's ',' 1087 green st. ',roswell,770-992-5383,southern/soul,693
+160,` harold \ 's barbecue ',' 171 mcdonough blvd. ',atlanta,404-627-9268,bbq,694
+161,` havana sandwich shop ',' 2905 buford hwy . ',atlanta,404-636-4094,cuban,695
+162,` house of chan ',' 2469 cobb pkwy . ',smyrna,770-955-9444,chinese,696
+163,` indian delights ',' 3675 satellite blvd. ',duluth,100-813-8212,indian,697
+164,` java jive ',' 790 ponce de leon ave. ',atlanta,404-876-6161,` coffee shops ',698
+165,` johnny rockets ( at ) ',' 2970 cobb pkwy . ',atlanta,770-955-6068,american,699
+166,` kalo \ 's coffee house ',' 1248 clairmont rd. ',decatur,404-325-3733,coffeehouses,700
+167,` la fonda latina ',' 4427 roswell rd. ',atlanta,404-303-8201,spanish,701
+168,` lettuce souprise you ( at ) ',' 3525 mall blvd. ',duluth,770-418-9969,cafeterias,702
+169,majestic,' 1031 ponce de leon ave. ',atlanta,404-875-0276,diners,703
+170,` morton \ 's of chicago ( atlanta ) ',' 303 peachtree st. ne ',atlanta,404-577-4366,steakhouses,704
+171,` my thai ',' 1248 clairmont rd. ',atlanta,404-636-4280,thai,705
+172,nava,' 3060 peachtree rd. ',atlanta,404-240-1984,southwestern,706
+173,` nuevo laredo cantina ',' 1495 chattahoochee ave. nw ',atlanta,404-352-9009,mexican,707
+174,` original pancake house ( at ) ',' 4330 peachtree rd. ',atlanta,404-237-4116,american,708
+175,` palm the ( atlanta ) ',' 3391 peachtree rd. ne ',atlanta,404-814-1955,steakhouses,709
+176,` rainbow restaurant ',' 2118 n. decatur rd. ',decatur,404-633-3538,vegetarian,710
+177,riviera,' 519 e. paces ferry rd. ',atlanta,404-262-7112,mediterranean,712
+178,` silver skillet the ',' 200 14th st. nw ',atlanta,404-874-1388,` coffee shops ',713
+179,soto,' 3330 piedmont rd. ',atlanta,404-233-2005,japanese,714
+180,` thelma \ 's kitchen ',' 764 marietta st. nw ',atlanta,404-688-5855,cafeterias,715
+181,tortillas,' 774 ponce de leon ave. ne ',atlanta,404-892-0193,tex-mex,716
+182,` van gogh \ 's restaurant & bar ','70 w. crossville rd. ',roswell,770-993-1156,` american ( new ) ',717
+183,veggieland,' 220 sandy springs circle ',atlanta,404-231-3111,vegetarian,718
+184,` white house restaurant ',' 3172 peachtree rd. ne ',atlanta,404-237-7601,diners,719
+185,zab-e-lee,' 4837 old national hwy . ',` college park ',404-768-2705,thai,720
+186,` bill \ 's place ',' 2315 clement st. ',` san francisco ',415-221-5262,hamburgers,721
+187,` cafe flore ',' 2298 market st. ',` san francisco ',415-621-8579,californian,722
+188,` caffe greco ',' 423 columbus ave. ',` san francisco ',415-397-6261,continental,723
+189,` campo santo ',' 240 columbus ave. ',` san francisco ',415-433-9623,mexican,724
+190,` cha cha cha \ 's ',' 1805 haight st. ',` san francisco ',415-386-5758,caribbean,725
+191,` doidge \ 's ',' 2217 union st. ',` san francisco ',415-921-2149,american,726
+192,` dottie \ 's true blue cafe ',' 522 jones st. ',` san francisco ',415-885-2767,diners,727
+193,` dusit thai ',' 3221 mission st. ',` san francisco ',415-826-4639,thai,728
+194,ebisu,' 1283 ninth ave. ',` san francisco ',415-566-1770,japanese,729
+195,'em erald garden restaurant ',' 1550 california st. ',` san francisco ',415-673-1155,vietnamese,730
+196,` eric \ 's chinese restaurant ',' 1500 church st. ',` san francisco ',415-282-0919,chinese,731
+197,` hamburger mary \ 's ',' 1582 folsom st. ',` san francisco ',415-626-1985,hamburgers,732
+198,` kelly \ 's on trinity ',' 333 bush st. ',` san francisco ',415-362-4454,californian,733
+199,` la cumbre ',' 515 valencia st. ',` san francisco ',415-863-8205,mexican,734
+200,` la mediterranee ',' 288 noe st. ',` san francisco ',415-431-7210,mediterranean,735
+201,` la taqueria ',' 2889 mission st. ',` san francisco ',415-285-7117,mexican,736
+202,` mario \ 's bohemian cigar store cafe ',' 2209 polk st. ',` san francisco ',415-776-8226,italian,737
+203,` marnee thai ',' 2225 irving st. ',` san francisco ',415-665-9500,thai,738
+204,` mel \ 's drive-in ',' 3355 geary st. ',` san francisco ',415-387-2244,hamburgers,739
+205,` mo \ 's burgers ',' 1322 grant st. ',` san francisco ',415-788-3779,hamburgers,740
+206,` phnom penh cambodian restaurant ',' 631 larkin st. ',` san francisco ',415-775-5979,cambodian,741
+207,` roosevelt tamale parlor ',' 2817 24th st. ',` san francisco ',415-550-9213,mexican,742
+208,` sally \ 's cafe & bakery ',' 300 de haro st. ',` san francisco ',415-626-6006,american,743
+209,` san francisco bbq ',' 1328 18th st. ',` san francisco ',415-431-8956,thai,744
+210,` slanted door ',' 584 valencia st. ',` san francisco ',415-861-8032,vietnamese,745
+211,` swan oyster depot ',' 1517 polk st. ',` san francisco ',415-673-1101,seafood,746
+212,` thep phanom ',' 400 waller st. ',` san francisco ',415-431-2526,thai,747
+213,` ti couz ',' 3108 16th st. ',` san francisco ',415-252-7373,french,748
+214,` trio cafe ',' 1870 fillmore st. ',` san francisco ',415-563-2248,american,749
+215,` tu lan ',' 8 sixth st. ',` san francisco ',415-626-0927,vietnamese,750
+216,` vicolo pizzeria ',' 201 ivy st. ',` san francisco ',415-863-2382,pizza,751
+217,` wa-ha-ka oaxaca mexican grill ',' 2141 polk st. ',` san francisco ',415-775-1055,mexican,752
+218,` arnie morton \ 's of chicago ',' 435 s. la cienega blvd. ',` los angeles ',310-246-1501,steakhouses,0
+219,` art \ 's deli ',' 12224 ventura blvd. ',` studio city ',818-762-1221,delis,1
+220,` bel-air hotel ',' 701 stone canyon rd. ',` bel air ',310-472-1211,californian,2
+221,` cafe bizou ',' 14016 ventura blvd. ',` sherman oaks ',818-788-3536,` french bistro ',3
+222,campanile,' 624 s. la brea ave. ',` los angeles ',213-938-1447,californian,4
+223,` chinois on main ',' 2709 main st. ',` santa monica ',310-392-9025,` pacific new wave ',5
+224,citrus,' 6703 melrose ave. ',` los angeles ',213-857-0034,californian,6
+225,` fenix at the argyle ',' 8358 sunset blvd. ',` w. hollywood ',213-848-6677,` french ( new ) ',7
+226,granita,' 23725 w. malibu rd. ',malibu,310-456-0488,californian,8
+227,` grill the ',' 9560 dayton way ',` beverly hills ',310-276-0615,` american ( traditional ) ',9
+228,katsu,' 1972 hillhurst ave. ',` los feliz ',213-665-1891,japanese,10
+229,` l \ ` orangerie ',' 903 n. la cienega blvd. ',` w. hollywood ',310-652-9770,` french ( classic ) ',11
+230,` le chardonnay ( los angeles ) ',' 8284 melrose ave. ',` los angeles ',213-655-8880,` french bistro ',12
+231,` locanda veneta ',' 8638 w. third st. ',` los angeles ',310-274-1893,italian,13
+232,matsuhisa,' 129 n. la cienega blvd. ',` beverly hills ',310-659-9639,seafood,14
+233,` palm the ( los angeles ) ',' 9001 santa monica blvd. ',` w. hollywood ',310-550-8811,steakhouses,15
+234,patina,' 5955 melrose ave. ',` los angeles ',213-467-1108,californian,16
+235,` philippe the original ',' 1001 n. alameda st. ',chinatown,213-628-3781,cafeterias,17
+236,` pinot bistro ',' 12969 ventura blvd. ',` studio city ',818-990-0500,` french bistro ',18
+237,` rex il ristorante ',' 617 s. olive st. ',` los angeles ',213-627-2300,` nuova cucina italian ',19
+238,` spago ( los angeles ) ',' 8795 sunset blvd. ',` w. hollywood ',310-652-4025,californian,20
+239,valentino,' 3115 pico blvd. ',` santa monica ',310-829-4313,italian,21
+240,` yujean kang \ 's ','67 n. raymond ave. ',pasadena,818-585-0855,chinese,22
+241,'21 club ','21 w. 52nd st. ',` new york city ',212-582-7200,` american ( new ) ',23
+242,aquavit,'13 w. 54th st. ',` new york city ',212-307-7311,scandinavian,24
+243,aureole,'34 e. 61st st. ',` new york city ',212-319-1660,` american ( new ) ',25
+244,` cafe lalo ',' 201 w. 83rd st. ',` new york city ',212-496-6031,coffeehouses,26
+245,` cafe des artistes ',' 1 w. 67th st. ',` new york city ',212-877-3500,` french ( classic ) ',27
+246,` carmine \ 's ',' 2450 broadway ',` new york city ',212-362-2200,italian,28
+247,` carnegie deli ',' 854 seventh ave. ',` new york city ',212-757-2245,delis,29
+248,chanterelle,' 2 harrison st. ',` new york city ',212-966-6960,` french ( new ) ',30
+249,daniel,'20 e. 76th st. ',` new york city ',212-288-0033,` french ( new ) ',31
+250,dawat,' 210 e. 58th st. ',` new york city ',212-355-7555,indian,32
+251,felidia,' 243 e. 58th st. ',` new york city ',212-758-1479,italian,33
+252,` four seasons ','99 e. 52nd st. ',` new york city ',212-754-9494,` american ( new ) ',34
+253,` gotham bar & grill ','12 e. 12th st. ',` new york city ',212-620-4020,` american ( new ) ',35
+254,` gramercy tavern ','42 e. 20th st. ',` new york city ',212-477-0777,` american ( new ) ',36
+255,` island spice ',' 402 w. 44th st. ',` new york city ',212-765-1737,caribbean,37
+256,` jo jo ',' 160 e. 64th st. ',` new york city ',212-223-5656,` french bistro ',38
+257,` la caravelle ','33 w. 55th st. ',` new york city ',212-586-4252,` french ( classic ) ',39
+258,` la cote basque ','60 w. 55th st. ',` new york city ',212-688-6525,` french ( classic ) ',40
+259,` le bernardin ',' 155 w. 51st st. ',` new york city ',212-489-1515,seafood,41
+260,` les celebrites ',' 155 w. 58th st. ',` new york city ',212-484-5113,` french ( classic ) ',42
+261,` lespinasse ( new york city ) ',' 2 e. 55th st. ',` new york city ',212-339-6719,asian,43
+262,lutece,' 249 e. 50th st. ',` new york city ',212-752-2225,` french ( classic ) ',44
+263,` manhattan ocean club ','57 w. 58th st. ',` new york city ',212-371-7777,seafood,45
+264,march,' 405 e. 58th st. ',` new york city ',212-754-6272,` american ( new ) ',46
+265,` mesa grill ',' 102 fifth ave. ',` new york city ',212-807-7400,southwestern,47
+266,` mi cocina ','57 jane st. ',` new york city ',212-627-8273,mexican,48
+267,montrachet,' 239 w. broadway ',` new york city ',212-219-2777,` french bistro ',49
+268,oceana,'55 e. 54th st. ',` new york city ',212-759-5941,seafood,50
+269,` park avenue cafe ( new york city ) ',' 100 e. 63rd st. ',` new york city ',212-644-1900,` american ( new ) ',51
+270,petrossian,' 182 w. 58th st. ',` new york city ',212-245-2214,russian,52
+271,picholine,'35 w. 64th st. ',` new york city ',212-724-8585,mediterranean,53
+272,pisces,'95 ave. a ',` new york city ',212-260-6660,seafood,54
+273,` rainbow room ','30 rockefeller plaza ',` new york city ',212-632-5000,` american ( new ) ',55
+274,` river cafe ',' 1 water st. ',brooklyn,718-522-5200,` american ( new ) ',56
+275,` san domenico ',' 240 central park s. ',` new york city ',212-265-5959,italian,57
+276,` second avenue deli ',' 156 second ave. ',` new york city ',212-677-0606,delis,58
+277,seryna,'11 e. 53rd st. ',` new york city ',212-980-9393,japanese,59
+278,` shun lee palace ',' 155 e. 55th st. ',` new york city ',212-371-8844,chinese,60
+279,` sign of the dove ',' 1110 third ave. ',` new york city ',212-861-8080,` american ( new ) ',61
+280,` smith & wollensky ',' 797 third ave. ',` new york city ',212-753-1530,steakhouses,62
+281,` tavern on the green ',` central park west ',` new york city ',212-873-3200,` american ( new ) ',63
+282,` uncle nick \ 's ',' 747 ninth ave. ',` new york city ',212-245-7992,greek,64
+283,` union square cafe ','21 e. 16th st. ',` new york city ',212-243-4020,` american ( new ) ',65
+284,` virgil \ 's real bbq ',' 152 w. 44th st. ',` new york city ',212-921-9494,bbq,66
+285,` chin \ 's ',' 3200 las vegas blvd. s. ',` las vegas ',702-733-8899,chinese,67
+286,` coyote cafe ( las vegas ) ',' 3799 las vegas blvd. s. ',` las vegas ',702-891-7349,southwestern,68
+287,` le montrachet bistro ',' 3000 paradise rd. ',` las vegas ',702-732-5651,` french bistro ',69
+288,` palace court ',' 3570 las vegas blvd. s. ',` las vegas ',702-731-7110,` french ( new ) ',70
+289,` second street grill ',' 200 e. fremont st. ',` las vegas ',702-385-6277,` pacific rim ',71
+290,` steak house the ',' 2880 las vegas blvd. s. ',` las vegas ',702-734-0410,steakhouses,72
+291,'till erman the ',' 2245 e. flamingo rd. ',` las vegas ',702-731-4036,steakhouses,73
+292,abruzzi,' 2355 peachtree rd. ne ',atlanta,404-261-8186,italian,74
+293,bacchanalia,' 3125 piedmont rd. ',atlanta,404-365-0410,californian,75
+294,` bone \ 's restaurant ',' 3130 piedmont rd. ne ',atlanta,404-237-2663,steakhouses,76
+295,` brasserie le coze ',' 3393 peachtree rd. ',atlanta,404-266-1440,` french bistro ',77
+296,` buckhead diner ',' 3073 piedmont rd. ',atlanta,404-262-3336,` american ( new ) ',78
+297,` ciboulette restaurant ',' 1529 piedmont ave. ',atlanta,404-874-7600,` french ( new ) ',79
+298,delectables,' 1 margaret mitchell sq. ',atlanta,404-681-2909,cafeterias,80
+299,` georgia grille ',' 2290 peachtree rd. ',atlanta,404-352-3517,southwestern,81
+300,` hedgerose heights inn the ',' 490 e. paces ferry rd. ne ',atlanta,404-233-7673,continental,82
+301,` heera of india ',' 595 piedmont ave. ',atlanta,404-876-4408,indian,83
+302,` indigo coastal grill ',' 1397 n. highland ave. ',atlanta,404-876-0676,eclectic,84
+303,` la grotta ',' 2637 peachtree rd. ne ',atlanta,404-231-1368,italian,85
+304,` mary mac \ 's tea room ',' 224 ponce de leon ave. ',atlanta,404-876-1800,southern/soul,86
+305,` nikolai \ 's roof ',' 255 courtland st. ',atlanta,404-221-6362,continental,87
+306,` pano \ 's & paul \ 's ',' 1232 w. paces ferry rd. ',atlanta,404-261-3662,` american ( new ) ',88
+307,` ritz-carlton cafe ( buckhead ) ',' 3434 peachtree rd. ne ',atlanta,404-237-2700,` american ( new ) ',89
+308,` ritz-carlton dining room ( buckhead ) ',' 3434 peachtree rd. ne ',atlanta,404-237-2700,` american ( new ) ',90
+309,` ritz-carlton restaurant ',' 181 peachtree st. ',atlanta,404-659-0400,` french ( classic ) ',91
+310,toulouse,' 293-b peachtree rd. ',atlanta,404-351-9533,` french ( new ) ',92
+311,` veni vidi vici ','41 14th st. ',atlanta,404-875-8424,italian,93
+312,` alain rondelli ',' 126 clement st. ',` san francisco ',415-387-0408,` french ( new ) ',94
+313,aqua,' 252 california st. ',` san francisco ',415-956-9662,` american ( new ) ',95
+314,boulevard,' 1 mission st. ',` san francisco ',415-543-6084,` american ( new ) ',96
+315,` cafe claude ',' 7 claude ln . ',` san francisco ',415-392-3505,` french bistro ',97
+316,` campton place ',' 340 stockton st. ',` san francisco ',415-955-5555,` american ( new ) ',98
+317,` chez michel ',' 804 north point st. ',` san francisco ',415-775-7036,californian,99
+318,` fleur de lys ',' 777 sutter st. ',` san francisco ',415-673-7779,` french ( new ) ',100
+319,fringale,' 570 fourth st. ',` san francisco ',415-543-0573,` french bistro ',101
+320,` hawthorne lane ','22 hawthorne st. ',` san francisco ',415-777-9779,californian,102
+321,` khan toke thai house ',' 5937 geary blvd. ',` san francisco ',415-668-6654,thai,103
+322,` la folie ',' 2316 polk st. ',` san francisco ',415-776-5577,` french ( new ) ',104
+323,` lulu restaurant-bis-cafe ',' 816 folsom st. ',` san francisco ',415-495-5775,mediterranean,105
+324,` masa \ 's ',' 648 bush st. ',` san francisco ',415-989-7154,` french ( new ) ',106
+325,mifune,' 1737 post st. ',` san francisco ',415-922-0337,japanese,107
+326,` plumpjack cafe ',' 3127 fillmore st. ',` san francisco ',415-563-4755,` american ( new ) ',108
+327,postrio,' 545 post st. ',` san francisco ',415-776-7825,californian,109
+328,` ritz-carlton dining room ( san francisco ) ',' 600 stockton st. ',` san francisco ',415-296-7465,` french ( new ) ',110
+329,` rose pistola ',' 532 columbus ave. ',` san francisco ',415-399-0499,italian,111
+330,` ritz-carlton cafe ( atlanta ) ',' 181 peachtree st. ',atlanta,404-659-0400,` american ( new ) ',711

datasets/Walmart-Amazon_dirty/matches.csv

@@ -0,0 +1,963 @@
+ltable_id,rtable_id
+3,4378
+5,20932
+6,9015
+7,1303
+15,4282
+15,17364
+15,17810
+19,17429
+20,16837
+25,6984
+29,6031
+30,11088
+33,11683
+34,3067
+39,10872
+41,6816
+42,172
+49,8096
+53,17892
+54,6572
+56,9164
+56,21964
+57,13302
+64,8391
+65,7156
+65,11129
+66,3668
+69,14641
+78,12347
+80,6526
+81,17194
+86,11617
+86,7610
+88,18980
+88,3269
+90,21917
+91,17687
+93,8350
+94,9051
+95,10966
+97,15904
+99,4520
+101,2593
+102,11759
+107,12090
+108,21709
+108,12441
+114,9223
+120,14795
+128,2701
+128,10640
+130,9890
+131,5595
+134,3740
+136,2576
+137,21120
+138,11468
+144,7970
+145,15891
+147,21192
+148,16398
+148,16350
+148,18936
+151,691
+155,3446
+157,16205
+159,9709
+159,5664
+164,18937
+172,1152
+173,1249
+178,19149
+179,8545
+181,12012
+182,7399
+184,2959
+186,1928
+191,1243
+192,11131
+194,6876
+194,20939
+199,5511
+201,6828
+205,10517
+211,400
+215,6601
+217,5274
+218,5046
+219,8696
+221,16624
+222,1354
+222,17534
+225,12627
+229,13258
+231,10436
+235,21783
+239,10617
+246,13589
+246,19031
+258,10646
+261,5192
+264,16640
+266,2949
+272,3965
+276,15426
+281,7824
+284,2675
+286,18626
+294,19911
+297,15653
+302,16807
+302,16641
+303,19299
+309,19291
+310,4720
+312,13299
+313,8870
+315,7705
+317,1082
+319,21483
+322,9990
+323,1244
+324,8519
+330,1599
+334,10661
+336,20174
+337,6115
+345,8386
+346,8304
+351,10048
+356,21591
+360,19150
+360,11566
+362,16307
+370,15071
+376,9389
+378,21690
+382,12582
+387,3169
+388,1450
+395,99
+396,7149
+396,7144
+396,17848
+398,16514
+399,713
+413,284
+415,17243
+415,20385
+424,8534
+426,14868
+428,6990
+429,3273
+431,18464
+446,5774
+447,13174
+448,13173
+452,16645
+453,10899
+453,4892
+454,1891
+455,19347
+461,18389
+463,11440
+465,7304
+465,15278
+468,20645
+469,2720
+469,19257
+470,10874
+472,15367
+475,12746
+475,20392
+476,2717
+480,376
+481,14853
+483,9130
+491,19957
+496,4496
+498,5577
+500,12544
+502,11435
+503,10206
+507,11953
+512,9060
+514,14881
+518,2138
+522,14014
+523,3626
+523,7630
+527,485
+528,21347
+528,76
+529,17879
+530,18531
+530,6615
+531,19817
+537,790
+538,17174
+539,11542
+540,9383
+541,9189
+542,13159
+544,13664
+545,7624
+545,7622
+547,9295
+548,3414
+550,19539
+550,2555
+555,19009
+558,5883
+559,20713
+559,21310
+561,18152
+563,18915
+575,1128
+575,5075
+577,21736
+577,19367
+579,15015
+584,965
+585,14867
+588,20108
+589,16953
+589,15495
+591,5594
+594,20843
+595,14869
+597,22043
+598,3626
+599,556
+600,4691
+600,12881
+604,15829
+611,8429
+614,22045
+615,17519
+617,19540
+625,18443
+630,12817
+632,13149
+635,7145
+636,9151
+639,7371
+639,10128
+645,6760
+648,4759
+652,7445
+652,6708
+656,10558
+658,21942
+660,10273
+662,17258
+663,7135
+669,12490
+678,8962
+680,843
+681,20251
+684,17573
+687,20212
+688,4783
+692,18704
+693,2611
+693,104
+694,20006
+695,12569
+699,4283
+699,3856
+705,16536
+705,12895
+707,11829
+707,8042
+709,1096
+714,16478
+715,7850
+717,20106
+719,6972
+720,13398
+722,12995
+723,9675
+725,16800
+725,16277
+734,5991
+735,12273
+737,6759
+738,18197
+739,11687
+741,11537
+742,8235
+745,11810
+752,7075
+759,8381
+765,15415
+769,6362
+770,11223
+771,5517
+781,17483
+784,21196
+784,12733
+785,10845
+788,14901
+797,6508
+801,13029
+801,8931
+803,4121
+805,14294
+810,1480
+810,11428
+811,453
+816,11998
+818,979
+828,10887
+829,8897
+832,19556
+832,4597
+836,8739
+838,715
+841,7503
+848,3153
+848,602
+853,6747
+857,7347
+861,11942
+862,5782
+864,11894
+870,14930
+871,20288
+872,15268
+874,5644
+875,4226
+887,12783
+892,21096
+893,11620
+895,12463
+898,7383
+898,11343
+898,15335
+902,6661
+903,16989
+905,19636
+906,3469
+907,13994
+908,13023
+916,8910
+917,17317
+919,17371
+920,3246
+925,21757
+926,4426
+931,6451
+932,2813
+934,13148
+935,14800
+936,14004
+938,15252
+939,3615
+941,2805
+944,14177
+949,7346
+950,15942
+955,4702
+961,14317
+973,21568
+974,19753
+975,13422
+979,9553
+983,18364
+986,6669
+991,12312
+992,19592
+996,2750
+997,4379
+999,4923
+1007,18142
+1007,11116
+1009,780
+1010,15003
+1011,13413
+1014,7028
+1022,12116
+1024,12674
+1029,16791
+1030,10555
+1031,13438
+1037,12168
+1038,5010
+1044,3009
+1066,8297
+1070,18620
+1071,16126
+1073,2522
+1074,20740
+1077,11884
+1080,14454
+1081,21587
+1083,20644
+1090,16634
+1092,13747
+1093,5462
+1093,16869
+1102,7853
+1104,1397
+1107,12700
+1108,10672
+1109,8091
+1119,3471
+1121,14109
+1125,8267
+1134,8789
+1136,17383
+1137,4429
+1138,9085
+1140,9097
+1141,9253
+1145,4787
+1145,2867
+1148,16756
+1149,9481
+1151,18330
+1153,9237
+1153,13245
+1155,14873
+1159,13109
+1165,3695
+1177,20811
+1178,18153
+1183,11288
+1186,16092
+1187,5479
+1189,2759
+1190,10706
+1193,4493
+1193,5592
+1194,14916
+1195,5364
+1197,4814
+1200,159
+1207,6898
+1208,16935
+1208,3254
+1208,18170
+1209,9717
+1209,14109
+1210,1324
+1217,5249
+1218,5267
+1224,8070
+1229,5842
+1230,4667
+1233,18428
+1233,12407
+1236,7615
+1237,6974
+1242,2465
+1244,10288
+1246,21594
+1246,17191
+1248,16298
+1252,9635
+1256,20411
+1262,12734
+1263,22051
+1266,4232
+1268,4380
+1280,17949
+1280,7637
+1282,1240
+1286,16535
+1287,15033
+1291,17829
+1292,18602
+1295,6405
+1298,8455
+1299,14223
+1299,15462
+1301,6494
+1305,2558
+1311,14760
+1315,15122
+1315,7783
+1315,21112
+1316,4506
+1317,20506
+1320,884
+1324,912
+1325,10507
+1334,8618
+1338,17543
+1340,8496
+1345,20145
+1345,14257
+1349,4659
+1349,12191
+1353,6819
+1355,559
+1356,11269
+1359,1903
+1360,994
+1362,17388
+1365,18372
+1367,12146
+1370,5172
+1377,12161
+1380,20890
+1385,17536
+1391,2976
+1394,19580
+1396,2438
+1397,20805
+1398,4586
+1400,14943
+1400,18727
+1402,20183
+1405,14894
+1412,16818
+1413,2788
+1417,7248
+1418,14715
+1423,9727
+1424,8192
+1426,10692
+1437,11704
+1439,19654
+1441,15284
+1451,4719
+1454,10760
+1456,18231
+1457,8346
+1460,14498
+1466,15575
+1466,13592
+1466,11641
+1467,16059
+1473,3250
+1473,3248
+1474,3961
+1475,8650
+1475,12654
+1476,11083
+1477,2432
+1478,1766
+1481,4162
+1482,8566
+1484,15558
+1485,1375
+1489,2077
+1498,5234
+1499,4637
+1504,4817
+1506,16666
+1508,19514
+1509,9332
+1511,4519
+1512,20844
+1513,1592
+1522,1025
+1522,21029
+1525,12543
+1528,16916
+1528,811
+1529,1441
+1534,1090
+1535,18999
+1539,4756
+1542,10127
+1545,21374
+1549,18948
+1550,11326
+1552,9156
+1558,12852
+1559,14553
+1560,19669
+1562,12169
+1563,430
+1564,6914
+1571,6998
+1573,5415
+1577,13540
+1578,3323
+1579,11460
+1584,11475
+1589,2645
+1591,3205
+1592,16658
+1595,13390
+1602,10608
+1602,19807
+1606,18687
+1607,13172
+1609,2509
+1610,10450
+1613,17218
+1614,11350
+1618,6364
+1621,16058
+1624,5013
+1626,13619
+1628,6288
+1632,20070
+1637,1270
+1640,9564
+1641,15009
+1641,11044
+1643,2779
+1651,13804
+1652,14384
+1653,1649
+1656,14169
+1657,16332
+1660,11172
+1666,6194
+1671,20794
+1684,17948
+1686,20038
+1686,17665
+1694,11026
+1694,20397
+1694,3793
+1695,17274
+1697,16578
+1697,21667
+1699,17103
+1701,18549
+1703,17701
+1704,16231
+1705,14645
+1706,17026
+1708,16226
+1713,21093
+1714,18512
+1716,17396
+1717,17261
+1718,8845
+1720,20522
+1726,18463
+1727,14918
+1728,12125
+1732,5720
+1733,21812
+1738,20132
+1742,15527
+1751,20117
+1753,16220
+1758,13845
+1758,9789
+1761,12647
+1761,4576
+1766,5888
+1771,235
+1774,18402
+1776,16437
+1776,15489
+1776,13179
+1777,3750
+1784,11403
+1789,11778
+1790,19800
+1791,19538
+1792,8494
+1802,14924
+1803,6872
+1810,577
+1813,3253
+1813,1607
+1827,2578
+1829,19353
+1832,432
+1832,13396
+1833,9342
+1834,16331
+1837,13677
+1841,18623
+1841,2326
+1843,4900
+1846,15547
+1850,17488
+1851,13348
+1860,5560
+1861,18357
+1862,15434
+1863,21325
+1866,14359
+1868,4494
+1875,9386
+1877,2927
+1882,10854
+1883,19123
+1884,17578
+1888,6729
+1891,15304
+1893,11120
+1895,7660
+1899,5458
+1901,10882
+1903,10674
+1904,15684
+1905,7800
+1907,10481
+1908,20111
+1919,543
+1922,20240
+1927,19026
+1930,15375
+1931,11214
+1932,9095
+1932,9016
+1934,9578
+1936,18140
+1936,11295
+1937,8500
+1943,15780
+1944,828
+1946,1315
+1949,17302
+1951,20443
+1952,1769
+1953,11438
+1957,17570
+1960,2284
+1963,6961
+1964,14803
+1966,656
+1968,21511
+1972,3590
+1972,3593
+1973,3155
+1975,8180
+1976,12093
+1980,4278
+1983,6913
+1984,1120
+1986,6413
+1987,10905
+1987,16252
+1988,2476
+1988,9474
+1992,1274
+1997,12451
+1999,20189
+2000,14202
+2004,19520
+2004,1747
+2006,13921
+2007,20461
+2013,9226
+2014,11740
+2014,9456
+2017,16793
+2018,11227
+2018,5207
+2019,16543
+2021,17451
+2021,17374
+2024,18710
+2028,12891
+2029,14035
+2030,13167
+2033,18679
+2037,8519
+2037,21543
+2039,7443
+2040,9550
+2047,8524
+2050,21912
+2052,6286
+2054,7601
+2056,3067
+2061,352
+2063,6308
+2069,21448
+2071,8814
+2074,17803
+2076,18297
+2078,3894
+2080,16295
+2083,14539
+2087,11143
+2088,22025
+2088,17263
+2089,7048
+2092,13767
+2093,16311
+2094,15247
+2096,2974
+2100,20731
+2101,15843
+2103,2963
+2106,1253
+2107,8358
+2118,20621
+2119,20314
+2121,4432
+2123,1794
+2125,16072
+2132,7358
+2133,8990
+2133,10434
+2136,4705
+2136,19006
+2139,101
+2140,6965
+2141,11048
+2142,13934
+2147,2887
+2149,4784
+2150,21343
+2151,7338
+2154,11211
+2155,4509
+2155,17864
+2155,4318
+2156,14545
+2158,9026
+2159,17899
+2160,9858
+2163,8469
+2164,3694
+2165,21076
+2167,17733
+2179,18387
+2184,11689
+2186,13583
+2188,2671
+2190,2310
+2190,21661
+2191,16173
+2192,3144
+2193,9730
+2194,5101
+2196,19484
+2196,17366
+2199,20395
+2199,18138
+2201,17198
+2209,17162
+2210,16560
+2211,4646
+2213,21798
+2216,17494
+2216,20675
+2219,245
+2221,11340
+2224,14686
+2229,14755
+2230,12949
+2231,17839
+2236,1662
+2237,1696
+2239,326
+2244,594
+2245,3566
+2248,8368
+2252,16783
+2253,11265
+2254,345
+2255,14015
+2257,8356
+2259,14834
+2260,13722
+2264,19007
+2265,18698
+2266,1616
+2266,13318
+2267,2527
+2273,5255
+2274,356
+2275,19662
+2276,12760
+2293,6548
+2295,21434
+2296,12892
+2297,10869
+2305,10515
+2315,9548
+2317,15825
+2324,20693
+2326,19268
+2327,8544
+2328,16352
+2331,21976
+2332,18200
+2335,8279
+2339,2236
+2340,3758
+2341,7893
+2342,638
+2343,4784
+2344,4984
+2344,6970
+2344,14944
+2345,15061
+2349,18166
+2353,13476
+2356,5527
+2360,17905
+2360,7249
+2364,16366
+2364,17915
+2365,8618
+2368,15351
+2371,10923
+2374,3051
+2378,11830
+2379,10267
+2379,264
+2381,3482
+2387,20821
+2392,15893
+2394,13251
+2396,18101
+2397,10117
+2405,8668
+2407,12311
+2408,6808
+2409,11714
+2412,7233
+2418,7679
+2422,8663
+2435,20419
+2438,2861
+2440,6643
+2441,21373
+2461,13243
+2462,14117
+2462,8175
+2468,16832
+2468,10813
+2471,19283
+2474,14437
+2476,6955
+2477,4884
+2478,16862
+2479,10669
+2481,6885
+2482,19119
+2483,11841
+2488,5635
+2496,12024
+2497,19525
+2501,4362
+2502,16032
+2503,20051
+2507,5038
+2508,11582
+2510,11625
+2511,21503
+2513,20426
+2515,4424
+2516,20309
+2519,2668
+2522,3324
+2523,10365
+2527,13282
+2531,19976
+2536,12333
+2538,2279
+2539,10705
+2540,5175
+2542,6690
+2543,2533
+2548,19117

datasets/iTunes-Amazon_dirty/matches.csv

@@ -0,0 +1,133 @@
+ltable_id,rtable_id
+110,53087
+147,50730
+205,41179
+249,53087
+251,52967
+251,53047
+254,49104
+416,33321
+426,33282
+431,33296
+433,33307
+467,362
+498,29283
+508,55899
+537,1878
+582,378
+588,55899
+592,43689
+598,43689
+642,24635
+668,24621
+836,23706
+1069,11277
+1148,7757
+1151,53282
+1161,27990
+1164,27994
+1193,28008
+1216,33988
+1358,42278
+1406,48287
+1413,51590
+1489,3898
+1514,49668
+1595,52925
+1636,27530
+1655,49479
+1683,23727
+1683,23781
+1735,10926
+1736,10938
+1776,33397
+1777,20232
+1897,51626
+1918,9358
+1946,23456
+2222,815
+2337,54230
+2368,33786
+2375,3581
+2394,29834
+2399,29849
+2456,24297
+2479,33786
+2490,4511
+2666,35475
+2742,17181
+2749,17161
+2782,31534
+2812,47863
+2834,53255
+2845,52867
+2901,21458
+2913,21458
+3093,22994
+3104,22995
+3292,7440
+3324,7403
+3354,17660
+3354,17643
+3356,39335
+3358,39320
+3376,7414
+3421,34320
+3503,54798
+3507,23084
+3734,47848
+3762,38431
+3771,38442
+3850,9120
+3864,8306
+3870,23889
+3870,23901
+3878,23925
+3887,16397
+3926,16397
+3944,18970
+4301,29695
+4312,6130
+4326,23361
+4384,23361
+4413,42103
+4468,31072
+4547,26140
+4587,26140
+4610,26152
+4659,41090
+4669,41083
+4747,5682
+4823,20588
+5001,3099
+5077,40465
+5083,40471
+5122,34113
+5143,43647
+5227,27254
+5363,47909
+5406,27824
+5518,9828
+5542,10251
+5702,37654
+5712,35057
+5726,4470
+5733,4465
+5761,21517
+5806,6499
+5827,6502
+5885,19151
+6047,4667
+6104,7981
+6218,29583
+6234,13713
+6280,47818
+6425,31102
+6429,31088
+6458,18807
+6532,38301
+6592,38375
+6634,824
+6724,17069
+6764,51438
+6857,39318

deprecated/inference_from_record_pairs.py

@@ -1,152 +0,0 @@
-import pandas as pd
-import time
-import Levenshtein
-import copy
-
-def my_Levenshtein_ratio(str1, str2):
-    return 1 - Levenshtein.distance(str1, str2) / max(len(str1), len(str2))
-
-
-def if_minimal(md, md_list, target_col):
-    # 假设这个md是minimal
-    minimal = True
-    for _ in md_list:
-        if _ != md:
-            # 假设列表中每一个md都使当前md不minimal
-            exist = True
-            # 如果左边任何一个大于，则假设不成立
-            for col in list(set(_.keys()) - {target_col}):
-                if _[col] > md[col]:
-                    exist = False
-            # 如果右边小于，假设也不成立
-            if _[target_col] < md[target_col]:
-                exist = False
-            # 任何一次假设成立，当前md不minimal
-            if exist:
-                minimal = False
-                break
-    return minimal
-
-
-def satisfy_confidence(md, df, conf_thresh, target_col):
-    support = 0
-    support_plus = 0
-    for row1 in df.itertuples():
-        i = row1[0]
-        df_slice = df[i + 1:]
-        for row2 in df_slice.itertuples():
-            left_satisfy = True
-            both_satisfy = True
-            for col in df.columns.values.tolist():
-                sim = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                if col == target_col:
-                    if sim < 1:
-                        both_satisfy = False
-                else:
-                    if sim < md[col]:
-                        left_satisfy = False
-                        both_satisfy = False
-            if left_satisfy:
-                support += 1
-            if both_satisfy:
-                support_plus += 1
-    if support == 0:
-        return False, 0.0
-    confidence = support_plus / support
-    return confidence >= conf_thresh, confidence
-
-
-def inference_from_record_pairs(path, threshold, target_col):
-    data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1')
-    data = data.astype(str)
-    columns = data.columns.values.tolist()
-
-    md_list = []
-    minimal_vio = []
-    init_md = {}
-    for col in columns:
-        init_md[col] = 1 if col == target_col else 0
-    md_list.append(init_md)
-
-    for row1 in data.itertuples():
-        # 获取当前行的索引，从后一行开始切片
-        i = row1[0]
-        data1 = data[i + 1:]
-        for row2 in data1.itertuples():
-            violated_mds = []
-            # sims是两行的相似度
-            sims = {}
-            for col in columns:
-                similarity = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                sims[col] = similarity
-
-            # 寻找violated md,从md列表中删除并加入vio列表
-            for md in md_list:
-                lhs_satis = True
-                rhs_satis = True
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] < md[col]:
-                        lhs_satis = False
-                if sims[target_col] < md[target_col]:
-                    rhs_satis = False
-                if lhs_satis == True and rhs_satis == False:
-                    md_list.remove(md)
-                    violated_mds.append(md)
-            minimal_vio.extend(violated_mds)
-
-            for vio_md in violated_mds:
-                # 特殊化右侧,我们需要右侧百分百相似，其实不需要降低右侧阈值
-                # if sims[target_col] >= threshold:
-                #     new_rhs = sims[target_col]
-                #     spec_r_md = copy.deepcopy(vio_md)
-                #     spec_r_md[target_col] = new_rhs
-                #     if if_minimal(spec_r_md, md_list, target_col):
-                #         md_list.append(spec_r_md)
-
-                # 特殊化左侧
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] + 0.001 <= 1:
-                        spec_l_md = copy.deepcopy(vio_md)
-                        spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.001
-                        if if_minimal(spec_l_md, md_list, target_col):
-                            md_list.append(spec_l_md)
-
-            for vio in minimal_vio:
-                if not if_minimal(vio, md_list, target_col):
-                    minimal_vio.remove(vio)
-
-    tmp = copy.deepcopy(minimal_vio)
-    for _ in tmp:
-        satis, conf = satisfy_confidence(_, data, 0.8, target_col)
-        if not satis:
-            minimal_vio.remove(_)
-
-    for _ in tmp:
-        if not if_minimal(_, minimal_vio, target_col):
-            minimal_vio.remove(_)
-
-    return md_list, minimal_vio
-
-
-if __name__ == '__main__':
-    # 目前可以仿照这个main函数写
-    path = "input/T_positive_with_id_concat_single_tuple.csv"
-    start = time.time()
-    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
-    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
-    # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
-    mds, mds_vio = inference_from_record_pairs(path, 0.7, 'id_concat')
-
-    # 将列表1写入本地，路径需自己修改
-    md_path = 'output/md.txt'
-    with open(md_path, 'w') as f:
-        for _ in mds:
-            f.write(str(_) + '\n')
-
-    # 将列表2写入本地，路径需自己修改
-    vio_path = 'output/vio.txt'
-    with open(vio_path, 'w') as f:
-        for _ in mds_vio:
-            f.write(str(_) + '\n')
-
-    print(time.time() - start)

draw.py

@@ -0,0 +1,55 @@
+import os
+
+import pyecharts
+from pyecharts.charts import Line
+from pyecharts import options as opts
+from pyecharts.globals import ThemeType
+
+if __name__ == '__main__':
+    dir_path = r'E:\Data\Research\Outcome\Walmart-Amazon_dirty'
+    filename_list = os.listdir(dir_path)
+    iter_list = []
+    precision = []
+    recall = []
+    f1 = []
+    interpretability = []
+    performance = []
+    for _ in filename_list:
+        if _.startswith('eval_result'):
+            it = int(_[12:13])
+            iter_list.append(str(it))
+            with open(dir_path + '\\' + _, 'r') as f:
+                # 读取每一行的md，加入该文件的md列表
+                for line in f.readlines():
+                    if line.startswith('Precision'):
+                        lt = line.split(' ')
+                        value = float(lt[2].replace('%', ''))/100
+                        precision.append(value)
+                    elif line.startswith('Recall'):
+                        lt = line.split(' ')
+                        value = float(lt[2].replace('%', ''))/100
+                        recall.append(value)
+                    elif line.startswith('F1'):
+                        lt = line.split(' ')
+                        value = float(lt[2].replace('%', ''))/100
+                        f1.append(value)
+                    elif line.startswith('interpretability'):
+                        lt = line.split(':')
+                        value = float(lt[1])
+                        interpretability.append(value)
+                    elif line.startswith('performance'):
+                        lt = line.split(':')
+                        value = float(lt[1])
+                        performance.append(value)
+
+    line = (
+        Line(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
+        .add_xaxis(iter_list)
+        .add_yaxis('Precision', precision)
+        .add_yaxis('Recall', recall)
+        .add_yaxis('F1', f1)
+        .add_yaxis('Interpretability', interpretability)
+        .add_yaxis('Performance', performance)
+        .set_global_opts(title_opts=opts.TitleOpts(title=dir_path.split('\\')[-1]))
+    )
+    line.render(dir_path + '\\' + "line.html")

functions/multi_process_infer_by_pairs.py

@@ -1,186 +0,0 @@
-import multiprocessing
-import pandas as pd
-import Levenshtein
-import copy
-
-
-conf_thresh = 0.8
-
-
-def my_Levenshtein_ratio(str1, str2):
-    if max(len(str1), len(str2)) == 0:
-        return 1
-    return 1 - Levenshtein.distance(str1, str2) / max(len(str1), len(str2))
-
-
-def if_minimal(md, md_list, target_col):
-    # 假设这个md是minimal
-    minimal = True
-    if md_list.count(md) > 1:
-        return False
-    for _ in md_list:
-        if _ != md:
-            # 假设列表中每一个md都使当前md不minimal
-            exist = True
-            # 如果左边任何一个大于，则假设不成立
-            for col in list(set(_.keys()) - {target_col}):
-                if _[col] > md[col]:
-                    exist = False
-            # 如果右边小于，假设也不成立
-            if _[target_col] < md[target_col]:
-                exist = False
-            # 任何一次假设成立，当前md不minimal
-            if exist:
-                minimal = False
-                break
-    return minimal
-
-
-def remove_by_confidence(md, l, relation, target_col, lock):
-    support, confidence = get_one_md_metadata(md, relation, target_col)
-    # todo: replace constant 0.8
-    if confidence < 0.8:
-        with lock:
-            l.remove(md)
-
-
-# def remove_by_confidence(md, l, relation, target_col):
-#     boolean, conf = satisfy_confidence(md, relation, 0.8, target_col)
-#     if not boolean:
-#         l.remove(md)
-#         print(md, '\t', conf)
-
-
-def inference_from_record_pairs(path, threshold, target_col):
-    data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1')
-    data.fillna("", inplace=True)
-    data = data.astype(str)
-    columns = data.columns.values.tolist()
-
-    md_list = []
-    minimal_vio = []
-    init_md = {}
-    for col in columns:
-        init_md[col] = 1 if col == target_col else 0
-    md_list.append(init_md)
-
-    for row1 in data.itertuples():
-        # 获取当前行的索引，从后一行开始切片
-        i = row1[0]
-        data1 = data[i + 1:]
-        for row2 in data1.itertuples():
-            violated_mds = []
-            # sims是两行的相似度
-            sims = {}
-            for col in columns:
-                similarity = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                sims[col] = similarity
-
-            # 寻找violated md,从md列表中删除并加入vio列表
-            # tmp_md_list = copy.deepcopy(md_list)
-            for md in md_list[:]:
-                lhs_satis = True
-                rhs_satis = True
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] < md[col]:
-                        lhs_satis = False
-                if sims[target_col] < md[target_col]:
-                    rhs_satis = False
-                if lhs_satis == True and rhs_satis == False:
-                    md_list.remove(md)
-                    violated_mds.append(md)
-            minimal_vio.extend(violated_mds)
-
-            for vio_md in violated_mds:
-                # 特殊化右侧,我们需要右侧百分百相似，其实不需要降低右侧阈值
-                # if sims[target_col] >= threshold:
-                #     new_rhs = sims[target_col]
-                #     spec_r_md = copy.deepcopy(vio_md)
-                #     spec_r_md[target_col] = new_rhs
-                #     if if_minimal(spec_r_md, md_list, target_col):
-                #         md_list.append(spec_r_md)
-
-                # 特殊化左侧
-                for col in list(set(columns) - {target_col}):
-                    if sims[col] + 0.001 <= 1:
-                        spec_l_md = copy.deepcopy(vio_md)
-                        spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.001
-                        if if_minimal(spec_l_md, md_list, target_col):
-                            md_list.append(spec_l_md)
-
-            # tmp_minimal_vio = copy.deepcopy(minimal_vio)
-            for vio in minimal_vio[:]:
-                if not if_minimal(vio, md_list, target_col):
-                    minimal_vio.remove(vio)
-
-    manager = multiprocessing.Manager()
-    lock = manager.Lock()
-    if len(minimal_vio) == 0:
-        return md_list, []
-    pool = multiprocessing.Pool(len(minimal_vio))
-    # tmp = copy.deepcopy(minimal_vio)
-    with manager:
-        proxy_minimal_vio = manager.list(minimal_vio)
-        for _ in minimal_vio[:]:
-            pool.apply_async(remove_by_confidence, args=(_, proxy_minimal_vio, data, target_col, lock))
-        pool.close()
-        pool.join()
-        minimal_vio = list(proxy_minimal_vio)
-
-    for _ in minimal_vio[:]:
-        if not if_minimal(_, minimal_vio, target_col):
-            minimal_vio.remove(_)
-
-    for _ in md_list[:]:
-        if not if_minimal(_, md_list, target_col):
-            md_list.remove(_)
-
-    return md_list, minimal_vio
-
-
-def get_mds_metadata(md_list, dataset_path, target_col):
-    data = pd.read_csv(dataset_path, low_memory=False, encoding='ISO-8859-1')
-    data.fillna("", inplace=True)
-    data = data.astype(str)
-
-    manager = multiprocessing.Manager()
-    if len(md_list) == 0:
-        return []
-    pool = multiprocessing.Pool(len(md_list))
-    result = []
-    with manager:
-        for _ in md_list:
-            task = pool.apply_async(get_one_md_metadata, args=(_, data, target_col))
-            support, confidence = task.get()
-            result.append({"md": _, "support": support, "confidence": confidence})
-        pool.close()
-        pool.join()
-    return result
-
-
-def get_one_md_metadata(md, dataframe, target_col):
-    support = 0
-    pre_confidence = 0
-    for row1 in dataframe.itertuples():
-        i = row1[0]
-        df_slice = dataframe[i + 1:]
-        for row2 in df_slice.itertuples():
-            left_satisfy = True
-            both_satisfy = True
-            for col in dataframe.columns.values.tolist():
-                sim = my_Levenshtein_ratio(getattr(row1, col), getattr(row2, col))
-                if col == target_col:
-                    if sim < 1:
-                        both_satisfy = False
-                else:
-                    if sim < md[col]:
-                        left_satisfy = False
-                        both_satisfy = False
-            if left_satisfy:
-                support += 1
-            if both_satisfy:
-                pre_confidence += 1
-
-    confidence = 0 if support == 0 else pre_confidence / support
-    # return {"md": md, "support": support, "confidence": confidence}
-    return support, confidence

generate_matches.py

@@ -0,0 +1,20 @@
+import csv
+
+import pandas as pd
+import json
+import sentence_transformers.util
+import torch
+from sentence_transformers import SentenceTransformer
+from torch import nn
+
+if __name__ == '__main__':
+    train = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\train.csv', encoding='ISO-8859-1')
+    valid = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\valid.csv', encoding='ISO-8859-1')
+    test = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\test.csv', encoding='ISO-8859-1')
+    train = train[train['label'] == 1]
+    valid = valid[valid['label'] == 1]
+    test = test[test['label'] == 1]
+    matches = pd.concat([train, valid, test])
+    matches.drop(columns=['label'], inplace=True)
+    matches = matches.sort_values(by='ltable_id')
+    matches.to_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\Fodors-Zagats\matches.csv', sep=',', index=False, header=True)

hpo/er_model_hpo.py

@@ -0,0 +1,258 @@
+import os
+
+import numpy as np
+import torch
+import json
+from ConfigSpace import Categorical, Configuration, ConfigurationSpace, Integer
+from ConfigSpace.conditions import InCondition
+from ConfigSpace.read_and_write import json as csj
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+import pandas as pd
+
+from smac import HyperparameterOptimizationFacade, Scenario
+from settings import *
+from ml_er.ml_entity_resolver import evaluate_prediction, load_mds, is_explicable, build_col_pairs_sim_tensor_dict
+
+# 数据在外部加载
+########################################################################################################################
+ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
+# ltable.fillna("", inplace=True)
+rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
+# rtable.fillna("", inplace=True)
+mappings = pd.read_csv(mapping_path)
+
+lid_mapping_list = []
+rid_mapping_list = []
+# 全部转为字符串
+# ltable = ltable.astype(str)
+# rtable = rtable.astype(str)
+# mappings = mappings.astype(str)
+matching_number = len(mappings)  # 所有阳性样本数，商品数据集应为1300
+
+for index, row in mappings.iterrows():
+    lid_mapping_list.append(row[mapping_lid])
+    rid_mapping_list.append(row[mapping_rid])
+# 仅保留两表中出现在映射表中的行，增大正样本比例
+selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
+# if len(lr_attrs_map) > 0:
+#     selected_ltable = selected_ltable.rename(columns=lr_attrs_map)  # 参照右表，修改左表中与右表对应但不同名的字段
+tables_id = rtable_id  # 不论左表右表ID字段名是否一致，经上一行调整，统一以右表为准
+selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
+selected_attrs = selected_ltable.columns.values.tolist()  # 两张表中的字段名
+########################################################################################################################
+
+
+class Classifier:
+    @property
+    def configspace(self) -> ConfigurationSpace:
+        # Build Configuration Space which defines all parameters and their ranges
+        cs = ConfigurationSpace(seed=0)
+        block_attr_items = selected_attrs[:]
+        block_attr_items.remove(tables_id)
+
+        block_attr = Categorical("block_attr", block_attr_items)
+        overlap_size = Integer("overlap_size", (1, 3), default=1)
+        ml_matcher = Categorical("ml_matcher", ["dt", "svm", "rf", "lg", "ln", "nb"], default="rf")
+        ml_blocker = Categorical("ml_blocker", ["over_lap", "attr_equiv"], default="over_lap")
+
+        use_overlap_size = InCondition(child=overlap_size, parent=ml_blocker, values=["over_lap"])
+        cs.add_hyperparameters([block_attr, overlap_size, ml_matcher, ml_blocker])
+        cs.add_conditions([use_overlap_size])
+        return cs
+
+    # train 就是整个函数 只需将返回结果由预测变成预测结果的评估
+    def train(self, config: Configuration, seed: int = 0) -> float:
+        # print(f"BRAINFUCK:{config.values()}")
+        cm.del_catalog()
+        attrs_with_l_prefix = ['ltable_' + i for i in selected_attrs]  # 字段名加左前缀
+        attrs_with_r_prefix = ['rtable_' + i for i in selected_attrs]  # 字段名加右前缀
+        cm.set_key(selected_ltable, tables_id)
+        cm.set_key(selected_rtable, tables_id)
+        if config["ml_blocker"] == "over_lap":
+            blocker = em.OverlapBlocker()
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
+                                             overlap_size=config["overlap_size"], show_progress=False,
+                                             allow_missing=True)
+        elif config["ml_blocker"] == "attr_equiv":
+            blocker = em.AttrEquivalenceBlocker()
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"], config["block_attr"],
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
+                                             allow_missing=True)
+
+        candidate['gold'] = 0
+        candidate = candidate.reset_index(drop=True)
+        candidate_match_rows = []
+        for line in candidate.itertuples():
+            l_id = getattr(line, 'ltable_' + tables_id)
+            map_row = mappings[mappings[mapping_lid] == l_id]
+
+            if map_row is not None:
+                r_id = map_row[mapping_rid]
+                for value in r_id:
+                    if value == getattr(line, 'rtable_' + tables_id):
+                        candidate_match_rows.append(line[0])
+            else:
+                continue
+        for _ in candidate_match_rows:
+            candidate.loc[_, 'gold'] = 1
+
+        candidate.fillna("", inplace=True)
+
+        # 裁剪负样本，保持正负样本数量一致
+        candidate_mismatch = candidate[candidate['gold'] == 0]
+        candidate_match = candidate[candidate['gold'] == 1]
+        if len(candidate_mismatch) > len(candidate_match):
+            candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
+        # 拼接正负样本
+        candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+        if len(candidate_for_train_test) == 0:
+            return 1
+        candidate_for_train_test = candidate_for_train_test.reset_index(drop=True)
+        cm.set_key(candidate_for_train_test, '_id')
+        cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + tables_id)
+        cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + tables_id)
+        cm.set_ltable(candidate_for_train_test, selected_ltable)
+        cm.set_rtable(candidate_for_train_test, selected_rtable)
+
+        # 分为训练测试集
+        train_proportion = 0.7
+        test_proportion = 0.3
+        sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0)
+        train_set = sets['train']
+        test_set = sets['test']
+
+        cm.set_key(train_set, '_id')
+        cm.set_fk_ltable(train_set, 'ltable_' + tables_id)
+        cm.set_fk_rtable(train_set, 'rtable_' + tables_id)
+        cm.set_ltable(train_set, selected_ltable)
+        cm.set_rtable(train_set, selected_rtable)
+
+        cm.set_key(test_set, '_id')
+        cm.set_fk_ltable(test_set, 'ltable_' + tables_id)
+        cm.set_fk_rtable(test_set, 'rtable_' + tables_id)
+        cm.set_ltable(test_set, selected_ltable)
+        cm.set_rtable(test_set, selected_rtable)
+
+        if config["ml_matcher"] == "dt":
+            matcher = em.DTMatcher(name='DecisionTree', random_state=0)
+        elif config["ml_matcher"] == "svm":
+            matcher = em.SVMMatcher(name='SVM', random_state=0)
+        elif config["ml_matcher"] == "rf":
+            matcher = em.RFMatcher(name='RF', random_state=0)
+        elif config["ml_matcher"] == "lg":
+            matcher = em.LogRegMatcher(name='LogReg', random_state=0)
+        elif config["ml_matcher"] == "ln":
+            matcher = em.LinRegMatcher(name='LinReg')
+        elif config["ml_matcher"] == "nb":
+            matcher = em.NBMatcher(name='NaiveBayes')
+        feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
+
+        train_feature_vecs = em.extract_feature_vecs(train_set,
+                                                     feature_table=feature_table,
+                                                     attrs_after=['gold'],
+                                                     show_progress=False)
+        train_feature_vecs.fillna(value=0, inplace=True)
+        test_feature_after = attrs_with_l_prefix[:]
+        test_feature_after.extend(attrs_with_r_prefix)
+        for _ in test_feature_after:
+            if _.endswith(tables_id):
+                test_feature_after.remove(_)
+        test_feature_after.append('gold')
+        test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
+                                                    attrs_after=test_feature_after, show_progress=False)
+        test_feature_vecs.fillna(value=0, inplace=True)
+        fit_exclude = ['_id', 'ltable_' + tables_id, 'rtable_' + tables_id, 'gold']
+        matcher.fit(table=train_feature_vecs, exclude_attrs=fit_exclude, target_attr='gold')
+
+        test_feature_after.extend(['_id', 'ltable_' + tables_id, 'rtable_' + tables_id])
+        predictions = matcher.predict(table=test_feature_vecs, exclude_attrs=test_feature_after,
+                                      append=True, target_attr='predicted', inplace=False)
+        eval_result = em.eval_matches(predictions, 'gold', 'predicted')
+        em.print_eval_summary(eval_result)
+        indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, candidate_for_train_test)
+        print(indicators)
+
+        # 计算可解释性
+        predictions_attrs = []
+        predictions_attrs.extend(attrs_with_l_prefix)
+        predictions_attrs.extend(attrs_with_r_prefix)
+        predictions_attrs.extend(['gold', 'predicted'])
+        predictions = predictions[predictions_attrs]
+        predictions = predictions.reset_index(drop=True)
+        predictions = predictions.astype(str)
+        sim_tensor_dict = build_col_pairs_sim_tensor_dict(predictions)
+
+        #  默认路径为 "../md_discovery/output/xxx.txt"
+        #  mds/vio  共2个md文件
+        md_paths = [md_output_dir + 'mds.txt', md_output_dir + 'vio.txt']
+        md_list = load_mds(md_paths)  # 从全局变量中读取所有的md
+        epl_match = 0  # 可解释，预测match
+        if len(md_list) > 0:
+            for line in predictions.itertuples():
+                if is_explicable(line, md_list, sim_tensor_dict) and str(getattr(line, 'predicted')) == str(1):
+                    epl_match += 1
+
+        ppre = predictions[predictions['predicted'] == str(1)]
+        interpretability = epl_match / len(ppre)  # 可解释性
+
+        if (indicators["block_recall"] < 0.8) and (indicators["block_recall"] < indicators["recall"]):
+            f1 = (2.0 * indicators["precision"] * indicators["block_recall"]) / (
+                        indicators["precision"] + indicators["block_recall"])
+        else:
+            f1 = indicators["F1"]
+        # if indicators["block_recall"] < 0.8:
+        #     return 1
+        # f1 = indicators["F1"]
+        performance = interpre_weight * interpretability + (1 - interpre_weight) * f1
+        return 1 - performance
+
+
+def ml_er_hpo():
+    classifier = Classifier()
+    cs = classifier.configspace
+    str_configspace = csj.write(cs)
+    dict_configspace = json.loads(str_configspace)
+    with open(hpo_output_dir + "configspace.json", "w") as f:
+        json.dump(dict_configspace, f)
+
+    # Next, we create an object, holding general information about the run
+    scenario = Scenario(
+        cs,
+        deterministic=True,
+        n_trials=10,  # We want to run max 50 trials (combination of config and seed)
+        n_workers=1
+    )
+
+    initial_design = HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=5)
+
+    # Now we use SMAC to find the best hyperparameters
+    smac = HyperparameterOptimizationFacade(
+        scenario,
+        classifier.train,
+        initial_design=initial_design,
+        overwrite=True,  # If the run exists, we overwrite it; alternatively, we can continue from last state
+    )
+
+    incumbent = smac.optimize()
+    incumbent_cost = smac.validate(incumbent)
+    default = cs.get_default_configuration()
+    default_cost = smac.validate(default)
+    print(f"Default Cost: {default_cost}")
+    print(f"Incumbent Cost: {incumbent_cost}")
+
+    if incumbent_cost > default_cost:
+        incumbent = default
+        print(f"Updated Incumbent Cost: {default_cost}")
+
+    print(f"Optimized Configuration:{incumbent.values()}")
+
+    incumbent_ndarray = incumbent.get_array()
+    np.save(hpo_output_dir + 'incumbent.npy', incumbent_ndarray)
+
+    return incumbent
+
+
+if __name__ == '__main__':
+    ml_er_hpo()

md_discovery/md_discover.py

@@ -0,0 +1,43 @@
+from md_discovery import tmp_discover
+from settings import *
+
+# # 若不输出support和confidence，使用以下两块代码
+# # 将列表1写入本地，路径需自己修改
+# md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt'
+# with open(md_path, 'w') as f:
+#     for _ in mds:
+#         f.write(str(_) + '\n')
+#
+# # 将列表2写入本地，路径需自己修改
+# vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt'
+# with open(vio_path, 'w') as f:
+#     for _ in vio:
+#         f.write(str(_) + '\n')
+
+
+def md_discover():
+    # 目前可以仿照这个main函数写
+    t_single_tuple_path = er_output_dir + "t_single_tuple.csv"
+    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
+    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值
+    mds_list, vio_list = tmp_discover.pairs_inference(t_single_tuple_path, similarity_threshold, target_attr)
+
+    # 将列表1写入本地，路径需自己修改
+    mds_path = md_output_dir + "mds.txt"
+    vio_path = md_output_dir + "vio.txt"
+    
+    with open(mds_path, 'w') as f:
+        for _ in mds_list:
+            f.write('Target:'+str(target_attr) + '\t')
+            f.write(str(_))
+            f.write('\n')
+
+    with open(vio_path, 'w') as f:
+        for _ in vio_list:
+            f.write('Target:'+str(target_attr) + '\t')
+            f.write(str(_))
+            f.write('\n')
+
+
+if __name__ == '__main__':
+    md_discover()

md_discovery/multi_process_infer_by_pairs.py

@@ -0,0 +1,276 @@
+import multiprocessing
+import pandas as pd
+import Levenshtein
+import copy
+import numpy as np
+import time
+import torch
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModel
+from settings import model, er_output_dir
+from sentence_transformers.util import cos_sim
+
+conf_thresh = 0.8
+
+
+def my_Levenshtein_ratio(str1, str2):
+    if max(len(str1), len(str2)) == 0:
+        return 1
+    return 1 - Levenshtein.distance(str1, str2) / max(len(str1), len(str2))
+
+
+def norm_cos_sim(embed1, embed2):
+    sim = cos_sim(embed1, embed2)
+    return sim.tolist()[0][0]/2 + 0.5
+
+
+def table_encode(tp_path, fn_path):
+    embedding_dic = {}
+
+    tp_data = pd.read_csv(tp_path, low_memory=False, encoding='ISO-8859-1')
+    tp_data.fillna("", inplace=True)
+    tp_data = tp_data.astype(str)
+    tp_length = tp_data.shape[0]
+    tp_width = tp_data.shape[1]
+    tp_sentences = []
+
+    for row in range(0, tp_length):
+        for col in range(0, tp_width):
+            cell_value = tp_data.values[row, col]
+            tp_sentences.append(cell_value)
+    tp_embedding = model.encode(tp_sentences, convert_to_tensor=True, device="cuda")
+
+    list_tp_embedding = tp_embedding.tolist()
+    for row in range(0, tp_length):
+        for col in range(0, tp_width):
+            cell_value = tp_data.values[row, col]
+            embedding_dic[cell_value] = list_tp_embedding[row * tp_width + col]
+
+
+    fn_data = pd.read_csv(fn_path, low_memory=False, encoding='ISO-8859-1')
+    fn_data.fillna("", inplace=True)
+    fn_data = fn_data.astype(str)
+    fn_length = fn_data.shape[0]
+    fn_width = fn_data.shape[1]
+    fn_sentences = []
+
+    for row in range(0, fn_length):
+        for col in range(0, fn_width):
+            cell_value = fn_data.values[row, col]
+            fn_sentences.append(cell_value)
+    fn_embedding = model.encode(fn_sentences, convert_to_tensor=True, device="cuda")
+
+    list_fn_embedding = fn_embedding.tolist()
+    for row in range(0, fn_length):
+        for col in range(0, fn_width):
+            cell_value = fn_data.values[row, col]
+            embedding_dic[cell_value] = list_fn_embedding[row * fn_width + col]
+
+    np.save('embedding_dic.npy', embedding_dic)
+
+
+def test_table_encode():
+    start = time.time()
+    table_encode(er_output_dir+'tp_single_tuple.csv', er_output_dir+'fn_single_tuple.csv')
+    print(time.time()-start)
+
+
+def test_load():
+    load_dict = np.load('embedding_dic.npy', allow_pickle=True).item()
+    a = load_dict['model- bdcd00105wi venor- bitdefender features- bitdefender antivirus v10- small box antivirus v10 delivers a one-two security punch integrating todays most powerful antivirus and antispyware modules into one convenient package. its easy to use and updates itself automatically making it truly an install and forget solution. * antivirus the purpose of the antivirus module is to ensure detection and removal of all viruses in the wild. bitdefender antivirus uses robust scan engines certified by icsa labs virus bulletin checkmark checkvir and tuv. - improved proactive detection b-have (behavioral heuristic analyzer in virtual environments) emulates a virtual computer-inside-a-computer where pieces of software are run in order to check for potential malware behavior. this bitdefender proprietary technology represents a new security layer that keeps the operating system safe from unknown viruses by detecting malicious pieces of code for which signatures have not yet been released. - permanent antivirus protection the new and improved bitdefender scanning engines will scan and disinfect infected files on access minimizing data loss. infected documents can now be recovered instead of being deleted. - new rootkit detection and removal a new bitdefender module looks for rootkits (malicious programs designed to control victim computers while staying hidden) and removes them on detection. - new web scanning web traffic is now filtered in real-time even before reaching your browser providing a safe and enjoyable web experience. - peer-2-peer and im applications protection filters against viruses that spread']
+    print(a)
+    print(1)
+# def test_lm_similarity():
+#     print(time.time())
+#     sentences = ['fun with reading & writing! is designed to help kids learn to read and write better through exercises puzzle-solving creative writing decoding and more!',
+#                   'based on the tween lifestyle brand launched in 2004 this action/adventure game will contain loads of adventures tailored specifically to the player\'s personality type. the evergirl brand features a clothing and accessories line with a companion web ...']
+#     embeddings = model.encode(sentences, convert_to_tensor=True)
+#     print(time.time())
+#     sim = cos_sim(embeddings[0], embeddings[1])
+#     print(time.time())
+#     # print(sim.tolist()[0][0]/2 + 0.5)
+
+
+def is_minimal(md, md_list, target_col):
+    # 假设这个md是minimal
+    minimal = True
+    if len(md_list) == 0:
+        return True
+    if md_list.count(md) > 1:
+        return False
+    for _ in md_list:
+        if _ != md:
+            # 假设列表中每一个md都使当前md不minimal
+            exist = True
+            # 如果左边任何一个大于，则假设不成立
+            for col in list(set(_.keys()) - {target_col}):
+                if _[col] > md[col]:
+                    exist = False
+            # 如果右边小于，假设也不成立
+            if _[target_col] < md[target_col]:
+                exist = False
+            # 任何一次假设成立，当前md不minimal
+            if exist:
+                minimal = False
+                break
+    return minimal
+
+
+def remove_by_confidence(md, l, relation, target_col, lock):
+    support, confidence = get_one_md_metadata(md, relation, target_col)
+    if confidence < 0.8:
+        with lock:
+            l.remove(md)
+
+
+# def remove_by_confidence(md, l, relation, target_col):
+#     boolean, conf = satisfy_confidence(md, relation, 0.8, target_col)
+#     if not boolean:
+#         l.remove(md)
+#         print(md, '\t', conf)
+
+
+def inference_from_record_pairs(path, threshold, target_col):
+    data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1')
+    data.fillna("", inplace=True)
+    data = data.astype(str)
+    columns = data.columns.values.tolist()
+    other_columns = list(set(columns) - {target_col})
+
+    md_list = []
+    minimal_vio = []
+    init_md = {}
+    for col in columns:
+        init_md[col] = 1 if col == target_col else 0
+    md_list.append(init_md)
+
+    for row1 in tqdm(data.itertuples()):
+        # 获取当前行的索引，从后一行开始切片
+        i = row1[0]
+        data1 = data[i + 1:]
+        for row2 in data1.itertuples():
+            violated_mds = []
+            # sims是两行的相似度
+            sims = {}
+            for col in columns:
+                similarity = norm_cos_sim(getattr(row1, col), getattr(row2, col))
+                sims[col] = similarity
+
+            # 寻找violated md,从md列表中删除并加入vio列表
+            # tmp_md_list = copy.deepcopy(md_list)
+            for md in md_list[:]:
+                lhs_satis = True
+                rhs_satis = True
+                for col in other_columns:
+                    if sims[col] + 0.0000001 < md[col]:
+                        lhs_satis = False
+                        break
+                if sims[target_col] + 0.0000001 < md[target_col]:
+                    rhs_satis = False
+                if lhs_satis == True and rhs_satis == False:
+                    md_list.remove(md)
+                    violated_mds.append(md)
+            # minimal_vio.extend(violated_mds)
+
+            for vio_md in violated_mds:
+                # 特殊化右侧,我们需要右侧百分百相似，其实不需要降低右侧阈值
+                # if sims[target_col] >= threshold:
+                #     new_rhs = sims[target_col]
+                #     spec_r_md = copy.deepcopy(vio_md)
+                #     spec_r_md[target_col] = new_rhs
+                #     if is_minimal(spec_r_md, md_list, target_col):
+                #         md_list.append(spec_r_md)
+
+                # 特殊化左侧
+                for col in other_columns:
+                    if sims[col] + 0.01 <= 1:
+                        spec_l_md = copy.deepcopy(vio_md)
+                        spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.01
+                        if is_minimal(spec_l_md, md_list, target_col):
+                            md_list.append(spec_l_md)
+
+            # for vio in minimal_vio[:]:
+            #     if not is_minimal(vio, md_list, target_col):
+            #         minimal_vio.remove(vio)
+
+    # fuck = len(minimal_vio)
+    # tmp = []
+    # for _ in minimal_vio:
+    #     if _ not in tmp:
+    #         tmp.append(_)
+    # minimal_vio = tmp
+
+    # manager = multiprocessing.Manager()
+    # lock = manager.Lock()
+    # if len(minimal_vio) == 0:
+    #     return md_list, []
+    # pool_size = len(minimal_vio) if len(minimal_vio) < 16 else 16
+    # pool = multiprocessing.Pool(pool_size)
+    # # tmp = copy.deepcopy(minimal_vio)
+    # with manager:
+    #     proxy_minimal_vio = manager.list(minimal_vio)
+    #     for _ in minimal_vio[:]:
+    #         pool.apply_async(remove_by_confidence, args=(_, proxy_minimal_vio, data, target_col, lock))
+    #     pool.close()
+    #     pool.join()
+    #     minimal_vio = list(proxy_minimal_vio)
+    #
+    # for _ in minimal_vio[:]:
+    #     if not is_minimal(_, minimal_vio, target_col):
+    #         minimal_vio.remove(_)
+    #
+    for _ in md_list[:]:
+        if not is_minimal(_, md_list, target_col):
+            md_list.remove(_)
+
+    return md_list, minimal_vio
+
+
+def get_mds_metadata(md_list, dataset_path, target_col):
+    data = pd.read_csv(dataset_path, low_memory=False, encoding='ISO-8859-1')
+    data.fillna("", inplace=True)
+    data = data.astype(str)
+
+    manager = multiprocessing.Manager()
+    if len(md_list) == 0:
+        return []
+    pool_size = len(md_list) if len(md_list) < 16 else 16
+    pool = multiprocessing.Pool(pool_size)
+    result = []
+    with manager:
+        for _ in md_list:
+            task = pool.apply_async(get_one_md_metadata, args=(_, data, target_col))
+            support, confidence = task.get()
+            result.append({"md": _, "support": support, "confidence": confidence})
+        pool.close()
+        pool.join()
+    return result
+
+
+def get_one_md_metadata(md, dataframe, target_col):
+    support = 0
+    pre_confidence = 0
+    columns = dataframe.columns.values.tolist()
+    for row1 in dataframe.itertuples():
+        i = row1[0]
+        df_slice = dataframe[i + 1:]
+        for row2 in df_slice.itertuples():
+            left_satisfy = True
+            both_satisfy = True
+            for col in columns:
+                sim = norm_cos_sim(getattr(row1, col), getattr(row2, col))
+                if col == target_col:
+                    if sim + 0.0000001 < 1:
+                        both_satisfy = False
+                else:
+                    if sim + 0.0000001 < md[col]:
+                        left_satisfy = False
+                        both_satisfy = False
+            if left_satisfy:
+                support += 1
+            if both_satisfy:
+                pre_confidence += 1
+
+    confidence = 0 if support == 0 else pre_confidence / support
+    # return {"md": md, "support": support, "confidence": confidence}
+    return support, confidence

md_discovery/tmp_discover.py

@@ -0,0 +1,202 @@
+import multiprocessing
+import time
+from concurrent.futures import ProcessPoolExecutor
+from multiprocessing.managers import SharedMemoryManager
+
+import numpy as np
+import pandas as pd
+import copy
+
+import torch
+from tqdm import tqdm
+
+from settings import model, md_output_dir, confidence_threshold, support_threshold
+
+
+def is_minimal(md, md_list, target_col):
+    # 假设这个md是minimal
+    if len(md_list) == 0:
+        return True
+    minimal = True
+    for _ in md_list:
+        if _ != md:
+            other_cols = list(set(_.keys()) - {target_col})
+            # 假设列表中每一个md都使当前md不minimal
+            exist = True
+            # 如果左边任何一个大于，则假设不成立
+            for col in other_cols:
+                if _[col] > md[col]:
+                    exist = False
+                    break
+            # 如果右边小于，假设也不成立
+            if _[target_col] < md[target_col]:
+                exist = False
+            # 任何一次假设成立，当前md不minimal
+            if exist:
+                minimal = False
+                break
+    return minimal
+
+
+def pairs_inference(path, threshold, target_col):
+    data = pd.read_csv(path, low_memory=False, encoding='ISO-8859-1')
+    data.fillna("", inplace=True)
+    data = data.astype(str)
+    columns = data.columns.values.tolist()
+    target_index = columns.index(target_col)
+    cols_but_target = list(set(columns) - {target_col})
+    length = data.shape[0]
+    width = data.shape[1]
+
+    sentences = []
+    for col in range(0, width):
+        for row in range(0, length):
+            cell_value = data.values[row, col]
+            sentences.append(cell_value)
+    embedding = model.encode(sentences, convert_to_tensor=True, device="cuda")
+    split_embedding = torch.split(embedding, length, dim=0)
+    table_tensor = torch.stack(split_embedding, dim=0, out=None)
+    norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2)
+    sim_tensor = torch.matmul(norm_table_tensor, norm_table_tensor.transpose(1, 2))
+
+    torch.save(sim_tensor, md_output_dir + "tensor.pt")
+
+    md_list = []
+    minimal_vio = []
+    init_md = {}
+    for col in columns:
+        init_md[col] = 1 if col == target_col else 0
+    md_list.append(init_md)
+
+    for row1 in range(0, length - 1):
+        terminate = False
+        for row2 in range(row1 + 1, length):
+            violated_mds = []
+            # sims是两行的相似度
+            sims = {}
+            for col_index in range(0, width):
+                col = columns[col_index]
+                similarity = sim_tensor[col_index, row1, row2].item()
+                sims[col] = similarity
+
+            # 寻找violated md,从md列表中删除并加入vio列表
+            for md in md_list[:]:
+                lhs_satis = True
+                rhs_satis = True
+                for col in cols_but_target:
+                    if sims[col] < md[col]:
+                        lhs_satis = False
+                        break
+                if sims[target_col] < md[target_col]:
+                    rhs_satis = False
+                if lhs_satis == True and rhs_satis == False:
+                    md_list.remove(md)
+                    violated_mds.append(md)
+
+            for vio_md in violated_mds:
+                # 特殊化左侧
+                for col in cols_but_target:
+                    if sims[col] + 0.01 <= 1:
+                        spec_l_md = copy.deepcopy(vio_md)
+                        spec_l_md[col] = threshold if sims[col] < threshold else sims[col] + 0.01
+                        if is_minimal(spec_l_md, md_list, target_col):
+                            md_list.append(spec_l_md)
+                if vio_md not in minimal_vio:
+                    minimal_vio.append(vio_md)
+            # for vio_md in violated_mds:
+            #     # 特殊化左侧
+            #     for col in cols_but_target:
+            #         if sims[col] < 1:
+            #             spec_l_md = copy.deepcopy(vio_md)
+            #             if sims[col] < threshold:
+            #                 spec_l_md[col] = threshold
+            #             else:
+            #                 if sims[col] + 0.01 <= 1:
+            #                     spec_l_md[col] = sims[col] + 0.01
+            #                 else:
+            #                     spec_l_md[col] = 1
+            #             if is_minimal(spec_l_md, md_list, target_col):
+            #                 md_list.append(spec_l_md)
+            #     if vio_md not in minimal_vio:
+            #         minimal_vio.append(vio_md)
+            if len(md_list) == 0:
+                terminate = True
+                break
+        if terminate:
+            break
+
+    tmp = []
+    for _ in md_list:
+        if _ not in tmp:
+            tmp.append(_)
+    md_list = tmp
+    if len(md_list) > 0:
+        for vio in minimal_vio[:]:
+            if not is_minimal(vio, md_list, target_col):
+                minimal_vio.remove(vio)
+        for _ in md_list[:]:
+            if not is_minimal(_, md_list, target_col):
+                md_list.remove(_)
+
+    print('mds_list\t', len(md_list), '\n')
+    print('vio_list\t', len(minimal_vio), '\n')
+
+    if len(minimal_vio) == 0:
+        return md_list, []
+
+    remove_list = []
+    # fuck = []
+    if len(md_list) > 0:
+        md_rm_list = []
+        for _ in md_list:
+            support, confidence = get_metrics(_, data, sim_tensor, target_col, target_index)
+            if support < support_threshold:
+                md_rm_list.append(_)
+        for _ in md_rm_list:
+            md_list.remove(_)
+    for md in minimal_vio:
+        support, confidence = get_metrics(md, data, sim_tensor, target_col, target_index)
+        # fuck.append((support, confidence))
+        if support < support_threshold:
+            remove_list.append(md)
+        if confidence < confidence_threshold and md not in remove_list:
+            remove_list.append(md)
+    # fuck_me = sorted(fuck, key=lambda x: x[1], reverse=True)
+    for _ in remove_list:
+        minimal_vio.remove(_)
+
+    for _ in minimal_vio[:]:
+        if not is_minimal(_, minimal_vio, target_col):
+            minimal_vio.remove(_)
+
+    print('\033[31m' + 'vio_length\t' + str(len(minimal_vio)) + '\033[0m')
+
+    return md_list, minimal_vio
+
+
+def get_metrics(current_md, data, sim_tensor, target_col, target_index):
+    columns = data.columns.values.tolist()
+    length = data.shape[0]
+    width = data.shape[1]
+
+    md_tensor = list(current_md.values())
+    md_tensor = torch.tensor(md_tensor, device='cuda')
+    md_tensor_2d = md_tensor.unsqueeze(1)
+    md_tensor_3d = md_tensor_2d.unsqueeze(2)
+    md_tensor_3d = md_tensor_3d.repeat(1, length, length)
+
+    sup_tensor = torch.ge(sim_tensor, md_tensor_3d)
+    ini_slice = torch.ones((length, length), dtype=torch.bool, device='cuda')
+    for i in range(0, width):
+        if i != target_index:
+            sup_tensor_slice = sup_tensor[i]
+            ini_slice = torch.logical_and(ini_slice, sup_tensor_slice)
+    sup_tensor_int = ini_slice.int()
+    support = torch.count_nonzero(sup_tensor_int).item()
+
+    ini_slice = torch.logical_and(ini_slice, sup_tensor[target_index])
+    conf_tensor_int = ini_slice.int()
+    confidence_numerator = torch.count_nonzero(conf_tensor_int).item()
+    confidence = confidence_numerator / support
+
+    return support, confidence

ml_er/ml_entity_resolver.py

@@ -0,0 +1,342 @@
+import json
+import os
+import sys
+
+import ConfigSpace
+import pandas
+import torch
+from py_entitymatching.debugmatcher.debug_gui_utils import _get_metric
+from ConfigSpace.read_and_write import json as csj
+import py_entitymatching as em
+import py_entitymatching.catalog.catalog_manager as cm
+import pandas as pd
+import six
+from ConfigSpace import Configuration
+from settings import *
+
+
+def process_prediction_for_md_discovery(pred: pd.DataFrame,
+                                        t_single_tuple_path: str = er_output_dir + "t_single_tuple.csv"):
+    # 提取预测表中真阳和假阴部分
+    tp = pred[(pred['gold'] == 1) & (pred['predicted'] == 1)]
+    fn = pred[(pred['gold'] == 1) & (pred['predicted'] == 0)]
+    # 拼成一张表
+    df = pd.concat([tp, fn])
+    # 将真阳/假阴表中左右ID调整一致
+    for index, row in df.iterrows():
+        df.loc[index, "rtable_" + rtable_id] = row["ltable_" + rtable_id]
+
+    pred_columns = pred.columns.values.tolist()
+    l_columns = []
+    r_columns = []
+    cols = []
+    # 将预测表中左表和右表字段名分别加入两个列表
+    for _ in pred_columns:
+        if _.startswith('ltable'):
+            l_columns.append(_)
+        elif _.startswith('rtable'):
+            r_columns.append(_)
+    # 将左表中字段名去掉前缀，作为统一的字段名列表(前提是两张表内对应字段名调整一致)
+    for _ in l_columns:
+        cols.append(_.replace('ltable_', ''))
+
+    ldf = df[l_columns]
+    rdf = df[r_columns]
+    ldf.columns = cols
+    rdf.columns = cols
+    t_single_tuple = pd.concat([ldf, rdf])
+
+    t_single_tuple.to_csv(t_single_tuple_path, sep=',', index=False, header=True, quoting=1)
+
+
+def evaluate_prediction(df: pd.DataFrame, labeled_attr: str, predicted_attr: str, matching_number: int,
+                        candidate: pd.DataFrame) -> dict:
+    new_df = df.reset_index(drop=False, inplace=False)
+    gold = new_df[labeled_attr]
+    predicted = new_df[predicted_attr]
+    gold_negative = gold[gold == 0].index.values
+    gold_positive = gold[gold == 1].index.values
+    predicted_negative = predicted[predicted == 0].index.values
+    predicted_positive = predicted[predicted == 1].index.values
+
+    false_positive_indices = list(set(gold_negative).intersection(predicted_positive))
+    true_positive_indices = list(set(gold_positive).intersection(predicted_positive))
+    false_negative_indices = list(set(gold_positive).intersection(predicted_negative))
+
+    num_true_positives = float(len(true_positive_indices))
+    num_false_positives = float(len(false_positive_indices))
+    num_false_negatives = float(len(false_negative_indices))
+
+    precision_denominator = num_true_positives + num_false_positives
+    recall_denominator = num_true_positives + num_false_negatives
+
+    precision = 0.0 if precision_denominator == 0.0 else num_true_positives / precision_denominator
+    recall = 0.0 if recall_denominator == 0.0 else num_true_positives / recall_denominator
+    F1 = 0.0 if precision == 0.0 and recall == 0.0 else (2.0 * precision * recall) / (precision + recall)
+    block_recall = len(candidate[candidate['gold'] == 1]) / matching_number
+
+    return {"precision": precision, "recall": recall, "F1": F1, "block_recall": block_recall}
+
+
+def load_mds(paths: list) -> list:
+    if len(paths) == 0:
+        return []
+    all_mds = []
+    # 传入md路径列表
+    for md_path in paths:
+        if not os.path.exists(md_path):
+            continue
+        mds = []
+        # 打开每一个md文件
+        with open(md_path, 'r') as f:
+            # 读取每一行的md，加入该文件的md列表
+            for line in f.readlines():
+                md_metadata = line.strip().split('\t')
+                # todo 如果MD文件的形式改了 这里也要改
+                md = eval(md_metadata[1])
+                mds.append(md)
+        all_mds.extend(mds)
+    return all_mds
+
+
+def is_explicable(row, all_mds: list, st_dict) -> bool:
+    attrs = all_mds[0].keys()  # 从第一条md中读取所有字段
+    for md in all_mds:
+        explicable = True  # 假设这条md能解释当前元组
+        for a in attrs:
+            if a != target_attr:
+                if st_dict[a][row[0]].item() < md[a]:
+                    explicable = False  # 任意一个字段的相似度达不到阈值，这条md就不能解释当前元组
+                    break  # 不再与当前md的其他相似度阈值比较，跳转到下一条md
+        if explicable:
+            return True  # 任意一条md能解释，直接返回
+    return False  # 遍历结束，不能解释
+
+
+def build_col_pairs_sim_tensor_dict(predictions: pandas.DataFrame):
+    predictions_attrs = predictions.columns.values.tolist()
+    col_tuple_list = []
+    for _ in predictions_attrs:
+        if _.startswith('ltable'):
+            left_index = predictions_attrs.index(_)
+            right_index = predictions_attrs.index(_.replace('ltable_', 'rtable_'))
+            col_tuple_list.append((left_index, right_index))
+
+    length = predictions.shape[0]
+    width = predictions.shape[1]
+    sentences = []
+    for col in range(0, width):
+        for row in range(0, length):
+            cell_value = predictions.values[row, col]
+            sentences.append(cell_value)
+    embedding = model.encode(sentences, convert_to_tensor=True, device="cuda")
+    split_embedding = torch.split(embedding, length, dim=0)
+    table_tensor = torch.stack(split_embedding, dim=0, out=None)
+    # prediction的归一化嵌入张量
+    norm_table_tensor = torch.nn.functional.normalize(table_tensor, dim=2)
+    sim_tensor_dict = {}
+    for col_tuple in col_tuple_list:
+        lattr_tensor = norm_table_tensor[col_tuple[0]]
+        rattr_tensor = norm_table_tensor[col_tuple[1]]
+        mul_tensor = lattr_tensor * rattr_tensor
+        sim_tensor = torch.sum(mul_tensor, 1)
+        sim_tensor_dict[predictions_attrs[col_tuple[0]].replace('ltable_', '')] = sim_tensor
+    return sim_tensor_dict
+
+
+def ml_er(iter_round: int, config: Configuration = None, ):
+    ltable = pd.read_csv(ltable_path, encoding='ISO-8859-1')
+    cm.set_key(ltable, ltable_id)
+    # ltable.fillna("", inplace=True)
+    rtable = pd.read_csv(rtable_path, encoding='ISO-8859-1')
+    cm.set_key(rtable, rtable_id)
+    # rtable.fillna("", inplace=True)
+    mappings = pd.read_csv(mapping_path, encoding='ISO-8859-1')
+
+    # 仅保留两表中出现在映射表中的行，增大正样本比例
+    lid_mapping_list = []
+    rid_mapping_list = []
+    # 全部转为字符串
+    # ltable = ltable.astype(str)
+    # rtable = rtable.astype(str)
+    # mappings = mappings.astype(str)
+    matching_number = len(mappings)  # 所有阳性样本数
+
+    for index, row in mappings.iterrows():
+        lid_mapping_list.append(row[mapping_lid])
+        rid_mapping_list.append(row[mapping_rid])
+
+    selected_ltable = ltable[ltable[ltable_id].isin(lid_mapping_list)]
+    # if len(lr_attrs_map) > 0:
+    #     selected_ltable = selected_ltable.rename(columns=lr_attrs_map)  # 参照右表，修改左表中与右表对应但不同名的字段
+    tables_id = rtable_id
+    selected_rtable = rtable[rtable[rtable_id].isin(rid_mapping_list)]
+    selected_attrs = selected_ltable.columns.values.tolist()  # 两张表中的字段名
+    items_but_id = selected_attrs[:]
+    items_but_id.remove(tables_id)  # 两张表中除了id的字段名
+    attrs_with_l_prefix = ['ltable_' + i for i in selected_attrs]
+    attrs_with_r_prefix = ['rtable_' + i for i in selected_attrs]
+    cm.set_key(selected_ltable, tables_id)
+    cm.set_key(selected_rtable, tables_id)
+
+    if config is not None:
+        ml_matcher = config["ml_matcher"]
+        if ml_matcher == "dt":
+            matcher = em.DTMatcher(name='DecisionTree', random_state=0)
+        elif ml_matcher == "svm":
+            matcher = em.SVMMatcher(name='SVM', random_state=0)
+        elif ml_matcher == "rf":
+            matcher = em.RFMatcher(name='RF', random_state=0)
+        elif ml_matcher == "lg":
+            matcher = em.LogRegMatcher(name='LogReg', random_state=0)
+        elif ml_matcher == "ln":
+            matcher = em.LinRegMatcher(name='LinReg')
+        elif ml_matcher == "nb":
+            matcher = em.NBMatcher(name='NaiveBayes')
+
+        if config["ml_blocker"] == "over_lap":
+            blocker = em.OverlapBlocker()
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"],
+                                             config["block_attr"], allow_missing=True,
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
+                                             overlap_size=config["overlap_size"], show_progress=False)
+        elif config["ml_blocker"] == "attr_equiv":
+            blocker = em.AttrEquivalenceBlocker()
+            candidate = blocker.block_tables(selected_ltable, selected_rtable, config["block_attr"],
+                                             config["block_attr"], allow_missing=True,
+                                             l_output_attrs=selected_attrs, r_output_attrs=selected_attrs)
+    else:
+        matcher = em.SVMMatcher(name='SVM', random_state=0)
+        blocker = em.OverlapBlocker()
+        candidate = blocker.block_tables(selected_ltable, selected_rtable, selected_attrs[-1], selected_attrs[-1],
+                                         l_output_attrs=selected_attrs, r_output_attrs=selected_attrs,
+                                         overlap_size=1, show_progress=False, allow_missing=True)
+
+    candidate['gold'] = 0
+    candidate = candidate.reset_index(drop=True)
+    candidate_match_rows = []
+    for row in candidate.itertuples():
+        l_id = getattr(row, 'ltable_' + tables_id)
+        map_row = mappings[mappings[mapping_lid] == l_id]
+
+        if map_row is not None:
+            r_id = map_row[mapping_rid]
+            for value in r_id:
+                if value == getattr(row, 'rtable_' + tables_id):
+                    candidate_match_rows.append(row[0])
+        else:
+            continue
+
+    for _ in candidate_match_rows:
+        candidate.loc[_, 'gold'] = 1
+
+    candidate.fillna(value="", inplace=True)
+
+    # 裁剪负样本，保持正负样本数量一致
+    candidate_mismatch = candidate[candidate['gold'] == 0]
+    candidate_match = candidate[candidate['gold'] == 1]
+    if len(candidate_mismatch) > len(candidate_match):
+        candidate_mismatch = candidate_mismatch.sample(n=len(candidate_match))
+    # 拼接正负样本
+    candidate_for_train_test = pd.concat([candidate_mismatch, candidate_match])
+    # 如果拼接后不重设索引可能导致索引重复
+    candidate_for_train_test = candidate_for_train_test.reset_index(drop=True)
+    cm.set_key(candidate_for_train_test, '_id')
+    cm.set_fk_ltable(candidate_for_train_test, 'ltable_' + tables_id)
+    cm.set_fk_rtable(candidate_for_train_test, 'rtable_' + tables_id)
+    cm.set_ltable(candidate_for_train_test, selected_ltable)
+    cm.set_rtable(candidate_for_train_test, selected_rtable)
+
+    # 分为训练测试集
+    train_proportion = 0.7
+    test_proportion = 0.3
+    sets = em.split_train_test(candidate_for_train_test, train_proportion=train_proportion, random_state=0)
+    train_set = sets['train']
+    test_set = sets['test']
+
+    feature_table = em.get_features_for_matching(selected_ltable, selected_rtable, validate_inferred_attr_types=False)
+
+    train_feature_vecs = em.extract_feature_vecs(train_set,
+                                                 feature_table=feature_table,
+                                                 attrs_after=['gold'],
+                                                 show_progress=False)
+    train_feature_vecs.fillna(value=0, inplace=True)
+
+    test_feature_after = attrs_with_l_prefix[:]
+    test_feature_after.extend(attrs_with_r_prefix)
+    for _ in test_feature_after:
+        if _.endswith(tables_id):
+            test_feature_after.remove(_)
+    test_feature_after.append('gold')
+    test_feature_vecs = em.extract_feature_vecs(test_set, feature_table=feature_table,
+                                                attrs_after=test_feature_after, show_progress=False)
+    test_feature_vecs.fillna(value=0, inplace=True)
+
+    fit_exclude = ['_id', 'ltable_' + tables_id, 'rtable_' + tables_id, 'gold']
+    matcher.fit(table=train_feature_vecs, exclude_attrs=fit_exclude, target_attr='gold')
+    test_feature_after.extend(['_id', 'ltable_' + tables_id, 'rtable_' + tables_id])
+    predictions = matcher.predict(table=test_feature_vecs, exclude_attrs=test_feature_after,
+                                  append=True, target_attr='predicted', inplace=False)
+    eval_result = em.eval_matches(predictions, 'gold', 'predicted')
+    em.print_eval_summary(eval_result)
+    indicators = evaluate_prediction(predictions, 'gold', 'predicted', matching_number, candidate_for_train_test)
+    print(indicators)
+
+    # 计算可解释性
+    ################################################################################################################
+    predictions_attrs = []
+    predictions_attrs.extend(attrs_with_l_prefix)
+    predictions_attrs.extend(attrs_with_r_prefix)
+    predictions_attrs.extend(['gold', 'predicted'])
+    predictions = predictions[predictions_attrs]
+    process_prediction_for_md_discovery(predictions)
+    predictions = predictions.reset_index(drop=True)
+    predictions = predictions.astype(str)
+    sim_tensor_dict = build_col_pairs_sim_tensor_dict(predictions)
+
+    md_paths = [md_output_dir + 'mds.txt', md_output_dir + 'vio.txt']
+    md_list = load_mds(md_paths)  # 从全局变量中读取所有的md
+    epl_match = 0  # 可解释，预测match
+    if len(md_list) > 0:
+        for row in predictions.itertuples():
+            if is_explicable(row, md_list, sim_tensor_dict) and str(getattr(row, 'predicted')) == str(1):
+                epl_match += 1
+
+    df = predictions[predictions['predicted'] == str(1)]
+    interpretability = epl_match / len(df)  # 可解释性
+    if (indicators["block_recall"] < 0.8) and (indicators["block_recall"] < indicators["recall"]):
+        f1 = (2.0 * indicators["precision"] * indicators["block_recall"]) / (
+                    indicators["precision"] + indicators["block_recall"])
+    else:
+        f1 = indicators["F1"]
+    performance = interpre_weight * interpretability + (1 - interpre_weight) * f1
+    ################################################################################################################
+
+    output_path = er_output_dir + "eval_result_" + str(iter_round) + ".txt"
+    with open(output_path, 'w') as f:
+        for key, value in six.iteritems(_get_metric(eval_result)):
+            f.write(key + " : " + value)
+            f.write('\n')
+        f.write('block_recall:' + str(indicators["block_recall"]) + '\n')
+        f.write('interpretability:' + str(interpretability) + '\n')
+        f.write('performance:' + str(performance) + '\n')
+
+
+if __name__ == '__main__':
+    iterations = 1
+    filename_list = os.listdir(er_output_dir)
+    if len(filename_list) > 0:
+        for _ in filename_list:
+            if _.startswith('eval_result'):
+                iterations = int(_[12:13]) + 1
+
+    if iterations > 1:
+        incumbent_array = np.load(hpo_output_dir + 'incumbent.npy')
+        with open(hpo_output_dir + "configspace.json", 'r') as f:
+            dict_configspace = json.load(f)
+        str_configspace = json.dumps(dict_configspace)
+        configspace = csj.read(str_configspace)
+        configuration = ConfigSpace.Configuration(configspace, vector=incumbent_array)
+        ml_er(iterations, configuration)
+    else:
+        ml_er(1)

script/get_support_and_confidence.py

@@ -1,48 +0,0 @@
-import time
-from functions.multi_process_infer_by_pairs import inference_from_record_pairs
-from functions.multi_process_infer_by_pairs import get_mds_metadata
-
-if __name__ == '__main__':
-    # 目前可以仿照这个main函数写
-    path = "/home/w/PycharmProjects/matching_dependency/input/T_positive_with_id_concat_single_tuple.csv"
-    start = time.time()
-    # 输入：csv文件路径，md左侧相似度阈值，md右侧目标字段
-    # 输出：2个md列表，列表1中md无violation,列表2中md有violation但confidence满足阈值(0.8)
-    # 例如此处输入参数要求md左侧相似度字段至少为0.7，右侧指向'id'字段
-    mds, mds_vio = inference_from_record_pairs(path, 0.1, 'id_concat')
-
-    # 如果不需要输出support和confidence，去掉下面两行
-    mds_meta = get_mds_metadata(mds, path, 'id_concat')
-    mds_vio_meta = get_mds_metadata(mds_vio, path, 'id_concat')
-
-    # # 若不输出support和confidence，使用以下两块代码
-    # # 将列表1写入本地，路径需自己修改
-    # md_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_md_list.txt'
-    # with open(md_path, 'w') as f:
-    #     for _ in mds:
-    #         f.write(str(_) + '\n')
-    #
-    # # 将列表2写入本地，路径需自己修改
-    # vio_path = '/home/w/A-New Folder/8.14/Paper Dataset/TP_vio_list.txt'
-    # with open(vio_path, 'w') as f:
-    #     for _ in mds_vio:
-    #         f.write(str(_) + '\n')
-
-    # 若输出support和confidence，使用以下两块代码
-    # 将列表1写入本地，路径需自己修改
-    md_path = "output/md.txt"
-    with open(md_path, 'w') as f:
-        for _ in mds_meta:
-            for i in _.keys():
-                f.write(i + ':' + str(_[i]) + '\t')
-            f.write('\n')
-
-    # 将列表2写入本地，路径需自己修改
-    vio_path = "output/vio.txt"
-    with open(vio_path, 'w') as f:
-        for _ in mds_vio_meta:
-            for i in _.keys():
-                f.write(i + ':' + str(_[i]) + '\t')
-            f.write('\n')
-
-    print(time.time() - start)

set_none.py

@@ -0,0 +1,27 @@
+import numpy as np
+import pandas as pd
+
+if __name__ == '__main__':
+    lt = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\tmdb.csv',
+                     encoding='ISO-8859-1', sep=',')
+    rt = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\tvdb.csv',
+                     encoding='ISO-8859-1', sep=',')
+    mapping = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\gt.csv',
+                          encoding='ISO-8859-1', sep=',')
+    # 删除列
+    # lt.drop(columns=['numberOfSeasons', 'numberOfEpisodes', 'birthDate', 'last_air_date', 'release_year', 'runtime',
+    #                  'genre_list', 'origin_country'], inplace=True)
+    # rt.drop(columns=['job'], inplace=True)
+    # 列名调整
+    # rt = rt.rename(columns={'birthDate': 'birthYear', 'release_year': 'startYear',
+    #                         'last_air_date': 'endYear', 'runtime': 'runtimeMinutes'})
+    # 列顺序调整
+    # rt = rt[['id', 'title', 'name', 'episodeNumber', 'seasonNumber', 'birthYear', 'endYear', 'startYear', 'genre_list',
+    #         'runtimeMinutes']]
+    # lt.to_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\tmdb.csv', sep=',', index=False, header=True, quoting=1)
+    # rt.to_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\tvdb.csv', sep=',', index=False, header=True, quoting=1)
+    # mapping.to_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\TMDB-TVDB\gt.csv',
+    #                sep=',', index=False, header=True, quoting=1)
+    print(1)
+
+

settings.py

@@ -0,0 +1,20 @@
+from sentence_transformers import SentenceTransformer
+import numpy as np
+
+ltable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\iTunes-Amazon_dirty\tableA.csv'
+rtable_path = r'E:\Data\Research\Projects\matching_dependency\datasets\iTunes-Amazon_dirty\tableB.csv'
+mapping_path = r'E:\Data\Research\Projects\matching_dependency\datasets\iTunes-Amazon_dirty\matches.csv'
+mapping_lid = 'ltable_id'  # mapping表中左表id名
+mapping_rid = 'rtable_id'  # mapping表中右表id名
+ltable_id = 'id'  # 左表id字段名称
+rtable_id = 'id'  # 右表id字段名称
+target_attr = 'id'  # 进行md挖掘时的目标字段
+# lr_attrs_map = {}  # 如果两个表中存在对应字段名称不一样的情况，将名称加入列表便于调整一致
+similarity_threshold = 0.16
+support_threshold = 70
+confidence_threshold = 0.3
+interpre_weight = 1  # 可解释性权重
+er_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\ml_er\\output\\'
+md_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\md_discovery\\output\\'
+hpo_output_dir = 'E:\\Data\\Research\\Projects\\matching_dependency\\hpo\\output\\'
+model = SentenceTransformer('E:\\Data\\Research\\Models\\roberta-large-nli-stsb-mean-tokens')

tfile.py

@@ -0,0 +1,141 @@
+import json
+import multiprocessing
+import os
+import time
+
+import ConfigSpace
+import numpy as np
+import pandas as pd
+import torch
+from tqdm import tqdm
+from ConfigSpace.read_and_write import json as csj
+from md_discovery import tmp_discover
+from settings import er_output_dir, similarity_threshold, target_attr, hpo_output_dir
+
+
+def fuck(i):
+    i = i * i + 1
+
+
+def test1():
+    li = [[[6, 6, 2],
+           [2, 4, 6],
+           [2, 4, 7],
+           [3, 6, 4]],
+          [[6, 2, 7],
+           [3, 2, 4],
+           [5, 3, 5],
+           [6, 2, 4]],
+          [[7, 2, 2],
+           [6, 3, 2],
+           [6, 4, 3],
+           [6, 5, 6]]]
+    tensor = torch.Tensor(li)
+    norm_tensor = torch.nn.functional.normalize(tensor, dim=2)
+    print(norm_tensor, '\n')
+    sim_ten = torch.matmul(norm_tensor, norm_tensor.transpose(1, 2))
+    print(sim_ten / 2 + 0.5, '\n')
+    print(sim_ten.size())
+
+
+def test2():
+    multiprocessing.set_start_method("spawn")
+    manager = multiprocessing.Manager()
+    lock = manager.Lock()
+    pool = multiprocessing.Pool(16)
+    with manager:
+        for _ in tqdm(range(0, 1000)):
+            result = pool.apply_async(fuck, args=(_,))
+            print(result)
+
+
+def test3():
+    dic = {'description': 0, 'id': 1, 'manufacturer': 0, 'name': 0.9309734582901001, 'price': 0.912541675567627}
+    ll = list(dic.values())
+    ten = torch.Tensor(ll)
+    t = ten.unsqueeze(1)
+    t = t.unsqueeze(2)
+    y = t.repeat(1, 742, 742)
+    print(ten)
+    print(y)
+    print(torch.isfinite(ten))
+    print(torch.count_nonzero(y).item())
+
+
+def test4():
+    one_bool_tensor = torch.ones((3, 3, 3), dtype=torch.bool)
+    print(torch.count_nonzero(one_bool_tensor).item())
+
+
+def test5():
+    ten1 = torch.tensor([[1, 2, 3],
+                         [7, 8, 9]])
+    ten2 = torch.tensor([[4, 5, 6],
+                         [11, 12, 15]])
+    result = ten1 * ten2
+    r = torch.sum(result, 1)
+    print('\n')
+    print(result)
+    print(r)
+
+
+def test6():
+    table_tensor = torch.tensor([[[1., 2., 3.],
+                                  [4., 5., 6.],
+                                  [7., 8., 9.]],
+                                 [[1., 2., 3.],
+                                  [4., 5., 6.],
+                                  [7., 8., 9.]]])
+    t = torch.tensor([[1., 2., 3.],
+                      [4., 5., 6.]])
+    norm1 = torch.nn.functional.normalize(table_tensor, dim=1)
+    norm2 = torch.nn.functional.normalize(table_tensor, dim=2)
+    print('\n')
+    print(norm1)
+    print(norm2)
+    print(t.shape)
+
+
+def test7():
+    iterations = 1
+    filename_list = os.listdir(er_output_dir)
+    if len(filename_list) > 0:
+        for _ in filename_list:
+            if _.startswith('eval_result'):
+                iterations = int(_[12:13]) + 1
+    print(iterations)
+
+
+def test8():
+    cum = np.load(hpo_output_dir + 'incumbent.npy')
+    with open(hpo_output_dir + "configspace.json", 'r') as load_f:
+        dict_configspace = json.load(load_f)
+    str_configspace = json.dumps(dict_configspace)
+    configspace = csj.read(str_configspace)
+    config = ConfigSpace.Configuration(configspace, vector=cum)
+    print(cum)
+
+
+def test9():
+    df = pd.read_json(r'./datasets/t.json', encoding='ISO-8859-1', lines=True)
+    df.to_csv(r'./datasets/s.csv')
+    d = pd.read_csv(r'./datasets/s.csv', encoding='ISO-8859-1')
+    print(1)
+
+
+def test10():
+    rtable = pd.read_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableB.csv',
+                         encoding='ISO-8859-1')
+    rtable.columns = ["id", "title", "authors", "venue", "year"]
+    rtable.to_csv(r'E:\Data\Research\Projects\matching_dependency\datasets\DBLP-GoogleScholar\tableB.csv',
+                  sep=',', index=False, header=True, quoting=1)
+
+
+if __name__ == '__main__':
+    start = time.time()
+    t_single_tuple_path = er_output_dir + "t_single_tuple.csv"
+    # tp_mds, tp_vio = inference_from_record_pairs(tp_single_tuple_path, similarity_threshold, target_attr)
+    tp_mds, tp_vio = tmp_discover.pairs_inference(t_single_tuple_path, similarity_threshold, target_attr)
+    print(time.time() - start)
+
+