#!/usr/bin/env python # -*- coding: utf-8 -*- #import gevent.monkey #gevent.monkey.patch_socket() from gevent import monkey, sleep monkey.patch_all() import gevent import web import httplib, urllib, base64 import json import time from multiprocessing.dummy import Pool as ThreadPool urls=( '/bof333?(.+)','get_user' ) app = web.application(urls,globals()) result=[] def api1(deinput): global data1 exp1 = "Composite(AA.AuId = %d )" % deinput[0] params1 = urllib.urlencode({ 'expr':exp1, 'offset': '0', 'model': 'latest', 'count': '1000000', 'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() conn.close() data1=json.loads(data) def api2(deinput): global data2 exp2 = "Composite(AA.AuId = %d )" % deinput[1] params1 = urllib.urlencode({ 'expr':exp2, 'offset': '0', 'model': 'latest', 'count': '1000000', 'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() conn.close() data2=json.loads(data) def api4(deinput): global data4 exp4 = 'Id = %d' % deinput[0] params1 = urllib.urlencode({ 'expr':exp4, 'model': 'latest', 'offset': '0', 'count': '1000000', 'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data4 = json.loads(data) conn.close() def api5(deinput): global data5 exp5 = 'Id = %d' % deinput[1] params1 = urllib.urlencode({ 'expr':exp5, 'count': '1000000', 'model': 'latest', 'attributes': 'F.FId,AA.AuId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data5 = json.loads(data) conn.close() def api3(deinput): global data3 exp3 = 'RId = %d' % deinput[1] params1 = urllib.urlencode({ 'expr':exp3, 'count': '1000000', 'model': 'latest', 'attributes': 'Id,F.FId,AA.AuId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data3 = json.loads(data) conn.close() def idid4api(Reid,deinput,FId2,AuId2,CJId2,Id3): now_time = time.clock() exe_time = (float)(now_time - start_time) if exe_time > 200: return exp4 = 'Id = %d' % Reid params1 = urllib.urlencode({ 'expr':exp4, 'count': '1000000', 'model': 'latest', 'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data4 = json.loads(data) conn.close() #print data4 try: ReId4= data4["entities"][0]["RId"] except (KeyError): ReId4 = [] try: FId4 = data4["entities"][0]["F"] except (KeyError): FId4 = [] try: AuId4= data4["entities"][0]["AA"] except (KeyError): AuId4=[] try: CJId4= data4["entities"][0]["J"]["JId"] except (KeyError): try: CJId4= data4["entities"][0]["C"]["CId"] except: CJId4=[] #id -> id -> id #id -> id -> id -> id for rid in ReId4: if deinput[1] == rid: result.append([deinput[0],Reid,deinput[1]]) for Id in Id3: if rid == Id: result.append([deinput[0],Reid,rid,deinput[1]]) #id -> id -> F.FId -> id for x in FId2: for y in FId4: if y['FId']==x['FId']: result.append([deinput[0],Reid,x['FId'],deinput[1]]) #id -> id -> AA.AuId -> id for x in AuId2: for y in AuId4: if y['AuId']==x['AuId']: result.append([deinput[0],Reid,x['AuId'],deinput[1]]) #id -> id -> C.CId/J.JId -> id if CJId4==CJId2: result.append([deinput[0],Reid,CJId4,deinput[1]]) def idauid3api(author2,deinput,AfId1): now_time = time.clock() exe_time = (float)(now_time - start_time) if exe_time > 200: return exp3 = "Composite(AA.AuId = %d )" % author2["AuId"] params1 = urllib.urlencode({ 'expr':exp3, 'model': 'latest', 'count': '1000000', 'attributes': 'AA.AuId,AA.AfId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data3 = json.loads(data) conn.close() AfId3=[] for i3 in data3["entities"]: try: for tmp in i3["AA"]: if tmp["AuId"] == author2["AuId"]: if tmp["AfId"] not in AfId3: AfId3.append(tmp["AfId"]) break except (KeyError): pass for x in AfId3: for y in AfId1: if x == y : result.append([deinput[0],author2["AuId"],x,deinput[1]]) def idauid4api(rid,deinput,Id1): now_time = time.clock() exe_time = (float)(now_time - start_time) if exe_time > 200: return exp4 = 'Id = %d' % rid params1 = urllib.urlencode({ 'expr':exp4, 'count': '1000000', 'model': 'latest', 'attributes': 'RId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data4 = json.loads(data) conn.close() #print data4 try: ReId4= data4["entities"][0]["RId"] except (KeyError): ReId4 = [] # AA.AuId -> id -> id -> id for rid4 in ReId4: for tmp in Id1: if rid4 == tmp: result.append([deinput[0],rid,rid4,deinput[1]]) def auidid4api(Aid,deinput,Id3): now_time = time.clock() exe_time = (float)(now_time - start_time) if exe_time > 200: return exp4 = 'Id = %d' % Aid params1 = urllib.urlencode({ 'expr':exp4, 'count': '1000000', 'attributes': 'RId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data4 = json.loads(data) conn.close() #print data4 try: ReId4= data4["entities"][0]["RId"] except (KeyError): ReId4 = [] # AA.AuId -> id -> id # AA.AuId -> id -> id -> id for rid in ReId4: if deinput[1] == rid: result.append([deinput[0],Aid,deinput[1]]) for tmp in Id3: if rid == tmp: result.append([deinput[0],Aid,rid,deinput[1]]) def auau3api(aaid1,deinput,Id2): now_time = time.clock() exe_time = (float)(now_time - start_time) if exe_time > 200: return exp3 = 'Id = %d' % aaid1 params1 = urllib.urlencode({ 'expr':exp3, 'count': '1000000', 'attributes': 'RId', 'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6', }) conn = httplib.HTTPSConnection('oxfordhk.azure-api.net') conn.request("GET", "/academic/v1.0/evaluate?%s" % params1) response = conn.getresponse() data = response.read() data3 = json.loads(data) conn.close() try: ReId3=data3["entities"][0]["RId"] except (KeyError): ReId3 = [] # AA.AuId -> Id -> Id -> AA.AuId for x in ReId3: for y in Id2: if x==y: result.append([deinput[0],aaid1,x,deinput[1]]) def ididfor(i): try: if i["Id"] not in Id3: Id3.append(i["Id"]) except(KeyError): pass # id -> F.FId -> id -> id try: FId_data3 = i["F"] for tmp in FId_data3: for x in FId1: if x["FId"]==tmp["FId"]: result.append([xxinput[0],tmp["FId"],i["Id"],xxinput[1]]) #FId3.append(tmp["FId"])# except(KeyError): pass # id -> AA.AuId -> id -> id try: AuId_data3=i["AA"] for tmp in AuId_data3: for x in AuId1: if x["AuId"]==tmp["AuId"]: result.append([xxinput[0],tmp["AuId"],i["Id"],xxinput[1]]) #AuId3.append(tmp["AuId"])#'''''''''''''''''''''need optimization except(KeyError): pass # id -> C.CId/J.JId -> id -> id try: CJId3=i["J"]["JId"]#'''''''''''''''''''''need optimization if CJId3==CJId1: result.append([xxinput[0],CJId3,i["Id"],xxinput[1]]) except(KeyError): try: CJId3=i["C"]["CId"] if CJId3==CJId1: result.append([xxinput[0],CJId3,i["Id"],xxinput[1]]) except(KeyError): pass def idauidfor(i1): Id1.append(i1["Id"]) for tmp in ReId2: if i1["Id"] == tmp:# Id -> Id -> AA.AuId (2-hop) result.append([xxinput[0],i1["Id"],xxinput[1]]) for tmp in i1["AA"]: for x in AuId2: if x["AuId"] == tmp["AuId"]: # Id -> AA.AuId -> Id -> AA.AuId (3-hop) (new add~~) result.append([xxinput[0],x["AuId"],i1["Id"],xxinput[1]]) try: FId1 = i1["F"] for tmp in FId1: for x in FId2: if x["FId"] == tmp["FId"]: # Id -> F.FId -> Id -> AA.AuId (3-hop) result.append([xxinput[0],x["FId"],i1["Id"],xxinput[1]]) except (KeyError): pass try: for tmp in i1["AA"]: if tmp["AuId"] == xxinput[1]: if tmp["AfId"] not in AfId1: AfId1.append(tmp["AfId"]) break except (KeyError): pass try: CJId1 = i1["J"]["JId"] # Id -> C.CId/J.JId -> Id -> AA.AuId (3-hop) if CJId1 == CJId2: result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]]) except (KeyError): try: CJId1= i1["C"]["CId"] if CJId1 == CJId2: result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]]) except: CJId1=[] def auididfor(i1): Id1.append(i1["Id"]) if i1["Id"]==xxinput[1]: result.append([xxinput[0],xxinput[1]]) # AA.AuId -> Id try: FId1 = i1["F"] for tmp in FId1: for x in FId2: if x["FId"]==tmp["FId"]: # AA.AuId -> Id -> F.FId -> Id result.append([xxinput[0],i1["Id"],tmp["FId"],xxinput[1]]) except (KeyError): pass try: AuId1= i1["AA"] for tmp in AuId1: for x in AuId2: if tmp["AuId"]==x["AuId"]: # AA.AuId -> Id -> AA.AuId -> Id result.append([xxinput[0],i1["Id"],tmp["AuId"],xxinput[1]]) except (KeyError): pass try: for tmp in AuId1: if tmp["AuId"]==xxinput[0]: if tmp["AfId"] not in AfId1: AfId1.append(tmp["AfId"]) break except(KeyError): pass try: CJId1= i1["J"]["JId"] # AA.AuId -> Id -> C.CId/J.JId -> Id if CJId1 == CJId2: result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]]) except (KeyError): try: CJId1= i1["C"]["CId"] if CJId1 == CJId2: result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]]) except: CJId1=[] class get_user: def GET(self,user): input_data = web.input() #input = [2157025439,2139398774] input=[int(input_data.id1),int(input_data.id2)] encodeinput = json.dumps(input) deinput = json.loads(encodeinput) print deinput threads=[] threads.append(gevent.spawn(api1,deinput)) threads.append(gevent.spawn(api2,deinput)) threads.append(gevent.spawn(api3,deinput)) threads.append(gevent.spawn(api4,deinput)) threads.append(gevent.spawn(api5,deinput)) gevent.joinall(threads) flag=0 try: print data1["entities"][0]["Id"] except(KeyError,IndexError): flag=10;#id1 is Id try: print data2["entities"][0]["Id"] except(KeyError,IndexError): flag+=1;#id2 is Id if flag == 0: result=auau(deinput,data1,data2) elif flag == 1: result=auidid(deinput,data1,data3,data5) elif flag == 10: result=idauid(deinput,data4,data2) elif flag == 11: result=idid(deinput,data4,data5,data3) else: print "FLAG : %d " % flag deresult = json.dumps(result) print deresult print "Total Size: %d " % len(result) return deresult def idid(deinput,data4,data5,data3): global result,start_time,xxinput,FId1,AuId1,CJId1,Id3 start_time = time.clock() xxinput=deinput result=[] Id3=[] print "XXXXXXXXXXididXXXXXXXXXXX" data1 = data4 data2 = data5 #''''''''''''''''''data 1''''''''''''''''''''''''''' try: ReId1=data1["entities"][0]["RId"] except (KeyError): ReId1 = [] try: FId1 = data1["entities"][0]["F"] except (KeyError): FId1 = [] try: AuId1= data1["entities"][0]["AA"] except (KeyError): AuId1=[] try: CJId1= data1["entities"][0]["J"]["JId"] except (KeyError): try: CJId1= data1["entities"][0]["C"]["CId"] except: CJId1=[] #''''''''''''''''''data 2''''''''''''''''''''''''''' try: FId2 = data2["entities"][0]["F"] except (KeyError): FId2 = [] try: AuId2= data2["entities"][0]["AA"] except (KeyError): AuId2=[] try: CJId2= data2["entities"][0]["J"]["JId"] except (KeyError): try: CJId2= data2["entities"][0]["C"]["CId"] except: CJId2=[] #''''''''''''''''''data 3'''''''''id->XXXX->id->id''''''''''''''''''maybe big enough > 10w pool = ThreadPool(4) pool.map(ididfor,data3["entities"]) pool.close() pool.join() #''''''''''''''''''operation''''''''''1 and 2 hop''''''''''''''''' #id -> id for Reid in ReId1: if deinput[1] == Reid: result.append([deinput[0],Reid]) break #id -> F.FId -> id for x in FId2: #print 'x: %d ' % x['FId'] for y in FId1: #print y['FId'] if y['FId']==x['FId']: result.append([deinput[0],x['FId'],deinput[1]]) #id -> AA.AuId -> id for x in AuId2: for y in AuId1: if y['AuId']==x['AuId']: result.append([deinput[0],x['AuId'],deinput[1]]) #id -> C.CId/J.JId -> id if CJId1==CJId2: result.append([deinput[0],CJId1,deinput[1]]) #''''''''''''''''''''4th API''''''''''''''''''''''''''''' #id -> id -> XXXXXX -> id ************************************************************need gevent threads=[] for Reid in ReId1: threads.append(gevent.spawn(idid4api,Reid,deinput,FId2,AuId2,CJId2,Id3)) gevent.joinall(threads) return result def idauid(deinput,data4,data2): global result,start_time,ReId2,xxinput,AuId2,FId2,CJId2,Id1,AfId1 Id1=[] AfId1=[] start_time = time.clock() xxinput=deinput result=[] print "XXXXXXXXXXidauidXXXXXXXXXXX" data1 = data2 data2 = data4 #'''''''''''''''''''''data 2'''''''''''''''''''''''''''''''' print ("Data 2") print data2 try: ReId2 = data2["entities"][0]["RId"] except (KeyError): ReId2 = [] try: FId2 = data2["entities"][0]["F"] except (KeyError): FId2 = [] try: AuId2 = data2["entities"][0]["AA"] except (KeyError): AuId2 = [] try: CJId2 = data2["entities"][0]["J"]["JId"] except (KeyError): try: CJId2 = data2["entities"][0]["C"]["CId"] except: CJId2 =[] for tmp in AuId2: if deinput[1] == tmp["AuId"]: result.append([deinput[0],deinput[1]]) # Id -> AA.AuId (1-hop) #'''''''''''''''''''''data 1'''''''''''''''''''''''''''''''' pool = ThreadPool(4) pool.map(idauidfor,data1["entities"]) pool.close() pool.join() #'''''''''''''''''''''''''3rd API''''''''''''''''''''''''''''''''' threads=[] for author2 in AuId2: threads.append(gevent.spawn(idauid3api,author2,deinput,AfId1)) gevent.joinall(threads) #************************************************************need gevent #''''''''''''''''''''4th API'''''''''''''''''''''''''''''''''''' threads=[] for rid in ReId2: threads.append(gevent.spawn(idauid4api,rid,deinput,Id1)) gevent.joinall(threads) return result def auidid(deinput,data1,data3,data5): #'''''''''''''''''''''''''1st API''''''''''''''''''''''''''''''''' global result,start_time,Id1,AfId1,xxinput,FId2,AuId2,CJId2 result=[] Id1=[] AfId1=[] xxinput = deinput start_time = time.clock() print "XXXXXXXXXXauididXXXXXXXXXXX" data2 = data5 #'''''''''''''''''''''data 2'''''''''''''''''''''''''''''''' try: FId2 = data2["entities"][0]["F"] except (KeyError): FId2 = [] try: AuId2= data2["entities"][0]["AA"] except (KeyError): AuId2=[] try: CJId2= data2["entities"][0]["J"]["JId"] except (KeyError): try: CJId2= data2["entities"][0]["C"]["CId"] except: CJId2=[] #'''''''''''''''''''''data 1'''''''''''''''''''''''''''''''' pool = ThreadPool(4) pool.map(auididfor,data1["entities"]) pool.close() pool.join() # AA.AuId -> AA.AfId -> AA.AuId -> Id for x in AuId2: for y in AfId1: try: if y==x["AfId"]: result.append([deinput[0],y,x["AuId"],deinput[1]]) except(KeyError): continue #'''''''''''''''''''''data 3'''''''''''''''''''''''''''''''' Id3 = [] for i3 in data3["entities"]: Id3.append(i3["Id"]) #************************************************************need gevent #''''''''''''''''''''4th API'''''''''''''''''''''''''''''''''''' threads=[] for Aid in Id1: threads.append(gevent.spawn(auidid4api,Aid,deinput,Id3)) gevent.joinall(threads) return result def auau(deinput,data1,data2): global result,start_time result=[] start_time = time.clock() print "XXXXXXXXXXauidauidXXXXXXXXXX" #'''''''''''''''''''''data 2'''''''''''''''''''''''''''''''' AfId2=[] for id2 in data2["entities"]: try: AuId2= id2["AA"] for tmp in AuId2: if tmp["AuId"]==deinput[1]: if tmp["AfId"] not in AfId2: AfId2.append(tmp["AfId"]) break except (KeyError): continue Id2=[] for id2 in data2["entities"]: Id2.append(id2["Id"]) #'''''''''''''''''''''data 1'''''''''''''''''''''''''''''''' #print data1 Id1=[] AfId1=[] for i1 in data1["entities"]: Id1.append(i1["Id"]) print Id1 for i1 in data1["entities"]: try: AuId1= i1["AA"] for tmp in AuId1: if tmp["AuId"]==deinput[0]: if tmp["AfId"] not in AfId1: AfId1.append(tmp["AfId"]) break except (KeyError): continue # AA.AuId -> AA.AfId -> AA.AuId for x in AfId1: for y in AfId2: if x==y: result.append([deinput[0],x,deinput[1]]) # AA.AuId -> Id -> AA.AuId for x in Id1: for y in Id2: if x==y: result.append([deinput[0],x,deinput[1]]) #'''''''''''''''''''''''''3rd API''''''''''''''''''''''''''''''''' threads=[] for aaid1 in Id1: threads.append(gevent.spawn(auau3api,aaid1,deinput,Id2)) gevent.joinall(threads) return result if __name__ == '__main__': app.run()