You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

757 lines
22 KiB

9 years ago
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#import gevent.monkey
#gevent.monkey.patch_socket()
from gevent import monkey, sleep
monkey.patch_all()
import gevent,os,sys
import web
import httplib, urllib, base64
import json
import time
#from multiprocessing.dummy import Pool as ThreadPool
#from multiprocessing import Pool
#import multiprocessing, logging
#logger = multiprocessing.log_to_stderr()
#logger.setLevel(multiprocessing.SUBDEBUG)
#from gevent.pool import Pool
from multiprocessing import Process, Lock
lock = Lock()
urls=(
'/bof333?(.+)','get_user'
)
app = web.application(urls,globals())
result=[]
def api1(deinput):
global data1
exp1 = "Composite(AA.AuId = %d )" % deinput[0]
params1 = urllib.urlencode({
'expr':exp1,
'offset': '0',
'model': 'latest',
'count': '1000000',
'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
conn.close()
data1=json.loads(data)
def api2(deinput):
global data2
exp2 = "Composite(AA.AuId = %d )" % deinput[1]
params1 = urllib.urlencode({
'expr':exp2,
'offset': '0',
'model': 'latest',
'count': '1000000',
'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
conn.close()
data2=json.loads(data)
def api4(deinput):
global data4
exp4 = 'Id = %d' % deinput[0]
params1 = urllib.urlencode({
'expr':exp4,
'model': 'latest',
'offset': '0',
'count': '1000000',
'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data4 = json.loads(data)
conn.close()
def api5(deinput):
global data5
exp5 = 'Id = %d' % deinput[1]
params1 = urllib.urlencode({
'expr':exp5,
'count': '1000000',
'model': 'latest',
'attributes': 'F.FId,AA.AuId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data5 = json.loads(data)
conn.close()
def api3(deinput):
global data3
exp3 = 'RId = %d' % deinput[1]
params1 = urllib.urlencode({
'expr':exp3,
'count': '1000000',
'model': 'latest',
'attributes': 'Id,F.FId,AA.AuId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data3 = json.loads(data)
conn.close()
def idid4api(Reid,deinput,FId2,AuId2,CJId2,Id3):
now_time = time.clock()
exe_time = (float)(now_time - start_time)
if exe_time > 200:
print ">200"
return
exp4 = 'Id = %d' % Reid
params1 = urllib.urlencode({
'expr':exp4,
'count': '1000000',
'model': 'latest',
'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data4 = json.loads(data)
conn.close()
#print data4
try:
ReId4 = data4["entities"][0]["RId"]
except (KeyError):
ReId4 = []
try:
FId4 = data4["entities"][0]["F"]
except (KeyError):
FId4 = []
try:
AuId4= data4["entities"][0]["AA"]
except (KeyError):
AuId4=[]
try:
CJId4= data4["entities"][0]["J"]["JId"]
except (KeyError):
try:
CJId4= data4["entities"][0]["C"]["CId"]
except:
CJId4=[]
#id -> id -> id #id -> id -> id -> id
for rid in ReId4:
if deinput[1] == rid:
result.append([deinput[0],Reid,deinput[1]])
for Id in Id3:
if rid == Id:
result.append([deinput[0],Reid,rid,deinput[1]])
#id -> id -> F.FId -> id
for x in FId2:
for y in FId4:
if y['FId']==x['FId']:
result.append([deinput[0],Reid,x['FId'],deinput[1]])
#id -> id -> AA.AuId -> id
for x in AuId2:
for y in AuId4:
if y['AuId']==x['AuId']:
result.append([deinput[0],Reid,x['AuId'],deinput[1]])
#id -> id -> C.CId/J.JId -> id
if CJId4==CJId2:
result.append([deinput[0],Reid,CJId4,deinput[1]])
def idauid3api(author2,deinput,AfId1):
now_time = time.clock()
exe_time = (float)(now_time - start_time)
if exe_time > 200:
return
exp3 = "Composite(AA.AuId = %d )" % author2["AuId"]
params1 = urllib.urlencode({
'expr':exp3,
'model': 'latest',
'count': '1000000',
'attributes': 'AA.AuId,AA.AfId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data3 = json.loads(data)
conn.close()
AfId3=[]
for i3 in data3["entities"]:
try:
for tmp in i3["AA"]:
if tmp["AuId"] == author2["AuId"]:
if tmp["AfId"] not in AfId3:
AfId3.append(tmp["AfId"])
break
except (KeyError):
pass
for x in AfId3:
for y in AfId1:
if x == y :
result.append([deinput[0],author2["AuId"],x,deinput[1]])
def idauid4api(rid,deinput,Id1):
now_time = time.clock()
exe_time = (float)(now_time - start_time)
if exe_time > 200:
return
exp4 = 'Id = %d' % rid
params1 = urllib.urlencode({
'expr':exp4,
'count': '1000000',
'model': 'latest',
'attributes': 'RId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data4 = json.loads(data)
conn.close()
#print data4
try:
ReId4= data4["entities"][0]["RId"]
except (KeyError):
ReId4 = []
# AA.AuId -> id -> id -> id
for rid4 in ReId4:
for tmp in Id1:
if rid4 == tmp:
result.append([deinput[0],rid,rid4,deinput[1]])
def auidid4api(Aid,deinput,Id3):
now_time = time.clock()
exe_time = (float)(now_time - start_time)
if exe_time > 200:
return
exp4 = 'Id = %d' % Aid
params1 = urllib.urlencode({
'expr':exp4,
'count': '1000000',
'attributes': 'RId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data4 = json.loads(data)
conn.close()
#print data4
try:
ReId4= data4["entities"][0]["RId"]
except (KeyError):
ReId4 = []
# AA.AuId -> id -> id # AA.AuId -> id -> id -> id
for rid in ReId4:
if deinput[1] == rid:
result.append([deinput[0],Aid,deinput[1]])
for tmp in Id3:
if rid == tmp:
result.append([deinput[0],Aid,rid,deinput[1]])
def auau3api(aaid1,deinput,Id2):
now_time = time.clock()
exe_time = (float)(now_time - start_time)
if exe_time > 200:
return
exp3 = 'Id = %d' % aaid1
params1 = urllib.urlencode({
'expr':exp3,
'count': '1000000',
'attributes': 'RId',
'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
})
conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
response = conn.getresponse()
data = response.read()
data3 = json.loads(data)
conn.close()
try:
ReId3=data3["entities"][0]["RId"]
except (KeyError):
ReId3 = []
# AA.AuId -> Id -> Id -> AA.AuId
for x in ReId3:
for y in Id2:
if x==y:
result.append([deinput[0],aaid1,x,deinput[1]])
def ididfor(lock,i):
#lock.acquire()
with lock:
if i["Id"] not in Id3:
Id3.append(i["Id"])
#lock.release()
# id -> F.FId -> id -> id
try:
FId_data3 = i["F"]
for tmp in FId_data3:
for x in FId1:
if x["FId"]==tmp["FId"]:
with lock:
result.append([xxinput[0],tmp["FId"],i["Id"],xxinput[1]])
#FId3.append(tmp["FId"])#
except(KeyError):
pass
# id -> AA.AuId -> id -> id
try:
AuId_data3=i["AA"]
for tmp in AuId_data3:
for x in AuId1:
if x["AuId"]==tmp["AuId"]:
with lock:
result.append([xxinput[0],tmp["AuId"],i["Id"],xxinput[1]])
#AuId3.append(tmp["AuId"])#'''''''''''''''''''''need optimization
except(KeyError):
pass
# id -> C.CId/J.JId -> id -> id
try:
CJId3=i["J"]["JId"]#'''''''''''''''''''''need optimization
if CJId3==CJId1:
with lock:
result.append([xxinput[0],CJId3,i["Id"],xxinput[1]])
except(KeyError):
try:
CJId3=i["C"]["CId"]
if CJId3==CJId1:
with lock:
result.append([xxinput[0],CJId3,i["Id"],xxinput[1]])
except(KeyError):
pass
def idauidfor(i1):
Id1.append(i1["Id"])
for tmp in ReId2:
if i1["Id"] == tmp:# Id -> Id -> AA.AuId (2-hop)
result.append([xxinput[0],i1["Id"],xxinput[1]])
for tmp in i1["AA"]:
for x in AuId2:
if x["AuId"] == tmp["AuId"]:
# Id -> AA.AuId -> Id -> AA.AuId (3-hop) (new add~~)
result.append([xxinput[0],x["AuId"],i1["Id"],xxinput[1]])
try:
FId1 = i1["F"]
for tmp in FId1:
for x in FId2:
if x["FId"] == tmp["FId"]:
# Id -> F.FId -> Id -> AA.AuId (3-hop)
result.append([xxinput[0],x["FId"],i1["Id"],xxinput[1]])
except (KeyError):
pass
try:
for tmp in i1["AA"]:
if tmp["AuId"] == xxinput[1]:
if tmp["AfId"] not in AfId1:
AfId1.append(tmp["AfId"])
break
except (KeyError):
pass
try:
CJId1 = i1["J"]["JId"] # Id -> C.CId/J.JId -> Id -> AA.AuId (3-hop)
if CJId1 == CJId2:
result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]])
except (KeyError):
try:
CJId1= i1["C"]["CId"]
if CJId1 == CJId2:
result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]])
except:
CJId1=[]
def auididfor(i1):
Id1.append(i1["Id"])
if i1["Id"]==xxinput[1]:
result.append([xxinput[0],xxinput[1]]) # AA.AuId -> Id
try:
FId1 = i1["F"]
for tmp in FId1:
for x in FId2:
if x["FId"]==tmp["FId"]:
# AA.AuId -> Id -> F.FId -> Id
result.append([xxinput[0],i1["Id"],tmp["FId"],xxinput[1]])
except (KeyError):
pass
try:
AuId1= i1["AA"]
for tmp in AuId1:
for x in AuId2:
if tmp["AuId"]==x["AuId"]:
# AA.AuId -> Id -> AA.AuId -> Id
result.append([xxinput[0],i1["Id"],tmp["AuId"],xxinput[1]])
except (KeyError):
pass
try:
for tmp in AuId1:
if tmp["AuId"]==xxinput[0]:
if tmp["AfId"] not in AfId1:
AfId1.append(tmp["AfId"])
break
except(KeyError):
pass
try:
CJId1= i1["J"]["JId"] # AA.AuId -> Id -> C.CId/J.JId -> Id
if CJId1 == CJId2:
result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]])
except (KeyError):
try:
CJId1= i1["C"]["CId"]
if CJId1 == CJId2:
result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]])
except:
CJId1=[]
def auau2for(id2):
Id2.append(id2["Id"])
try:
AuId2= id2["AA"]
for tmp in AuId2:
if tmp["AuId"]==xxinput[1]:
if tmp["AfId"] not in AfId2:
AfId2.append(tmp["AfId"])
break
except (KeyError):
pass
def auau3for(i1):
Id1.append(i1["Id"])
try:
AuId1= i1["AA"]
for tmp in AuId1:
if tmp["AuId"]==xxinput[0]:
if tmp["AfId"] not in AfId1:
AfId1.append(tmp["AfId"])
break
except (KeyError):
pass
class get_user:
def GET(self,user):
input_data = web.input()
#input = [2157025439,2139398774]
input=[int(input_data.id1),int(input_data.id2)]
encodeinput = json.dumps(input)
deinput = json.loads(encodeinput)
print deinput
threads=[]
threads.append(gevent.spawn(api1,deinput))
threads.append(gevent.spawn(api2,deinput))
threads.append(gevent.spawn(api3,deinput))
threads.append(gevent.spawn(api4,deinput))
threads.append(gevent.spawn(api5,deinput))
gevent.joinall(threads)
flag=0
try:
print data1["entities"][0]["Id"]
except(KeyError,IndexError):
flag=10;#id1 is Id
try:
print data2["entities"][0]["Id"]
except(KeyError,IndexError):
flag+=1;#id2 is Id
if flag == 0:
result=auau(deinput,data1,data2)
elif flag == 1:
result=auidid(deinput,data1,data3,data5)
elif flag == 10:
result=idauid(deinput,data4,data2)
elif flag == 11:
result=idid(deinput,data4,data5,data3)
else:
print "FLAG : %d " % flag
deresult = json.dumps(result)
print deresult
print "Total Size: %d " % len(result)
return deresult
def idid(deinput,data4,data5,data3):
global result,start_time,xxinput,FId1,AuId1,CJId1,Id3
start_time = time.clock()
xxinput=deinput
result=[]
Id3=[]
print "XXXXXXXXXXididXXXXXXXXXXX"
data1 = data4
data2 = data5
#''''''''''''''''''data 1'''''''''''''''''''''''''''
print "before data1"
try:
ReId1=data1["entities"][0]["RId"]
except (KeyError):
ReId1 = []
try:
FId1 = data1["entities"][0]["F"]
except (KeyError):
FId1 = []
try:
AuId1= data1["entities"][0]["AA"]
except (KeyError):
AuId1=[]
try:
CJId1= data1["entities"][0]["J"]["JId"]
except (KeyError):
try:
CJId1= data1["entities"][0]["C"]["CId"]
except:
CJId1=[]
print "before data2"
#''''''''''''''''''data 2'''''''''''''''''''''''''''
try:
FId2 = data2["entities"][0]["F"]
except (KeyError):
FId2 = []
try:
AuId2= data2["entities"][0]["AA"]
except (KeyError):
AuId2=[]
try:
CJId2= data2["entities"][0]["J"]["JId"]
except (KeyError):
try:
CJId2= data2["entities"][0]["C"]["CId"]
except:
CJId2=[]
print "threads"
#''''''''''''''''''data 3'''''''''id->XXXX->id->id''''''''''''''''''maybe big enough > 10w
#p = Pool()
#for i in data3["entities"]:
# p.map(ididfor,i)
#p.map(ididfor,data3["entities"])
#p.close()
#p.join()
threads = []
for i in data3["entities"]:
p=Process(target=ididfor,args=(lock,i))
p.daemon = True
threads.append(p)
for i in range(len(threads)):
threads[i].start()
for j in range(len(threads)):
threads[j].join()
'''for data in data3["entities"]:
Process(target=ididfor,args=(data,)).start()
print "threads done"'''
print Id3
'''for i in data3["entities"]:
p=Process(target=ididfor,args=(i,))
p.start()'''
#''''''''''''''''''operation''''''''''1 and 2 hop'''''''''''''''''
#id -> id
for Reid in ReId1:
if deinput[1] == Reid:
result.append([deinput[0],Reid])
break
#id -> F.FId -> id
for x in FId2:
#print 'x: %d ' % x['FId']
for y in FId1:
#print y['FId']
if y['FId']==x['FId']:
result.append([deinput[0],x['FId'],deinput[1]])
#id -> AA.AuId -> id
for x in AuId2:
for y in AuId1:
if y['AuId']==x['AuId']:
result.append([deinput[0],x['AuId'],deinput[1]])
#id -> C.CId/J.JId -> id
if CJId1==CJId2:
result.append([deinput[0],CJId1,deinput[1]])
print "before gevent"
#''''''''''''''''''''4th API'''''''''''''''''''''''''''''
#id -> id -> XXXXXX -> id ************************************************************need gevent
threads=[]
for Reid in ReId1:
threads.append(gevent.spawn(idid4api,Reid,deinput,FId2,AuId2,CJId2,Id3))
gevent.joinall(threads)
return result
def idauid(deinput,data4,data2):
global result,start_time,ReId2,xxinput,AuId2,FId2,CJId2,Id1,AfId1
Id1=[]
AfId1=[]
start_time = time.clock()
xxinput=deinput
result=[]
print "XXXXXXXXXXidauidXXXXXXXXXXX"
data1 = data2
data2 = data4
#'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
print ("Data 2")
print data2
try:
ReId2 = data2["entities"][0]["RId"]
except (KeyError):
ReId2 = []
try:
FId2 = data2["entities"][0]["F"]
except (KeyError):
FId2 = []
try:
AuId2 = data2["entities"][0]["AA"]
except (KeyError):
AuId2 = []
try:
CJId2 = data2["entities"][0]["J"]["JId"]
except (KeyError):
try:
CJId2 = data2["entities"][0]["C"]["CId"]
except:
CJId2 =[]
for tmp in AuId2:
if deinput[1] == tmp["AuId"]:
result.append([deinput[0],deinput[1]]) # Id -> AA.AuId (1-hop)
#'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
pool = ThreadPool(4)
pool.map(idauidfor,data1["entities"])
pool.close()
pool.join()
#'''''''''''''''''''''''''3rd API'''''''''''''''''''''''''''''''''
threads=[]
for author2 in AuId2:
threads.append(gevent.spawn(idauid3api,author2,deinput,AfId1))
gevent.joinall(threads)
#************************************************************need gevent
#''''''''''''''''''''4th API''''''''''''''''''''''''''''''''''''
threads=[]
for rid in ReId2:
threads.append(gevent.spawn(idauid4api,rid,deinput,Id1))
gevent.joinall(threads)
return result
def auidid(deinput,data1,data3,data5):
#'''''''''''''''''''''''''1st API'''''''''''''''''''''''''''''''''
global result,start_time,Id1,AfId1,xxinput,FId2,AuId2,CJId2
result=[]
Id1=[]
AfId1=[]
xxinput = deinput
start_time = time.clock()
print "XXXXXXXXXXauididXXXXXXXXXXX"
data2 = data5
#'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
try:
FId2 = data2["entities"][0]["F"]
except (KeyError):
FId2 = []
try:
AuId2= data2["entities"][0]["AA"]
except (KeyError):
AuId2=[]
try:
CJId2= data2["entities"][0]["J"]["JId"]
except (KeyError):
try:
CJId2= data2["entities"][0]["C"]["CId"]
except:
CJId2=[]
#'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
pool = ThreadPool(4)
pool.map(auididfor,data1["entities"])
pool.close()
pool.join()
# AA.AuId -> AA.AfId -> AA.AuId -> Id
for x in AuId2:
for y in AfId1:
try:
if y==x["AfId"]:
result.append([deinput[0],y,x["AuId"],deinput[1]])
except(KeyError):
continue
#'''''''''''''''''''''data 3''''''''''''''''''''''''''''''''
Id3 = []
for i3 in data3["entities"]:
Id3.append(i3["Id"])
#************************************************************need gevent
#''''''''''''''''''''4th API''''''''''''''''''''''''''''''''''''
threads=[]
for Aid in Id1:
threads.append(gevent.spawn(auidid4api,Aid,deinput,Id3))
gevent.joinall(threads)
return result
def auau(deinput,data1,data2):
global result,start_time,AfId2,xxinput,Id2,Id1,AfId1
result=[]
AfId2=[]
Id2=[]
Id1=[]
AfId1=[]
xxinput = deinput
start_time = time.clock()
print "XXXXXXXXXXauidauidXXXXXXXXXX"
#'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
pool = ThreadPool(4)
pool.map(auau2for,data2["entities"])
pool.close()
pool.join()
#'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
#print data1
pool = ThreadPool(4)
pool.map(auau3for,data1["entities"])
pool.close()
pool.join()
# AA.AuId -> AA.AfId -> AA.AuId
for x in AfId1:
for y in AfId2:
if x==y:
result.append([deinput[0],x,deinput[1]])
# AA.AuId -> Id -> AA.AuId
for x in Id1:
for y in Id2:
if x==y:
result.append([deinput[0],x,deinput[1]])
#'''''''''''''''''''''''''3rd API'''''''''''''''''''''''''''''''''
threads=[]
for aaid1 in Id1:
threads.append(gevent.spawn(auau3api,aaid1,deinput,Id2))
gevent.joinall(threads)
return result
if __name__ == '__main__':
app.run()