academic_R/test1 - 副本 (3).py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#import gevent.monkey
#gevent.monkey.patch_socket()
from gevent import monkey, sleep
monkey.patch_all()
import gevent
import web
import httplib, urllib, base64
import json
import time
from multiprocessing.dummy import Pool as ThreadPool



urls=(
    '/bof333?(.+)','get_user'
)
app = web.application(urls,globals())

result=[]

def api1(deinput):
    global data1
    exp1 = "Composite(AA.AuId = %d )" % deinput[0]
    params1 = urllib.urlencode({
        'expr':exp1,
        'offset': '0',
        'model': 'latest',
        'count': '1000000',
        'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    conn.close()
    data1=json.loads(data)

def api2(deinput):
    global data2
    exp2 = "Composite(AA.AuId = %d )" % deinput[1]
    params1 = urllib.urlencode({
        'expr':exp2,
        'offset': '0',
        'model': 'latest',
        'count': '1000000',
        'attributes': 'Id,RId,F.FId,AA.AuId,AA.AfId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    conn.close()
    data2=json.loads(data)

def api4(deinput):
    global data4
    exp4 = 'Id = %d' % deinput[0]
    params1 = urllib.urlencode({
        'expr':exp4,
        'model': 'latest',
        'offset': '0',
        'count': '1000000',
        'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data4 = json.loads(data)
    conn.close()

def api5(deinput):
    global data5
    exp5 = 'Id = %d' % deinput[1]
    params1 = urllib.urlencode({
        'expr':exp5,
        'count': '1000000',
        'model': 'latest',
        'attributes': 'F.FId,AA.AuId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data5 = json.loads(data)
    conn.close()

def api3(deinput):
    global data3
    exp3 = 'RId = %d' % deinput[1]
    params1 = urllib.urlencode({
        'expr':exp3,
        'count': '1000000',
        'model': 'latest',
        'attributes': 'Id,F.FId,AA.AuId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data3 = json.loads(data)
    conn.close()


def idid4api(Reid,deinput,FId2,AuId2,CJId2,Id3):
    now_time = time.clock()
    exe_time = (float)(now_time - start_time)
    if exe_time > 200:
        return
    exp4 = 'Id = %d' % Reid
    params1 = urllib.urlencode({
        'expr':exp4,
        'count': '1000000',
        'model': 'latest',
        'attributes': 'RId,F.FId,AA.AuId,C.CId,J.JId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data4 = json.loads(data)
    conn.close()
    #print data4
    try:
        ReId4= data4["entities"][0]["RId"]
    except (KeyError):
        ReId4 = []
    try:
        FId4 = data4["entities"][0]["F"]
    except (KeyError):
        FId4 = []
    try:
        AuId4= data4["entities"][0]["AA"]
    except (KeyError):
        AuId4=[]
    try:
        CJId4= data4["entities"][0]["J"]["JId"]
    except (KeyError):
        try:
            CJId4= data4["entities"][0]["C"]["CId"]
        except:
            CJId4=[]

    #id -> id -> id     #id -> id -> id -> id
    for rid in ReId4:
        if deinput[1] == rid:
            result.append([deinput[0],Reid,deinput[1]])
        for Id in Id3:
            if rid == Id:
            	result.append([deinput[0],Reid,rid,deinput[1]])
    #id -> id -> F.FId -> id
    for x in FId2:
        for y in FId4:
            if y['FId']==x['FId']:
                result.append([deinput[0],Reid,x['FId'],deinput[1]])
    #id -> id -> AA.AuId -> id
    for x in AuId2:
        for y in AuId4:
            if y['AuId']==x['AuId']:
                result.append([deinput[0],Reid,x['AuId'],deinput[1]])
    #id -> id -> C.CId/J.JId -> id
    if CJId4==CJId2:
        result.append([deinput[0],Reid,CJId4,deinput[1]])

def idauid3api(author2,deinput,AfId1):
    now_time = time.clock()
    exe_time = (float)(now_time - start_time)
    if exe_time > 200:
        return
    exp3 = "Composite(AA.AuId = %d )" % author2["AuId"]
    params1 = urllib.urlencode({
        'expr':exp3,
        'model': 'latest',
        'count': '1000000',
        'attributes': 'AA.AuId,AA.AfId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data3 = json.loads(data)
    conn.close()
    AfId3=[]
    for i3 in data3["entities"]:
        try:
            for tmp in i3["AA"]:
                if tmp["AuId"] == author2["AuId"]:
                    if tmp["AfId"] not in AfId3:
                        AfId3.append(tmp["AfId"])
                        break
        except (KeyError):
            pass
    for x in AfId3:
        for y in AfId1:
            if x == y :
                result.append([deinput[0],author2["AuId"],x,deinput[1]])

def idauid4api(rid,deinput,Id1):
    now_time = time.clock()
    exe_time = (float)(now_time - start_time)
    if exe_time > 200:
        return
    exp4 = 'Id = %d' % rid
    params1 = urllib.urlencode({
        'expr':exp4,
        'count': '1000000',
        'model': 'latest',
        'attributes': 'RId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data4 = json.loads(data)
    conn.close()
    #print data4
    try:
        ReId4= data4["entities"][0]["RId"]
    except (KeyError):
        ReId4 = []
    # AA.AuId -> id -> id -> id
    for rid4 in ReId4:
        for tmp in Id1:
            if rid4 == tmp:
            	result.append([deinput[0],rid,rid4,deinput[1]])

def auidid4api(Aid,deinput,Id3):
    now_time = time.clock()
    exe_time = (float)(now_time - start_time)
    if exe_time > 200:
        return
    exp4 = 'Id = %d' % Aid
    params1 = urllib.urlencode({
        'expr':exp4,
        'count': '1000000',
        'attributes': 'RId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data4 = json.loads(data)
    conn.close()
    #print data4
    try:
        ReId4= data4["entities"][0]["RId"]
    except (KeyError):
        ReId4 = []
    # AA.AuId -> id -> id     # AA.AuId -> id -> id -> id
    for rid in ReId4:
        if deinput[1] == rid:
            result.append([deinput[0],Aid,deinput[1]])
        for tmp in Id3:
            if rid == tmp:
                result.append([deinput[0],Aid,rid,deinput[1]])

def auau3api(aaid1,deinput,Id2):
    now_time = time.clock()
    exe_time = (float)(now_time - start_time)
    if exe_time > 200:
        return
    exp3 = 'Id = %d' % aaid1
    params1 = urllib.urlencode({
        'expr':exp3,
        'count': '1000000',
        'attributes': 'RId',
        'subscription-key': 'f7cc29509a8443c5b3a5e56b0e38b5a6',
    })
    conn = httplib.HTTPSConnection('oxfordhk.azure-api.net')
    conn.request("GET", "/academic/v1.0/evaluate?%s" % params1)
    response = conn.getresponse()
    data = response.read()
    data3 = json.loads(data)
    conn.close()
    try:
        ReId3=data3["entities"][0]["RId"]
    except (KeyError):
        ReId3 = []
    # AA.AuId -> Id -> Id -> AA.AuId
    for x in ReId3:
        for y in Id2:
            if x==y:
                result.append([deinput[0],aaid1,x,deinput[1]])
  
def ididfor(i):
    try:
        if i["Id"] not in Id3:
            Id3.append(i["Id"])
    except(KeyError):
        pass
# id -> F.FId -> id -> id
    try:
        FId_data3 = i["F"]
        for tmp in FId_data3:
            for x in FId1:
                if x["FId"]==tmp["FId"]:
                    result.append([xxinput[0],tmp["FId"],i["Id"],xxinput[1]])
            #FId3.append(tmp["FId"])#
    except(KeyError):
        pass
# id -> AA.AuId -> id -> id
    try:
        AuId_data3=i["AA"]
        for tmp in AuId_data3:
            for x in AuId1:
                if x["AuId"]==tmp["AuId"]:
                    result.append([xxinput[0],tmp["AuId"],i["Id"],xxinput[1]])
            #AuId3.append(tmp["AuId"])#'''''''''''''''''''''need optimization
    except(KeyError):
        pass
# id -> C.CId/J.JId -> id -> id
    try:
        CJId3=i["J"]["JId"]#'''''''''''''''''''''need optimization
        if CJId3==CJId1:
            result.append([xxinput[0],CJId3,i["Id"],xxinput[1]])
    except(KeyError):
        try:
            CJId3=i["C"]["CId"]
            if CJId3==CJId1:
                result.append([xxinput[0],CJId3,i["Id"],xxinput[1]])
        except(KeyError):
            pass 

def idauidfor(i1):
    Id1.append(i1["Id"])
    for tmp in ReId2:
        if i1["Id"] == tmp:# Id -> Id -> AA.AuId (2-hop)
            result.append([xxinput[0],i1["Id"],xxinput[1]])
    for tmp in i1["AA"]:
        for x in AuId2:
            if x["AuId"] == tmp["AuId"]:
                # Id -> AA.AuId -> Id -> AA.AuId (3-hop) (new add~~)
                result.append([xxinput[0],x["AuId"],i1["Id"],xxinput[1]])
    try:
        FId1 = i1["F"]
        for tmp in FId1:
            for x in FId2:
                if x["FId"] == tmp["FId"]:
                    # Id -> F.FId -> Id -> AA.AuId (3-hop)
                    result.append([xxinput[0],x["FId"],i1["Id"],xxinput[1]])
    except (KeyError):
		pass
    try:
        for tmp in i1["AA"]:
            if tmp["AuId"] == xxinput[1]:
                if tmp["AfId"] not in AfId1:
                    AfId1.append(tmp["AfId"])
                    break
    except (KeyError):
        pass
    try:
        CJId1 = i1["J"]["JId"] # Id -> C.CId/J.JId -> Id -> AA.AuId (3-hop)
        if CJId1 == CJId2:
            result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]])
    except (KeyError):
        try:
            CJId1= i1["C"]["CId"]
            if CJId1 == CJId2:
                result.append([xxinput[0],CJId2,i1["Id"],xxinput[1]])
        except:
            CJId1=[]

def auididfor(i1):
    Id1.append(i1["Id"])
    if i1["Id"]==xxinput[1]:
        result.append([xxinput[0],xxinput[1]])  # AA.AuId -> Id
    try:
        FId1 = i1["F"]
        for tmp in FId1:
            for x in FId2:
                if x["FId"]==tmp["FId"]:
                    # AA.AuId -> Id -> F.FId -> Id
                    result.append([xxinput[0],i1["Id"],tmp["FId"],xxinput[1]])
    except (KeyError):
        pass
    try:
        AuId1= i1["AA"]
        for tmp in AuId1:
            for x in AuId2:
                if tmp["AuId"]==x["AuId"]:
                    # AA.AuId -> Id -> AA.AuId -> Id
                    result.append([xxinput[0],i1["Id"],tmp["AuId"],xxinput[1]])
    except (KeyError):
        pass
    try:
        for tmp in AuId1:
            if tmp["AuId"]==xxinput[0]:
                if tmp["AfId"] not in AfId1:
                    AfId1.append(tmp["AfId"])
                    break
    except(KeyError):
        pass
    try:
        CJId1= i1["J"]["JId"] # AA.AuId -> Id -> C.CId/J.JId -> Id
        if CJId1 == CJId2:
            result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]])
    except (KeyError):
        try:
            CJId1= i1["C"]["CId"]
            if CJId1 == CJId2:
                result.append([xxinput[0],i1["Id"],CJId1,xxinput[1]])
        except:
            CJId1=[]

class get_user:
    def GET(self,user):
        input_data = web.input()
        #input = [2157025439,2139398774]
        input=[int(input_data.id1),int(input_data.id2)]
        encodeinput = json.dumps(input)
        deinput = json.loads(encodeinput)
        print deinput
        threads=[]
        threads.append(gevent.spawn(api1,deinput))
        threads.append(gevent.spawn(api2,deinput))
        threads.append(gevent.spawn(api3,deinput))
        threads.append(gevent.spawn(api4,deinput))
        threads.append(gevent.spawn(api5,deinput))
        gevent.joinall(threads)
        flag=0
        try:
            print data1["entities"][0]["Id"]
        except(KeyError,IndexError):
            flag=10;#id1 is Id
        try:
            print data2["entities"][0]["Id"]
        except(KeyError,IndexError):
            flag+=1;#id2 is Id
        if flag == 0:
            result=auau(deinput,data1,data2)
        elif flag == 1:
            result=auidid(deinput,data1,data3,data5)
        elif flag == 10:
            result=idauid(deinput,data4,data2)
        elif flag == 11:
            result=idid(deinput,data4,data5,data3)
        else:
            print "FLAG : %d " % flag
        deresult = json.dumps(result)
        print deresult
        print "Total Size: %d " % len(result)
        return deresult
        

def idid(deinput,data4,data5,data3):
    global result,start_time,xxinput,FId1,AuId1,CJId1,Id3
    start_time = time.clock()
    xxinput=deinput
    result=[]
    Id3=[]
    print "XXXXXXXXXXididXXXXXXXXXXX"
    data1 = data4
    data2 = data5
    #''''''''''''''''''data 1'''''''''''''''''''''''''''
    try:
        ReId1=data1["entities"][0]["RId"]
    except (KeyError):
        ReId1 = []
    try:
        FId1 = data1["entities"][0]["F"]
    except (KeyError):
        FId1 = []
    try:
        AuId1= data1["entities"][0]["AA"]
    except (KeyError):
        AuId1=[]
    try:
        CJId1= data1["entities"][0]["J"]["JId"]
    except (KeyError):
        try:
            CJId1= data1["entities"][0]["C"]["CId"]
        except:
            CJId1=[]

    #''''''''''''''''''data 2'''''''''''''''''''''''''''
    try:
        FId2 = data2["entities"][0]["F"]
    except (KeyError):
        FId2 = []
    try:
        AuId2= data2["entities"][0]["AA"]
    except (KeyError):
        AuId2=[]
    try:
        CJId2= data2["entities"][0]["J"]["JId"]
    except (KeyError):
        try:
            CJId2= data2["entities"][0]["C"]["CId"]
        except:
            CJId2=[]

    #''''''''''''''''''data 3'''''''''id->XXXX->id->id''''''''''''''''''maybe big enough > 10w
    pool = ThreadPool(4)
    pool.map(ididfor,data3["entities"])
    pool.close()
    pool.join()

    #''''''''''''''''''operation''''''''''1 and 2 hop'''''''''''''''''
    #id -> id
    for Reid in ReId1:
        if deinput[1] == Reid:
            result.append([deinput[0],Reid])
            break
    #id -> F.FId -> id
    for x in FId2:
        #print 'x: %d ' % x['FId']
        for y in FId1:
            #print y['FId']
            if y['FId']==x['FId']:
                result.append([deinput[0],x['FId'],deinput[1]])
    #id -> AA.AuId -> id
    for x in AuId2:
        for y in AuId1:
            if y['AuId']==x['AuId']:
                result.append([deinput[0],x['AuId'],deinput[1]])
    #id -> C.CId/J.JId -> id
    if CJId1==CJId2:
        result.append([deinput[0],CJId1,deinput[1]])

    #''''''''''''''''''''4th API'''''''''''''''''''''''''''''
    #id -> id -> XXXXXX -> id ************************************************************need gevent
    threads=[]
    for Reid in ReId1:
        threads.append(gevent.spawn(idid4api,Reid,deinput,FId2,AuId2,CJId2,Id3))
    gevent.joinall(threads)
    return result

def idauid(deinput,data4,data2):
    global result,start_time,ReId2,xxinput,AuId2,FId2,CJId2,Id1,AfId1
    Id1=[]
    AfId1=[]
    start_time = time.clock()
    xxinput=deinput
    result=[]
    print "XXXXXXXXXXidauidXXXXXXXXXXX"

    data1 = data2
    data2 = data4

    #'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
    print ("Data 2")
    print data2

    try:
        ReId2 = data2["entities"][0]["RId"]
    except (KeyError):
        ReId2 = []
    try:
        FId2 = data2["entities"][0]["F"]
    except (KeyError):
        FId2 = []
    try:
        AuId2 = data2["entities"][0]["AA"]
    except (KeyError):
        AuId2 = []
    try:
        CJId2 = data2["entities"][0]["J"]["JId"]
    except (KeyError):
        try:
            CJId2 = data2["entities"][0]["C"]["CId"]
        except:
            CJId2 =[]

    for tmp in AuId2:
        if deinput[1] == tmp["AuId"]:
            result.append([deinput[0],deinput[1]])  # Id -> AA.AuId (1-hop)


    #'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
    pool = ThreadPool(4)
    pool.map(idauidfor,data1["entities"])
    pool.close()
    pool.join()

    #'''''''''''''''''''''''''3rd API'''''''''''''''''''''''''''''''''
    threads=[]
    for author2 in AuId2:
        threads.append(gevent.spawn(idauid3api,author2,deinput,AfId1))
    gevent.joinall(threads)

    #************************************************************need gevent
    #''''''''''''''''''''4th API''''''''''''''''''''''''''''''''''''
    threads=[]
    for rid in ReId2:
        threads.append(gevent.spawn(idauid4api,rid,deinput,Id1))
    gevent.joinall(threads)
    return result

def auidid(deinput,data1,data3,data5):
    #'''''''''''''''''''''''''1st API'''''''''''''''''''''''''''''''''
    global result,start_time,Id1,AfId1,xxinput,FId2,AuId2,CJId2
    result=[]
    Id1=[]
    AfId1=[]
    xxinput = deinput
    start_time = time.clock()
    print "XXXXXXXXXXauididXXXXXXXXXXX"
    data2 = data5
    #'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
    try:
        FId2 = data2["entities"][0]["F"]
    except (KeyError):
        FId2 = []
    try:
        AuId2= data2["entities"][0]["AA"]
    except (KeyError):
        AuId2=[]
    try:
        CJId2= data2["entities"][0]["J"]["JId"]
    except (KeyError):
        try:
            CJId2= data2["entities"][0]["C"]["CId"]
        except:
            CJId2=[]

    #'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
    pool = ThreadPool(4)
    pool.map(auididfor,data1["entities"])
    pool.close()
    pool.join()

    # AA.AuId -> AA.AfId -> AA.AuId -> Id
    for x in AuId2:
        for y in AfId1:
            try:
                if y==x["AfId"]:
                    result.append([deinput[0],y,x["AuId"],deinput[1]])
            except(KeyError):
                continue

    #'''''''''''''''''''''data 3''''''''''''''''''''''''''''''''
    Id3 = []
    for i3 in data3["entities"]:
        Id3.append(i3["Id"])

    #************************************************************need gevent
    #''''''''''''''''''''4th API''''''''''''''''''''''''''''''''''''
    threads=[]
    for Aid in Id1:
        threads.append(gevent.spawn(auidid4api,Aid,deinput,Id3))
    gevent.joinall(threads)
    return result

def auau(deinput,data1,data2):
    global result,start_time
    result=[]
    start_time = time.clock()
    print "XXXXXXXXXXauidauidXXXXXXXXXX"
    #'''''''''''''''''''''data 2''''''''''''''''''''''''''''''''
    AfId2=[]
    for id2 in data2["entities"]:
        try:
            AuId2= id2["AA"]
            for tmp in AuId2:
                if tmp["AuId"]==deinput[1]:
                    if tmp["AfId"] not in AfId2:
                        AfId2.append(tmp["AfId"])
                        break
        except (KeyError):
            continue
    Id2=[]
    for id2 in data2["entities"]:
        Id2.append(id2["Id"])
    #'''''''''''''''''''''data 1''''''''''''''''''''''''''''''''
    #print data1
    Id1=[]
    AfId1=[]
    for i1 in data1["entities"]:
        Id1.append(i1["Id"])
    print Id1
    for i1 in data1["entities"]:
        try:
            AuId1= i1["AA"]
            for tmp in AuId1:
                if tmp["AuId"]==deinput[0]:
                    if tmp["AfId"] not in AfId1:
                        AfId1.append(tmp["AfId"])
                        break
        except (KeyError):
            continue

    # AA.AuId -> AA.AfId -> AA.AuId
    for x in AfId1:
        for y in AfId2:
            if x==y:
                result.append([deinput[0],x,deinput[1]])

    # AA.AuId -> Id -> AA.AuId
    for x in Id1:
        for y in Id2:
            if x==y:
                result.append([deinput[0],x,deinput[1]])

    #'''''''''''''''''''''''''3rd API'''''''''''''''''''''''''''''''''
    threads=[]
    for aaid1 in Id1:
        threads.append(gevent.spawn(auau3api,aaid1,deinput,Id2))
    gevent.joinall(threads)
    return result

if __name__ == '__main__':
    app.run()