[sqlite] index on proc_uid

Summary: There is still a bug in using marshalled values as keys (exposed up the diff stack) where structurally equal procnames get serialised to distinct keys.  This diff addresses that bug by using the existing functionality for string unique IDs.

Reviewed By: ezgicicek

Differential Revision: D23133761

fbshipit-source-id: 3cbafb51b
master
Nikos Gorogiannis 4 years ago committed by Facebook GitHub Bot
parent 343d6b1bea
commit e9a6195b52

@ -34,16 +34,17 @@ let should_try_to_update =
{|
SELECT attr_kind
FROM procedures
WHERE proc_name = :pname
WHERE proc_uid = :uid
AND attr_kind > :akind
|}
in
fun pname_blob akind ->
fun proc_uid attr_kind ->
ResultsDatabase.with_registered_statement find_more_defined_statement ~f:(fun db find_stmt ->
Sqlite3.bind find_stmt 1 (* :pname *) pname_blob
|> SqliteUtils.check_result_code db ~log:"replace bind pname" ;
Sqlite3.bind find_stmt 2 (* :akind *) (Sqlite3.Data.INT (int64_of_attributes_kind akind))
|> SqliteUtils.check_result_code db ~log:"replace bind attribute kind" ;
Sqlite3.bind find_stmt 1 (* uid *) (Sqlite3.Data.TEXT proc_uid)
|> SqliteUtils.check_result_code db ~log:"should update bind pname_uid" ;
Sqlite3.bind find_stmt 2
(* :akind *) (Sqlite3.Data.INT (int64_of_attributes_kind attr_kind))
|> SqliteUtils.check_result_code db ~log:"should update bind attribute kind" ;
SqliteUtils.result_single_column_option ~finalize:false ~log:"Attributes.replace" db
find_stmt
|> (* there is no entry with a strictly larger "definedness" for that proc name *)
@ -52,35 +53,36 @@ let should_try_to_update =
let find =
let select_statement =
ResultsDatabase.register_statement "SELECT proc_attributes FROM procedures WHERE proc_name = :k"
ResultsDatabase.register_statement "SELECT proc_attributes FROM procedures WHERE proc_uid = :k"
in
fun pname_blob ->
fun proc_uid ->
ResultsDatabase.with_registered_statement select_statement ~f:(fun db select_stmt ->
Sqlite3.bind select_stmt 1 pname_blob
Sqlite3.bind select_stmt 1 (Sqlite3.Data.TEXT proc_uid)
|> SqliteUtils.check_result_code db ~log:"find bind proc name" ;
SqliteUtils.result_single_column_option ~finalize:false ~log:"Attributes.find" db
select_stmt
|> Option.map ~f:ProcAttributes.SQLite.deserialize )
let load pname = find (Procname.SQLite.serialize pname)
let load pname = find (Procname.to_unique_id pname)
let store ~proc_desc (attr : ProcAttributes.t) =
let pname = attr.proc_name in
let akind = proc_kind_of_attr attr in
let key = Procname.SQLite.serialize attr.proc_name in
if should_try_to_update key akind then
let source_file_blob = SourceFile.SQLite.serialize attr.loc.Location.file in
let attributes_blob = ProcAttributes.SQLite.serialize attr in
let pname_str = Procname.to_string attr.proc_name in
let akind_int64 = int64_of_attributes_kind akind in
let proc_desc_blob = Procdesc.SQLite.serialize proc_desc in
let callees_blob =
let proc_uid = Procname.to_unique_id pname in
if should_try_to_update proc_uid akind then
let proc_name = Procname.SQLite.serialize pname in
let proc_name_hum = Procname.to_string pname in
let attr_kind = int64_of_attributes_kind akind in
let cfg = Procdesc.SQLite.serialize proc_desc in
let proc_attributes = ProcAttributes.SQLite.serialize attr in
let source_file = SourceFile.SQLite.serialize attr.loc.Location.file in
let callees =
Option.map proc_desc ~f:Procdesc.get_static_callees
|> Option.value ~default:[] |> Procname.SQLiteList.serialize
in
DBWriter.replace_attributes ~pname_str ~pname:key ~akind:akind_int64
~source_file:source_file_blob ~attributes:attributes_blob ~proc_desc:proc_desc_blob
~callees:callees_blob
DBWriter.replace_attributes ~proc_uid ~proc_name ~proc_name_hum ~attr_kind ~source_file
~proc_attributes ~cfg ~callees
let find_file_capturing_procedure pname =

@ -837,11 +837,11 @@ end)
let load =
let load_statement =
ResultsDatabase.register_statement "SELECT cfg FROM procedures WHERE proc_name = :k"
ResultsDatabase.register_statement "SELECT cfg FROM procedures WHERE proc_uid = :k"
in
fun pname ->
ResultsDatabase.with_registered_statement load_statement ~f:(fun db stmt ->
Procname.SQLite.serialize pname |> Sqlite3.bind stmt 1
|> SqliteUtils.check_result_code db ~log:"load bind proc name" ;
Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT (Procname.to_unique_id pname))
|> SqliteUtils.check_result_code db ~log:"load bind proc_uid" ;
SqliteUtils.result_single_column_option ~finalize:false ~log:"Procdesc.load" db stmt
|> Option.bind ~f:SQLite.deserialize )

@ -221,13 +221,13 @@ module OnDisk = struct
let spec_of_procname =
let load_statement =
ResultsDatabase.register_statement
"SELECT analysis_summary, report_summary FROM specs WHERE proc_name = :k"
"SELECT analysis_summary, report_summary FROM specs WHERE proc_uid = :k"
in
fun proc_name ->
ResultsDatabase.with_registered_statement load_statement ~f:(fun db load_stmt ->
Sqlite3.bind load_stmt 1 (Procname.SQLite.serialize proc_name)
|> SqliteUtils.check_result_code db ~log:"load proc specs bind proc_name" ;
SqliteUtils.result_option ~finalize:false db ~log:"load proc specs run" load_stmt
ResultsDatabase.with_registered_statement load_statement ~f:(fun db stmt ->
Sqlite3.bind stmt 1 (Sqlite3.Data.TEXT (Procname.to_unique_id proc_name))
|> SqliteUtils.check_result_code db ~log:"load proc specs bind proc_uid" ;
SqliteUtils.result_option ~finalize:false db ~log:"load proc specs run" stmt
~read_row:(fun stmt ->
let analysis_summary = Sqlite3.column stmt 0 |> AnalysisSummary.SQLite.deserialize in
let report_summary = Sqlite3.column stmt 1 |> ReportSummary.SQLite.deserialize in
@ -279,7 +279,7 @@ module OnDisk = struct
else
let analysis_summary = AnalysisSummary.of_full_summary summary in
let report_summary = ReportSummary.of_full_summary summary in
DBWriter.store_spec
DBWriter.store_spec ~proc_uid:(Procname.to_unique_id proc_name)
~proc_name:(Procname.SQLite.serialize proc_name)
~analysis_summary:(AnalysisSummary.SQLite.serialize analysis_summary)
~report_summary:(ReportSummary.SQLite.serialize report_summary)
@ -317,7 +317,7 @@ module OnDisk = struct
let filename = specs_filename_of_procname pname |> DB.filename_to_string in
(* Unix_error is raised if the file isn't present so do nothing in this case *)
try Unix.unlink filename with Unix.Unix_error _ -> ()
else DBWriter.delete_spec ~proc_name:(Procname.SQLite.serialize pname)
else DBWriter.delete_spec ~proc_uid:(Procname.to_unique_id pname)
let iter_filtered_specs ~filter ~f =
@ -325,7 +325,7 @@ module OnDisk = struct
let dummy_source_file = SourceFile.invalid __FILE__ in
(* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *)
Sqlite3.prepare db
"SELECT proc_name, analysis_summary, report_summary FROM specs ORDER BY proc_name ASC"
"SELECT proc_name, analysis_summary, report_summary FROM specs ORDER BY proc_uid ASC"
|> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs")
~f:(fun stmt ->
let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in
@ -340,7 +340,7 @@ module OnDisk = struct
let db = ResultsDatabase.get_database () in
let dummy_source_file = SourceFile.invalid __FILE__ in
(* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *)
Sqlite3.prepare db "SELECT proc_name, report_summary FROM specs ORDER BY proc_name ASC"
Sqlite3.prepare db "SELECT proc_name, report_summary FROM specs ORDER BY proc_uid ASC"
|> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs")
~f:(fun stmt ->
let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in

@ -31,36 +31,38 @@ module Implementation = struct
ResultsDatabase.register_statement
{|
INSERT OR REPLACE INTO procedures
SELECT :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees
SELECT :uid, :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees
FROM (
SELECT NULL
FROM (
SELECT COALESCE(MAX(attr_kind),-1) AS attr_kind,
COALESCE(MAX(source_file),"") AS source_file
FROM procedures
WHERE proc_name = :pname )
WHERE proc_uid = :uid )
WHERE attr_kind < :akind
OR (attr_kind = :akind AND source_file <= :sfile) )
|}
in
fun ~pname_str ~pname ~akind ~source_file ~attributes ~proc_desc ~callees ->
fun ~proc_uid ~proc_name ~proc_name_hum ~attr_kind ~source_file ~proc_attributes ~cfg ~callees ->
ResultsDatabase.with_registered_statement attribute_replace_statement
~f:(fun db replace_stmt ->
Sqlite3.bind replace_stmt 1 (* :pname *) pname
|> SqliteUtils.check_result_code db ~log:"replace bind pname" ;
Sqlite3.bind replace_stmt 2 (* :proc_name_hum *) (Sqlite3.Data.TEXT pname_str)
Sqlite3.bind replace_stmt 1 (* :proc_uid *) (Sqlite3.Data.TEXT proc_uid)
|> SqliteUtils.check_result_code db ~log:"replace bind proc_uid" ;
Sqlite3.bind replace_stmt 2 (* :pname *) proc_name
|> SqliteUtils.check_result_code db ~log:"replace bind proc_name" ;
Sqlite3.bind replace_stmt 3 (* :proc_name_hum *) (Sqlite3.Data.TEXT proc_name_hum)
|> SqliteUtils.check_result_code db ~log:"replace bind proc_name_hum" ;
Sqlite3.bind replace_stmt 3 (* :akind *) (Sqlite3.Data.INT akind)
|> SqliteUtils.check_result_code db ~log:"replace bind attribute kind" ;
Sqlite3.bind replace_stmt 4 (* :sfile *) source_file
|> SqliteUtils.check_result_code db ~log:"replace bind source file" ;
Sqlite3.bind replace_stmt 5 (* :pattr *) attributes
|> SqliteUtils.check_result_code db ~log:"replace bind proc attributes" ;
Sqlite3.bind replace_stmt 6 (* :cfg *) proc_desc
Sqlite3.bind replace_stmt 4 (* :akind *) (Sqlite3.Data.INT attr_kind)
|> SqliteUtils.check_result_code db ~log:"replace bind attr_kind" ;
Sqlite3.bind replace_stmt 5 (* :sfile *) source_file
|> SqliteUtils.check_result_code db ~log:"replace bind source source_file" ;
Sqlite3.bind replace_stmt 6 (* :pattr *) proc_attributes
|> SqliteUtils.check_result_code db ~log:"replace bind proc proc_attributes" ;
Sqlite3.bind replace_stmt 7 (* :cfg *) cfg
|> SqliteUtils.check_result_code db ~log:"replace bind cfg" ;
Sqlite3.bind replace_stmt 7 (* :callees *) callees
Sqlite3.bind replace_stmt 8 (* :callees *) callees
|> SqliteUtils.check_result_code db ~log:"replace bind callees" ;
SqliteUtils.result_unit db ~finalize:false ~log:"Attributes.replace" replace_stmt )
SqliteUtils.result_unit db ~finalize:false ~log:"replace_attributes" replace_stmt )
let add_source_file =
@ -108,6 +110,7 @@ module Implementation = struct
{|
INSERT OR REPLACE INTO memdb.procedures
SELECT
sub.proc_uid,
sub.proc_name,
sub.proc_name_hum,
sub.attr_kind,
@ -118,7 +121,7 @@ module Implementation = struct
FROM (
attached.procedures AS sub
LEFT OUTER JOIN memdb.procedures AS main
ON sub.proc_name = main.proc_name )
ON sub.proc_uid = main.proc_uid )
WHERE
main.attr_kind IS NULL
OR main.attr_kind < sub.attr_kind
@ -180,27 +183,32 @@ module Implementation = struct
let store_spec =
let store_statement =
ResultsDatabase.register_statement
"INSERT OR REPLACE INTO specs VALUES (:proc_name, :analysis_summary, :report_summary)"
{|
INSERT OR REPLACE INTO specs
VALUES (:proc_uid, :proc_name, :analysis_summary, :report_summary)
|}
in
fun ~proc_name ~analysis_summary ~report_summary ->
fun ~proc_uid ~proc_name ~analysis_summary ~report_summary ->
ResultsDatabase.with_registered_statement store_statement ~f:(fun db store_stmt ->
Sqlite3.bind store_stmt 1 proc_name
Sqlite3.bind store_stmt 1 (Sqlite3.Data.TEXT proc_uid)
|> SqliteUtils.check_result_code db ~log:"store spec bind proc_uid" ;
Sqlite3.bind store_stmt 2 proc_name
|> SqliteUtils.check_result_code db ~log:"store spec bind proc_name" ;
Sqlite3.bind store_stmt 2 analysis_summary
Sqlite3.bind store_stmt 3 analysis_summary
|> SqliteUtils.check_result_code db ~log:"store spec bind analysis_summary" ;
Sqlite3.bind store_stmt 3 report_summary
Sqlite3.bind store_stmt 4 report_summary
|> SqliteUtils.check_result_code db ~log:"store spec bind report_summary" ;
SqliteUtils.result_unit ~finalize:false ~log:"store spec" db store_stmt )
let delete_spec =
let delete_statement =
ResultsDatabase.register_statement "DELETE FROM specs WHERE proc_name = :k"
ResultsDatabase.register_statement "DELETE FROM specs WHERE proc_uid = :k"
in
fun ~proc_name ->
fun ~proc_uid ->
ResultsDatabase.with_registered_statement delete_statement ~f:(fun db delete_stmt ->
Sqlite3.bind delete_stmt 1 proc_name
|> SqliteUtils.check_result_code db ~log:"delete spec bind proc_name" ;
Sqlite3.bind delete_stmt 1 (Sqlite3.Data.TEXT proc_uid)
|> SqliteUtils.check_result_code db ~log:"delete spec bind proc_uid" ;
SqliteUtils.result_unit ~finalize:false ~log:"store spec" db delete_stmt )
@ -217,19 +225,23 @@ module Command = struct
; integer_type_widths: Sqlite3.Data.t
; proc_names: Sqlite3.Data.t }
| DeleteAllSpecs
| DeleteSpec of {proc_name: Sqlite3.Data.t}
| DeleteSpec of {proc_uid: string}
| Handshake
| MarkAllSourceFilesStale
| Merge of {infer_deps_file: string}
| StoreSpec of
{proc_name: Sqlite3.Data.t; analysis_summary: Sqlite3.Data.t; report_summary: Sqlite3.Data.t}
{ proc_uid: string
; proc_name: Sqlite3.Data.t
; analysis_summary: Sqlite3.Data.t
; report_summary: Sqlite3.Data.t }
| ReplaceAttributes of
{ pname_str: string
; pname: Sqlite3.Data.t
; akind: int64
{ proc_uid: string
; proc_name: Sqlite3.Data.t
; proc_name_hum: string
; attr_kind: int64
; source_file: Sqlite3.Data.t
; attributes: Sqlite3.Data.t
; proc_desc: Sqlite3.Data.t
; proc_attributes: Sqlite3.Data.t
; cfg: Sqlite3.Data.t
; callees: Sqlite3.Data.t }
| ResetCaptureTables
| Terminate
@ -267,19 +279,21 @@ module Command = struct
Implementation.add_source_file ~source_file ~tenv ~integer_type_widths ~proc_names
| DeleteAllSpecs ->
Implementation.delete_all_specs ()
| DeleteSpec {proc_name} ->
Implementation.delete_spec ~proc_name
| DeleteSpec {proc_uid} ->
Implementation.delete_spec ~proc_uid
| Handshake ->
()
| MarkAllSourceFilesStale ->
Implementation.mark_all_source_files_stale ()
| Merge {infer_deps_file} ->
Implementation.merge infer_deps_file
| StoreSpec {proc_name; analysis_summary; report_summary} ->
Implementation.store_spec ~proc_name ~analysis_summary ~report_summary
| ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees} ->
Implementation.replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes
~proc_desc ~callees
| StoreSpec {proc_uid; proc_name; analysis_summary; report_summary} ->
Implementation.store_spec ~proc_uid ~proc_name ~analysis_summary ~report_summary
| ReplaceAttributes
{proc_uid; proc_name; proc_name_hum; attr_kind; source_file; proc_attributes; cfg; callees}
->
Implementation.replace_attributes ~proc_uid ~proc_name ~proc_name_hum ~attr_kind
~source_file ~proc_attributes ~cfg ~callees
| ResetCaptureTables ->
Implementation.reset_capture_tables ()
| Terminate ->
@ -383,8 +397,11 @@ let start () = Server.start ()
let stop () = Server.send Command.Terminate
let replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes ~proc_desc ~callees =
perform (ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees})
let replace_attributes ~proc_uid ~proc_name ~proc_name_hum ~attr_kind ~source_file ~proc_attributes
~cfg ~callees =
perform
(ReplaceAttributes
{proc_uid; proc_name; proc_name_hum; attr_kind; source_file; proc_attributes; cfg; callees})
let add_source_file ~source_file ~tenv ~integer_type_widths ~proc_names =
@ -399,10 +416,10 @@ let canonicalize () = perform Vacuum
let reset_capture_tables () = perform ResetCaptureTables
let store_spec ~proc_name ~analysis_summary ~report_summary =
perform (StoreSpec {proc_name; analysis_summary; report_summary})
let store_spec ~proc_uid ~proc_name ~analysis_summary ~report_summary =
perform (StoreSpec {proc_uid; proc_name; analysis_summary; report_summary})
let delete_spec ~proc_name = perform (DeleteSpec {proc_name})
let delete_spec ~proc_uid = perform (DeleteSpec {proc_uid})
let delete_all_specs () = perform DeleteAllSpecs

@ -12,12 +12,13 @@ val use_daemon : bool
(** indicates that there should be a daemon running *)
val replace_attributes :
pname_str:string
-> pname:Sqlite3.Data.t
-> akind:int64
proc_uid:string
-> proc_name:Sqlite3.Data.t
-> proc_name_hum:string
-> attr_kind:int64
-> source_file:Sqlite3.Data.t
-> attributes:Sqlite3.Data.t
-> proc_desc:Sqlite3.Data.t
-> proc_attributes:Sqlite3.Data.t
-> cfg:Sqlite3.Data.t
-> callees:Sqlite3.Data.t
-> unit
@ -42,11 +43,12 @@ val start : unit -> unit
val stop : unit -> unit
val store_spec :
proc_name:Sqlite3.Data.t
proc_uid:string
-> proc_name:Sqlite3.Data.t
-> analysis_summary:Sqlite3.Data.t
-> report_summary:Sqlite3.Data.t
-> unit
val delete_spec : proc_name:Sqlite3.Data.t -> unit
val delete_spec : proc_uid:string -> unit
val delete_all_specs : unit -> unit

@ -15,7 +15,8 @@ let procedures_schema prefix =
Printf.sprintf
{|
CREATE TABLE IF NOT EXISTS %sprocedures
( proc_name BLOB PRIMARY KEY NOT NULL
( proc_uid TEXT PRIMARY KEY NOT NULL
, proc_name BLOB NOT NULL
, proc_name_hum TEXT
, attr_kind INTEGER NOT NULL
, source_file TEXT NOT NULL
@ -44,7 +45,8 @@ let specs_schema prefix =
Printf.sprintf
{|
CREATE TABLE IF NOT EXISTS %sspecs
( proc_name BLOB PRIMARY KEY NOT NULL
( proc_uid TEXT PRIMARY KEY NOT NULL
, proc_name BLOB NOT NULL
, analysis_summary BLOB NOT NULL
, report_summary BLOB NOT NULL
)

@ -1 +1 @@
templates.cpp, unused_var_template_bad<int>, 1, DEAD_STORE, no_bucket, ERROR, [Write of unused value]
templates.cpp, unused_var_template_bad<char>, 1, DEAD_STORE, no_bucket, ERROR, [Write of unused value]

@ -2,28 +2,28 @@
digraph callgraph {
N1 [ label = "Test.<init>()", flag = true ];
N5 [ label = "void PrintStream.println(String)", flag = false ];
N5 -> N3 ;
N5 [ label = "void Test.main(String[])", flag = true ];
N0 [ label = "Object.<init>()", flag = false ];
N0 -> N2 ;
N0 -> N7 ;
N0 -> N1 ;
N7 [ label = "void Test.complexityIncrease(int)", flag = true ];
N7 -> N8 ;
N7 [ label = "Unchanged.<init>()", flag = false ];
N6 [ label = "void Test.complexityDecrease(int)", flag = true ];
N6 -> N8 ;
N6 [ label = "void Test.newMethod(int)", flag = true ];
N4 [ label = "void Test.newMethod(int)", flag = true ];
N4 [ label = "void Test.complexityIncrease(int)", flag = true ];
N4 -> N5 ;
N2 [ label = "Unchanged.<init>()", flag = false ];
N2 [ label = "void Unchanged.orderN(int)", flag = false ];
N2 -> N6 ;
N2 -> N4 ;
N2 -> N3 ;
N8 [ label = "void Test.main(String[])", flag = true ];
N8 [ label = "void PrintStream.println(String)", flag = false ];
N8 -> N2 ;
N3 [ label = "void Unchanged.orderN(int)", flag = false ];
N3 -> N7 ;
N3 -> N6 ;
N3 -> N4 ;
N3 [ label = "void Test.complexityDecrease(int)", flag = true ];
N3 -> N5 ;
}

@ -1,2 +1,2 @@
build_systems/codetoanalyze/objc_missing_fld/A.m, badOnlyOneNDA, 5, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure badOnlyOneNDA(),start of procedure predA(),start of procedure implOnlyFn:,return from a call to A.implOnlyFn:,start of procedure delegate,return from a call to A.delegate,Condition is true,return from a call to predA,Taking false branch]
build_systems/codetoanalyze/objc_missing_fld/B.m, badOnlyOneNDB, 5, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure badOnlyOneNDB(),start of procedure predB(),Skipping implOnlyFn:: method has no implementation,start of procedure delegate,return from a call to A.delegate,Condition is true,return from a call to predB,Taking false branch]
build_systems/codetoanalyze/objc_missing_fld/B.m, badOnlyOneNDB, 5, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure badOnlyOneNDB(),start of procedure predB(),start of procedure implOnlyFn:,return from a call to A.implOnlyFn:,start of procedure delegate,return from a call to A.delegate,Condition is true,return from a call to predB,Taking false branch]

Loading…
Cancel
Save