From c98783a45aae11c6d4cb2bb7a03090a613b454e1 Mon Sep 17 00:00:00 2001 From: Nikos Gorogiannis Date: Fri, 14 Aug 2020 06:27:41 -0700 Subject: [PATCH] [specs] store analysis and report artefacts separately Summary: Constructing the report is done by reading all the summaries, and using certain parts thereof. However, the payloads, which typically account for the greatest size, are not used (with the exception of costs). This diff splits the storage of summaries into analysis and report summaries, and only reads and deserialises the latter for the report phase. This makes a big difference for runs with a large number of procedures. Reviewed By: jvillard Differential Revision: D23105072 fbshipit-source-id: 359067a0f --- infer/src/backend/Summary.ml | 137 ++++++++++++++++++++------- infer/src/backend/Summary.mli | 8 +- infer/src/base/DBWriter.ml | 37 ++++---- infer/src/base/DBWriter.mli | 6 +- infer/src/base/ResultsDatabase.ml | 3 +- infer/src/integration/JsonReports.ml | 21 ++-- 6 files changed, 144 insertions(+), 68 deletions(-) diff --git a/infer/src/backend/Summary.ml b/infer/src/backend/Summary.ml index 84c572f52..466447a9a 100644 --- a/infer/src/backend/Summary.ml +++ b/infer/src/backend/Summary.ml @@ -9,7 +9,6 @@ open! IStd module F = Format module L = Logging -module CLOpt = CommandLineOption module Stats = struct type t = @@ -69,6 +68,8 @@ include struct [@@deriving fields] end +type full_summary = t + let get_status summary = summary.status let get_proc_desc summary = summary.proc_desc @@ -118,9 +119,58 @@ let pp_html source fmt summary = F.fprintf fmt "@\n" -module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct - type nonrec t = t -end) +module ReportSummary = struct + type t = {loc: Location.t; cost_opt: CostDomain.summary option; err_log: Errlog.t} + + let of_full_summary (f : full_summary) = + ({loc= get_loc f; cost_opt= f.payloads.Payloads.cost; err_log= f.err_log} : t) + + + module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct + type nonrec t = t + end) +end + +module AnalysisSummary = struct + include struct + (* ignore dead modules added by @@deriving fields *) + [@@@warning "-60"] + + type t = + { payloads: Payloads.t + ; mutable sessions: int + ; stats: Stats.t + ; status: Status.t + ; proc_desc: Procdesc.t + ; mutable callee_pnames: Procname.Set.t } + [@@deriving fields] + end + + let of_full_summary (f : full_summary) = + ( { payloads= f.payloads + ; sessions= f.sessions + ; stats= f.stats + ; status= f.status + ; proc_desc= f.proc_desc + ; callee_pnames= f.callee_pnames } + : t ) + + + module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct + type nonrec t = t + end) +end + +let mk_full_summary (report_summary : ReportSummary.t) (analysis_summary : AnalysisSummary.t) = + ( { payloads= analysis_summary.payloads + ; sessions= analysis_summary.sessions + ; stats= analysis_summary.stats + ; status= analysis_summary.status + ; proc_desc= analysis_summary.proc_desc + ; callee_pnames= analysis_summary.callee_pnames + ; err_log= report_summary.err_log } + : full_summary ) + module OnDisk = struct type cache = t Procname.Hash.t @@ -181,15 +231,18 @@ module OnDisk = struct let spec_of_procname = let load_statement = - ResultsDatabase.register_statement "SELECT spec FROM specs WHERE proc_name = :k" + ResultsDatabase.register_statement + "SELECT analysis_summary, report_summary FROM specs WHERE proc_name = :k" in fun proc_name -> ResultsDatabase.with_registered_statement load_statement ~f:(fun db load_stmt -> Sqlite3.bind load_stmt 1 (Procname.SQLite.serialize proc_name) |> SqliteUtils.check_result_code db ~log:"load proc specs bind proc_name" ; - SqliteUtils.result_single_column_option ~finalize:false ~log:"load proc specs run" db - load_stmt - |> Option.map ~f:SQLite.deserialize ) + SqliteUtils.result_option ~finalize:false db ~log:"load proc specs run" load_stmt + ~read_row:(fun stmt -> + let analysis_summary = Sqlite3.column stmt 0 |> AnalysisSummary.SQLite.deserialize in + let report_summary = Sqlite3.column stmt 1 |> ReportSummary.SQLite.deserialize in + mk_full_summary report_summary analysis_summary ) ) (** Load procedure summary for the given procedure name and update spec table *) @@ -236,9 +289,12 @@ module OnDisk = struct (specs_filename_of_procname proc_name) ~data:final_summary else + let analysis_summary = AnalysisSummary.of_full_summary final_summary in + let report_summary = ReportSummary.of_full_summary final_summary in DBWriter.store_spec ~proc_name:(Procname.SQLite.serialize proc_name) - ~spec:(SQLite.serialize final_summary) + ~analysis_summary:(AnalysisSummary.SQLite.serialize analysis_summary) + ~report_summary:(ReportSummary.SQLite.serialize report_summary) let reset proc_desc = @@ -274,39 +330,56 @@ module OnDisk = struct else DBWriter.delete_spec ~proc_name:(Procname.SQLite.serialize pname) - let iter_specs = - let iter_statement = - (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *) - ResultsDatabase.register_statement "SELECT spec FROM specs ORDER BY proc_name ASC" - in - fun ~f -> - ResultsDatabase.with_registered_statement iter_statement ~f:(fun db stmt -> - SqliteUtils.result_fold_single_column_rows ~finalize:false db stmt - ~log:"iter over all specs" ~init:() ~f:(fun () sqlite_spec -> - let summary : t = SQLite.deserialize sqlite_spec in - let () = f summary in - () ) ) + let iter_filtered_specs ~filter ~f = + let db = ResultsDatabase.get_database () in + let dummy_source_file = SourceFile.invalid __FILE__ in + (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *) + Sqlite3.prepare db + "SELECT proc_name, analysis_summary, report_summary FROM specs ORDER BY proc_name ASC" + |> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs") + ~f:(fun stmt -> + let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in + if filter dummy_source_file proc_name then + let analysis_summary = Sqlite3.column stmt 1 |> AnalysisSummary.SQLite.deserialize in + let report_summary = Sqlite3.column stmt 2 |> ReportSummary.SQLite.deserialize in + let spec = mk_full_summary report_summary analysis_summary in + f spec ) - let iter_over_filter ~filter ~f = + let iter_filtered_report_summaries ~filter ~f = let db = ResultsDatabase.get_database () in + let dummy_source_file = SourceFile.invalid __FILE__ in (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *) - Sqlite3.prepare db "SELECT proc_name, spec FROM specs ORDER BY proc_name ASC" + Sqlite3.prepare db "SELECT proc_name, report_summary FROM specs ORDER BY proc_name ASC" |> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs") ~f:(fun stmt -> let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in - let spec = Sqlite3.column stmt 1 |> SQLite.deserialize in - if filter (SourceFile.invalid "invalid") proc_name then f spec ) + if filter dummy_source_file proc_name then + let ({loc; cost_opt; err_log} : ReportSummary.t) = + Sqlite3.column stmt 1 |> ReportSummary.SQLite.deserialize + in + f proc_name loc cost_opt err_log ) + + + let make_filtered_iterator_from_config ~iter ~f = + let filter = + if Option.is_some Config.procedures_filter then ( + if Config.test_filtering then ( + Inferconfig.test () ; + L.exit 0 ) ; + Lazy.force Filtering.procedures_filter ) + else fun _ _ -> true + in + iter ~filter ~f + + + let iter_report_summaries_from_config ~f = + make_filtered_iterator_from_config ~iter:iter_filtered_report_summaries ~f - let iter_specs_from_config ~f = - if CLOpt.is_originator && Option.is_some Config.procedures_filter then ( - if Config.test_filtering then ( - Inferconfig.test () ; - L.exit 0 ) ; - iter_over_filter ~filter:(Lazy.force Filtering.procedures_filter) ~f ) - else iter_specs ~f + let iter_specs_from_config ~f = make_filtered_iterator_from_config ~iter:iter_filtered_specs ~f + let iter_specs ~f = iter_filtered_specs ~filter:(fun _ _ -> true) ~f let pp_specs_from_config fmt = iter_specs_from_config ~f:(fun summary -> diff --git a/infer/src/backend/Summary.mli b/infer/src/backend/Summary.mli index dbed10667..d93163327 100644 --- a/infer/src/backend/Summary.mli +++ b/infer/src/backend/Summary.mli @@ -50,8 +50,6 @@ val get_proc_desc : t -> Procdesc.t val get_err_log : t -> Errlog.t -val get_loc : t -> Location.t - val get_status : t -> Status.t (** Return the status (active v.s. inactive) of a procedure summary *) @@ -87,9 +85,9 @@ module OnDisk : sig val iter_specs : f:(t -> unit) -> unit (** Iterates over all stored summaries *) - val iter_specs_from_config : f:(t -> unit) -> unit - (** Iterates over all stored summaries, or over the summaries of the list of procedure filenames - passed on the command line *) + val iter_report_summaries_from_config : + f:(Procname.t -> Location.t -> CostDomain.summary option -> Errlog.t -> unit) -> unit + (** Iterates over all analysis artefacts listed above, for each procedure *) val pp_specs_from_config : Format.formatter -> unit (** pretty print all stored summaries *) diff --git a/infer/src/base/DBWriter.ml b/infer/src/base/DBWriter.ml index 9dd28c84a..32bb6dce6 100644 --- a/infer/src/base/DBWriter.ml +++ b/infer/src/base/DBWriter.ml @@ -177,14 +177,17 @@ module Implementation = struct let store_spec = let store_statement = - ResultsDatabase.register_statement "INSERT OR REPLACE INTO specs VALUES (:proc_name, :spec)" + ResultsDatabase.register_statement + "INSERT OR REPLACE INTO specs VALUES (:proc_name, :analysis_summary, :report_summary)" in - fun ~proc_name ~spec -> + fun ~proc_name ~analysis_summary ~report_summary -> ResultsDatabase.with_registered_statement store_statement ~f:(fun db store_stmt -> Sqlite3.bind store_stmt 1 proc_name |> SqliteUtils.check_result_code db ~log:"store spec bind proc_name" ; - Sqlite3.bind store_stmt 2 spec - |> SqliteUtils.check_result_code db ~log:"store spec bind spec" ; + Sqlite3.bind store_stmt 2 analysis_summary + |> SqliteUtils.check_result_code db ~log:"store spec bind analysis_summary" ; + Sqlite3.bind store_stmt 3 report_summary + |> SqliteUtils.check_result_code db ~log:"store spec bind report_summary" ; SqliteUtils.result_unit ~finalize:false ~log:"store spec" db store_stmt ) @@ -210,7 +213,8 @@ module Command = struct | Handshake | MarkAllSourceFilesStale | Merge of {infer_deps_file: string} - | StoreSpec of {proc_name: Sqlite3.Data.t; spec: Sqlite3.Data.t} + | StoreSpec of + {proc_name: Sqlite3.Data.t; analysis_summary: Sqlite3.Data.t; report_summary: Sqlite3.Data.t} | ReplaceAttributes of { pname_str: string ; pname: Sqlite3.Data.t @@ -259,8 +263,8 @@ module Command = struct Implementation.mark_all_source_files_stale () | Merge {infer_deps_file} -> Implementation.merge infer_deps_file - | StoreSpec {proc_name; spec} -> - Implementation.store_spec ~proc_name ~spec + | StoreSpec {proc_name; analysis_summary; report_summary} -> + Implementation.store_spec ~proc_name ~analysis_summary ~report_summary | ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees} -> Implementation.replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes ~proc_desc ~callees @@ -368,22 +372,23 @@ let start () = Server.start () let stop () = Server.send Command.Terminate let replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes ~proc_desc ~callees = - Command.ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees} - |> perform + perform (ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees}) let add_source_file ~source_file ~tenv ~integer_type_widths ~proc_names = - Command.AddSourceFile {source_file; tenv; integer_type_widths; proc_names} |> perform + perform (AddSourceFile {source_file; tenv; integer_type_widths; proc_names}) -let mark_all_source_files_stale () = perform Command.MarkAllSourceFilesStale +let mark_all_source_files_stale () = perform MarkAllSourceFilesStale -let merge ~infer_deps_file = Command.Merge {infer_deps_file} |> perform +let merge ~infer_deps_file = perform (Merge {infer_deps_file}) -let canonicalize () = perform Command.Vacuum +let canonicalize () = perform Vacuum -let reset_capture_tables () = perform Command.ResetCaptureTables +let reset_capture_tables () = perform ResetCaptureTables -let store_spec ~proc_name ~spec = perform (Command.StoreSpec {proc_name; spec}) +let store_spec ~proc_name ~analysis_summary ~report_summary = + perform (StoreSpec {proc_name; analysis_summary; report_summary}) -let delete_spec ~proc_name = perform (Command.DeleteSpec {proc_name}) + +let delete_spec ~proc_name = perform (DeleteSpec {proc_name}) diff --git a/infer/src/base/DBWriter.mli b/infer/src/base/DBWriter.mli index d3b3724f8..cd6695f89 100644 --- a/infer/src/base/DBWriter.mli +++ b/infer/src/base/DBWriter.mli @@ -41,6 +41,10 @@ val start : unit -> unit val stop : unit -> unit -val store_spec : proc_name:Sqlite3.Data.t -> spec:Sqlite3.Data.t -> unit +val store_spec : + proc_name:Sqlite3.Data.t + -> analysis_summary:Sqlite3.Data.t + -> report_summary:Sqlite3.Data.t + -> unit val delete_spec : proc_name:Sqlite3.Data.t -> unit diff --git a/infer/src/base/ResultsDatabase.ml b/infer/src/base/ResultsDatabase.ml index e9761c247..f5d1bbb6e 100644 --- a/infer/src/base/ResultsDatabase.ml +++ b/infer/src/base/ResultsDatabase.ml @@ -41,7 +41,8 @@ let specs_schema prefix = {| CREATE TABLE IF NOT EXISTS %sspecs ( proc_name TEXT PRIMARY KEY - , spec BLOB NOT NULL + , analysis_summary BLOB NOT NULL + , report_summary BLOB NOT NULL )|} prefix diff --git a/infer/src/integration/JsonReports.ml b/infer/src/integration/JsonReports.ml index 613855dae..5de3c282e 100644 --- a/infer/src/integration/JsonReports.ml +++ b/infer/src/integration/JsonReports.ml @@ -282,20 +282,15 @@ let mk_error_filter filters proc_name file error_name = && filters.Inferconfig.proc_filter proc_name -let collect_issues summary issues_acc = - let err_log = Summary.get_err_log summary in - let proc_name = Summary.get_proc_name summary in - let proc_location = Summary.get_loc summary in +let collect_issues proc_name proc_location err_log issues_acc = Errlog.fold (fun err_key err_data acc -> {Issue.proc_name; proc_location; err_key; err_data} :: acc) err_log issues_acc -let write_costs summary (outfile : Utils.outfile) = - let proc_name = Summary.get_proc_name summary in +let write_costs proc_name loc cost_opt (outfile : Utils.outfile) = if not (Cost.is_report_suppressed proc_name) then - JsonCostsPrinter.pp outfile.fmt - {loc= Summary.get_loc summary; proc_name; cost_opt= summary.Summary.payloads.Payloads.cost} + JsonCostsPrinter.pp outfile.fmt {loc; proc_name; cost_opt} (** Process lint issues of a procedure *) @@ -305,17 +300,17 @@ let write_lint_issues filters (issues_outf : Utils.outfile) linereader procname (** Process a summary *) -let process_summary ~costs_outf summary issues_acc = - write_costs summary costs_outf ; - collect_issues summary issues_acc +let process_summary ~costs_outf proc_name loc cost_opt err_log issues_acc = + write_costs proc_name loc cost_opt costs_outf ; + collect_issues proc_name loc err_log issues_acc let process_all_summaries_and_issues ~issues_outf ~costs_outf = let linereader = LineReader.create () in let filters = Inferconfig.create_filters () in let all_issues = ref [] in - Summary.OnDisk.iter_specs_from_config ~f:(fun summary -> - all_issues := process_summary ~costs_outf summary !all_issues ) ; + Summary.OnDisk.iter_report_summaries_from_config ~f:(fun proc_name loc cost_opt err_log -> + all_issues := process_summary ~costs_outf proc_name loc cost_opt err_log !all_issues ) ; all_issues := Issue.sort_filter_issues !all_issues ; List.iter ~f:(fun {Issue.proc_name; proc_location; err_key; err_data} ->