[specs] store analysis and report artefacts separately

Summary:
Constructing the report is done by reading all the summaries, and using certain parts thereof.  However, the payloads, which typically account for the greatest size, are not used (with the exception of costs).

This diff splits the storage of summaries into analysis and report summaries, and only reads and deserialises the latter for the report phase. This makes a big difference for runs with a large number of procedures.

Reviewed By: jvillard

Differential Revision: D23105072

fbshipit-source-id: 359067a0f

infer/src/backend/Summary.ml

@@ -9,7 +9,6 @@
 open! IStd
 module F = Format
 module L = Logging
-module CLOpt = CommandLineOption
 
 module Stats = struct
   type t =
@@ -69,6 +68,8 @@ include struct
   [@@deriving fields]
 end
 
+type full_summary = t
+
 let get_status summary = summary.status
 
 let get_proc_desc summary = summary.proc_desc
@@ -118,9 +119,58 @@ let pp_html source fmt summary =
   F.fprintf fmt "</LISTING>@\n"
 
 
-module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct
+module ReportSummary = struct
-  type nonrec t = t
+  type t = {loc: Location.t; cost_opt: CostDomain.summary option; err_log: Errlog.t}
-end)
+
+  let of_full_summary (f : full_summary) =
+    ({loc= get_loc f; cost_opt= f.payloads.Payloads.cost; err_log= f.err_log} : t)
+
+
+  module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct
+    type nonrec t = t
+  end)
+end
+
+module AnalysisSummary = struct
+  include struct
+    (* ignore dead modules added by @@deriving fields *)
+    [@@@warning "-60"]
+
+    type t =
+      { payloads: Payloads.t
+      ; mutable sessions: int
+      ; stats: Stats.t
+      ; status: Status.t
+      ; proc_desc: Procdesc.t
+      ; mutable callee_pnames: Procname.Set.t }
+    [@@deriving fields]
+  end
+
+  let of_full_summary (f : full_summary) =
+    ( { payloads= f.payloads
+      ; sessions= f.sessions
+      ; stats= f.stats
+      ; status= f.status
+      ; proc_desc= f.proc_desc
+      ; callee_pnames= f.callee_pnames }
+      : t )
+
+
+  module SQLite = SqliteUtils.MarshalledDataNOTForComparison (struct
+    type nonrec t = t
+  end)
+end
+
+let mk_full_summary (report_summary : ReportSummary.t) (analysis_summary : AnalysisSummary.t) =
+  ( { payloads= analysis_summary.payloads
+    ; sessions= analysis_summary.sessions
+    ; stats= analysis_summary.stats
+    ; status= analysis_summary.status
+    ; proc_desc= analysis_summary.proc_desc
+    ; callee_pnames= analysis_summary.callee_pnames
+    ; err_log= report_summary.err_log }
+    : full_summary )
+
 
 module OnDisk = struct
   type cache = t Procname.Hash.t
@@ -181,15 +231,18 @@ module OnDisk = struct
 
   let spec_of_procname =
     let load_statement =
-      ResultsDatabase.register_statement "SELECT spec FROM specs WHERE proc_name = :k"
+      ResultsDatabase.register_statement
+        "SELECT analysis_summary, report_summary FROM specs WHERE proc_name = :k"
     in
     fun proc_name ->
       ResultsDatabase.with_registered_statement load_statement ~f:(fun db load_stmt ->
           Sqlite3.bind load_stmt 1 (Procname.SQLite.serialize proc_name)
           |> SqliteUtils.check_result_code db ~log:"load proc specs bind proc_name" ;
-          SqliteUtils.result_single_column_option ~finalize:false ~log:"load proc specs run" db
+          SqliteUtils.result_option ~finalize:false db ~log:"load proc specs run" load_stmt
-            load_stmt
+            ~read_row:(fun stmt ->
-          |> Option.map ~f:SQLite.deserialize )
+              let analysis_summary = Sqlite3.column stmt 0 |> AnalysisSummary.SQLite.deserialize in
+              let report_summary = Sqlite3.column stmt 1 |> ReportSummary.SQLite.deserialize in
+              mk_full_summary report_summary analysis_summary ) )
 
 
   (** Load procedure summary for the given procedure name and update spec table *)
@@ -236,9 +289,12 @@ module OnDisk = struct
         (specs_filename_of_procname proc_name)
         ~data:final_summary
     else
+      let analysis_summary = AnalysisSummary.of_full_summary final_summary in
+      let report_summary = ReportSummary.of_full_summary final_summary in
       DBWriter.store_spec
         ~proc_name:(Procname.SQLite.serialize proc_name)
-        ~spec:(SQLite.serialize final_summary)
+        ~analysis_summary:(AnalysisSummary.SQLite.serialize analysis_summary)
+        ~report_summary:(ReportSummary.SQLite.serialize report_summary)
 
 
   let reset proc_desc =
@@ -274,39 +330,56 @@ module OnDisk = struct
     else DBWriter.delete_spec ~proc_name:(Procname.SQLite.serialize pname)
 
 
-  let iter_specs =
+  let iter_filtered_specs ~filter ~f =
-    let iter_statement =
+    let db = ResultsDatabase.get_database () in
-      (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *)
+    let dummy_source_file = SourceFile.invalid __FILE__ in
-      ResultsDatabase.register_statement "SELECT spec FROM specs ORDER BY proc_name ASC"
+    (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *)
-    in
+    Sqlite3.prepare db
-    fun ~f ->
+      "SELECT proc_name, analysis_summary, report_summary FROM specs ORDER BY proc_name ASC"
-      ResultsDatabase.with_registered_statement iter_statement ~f:(fun db stmt ->
+    |> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs")
-          SqliteUtils.result_fold_single_column_rows ~finalize:false db stmt
+         ~f:(fun stmt ->
-            ~log:"iter over all specs" ~init:() ~f:(fun () sqlite_spec ->
+           let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in
-              let summary : t = SQLite.deserialize sqlite_spec in
+           if filter dummy_source_file proc_name then
-              let () = f summary in
+             let analysis_summary = Sqlite3.column stmt 1 |> AnalysisSummary.SQLite.deserialize in
-              () ) )
+             let report_summary = Sqlite3.column stmt 2 |> ReportSummary.SQLite.deserialize in
+             let spec = mk_full_summary report_summary analysis_summary in
+             f spec )
 
 
-  let iter_over_filter ~filter ~f =
+  let iter_filtered_report_summaries ~filter ~f =
     let db = ResultsDatabase.get_database () in
+    let dummy_source_file = SourceFile.invalid __FILE__ in
     (* NB the order is deterministic, but it is over a serialised value, so it is arbitrary *)
-    Sqlite3.prepare db "SELECT proc_name, spec FROM specs ORDER BY proc_name ASC"
+    Sqlite3.prepare db "SELECT proc_name, report_summary FROM specs ORDER BY proc_name ASC"
     |> Container.iter ~fold:(SqliteUtils.result_fold_rows db ~log:"iter over filtered specs")
          ~f:(fun stmt ->
            let proc_name = Sqlite3.column stmt 0 |> Procname.SQLite.deserialize in
-           let spec = Sqlite3.column stmt 1 |> SQLite.deserialize in
+           if filter dummy_source_file proc_name then
-           if filter (SourceFile.invalid "invalid") proc_name then f spec )
+             let ({loc; cost_opt; err_log} : ReportSummary.t) =
+               Sqlite3.column stmt 1 |> ReportSummary.SQLite.deserialize
+             in
+             f proc_name loc cost_opt err_log )
+
+
+  let make_filtered_iterator_from_config ~iter ~f =
+    let filter =
+      if Option.is_some Config.procedures_filter then (
+        if Config.test_filtering then (
+          Inferconfig.test () ;
+          L.exit 0 ) ;
+        Lazy.force Filtering.procedures_filter )
+      else fun _ _ -> true
+    in
+    iter ~filter ~f
+
+
+  let iter_report_summaries_from_config ~f =
+    make_filtered_iterator_from_config ~iter:iter_filtered_report_summaries ~f
 
 
-  let iter_specs_from_config ~f =
+  let iter_specs_from_config ~f = make_filtered_iterator_from_config ~iter:iter_filtered_specs ~f
-    if CLOpt.is_originator && Option.is_some Config.procedures_filter then (
-      if Config.test_filtering then (
-        Inferconfig.test () ;
-        L.exit 0 ) ;
-      iter_over_filter ~filter:(Lazy.force Filtering.procedures_filter) ~f )
-    else iter_specs ~f
 
+  let iter_specs ~f = iter_filtered_specs ~filter:(fun _ _ -> true) ~f
 
   let pp_specs_from_config fmt =
     iter_specs_from_config ~f:(fun summary ->

infer/src/backend/Summary.mli

@@ -50,8 +50,6 @@ val get_proc_desc : t -> Procdesc.t
 
 val get_err_log : t -> Errlog.t
 
-val get_loc : t -> Location.t
-
 val get_status : t -> Status.t
 (** Return the status (active v.s. inactive) of a procedure summary *)
 
@@ -87,9 +85,9 @@ module OnDisk : sig
   val iter_specs : f:(t -> unit) -> unit
   (** Iterates over all stored summaries *)
 
-  val iter_specs_from_config : f:(t -> unit) -> unit
+  val iter_report_summaries_from_config :
-  (** Iterates over all stored summaries, or over the summaries of the list of procedure filenames
+    f:(Procname.t -> Location.t -> CostDomain.summary option -> Errlog.t -> unit) -> unit
-      passed on the command line *)
+  (** Iterates over all analysis artefacts listed above, for each procedure *)
 
   val pp_specs_from_config : Format.formatter -> unit
   (** pretty print all stored summaries *)

infer/src/base/DBWriter.ml

@@ -177,14 +177,17 @@ module Implementation = struct
 
   let store_spec =
     let store_statement =
-      ResultsDatabase.register_statement "INSERT OR REPLACE INTO specs VALUES (:proc_name, :spec)"
+      ResultsDatabase.register_statement
+        "INSERT OR REPLACE INTO specs VALUES (:proc_name, :analysis_summary, :report_summary)"
     in
-    fun ~proc_name ~spec ->
+    fun ~proc_name ~analysis_summary ~report_summary ->
       ResultsDatabase.with_registered_statement store_statement ~f:(fun db store_stmt ->
           Sqlite3.bind store_stmt 1 proc_name
           |> SqliteUtils.check_result_code db ~log:"store spec bind proc_name" ;
-          Sqlite3.bind store_stmt 2 spec
+          Sqlite3.bind store_stmt 2 analysis_summary
-          |> SqliteUtils.check_result_code db ~log:"store spec bind spec" ;
+          |> SqliteUtils.check_result_code db ~log:"store spec bind analysis_summary" ;
+          Sqlite3.bind store_stmt 3 report_summary
+          |> SqliteUtils.check_result_code db ~log:"store spec bind report_summary" ;
           SqliteUtils.result_unit ~finalize:false ~log:"store spec" db store_stmt )
 
 
@@ -210,7 +213,8 @@ module Command = struct
     | Handshake
     | MarkAllSourceFilesStale
     | Merge of {infer_deps_file: string}
-    | StoreSpec of {proc_name: Sqlite3.Data.t; spec: Sqlite3.Data.t}
+    | StoreSpec of
+        {proc_name: Sqlite3.Data.t; analysis_summary: Sqlite3.Data.t; report_summary: Sqlite3.Data.t}
     | ReplaceAttributes of
         { pname_str: string
         ; pname: Sqlite3.Data.t
@@ -259,8 +263,8 @@ module Command = struct
         Implementation.mark_all_source_files_stale ()
     | Merge {infer_deps_file} ->
         Implementation.merge infer_deps_file
-    | StoreSpec {proc_name; spec} ->
+    | StoreSpec {proc_name; analysis_summary; report_summary} ->
-        Implementation.store_spec ~proc_name ~spec
+        Implementation.store_spec ~proc_name ~analysis_summary ~report_summary
     | ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees} ->
         Implementation.replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes
           ~proc_desc ~callees
@@ -368,22 +372,23 @@ let start () = Server.start ()
 let stop () = Server.send Command.Terminate
 
 let replace_attributes ~pname_str ~pname ~akind ~source_file ~attributes ~proc_desc ~callees =
-  Command.ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees}
+  perform (ReplaceAttributes {pname_str; pname; akind; source_file; attributes; proc_desc; callees})
-  |> perform
 
 
 let add_source_file ~source_file ~tenv ~integer_type_widths ~proc_names =
-  Command.AddSourceFile {source_file; tenv; integer_type_widths; proc_names} |> perform
+  perform (AddSourceFile {source_file; tenv; integer_type_widths; proc_names})
 
 
-let mark_all_source_files_stale () = perform Command.MarkAllSourceFilesStale
+let mark_all_source_files_stale () = perform MarkAllSourceFilesStale
 
-let merge ~infer_deps_file = Command.Merge {infer_deps_file} |> perform
+let merge ~infer_deps_file = perform (Merge {infer_deps_file})
 
-let canonicalize () = perform Command.Vacuum
+let canonicalize () = perform Vacuum
 
-let reset_capture_tables () = perform Command.ResetCaptureTables
+let reset_capture_tables () = perform ResetCaptureTables
 
-let store_spec ~proc_name ~spec = perform (Command.StoreSpec {proc_name; spec})
+let store_spec ~proc_name ~analysis_summary ~report_summary =
+  perform (StoreSpec {proc_name; analysis_summary; report_summary})
 
-let delete_spec ~proc_name = perform (Command.DeleteSpec {proc_name})
+
+let delete_spec ~proc_name = perform (DeleteSpec {proc_name})

infer/src/base/DBWriter.mli

@@ -41,6 +41,10 @@ val start : unit -> unit
 
 val stop : unit -> unit
 
-val store_spec : proc_name:Sqlite3.Data.t -> spec:Sqlite3.Data.t -> unit
+val store_spec :
+     proc_name:Sqlite3.Data.t
+  -> analysis_summary:Sqlite3.Data.t
+  -> report_summary:Sqlite3.Data.t
+  -> unit
 
 val delete_spec : proc_name:Sqlite3.Data.t -> unit

infer/src/base/ResultsDatabase.ml

@@ -41,7 +41,8 @@ let specs_schema prefix =
     {|
       CREATE TABLE IF NOT EXISTS %sspecs
       ( proc_name TEXT PRIMARY KEY
-      , spec BLOB NOT NULL
+      , analysis_summary BLOB NOT NULL
+      , report_summary BLOB NOT NULL
     )|}
     prefix
 

infer/src/integration/JsonReports.ml

@@ -282,20 +282,15 @@ let mk_error_filter filters proc_name file error_name =
   && filters.Inferconfig.proc_filter proc_name
 
 
-let collect_issues summary issues_acc =
+let collect_issues proc_name proc_location err_log issues_acc =
-  let err_log = Summary.get_err_log summary in
-  let proc_name = Summary.get_proc_name summary in
-  let proc_location = Summary.get_loc summary in
   Errlog.fold
     (fun err_key err_data acc -> {Issue.proc_name; proc_location; err_key; err_data} :: acc)
     err_log issues_acc
 
 
-let write_costs summary (outfile : Utils.outfile) =
+let write_costs proc_name loc cost_opt (outfile : Utils.outfile) =
-  let proc_name = Summary.get_proc_name summary in
   if not (Cost.is_report_suppressed proc_name) then
-    JsonCostsPrinter.pp outfile.fmt
+    JsonCostsPrinter.pp outfile.fmt {loc; proc_name; cost_opt}
-      {loc= Summary.get_loc summary; proc_name; cost_opt= summary.Summary.payloads.Payloads.cost}
 
 
 (** Process lint issues of a procedure *)
@@ -305,17 +300,17 @@ let write_lint_issues filters (issues_outf : Utils.outfile) linereader procname
 
 
 (** Process a summary *)
-let process_summary ~costs_outf summary issues_acc =
+let process_summary ~costs_outf proc_name loc cost_opt err_log issues_acc =
-  write_costs summary costs_outf ;
+  write_costs proc_name loc cost_opt costs_outf ;
-  collect_issues summary issues_acc
+  collect_issues proc_name loc err_log issues_acc
 
 
 let process_all_summaries_and_issues ~issues_outf ~costs_outf =
   let linereader = LineReader.create () in
   let filters = Inferconfig.create_filters () in
   let all_issues = ref [] in
-  Summary.OnDisk.iter_specs_from_config ~f:(fun summary ->
+  Summary.OnDisk.iter_report_summaries_from_config ~f:(fun proc_name loc cost_opt err_log ->
-      all_issues := process_summary ~costs_outf summary !all_issues ) ;
+      all_issues := process_summary ~costs_outf proc_name loc cost_opt err_log !all_issues ) ;
   all_issues := Issue.sort_filter_issues !all_issues ;
   List.iter
     ~f:(fun {Issue.proc_name; proc_location; err_key; err_data} ->