[cli] add the ability to rerun the analysis on some files with different options

Summary:
When `--reanalyze` is passed, mark the summaries of procedures matching
`--procedures-filter` as needing to be analysed before running the analysis.

This allows one to, for instance, re-run the analysis in debug mode on only
some files or procedures. However, this won't work for the Java Buck
integration since the summaries are hidden away in buck-out.

Reviewed By: mbouaziz

Differential Revision: D8783668

fbshipit-source-id: 9032d83

infer/src/IR/Filtering.ml

@@ -6,6 +6,10 @@
  *)
 open! IStd
 
+type source_files_filter = SourceFile.t -> bool
+
+type procedures_filter = SourceFile.t -> Typ.Procname.t -> bool
+
 let filter_of_regexp_opt ~to_string r =
   match r with
   | None ->
@@ -18,8 +22,10 @@ let ( &&& ) filter1 filter2 x1 x2 = filter1 x1 && filter2 x2
 
 let mk_source_file_filter ~filter =
   let regexp_opt = Option.map ~f:Str.regexp filter in
-  Staged.stage (filter_of_regexp_opt ~to_string:SourceFile.to_string regexp_opt)
+  filter_of_regexp_opt ~to_string:SourceFile.to_string regexp_opt
+
 
+let source_files_filter = lazy (mk_source_file_filter ~filter:Config.source_files_filter)
 
 let mk_procedure_name_filter ~filter =
   let source_file_regexp, proc_name_regexp =
@@ -39,5 +45,7 @@ let mk_procedure_name_filter ~filter =
     filter_of_regexp_opt ~to_string:SourceFile.to_string source_file_regexp
   in
   let proc_name_filter = filter_of_regexp_opt ~to_string:Typ.Procname.to_string proc_name_regexp in
-  let filter = source_file_filter &&& proc_name_filter in
+  source_file_filter &&& proc_name_filter
-  Staged.stage filter
+
+
+let procedures_filter = lazy (mk_procedure_name_filter ~filter:Config.procedures_filter)

infer/src/IR/Filtering.mli

@@ -7,7 +7,12 @@
 
 open! IStd
 
-val mk_source_file_filter : filter:string option -> (SourceFile.t -> bool) Staged.t
+type source_files_filter = SourceFile.t -> bool
 
-val mk_procedure_name_filter :
+type procedures_filter = SourceFile.t -> Typ.Procname.t -> bool
-  filter:string option -> (SourceFile.t -> Typ.Procname.t -> bool) Staged.t
+
+val source_files_filter : source_files_filter Lazy.t
+(** filter corresponding to `--source-files-filter` *)
+
+val procedures_filter : procedures_filter Lazy.t
+(** filter corresponding to `--procedures-filter` *)

infer/src/IR/SourceFiles.ml

@@ -71,14 +71,16 @@ let add source_file cfg tenv =
       SqliteUtils.result_unit ~finalize:false ~log:"Cfg.store" db store_stmt )
 
 
-let get_all () =
+let get_all ~filter () =
   let db = ResultsDatabase.get_database () in
   (* we could also register this statement but it's typically used only once per run so just prepare
      it inside the function *)
   Sqlite3.prepare db "SELECT source_file FROM source_files"
-  |> IContainer.rev_map_to_list
+  |> IContainer.rev_filter_map_to_list
-       ~fold:(SqliteUtils.result_fold_single_column_rows db ~log:"getting all source files")
+       ~fold:(SqliteUtils.result_fold_single_column_rows db ~log:"getting all source files") ~f:
-       ~f:SourceFile.SQLite.deserialize
+       (fun column ->
+         let source_file = SourceFile.SQLite.deserialize column in
+         Option.some_if (filter source_file) source_file )
 
 
 let load_proc_names_statement =
@@ -151,8 +153,7 @@ let select_all_source_files_statement =
   ResultsDatabase.register_statement "SELECT * FROM source_files"
 
 
-let pp_all ?filter ~cfgs ~type_environment ~procedure_names ~freshly_captured fmt () =
+let pp_all ~filter ~cfgs ~type_environment ~procedure_names ~freshly_captured fmt () =
-  let filter = Staged.unstage (Filtering.mk_source_file_filter ~filter) in
   let pp_procnames fmt procs =
     F.fprintf fmt "@[<v>" ;
     List.iter ~f:(F.fprintf fmt "%a@," Typ.Procname.pp) procs ;

infer/src/IR/SourceFiles.mli

@@ -10,7 +10,7 @@ open! IStd
 val add : SourceFile.t -> Cfg.t -> Tenv.per_file -> unit
 (** Add or replace the row corresponding to the source file into the database. *)
 
-val get_all : unit -> SourceFile.t list
+val get_all : filter:Filtering.source_files_filter -> unit -> SourceFile.t list
 (** get all the source files in the database *)
 
 val proc_names_of_source : SourceFile.t -> Typ.Procname.t list
@@ -29,5 +29,5 @@ val mark_all_stale : unit -> unit
 (** mark all source files as stale; do be called at the start of a new capture phase *)
 
 val pp_all :
-  ?filter:string -> cfgs:bool -> type_environment:bool -> procedure_names:bool
+  filter:Filtering.source_files_filter -> cfgs:bool -> type_environment:bool
-  -> freshly_captured:bool -> Format.formatter -> unit -> unit
+  -> procedure_names:bool -> freshly_captured:bool -> Format.formatter -> unit -> unit

infer/src/backend/InferAnalyze.ml

@@ -73,8 +73,11 @@ let main ~changed_files =
   | None ->
       () ) ;
   register_active_checkers () ;
-  DB.Results_dir.clean_specs_dir () ;
+  if Config.reanalyze then Summary.reset_all ~filter:(Lazy.force Filtering.procedures_filter) ()
-  let all_source_files = SourceFiles.get_all () in
+  else DB.Results_dir.clean_specs_dir () ;
+  let all_source_files =
+    SourceFiles.get_all ~filter:(Lazy.force Filtering.source_files_filter) ()
+  in
   let source_files_to_analyze =
     List.filter ~f:(source_file_should_be_analyzed ~changed_files) all_source_files
   in

infer/src/backend/Procedures.ml

@@ -7,10 +7,19 @@
 open! IStd
 module F = Format
 
-let pp_all ?filter ~proc_name:proc_name_cond ~attr_kind ~source_file:source_file_cond
+let get_all ~filter () =
+  let db = ResultsDatabase.get_database () in
+  let stmt = Sqlite3.prepare db "SELECT source_file, proc_name FROM procedures" in
+  SqliteUtils.result_fold_rows db ~log:"reading all procedure names" stmt ~init:[] ~f:
+    (fun rev_results stmt ->
+      let source_file = Sqlite3.column stmt 0 |> SourceFile.SQLite.deserialize in
+      let proc_name = Sqlite3.column stmt 1 |> Typ.Procname.SQLite.deserialize in
+      if filter source_file proc_name then proc_name :: rev_results else rev_results )
+
+
+let pp_all ~filter ~proc_name:proc_name_cond ~attr_kind ~source_file:source_file_cond
     ~proc_attributes fmt () =
   let db = ResultsDatabase.get_database () in
-  let filter = Filtering.mk_procedure_name_filter ~filter |> Staged.unstage in
   let pp_if ?(new_line= false) condition title pp fmt x =
     if condition then (
       if new_line then F.fprintf fmt "@[<v2>" else F.fprintf fmt "@[<h>" ;

infer/src/backend/Procedures.mli

@@ -7,6 +7,8 @@
 
 open! IStd
 
+val get_all : filter:Filtering.procedures_filter -> unit -> Typ.Procname.t list
+
 val pp_all :
-  ?filter:string -> proc_name:bool -> attr_kind:bool -> source_file:bool -> proc_attributes:bool
+  filter:Filtering.procedures_filter -> proc_name:bool -> attr_kind:bool -> source_file:bool
-  -> Format.formatter -> unit -> unit
+  -> proc_attributes:bool -> Format.formatter -> unit -> unit

infer/src/backend/Summary.ml

@@ -256,4 +256,12 @@ let dummy =
 (** Reset a summary rebuilding the dependents and preserving the proc attributes if present. *)
 let reset proc_desc = init_summary proc_desc
 
-(* =============== END of support for spec tables =============== *)
+let reset_all ~filter () =
+  let reset proc_name =
+    let filename = res_dir_specs_filename proc_name in
+    Serialization.read_from_file summary_serializer filename
+    |> Option.iter ~f:(fun summary ->
+           let blank_summary = reset summary.proc_desc in
+           Serialization.write_to_file summary_serializer filename ~data:blank_summary )
+  in
+  Procedures.get_all ~filter () |> List.iter ~f:reset

infer/src/backend/Summary.mli

@@ -111,3 +111,5 @@ val proc_is_library : ProcAttributes.t -> bool
 
 val store : t -> unit
 (** Save summary for the procedure into the spec database *)
+
+val reset_all : filter:Filtering.procedures_filter -> unit -> unit

infer/src/base/Config.ml

@@ -1801,6 +1801,8 @@ and reactive_capture =
     "Compile source files only when required by analyzer (clang only)"
 
 
+and reanalyze = CLOpt.mk_bool ~long:"reanalyze" "Rerun the analysis"
+
 and relative_path_backtack =
   CLOpt.mk_int ~long:"backtrack-level" ~default:0 ~meta:"int"
     "Maximum level of backtracking to convert an absolute path to path relative to the common \
@@ -2762,6 +2764,8 @@ and reactive_mode = !reactive || InferCommand.(equal Diff) command
 
 and reactive_capture = !reactive_capture
 
+and reanalyze = !reanalyze
+
 and relative_path_backtack = !relative_path_backtack
 
 and report = !report

infer/src/base/Config.mli

@@ -538,6 +538,8 @@ val reactive_mode : bool
 
 val reactive_capture : bool
 
+val reanalyze : bool
+
 val report_current : string option
 
 val report_formatter : [`No_formatter | `Phabricator_formatter]

infer/src/infer.ml

@@ -145,14 +145,16 @@ let () =
     | Explore when Config.procedures ->
         L.result "%a"
           Config.(
-            Procedures.pp_all ?filter:procedures_filter ~proc_name:procedures_name
+            Procedures.pp_all
-              ~attr_kind:procedures_definedness ~source_file:procedures_source_file
+              ~filter:(Lazy.force Filtering.procedures_filter)
-              ~proc_attributes:procedures_attributes)
+              ~proc_name:procedures_name ~attr_kind:procedures_definedness
+              ~source_file:procedures_source_file ~proc_attributes:procedures_attributes)
           ()
     | Explore when Config.source_files ->
         L.result "%a"
-          (SourceFiles.pp_all ?filter:Config.source_files_filter ~cfgs:Config.source_files_cfgs
+          (SourceFiles.pp_all
-             ~type_environment:Config.source_files_type_environment
+             ~filter:(Lazy.force Filtering.source_files_filter)
+             ~cfgs:Config.source_files_cfgs ~type_environment:Config.source_files_type_environment
              ~procedure_names:Config.source_files_procedure_names
              ~freshly_captured:Config.source_files_freshly_captured)
           ()