[db] record symbolic capture timestamps for source files

Summary:
Record "capture phases" in the runstate and in the source files table of the
database. Use this instead of filesystem timestamps to decide which files need
re-analyzing in the reactive analysis.

Reviewed By: jeremydubreil

Differential Revision: D6760833

fbshipit-source-id: 7955621

.gitignore

@@ -11,7 +11,6 @@
 *.log
 *.orig
 *.rej
-*.start
 
 # generated by build and tests
 /_build

infer/src/IR/Cfg.ml

@@ -270,7 +270,7 @@ let mark_unchanged_pdescs cfg_new cfg_old =
 
 let store_statement =
   ResultsDatabase.register_statement
-    "INSERT OR REPLACE INTO source_files VALUES (:source, :cfgs, :proc_names)"
+    "INSERT OR REPLACE INTO source_files VALUES (:source, :cfgs, :proc_names, :timestamp)"
 
 
 let store source_file cfg =
@@ -291,6 +291,9 @@ let store source_file cfg =
       get_all_proc_names cfg |> Typ.Procname.SQLiteList.serialize |> Sqlite3.bind store_stmt 3
       (* :proc_names *)
       |> SqliteUtils.check_sqlite_error db ~log:"store bind proc names" ;
+      Sqlite3.bind store_stmt 4 (Sqlite3.Data.INT Int64.one)
+      (* :freshly_captured *)
+      |> SqliteUtils.check_sqlite_error db ~log:"store freshness" ;
       SqliteUtils.sqlite_unit_step ~finalize:false ~log:"Cfg.store" db store_stmt )
 
 

infer/src/backend/InferAnalyze.ml

@@ -78,10 +78,7 @@ let cluster_should_be_analyzed ~changed_files cluster =
   (* whether [fname] is one of the [changed_files] *)
   let is_changed_file = Option.map changed_files ~f:(SourceFile.Set.mem cluster) in
   let check_modified () =
-    let modified =
-      DB.source_dir_from_source_file cluster |> DB.source_dir_to_string |> DB.filename_from_string
-      |> DB.file_was_updated_after_start
-    in
+    let modified = SourceFiles.is_freshly_captured cluster in
     if modified then L.debug Analysis Medium "Modified: %a@\n" SourceFile.pp cluster ;
     modified
   in

infer/src/backend/infer.ml

@@ -44,7 +44,10 @@ let setup () =
               ( Driver.(equal_mode driver_mode Analyze)
               || Config.(continue_capture || infer_is_clang || infer_is_javac || reactive_mode) )
       then ResultsDir.remove_results_dir () ;
-      ResultsDir.create_results_dir ()
+      ResultsDir.create_results_dir () ;
+      if CLOpt.is_originator && not Config.continue_capture
+         && not Driver.(equal_mode driver_mode Analyze)
+      then SourceFiles.mark_all_stale ()
   | Explore ->
       ResultsDir.assert_results_dir "please run an infer analysis first"
   | Events ->

infer/src/backend/mergeCapture.ml

@@ -134,7 +134,7 @@ let process_merge_file deps_file =
           else target_results_dir
         in
         let skiplevels = 2 in
-        (* Don't link toplevel files, definitely not .start *)
+        (* Don't link toplevel files *)
         if should_link ~target ~target_results_dir ~stats infer_out_src infer_out_dst then
           slink ~stats ~skiplevels infer_out_src infer_out_dst
     | _ ->

infer/src/base/Config.ml

@@ -287,8 +287,6 @@ let specs_dir_name = "specs"
 
 let specs_files_suffix = ".specs"
 
-let start_filename = ".start"
-
 (** Enable detailed tracing information during array abstraction *)
 let trace_absarray = false
 

infer/src/base/Config.mli

@@ -209,8 +209,6 @@ val specs_dir_name : string
 
 val specs_files_suffix : string
 
-val start_filename : string
-
 val trace_absarray : bool
 
 val undo_join : bool

infer/src/base/DB.ml

@@ -229,27 +229,6 @@ let is_source_file path =
   List.exists ~f:(fun ext -> Filename.check_suffix path ext) Config.source_file_extentions
 
 
-let infer_start_time =
-  lazy
-    (file_modified_time (Results_dir.path_to_filename Results_dir.Abs_root [Config.start_filename]))
-
-
-(** Return whether filename was updated after analysis started. File doesn't have to exist *)
-let file_was_updated_after_start fname =
-  if file_exists fname then
-    let file_mtime = file_modified_time fname in
-    file_mtime > Lazy.force infer_start_time
-  else (* since file doesn't exist, it wasn't modified *)
-    false
-
-
-(** Mark a file as updated by changing its timestamps to be one second in the future.
-    This guarantees that it appears updated after start. *)
-let mark_file_updated fname =
-  let near_future = Unix.gettimeofday () +. 1. in
-  Unix.utimes fname ~access:near_future ~modif:near_future
-
-
 (** Fold over all file paths recursively under [dir] which match [p]. *)
 let fold_paths_matching ~dir ~p ~init ~f =
   let rec paths path_list dir =

infer/src/base/DB.mli

@@ -32,13 +32,6 @@ val file_exists : filename -> bool
 val file_modified_time : ?symlink:bool -> filename -> float
 (** Return the time when a file was last modified. The file must exist. *)
 
-val mark_file_updated : string -> unit
-(** Mark a file as updated by changing its timestamps to be one second in the future.
-    This guarantees that it appears updated after start. *)
-
-val file_was_updated_after_start : filename -> bool
-(** Return whether filename was updated after analysis started. File doesn't have to exist *)
-
 (** {2 Results Directory} *)
 
 module Results_dir : sig

infer/src/base/MergeResults.ml

@@ -34,7 +34,12 @@ WHERE
 
 let merge_source_files_table ~db_file =
   let db = ResultsDatabase.get_database () in
-  Sqlite3.exec db "INSERT OR REPLACE INTO source_files SELECT * FROM attached.source_files"
+  Sqlite3.exec db
+    {|
+    INSERT OR REPLACE INTO source_files
+    SELECT source_file, cfgs, procedure_names, 1
+    FROM attached.source_files
+|}
   |> SqliteUtils.check_sqlite_error db
        ~log:(Printf.sprintf "copying source_files of database '%s'" db_file)
 

infer/src/base/ResultsDatabase.ml

@@ -29,7 +29,8 @@ let source_files_schema =
   {|CREATE TABLE IF NOT EXISTS source_files
   ( source_file TEXT PRIMARY KEY
   , cfgs BLOB NOT NULL
-  , procedure_names BLOB NOT NULL )|}
+  , procedure_names BLOB NOT NULL
+  , freshly_captured INT NOT NULL )|}
 
 
 let schema_hum = Printf.sprintf "%s;\n%s" procedures_schema source_files_schema

infer/src/base/SourceFiles.ml

@@ -7,6 +7,7 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  *)
 open! IStd
+module L = Logging
 
 let get_all () =
   let db = ResultsDatabase.get_database () in
@@ -29,14 +30,48 @@ let proc_names_of_source source =
       |> Option.value_map ~default:[] ~f:Typ.Procname.SQLiteList.deserialize )
 
 
-let exists_statement =
+let exists_source_statement =
   ResultsDatabase.register_statement "SELECT 1 FROM source_files WHERE source_file = :k"
 
 
 let is_captured source =
-  ResultsDatabase.with_registered_statement exists_statement ~f:(fun db exists_stmt ->
+  ResultsDatabase.with_registered_statement exists_source_statement ~f:(fun db exists_stmt ->
       SourceFile.SQLite.serialize source |> Sqlite3.bind exists_stmt 1
       (* :k *)
       |> SqliteUtils.check_sqlite_error db ~log:"load captured source file" ;
       SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_captured" db exists_stmt
       |> Option.is_some )
+
+
+let is_non_empty_statement =
+  ResultsDatabase.register_statement "SELECT 1 FROM source_files LIMIT 1"
+
+
+let is_empty () =
+  ResultsDatabase.with_registered_statement is_non_empty_statement ~f:(fun db stmt ->
+      SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_empty" db stmt
+      |> Option.is_none )
+
+
+let is_freshly_captured_statement =
+  ResultsDatabase.register_statement
+    "SELECT freshly_captured FROM source_files WHERE source_file = :k"
+
+
+let is_freshly_captured source =
+  ResultsDatabase.with_registered_statement is_freshly_captured_statement ~f:(fun db load_stmt ->
+      SourceFile.SQLite.serialize source |> Sqlite3.bind load_stmt 1
+      |> SqliteUtils.check_sqlite_error db ~log:"load bind source file" ;
+      SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_freshly_captured" db
+        load_stmt
+      |> Option.value_map ~default:false ~f:(function [@warning "-8"] Sqlite3.Data.INT p ->
+             Int64.equal p Int64.one ) )
+
+
+let mark_all_stale_statement =
+  ResultsDatabase.register_statement "UPDATE source_files SET freshly_captured = 0"
+
+
+let mark_all_stale () =
+  ResultsDatabase.with_registered_statement mark_all_stale_statement ~f:(fun db stmt ->
+      SqliteUtils.sqlite_unit_step db ~finalize:false ~log:"mark_all_stale" stmt )

infer/src/base/SourceFiles.mli

@@ -15,3 +15,12 @@ val proc_names_of_source : SourceFile.t -> Typ.Procname.t list
 
 val is_captured : SourceFile.t -> bool
 (** has the source file been captured? *)
+
+val is_empty : unit -> bool
+(** whether there exists at least one captured source file *)
+
+val is_freshly_captured : SourceFile.t -> bool
+(** whether the source file was captured in the last capture phase *)
+
+val mark_all_stale : unit -> unit
+(** mark all source files as stale; do be called at the start of a new capture phase *)

infer/src/clang/cFrontend.ml

@@ -50,7 +50,6 @@ let do_source_file (translation_unit_context: CFrontend_config.translation_unit_
     "@\n End building call/cfg graph for '%a'.@\n" SourceFile.pp source_file ;
   (* This part below is a boilerplate in every frontends. *)
   (* This could be moved in the cfg_infer module *)
-  let source_dir = DB.source_dir_from_source_file source_file in
   NullabilityPreanalysis.analysis cfg tenv ;
   Cfg.store source_file cfg ;
   Tenv.sort_fields_tenv tenv ;
@@ -60,13 +59,4 @@ let do_source_file (translation_unit_context: CFrontend_config.translation_unit_
      || Option.is_some Config.icfg_dotty_outfile
   then Dotty.print_icfg_dotty source_file cfg ;
   L.(debug Capture Verbose) "%a" Cfg.pp_proc_signatures cfg ;
-  let procedures_translated_summary =
-    EventLogger.ProceduresTranslatedSummary
-      { procedures_translated_total= !CFrontend_config.procedures_attempted
-      ; procedures_translated_failed= !CFrontend_config.procedures_failed
-      ; lang= CFrontend_config.string_of_clang_lang translation_unit_context.lang
-      ; source_file= translation_unit_context.source_file }
-  in
-  EventLogger.log procedures_translated_summary ;
-  (* NOTE: nothing should be written to source_dir after this *)
-  DB.mark_file_updated (DB.source_dir_to_string source_dir)
+  ()

infer/src/integration/Driver.ml

@@ -126,24 +126,6 @@ let clean_results_dir () =
   delete_temp_results Config.results_dir
 
 
-let check_captured_empty mode =
-  let clean_command_opt = clean_compilation_command mode in
-  if Config.capture && Utils.directory_is_empty Config.captured_dir then
-    let nothing_to_compile_msg = "Nothing to compile." in
-    let please_run_capture_msg =
-      match mode with Analyze -> " Have you run `infer capture`?" | _ -> ""
-    in
-    ( match clean_command_opt with
-    | Some clean_command ->
-        L.user_warning "%s%s Try running `%s` first.@." nothing_to_compile_msg
-          please_run_capture_msg clean_command
-    | None ->
-        L.user_warning "%s%s Try cleaning the build first.@." nothing_to_compile_msg
-          please_run_capture_msg ) ;
-    true
-  else false
-
-
 let register_perf_stats_report () =
   let stats_dir = Filename.concat Config.results_dir Config.backend_stats_dir_name in
   let stats_base = Config.perf_stats_prefix ^ ".json" in
@@ -161,17 +143,6 @@ let reset_duplicates_file () =
   create ()
 
 
-(* Create the .start file, and update the timestamp unless in continue mode *)
-let touch_start_file_unless_continue () =
-  let start = Config.results_dir ^/ Config.start_filename in
-  let delete () = Unix.unlink start in
-  let create () =
-    Unix.close (Unix.openfile ~perm:0o0666 ~mode:[Unix.O_CREAT; Unix.O_WRONLY] start)
-  in
-  if not (Sys.file_exists start = `Yes) then create ()
-  else if not Config.continue_capture then ( delete () ; create () )
-
-
 let command_error_handling ~always_die ~prog ~args = function
   | Ok _ ->
       ()
@@ -382,6 +353,22 @@ let report ?(suppress_console= false) () =
           (String.concat ~sep:" " args)
 
 
+let error_nothing_to_analyze mode =
+  let clean_command_opt = clean_compilation_command mode in
+  let nothing_to_compile_msg = "Nothing to compile." in
+  let please_run_capture_msg =
+    match mode with Analyze -> " Have you run `infer capture`?" | _ -> ""
+  in
+  ( match clean_command_opt with
+  | Some clean_command ->
+      L.user_warning "%s%s Try running `%s` first.@." nothing_to_compile_msg please_run_capture_msg
+        clean_command
+  | None ->
+      L.user_warning "%s%s Try cleaning the build first.@." nothing_to_compile_msg
+        please_run_capture_msg ) ;
+  L.user_error "There was nothing to analyze.@\n@."
+
+
 let analyze_and_report ?suppress_console_report ~changed_files mode =
   let should_analyze, should_report =
     match (Config.command, mode, Config.analyzer) with
@@ -400,18 +387,17 @@ let analyze_and_report ?suppress_console_report ~changed_files mode =
   in
   let should_merge =
     match mode with
-    | PythonCapture (BBuck, _) when Config.flavors && InferCommand.(equal Run) Config.command ->
+    | PythonCapture (BBuck, _) when Config.flavors && InferCommand.equal Run Config.command ->
         (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *)
         true
     | _ ->
         (* else rely on the command line value *) Config.merge
   in
   if should_merge then MergeCapture.merge_captured_targets () ;
-  if (should_analyze || should_report)
-     && (Sys.file_exists Config.captured_dir <> `Yes || check_captured_empty mode)
-  then L.user_error "There was nothing to analyze.@\n@."
-  else if should_analyze then execute_analyze ~changed_files ;
-  if should_report && Config.report then report ?suppress_console:suppress_console_report ()
+  if should_analyze || should_report then (
+    if SourceFiles.is_empty () then error_nothing_to_analyze mode
+    else if should_analyze then execute_analyze ~changed_files ;
+    if should_report && Config.report then report ?suppress_console:suppress_console_report () )
 
 
 (** as the Config.fail_on_bug flag mandates, exit with error when an issue is reported *)
@@ -553,8 +539,6 @@ let run_prologue mode =
      mono-threaded execution. *)
   Unix.unsetenv "MAKEFLAGS" ;
   if Config.developer_mode then register_perf_stats_report () ;
-  if not Config.buck_cache_mode && not Config.infer_is_clang && not Config.infer_is_javac then
-    touch_start_file_unless_continue () ;
   ()
 
 

infer/src/java/jMain.ml

@@ -29,11 +29,9 @@ let init_global_state source_file =
 
 
 let store_icfg source_file cfg =
-  let source_dir = DB.source_dir_from_source_file source_file in
   Cfg.store source_file cfg ;
   if Config.debug_mode || Config.frontend_tests then Dotty.print_icfg_dotty source_file cfg ;
-  (* NOTE: nothing should be written to source_dir after this *)
-  DB.mark_file_updated (DB.source_dir_to_string source_dir)
+  ()
 
 
 (* Given a source file, its code is translated, and the call-graph, control-flow-graph and type *)

infer/tests/build_systems/buck_flavors_run/Makefile

@@ -26,5 +26,5 @@ infer-out/report.json: $(CLANG_DEPS) $(SOURCES) $(MAKEFILE_LIST)
 	$(QUIET)$(REMOVE_DIR) buck-out && \
 	$(call silent_on_success,Testing infer-run Buck flavors integration,\
 	NO_BUCKD=1 \
-	$(INFER_BIN) $(INFER_OPTIONS) run --flavors --results-dir $(CURDIR)/infer-out -- \
+	$(INFER_BIN) $(INFER_OPTIONS) run --flavors --results-dir $(CURDIR)/infer-out --reactive -- \
 	  $(BUCK) build --no-cache $(BUCK_TARGET))