From 70c2a50b4e2b4a293e7a0185aa62a33bd7bb0888 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Tue, 6 Feb 2018 08:31:53 -0800 Subject: [PATCH] [db] record symbolic capture timestamps for source files Summary: Record "capture phases" in the runstate and in the source files table of the database. Use this instead of filesystem timestamps to decide which files need re-analyzing in the reactive analysis. Reviewed By: jeremydubreil Differential Revision: D6760833 fbshipit-source-id: 7955621 --- .gitignore | 1 - infer/src/IR/Cfg.ml | 5 +- infer/src/backend/InferAnalyze.ml | 5 +- infer/src/backend/infer.ml | 5 +- infer/src/backend/mergeCapture.ml | 2 +- infer/src/base/Config.ml | 2 - infer/src/base/Config.mli | 2 - infer/src/base/DB.ml | 21 ------- infer/src/base/DB.mli | 7 --- infer/src/base/MergeResults.ml | 7 ++- infer/src/base/ResultsDatabase.ml | 3 +- infer/src/base/SourceFiles.ml | 39 ++++++++++++- infer/src/base/SourceFiles.mli | 9 +++ infer/src/clang/cFrontend.ml | 12 +--- infer/src/integration/Driver.ml | 58 +++++++------------ infer/src/java/jMain.ml | 4 +- .../build_systems/buck_flavors_run/Makefile | 2 +- 17 files changed, 88 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index 4409230d9..72b746c12 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ *.log *.orig *.rej -*.start # generated by build and tests /_build diff --git a/infer/src/IR/Cfg.ml b/infer/src/IR/Cfg.ml index 2e92e57fe..c613aa0df 100644 --- a/infer/src/IR/Cfg.ml +++ b/infer/src/IR/Cfg.ml @@ -270,7 +270,7 @@ let mark_unchanged_pdescs cfg_new cfg_old = let store_statement = ResultsDatabase.register_statement - "INSERT OR REPLACE INTO source_files VALUES (:source, :cfgs, :proc_names)" + "INSERT OR REPLACE INTO source_files VALUES (:source, :cfgs, :proc_names, :timestamp)" let store source_file cfg = @@ -291,6 +291,9 @@ let store source_file cfg = get_all_proc_names cfg |> Typ.Procname.SQLiteList.serialize |> Sqlite3.bind store_stmt 3 (* :proc_names *) |> SqliteUtils.check_sqlite_error db ~log:"store bind proc names" ; + Sqlite3.bind store_stmt 4 (Sqlite3.Data.INT Int64.one) + (* :freshly_captured *) + |> SqliteUtils.check_sqlite_error db ~log:"store freshness" ; SqliteUtils.sqlite_unit_step ~finalize:false ~log:"Cfg.store" db store_stmt ) diff --git a/infer/src/backend/InferAnalyze.ml b/infer/src/backend/InferAnalyze.ml index a9e45cfbd..2a2bf2dcb 100644 --- a/infer/src/backend/InferAnalyze.ml +++ b/infer/src/backend/InferAnalyze.ml @@ -78,10 +78,7 @@ let cluster_should_be_analyzed ~changed_files cluster = (* whether [fname] is one of the [changed_files] *) let is_changed_file = Option.map changed_files ~f:(SourceFile.Set.mem cluster) in let check_modified () = - let modified = - DB.source_dir_from_source_file cluster |> DB.source_dir_to_string |> DB.filename_from_string - |> DB.file_was_updated_after_start - in + let modified = SourceFiles.is_freshly_captured cluster in if modified then L.debug Analysis Medium "Modified: %a@\n" SourceFile.pp cluster ; modified in diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index 10e13e5ea..a48d685d4 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -44,7 +44,10 @@ let setup () = ( Driver.(equal_mode driver_mode Analyze) || Config.(continue_capture || infer_is_clang || infer_is_javac || reactive_mode) ) then ResultsDir.remove_results_dir () ; - ResultsDir.create_results_dir () + ResultsDir.create_results_dir () ; + if CLOpt.is_originator && not Config.continue_capture + && not Driver.(equal_mode driver_mode Analyze) + then SourceFiles.mark_all_stale () | Explore -> ResultsDir.assert_results_dir "please run an infer analysis first" | Events -> diff --git a/infer/src/backend/mergeCapture.ml b/infer/src/backend/mergeCapture.ml index 6dd76ff00..4d0e04365 100644 --- a/infer/src/backend/mergeCapture.ml +++ b/infer/src/backend/mergeCapture.ml @@ -134,7 +134,7 @@ let process_merge_file deps_file = else target_results_dir in let skiplevels = 2 in - (* Don't link toplevel files, definitely not .start *) + (* Don't link toplevel files *) if should_link ~target ~target_results_dir ~stats infer_out_src infer_out_dst then slink ~stats ~skiplevels infer_out_src infer_out_dst | _ -> diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index 383285d83..813fedfc9 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -287,8 +287,6 @@ let specs_dir_name = "specs" let specs_files_suffix = ".specs" -let start_filename = ".start" - (** Enable detailed tracing information during array abstraction *) let trace_absarray = false diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index c56d91c8a..2984ccfdc 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -209,8 +209,6 @@ val specs_dir_name : string val specs_files_suffix : string -val start_filename : string - val trace_absarray : bool val undo_join : bool diff --git a/infer/src/base/DB.ml b/infer/src/base/DB.ml index b5c78d8ca..577909fa8 100644 --- a/infer/src/base/DB.ml +++ b/infer/src/base/DB.ml @@ -229,27 +229,6 @@ let is_source_file path = List.exists ~f:(fun ext -> Filename.check_suffix path ext) Config.source_file_extentions -let infer_start_time = - lazy - (file_modified_time (Results_dir.path_to_filename Results_dir.Abs_root [Config.start_filename])) - - -(** Return whether filename was updated after analysis started. File doesn't have to exist *) -let file_was_updated_after_start fname = - if file_exists fname then - let file_mtime = file_modified_time fname in - file_mtime > Lazy.force infer_start_time - else (* since file doesn't exist, it wasn't modified *) - false - - -(** Mark a file as updated by changing its timestamps to be one second in the future. - This guarantees that it appears updated after start. *) -let mark_file_updated fname = - let near_future = Unix.gettimeofday () +. 1. in - Unix.utimes fname ~access:near_future ~modif:near_future - - (** Fold over all file paths recursively under [dir] which match [p]. *) let fold_paths_matching ~dir ~p ~init ~f = let rec paths path_list dir = diff --git a/infer/src/base/DB.mli b/infer/src/base/DB.mli index 86e149159..c01c6cf84 100644 --- a/infer/src/base/DB.mli +++ b/infer/src/base/DB.mli @@ -32,13 +32,6 @@ val file_exists : filename -> bool val file_modified_time : ?symlink:bool -> filename -> float (** Return the time when a file was last modified. The file must exist. *) -val mark_file_updated : string -> unit -(** Mark a file as updated by changing its timestamps to be one second in the future. - This guarantees that it appears updated after start. *) - -val file_was_updated_after_start : filename -> bool -(** Return whether filename was updated after analysis started. File doesn't have to exist *) - (** {2 Results Directory} *) module Results_dir : sig diff --git a/infer/src/base/MergeResults.ml b/infer/src/base/MergeResults.ml index dab9304f8..5b027ee76 100644 --- a/infer/src/base/MergeResults.ml +++ b/infer/src/base/MergeResults.ml @@ -34,7 +34,12 @@ WHERE let merge_source_files_table ~db_file = let db = ResultsDatabase.get_database () in - Sqlite3.exec db "INSERT OR REPLACE INTO source_files SELECT * FROM attached.source_files" + Sqlite3.exec db + {| + INSERT OR REPLACE INTO source_files + SELECT source_file, cfgs, procedure_names, 1 + FROM attached.source_files +|} |> SqliteUtils.check_sqlite_error db ~log:(Printf.sprintf "copying source_files of database '%s'" db_file) diff --git a/infer/src/base/ResultsDatabase.ml b/infer/src/base/ResultsDatabase.ml index f816d0912..447f28dc8 100644 --- a/infer/src/base/ResultsDatabase.ml +++ b/infer/src/base/ResultsDatabase.ml @@ -29,7 +29,8 @@ let source_files_schema = {|CREATE TABLE IF NOT EXISTS source_files ( source_file TEXT PRIMARY KEY , cfgs BLOB NOT NULL - , procedure_names BLOB NOT NULL )|} + , procedure_names BLOB NOT NULL + , freshly_captured INT NOT NULL )|} let schema_hum = Printf.sprintf "%s;\n%s" procedures_schema source_files_schema diff --git a/infer/src/base/SourceFiles.ml b/infer/src/base/SourceFiles.ml index 5ee239322..cd07027d3 100644 --- a/infer/src/base/SourceFiles.ml +++ b/infer/src/base/SourceFiles.ml @@ -7,6 +7,7 @@ * of patent rights can be found in the PATENTS file in the same directory. *) open! IStd +module L = Logging let get_all () = let db = ResultsDatabase.get_database () in @@ -29,14 +30,48 @@ let proc_names_of_source source = |> Option.value_map ~default:[] ~f:Typ.Procname.SQLiteList.deserialize ) -let exists_statement = +let exists_source_statement = ResultsDatabase.register_statement "SELECT 1 FROM source_files WHERE source_file = :k" let is_captured source = - ResultsDatabase.with_registered_statement exists_statement ~f:(fun db exists_stmt -> + ResultsDatabase.with_registered_statement exists_source_statement ~f:(fun db exists_stmt -> SourceFile.SQLite.serialize source |> Sqlite3.bind exists_stmt 1 (* :k *) |> SqliteUtils.check_sqlite_error db ~log:"load captured source file" ; SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_captured" db exists_stmt |> Option.is_some ) + + +let is_non_empty_statement = + ResultsDatabase.register_statement "SELECT 1 FROM source_files LIMIT 1" + + +let is_empty () = + ResultsDatabase.with_registered_statement is_non_empty_statement ~f:(fun db stmt -> + SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_empty" db stmt + |> Option.is_none ) + + +let is_freshly_captured_statement = + ResultsDatabase.register_statement + "SELECT freshly_captured FROM source_files WHERE source_file = :k" + + +let is_freshly_captured source = + ResultsDatabase.with_registered_statement is_freshly_captured_statement ~f:(fun db load_stmt -> + SourceFile.SQLite.serialize source |> Sqlite3.bind load_stmt 1 + |> SqliteUtils.check_sqlite_error db ~log:"load bind source file" ; + SqliteUtils.sqlite_result_step ~finalize:false ~log:"SourceFiles.is_freshly_captured" db + load_stmt + |> Option.value_map ~default:false ~f:(function [@warning "-8"] Sqlite3.Data.INT p -> + Int64.equal p Int64.one ) ) + + +let mark_all_stale_statement = + ResultsDatabase.register_statement "UPDATE source_files SET freshly_captured = 0" + + +let mark_all_stale () = + ResultsDatabase.with_registered_statement mark_all_stale_statement ~f:(fun db stmt -> + SqliteUtils.sqlite_unit_step db ~finalize:false ~log:"mark_all_stale" stmt ) diff --git a/infer/src/base/SourceFiles.mli b/infer/src/base/SourceFiles.mli index 331ed7d59..be98f3df6 100644 --- a/infer/src/base/SourceFiles.mli +++ b/infer/src/base/SourceFiles.mli @@ -15,3 +15,12 @@ val proc_names_of_source : SourceFile.t -> Typ.Procname.t list val is_captured : SourceFile.t -> bool (** has the source file been captured? *) + +val is_empty : unit -> bool +(** whether there exists at least one captured source file *) + +val is_freshly_captured : SourceFile.t -> bool +(** whether the source file was captured in the last capture phase *) + +val mark_all_stale : unit -> unit +(** mark all source files as stale; do be called at the start of a new capture phase *) diff --git a/infer/src/clang/cFrontend.ml b/infer/src/clang/cFrontend.ml index 1ecee9f18..de16e79e7 100644 --- a/infer/src/clang/cFrontend.ml +++ b/infer/src/clang/cFrontend.ml @@ -50,7 +50,6 @@ let do_source_file (translation_unit_context: CFrontend_config.translation_unit_ "@\n End building call/cfg graph for '%a'.@\n" SourceFile.pp source_file ; (* This part below is a boilerplate in every frontends. *) (* This could be moved in the cfg_infer module *) - let source_dir = DB.source_dir_from_source_file source_file in NullabilityPreanalysis.analysis cfg tenv ; Cfg.store source_file cfg ; Tenv.sort_fields_tenv tenv ; @@ -60,13 +59,4 @@ let do_source_file (translation_unit_context: CFrontend_config.translation_unit_ || Option.is_some Config.icfg_dotty_outfile then Dotty.print_icfg_dotty source_file cfg ; L.(debug Capture Verbose) "%a" Cfg.pp_proc_signatures cfg ; - let procedures_translated_summary = - EventLogger.ProceduresTranslatedSummary - { procedures_translated_total= !CFrontend_config.procedures_attempted - ; procedures_translated_failed= !CFrontend_config.procedures_failed - ; lang= CFrontend_config.string_of_clang_lang translation_unit_context.lang - ; source_file= translation_unit_context.source_file } - in - EventLogger.log procedures_translated_summary ; - (* NOTE: nothing should be written to source_dir after this *) - DB.mark_file_updated (DB.source_dir_to_string source_dir) + () diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index ee80081e8..e21097ec3 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -126,24 +126,6 @@ let clean_results_dir () = delete_temp_results Config.results_dir -let check_captured_empty mode = - let clean_command_opt = clean_compilation_command mode in - if Config.capture && Utils.directory_is_empty Config.captured_dir then - let nothing_to_compile_msg = "Nothing to compile." in - let please_run_capture_msg = - match mode with Analyze -> " Have you run `infer capture`?" | _ -> "" - in - ( match clean_command_opt with - | Some clean_command -> - L.user_warning "%s%s Try running `%s` first.@." nothing_to_compile_msg - please_run_capture_msg clean_command - | None -> - L.user_warning "%s%s Try cleaning the build first.@." nothing_to_compile_msg - please_run_capture_msg ) ; - true - else false - - let register_perf_stats_report () = let stats_dir = Filename.concat Config.results_dir Config.backend_stats_dir_name in let stats_base = Config.perf_stats_prefix ^ ".json" in @@ -161,17 +143,6 @@ let reset_duplicates_file () = create () -(* Create the .start file, and update the timestamp unless in continue mode *) -let touch_start_file_unless_continue () = - let start = Config.results_dir ^/ Config.start_filename in - let delete () = Unix.unlink start in - let create () = - Unix.close (Unix.openfile ~perm:0o0666 ~mode:[Unix.O_CREAT; Unix.O_WRONLY] start) - in - if not (Sys.file_exists start = `Yes) then create () - else if not Config.continue_capture then ( delete () ; create () ) - - let command_error_handling ~always_die ~prog ~args = function | Ok _ -> () @@ -382,6 +353,22 @@ let report ?(suppress_console= false) () = (String.concat ~sep:" " args) +let error_nothing_to_analyze mode = + let clean_command_opt = clean_compilation_command mode in + let nothing_to_compile_msg = "Nothing to compile." in + let please_run_capture_msg = + match mode with Analyze -> " Have you run `infer capture`?" | _ -> "" + in + ( match clean_command_opt with + | Some clean_command -> + L.user_warning "%s%s Try running `%s` first.@." nothing_to_compile_msg please_run_capture_msg + clean_command + | None -> + L.user_warning "%s%s Try cleaning the build first.@." nothing_to_compile_msg + please_run_capture_msg ) ; + L.user_error "There was nothing to analyze.@\n@." + + let analyze_and_report ?suppress_console_report ~changed_files mode = let should_analyze, should_report = match (Config.command, mode, Config.analyzer) with @@ -400,18 +387,17 @@ let analyze_and_report ?suppress_console_report ~changed_files mode = in let should_merge = match mode with - | PythonCapture (BBuck, _) when Config.flavors && InferCommand.(equal Run) Config.command -> + | PythonCapture (BBuck, _) when Config.flavors && InferCommand.equal Run Config.command -> (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *) true | _ -> (* else rely on the command line value *) Config.merge in if should_merge then MergeCapture.merge_captured_targets () ; - if (should_analyze || should_report) - && (Sys.file_exists Config.captured_dir <> `Yes || check_captured_empty mode) - then L.user_error "There was nothing to analyze.@\n@." - else if should_analyze then execute_analyze ~changed_files ; - if should_report && Config.report then report ?suppress_console:suppress_console_report () + if should_analyze || should_report then ( + if SourceFiles.is_empty () then error_nothing_to_analyze mode + else if should_analyze then execute_analyze ~changed_files ; + if should_report && Config.report then report ?suppress_console:suppress_console_report () ) (** as the Config.fail_on_bug flag mandates, exit with error when an issue is reported *) @@ -553,8 +539,6 @@ let run_prologue mode = mono-threaded execution. *) Unix.unsetenv "MAKEFLAGS" ; if Config.developer_mode then register_perf_stats_report () ; - if not Config.buck_cache_mode && not Config.infer_is_clang && not Config.infer_is_javac then - touch_start_file_unless_continue () ; () diff --git a/infer/src/java/jMain.ml b/infer/src/java/jMain.ml index 612c25b99..abd5e4aac 100644 --- a/infer/src/java/jMain.ml +++ b/infer/src/java/jMain.ml @@ -29,11 +29,9 @@ let init_global_state source_file = let store_icfg source_file cfg = - let source_dir = DB.source_dir_from_source_file source_file in Cfg.store source_file cfg ; if Config.debug_mode || Config.frontend_tests then Dotty.print_icfg_dotty source_file cfg ; - (* NOTE: nothing should be written to source_dir after this *) - DB.mark_file_updated (DB.source_dir_to_string source_dir) + () (* Given a source file, its code is translated, and the call-graph, control-flow-graph and type *) diff --git a/infer/tests/build_systems/buck_flavors_run/Makefile b/infer/tests/build_systems/buck_flavors_run/Makefile index 8ed53f89b..164bcb532 100644 --- a/infer/tests/build_systems/buck_flavors_run/Makefile +++ b/infer/tests/build_systems/buck_flavors_run/Makefile @@ -26,5 +26,5 @@ infer-out/report.json: $(CLANG_DEPS) $(SOURCES) $(MAKEFILE_LIST) $(QUIET)$(REMOVE_DIR) buck-out && \ $(call silent_on_success,Testing infer-run Buck flavors integration,\ NO_BUCKD=1 \ - $(INFER_BIN) $(INFER_OPTIONS) run --flavors --results-dir $(CURDIR)/infer-out -- \ + $(INFER_BIN) $(INFER_OPTIONS) run --flavors --results-dir $(CURDIR)/infer-out --reactive -- \ $(BUCK) build --no-cache $(BUCK_TARGET))