[driver] persist some global state across infer runs

Summary:
Record the db schema, infer version, and run dates into
infer-out/.infer_runstate.json. This allows us to check on startup whether the
results directory was generated using a compatible version of infer or not, and
give a better error message in the latter case than some SQLite error about
mismatching tables.

This will be used in a follow-up diff to record capture phases too, and avoid
relying on filesystem timestamps of the infer-out/capture/foo/ directories for
reactive analysis.

Had to change some tests Makefiles to make sure they do not attempt to re-use
stale infer-out directories, which would now fail the run.

The stale infer-out directory gets deleted if `--force-delete-results-dir` is
passed (but a warning still gets printed).

Reviewed By: mbouaziz

Differential Revision: D6760787

fbshipit-source-id: f36f7df
master
Jules Villard 7 years ago committed by Facebook Github Bot
parent 0c9b025857
commit 66ad5c3018

1
.gitignore vendored

@ -43,6 +43,7 @@ duplicates.txt
/infer/tests/build_systems/diff/src /infer/tests/build_systems/diff/src
/infer/tests/build_systems/diff_*/src /infer/tests/build_systems/diff_*/src
/infer/tests/build_systems/buck_flavors_deterministic/capture_hash-*.sha /infer/tests/build_systems/buck_flavors_deterministic/capture_hash-*.sha
/infer/tests/build_systems/genrule/report.json
/_release /_release
# generated by oUnit # generated by oUnit

@ -21,7 +21,7 @@ INFER_MAIN = infer
#### Checkers declarations #### #### Checkers declarations ####
INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/stacktree INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/stacktree
INFER_ATDGEN_TYPES = j t INFER_ATDGEN_TYPES = j t
INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd) INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd)
INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli) INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli)

@ -0,0 +1,23 @@
type infer_version = {
major: int;
minor: int;
patch: int;
commit: string;
}
type command = string wrap <ocaml
t="InferCommand.t"
wrap="InferCommand.of_string"
unwrap="InferCommand.to_string"
>
type run_info = {
date: string;
command: command;
infer_version: infer_version;
}
type t = {
run_sequence: run_info list; (** successive runs that re-used the same results directory *)
results_dir_format: string; (** to check if the versions of the results dir are compatible *)
}

@ -26,7 +26,7 @@ let run driver_mode =
let setup () = let setup () =
match Config.command with ( match Config.command with
| Analyze -> | Analyze ->
ResultsDir.assert_results_dir "have you run capture before?" ResultsDir.assert_results_dir "have you run capture before?"
| Report | ReportDiff -> | Report | ReportDiff ->
@ -45,7 +45,9 @@ let setup () =
| Explore -> | Explore ->
ResultsDir.assert_results_dir "please run an infer analysis first" ResultsDir.assert_results_dir "please run an infer analysis first"
| Events -> | Events ->
ResultsDir.assert_results_dir "have you run infer before?" ResultsDir.assert_results_dir "have you run infer before?" ) ;
if CLOpt.is_originator then ( RunState.add_run_to_sequence () ; RunState.store () ) ;
()
let print_active_checkers () = let print_active_checkers () =

@ -17,28 +17,32 @@ let database_filename = "results.db"
let database_fullpath = Config.results_dir ^/ database_filename let database_fullpath = Config.results_dir ^/ database_filename
let create_procedures_table db = let procedures_schema =
(* it would be nice to use "WITHOUT ROWID" here but ancient versions of sqlite do not support {|CREATE TABLE IF NOT EXISTS procedures
it *)
SqliteUtils.exec db ~log:"creating procedures table"
~stmt:
{|
CREATE TABLE IF NOT EXISTS procedures
( proc_name TEXT PRIMARY KEY ( proc_name TEXT PRIMARY KEY
, attr_kind INTEGER NOT NULL , attr_kind INTEGER NOT NULL
, source_file TEXT NOT NULL , source_file TEXT NOT NULL
, proc_attributes BLOB NOT NULL )|} , proc_attributes BLOB NOT NULL )|}
let create_source_files_table db = let source_files_schema =
SqliteUtils.exec db ~log:"creating source_files table" {|CREATE TABLE IF NOT EXISTS source_files
~stmt:
{|
CREATE TABLE IF NOT EXISTS source_files
( source_file TEXT PRIMARY KEY ( source_file TEXT PRIMARY KEY
, cfgs BLOB NOT NULL )|} , cfgs BLOB NOT NULL )|}
let schema_hum = Printf.sprintf "%s;\n%s" procedures_schema source_files_schema
let create_procedures_table db =
(* it would be nice to use "WITHOUT ROWID" here but ancient versions of sqlite do not support
it *)
SqliteUtils.exec db ~log:"creating procedures table" ~stmt:procedures_schema
let create_source_files_table db =
SqliteUtils.exec db ~log:"creating source_files table" ~stmt:source_files_schema
let create_db () = let create_db () =
let temp_db = Filename.temp_file ~in_dir:Config.results_dir database_filename ".tmp" in let temp_db = Filename.temp_file ~in_dir:Config.results_dir database_filename ".tmp" in
let db = Sqlite3.db_open ~mutex:`FULL temp_db in let db = Sqlite3.db_open ~mutex:`FULL temp_db in

@ -13,6 +13,9 @@ val database_filename : string
val database_fullpath : string val database_fullpath : string
(** the absolute path to the database file *) (** the absolute path to the database file *)
val schema_hum : string
(** some human-readable string describing the tables *)
val get_database : unit -> Sqlite3.db val get_database : unit -> Sqlite3.db
(** The results database. You should always use this function to access the database, as the connection to it may change during the execution (see [new_database_connection]). *) (** The results database. You should always use this function to access the database, as the connection to it may change during the execution (see [new_database_connection]). *)

@ -8,6 +8,7 @@
*) *)
open! IStd open! IStd
open! PVariant open! PVariant
module CLOpt = CommandLineOption
module L = Logging module L = Logging
let results_dir_dir_markers = let results_dir_dir_markers =
@ -30,16 +31,22 @@ let is_results_dir ~check_correct_version () =
Result.ok_if_true has_all_markers ~error:(Printf.sprintf "'%s' not found" !not_found) Result.ok_if_true has_all_markers ~error:(Printf.sprintf "'%s' not found" !not_found)
let non_empty_directory_exists results_dir =
(* Look if [results_dir] exists and is a non-empty directory. If it's an empty directory, leave it
alone. This allows users to create a temporary directory for the infer results without infer
removing it to recreate it, which could be racy. *)
Sys.is_directory results_dir = `Yes && not (Utils.directory_is_empty results_dir)
let remove_results_dir () = let remove_results_dir () =
(* Look if file exists, it may not be a directory but that will be caught by the call to [is_results_dir]. If it's an empty directory, leave it alone. This allows users to create a temporary directory for the infer results without infer removing it to recreate it, which could be racy. *) if non_empty_directory_exists Config.results_dir then (
if Sys.file_exists Config.results_dir = `Yes && not (Utils.directory_is_empty Config.results_dir)
then (
if not Config.force_delete_results_dir then if not Config.force_delete_results_dir then
Result.iter_error (is_results_dir ~check_correct_version:false ()) ~f:(fun err -> Result.iter_error (is_results_dir ~check_correct_version:false ()) ~f:(fun err ->
L.(die UserError) L.(die UserError)
"ERROR: '%s' exists but does not seem to be an infer results directory: %s@\nERROR: Please delete '%s' and try again@." "ERROR: '%s' exists but does not seem to be an infer results directory: %s@\nERROR: Please delete '%s' and try again@."
Config.results_dir err Config.results_dir ) ; Config.results_dir err Config.results_dir ) ;
Utils.rmtree Config.results_dir ) Utils.rmtree Config.results_dir ) ;
RunState.reset ()
let prepare_logging_and_db () = let prepare_logging_and_db () =
@ -50,17 +57,30 @@ let prepare_logging_and_db () =
let create_results_dir () = let create_results_dir () =
if non_empty_directory_exists Config.results_dir then
RunState.load_and_validate ()
|> Result.iter_error ~f:(fun error ->
if Config.force_delete_results_dir then (
L.user_warning
"%s@\nDeleting results dir because --force-delete-results-dir was passed@." error ;
remove_results_dir () )
else L.die UserError "%s@\nPlease remove '%s' and try again" error Config.results_dir ) ;
Unix.mkdir_p Config.results_dir ; Unix.mkdir_p Config.results_dir ;
Unix.mkdir_p (Config.results_dir ^/ Config.events_dir_name) ; Unix.mkdir_p (Config.results_dir ^/ Config.events_dir_name) ;
List.iter ~f:Unix.mkdir_p results_dir_dir_markers ; List.iter ~f:Unix.mkdir_p results_dir_dir_markers ;
prepare_logging_and_db () prepare_logging_and_db () ;
()
let assert_results_dir advice = let assert_results_dir advice =
Result.iter_error (is_results_dir ~check_correct_version:true ()) ~f:(fun err -> Result.iter_error (is_results_dir ~check_correct_version:true ()) ~f:(fun err ->
L.(die UserError) L.(die UserError)
"ERROR: No results directory at '%s': %s@\nERROR: %s@." Config.results_dir err advice ) ; "ERROR: No results directory at '%s': %s@\nERROR: %s@." Config.results_dir err advice ) ;
prepare_logging_and_db () RunState.load_and_validate ()
|> Result.iter_error ~f:(fun error ->
L.die UserError "%s@\nPlease remove '%s' and try again" error Config.results_dir ) ;
prepare_logging_and_db () ;
()
let delete_capture_and_analysis_data () = let delete_capture_and_analysis_data () =

@ -0,0 +1,68 @@
(*
* Copyright (c) 2018 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*)
open! IStd
let run_time_string = Time.now () |> Time.to_string
let state0 =
let open Runstate_t in
{ run_sequence= []
; results_dir_format=
Printf.sprintf "db_filename: %s\ndb_schema: %s" ResultsDatabase.database_filename
ResultsDatabase.schema_hum }
let state : Runstate_t.t ref = ref state0
let add_run_to_sequence () =
let run =
{ Runstate_t.infer_version= Version.{Runstate_t.major; minor; patch; commit}
; date= run_time_string
; command= Config.command }
in
Runstate_t.(state := {(!state) with run_sequence= run :: !state.run_sequence})
let state_filename = ".infer_runstate.json"
let state_file_path = Config.results_dir ^/ state_filename
let store () =
Utils.with_file_out state_file_path ~f:(fun oc ->
Runstate_j.string_of_t !state |> Out_channel.output_string oc )
let load_and_validate () =
let error msg =
Printf.ksprintf
(fun err_msg ->
Error
(Printf.sprintf
"Incompatible results directory '%s':\n%s\nWas '%s' created using an older version of infer?"
Config.results_dir err_msg Config.results_dir) )
msg
in
if Sys.file_exists state_file_path <> `Yes then error "save state not found"
else
try
let loaded_state = Ag_util.Json.from_file Runstate_j.read_t state_file_path in
if not
(String.equal !state.Runstate_t.results_dir_format
loaded_state.Runstate_t.results_dir_format)
then
error "Incompatible formats: found\n %s\n\nbut expected this format:\n %s\n\n"
loaded_state.results_dir_format !state.Runstate_t.results_dir_format
else (
state := loaded_state ;
Ok () )
with _ -> Error "error reading the save state"
let reset () = state := state0

@ -0,0 +1,22 @@
(*
* Copyright (c) 2018 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*)
open! IStd
val add_run_to_sequence : unit -> unit
(** add an entry with the current run date *)
val store : unit -> unit
(** save the current state to disk *)
val load_and_validate : unit -> (unit, string) Result.t
(** attempt to load state from disk *)
val reset : unit -> unit
(** reset the in-memory state to what it would be if this were a fresh run of infer *)

@ -9,6 +9,12 @@
open! IStd open! IStd
val major : int
val minor : int
val patch : int
val commit : string val commit : string
val versionString : string val versionString : string

@ -127,13 +127,19 @@ let clean_results_dir () =
let check_captured_empty mode = let check_captured_empty mode =
let clean_command_opt = clean_compilation_command mode in let clean_command_opt = clean_compilation_command mode in
if Config.capture && Utils.directory_is_empty Config.captured_dir then ( if Config.capture && Utils.directory_is_empty Config.captured_dir then
let nothing_to_compile_msg = "Nothing to compile." in
let please_run_capture_msg =
match mode with Analyze -> " Have you run `infer capture`?" | _ -> ""
in
( match clean_command_opt with ( match clean_command_opt with
| Some clean_command -> | Some clean_command ->
L.user_warning "@\nNothing to compile. Try running `%s` first.@." clean_command L.user_warning "%s%s Try running `%s` first.@." nothing_to_compile_msg
please_run_capture_msg clean_command
| None -> | None ->
L.user_warning "@\nNothing to compile. Try cleaning the build first.@." ) ; L.user_warning "%s%s Try cleaning the build first.@." nothing_to_compile_msg
true ) please_run_capture_msg ) ;
true
else false else false

@ -45,7 +45,6 @@ previous.exp.test: $(PREVIOUS_REPORT)
test: compare_reports test: compare_reports
print: current.exp.test previous.exp.test print: current.exp.test previous.exp.test
replace: replace_reports replace: replace_reports
$(DIFFERENTIAL_REPORT): $(MODIFIED_FILES_FILE)
$(CURRENT_REPORT): $(CURRENT_REPORT):
$(QUIET)$(COPY) src/com/example/DiffClass1.java.current src/com/example/DiffClass1.java $(QUIET)$(COPY) src/com/example/DiffClass1.java.current src/com/example/DiffClass1.java

@ -16,7 +16,10 @@ OBJECTS = $(ROOT_DIR)/buck-out/genruletest/gen/infer/tests/build_systems/genrule
JSON_REPORT = $(ROOT_DIR)/buck-out/gen/infer/tests/build_systems/genrule/module2/module2_infer/infer_out/report.json JSON_REPORT = $(ROOT_DIR)/buck-out/gen/infer/tests/build_systems/genrule/module2/module2_infer/infer_out/report.json
INFER_OPTIONS = --project-root $(ROOT_DIR) INFER_OPTIONS = --project-root $(ROOT_DIR)
INFERPRINT_OPTIONS = --project-root $(ROOT_DIR) --issues-tests INFERPRINT_OPTIONS = --project-root $(ROOT_DIR) --issues-tests
CLEAN_EXTRA = $(ROOT_DIR)/buck-out/genruletest CLEAN_EXTRA = $(ROOT_DIR)/buck-out/genruletest report.json
# fake infer-out because we only copy the results.json from buck-out.
INFER_OUT = .
include $(TESTS_DIR)/java.make include $(TESTS_DIR)/java.make
include $(TESTS_DIR)/infer.make include $(TESTS_DIR)/infer.make
@ -37,8 +40,10 @@ $(JSON_REPORT): $(JAVA_DEPS) $(JAVA_SOURCE_FILES) $(MAKEFILE_LIST)
INFER_BIN="$(INFER_BIN)" NO_BUCKD=1 $(BUCK) build --no-cache $(INFER_TARGET)) INFER_BIN="$(INFER_BIN)" NO_BUCKD=1 $(BUCK) build --no-cache $(INFER_TARGET))
$(QUIET)touch $@ $(QUIET)touch $@
infer-out/report.json: $(JSON_REPORT) $(MAKEFILE_LIST) report.json: $(JSON_REPORT) $(MAKEFILE_LIST)
$(QUIET)$(REMOVE_DIR) infer-out
$(QUIET)$(MKDIR_P) infer-out
# the report contains absolute paths # the report contains absolute paths
$(QUIET)sed -e 's#$(abspath $(TESTS_DIR))/##g' $< > $@ $(QUIET)sed -e 's#$(abspath $(TESTS_DIR))/##g' $< > $@
issues.exp.test$(TEST_SUFFIX): report.json $(INFER_BIN)
$(QUIET)$(INFER_BIN) report -q -a $(ANALYZER) \
$(INFERPRINT_OPTIONS) $@ --from-json-report $<

@ -11,7 +11,6 @@
include $(TESTS_DIR)/base.make include $(TESTS_DIR)/base.make
INFER_OUT = infer-out INFER_OUT = infer-out
DIFFERENTIAL_REPORT = $(INFER_OUT)/differential/introduced.json
EXPECTED_TEST_OUTPUT = introduced.exp.test EXPECTED_TEST_OUTPUT = introduced.exp.test
INFERPRINT_ISSUES_FIELDS = \ INFERPRINT_ISSUES_FIELDS = \
"bug_type,file,procedure,line_offset,procedure_id,procedure_id_without_crc" "bug_type,file,procedure,line_offset,procedure_id,procedure_id_without_crc"
@ -30,14 +29,13 @@ $(PREVIOUS_REPORT): $(CURRENT_REPORT)
.PHONY: analyze .PHONY: analyze
analyze: $(CURRENT_REPORT) $(PREVIOUS_REPORT) analyze: $(CURRENT_REPORT) $(PREVIOUS_REPORT)
$(DIFFERENTIAL_REPORT): $(CURRENT_REPORT) $(PREVIOUS_REPORT) $(MAKEFILE_LIST) $(EXPECTED_TEST_OUTPUT): $(CURRENT_REPORT) $(PREVIOUS_REPORT) $(MODIFIED_FILES_FILE) \
$(INFER_BIN) $(MAKEFILE_LIST)
$(QUIET)$(REMOVE_DIR) $(INFER_OUT) $(QUIET)$(REMOVE_DIR) $(INFER_OUT)
$(QUIET)$(call silent_on_success,Computing results difference in $(TEST_REL_DIR),\ $(QUIET)$(call silent_on_success,Computing results difference in $(TEST_REL_DIR),\
$(INFER_BIN) -o $(INFER_OUT) --project-root $(CURDIR) reportdiff \ $(INFER_BIN) -o $(INFER_OUT) --project-root $(CURDIR) reportdiff \
--report-current $(CURRENT_REPORT) --report-previous $(PREVIOUS_REPORT) \ --report-current $(CURRENT_REPORT) --report-previous $(PREVIOUS_REPORT) \
$(DIFFERENTIAL_ARGS)) $(DIFFERENTIAL_ARGS))
$(EXPECTED_TEST_OUTPUT): $(DIFFERENTIAL_REPORT) $(INFER_BIN) $(MAKEFILE_LIST)
$(QUIET)$(INFER_BIN) report -o $(INFER_OUT) \ $(QUIET)$(INFER_BIN) report -o $(INFER_OUT) \
--issues-fields $(INFERPRINT_ISSUES_FIELDS) \ --issues-fields $(INFERPRINT_ISSUES_FIELDS) \
--from-json-report $(INFER_OUT)/differential/introduced.json \ --from-json-report $(INFER_OUT)/differential/introduced.json \

@ -5,6 +5,8 @@
# LICENSE file in the root directory of this source tree. An additional grant # LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory. # of patent rights can be found in the PATENTS file in the same directory.
INFER_OUT ?= infer-out$(TEST_SUFFIX)
include $(TESTS_DIR)/base.make include $(TESTS_DIR)/base.make
# useful to print non-default analyzer # useful to print non-default analyzer
@ -12,7 +14,7 @@ ANALYZER_STRING=$(shell if [ -n $(ANALYZER) ]; then printf ' ($(ANALYZER))'; fi)
default: compile default: compile
issues.exp.test$(TEST_SUFFIX): infer-out$(TEST_SUFFIX)/report.json $(INFER_BIN) issues.exp.test$(TEST_SUFFIX): $(INFER_OUT)/report.json $(INFER_BIN)
$(QUIET)$(INFER_BIN) report -q -a $(ANALYZER) --results-dir $(<D) \ $(QUIET)$(INFER_BIN) report -q -a $(ANALYZER) --results-dir $(<D) \
$(INFERPRINT_OPTIONS) $@ --from-json-report $< $(INFERPRINT_OPTIONS) $@ --from-json-report $<
@ -20,7 +22,7 @@ issues.exp.test$(TEST_SUFFIX): infer-out$(TEST_SUFFIX)/report.json $(INFER_BIN)
compile: $(OBJECTS) compile: $(OBJECTS)
.PHONY: analyze .PHONY: analyze
analyze: infer-out$(TEST_SUFFIX)/report.json analyze: $(INFER_OUT)/report.json
.PHONY: print .PHONY: print
print: issues.exp.test$(TEST_SUFFIX) print: issues.exp.test$(TEST_SUFFIX)
@ -36,5 +38,7 @@ replace: issues.exp.test$(TEST_SUFFIX)
.PHONY: clean .PHONY: clean
clean: clean:
$(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) infer-out$(TEST_SUFFIX) \ $(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) $(OBJECTS) $(CLEAN_EXTRA)
$(OBJECTS) $(CLEAN_EXTRA) ifneq ($(INFER_OUT),.)
$(REMOVE_DIR) $(INFER_OUT)
endif

Loading…
Cancel
Save