From a2a7e07708867471014b9aa3ed9c43fc0042ab6a Mon Sep 17 00:00:00 2001 From: Josh Berdine Date: Wed, 22 Jun 2016 08:51:18 -0700 Subject: [PATCH] Add OCaml toplevel driver executable Summary: This diff changes the toplevel 'infer' executable from the current python script to an OCaml binary. Currently this executable only parses command line arguments, sets up environment variables, and invokes the existing python script. This improves infer's command-line and configuration interface, since passing arguments to the frontends or backend no longer requires manually setting environment variables, and arguments for the toplevel can now also be specified in .inferconfig. Simplification and migration of functionality from the python script is left for the future. Reviewed By: martinoluca, jvillard Differential Revision: D3450662 fbshipit-source-id: 1b52302 --- .gitignore | 1 + FILES.md | 2 +- Makefile | 20 ++- Makefile.config.in | 4 +- infer/lib/python/{infer => infer.py} | 0 infer/lib/python/inferlib/analyze.py | 1 - infer/lib/python/inferlib/bucklib.py | 2 +- infer/src/Makefile | 10 +- infer/src/backend/CommandLineOption.ml | 26 ++- infer/src/backend/CommandLineOption.mli | 13 +- infer/src/backend/config.ml | 214 ++++++++++++++++++------ infer/src/backend/config.mli | 38 +++-- infer/src/backend/infer.ml | 8 +- infer/src/backend/infer.mli | 12 ++ infer/src/backend/utils.ml | 29 +++- infer/src/backend/utils.mli | 7 +- 16 files changed, 289 insertions(+), 98 deletions(-) rename infer/lib/python/{infer => infer.py} (100%) create mode 100644 infer/src/backend/infer.mli diff --git a/.gitignore b/.gitignore index 6c37fbb88..96c53f1da 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ buck-out/ /infer/bin/InferUnit /infer/bin/Typeprop /infer/bin/infer +/infer/lib/infer /infer/bin/inferTraceBugs /infer/src/backend/version.ml /infer/models/java/models/ diff --git a/FILES.md b/FILES.md index 27a4d655b..47ad6cac1 100644 --- a/FILES.md +++ b/FILES.md @@ -2,7 +2,7 @@ ## Top-level commands -*infer* : Main command to run Infer. It's a python script. Check out the docs for instructions on how to use it. +*infer* : Main command to run Infer. Check out the docs for instructions on how to use it. *inferTest* : Shell script for running Infer's tests. Uses Buck for running the tests. Usage: inferTest {c, objc, java} for the tests about the analysis of C, Objective-C, or Java files. diff --git a/Makefile b/Makefile index 47ae0d85e..3b589984a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,12 @@ TARGETS_TO_TEST := $(shell echo $(TARGETS_TO_TEST)) all: infer inferTraceBugs -$(INFER_BIN_RELPATH) $(INFERTRACEBUGS_BIN_RELPATH): +$(INFER_BIN_SYMLINK): + ($(REMOVE) $@ && \ + cd $(@D) && \ + $(LN_S) ../lib/$(@F) $(@F)) + +$(INFERTRACEBUGS_BIN_RELPATH): ($(REMOVE) $@ && \ cd $(@D) && \ $(LN_S) ../lib/python/$(@F) $(@F)) @@ -36,7 +41,7 @@ ifeq ($(BUILD_C_ANALYZERS),yes) src_build: clang_plugin endif -infer: $(INFER_BIN_RELPATH) src_build +infer: $(INFER_BIN_SYMLINK) src_build ifeq ($(BUILD_JAVA_ANALYZERS),yes) $(MAKE) -C $(ANNOTATIONS_DIR) endif @@ -213,19 +218,20 @@ endif @for i in $$(find infer/lib/python/inferlib/* -type f); do \ $(INSTALL_DATA) -C $$i $(DESTDIR)$(libdir)/infer/$$i; \ done - $(INSTALL_PROGRAM) -C infer/lib/python/infer \ - $(DESTDIR)$(libdir)/infer/infer/lib/python/infer + $(INSTALL_PROGRAM) -C infer/lib/python/infer.py \ + $(DESTDIR)$(libdir)/infer/infer/lib/python/infer.py $(INSTALL_PROGRAM) -C infer/lib/python/inferTraceBugs \ $(DESTDIR)$(libdir)/infer/infer/lib/python/inferTraceBugs + $(INSTALL_PROGRAM) -C $(INFER_BIN) $(DESTDIR)$(libdir)/infer/infer/lib/ $(INSTALL_PROGRAM) -C $(INFERANALYZE_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ $(INSTALL_PROGRAM) -C $(INFERPRINT_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ $(INSTALL_PROGRAM) -C $(INFERSTATS_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ (cd $(DESTDIR)$(libdir)/infer/infer/bin/ && \ $(REMOVE) infer && \ - $(LN_S) $(libdir)/infer/infer/lib/python/infer infer) + $(LN_S) $(libdir)/infer/infer/lib/infer infer) (cd $(DESTDIR)$(bindir)/ && \ $(REMOVE) infer && \ - $(LN_S) $(libdir)/infer/infer/lib/python/infer infer) + $(LN_S) $(libdir)/infer/infer/lib/infer infer) (cd $(DESTDIR)$(bindir)/ && \ $(REMOVE) inferTraceBugs && \ $(LN_S) $(libdir)/infer/infer/lib/python/inferTraceBugs inferTraceBugs) @@ -245,7 +251,7 @@ endif $(MAKE) -C $(SRC_DIR) clean $(MAKE) -C $(ANNOTATIONS_DIR) clean $(MAKE) -C $(MODELS_DIR) clean - $(REMOVE) $(INFER_BIN_RELPATH) $(INFERTRACEBUGS_BIN_RELPATH) + $(REMOVE) $(INFER_BIN_SYMLINK) $(INFERTRACEBUGS_BIN_RELPATH) ifeq ($(IS_FACEBOOK_TREE),yes) $(MAKE) -C facebook clean endif diff --git a/Makefile.config.in b/Makefile.config.in index d0ecf6868..bd523bf8d 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -98,10 +98,10 @@ INFERJAVA_BIN = $(BIN_DIR)/InferJava INFERSTATS_BIN = $(BIN_DIR)/InferStatsAggregator INFERPRINT_BIN = $(BIN_DIR)/InferPrint INFERUNIT_BIN = $(BIN_DIR)/InferUnit -INFER_BIN = $(BIN_DIR)/infer +INFER_BIN = $(LIB_DIR)/infer INFERTRACEBUGS_BIN = $(BIN_DIR)/inferTraceBugs # paths relative to $(ROOT_DIR) -INFER_BIN_RELPATH = infer/bin/infer +INFER_BIN_SYMLINK = infer/bin/infer INFERTRACEBUGS_BIN_RELPATH = infer/bin/inferTraceBugs ifeq ($(BUILD_JAVA_ANALYZERS),yes) diff --git a/infer/lib/python/infer b/infer/lib/python/infer.py similarity index 100% rename from infer/lib/python/infer rename to infer/lib/python/infer.py diff --git a/infer/lib/python/inferlib/analyze.py b/infer/lib/python/inferlib/analyze.py index 9f92e8b4f..c11ca917a 100644 --- a/infer/lib/python/inferlib/analyze.py +++ b/infer/lib/python/inferlib/analyze.py @@ -153,7 +153,6 @@ infer_group.add_argument('--infer_cache', metavar='', infer_group.add_argument('-pr', '--project_root', dest='project_root', default=get_pwd(), - type=utils.decode, help='Location of the project root ' '(default is current directory)') diff --git a/infer/lib/python/inferlib/bucklib.py b/infer/lib/python/inferlib/bucklib.py index 161368576..a94d5a634 100644 --- a/infer/lib/python/inferlib/bucklib.py +++ b/infer/lib/python/inferlib/bucklib.py @@ -49,7 +49,7 @@ INFER_SCRIPT = """\ import subprocess import sys -cmd = ['{0}'] + {1} + ['--', 'javac'] + sys.argv[1:] +cmd = {1} + ['--', 'javac'] + sys.argv[1:] subprocess.check_call(cmd) """ diff --git a/infer/src/Makefile b/infer/src/Makefile index 1292001f9..2e26aa5b0 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -45,6 +45,7 @@ endif #### Backend declarations #### +INFER_MAIN = backend/infer INFERANALYZE_MAIN = backend/inferanalyze #### InferPrint declarations #### @@ -130,6 +131,7 @@ OCAMLBUILD_ALL = $(OCAMLBUILD_BASE) $(JAVA_OCAMLBUILD_OPTIONS) # list of ocamlbuild targets common to all build targets -- native version INFER_BASE_TARGETS = \ + $(INFER_MAIN).native \ $(INFERANALYZE_MAIN).native \ $(INFERPRINT_MAIN).native \ $(INFERUNIT_MAIN).native \ @@ -166,6 +168,7 @@ all: infer infer: init $(INFERPRINT_ATDGEN_STUBS) $(OCAMLBUILD_CONFIG) -build-dir $(INFER_BUILD_DIR) $(INFER_CONFIG_TARGETS) + $(COPY) $(INFER_BUILD_DIR)/$(INFER_MAIN).native $(INFER_BIN) $(COPY) $(INFER_BUILD_DIR)/$(INFERANALYZE_MAIN).native $(INFERANALYZE_BIN) $(COPY) $(INFER_BUILD_DIR)/$(INFERPRINT_MAIN).native $(INFERPRINT_BIN) $(COPY) $(INFER_BUILD_DIR)/$(CHECKCOPYRIGHT_MAIN).native $(CHECKCOPYRIGHT_BIN) @@ -226,7 +229,7 @@ test_build: init $(INFERPRINT_ATDGEN_STUBS) $(CLANG_ATDGEN_STUBS) $(INFER_CLANG_ $(DEPENDENCIES_DIR)/ocamldot/ocamldot: $(MAKE) -C $(DEPENDENCIES_DIR)/ocamldot -roots:=Inferanalyze CMain JMain Inferprint +roots:=Infer Inferanalyze CMain JMain Inferprint src_dirs:=$(shell find * -type d) src_files:=$(shell find $(src_dirs) -regex '.*\.ml\(i\)*' -not -path facebook/scripts/eradicate_stats.ml) inc_flags:=$(foreach dir,$(src_dirs),-I $(dir)) @@ -317,8 +320,9 @@ endif $(REMOVE) backend/version.ml $(REMOVE) backend/version.ml.tmp.* $(REMOVE) backend/jsonbug_{j,t}.ml{,i} - $(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(INFERLLVM_BIN) $(INFERUNIT_BIN) - $(REMOVE) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(CHECKCOPYRIGHT_BIN) $(STATSAGGREGATOR_BIN) + $(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(STATSAGGREGATOR_BIN) + $(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(INFERLLVM_BIN) + $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(REMOVE) $(CLANG_ATDGEN_STUBS) $(REMOVE) $(INFER_CLANG_FCP_MIRRORED_FILES) $(REMOVE) mod_dep.dot diff --git a/infer/src/backend/CommandLineOption.ml b/infer/src/backend/CommandLineOption.ml index 5de47e6b2..4cf6aa49a 100644 --- a/infer/src/backend/CommandLineOption.ml +++ b/infer/src/backend/CommandLineOption.ml @@ -144,7 +144,9 @@ let mk ?(deprecated=[]) ?(exes=[]) let closure = mk_setter variable in let setter str = try closure str - with _ -> raise (Arg.Bad ("bad value " ^ str ^ " for flag " ^ long)) in + with exc -> + raise (Arg.Bad ("bad value " ^ str ^ " for flag " ^ long + ^ " (" ^ (Printexc.to_string exc) ^ ")")) in let spec = mk_spec setter in let doc = let default_string = default_to_string default in @@ -297,7 +299,7 @@ let mk_symbol_seq ?(default=[]) ~symbols ?(deprecated=[]) ~long ?short ?exes ?(m let sym_to_str = IList.map (fun (x,y) -> (y,x)) symbols in let of_string str = IList.assoc string_equal str symbols in let to_string sym = IList.assoc ( = ) sym sym_to_str in - mk ~deprecated ~long ?short ~default ?exes ~meta:(" ,-separated sequence" ^ meta) doc + mk ~deprecated ~long ?short ~default ?exes ~meta:(",-separated sequence" ^ meta) doc ~default_to_string:(fun syms -> String.concat " " (IList.map to_string syms)) ~mk_setter:(fun var str_seq -> var := IList.map of_string (Str.split (Str.regexp_string ",") str_seq)) @@ -314,13 +316,22 @@ let mk_set_from_json ~default ~default_to_string ~f ~decode_json:(fun json -> [dashdash long; Yojson.Basic.to_string json]) ~mk_spec:(fun set -> Arg.String set) +(* A ref to a function used during argument parsing to process anonymous arguments. By default, + anonymous arguments are rejected. *) let anon_fun = ref (fun arg -> raise (Arg.Bad ("unexpected anonymous argument: " ^ arg))) +(* Clients declare that anonymous arguments are acceptable by calling [mk_anon], which returns a ref + storing the anonymous arguments. *) let mk_anon () = let anon = ref [] in anon_fun := (fun arg -> anon := arg :: !anon) ; anon +let mk_rest ?(exes=[]) doc = + let rest = ref [] in + let spec = Arg.Rest (fun arg -> rest := arg :: !rest) in + add exes {long = "--"; short = ""; meta = ""; doc; spec; decode_json = fun _ -> []} ; + rest let decode_inferconfig_to_argv path = let json = match read_optional_json_file path with @@ -378,7 +389,7 @@ let prefix_before_rest args = prefix_before_rest_ [] args -let parse ?(incomplete=false) ?config_file env_var exe_usage = +let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file env_var exe_usage = let curr_speclist = ref [] and full_speclist = ref [] in @@ -446,13 +457,18 @@ let parse ?(incomplete=false) ?config_file env_var exe_usage = let json_args = decode_inferconfig_to_argv path in (* read .inferconfig first, as both env vars and command-line options overwrite it *) json_args @ env_cl_args in + let argv = Array.of_list (exe_name :: all_args) in let current = ref 0 in + (* tests if msg indicates an unknown option, as opposed to a known option with bad argument *) + let is_unknown msg = + let prefix = exe_name ^ ": unknown option" in + prefix = (String.sub msg 0 (String.length prefix)) in let rec parse_loop () = try - Arg.parse_argv_dynamic ~current (Array.of_list (exe_name :: all_args)) - curr_speclist !anon_fun usage_msg + Arg.parse_argv_dynamic ~current argv curr_speclist !anon_fun usage_msg with | Arg.Bad _ when incomplete -> parse_loop () + | Arg.Bad msg when accept_unknown && is_unknown msg -> !anon_fun argv.(!current) ; parse_loop () | Arg.Bad usage_msg -> Pervasives.prerr_string usage_msg; exit 2 | Arg.Help usage_msg -> Pervasives.print_string usage_msg; exit 0 in diff --git a/infer/src/backend/CommandLineOption.mli b/infer/src/backend/CommandLineOption.mli index 78c925e16..280f578d4 100644 --- a/infer/src/backend/CommandLineOption.mli +++ b/infer/src/backend/CommandLineOption.mli @@ -90,6 +90,13 @@ val mk_anon : unit -> string list ref +(** [mk_rest doc] defines a [string list ref] of the command line arguments following ["--"], in the + reverse order they appeared on the command line. For example, calling [mk_rest] and parsing + [exe -opt1 -opt2 -- arg1 arg2] will result in the returned ref containing [arg2; arg1]. *) +val mk_rest : + ?exes:exe list -> string -> + string list ref + (** [parse env_var exe_usage] parses command line arguments as specified by preceding calls to the [mk_*] functions, and returns a function that prints the usage message and help text then exits. The decoded values of the inferconfig file [config_file], if provided, and of the environment @@ -97,6 +104,8 @@ val mk_anon : the command line supersede those specified in the environment variable, which themselves supersede those passed via the config file. WARNING: An argument will be interpreted as many times as it appears in all of the config file, the environment variable, and the command - line. *) -val parse : ?incomplete:bool -> ?config_file:string -> + line. The [env_var] is set to the full set of options parsed. If [incomplete] is set, unknown + options are ignored, and [env_var] is not set. If [accept_unknown] is set, unknown options are + treated the same as anonymous arguments. *) +val parse : ?incomplete:bool -> ?accept_unknown:bool -> ?config_file:string -> string -> (exe -> Arg.usage_msg) -> (int -> 'a) diff --git a/infer/src/backend/config.ml b/infer/src/backend/config.ml index 0a5bcf09e..c524aeba1 100644 --- a/infer/src/backend/config.ml +++ b/infer/src/backend/config.ml @@ -35,6 +35,15 @@ let string_of_language = function type clang_lang = C | CPP | OBJC | OBJCPP +let ml_bucket_symbols = [ + ("all", `MLeak_all); + ("cf", `MLeak_cf); + ("arc", `MLeak_arc); + ("narc", `MLeak_no_arc); + ("cpp", `MLeak_cpp); + ("unknown_origin", `MLeak_unknown); +] + type os_type = Unix | Win32 | Cygwin type zip_library = { @@ -43,6 +52,17 @@ type zip_library = { models: bool; } +let whitelisted_cpp_methods = [ + ["std"; "move"]; + ["google"; "CheckNotNull"]; + ["google"; "GetReferenceableValue"]; + ["google"; "Check_NEImpl"]; + ["google"; "Check_LEImpl"]; + ["google"; "Check_GTImpl"]; + ["google"; "Check_GEImpl"]; + ["google"; "Check_EQImpl"] +] + (** Constant configuration values *) @@ -183,7 +203,7 @@ let buck_generated_folder = "buck-out/gen" let version_string = "Infer version " ^ Version.versionString - ^ "\nCopyright 2009 - present Facebook. All Rights Reserved.\n" + ^ "\nCopyright 2009 - present Facebook. All Rights Reserved." (** System call configuration values *) @@ -195,27 +215,21 @@ let initial_analysis_time = Unix.time () (** Path to lib/specs to retrieve the default models *) let models_dir = let bin_dir = Filename.dirname Sys.executable_name in - let lib_dir = Filename.concat (Filename.concat bin_dir Filename.parent_dir_name) "lib" in - let lib_specs_dir = Filename.concat lib_dir specs_dir_name in + let lib_dir = bin_dir // Filename.parent_dir_name // "lib" in + let lib_specs_dir = lib_dir // specs_dir_name in lib_specs_dir let cpp_models_dir = let bin_dir = Filename.dirname Sys.executable_name in - let cpp_models_dir = - Filename.concat (Filename.concat bin_dir Filename.parent_dir_name) - "models/cpp/include" in - cpp_models_dir + bin_dir // Filename.parent_dir_name // "models" // "cpp" // "include" -let whitelisted_cpp_methods = [ - ["std"; "move"]; - ["google"; "CheckNotNull"]; - ["google"; "GetReferenceableValue"]; - ["google"; "Check_NEImpl"]; - ["google"; "Check_LEImpl"]; - ["google"; "Check_GTImpl"]; - ["google"; "Check_GEImpl"]; - ["google"; "Check_EQImpl"] -] +let ncpu = + try + with_process_in + "getconf _NPROCESSORS_ONLN 2>/dev/null" + (fun chan -> Scanf.fscanf chan "%d" (fun n -> n)) + with _ -> + 1 let os_type = match Sys.os_type with | "Win32" -> Win32 @@ -321,6 +335,38 @@ let patterns_of_json_with_key json_key json = (** Command Line options *) +(* The working directory of the initial invocation of infer, to which paths passed as command line + options are relative. *) +let init_work_dir = + try + Sys.getenv "INFER_CWD" + with Not_found -> + let cwd = + (* Use PWD if it denotes the same inode as ., to try to avoid paths with symlinks resolved *) + (* Approach is borrowed from llvm implementation of *) + (* llvm::sys::fs::current_path (implemented in Path.inc file) *) + try + let pwd = Unix.getenv "PWD" in + let pwd_stat = Unix.stat pwd in + let dot_stat = Unix.stat "." in + if pwd_stat.st_dev = dot_stat.st_dev && pwd_stat.st_ino = dot_stat.st_ino then + pwd + else + Sys.getcwd () + with _ -> + Sys.getcwd () + in + Unix.putenv "INFER_CWD" cwd ; + cwd + +(* Resolve relative paths passed as command line options, i.e., with respect to the working + directory of the initial invocation of infer. *) +let resolve path = + if Filename.is_relative path then + init_work_dir // path + else + path + (* Declare the phase 1 options *) let inferconfig_home = @@ -328,10 +374,10 @@ let inferconfig_home = ~exes:CLOpt.[Analyze] ~meta:"dir" "Path to the .inferconfig file" and project_root = - CLOpt.mk_string_opt ~deprecated:["project_root"] ~long:"project-root" ~short:"pr" - ?default:(if CLOpt.(current_exe = Print) then Some (Sys.getcwd ()) else None) + CLOpt.mk_string_opt ~deprecated:["project_root"; "-project_root"] ~long:"project-root" ~short:"pr" + ?default:CLOpt.(match current_exe with Print | Toplevel -> Some (Sys.getcwd ()) | _ -> None) ~f:filename_to_absolute - ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print] + ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print;Toplevel] ~meta:"dir" "Specify the root directory of the project" (* Parse the phase 1 options, ignoring the rest *) @@ -345,7 +391,7 @@ and project_root = !project_root let inferconfig_path = match inferconfig_home, project_root with - | Some dir, _ | _, Some dir -> Filename.concat dir inferconfig_file + | Some dir, _ | _, Some dir -> dir // inferconfig_file | None, None -> inferconfig_file (* Proceed to declare and parse the remaining options *) @@ -361,6 +407,14 @@ let inferconfig_path = let anon_args = CLOpt.mk_anon () +and rest = + CLOpt.mk_rest + ~exes:CLOpt.[Toplevel] "Stop argument processing, use remaining arguments as a build command" + +and absolute_paths = + CLOpt.mk_bool ~long:"absolute-paths" + ~exes:CLOpt.[Toplevel] "Report errors using absolute paths" + (** Flag for abstracting fields of structs 0 = no 1 = forget some fields during matching (and so lseg abstraction) *) @@ -432,10 +486,14 @@ and analysis_stops = (** Setup the analyzer in order to filter out errors for this analyzer only *) and analyzer = - CLOpt.mk_symbol_opt ~deprecated:["analyzer"] ~long:"analyzer" + CLOpt.mk_symbol_opt ~deprecated:["analyzer"] ~long:"analyzer" ~short:"a" "Specify the analyzer for the path filtering" ~symbols:Utils.string_to_analyzer +and android_harness = + CLOpt.mk_bool ~deprecated:["harness"] ~long:"android-harness" + "Create harness to detect bugs involving the Android lifecycle" + (** if true, completely ignore the possibility that errors can be caused by unknown procedures during the symbolic execution phase *) and angelic_execution = @@ -455,6 +513,14 @@ and ast_file = CLOpt.mk_string_opt ~long:"ast-file" ~short:"ast" ~meta:"file" "AST file for the translation" +and blacklist = + CLOpt.mk_string_opt ~deprecated:["-blacklist-regex"] ~long:"blacklist" + ~meta:"regex" "Skip analysis of files matched by the specified regular expression" + +and buck = + CLOpt.mk_bool ~long:"buck" + "To use when run with buck" + and buck_out = CLOpt.mk_string_opt ~long:"buck-out" ~exes:CLOpt.[StatsAggregator] ~meta:"dir" "Specify the root directory of buck-out" @@ -526,7 +592,7 @@ and curr_language = var and cxx_experimental = - CLOpt.mk_bool ~deprecated:["cxx-experimental"] ~long:"cxx-experimental" + CLOpt.mk_bool ~deprecated:["cxx-experimental"] ~long:"cxx" "Analyze C++ methods, still experimental" and debug, print_types, write_dotty = @@ -547,6 +613,10 @@ and debug, print_types, write_dotty = in (debug, print_types, write_dotty) +and debug_exceptions = + CLOpt.mk_bool ~long:"debug-exceptions" + "Generate lightweight debugging information: just print the internal exceptions during analysis" + (* The classes in the given jar file will be translated. No sources needed *) and dependencies = CLOpt.mk_bool ~deprecated:["dependencies"] ~long:"dependencies" @@ -589,17 +659,29 @@ and err_file = CLOpt.mk_string ~deprecated:["err_file"] ~long:"err-file" ~default:"" ~exes:CLOpt.[Analyze] ~meta:"file" "use file for the err channel" -(* Generate harness for Android code *) -and harness = - CLOpt.mk_bool ~deprecated:["harness"] ~long:"harness" - "Create Android lifecycle harness" +and failures_allowed = + CLOpt.mk_bool ~deprecated_no:["-no_failures_allowed"] ~long:"failures-allowed" ~default:true + "Fail if at least one of the translations fails" + +and filtering = + CLOpt.mk_bool ~long:"filtering" ~short:"f" ~default:true + "Also show the results from the experimental checks. Warning: some checks may contain many \ + false alarms." + +and flavors = + CLOpt.mk_bool ~deprecated:["-use-flavors"] ~long:"flavors" + "Buck integration using flavors." + +and frontend_stats = + CLOpt.mk_bool ~long:"frontend-stats" ~short:"fs" + "Output statistics about the capture phase to *.o.astlog" and headers = - CLOpt.mk_bool ~deprecated:["headers"] ~deprecated_no:["no_headers"] ~long:"headers" - "Translate code in header files" + CLOpt.mk_bool ~deprecated:["headers"] ~deprecated_no:["no_headers"] ~long:"headers" ~short:"hd" + "Analyze code in header files" and infer_cache = - CLOpt.mk_string_opt ~deprecated:["infer_cache"] ~long:"infer-cache" + CLOpt.mk_string_opt ~deprecated:["infer_cache"; "-infer_cache"] ~long:"infer-cache" ~f:filename_to_absolute ~meta:"dir" "Select a directory to contain the infer cache" @@ -611,6 +693,10 @@ and iterations = "Specify the maximum number of operations for each function, expressed as a multiple \ of symbolic operations" +and jobs = + CLOpt.mk_int ~deprecated:["-multicore"] ~long:"jobs" ~short:"j" ~default:ncpu + ~exes:CLOpt.[Toplevel] ~meta:"int" "Run the specified number of analysis jobs simultaneously" + (** Flag to tune the final information-loss check used by the join 0 = use the most aggressive join for preconditions 1 = use the least aggressive join for preconditions @@ -624,11 +710,19 @@ and latex = CLOpt.mk_option ~deprecated:["latex"] ~long:"latex" ~f:create_outfile ~meta:"file.tex" "Print latex report to file.tex" +and load_average = + CLOpt.mk_option ~long:"load-average" ~short:"l" ~f:(fun s -> Some (float_of_string s)) + ~meta:"float" + "Do not start new parallel jobs if the load average is greater than that specified" + (** name of the file to load analysis results from *) and load_results = CLOpt.mk_string_opt ~deprecated:["load_results"] ~long:"load-results" ~meta:"file.iar" "Load analysis results from Infer Analysis Results file file.iar" +and llvm = + CLOpt.mk_bool ~long:"llvm" "Analyze C or C++ using the experimental LLVM frontend" + (** name of the makefile to create with clusters and dependencies *) and makefile = CLOpt.mk_string ~deprecated:["makefile"] ~long:"makefile" ~default:"" @@ -641,26 +735,22 @@ and merge = (** List of obj memory leak buckets to be checked in Objective-C/C++ *) and ml_buckets = - CLOpt.mk_symbol_seq ~deprecated:["ml_buckets"] ~long:"ml-buckets" ~default:[`MLeak_cf] + CLOpt.mk_symbol_seq ~deprecated:["ml_buckets"; "-ml_buckets"] ~long:"ml-buckets" + ~default:[`MLeak_cf] + ~exes:CLOpt.[Toplevel] "Specify the memory leak buckets to be checked: \ 'cf' checks leaks from Core Foundation, \ 'arc' from code compiled in ARC mode, \ 'narc' from code not compiled in ARC mode, \ 'cpp' from C++ code" - ~symbols:[ - ("all", `MLeak_all); - ("cf", `MLeak_cf); - ("arc", `MLeak_arc); - ("narc", `MLeak_no_arc); - ("cpp", `MLeak_cpp); - ("unknown_origin", `MLeak_unknown)] + ~symbols:ml_bucket_symbols and models_file = CLOpt.mk_string_opt ~deprecated:["models"] ~long:"models" ~exes:CLOpt.[Analyze;Java] ~meta:"zip file" "add a zip file containing the models" and models_mode = - CLOpt.mk_bool ~deprecated:["models_mode"] ~long:"models-mode" + CLOpt.mk_bool ~deprecated:["models_mode"; "-models_mode"] ~long:"models-mode" "Mode for computing the models" and modified_targets = @@ -759,10 +849,10 @@ and reports_include_ml_loc = "Include the location (in the Infer source code) from where reports are generated" and results_dir = - CLOpt.mk_string ~deprecated:["results_dir"] ~long:"results-dir" - ~default:(Filename.concat (Sys.getcwd ()) "infer-out") + CLOpt.mk_string ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o" + ~default:(Sys.getcwd () // "infer-out") ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print;StatsAggregator] - ~meta:"dir" "Specify the project results directory" + ~meta:"dir" "Write results in the specified directory" (** name of the file to load save results to *) and save_results = @@ -813,13 +903,14 @@ and specs_library = let validate_path path = if Filename.is_relative path then failwith ("Failing because path " ^ path ^ " is not absolute") in - match read_file fname with + match read_file (resolve fname) with | Some pathlist -> IList.iter validate_path pathlist; pathlist - | None -> failwith ("cannot read file " ^ fname) + | None -> failwith ("cannot read file " ^ fname ^ " from cwd " ^ (Sys.getcwd ())) in - CLOpt.mk_string ~deprecated:["specs-dir-list-file"] ~long:"specs-library-index" + CLOpt.mk_string ~deprecated:["specs-dir-list-file"; "-specs-dir-list-file"] + ~long:"specs-library-index" ~default:"" ~f:(fun file -> specs_library := (read_specs_dir_list_file file) @ !specs_library; "") ~exes:CLOpt.[Analyze] ~meta:"file" @@ -870,7 +961,7 @@ and test_filtering = "List all the files Infer can report on (should be call at the root of the project)" and testing_mode = - CLOpt.mk_bool ~deprecated:["testing_mode"] ~long:"testing-mode" + CLOpt.mk_bool ~deprecated:["testing_mode"; "-testing_mode"] ~long:"testing-mode" ~short:"tm" "Mode for testing, where no headers are translated, and dot files are created" (** Flag set to enable detailed tracing informatin during error explanation *) @@ -1046,7 +1137,7 @@ let use_jar_cache = true let exe_usage (exe : CLOpt.exe) = match exe with | Analyze -> - version_string ^ "\ + version_string ^ "\n\ Usage: InferAnalyze [options]\n\ Analyze the files captured in the project results directory, \ which can be specified with the --results-dir option." @@ -1063,7 +1154,7 @@ let exe_usage (exe : CLOpt.exe) = To process all the .specs in the results directory, use option --results-dir \ Each spec is printed to standard output unless option -q is used." | StatsAggregator -> - "Usage: InferStatsAggregator --results-dir --buck-out \n \ + "Usage: InferStatsAggregator --results-dir --buck-out \n\ Aggregates all the perf stats generated by Buck on each target" | Toplevel -> version_string @@ -1072,7 +1163,8 @@ let post_parsing_initialization () = F.set_margin !margin ; if !version then ( - F.fprintf F.std_formatter "%s@." version_string ; + (* TODO(11791235) change back to stdout once buck integration is fixed *) + F.fprintf F.err_formatter "%s@." version_string ; exit 0 ); if !version_json then ( @@ -1115,7 +1207,7 @@ let post_parsing_initialization () = let zip_channel = Zip.open_in zip_filename in let entries = Zip.entries zip_channel in let extract_entry entry = - let dest_file = Filename.concat dest_dir (Filename.basename entry.Zip.filename) in + let dest_file = dest_dir // (Filename.basename entry.Zip.filename) in if Filename.check_suffix entry.Zip.filename specs_files_suffix then Zip.copy_entry_to_file zip_channel entry dest_file in IList.iter extract_entry entries; @@ -1123,7 +1215,7 @@ let post_parsing_initialization () = in let basename = Filename.basename zip_filename in let key = basename ^ string_crc_hex32 zip_filename in - let key_dir = Filename.concat cache_dir key in + let key_dir = cache_dir // key in if (mkdir key_dir) then extract_specs key_dir zip_filename; specs_library := !specs_library @ [key_dir] @@ -1156,7 +1248,8 @@ let post_parsing_initialization () = let parse_args_and_return_usage_exit = - let usage_exit = CLOpt.parse ~config_file:inferconfig_path "INFER_ARGS" exe_usage in + let usage_exit = + CLOpt.parse ~accept_unknown:true ~config_file:inferconfig_path "INFER_ARGS" exe_usage in if !debug || (!developer_mode && not (CLOpt.current_exe = CLOpt.Print)) then prerr_endline ((Filename.basename Sys.executable_name) ^ " got args " @@ -1170,8 +1263,10 @@ let print_usage_exit () = (** Freeze initialized configuration values *) -let anon_args = !anon_args +let anon_args = IList.rev !anon_args +and rest = !rest and abs_struct = !abs_struct +and absolute_paths = !absolute_paths and allow_specs_cleanup = !allow_specs_cleanup and analysis_path_regex_whitelist_options = IList.map (fun (a, b) -> (a, !b)) analysis_path_regex_whitelist_options @@ -1187,6 +1282,8 @@ and angelic_execution = !angelic_execution and arc_mode = objc_arc and array_level = !array_level and ast_file = !ast_file +and blacklist = !blacklist +and buck = !buck and buck_out = !buck_out and bugs_csv = !bugs_csv and bugs_json = !bugs_json @@ -1201,9 +1298,10 @@ and clang_lang = !clang_lang and cluster_cmdline = !cluster and code_query = !code_query and continue_capture = !continue -and create_harness = !harness +and create_harness = !android_harness and cxx_experimental = !cxx_experimental and debug_mode = !debug +and debug_exceptions = !debug_exceptions and dependency_mode = !dependencies and developer_mode = !developer_mode and disable_checks = !disable_checks @@ -1211,12 +1309,20 @@ and dotty_cfg_libs = !dotty_cfg_libs and enable_checks = !enable_checks and eradicate = !eradicate and err_file_cmdline = !err_file +and failures_allowed = !failures_allowed +and filtering = !filtering +and flavors = !flavors +and frontend_stats = !frontend_stats +and headers = !headers and infer_cache = !infer_cache and iterations = !iterations and javac_verbose_out = !verbose_out +and jobs = !jobs and join_cond = !join_cond and latex = !latex +and load_average = !load_average and load_analysis_results = !load_results +and llvm = !llvm and makefile_cmdline = !makefile and merge = !merge and ml_buckets = !ml_buckets diff --git a/infer/src/backend/config.mli b/infer/src/backend/config.mli index 6f3fa3e55..72061a304 100644 --- a/infer/src/backend/config.mli +++ b/infer/src/backend/config.mli @@ -30,6 +30,10 @@ type pattern = type clang_lang = C | CPP | OBJC | OBJCPP +val ml_bucket_symbols : + (string * [ `MLeak_all | `MLeak_arc | `MLeak_cf | `MLeak_cpp | `MLeak_no_arc | `MLeak_unknown ]) + list + type os_type = Unix | Win32 | Cygwin type zip_library = { @@ -48,9 +52,12 @@ val assign : string val attributes_dir_name : string val backend_stats_dir_name : string val bound_error_allowed_in_procedure_call : bool +val buck_generated_folder : string val buck_infer_deps_file_name : string -val checks_disabled_by_default : string list val captured_dir_name : string +val checks_disabled_by_default : string list +val cpp_models_dir : string +val csl_analysis : bool val default_failure_name : string val default_in_zip_results_dir : string val dotty_output : string @@ -61,25 +68,24 @@ val idempotent_getters : bool val incremental_procs : bool val initial_analysis_time : float val ivar_attributes : string +val load_average : float option +val log_analysis_crash : string val log_analysis_file : string val log_analysis_procedure : string -val log_analysis_wallclock_timeout : string -val log_analysis_symops_timeout : string val log_analysis_recursion_timeout : string -val log_analysis_crash : string -val buck_generated_folder : string +val log_analysis_symops_timeout : string +val log_analysis_wallclock_timeout : string val max_recursion : int val meet_level : int val models_dir : string -val cpp_models_dir : string -val whitelisted_cpp_methods : string list list +val ncpu : int val nsnotification_center_checker_backend : bool val objc_method_call_semantics : bool val os_type : os_type +val patterns_modeled_expensive : pattern list val patterns_never_returning_null : pattern list -val patterns_suppress_warnings : pattern list val patterns_skip_translation : pattern list -val patterns_modeled_expensive : pattern list +val patterns_suppress_warnings : pattern list val perf_stats_prefix : string val proc_stats_filename : string val property_attributes : string @@ -94,11 +100,11 @@ val specs_dir_name : string val specs_files_suffix : string val start_filename : string val taint_analysis : bool -val csl_analysis : bool val trace_absarray : bool val undo_join : bool val unsafe_unret : string val weak : string +val whitelisted_cpp_methods : string list list (** Configuration values specified by environment variables *) @@ -115,7 +121,9 @@ val sound_dynamic_dispatch : bool (** Configuration values specified by command-line options *) val anon_args : string list +val rest : string list val abs_struct : int +val absolute_paths : bool val allow_specs_cleanup : bool val analysis_path_regex_whitelist : analyzer -> string list val analysis_path_regex_blacklist : analyzer -> string list @@ -126,6 +134,8 @@ val analyzer : analyzer option val angelic_execution : bool val array_level : int val ast_file : string option +val blacklist : string option +val buck : bool val buck_out : string option val bugs_csv : outfile option val bugs_json : outfile option @@ -142,6 +152,7 @@ val continue_capture : bool val create_harness : bool val cxx_experimental : bool val debug_mode : bool +val debug_exceptions : bool val dependency_mode : bool val developer_mode : bool val disable_checks : string list @@ -149,12 +160,19 @@ val dotty_cfg_libs : bool val enable_checks : string list val eradicate : bool val err_file_cmdline : string +val failures_allowed : bool +val filtering : bool +val flavors : bool +val frontend_stats : bool +val headers : bool val infer_cache : string option val iterations : int val javac_verbose_out : string +val jobs : int val join_cond : int val latex : outfile option val load_analysis_results : string option +val llvm : bool val makefile_cmdline : string val merge : bool val ml_buckets : diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index 30d19e4c9..3f2c59ce1 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -27,14 +27,13 @@ let set_env_for_clang_wrapper () = let () = set_env_for_clang_wrapper () ; - let ( / ) = Filename.concat in (* The infer executable in the bin directory is a symbolic link to the real binary in the lib directory, so that the python script in the lib directory can be found relative to it. *) let real_exe = match Unix.readlink Sys.executable_name with | link when Filename.is_relative link -> (* Sys.executable_name is a relative symbolic link *) - (Filename.dirname Sys.executable_name) / link + (Filename.dirname Sys.executable_name) // link | link -> (* Sys.executable_name is an absolute symbolic link *) link @@ -42,12 +41,13 @@ let () = (* Sys.executable_name is not a symbolic link *) Sys.executable_name in - let infer_py = (Filename.dirname real_exe) / "python" / "infer.py" in + let infer_py = (Filename.dirname real_exe) // "python" // "infer.py" in let build_cmd = IList.rev Config.rest in let buck = match build_cmd with "buck" :: _ -> true | _ -> false in let args_py = Array.of_list ( infer_py :: + Config.anon_args @ (match Config.analyzer with None -> [] | Some a -> ["--analyzer"; Utils.string_of_analyzer a]) @ (match Config.blacklist with @@ -67,7 +67,7 @@ let () = ["--use-flavors"]) @ (match Config.infer_cache with None -> [] | Some s -> ["--infer_cache"; s]) @ - "--multicore" :: (string_of_int Config.multicore) :: + "--multicore" :: (string_of_int Config.jobs) :: "--out" :: Config.results_dir :: (match Config.project_root with None -> [] | Some pr -> ["--project_root"; pr]) @ diff --git a/infer/src/backend/infer.mli b/infer/src/backend/infer.mli new file mode 100644 index 000000000..042850972 --- /dev/null +++ b/infer/src/backend/infer.mli @@ -0,0 +1,12 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +open! Utils + +(** Top-level driver that orchestrates build system integration, frontends, and backend *) diff --git a/infer/src/backend/utils.ml b/infer/src/backend/utils.ml index a3a9f4579..09262b28b 100644 --- a/infer/src/backend/utils.ml +++ b/infer/src/backend/utils.ml @@ -387,6 +387,8 @@ let do_outf outf_opt f = let close_outf outf = close_out outf.out_c +let ( // ) = Filename.concat + (** convert a filename to absolute path and normalize by removing occurrences of "." and ".." *) module FileNormalize = struct let rec fname_to_list_rev fname = @@ -405,7 +407,7 @@ module FileNormalize = struct (* concatenate a list of strings representing a path into a filename *) let rec list_to_fname base path = match path with | [] -> base - | x :: path' -> list_to_fname (Filename.concat base x) path' + | x :: path' -> list_to_fname (base // x) path' (* normalize a path where done_l is a reversed path from the root already normalized *) (* and todo_l is the path still to normalize *) @@ -429,7 +431,7 @@ module FileNormalize = struct let is_relative = Filename.is_relative fname in let must_normalize = fname_contains_current_parent fname in let simple_case () = - if is_relative then Filename.concat (Unix.getcwd ()) fname + if is_relative then Unix.getcwd () // fname else fname in if must_normalize then begin let done_l, todo_l = @@ -473,7 +475,7 @@ let filename_to_relative root fname = String.sub s2 (n1 + 1) (n2 - (n1 + 1)) else s2 in let norm_root = (* norm_root is root without any trailing / *) - Filename.concat (Filename.dirname root) (Filename.basename root) in + Filename.dirname root // Filename.basename root in let remainder = (* remove the path prefix to root including trailing / *) string_strict_subtract norm_root fname in remainder @@ -506,7 +508,7 @@ let next compare = let directory_fold f init path = let collect current_dir (accu, dirs) path = - let full_path = (Filename.concat current_dir path) in + let full_path = current_dir // path in try if Sys.is_directory full_path then (accu, full_path:: dirs) @@ -528,7 +530,7 @@ let directory_fold f init path = let directory_iter f path = let apply current_dir dirs path = - let full_path = (Filename.concat current_dir path) in + let full_path = current_dir // path in try if Sys.is_directory full_path then full_path:: dirs @@ -548,10 +550,11 @@ let directory_iter f path = else f path -type analyzer = Infer | Eradicate | Checkers | Tracing +type analyzer = Capture | Compile | Infer | Eradicate | Checkers | Tracing let string_to_analyzer = - [("infer", Infer); ("eradicate", Eradicate); ("checkers", Checkers); ("tracing", Tracing)] + [("capture", Capture); ("compile", Compile); + ("infer", Infer); ("eradicate", Eradicate); ("checkers", Checkers); ("tracing", Tracing)] let analyzers = IList.map snd string_to_analyzer @@ -600,6 +603,18 @@ let with_file file ~f = let write_json_to_file destfile json = with_file destfile ~f:(fun oc -> Yojson.Basic.pretty_to_channel oc json) +let with_process_in command read = + let chan = Unix.open_process_in command in + let res = + try + read chan + with exc -> + Unix.close_process_in chan |> ignore ; + raise exc + in + Unix.close_process_in chan |> ignore ; + res + let failwithf fmt = Format.kfprintf (fun _ -> failwith (Format.flush_str_formatter ())) Format.str_formatter fmt diff --git a/infer/src/backend/utils.mli b/infer/src/backend/utils.mli index 602d2ec9b..05fe17581 100644 --- a/infer/src/backend/utils.mli +++ b/infer/src/backend/utils.mli @@ -193,6 +193,9 @@ val copy_file : string -> string -> int option (** read a source file and return a list of lines, or None in case of error *) val read_file : string -> string list option +(** Filename.concat *) +val ( // ) : string -> string -> string + (** Convert a filename to an absolute one if it is relative, and normalize "." and ".." *) val filename_to_absolute : string -> string @@ -254,7 +257,7 @@ val directory_fold : ('a -> string -> 'a) -> 'a -> string -> 'a val directory_iter : (string -> unit) -> string -> unit (** Various kind of analyzers *) -type analyzer = Infer | Eradicate | Checkers | Tracing +type analyzer = Capture | Compile | Infer | Eradicate | Checkers | Tracing (** Association list of analyzers and their names *) val string_to_analyzer : (string * analyzer) list @@ -270,6 +273,8 @@ val read_optional_json_file : string -> (Yojson.Basic.json, string) result val write_json_to_file : string -> Yojson.Basic.json -> unit +val with_process_in: string -> (in_channel -> 'a) -> 'a + val failwithf : ('a, Format.formatter, unit, 'b) format4 -> 'a val invalid_argf : ('a, Format.formatter, unit, 'b) format4 -> 'a