Add OCaml toplevel driver executable

Summary:
This diff changes the toplevel 'infer' executable from the current
python script to an OCaml binary.  Currently this executable only parses
command line arguments, sets up environment variables, and invokes the
existing python script.  This improves infer's command-line and
configuration interface, since passing arguments to the frontends or
backend no longer requires manually setting environment variables, and
arguments for the toplevel can now also be specified in .inferconfig.
Simplification and migration of functionality from the python script is
left for the future.

Reviewed By: martinoluca, jvillard

Differential Revision: D3450662

fbshipit-source-id: 1b52302

.gitignore

@@ -71,6 +71,7 @@ buck-out/
 /infer/bin/InferUnit
 /infer/bin/Typeprop
 /infer/bin/infer
+/infer/lib/infer
 /infer/bin/inferTraceBugs
 /infer/src/backend/version.ml
 /infer/models/java/models/

FILES.md

@@ -2,7 +2,7 @@
 
 ## Top-level commands
 
-*infer* : Main command to run Infer. It's a python script. Check out the docs for instructions on how to use it.
+*infer* : Main command to run Infer. Check out the docs for instructions on how to use it.
 
 *inferTest* : Shell script for running Infer's tests. Uses Buck for running the tests.
 Usage: inferTest {c, objc, java} for the tests about the analysis of C, Objective-C, or Java files.

Makefile

@@ -22,7 +22,12 @@ TARGETS_TO_TEST := $(shell echo $(TARGETS_TO_TEST))
 
 all: infer inferTraceBugs
 
-$(INFER_BIN_RELPATH) $(INFERTRACEBUGS_BIN_RELPATH):
+$(INFER_BIN_SYMLINK):
+	($(REMOVE) $@ && \
+	 cd $(@D) && \
+	 $(LN_S) ../lib/$(@F) $(@F))
+
+$(INFERTRACEBUGS_BIN_RELPATH):
 	($(REMOVE) $@ && \
 	 cd $(@D) && \
 	 $(LN_S) ../lib/python/$(@F) $(@F))
@@ -36,7 +41,7 @@ ifeq ($(BUILD_C_ANALYZERS),yes)
 src_build: clang_plugin
 endif
 
-infer: $(INFER_BIN_RELPATH) src_build
+infer: $(INFER_BIN_SYMLINK) src_build
 ifeq ($(BUILD_JAVA_ANALYZERS),yes)
 	$(MAKE) -C $(ANNOTATIONS_DIR)
 endif
@@ -213,19 +218,20 @@ endif
 	@for i in $$(find infer/lib/python/inferlib/* -type f); do \
 	  $(INSTALL_DATA) -C $$i $(DESTDIR)$(libdir)/infer/$$i; \
 	done
-	$(INSTALL_PROGRAM) -C       infer/lib/python/infer \
+	$(INSTALL_PROGRAM) -C       infer/lib/python/infer.py \
-	  $(DESTDIR)$(libdir)/infer/infer/lib/python/infer
+	  $(DESTDIR)$(libdir)/infer/infer/lib/python/infer.py
 	$(INSTALL_PROGRAM) -C       infer/lib/python/inferTraceBugs \
 	  $(DESTDIR)$(libdir)/infer/infer/lib/python/inferTraceBugs
+	$(INSTALL_PROGRAM) -C $(INFER_BIN) $(DESTDIR)$(libdir)/infer/infer/lib/
 	$(INSTALL_PROGRAM) -C $(INFERANALYZE_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	$(INSTALL_PROGRAM) -C $(INFERPRINT_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	$(INSTALL_PROGRAM) -C $(INFERSTATS_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	(cd $(DESTDIR)$(libdir)/infer/infer/bin/ && \
 	 $(REMOVE) infer && \
-	 $(LN_S) $(libdir)/infer/infer/lib/python/infer infer)
+	 $(LN_S) $(libdir)/infer/infer/lib/infer infer)
 	(cd $(DESTDIR)$(bindir)/ && \
 	 $(REMOVE) infer && \
-	 $(LN_S) $(libdir)/infer/infer/lib/python/infer infer)
+	 $(LN_S) $(libdir)/infer/infer/lib/infer infer)
 	(cd $(DESTDIR)$(bindir)/ && \
 	 $(REMOVE) inferTraceBugs && \
 	 $(LN_S) $(libdir)/infer/infer/lib/python/inferTraceBugs inferTraceBugs)
@@ -245,7 +251,7 @@ endif
 	$(MAKE) -C $(SRC_DIR) clean
 	$(MAKE) -C $(ANNOTATIONS_DIR) clean
 	$(MAKE) -C $(MODELS_DIR) clean
-	$(REMOVE) $(INFER_BIN_RELPATH) $(INFERTRACEBUGS_BIN_RELPATH)
+	$(REMOVE) $(INFER_BIN_SYMLINK) $(INFERTRACEBUGS_BIN_RELPATH)
 ifeq ($(IS_FACEBOOK_TREE),yes)
 	$(MAKE) -C facebook clean
 endif

Makefile.config.in

@@ -98,10 +98,10 @@ INFERJAVA_BIN = $(BIN_DIR)/InferJava
 INFERSTATS_BIN = $(BIN_DIR)/InferStatsAggregator
 INFERPRINT_BIN = $(BIN_DIR)/InferPrint
 INFERUNIT_BIN = $(BIN_DIR)/InferUnit
-INFER_BIN = $(BIN_DIR)/infer
+INFER_BIN = $(LIB_DIR)/infer
 INFERTRACEBUGS_BIN = $(BIN_DIR)/inferTraceBugs
 # paths relative to $(ROOT_DIR)
-INFER_BIN_RELPATH = infer/bin/infer
+INFER_BIN_SYMLINK = infer/bin/infer
 INFERTRACEBUGS_BIN_RELPATH = infer/bin/inferTraceBugs
 
 ifeq ($(BUILD_JAVA_ANALYZERS),yes)

infer/lib/python/inferlib/analyze.py

@@ -153,7 +153,6 @@ infer_group.add_argument('--infer_cache', metavar='<directory>',
 infer_group.add_argument('-pr', '--project_root',
                          dest='project_root',
                          default=get_pwd(),
-                         type=utils.decode,
                          help='Location of the project root '
                          '(default is current directory)')
 

infer/lib/python/inferlib/bucklib.py

@@ -49,7 +49,7 @@ INFER_SCRIPT = """\
 import subprocess
 import sys
 
-cmd = ['{0}'] + {1} + ['--', 'javac'] + sys.argv[1:]
+cmd = {1} + ['--', 'javac'] + sys.argv[1:]
 subprocess.check_call(cmd)
 """
 

infer/src/Makefile

@@ -45,6 +45,7 @@ endif
 
 #### Backend declarations ####
 
+INFER_MAIN = backend/infer
 INFERANALYZE_MAIN = backend/inferanalyze
 
 #### InferPrint declarations ####
@@ -130,6 +131,7 @@ OCAMLBUILD_ALL = $(OCAMLBUILD_BASE) $(JAVA_OCAMLBUILD_OPTIONS)
 
 # list of ocamlbuild targets common to all build targets -- native version
 INFER_BASE_TARGETS = \
+  $(INFER_MAIN).native \
   $(INFERANALYZE_MAIN).native \
   $(INFERPRINT_MAIN).native \
   $(INFERUNIT_MAIN).native \
@@ -166,6 +168,7 @@ all: infer
 
 infer: init $(INFERPRINT_ATDGEN_STUBS)
 	$(OCAMLBUILD_CONFIG) -build-dir $(INFER_BUILD_DIR) $(INFER_CONFIG_TARGETS)
+	$(COPY) $(INFER_BUILD_DIR)/$(INFER_MAIN).native $(INFER_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERANALYZE_MAIN).native $(INFERANALYZE_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERPRINT_MAIN).native $(INFERPRINT_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(CHECKCOPYRIGHT_MAIN).native $(CHECKCOPYRIGHT_BIN)
@@ -226,7 +229,7 @@ test_build: init $(INFERPRINT_ATDGEN_STUBS) $(CLANG_ATDGEN_STUBS) $(INFER_CLANG_
 $(DEPENDENCIES_DIR)/ocamldot/ocamldot:
 	$(MAKE) -C $(DEPENDENCIES_DIR)/ocamldot
 
-roots:=Inferanalyze CMain JMain Inferprint
+roots:=Infer Inferanalyze CMain JMain Inferprint
 src_dirs:=$(shell find * -type d)
 src_files:=$(shell find $(src_dirs) -regex '.*\.ml\(i\)*' -not -path facebook/scripts/eradicate_stats.ml)
 inc_flags:=$(foreach dir,$(src_dirs),-I $(dir))
@@ -317,8 +320,9 @@ endif
 	$(REMOVE) backend/version.ml
 	$(REMOVE) backend/version.ml.tmp.*
 	$(REMOVE) backend/jsonbug_{j,t}.ml{,i}
-	$(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(INFERLLVM_BIN) $(INFERUNIT_BIN)
+	$(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(STATSAGGREGATOR_BIN)
-	$(REMOVE) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(CHECKCOPYRIGHT_BIN) $(STATSAGGREGATOR_BIN)
+	$(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(INFERLLVM_BIN)
+	$(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN)
 	$(REMOVE) $(CLANG_ATDGEN_STUBS)
 	$(REMOVE) $(INFER_CLANG_FCP_MIRRORED_FILES)
 	$(REMOVE) mod_dep.dot

infer/src/backend/CommandLineOption.ml

@@ -144,7 +144,9 @@ let mk ?(deprecated=[]) ?(exes=[])
   let closure = mk_setter variable in
   let setter str =
     try closure str
-    with _ -> raise (Arg.Bad ("bad value " ^ str ^ " for flag " ^ long)) in
+    with exc ->
+      raise (Arg.Bad ("bad value " ^ str ^ " for flag " ^ long
+                      ^ " (" ^ (Printexc.to_string exc) ^ ")")) in
   let spec = mk_spec setter in
   let doc =
     let default_string = default_to_string default in
@@ -297,7 +299,7 @@ let mk_symbol_seq ?(default=[]) ~symbols ?(deprecated=[]) ~long ?short ?exes ?(m
   let sym_to_str = IList.map (fun (x,y) -> (y,x)) symbols in
   let of_string str = IList.assoc string_equal str symbols in
   let to_string sym = IList.assoc ( = ) sym sym_to_str in
-  mk ~deprecated ~long ?short ~default ?exes ~meta:(" ,-separated sequence" ^ meta) doc
+  mk ~deprecated ~long ?short ~default ?exes ~meta:(",-separated sequence" ^ meta) doc
     ~default_to_string:(fun syms -> String.concat " " (IList.map to_string syms))
     ~mk_setter:(fun var str_seq ->
         var := IList.map of_string (Str.split (Str.regexp_string ",") str_seq))
@@ -314,13 +316,22 @@ let mk_set_from_json ~default ~default_to_string ~f
     ~decode_json:(fun json -> [dashdash long; Yojson.Basic.to_string json])
     ~mk_spec:(fun set -> Arg.String set)
 
+(* A ref to a function used during argument parsing to process anonymous arguments. By default,
+   anonymous arguments are rejected. *)
 let anon_fun = ref (fun arg -> raise (Arg.Bad ("unexpected anonymous argument: " ^ arg)))
 
+(* Clients declare that anonymous arguments are acceptable by calling [mk_anon], which returns a ref
+   storing the anonymous arguments. *)
 let mk_anon () =
   let anon = ref [] in
   anon_fun := (fun arg -> anon := arg :: !anon) ;
   anon
 
+let mk_rest ?(exes=[]) doc =
+  let rest = ref [] in
+  let spec = Arg.Rest (fun arg -> rest := arg :: !rest) in
+  add exes {long = "--"; short = ""; meta = ""; doc; spec; decode_json = fun _ -> []} ;
+  rest
 
 let decode_inferconfig_to_argv path =
   let json = match read_optional_json_file path with
@@ -378,7 +389,7 @@ let prefix_before_rest args =
   prefix_before_rest_ [] args
 
 
-let parse ?(incomplete=false) ?config_file env_var exe_usage =
+let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file env_var exe_usage =
   let curr_speclist = ref []
   and full_speclist = ref []
   in
@@ -446,13 +457,18 @@ let parse ?(incomplete=false) ?config_file env_var exe_usage =
         let json_args = decode_inferconfig_to_argv path in
         (* read .inferconfig first, as both env vars and command-line options overwrite it *)
         json_args @ env_cl_args in
+  let argv = Array.of_list (exe_name :: all_args) in
   let current = ref 0 in
+  (* tests if msg indicates an unknown option, as opposed to a known option with bad argument *)
+  let is_unknown msg =
+    let prefix = exe_name ^ ": unknown option" in
+    prefix = (String.sub msg 0 (String.length prefix)) in
   let rec parse_loop () =
     try
-      Arg.parse_argv_dynamic ~current (Array.of_list (exe_name :: all_args))
+      Arg.parse_argv_dynamic ~current argv curr_speclist !anon_fun usage_msg
-        curr_speclist !anon_fun usage_msg
     with
     | Arg.Bad _ when incomplete -> parse_loop ()
+    | Arg.Bad msg when accept_unknown && is_unknown msg -> !anon_fun argv.(!current) ; parse_loop ()
     | Arg.Bad usage_msg -> Pervasives.prerr_string usage_msg; exit 2
     | Arg.Help usage_msg -> Pervasives.print_string usage_msg; exit 0
   in

infer/src/backend/CommandLineOption.mli

@@ -90,6 +90,13 @@ val mk_anon :
   unit ->
   string list ref
 
+(** [mk_rest doc] defines a [string list ref] of the command line arguments following ["--"], in the
+    reverse order they appeared on the command line.  For example, calling [mk_rest] and parsing
+    [exe -opt1 -opt2 -- arg1 arg2] will result in the returned ref containing [arg2; arg1]. *)
+val mk_rest :
+  ?exes:exe list -> string ->
+  string list ref
+
 (** [parse env_var exe_usage] parses command line arguments as specified by preceding calls to the
     [mk_*] functions, and returns a function that prints the usage message and help text then exits.
     The decoded values of the inferconfig file [config_file], if provided, and of the environment
@@ -97,6 +104,8 @@ val mk_anon :
     the command line supersede those specified in the environment variable, which themselves
     supersede those passed via the config file.  WARNING: An argument will be interpreted as many
     times as it appears in all of the config file, the environment variable, and the command
-    line. *)
+    line. The [env_var] is set to the full set of options parsed.  If [incomplete] is set, unknown
-val parse : ?incomplete:bool -> ?config_file:string ->
+    options are ignored, and [env_var] is not set.  If [accept_unknown] is set, unknown options are
+    treated the same as anonymous arguments. *)
+val parse : ?incomplete:bool -> ?accept_unknown:bool -> ?config_file:string ->
   string -> (exe -> Arg.usage_msg) -> (int -> 'a)

infer/src/backend/config.ml

@@ -35,6 +35,15 @@ let string_of_language = function
 
 type clang_lang = C | CPP | OBJC | OBJCPP
 
+let ml_bucket_symbols = [
+  ("all", `MLeak_all);
+  ("cf", `MLeak_cf);
+  ("arc", `MLeak_arc);
+  ("narc", `MLeak_no_arc);
+  ("cpp", `MLeak_cpp);
+  ("unknown_origin", `MLeak_unknown);
+]
+
 type os_type = Unix | Win32 | Cygwin
 
 type zip_library = {
@@ -43,6 +52,17 @@ type zip_library = {
   models: bool;
 }
 
+let whitelisted_cpp_methods = [
+  ["std"; "move"];
+  ["google"; "CheckNotNull"];
+  ["google"; "GetReferenceableValue"];
+  ["google"; "Check_NEImpl"];
+  ["google"; "Check_LEImpl"];
+  ["google"; "Check_GTImpl"];
+  ["google"; "Check_GEImpl"];
+  ["google"; "Check_EQImpl"]
+]
+
 
 (** Constant configuration values *)
 
@@ -183,7 +203,7 @@ let buck_generated_folder = "buck-out/gen"
 let version_string =
   "Infer version "
   ^ Version.versionString
-  ^ "\nCopyright 2009 - present Facebook. All Rights Reserved.\n"
+  ^ "\nCopyright 2009 - present Facebook. All Rights Reserved."
 
 
 (** System call configuration values *)
@@ -195,27 +215,21 @@ let initial_analysis_time = Unix.time ()
 (** Path to lib/specs to retrieve the default models *)
 let models_dir =
   let bin_dir = Filename.dirname Sys.executable_name in
-  let lib_dir = Filename.concat (Filename.concat bin_dir Filename.parent_dir_name) "lib" in
+  let lib_dir = bin_dir // Filename.parent_dir_name // "lib" in
-  let lib_specs_dir = Filename.concat lib_dir specs_dir_name in
+  let lib_specs_dir = lib_dir // specs_dir_name in
   lib_specs_dir
 
 let cpp_models_dir =
   let bin_dir = Filename.dirname Sys.executable_name in
-  let cpp_models_dir =
+  bin_dir // Filename.parent_dir_name // "models" // "cpp" // "include"
-    Filename.concat (Filename.concat bin_dir Filename.parent_dir_name)
-      "models/cpp/include" in
-  cpp_models_dir
 
-let whitelisted_cpp_methods = [
+let ncpu =
-  ["std"; "move"];
+  try
-  ["google"; "CheckNotNull"];
+    with_process_in
-  ["google"; "GetReferenceableValue"];
+      "getconf _NPROCESSORS_ONLN 2>/dev/null"
-  ["google"; "Check_NEImpl"];
+      (fun chan -> Scanf.fscanf chan "%d" (fun n -> n))
-  ["google"; "Check_LEImpl"];
+  with _ ->
-  ["google"; "Check_GTImpl"];
+    1
-  ["google"; "Check_GEImpl"];
-  ["google"; "Check_EQImpl"]
-]
 
 let os_type = match Sys.os_type with
   | "Win32" -> Win32
@@ -321,6 +335,38 @@ let patterns_of_json_with_key json_key json =
 
 (** Command Line options *)
 
+(* The working directory of the initial invocation of infer, to which paths passed as command line
+   options are relative. *)
+let init_work_dir =
+  try
+    Sys.getenv "INFER_CWD"
+  with Not_found ->
+    let cwd =
+      (* Use PWD if it denotes the same inode as ., to try to avoid paths with symlinks resolved *)
+      (* Approach is borrowed from llvm implementation of *)
+      (* llvm::sys::fs::current_path (implemented in Path.inc file) *)
+      try
+        let pwd = Unix.getenv "PWD" in
+        let pwd_stat = Unix.stat pwd in
+        let dot_stat = Unix.stat "." in
+        if pwd_stat.st_dev = dot_stat.st_dev && pwd_stat.st_ino = dot_stat.st_ino then
+          pwd
+        else
+          Sys.getcwd ()
+      with _ ->
+        Sys.getcwd ()
+    in
+    Unix.putenv "INFER_CWD" cwd ;
+    cwd
+
+(* Resolve relative paths passed as command line options, i.e., with respect to the working
+   directory of the initial invocation of infer. *)
+let resolve path =
+  if Filename.is_relative path then
+    init_work_dir // path
+  else
+    path
+
 (* Declare the phase 1 options *)
 
 let inferconfig_home =
@@ -328,10 +374,10 @@ let inferconfig_home =
     ~exes:CLOpt.[Analyze] ~meta:"dir" "Path to the .inferconfig file"
 
 and project_root =
-  CLOpt.mk_string_opt ~deprecated:["project_root"] ~long:"project-root" ~short:"pr"
+  CLOpt.mk_string_opt ~deprecated:["project_root"; "-project_root"] ~long:"project-root" ~short:"pr"
-    ?default:(if CLOpt.(current_exe = Print) then Some (Sys.getcwd ()) else None)
+    ?default:CLOpt.(match current_exe with Print | Toplevel -> Some (Sys.getcwd ()) | _ -> None)
     ~f:filename_to_absolute
-    ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print]
+    ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print;Toplevel]
     ~meta:"dir" "Specify the root directory of the project"
 
 (* Parse the phase 1 options, ignoring the rest *)
@@ -345,7 +391,7 @@ and project_root = !project_root
 
 let inferconfig_path =
   match inferconfig_home, project_root with
-  | Some dir, _ | _, Some dir -> Filename.concat dir inferconfig_file
+  | Some dir, _ | _, Some dir -> dir // inferconfig_file
   | None, None -> inferconfig_file
 
 (* Proceed to declare and parse the remaining options *)
@@ -361,6 +407,14 @@ let inferconfig_path =
 let anon_args =
   CLOpt.mk_anon ()
 
+and rest =
+  CLOpt.mk_rest
+    ~exes:CLOpt.[Toplevel] "Stop argument processing, use remaining arguments as a build command"
+
+and absolute_paths =
+  CLOpt.mk_bool ~long:"absolute-paths"
+    ~exes:CLOpt.[Toplevel] "Report errors using absolute paths"
+
 (** Flag for abstracting fields of structs
     0 = no
     1 = forget some fields during matching (and so lseg abstraction) *)
@@ -432,10 +486,14 @@ and analysis_stops =
 
 (** Setup the analyzer in order to filter out errors for this analyzer only *)
 and analyzer =
-  CLOpt.mk_symbol_opt ~deprecated:["analyzer"] ~long:"analyzer"
+  CLOpt.mk_symbol_opt ~deprecated:["analyzer"] ~long:"analyzer" ~short:"a"
     "Specify the analyzer for the path filtering"
     ~symbols:Utils.string_to_analyzer
 
+and android_harness =
+  CLOpt.mk_bool ~deprecated:["harness"] ~long:"android-harness"
+    "Create harness to detect bugs involving the Android lifecycle"
+
 (** if true, completely ignore the possibility that errors can be caused by unknown procedures
     during the symbolic execution phase *)
 and angelic_execution =
@@ -455,6 +513,14 @@ and ast_file =
   CLOpt.mk_string_opt ~long:"ast-file" ~short:"ast"
     ~meta:"file" "AST file for the translation"
 
+and blacklist =
+  CLOpt.mk_string_opt ~deprecated:["-blacklist-regex"] ~long:"blacklist"
+    ~meta:"regex" "Skip analysis of files matched by the specified regular expression"
+
+and buck =
+  CLOpt.mk_bool ~long:"buck"
+    "To use when run with buck"
+
 and buck_out =
   CLOpt.mk_string_opt ~long:"buck-out"
     ~exes:CLOpt.[StatsAggregator] ~meta:"dir" "Specify the root directory of buck-out"
@@ -526,7 +592,7 @@ and curr_language =
   var
 
 and cxx_experimental =
-  CLOpt.mk_bool ~deprecated:["cxx-experimental"] ~long:"cxx-experimental"
+  CLOpt.mk_bool ~deprecated:["cxx-experimental"] ~long:"cxx"
     "Analyze C++ methods, still experimental"
 
 and debug, print_types, write_dotty =
@@ -547,6 +613,10 @@ and debug, print_types, write_dotty =
   in
   (debug, print_types, write_dotty)
 
+and debug_exceptions =
+  CLOpt.mk_bool ~long:"debug-exceptions"
+    "Generate lightweight debugging information: just print the internal exceptions during analysis"
+
 (* The classes in the given jar file will be translated. No sources needed *)
 and dependencies =
   CLOpt.mk_bool ~deprecated:["dependencies"] ~long:"dependencies"
@@ -589,17 +659,29 @@ and err_file =
   CLOpt.mk_string ~deprecated:["err_file"] ~long:"err-file" ~default:""
     ~exes:CLOpt.[Analyze] ~meta:"file" "use file for the err channel"
 
-(* Generate harness for Android code *)
+and failures_allowed =
-and harness =
+  CLOpt.mk_bool ~deprecated_no:["-no_failures_allowed"] ~long:"failures-allowed" ~default:true
-  CLOpt.mk_bool ~deprecated:["harness"] ~long:"harness"
+    "Fail if at least one of the translations fails"
-    "Create Android lifecycle harness"
+
+and filtering =
+  CLOpt.mk_bool ~long:"filtering" ~short:"f" ~default:true
+    "Also show the results from the experimental checks. Warning: some checks may contain many \
+     false alarms."
+
+and flavors =
+  CLOpt.mk_bool ~deprecated:["-use-flavors"] ~long:"flavors"
+    "Buck integration using flavors."
+
+and frontend_stats =
+  CLOpt.mk_bool ~long:"frontend-stats" ~short:"fs"
+    "Output statistics about the capture phase to *.o.astlog"
 
 and headers =
-  CLOpt.mk_bool ~deprecated:["headers"] ~deprecated_no:["no_headers"] ~long:"headers"
+  CLOpt.mk_bool ~deprecated:["headers"] ~deprecated_no:["no_headers"] ~long:"headers" ~short:"hd"
-    "Translate code in header files"
+    "Analyze code in header files"
 
 and infer_cache =
-  CLOpt.mk_string_opt ~deprecated:["infer_cache"] ~long:"infer-cache"
+  CLOpt.mk_string_opt ~deprecated:["infer_cache"; "-infer_cache"] ~long:"infer-cache"
     ~f:filename_to_absolute
     ~meta:"dir" "Select a directory to contain the infer cache"
 
@@ -611,6 +693,10 @@ and iterations =
     "Specify the maximum number of operations for each function, expressed as a multiple \
      of symbolic operations"
 
+and jobs =
+  CLOpt.mk_int ~deprecated:["-multicore"] ~long:"jobs" ~short:"j" ~default:ncpu
+    ~exes:CLOpt.[Toplevel] ~meta:"int" "Run the specified number of analysis jobs simultaneously"
+
 (** Flag to tune the final information-loss check used by the join
     0 = use the most aggressive join for preconditions
     1 = use the least aggressive join for preconditions
@@ -624,11 +710,19 @@ and latex =
   CLOpt.mk_option ~deprecated:["latex"] ~long:"latex" ~f:create_outfile
     ~meta:"file.tex" "Print latex report to file.tex"
 
+and load_average =
+  CLOpt.mk_option ~long:"load-average" ~short:"l" ~f:(fun s -> Some (float_of_string s))
+    ~meta:"float"
+    "Do not start new parallel jobs if the load average is greater than that specified"
+
 (** name of the file to load analysis results from *)
 and load_results =
   CLOpt.mk_string_opt ~deprecated:["load_results"] ~long:"load-results"
     ~meta:"file.iar" "Load analysis results from Infer Analysis Results file file.iar"
 
+and llvm =
+  CLOpt.mk_bool ~long:"llvm" "Analyze C or C++ using the experimental LLVM frontend"
+
 (** name of the makefile to create with clusters and dependencies *)
 and makefile =
   CLOpt.mk_string ~deprecated:["makefile"] ~long:"makefile" ~default:""
@@ -641,26 +735,22 @@ and merge =
 
 (** List of obj memory leak buckets to be checked in Objective-C/C++ *)
 and ml_buckets =
-  CLOpt.mk_symbol_seq ~deprecated:["ml_buckets"] ~long:"ml-buckets" ~default:[`MLeak_cf]
+  CLOpt.mk_symbol_seq ~deprecated:["ml_buckets"; "-ml_buckets"] ~long:"ml-buckets"
+    ~default:[`MLeak_cf]
+    ~exes:CLOpt.[Toplevel]
     "Specify the memory leak buckets to be checked: \
      'cf' checks leaks from Core Foundation, \
      'arc' from code compiled in ARC mode, \
      'narc' from code not compiled in ARC mode, \
      'cpp' from C++ code"
-    ~symbols:[
+    ~symbols:ml_bucket_symbols
-      ("all", `MLeak_all);
-      ("cf", `MLeak_cf);
-      ("arc", `MLeak_arc);
-      ("narc", `MLeak_no_arc);
-      ("cpp", `MLeak_cpp);
-      ("unknown_origin", `MLeak_unknown)]
 
 and models_file =
   CLOpt.mk_string_opt ~deprecated:["models"] ~long:"models"
     ~exes:CLOpt.[Analyze;Java] ~meta:"zip file" "add a zip file containing the models"
 
 and models_mode =
-  CLOpt.mk_bool ~deprecated:["models_mode"] ~long:"models-mode"
+  CLOpt.mk_bool ~deprecated:["models_mode"; "-models_mode"] ~long:"models-mode"
     "Mode for computing the models"
 
 and modified_targets =
@@ -759,10 +849,10 @@ and reports_include_ml_loc =
     "Include the location (in the Infer source code) from where reports are generated"
 
 and results_dir =
-  CLOpt.mk_string ~deprecated:["results_dir"] ~long:"results-dir"
+  CLOpt.mk_string ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o"
-    ~default:(Filename.concat (Sys.getcwd ()) "infer-out")
+    ~default:(Sys.getcwd () // "infer-out")
     ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print;StatsAggregator]
-    ~meta:"dir" "Specify the project results directory"
+    ~meta:"dir" "Write results in the specified directory"
 
 (** name of the file to load save results to *)
 and save_results =
@@ -813,13 +903,14 @@ and specs_library =
       let validate_path path =
         if Filename.is_relative path then
           failwith ("Failing because path " ^ path ^ " is not absolute") in
-      match read_file fname with
+      match read_file (resolve fname) with
       | Some pathlist ->
           IList.iter validate_path pathlist;
           pathlist
-      | None -> failwith ("cannot read file " ^ fname)
+      | None -> failwith ("cannot read file " ^ fname ^ " from cwd " ^ (Sys.getcwd ()))
     in
-    CLOpt.mk_string ~deprecated:["specs-dir-list-file"] ~long:"specs-library-index"
+    CLOpt.mk_string ~deprecated:["specs-dir-list-file"; "-specs-dir-list-file"]
+      ~long:"specs-library-index"
       ~default:""
       ~f:(fun file -> specs_library := (read_specs_dir_list_file file) @ !specs_library; "")
       ~exes:CLOpt.[Analyze] ~meta:"file"
@@ -870,7 +961,7 @@ and test_filtering =
     "List all the files Infer can report on (should be call at the root of the project)"
 
 and testing_mode =
-  CLOpt.mk_bool ~deprecated:["testing_mode"] ~long:"testing-mode"
+  CLOpt.mk_bool ~deprecated:["testing_mode"; "-testing_mode"] ~long:"testing-mode" ~short:"tm"
     "Mode for testing, where no headers are translated, and dot files are created"
 
 (** Flag set to enable detailed tracing informatin during error explanation *)
@@ -1046,7 +1137,7 @@ let use_jar_cache = true
 let exe_usage (exe : CLOpt.exe) =
   match exe with
   | Analyze ->
-      version_string ^ "\
+      version_string ^ "\n\
       Usage: InferAnalyze [options]\n\
       Analyze the files captured in the project results directory, \
       which can be specified with the --results-dir option."
@@ -1063,7 +1154,7 @@ let exe_usage (exe : CLOpt.exe) =
        To process all the .specs in the results directory, use option --results-dir \
        Each spec is printed to standard output unless option -q is used."
   | StatsAggregator ->
-      "Usage: InferStatsAggregator --results-dir <dir> --buck-out <dir>\n \
+      "Usage: InferStatsAggregator --results-dir <dir> --buck-out <dir>\n\
        Aggregates all the perf stats generated by Buck on each target"
   | Toplevel ->
       version_string
@@ -1072,7 +1163,8 @@ let post_parsing_initialization () =
   F.set_margin !margin ;
 
   if !version then (
-    F.fprintf F.std_formatter "%s@." version_string ;
+    (* TODO(11791235) change back to stdout once buck integration is fixed *)
+    F.fprintf F.err_formatter "%s@." version_string ;
     exit 0
   );
   if !version_json then (
@@ -1115,7 +1207,7 @@ let post_parsing_initialization () =
           let zip_channel = Zip.open_in zip_filename in
           let entries = Zip.entries zip_channel in
           let extract_entry entry =
-            let dest_file = Filename.concat dest_dir (Filename.basename entry.Zip.filename) in
+            let dest_file = dest_dir // (Filename.basename entry.Zip.filename) in
             if Filename.check_suffix entry.Zip.filename specs_files_suffix
             then Zip.copy_entry_to_file zip_channel entry dest_file in
           IList.iter extract_entry entries;
@@ -1123,7 +1215,7 @@ let post_parsing_initialization () =
         in
         let basename = Filename.basename zip_filename in
         let key = basename ^ string_crc_hex32 zip_filename in
-        let key_dir = Filename.concat cache_dir key in
+        let key_dir = cache_dir // key in
         if (mkdir key_dir)
         then extract_specs key_dir zip_filename;
         specs_library := !specs_library @ [key_dir]
@@ -1156,7 +1248,8 @@ let post_parsing_initialization () =
 
 
 let parse_args_and_return_usage_exit =
-  let usage_exit = CLOpt.parse ~config_file:inferconfig_path "INFER_ARGS" exe_usage in
+  let usage_exit =
+    CLOpt.parse ~accept_unknown:true ~config_file:inferconfig_path "INFER_ARGS" exe_usage in
   if !debug || (!developer_mode && not (CLOpt.current_exe = CLOpt.Print)) then
     prerr_endline
       ((Filename.basename Sys.executable_name) ^ " got args "
@@ -1170,8 +1263,10 @@ let print_usage_exit () =
 
 (** Freeze initialized configuration values *)
 
-let anon_args = !anon_args
+let anon_args = IList.rev !anon_args
+and rest = !rest
 and abs_struct = !abs_struct
+and absolute_paths = !absolute_paths
 and allow_specs_cleanup = !allow_specs_cleanup
 and analysis_path_regex_whitelist_options =
   IList.map (fun (a, b) -> (a, !b)) analysis_path_regex_whitelist_options
@@ -1187,6 +1282,8 @@ and angelic_execution = !angelic_execution
 and arc_mode = objc_arc
 and array_level = !array_level
 and ast_file = !ast_file
+and blacklist = !blacklist
+and buck = !buck
 and buck_out = !buck_out
 and bugs_csv = !bugs_csv
 and bugs_json = !bugs_json
@@ -1201,9 +1298,10 @@ and clang_lang = !clang_lang
 and cluster_cmdline = !cluster
 and code_query = !code_query
 and continue_capture = !continue
-and create_harness = !harness
+and create_harness = !android_harness
 and cxx_experimental = !cxx_experimental
 and debug_mode = !debug
+and debug_exceptions = !debug_exceptions
 and dependency_mode = !dependencies
 and developer_mode = !developer_mode
 and disable_checks = !disable_checks
@@ -1211,12 +1309,20 @@ and dotty_cfg_libs = !dotty_cfg_libs
 and enable_checks = !enable_checks
 and eradicate = !eradicate
 and err_file_cmdline = !err_file
+and failures_allowed = !failures_allowed
+and filtering = !filtering
+and flavors = !flavors
+and frontend_stats = !frontend_stats
+and headers = !headers
 and infer_cache = !infer_cache
 and iterations = !iterations
 and javac_verbose_out = !verbose_out
+and jobs = !jobs
 and join_cond = !join_cond
 and latex = !latex
+and load_average = !load_average
 and load_analysis_results = !load_results
+and llvm = !llvm
 and makefile_cmdline = !makefile
 and merge = !merge
 and ml_buckets = !ml_buckets

infer/src/backend/config.mli

@@ -30,6 +30,10 @@ type pattern =
 
 type clang_lang = C | CPP | OBJC | OBJCPP
 
+val ml_bucket_symbols :
+  (string * [ `MLeak_all | `MLeak_arc | `MLeak_cf | `MLeak_cpp | `MLeak_no_arc | `MLeak_unknown ])
+    list
+
 type os_type = Unix | Win32 | Cygwin
 
 type zip_library = {
@@ -48,9 +52,12 @@ val assign : string
 val attributes_dir_name : string
 val backend_stats_dir_name : string
 val bound_error_allowed_in_procedure_call : bool
+val buck_generated_folder : string
 val buck_infer_deps_file_name : string
-val checks_disabled_by_default : string list
 val captured_dir_name : string
+val checks_disabled_by_default : string list
+val cpp_models_dir : string
+val csl_analysis : bool
 val default_failure_name : string
 val default_in_zip_results_dir : string
 val dotty_output : string
@@ -61,25 +68,24 @@ val idempotent_getters : bool
 val incremental_procs : bool
 val initial_analysis_time : float
 val ivar_attributes : string
+val load_average : float option
+val log_analysis_crash : string
 val log_analysis_file : string
 val log_analysis_procedure : string
-val log_analysis_wallclock_timeout : string
-val log_analysis_symops_timeout : string
 val log_analysis_recursion_timeout : string
-val log_analysis_crash : string
+val log_analysis_symops_timeout : string
-val buck_generated_folder : string
+val log_analysis_wallclock_timeout : string
 val max_recursion : int
 val meet_level : int
 val models_dir : string
-val cpp_models_dir : string
+val ncpu : int
-val whitelisted_cpp_methods : string list list
 val nsnotification_center_checker_backend : bool
 val objc_method_call_semantics : bool
 val os_type : os_type
+val patterns_modeled_expensive : pattern list
 val patterns_never_returning_null : pattern list
-val patterns_suppress_warnings : pattern list
 val patterns_skip_translation : pattern list
-val patterns_modeled_expensive : pattern list
+val patterns_suppress_warnings : pattern list
 val perf_stats_prefix : string
 val proc_stats_filename : string
 val property_attributes : string
@@ -94,11 +100,11 @@ val specs_dir_name : string
 val specs_files_suffix : string
 val start_filename : string
 val taint_analysis : bool
-val csl_analysis : bool
 val trace_absarray : bool
 val undo_join : bool
 val unsafe_unret : string
 val weak : string
+val whitelisted_cpp_methods : string list list
 
 
 (** Configuration values specified by environment variables *)
@@ -115,7 +121,9 @@ val sound_dynamic_dispatch : bool
 (** Configuration values specified by command-line options *)
 
 val anon_args : string list
+val rest : string list
 val abs_struct : int
+val absolute_paths : bool
 val allow_specs_cleanup : bool
 val analysis_path_regex_whitelist : analyzer -> string list
 val analysis_path_regex_blacklist : analyzer -> string list
@@ -126,6 +134,8 @@ val analyzer : analyzer option
 val angelic_execution : bool
 val array_level : int
 val ast_file : string option
+val blacklist : string option
+val buck : bool
 val buck_out : string option
 val bugs_csv : outfile option
 val bugs_json : outfile option
@@ -142,6 +152,7 @@ val continue_capture : bool
 val create_harness : bool
 val cxx_experimental : bool
 val debug_mode : bool
+val debug_exceptions : bool
 val dependency_mode : bool
 val developer_mode : bool
 val disable_checks : string list
@@ -149,12 +160,19 @@ val dotty_cfg_libs : bool
 val enable_checks : string list
 val eradicate : bool
 val err_file_cmdline : string
+val failures_allowed : bool
+val filtering : bool
+val flavors : bool
+val frontend_stats : bool
+val headers : bool
 val infer_cache : string option
 val iterations : int
 val javac_verbose_out : string
+val jobs : int
 val join_cond : int
 val latex : outfile option
 val load_analysis_results : string option
+val llvm : bool
 val makefile_cmdline : string
 val merge : bool
 val ml_buckets :

infer/src/backend/infer.ml

@@ -27,14 +27,13 @@ let set_env_for_clang_wrapper () =
 
 let () =
   set_env_for_clang_wrapper () ;
-  let ( / ) = Filename.concat in
   (* The infer executable in the bin directory is a symbolic link to the real binary in the lib
      directory, so that the python script in the lib directory can be found relative to it. *)
   let real_exe =
     match Unix.readlink Sys.executable_name with
     | link when Filename.is_relative link ->
         (* Sys.executable_name is a relative symbolic link *)
-        (Filename.dirname Sys.executable_name) / link
+        (Filename.dirname Sys.executable_name) // link
     | link ->
         (* Sys.executable_name is an absolute symbolic link *)
         link
@@ -42,12 +41,13 @@ let () =
         (* Sys.executable_name is not a symbolic link *)
         Sys.executable_name
   in
-  let infer_py = (Filename.dirname real_exe) / "python" / "infer.py" in
+  let infer_py = (Filename.dirname real_exe) // "python" // "infer.py" in
   let build_cmd = IList.rev Config.rest in
   let buck = match build_cmd with "buck" :: _ -> true | _ -> false in
   let args_py =
     Array.of_list (
       infer_py ::
+      Config.anon_args @
       (match Config.analyzer with None -> [] | Some a ->
         ["--analyzer"; Utils.string_of_analyzer a]) @
       (match Config.blacklist with
@@ -67,7 +67,7 @@ let () =
          ["--use-flavors"]) @
       (match Config.infer_cache with None -> [] | Some s ->
         ["--infer_cache"; s]) @
-      "--multicore" :: (string_of_int Config.multicore) ::
+      "--multicore" :: (string_of_int Config.jobs) ::
       "--out" :: Config.results_dir ::
       (match Config.project_root with None -> [] | Some pr ->
         ["--project_root"; pr]) @

infer/src/backend/infer.mli

@@ -0,0 +1,12 @@
+(*
+ * Copyright (c) 2016 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *)
+
+open! Utils
+
+(** Top-level driver that orchestrates build system integration, frontends, and backend *)

infer/src/backend/utils.ml

@@ -387,6 +387,8 @@ let do_outf outf_opt f =
 let close_outf outf =
   close_out outf.out_c
 
+let ( // ) = Filename.concat
+
 (** convert a filename to absolute path and normalize by removing occurrences of "." and ".." *)
 module FileNormalize = struct
   let rec fname_to_list_rev fname =
@@ -405,7 +407,7 @@ module FileNormalize = struct
   (* concatenate a list of strings representing a path into a filename *)
   let rec list_to_fname base path = match path with
     | [] -> base
-    | x :: path' -> list_to_fname (Filename.concat base x) path'
+    | x :: path' -> list_to_fname (base // x) path'
 
   (* normalize a path where done_l is a reversed path from the root already normalized *)
   (* and todo_l is the path still to normalize *)
@@ -429,7 +431,7 @@ module FileNormalize = struct
     let is_relative = Filename.is_relative fname in
     let must_normalize = fname_contains_current_parent fname in
     let simple_case () =
-      if is_relative then Filename.concat (Unix.getcwd ()) fname
+      if is_relative then Unix.getcwd () // fname
       else fname in
     if must_normalize then begin
       let done_l, todo_l =
@@ -473,7 +475,7 @@ let filename_to_relative root fname =
       String.sub s2 (n1 + 1) (n2 - (n1 + 1))
     else s2 in
   let norm_root = (* norm_root is root without any trailing / *)
-    Filename.concat (Filename.dirname root) (Filename.basename root) in
+    Filename.dirname root // Filename.basename root in
   let remainder = (* remove the path prefix to root including trailing / *)
     string_strict_subtract norm_root fname in
   remainder
@@ -506,7 +508,7 @@ let next compare =
 
 let directory_fold f init path =
   let collect current_dir (accu, dirs) path =
-    let full_path = (Filename.concat current_dir path) in
+    let full_path = current_dir // path in
     try
       if Sys.is_directory full_path then
         (accu, full_path:: dirs)
@@ -528,7 +530,7 @@ let directory_fold f init path =
 
 let directory_iter f path =
   let apply current_dir dirs path =
-    let full_path = (Filename.concat current_dir path) in
+    let full_path = current_dir // path in
     try
       if Sys.is_directory full_path then
         full_path:: dirs
@@ -548,10 +550,11 @@ let directory_iter f path =
   else
     f path
 
-type analyzer = Infer | Eradicate | Checkers | Tracing
+type analyzer = Capture | Compile | Infer | Eradicate | Checkers | Tracing
 
 let string_to_analyzer =
-  [("infer", Infer); ("eradicate", Eradicate); ("checkers", Checkers); ("tracing", Tracing)]
+  [("capture", Capture); ("compile", Compile);
+   ("infer", Infer); ("eradicate", Eradicate); ("checkers", Checkers); ("tracing", Tracing)]
 
 let analyzers = IList.map snd string_to_analyzer
 
@@ -600,6 +603,18 @@ let with_file file ~f =
 let write_json_to_file destfile json =
   with_file destfile ~f:(fun oc -> Yojson.Basic.pretty_to_channel oc json)
 
+let with_process_in command read =
+  let chan = Unix.open_process_in command in
+  let res =
+    try
+      read chan
+    with exc ->
+      Unix.close_process_in chan |> ignore ;
+      raise exc
+  in
+  Unix.close_process_in chan |> ignore ;
+  res
+
 let failwithf fmt =
   Format.kfprintf (fun _ -> failwith (Format.flush_str_formatter ()))
     Format.str_formatter fmt

infer/src/backend/utils.mli

@@ -193,6 +193,9 @@ val copy_file : string -> string -> int option
 (** read a source file and return a list of lines, or None in case of error *)
 val read_file : string -> string list option
 
+(** Filename.concat *)
+val ( // ) : string -> string -> string
+
 (** Convert a filename to an absolute one if it is relative, and normalize "." and ".." *)
 val filename_to_absolute : string -> string
 
@@ -254,7 +257,7 @@ val directory_fold : ('a -> string -> 'a) -> 'a -> string -> 'a
 val directory_iter : (string -> unit) -> string -> unit
 
 (** Various kind of analyzers *)
-type analyzer = Infer | Eradicate | Checkers | Tracing
+type analyzer = Capture | Compile | Infer | Eradicate | Checkers | Tracing
 
 (** Association list of analyzers and their names *)
 val string_to_analyzer : (string * analyzer) list
@@ -270,6 +273,8 @@ val read_optional_json_file : string -> (Yojson.Basic.json, string) result
 
 val write_json_to_file : string -> Yojson.Basic.json -> unit
 
+val with_process_in: string -> (in_channel -> 'a) -> 'a
+
 val failwithf : ('a, Format.formatter, unit, 'b) format4 -> 'a
 
 val invalid_argf : ('a, Format.formatter, unit, 'b) format4 -> 'a