diff --git a/.gitignore b/.gitignore index 6fe2e3ef0..3b781a1ea 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ buck-out/ /infer/bin/InferLLVM /infer/bin/InferPrint /infer/bin/InferStatsAggregator +/infer/bin/InferBuckCompilationDatabase /infer/bin/InferUnit /infer/bin/Typeprop /infer/bin/infer diff --git a/Makefile.config.in b/Makefile.config.in index bd523bf8d..bd8b58154 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -98,6 +98,7 @@ INFERJAVA_BIN = $(BIN_DIR)/InferJava INFERSTATS_BIN = $(BIN_DIR)/InferStatsAggregator INFERPRINT_BIN = $(BIN_DIR)/InferPrint INFERUNIT_BIN = $(BIN_DIR)/InferUnit +INFER_BUCK_COMPILATION_DATABASE_BIN = $(BIN_DIR)/InferBuckCompilationDatabase INFER_BIN = $(LIB_DIR)/infer INFERTRACEBUGS_BIN = $(BIN_DIR)/inferTraceBugs # paths relative to $(ROOT_DIR) diff --git a/infer/lib/python/infer.py b/infer/lib/python/infer.py index 01d53a215..fcf90ddcd 100755 --- a/infer/lib/python/infer.py +++ b/infer/lib/python/infer.py @@ -203,7 +203,9 @@ def main(): global_argparser.print_help() sys.exit(os.EX_OK) - if not (mod_name == 'buck' or mod_name == 'javac'): + buck_not_in_compilation_database_mode = \ + mod_name == 'buck' and not args.use_compilation_database + if not (buck_not_in_compilation_database_mode or mod_name == 'javac'): # Something should be already captured, otherwise analysis would fail if not os.path.exists(os.path.join(args.infer_out, 'captured')): print('There was nothing to analyze, exiting') diff --git a/infer/lib/python/inferlib/capture/buck.py b/infer/lib/python/inferlib/capture/buck.py index f2861b144..8992db20d 100644 --- a/infer/lib/python/inferlib/capture/buck.py +++ b/infer/lib/python/inferlib/capture/buck.py @@ -56,6 +56,12 @@ def create_argparser(group_name=MODULE_NAME): 'and not for Java. Note: this flag should be used ' 'in combination with passing the #infer flavor ' 'to the Buck target.') + group.add_argument('--use-compilation-database', action='store_true', + help='Run Infer analysis through the use of the flavor ' + 'compilation database. Currently this is ' + 'supported only for the cxx_* targets of Buck ' + '- e.g. cxx_library, cxx_binary - and not for ' + 'Java.') group.add_argument('--xcode-developer-dir', help='Specify the path to Xcode developer directory ' '(requires --use-flavors to work)') @@ -84,8 +90,11 @@ class BuckAnalyzer: def capture(self): try: - if self.args.use_flavors: + if self.args.use_flavors and \ + not self.args.use_compilation_database: return self.capture_with_flavors() + elif self.args.use_compilation_database: + return self.capture_with_compilation_database() else: return self.capture_without_flavors() except subprocess.CalledProcessError as exc: @@ -196,6 +205,15 @@ class BuckAnalyzer: issues.print_and_save_errors(merged_reports_path, bugs_out, xml_out) return os.EX_OK + def capture_with_compilation_database(self): + buck_args = self.cmd + cmd = [utils.get_cmd_in_bin_dir('InferBuckCompilationDatabase')] + for arg in buck_args: + cmd += ['--Xbuck'] + [arg] + if self.args.project_root: + cmd += ['--project-root'] + [self.args.project_root] + return subprocess.check_call(cmd) + def capture_without_flavors(self): # Java is a special case, and we run the analysis from here buck_wrapper = bucklib.Wrapper(self.args, self.cmd) diff --git a/infer/src/Makefile b/infer/src/Makefile index 1eb864c28..0bf6c6cea 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -70,6 +70,11 @@ UNIT_SOURCES = unit INFERUNIT_MAIN = $(UNIT_SOURCES)/inferunit +#### Infer integration declarations #### +INTEGRATION_SOURCES = integration + +BUCK_COMPILATION_DATABASE_MAIN = $(INTEGRATION_SOURCES)/BuckCompilationDatabase + #### Java declarations #### JAVA_OCAMLBUILD_OPTIONS = -pkgs javalib,ptrees,sawja @@ -149,7 +154,8 @@ INFER_BASE_TARGETS = \ INFER_ALL_TARGETS = $(INFER_BASE_TARGETS) \ $(INFERJAVA_MAIN).native \ $(INFERCLANG_MAIN).native \ - $(INFERLLVM_MAIN).native + $(INFERLLVM_MAIN).native \ + $(BUCK_COMPILATION_DATABASE_MAIN).native # configure-aware ocamlbuild commands and targets OCAMLBUILD_CONFIG = $(OCAMLBUILD_BASE) @@ -162,6 +168,7 @@ DEPENDENCIES += java endif ifeq ($(BUILD_C_ANALYZERS),yes) INFER_CONFIG_TARGETS += $(INFERCLANG_MAIN).native +INFER_CONFIG_TARGETS += $(BUCK_COMPILATION_DATABASE_MAIN).native DEPENDENCIES += clang endif ifeq ($(BUILD_LLVM_ANALYZERS),yes) @@ -181,6 +188,7 @@ infer: init $(STACKTREE_ATDGEN_STUBS) $(INFERPRINT_ATDGEN_STUBS) $(COPY) $(INFER_BUILD_DIR)/$(CHECKCOPYRIGHT_MAIN).native $(CHECKCOPYRIGHT_BIN) $(COPY) $(INFER_BUILD_DIR)/$(STATSAGGREGATOR_MAIN).native $(STATSAGGREGATOR_BIN) $(COPY) $(INFER_BUILD_DIR)/$(INFERUNIT_MAIN).native $(INFERUNIT_BIN) + $(COPY) $(INFER_BUILD_DIR)/$(BUCK_COMPILATION_DATABASE_MAIN).native $(INFER_BUCK_COMPILATION_DATABASE_BIN) ifeq ($(BUILD_LLVM_ANALYZERS),yes) $(COPY) $(INFER_BUILD_DIR)/$(INFERLLVM_MAIN).native $(INFERLLVM_BIN) endif @@ -189,6 +197,7 @@ ifeq ($(BUILD_JAVA_ANALYZERS),yes) endif ifeq ($(BUILD_C_ANALYZERS),yes) $(COPY) $(INFER_BUILD_DIR)/$(INFERCLANG_MAIN).native $(INFERCLANG_BIN) + $(COPY) $(INFER_BUILD_DIR)/$(BUCK_COMPILATION_DATABASE_MAIN).native $(INFER_BUCK_COMPILATION_DATABASE_BIN) endif ifeq ($(ENABLE_OCAML_ANNOT),yes) rsync -a --include '*/' --include '*.annot' --exclude '*' $(INFER_BUILD_DIR)/ $(ANNOT_DIR)/ @@ -236,7 +245,8 @@ test_build: init $(STACKTREE_ATDGEN_STUBS) $(INFERPRINT_ATDGEN_STUBS) $(CLANG_AT $(DEPENDENCIES_DIR)/ocamldot/ocamldot: $(MAKE) -C $(DEPENDENCIES_DIR)/ocamldot -roots:=Infer InferAnalyze CMain JMain InferPrint +roots:=Infer InferAnalyze CMain JMain InferPrint BuckCompilationDatabase + src_dirs:=$(shell find * -type d) ml_src_files:=$(shell find $(src_dirs) -regex '.*\.ml\(i\)*' -not -path facebook/scripts/eradicate_stats.ml) re_src_files:=$(shell find $(src_dirs) -regex '.*\.re\(i\)*') @@ -360,7 +370,7 @@ endif $(REMOVE) checkers/stacktree_{j,t}.ml{,i} $(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(STATSAGGREGATOR_BIN) $(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(INFERLLVM_BIN) - $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) + $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(INFER_BUCK_COMPILATION_DATABASE_BIN) $(REMOVE) $(CLANG_ATDGEN_STUBS) $(REMOVE) $(INFER_CLANG_FCP_MIRRORED_FILES) $(REMOVE) mod_dep.dot diff --git a/infer/src/backend/CommandLineOption.ml b/infer/src/backend/CommandLineOption.ml index c17603ce9..a0de4a80c 100644 --- a/infer/src/backend/CommandLineOption.ml +++ b/infer/src/backend/CommandLineOption.ml @@ -17,9 +17,12 @@ module YBU = Yojson.Basic.Util (** Each command line option may appear in the --help list of any executable, these tags are used to specify which executables for which an option will be documented. *) -type exe = Analyze | Clang | Java | Llvm | Print | StatsAggregator | Toplevel | Interactive +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Llvm | Print | + StatsAggregator | Toplevel + let exes = [ + ("InferBuckCompilationDatabase", BuckCompilationDatabase); ("InferAnalyze", Analyze); ("InferClang", Clang); ("InferJava", Java); @@ -38,7 +41,6 @@ let current_exe = try IList.assoc string_equal (Filename.basename Sys.executable_name) exes with Not_found -> Toplevel - type desc = { long: string; short: string; meta: string; doc: string; spec: Arg.spec; (** how to go from an option in the json config file to a list of command-line options *) diff --git a/infer/src/backend/CommandLineOption.mli b/infer/src/backend/CommandLineOption.mli index cd9f036f1..741c5974b 100644 --- a/infer/src/backend/CommandLineOption.mli +++ b/infer/src/backend/CommandLineOption.mli @@ -11,7 +11,8 @@ open! Utils -type exe = Analyze | Clang | Java | Llvm | Print | StatsAggregator | Toplevel | Interactive +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Llvm | Print | + StatsAggregator | Toplevel val current_exe : exe diff --git a/infer/src/backend/DB.ml b/infer/src/backend/DB.ml index 14f80082d..22f50530b 100644 --- a/infer/src/backend/DB.ml +++ b/infer/src/backend/DB.ml @@ -387,3 +387,9 @@ let fold_paths_matching ~dir ~p ~init ~f = matcher function p *) let paths_matching dir p = fold_paths_matching ~dir ~p ~init:[] ~f:(fun x xs -> x :: xs) + +let read_changed_files_index = + match Config.changed_files_index with + | None -> None + | Some fname -> + read_file (source_file_to_abs_path (source_file_from_string fname)) diff --git a/infer/src/backend/DB.mli b/infer/src/backend/DB.mli index 32bc16b70..f533e0840 100644 --- a/infer/src/backend/DB.mli +++ b/infer/src/backend/DB.mli @@ -174,3 +174,5 @@ val fold_paths_matching : (** Return all file paths recursively under the given directory which match the given predicate *) val paths_matching : string -> (string -> bool) -> string list + +val read_changed_files_index : string list option diff --git a/infer/src/backend/Process.ml b/infer/src/backend/Process.ml new file mode 100644 index 000000000..f720a541d --- /dev/null +++ b/infer/src/backend/Process.ml @@ -0,0 +1,106 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +open! Utils + +(** Prints information about a unix error *) +let print_unix_error cmd e = + match e with + | Unix.Unix_error(err, _, _) -> + Logging.err "Cannot execute %s : %s\n%!" + cmd (Unix.error_message err) + | _ -> () + +(** Prints an error message to a log file, prints a message saying that the error can be + found in that file, and exits, with default code 1 or a given code. *) +let print_error_and_exit ?(exit_code=1) f el = + Logging.do_err f el; + let log_file = snd (Config.tmp_log_files_of_current_exe ()) in + Logging.stderr "\nAn error occured. Please, find details in %s\n\n%!" log_file; + exit exit_code + +(** Executes a command and catches a potential exeption and prints it. *) +let exec_command cmd args env = + try Unix.execvpe cmd args env + with (Unix.Unix_error _ as e) -> + print_unix_error cmd e + +(** Given an command to be executed, creates a process to execute this command, + and waits for it to terminate. The standard out and error are not redirected. + If the commands fails to execute, prints an error message and exits. *) +let create_process_and_wait cmd = + let pid = Unix.create_process cmd.(0) cmd Unix.stdin Unix.stdout Unix.stderr in + let _, status = Unix.waitpid [] pid in + let exit_code = match status with + | Unix.WEXITED i -> i + | _ -> 1 in + if exit_code <> 0 then + print_error_and_exit ~exit_code:exit_code + "Failed to execute: %s" (String.concat " " (Array.to_list cmd)) + +(** Given a process id and a function that describes the command that the process id + represents, prints a message explaining the command and its status, if in debug or stats mode. + It also prints a dot to show progress of jobs being finished. *) +let print_status f pid (status : Unix. process_status) = + if Config.debug_mode || Config.stats_mode then + (let program = f pid in + match status with + | WEXITED status -> + if status = 0 then + Logging.out "%s OK \n%!" program + else + Logging.err "%s exited with code %d\n%!" program status + | WSIGNALED signal -> + Logging.err "%s killed by signal %d\n%!" program signal + | WSTOPPED _ -> + Logging.err "%s stopped (???)\n%!" program); + Logging.stdout ".%!" + +let start_current_jobs_count () = ref 0 + +let waited_for_jobs = ref 0 + +(** [wait_for_son pid_child f jobs_count] wait for pid_child + and all the other children and update the current jobs count. + Use f to print the job status *) +let rec wait_for_child pid_child f current_jobs_count = + let pid, status = Unix.wait () in + Pervasives.decr current_jobs_count; + Pervasives.incr waited_for_jobs; + print_status f pid status; + if pid <> pid_child then + wait_for_child pid_child f current_jobs_count + +let pid_to_program jobsMap pid = + try + IntMap.find pid jobsMap + with Not_found -> "" + +(** [run_jobs_in_parallel jobs_stack run_job cmd_to_string ] runs the jobs in + the given stack, by spawning the jobs in batches of n, where n is Config.jobs. It + then waits for all those jobs and starts a new batch and so on. cmd_to_string + is used for printing information about the job's status. *) +let run_jobs_in_parallel jobs_stack run_job cmd_to_string = + let run_job () = + let jobs_map = ref IntMap.empty in + let current_jobs_count = start_current_jobs_count () in + while not (Stack.is_empty jobs_stack) do + let job_cmd = Stack.pop jobs_stack in + Pervasives.incr current_jobs_count; + match Unix.fork () with + | 0 -> + run_job job_cmd + | pid_child -> + jobs_map := IntMap.add pid_child (cmd_to_string job_cmd) !jobs_map; + if Stack.length jobs_stack = 0 || !current_jobs_count >= Config.jobs then + wait_for_child pid_child (pid_to_program !jobs_map) current_jobs_count + done in + run_job (); + Logging.stdout ".\n%!"; + Logging.out "Waited for %d jobs" !waited_for_jobs diff --git a/infer/src/backend/Process.mli b/infer/src/backend/Process.mli new file mode 100644 index 000000000..126561c2f --- /dev/null +++ b/infer/src/backend/Process.mli @@ -0,0 +1,34 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +open! Utils + +(** Given an command to be executed, creates a process to execute this command, + and waits for its execution. The standard out and error are not redirected. + If the commands fails to execute, prints an error message and exits. *) +val create_process_and_wait : string array -> unit + +(** Given an command to be executed, creates a process to execute this command, + and waits for its execution. The standard out and error are not redirected. + If the commands fails to execute, prints an error message and exits. *) +val exec_command : string -> string array -> string array -> unit + +(** Prints an error message to a log file, prints a message saying that the error can be + found in that file, and exist, with default code 1 or a given code. *) +val print_error_and_exit : + ?exit_code:int -> ('a -> unit, Format.formatter, unit) format -> 'a -> 'b + +(** Prints information about a unix error *) +val print_unix_error : string -> exn -> unit + +(** [run_jobs_in_parallel jobs_stack run_job cmd_to_string ] runs the jobs in + the given stack, by spawning the jobs in batches of n, where n is Config.jobs. It + then waits for all those jobs and starts a new batch and so on. cmd_to_string + is used for printing information about the job's status. *) +val run_jobs_in_parallel : 'a Stack.t -> ('a -> unit) -> ('a -> string) -> unit diff --git a/infer/src/backend/config.ml b/infer/src/backend/config.ml index 88dce719e..30dc6ec13 100644 --- a/infer/src/backend/config.ml +++ b/infer/src/backend/config.ml @@ -100,6 +100,8 @@ let checks_disabled_by_default = [ "UNSAFE_GUARDED_BY_ACCESS"; ] +let clang_build_output_dir_name = "build_output" + (** Experimental: if true do some specialized analysis of concurrent constructs. *) let csl_analysis = true @@ -146,6 +148,8 @@ let log_analysis_symops_timeout = "S" let log_analysis_recursion_timeout = "R" let log_analysis_crash = "C" +let log_dir_name = "log" + (** Maximum level of recursion during the analysis, after which a timeout is generated *) let max_recursion = 5 @@ -228,16 +232,20 @@ let version_string = Unix.time *) let initial_analysis_time = Unix.time () -(** Path to lib/specs to retrieve the default models *) -let models_dir = +let lib_dir = let bin_dir = Filename.dirname Sys.executable_name in let lib_dir = bin_dir // Filename.parent_dir_name // "lib" in - let lib_specs_dir = lib_dir // specs_dir_name in - lib_specs_dir + lib_dir + +(** Path to lib/specs to retrieve the default models *) +let models_dir = + lib_dir // specs_dir_name let cpp_models_dir = - let bin_dir = Filename.dirname Sys.executable_name in - bin_dir // Filename.parent_dir_name // "models" // "cpp" // "include" + lib_dir // "models" // "cpp" // "include" + +let wrappers_dir = + lib_dir // "wrappers" let ncpu = try @@ -391,7 +399,8 @@ let inferconfig_home = and project_root = CLOpt.mk_string_opt ~deprecated:["project_root"; "-project_root"] ~long:"project-root" ~short:"pr" - ?default:CLOpt.(match current_exe with Print | Toplevel -> Some (Sys.getcwd ()) | _ -> None) + ?default:CLOpt.(match current_exe with Print | Toplevel | StatsAggregator -> + Some (Sys.getcwd ()) | _ -> None) ~f:resolve ~exes:CLOpt.[Analyze;Clang;Java;Llvm;Print;Toplevel] ~meta:"dir" "Specify the root directory of the project" @@ -568,7 +577,7 @@ and buck = and buck_build_args = CLOpt.mk_string_list ~long:"Xbuck" - ~exes:CLOpt.[Toplevel] + ~exes:CLOpt.[Toplevel;BuckCompilationDatabase] "Pass values as command-line arguments to invocations of `buck build` (Buck flavors only)" and buck_out = @@ -1092,6 +1101,11 @@ and unsafe_malloc = ~exes:CLOpt.[Analyze] "Assume that malloc(3) never returns null." +and use_compilation_database = + CLOpt.mk_symbol_opt ~long:"use-compilation-database" + "Buck integration using the compilation database, with or without dependencies." + ~symbols:[("deps", `Deps); ("no-deps", `NoDeps)] + (** Set the path to the javac verbose output *) and verbose_out = CLOpt.mk_string ~deprecated:["verbose_out"] ~long:"verbose-out" ~default:"" @@ -1179,9 +1193,9 @@ let exe_usage (exe : CLOpt.exe) = match exe with | Analyze -> version_string ^ "\n\ - Usage: InferAnalyze [options]\n\ - Analyze the files captured in the project results directory, \ - which can be specified with the --results-dir option." + Usage: InferAnalyze [options]\n\ + Analyze the files captured in the project results directory, \ + which can be specified with the --results-dir option." | Clang -> "Usage: InferClang -c -ast --results-dir [options] \n\ Translate the given files using clang into infer internal representation for later analysis." @@ -1204,6 +1218,11 @@ let exe_usage (exe : CLOpt.exe) = version_string | Interactive -> "Usage: interactive ocaml toplevel. To pass infer config options use env variable" + | BuckCompilationDatabase -> + "Usage: BuckCompilationDatabase --Xbuck //target \n\ + Runs buck with the flavor compilation-database or uber-compilation-database. It then \n\ + reads the compilation database emited in json and runs the capture in parallel for \n\ + those commands" let post_parsing_initialization () = F.set_margin !margin ; @@ -1434,6 +1453,7 @@ and trace_join = !trace_join and trace_rearrange = !trace_rearrange and type_size = !type_size and unsafe_malloc = !unsafe_malloc +and use_compilation_database = !use_compilation_database and whole_seconds = !whole_seconds and worklist_mode = !worklist_mode and write_dotty = !write_dotty @@ -1473,6 +1493,42 @@ let patterns_suppress_warnings = if CLOpt.(current_exe <> Java) then [] else error ("Error: The option " ^ suppress_warnings_annotations_long ^ " was not provided") +(** Name of files for logging the output in the specific executable *) +let log_files_of_current_exe = + let prefix = + match CLOpt.current_exe with + | Analyze -> "analyze" + | BuckCompilationDatabase -> "buck_compilation_database" + | Clang -> "clang" + | Interactive -> "interactive" + | Java -> "java" + | Llvm -> "llvm" + | Print -> "print" + | StatsAggregator -> "stats_agregator" + | Toplevel -> "top_level" in + prefix ^ "_out", prefix ^ "_err" + +(** should_log_exe exe = true means that files for logging in the log folder will be created + and uses of Logging.out or Logging.err will log in those files *) +let should_log_current_exe = + match CLOpt.current_exe with + | Analyze -> debug_mode || stats_mode + | BuckCompilationDatabase -> true + | _ -> false + +let tmp_log_files_of_current_exe () = + let out_name, err_name = log_files_of_current_exe in + let log_dir = results_dir // log_dir_name in + let out_file = + if out_file_cmdline = "" then + Filename.temp_file ~temp_dir:log_dir out_name "" + else out_file_cmdline in + let err_file = + if err_file_cmdline = "" then + Filename.temp_file ~temp_dir:log_dir err_name "" + else err_file_cmdline in + out_file, err_file + (** Global variables *) let set_reference_and_call_function reference value f x = diff --git a/infer/src/backend/config.mli b/infer/src/backend/config.mli index 76eeeaa84..17bfa105a 100644 --- a/infer/src/backend/config.mli +++ b/infer/src/backend/config.mli @@ -68,6 +68,7 @@ val buck_generated_folder : string val buck_infer_deps_file_name : string val captured_dir_name : string val checks_disabled_by_default : string list +val clang_build_output_dir_name : string val cpp_models_dir : string val csl_analysis : bool val default_failure_name : string @@ -90,6 +91,7 @@ val log_analysis_procedure : string val log_analysis_recursion_timeout : string val log_analysis_symops_timeout : string val log_analysis_wallclock_timeout : string +val log_dir_name : string val max_recursion : int val meet_level : int val models_dir : string @@ -119,6 +121,7 @@ val undo_join : bool val unsafe_unret : string val weak : string val whitelisted_cpp_methods : string list list +val wrappers_dir : string (** Configuration values specified by environment variables *) @@ -251,6 +254,7 @@ val trace_join : bool val trace_rearrange : bool val type_size : bool val unsafe_malloc : bool +val use_compilation_database : [ `Deps | `NoDeps ] option val whole_seconds : bool val worklist_mode : int val write_dotty : bool @@ -281,6 +285,7 @@ val nLOC : int ref val pp_simple : bool ref + (** Global variables with initial values specified by command-line options *) val abs_val : int ref @@ -299,3 +304,13 @@ val curr_language : language ref (** Command Line Interface Documentation *) val print_usage_exit : unit -> 'a + +(** Name of files for logging the output in the current executable *) +val log_files_of_current_exe : string * string + +(** Name of current temporary files for logging the output in the current executable *) +val tmp_log_files_of_current_exe : unit -> string * string + +(** should_log_exe = true means that files for logging in the log folder will be created + and uses of Logging.out or Logging.err will log in those files *) +val should_log_current_exe : bool diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index 29e0ce906..687829e4a 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -65,8 +65,8 @@ let () = (if not Config.absolute_paths then [] else ["--absolute-paths"]) @ (match Config.analyzer with None -> [] | Some a -> - ["--analyzer"; - IList.assoc (=) a (IList.map (fun (n,a) -> (a,n)) Config.string_to_analyzer)]) @ + ["--analyzer"; + IList.assoc (=) a (IList.map (fun (n,a) -> (a,n)) Config.string_to_analyzer)]) @ (match Config.blacklist with | Some s when in_buck_mode -> ["--blacklist-regex"; s] | _ -> []) @ @@ -92,20 +92,22 @@ let () = ["--frontend-stats"]) @ (if not Config.flavors || not in_buck_mode then [] else ["--use-flavors"]) @ + (if Option.is_none Config.use_compilation_database || not in_buck_mode then [] else + ["--use-compilation-database"]) @ (match Config.infer_cache with None -> [] | Some s -> - ["--infer_cache"; s]) @ + ["--infer_cache"; s]) @ "-j" :: (string_of_int Config.jobs) :: (match Config.load_average with None -> [] | Some f -> - ["-l"; string_of_float f]) @ + ["-l"; string_of_float f]) @ (if not Config.pmd_xml then [] else ["--pmd-xml"]) @ (if not Config.reactive_mode then [] else ["--reactive"]) @ "--out" :: Config.results_dir :: (match Config.project_root with None -> [] | Some pr -> - ["--project_root"; pr]) @ + ["--project_root"; pr]) @ (match Config.xcode_developer_dir with None -> [] | Some d -> - ["--xcode-developer-dir"; d]) @ + ["--xcode-developer-dir"; d]) @ (if Config.rest = [] then [] else ("--" :: build_cmd)) ) in diff --git a/infer/src/backend/logging.ml b/infer/src/backend/logging.ml index bca328951..936d32bd9 100644 --- a/infer/src/backend/logging.ml +++ b/infer/src/backend/logging.ml @@ -87,23 +87,15 @@ let out_formatter, err_formatter = with Sys_error _ -> failwithf "@.ERROR: cannot open output file %s@." fname in - if Config.developer_mode - && Sys.file_exists Config.results_dir + if Sys.file_exists Config.results_dir && Sys.is_directory Config.results_dir + && Config.should_log_current_exe then - let log_dir_name = "log" in - let analyzer_out_name = "analyzer_out" in - let analyzer_err_name = "analyzer_err" in - let log_dir = Filename.concat Config.results_dir log_dir_name in + let log_dir = Config.results_dir // Config.log_dir_name in create_dir log_dir; - let analyzer_out_file = - if Config.out_file_cmdline = "" then Filename.concat log_dir analyzer_out_name - else Config.out_file_cmdline in - let analyzer_err_file = - if Config.err_file_cmdline = "" then Filename.concat log_dir analyzer_err_name - else Config.err_file_cmdline in - let out_fmt, out_chan = open_output_file analyzer_out_file in - let err_fmt, err_chan = open_output_file analyzer_err_file in + let out_file, err_file = Config.tmp_log_files_of_current_exe () in + let out_fmt, out_chan = open_output_file out_file in + let err_fmt, err_chan = open_output_file err_file in Pervasives.at_exit (fun () -> F.pp_print_flush out_fmt () ; F.pp_print_flush err_fmt () ; @@ -134,19 +126,27 @@ let set_delayed_prints new_delayed_actions = let do_print fmt fmt_string = F.fprintf fmt fmt_string -let do_print_in_developer_mode fmt fmt_string = - if Config.developer_mode then +let do_print_in_debug_mode fmt fmt_string = + if Config.debug_mode || Config.stats_mode then F.fprintf fmt fmt_string else F.ifprintf fmt fmt_string -(** print to the current out stream (note: only prints in developer mode) *) +(** print to the current out stream (note: only prints in debug mode) *) let out fmt_string = - do_print_in_developer_mode out_formatter fmt_string + do_print_in_debug_mode out_formatter fmt_string -(** print to the current err stream (note: only prints in developer mode) *) +(** print to the current out stream *) +let do_out fmt_string = + do_print out_formatter fmt_string + +(** print to the current err stream (note: only prints in debug mode) *) let err fmt_string = - do_print_in_developer_mode err_formatter fmt_string + do_print_in_debug_mode err_formatter fmt_string + +(** print to the current err stream *) +let do_err fmt_string = + do_print err_formatter fmt_string (** print immediately to standard error *) let stderr fmt_string = diff --git a/infer/src/backend/logging.mli b/infer/src/backend/logging.mli index 36563dac4..b3ce1e9f6 100644 --- a/infer/src/backend/logging.mli +++ b/infer/src/backend/logging.mli @@ -75,9 +75,15 @@ val reset_delayed_prints : unit -> unit (** print to the current out stream (note: only prints in developer mode) *) val out : ('a, Format.formatter, unit) format -> 'a -(** print to the current err stream (note: only prints in developer mode) *) +(** print to the current error stream (note: only prints in developer mode) *) val err : ('a, Format.formatter, unit) format -> 'a +(** print to the current out stream *) +val do_out : ('a, Format.formatter, unit) format -> 'a + +(** print to the current err stream *) +val do_err : ('a, Format.formatter, unit) format -> 'a + (** print immediately to standard error *) val stderr : ('a, Format.formatter, unit) format -> 'a diff --git a/infer/src/backend/ondemand.ml b/infer/src/backend/ondemand.ml index 0f9d88ab8..15183150b 100644 --- a/infer/src/backend/ondemand.ml +++ b/infer/src/backend/ondemand.ml @@ -18,12 +18,7 @@ let trace () = Config.from_env_variable "INFER_TRACE_ONDEMAND" (** Read the directories to analyze from the ondemand file. *) let read_dirs_to_analyze () = - let lines_opt = match Config.changed_files_index with - | None -> - None - | Some fname -> - read_file (DB.source_file_to_abs_path (DB.source_file_from_string fname)) in - match lines_opt with + match DB.read_changed_files_index with | None -> None | Some lines -> diff --git a/infer/src/integration/BuckCompilationDatabase.ml b/infer/src/integration/BuckCompilationDatabase.ml new file mode 100644 index 000000000..f97970ad2 --- /dev/null +++ b/infer/src/integration/BuckCompilationDatabase.ml @@ -0,0 +1,196 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +open! Utils +module YBU = Yojson.Basic.Util + +type compilation_data = { + dir : string; + command : string; + args : string list; +} + +let capture_text = + if Config.analyzer = Some Config.Linters then "linting" + else "translating" + +let swap_command cmd = + let clang = "clang" in + let clangplusplus = "clang++" in + if Utils.string_is_suffix clang cmd then + Config.wrappers_dir // clang + else if Utils.string_is_suffix clangplusplus cmd then + Config.wrappers_dir // clangplusplus + else assert false +(* The command in the compilation database json + emited by buck can only be clang or clang++ *) + +(** Read the files to compile from the changed files index. *) +let read_files_to_compile () = + let changed_files = DB.SourceFileSet.empty in + match DB.read_changed_files_index with + | None -> + (match Config.changed_files_index with + | Some index -> + Process.print_error_and_exit "Error reading the changed files index %s.\n%!" index + | None -> changed_files) + | Some lines -> + IList.fold_left + (fun changed_files line -> + DB.SourceFileSet.add (DB.abs_source_file_from_path line) changed_files) + changed_files lines + +(** Add file to compilation database, only if it is in changed_files. *) +let add_file_to_compilation_database file_path cmd_options changed_files compilation_database = + let add_file = + match Config.changed_files_index with + | Some _ -> DB.SourceFileSet.mem (DB.source_file_from_string file_path) changed_files + | None -> true in + if add_file then + compilation_database := StringMap.add file_path cmd_options !compilation_database + +(** We have to replace the .o files because the path in buck-out doesn't exist at this point. + Moreover, in debug mode we create debug files in the place where the .o files are created, + so having all that in the results directory is convenient for finding the files and for + scanning the directory for running clang_frontend_stats. *) +let replace_clang_args arg = + if Filename.check_suffix arg ".o" then + let dir = Config.results_dir // Config.clang_build_output_dir_name in + let abbrev_source_file = DB.source_file_encoding (DB.source_file_from_string arg) in + dir // abbrev_source_file + else arg + +(* Doing this argument manipulation here rather than in the wrappers because it seems to + be needed only with this integration.*) +let remove_clang_arg arg args = + if (arg = "-include-pch") || (Filename.check_suffix arg ".gch") + then args else arg :: args + +(** Parse the compilation database json file into the compilationDatabase + map. The json file consists of an array of json objects that contain the file + to be compiled, the directory to be compiled in, and the compilation command as a list + and as a string. We pack this information into the compilationDatabase map, and remove the + clang invocation part, because we will use a clang wrapper. *) +let decode_compilation_database changed_files compilation_database _ path = + let collect_arguments compilation_argument args = + match compilation_argument with + | `String arg -> + let arg' = replace_clang_args arg in + remove_clang_arg arg' args + | _ -> failwith ("Json file doesn't have the expected format") in + let json = Yojson.Basic.from_file path in + let rec parse_json json = + match json with + | `List arguments -> + IList.iter parse_json arguments + | `Assoc [ ("directory", `String dir); + ("file", `String file_path); + ("arguments", `List compilation_arguments); + ("command", `String _) ] -> + (match IList.fold_right collect_arguments compilation_arguments [] with + | [] -> failwith ("Command cannot be empty") + | cmd :: args -> + let wrapper_cmd = swap_command cmd in + let compilation_data = { dir; command = wrapper_cmd; args;} in + add_file_to_compilation_database file_path compilation_data changed_files + compilation_database) + | _ -> + failwith ("Json file doesn't have the expected format") in + parse_json json + +(** The buck targets are assumed to start with //, aliases are not supported. *) +let check_args_for_targets args = + if not (IList.exists (Utils.string_is_prefix "//") args) then + let args_s = String.concat " " args in + Process.print_error_and_exit + "Error reading buck command %s. Please, pass buck targets, aliases are not allowed.\n%!" + args_s + +let add_flavor_to_targets args = + let flavor = + match Config.use_compilation_database with + | Some `Deps -> "#uber-compilation-database" + | Some `NoDeps -> "#compilation-database" + | _ -> assert false (* cannot happen *) in + let process_arg arg = + (* Targets are assumed to start with //, aliases are not allowed *) + if Utils.string_is_prefix "//" arg then arg ^ flavor + else arg in + IList.map process_arg args + +let create_files_stack compilation_database = + let stack = Stack.create () in + let add_to_stack file _ = + Stack.push file stack in + StringMap.iter add_to_stack !compilation_database; + stack + +let run_compilation_file compilation_database file = + try + let compilation_data = StringMap.find file !compilation_database in + Unix.chdir compilation_data.dir; + let args = Array.of_list compilation_data.args in + let env = Array.append + (Unix.environment()) + (Array.of_list [ + "INFER_RESULTS_DIR="^Config.results_dir; + "FCP_RUN_SYNTAX_ONLY=1"]) in + Process.exec_command compilation_data.command args env + with Not_found -> + Process.print_error_and_exit "Failed to find compilation data for %s \n%!" file + +let run_compilation_database compilation_database = + let number_of_files = StringMap.cardinal !compilation_database in + Logging.out "Starting %s %d files \n%!" capture_text number_of_files; + Logging.stdout "Starting %s %d files \n%!" capture_text number_of_files; + let jobsStack = create_files_stack compilation_database in + let capture_text_upper = String.capitalize capture_text in + let job_to_string = fun file -> capture_text_upper ^ " " ^ file in + Process.run_jobs_in_parallel jobsStack (run_compilation_file compilation_database) job_to_string + + +(** Computes the compilation database: a map from a file path to info to compile the file, i.e. + the dir where the compilation should be executed and the arguments to clang.*) +let get_compilation_database changed_files = + let cmd = IList.rev Config.buck_build_args in + match cmd with + | buck :: build :: args -> + (check_args_for_targets args; + let args_with_flavor = add_flavor_to_targets args in + let buck_build = Array.of_list (buck :: build :: args_with_flavor) in + Process.create_process_and_wait buck_build; + let buck_targets_list = buck :: "targets" :: "--show-output" :: args_with_flavor in + let buck_targets = String.concat " " buck_targets_list in + try + match Utils.with_process_in buck_targets Std.input_list with + | [] -> Logging.stdout "There are no files to process, exiting."; exit 0 + | lines -> + let scan_output compilation_database_files chan = + Scanf.sscanf chan "%s %s" + (fun target file -> StringMap.add target file compilation_database_files) in + (* Map from targets to json output *) + let compilation_database_files = IList.fold_left scan_output StringMap.empty lines in + let compilation_database = ref StringMap.empty in + StringMap.iter + (decode_compilation_database changed_files compilation_database) + compilation_database_files; + compilation_database + with Unix.Unix_error (err, _, _) -> + Process.print_error_and_exit + "Cannot execute %s\n%!" + (buck_targets ^ " " ^ (Unix.error_message err))) + | _ -> + let cmd = String.concat " " cmd in + Process.print_error_and_exit "Incorrect buck command: %s. Please use buck build " cmd + +let () = + let changed_files = read_files_to_compile () in + let compilation_database = get_compilation_database changed_files in + DB.create_dir (Config.results_dir // Config.clang_build_output_dir_name); + run_compilation_database compilation_database