From bf14034289c3a226c89e646c30e6b1cf16a355fe Mon Sep 17 00:00:00 2001 From: Dulma Churchill Date: Mon, 7 Nov 2016 07:35:29 -0800 Subject: [PATCH] [clang] Do compilation database capture in OCaml directly Reviewed By: jberdine Differential Revision: D4124802 fbshipit-source-id: bb41b8b --- .gitignore | 2 +- Makefile | 1 - Makefile.config.in | 1 - infer/lib/python/inferlib/capture/buck.py | 12 +- .../capture/clang-compilation-database.py | 36 ------ infer/src/Makefile | 12 +- infer/src/backend/infer.ml | 120 ++++++++++-------- infer/src/base/CommandLineOption.ml | 5 +- infer/src/base/CommandLineOption.mli | 2 +- infer/src/base/Config.ml | 15 +-- infer/src/base/Config.mli | 2 - infer/src/base/Logging.ml | 3 +- .../CaptureCompilationDatabase.ml} | 14 +- .../src/clang/CaptureCompilationDatabase.mli | 18 +++ 14 files changed, 98 insertions(+), 145 deletions(-) delete mode 100644 infer/lib/python/inferlib/capture/clang-compilation-database.py rename infer/src/{integration/BuckCompilationDatabase.ml => clang/CaptureCompilationDatabase.ml} (93%) create mode 100644 infer/src/clang/CaptureCompilationDatabase.mli diff --git a/.gitignore b/.gitignore index 6be662e86..78492413c 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,7 @@ buck-out/ /infer/bin/InferClang++ /infer/bin/InferJava /infer/bin/InferPrint -/infer/bin/InferBuckCompilationDatabase + /infer/bin/InferUnit /infer/bin/Typeprop /infer/bin/infer diff --git a/Makefile b/Makefile index 3d10531b7..a2c72da57 100644 --- a/Makefile +++ b/Makefile @@ -268,7 +268,6 @@ ifeq ($(BUILD_C_ANALYZERS),yes) $(INSTALL_PROGRAM) -C $(INFERCLANG_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ (cd $(DESTDIR)$(libdir)/infer/infer/bin/ && \ $(LN_S) -f $(INFERCLANG_BIN) $(INFERCLANG_BIN)++) - $(INSTALL_PROGRAM) -C $(INFER_BUCK_COMPILATION_DATABASE_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ endif ifneq ($(XCODE_SELECT),no) @for i in $$(find infer/lib/xcode_wrappers/*); do \ diff --git a/Makefile.config.in b/Makefile.config.in index 7000fda78..4c708ac34 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -97,7 +97,6 @@ INFERCLANG_BIN = $(BIN_DIR)/InferClang INFERJAVA_BIN = $(BIN_DIR)/InferJava INFERPRINT_BIN = $(BIN_DIR)/InferPrint INFERUNIT_BIN = $(BIN_DIR)/InferUnit -INFER_BUCK_COMPILATION_DATABASE_BIN = $(BIN_DIR)/InferBuckCompilationDatabase INFER_BIN = $(BIN_DIR)/infer INFERTRACEBUGS_BIN = $(BIN_DIR)/inferTraceBugs INFERTRACEBUGS_BIN_RELPATH = infer/bin/inferTraceBugs diff --git a/infer/lib/python/inferlib/capture/buck.py b/infer/lib/python/inferlib/capture/buck.py index 8913700fc..997f62279 100644 --- a/infer/lib/python/inferlib/capture/buck.py +++ b/infer/lib/python/inferlib/capture/buck.py @@ -89,11 +89,8 @@ class BuckAnalyzer: def capture(self): try: - if self.args.use_flavors and \ - not self.args.use_compilation_database: + if self.args.use_flavors: return self.capture_with_flavors() - elif self.args.use_compilation_database: - return self.capture_with_compilation_database() else: return self.capture_without_flavors() except subprocess.CalledProcessError as exc: @@ -211,13 +208,6 @@ class BuckAnalyzer: merged_reports_path, bugs_out, xml_out) return os.EX_OK - def capture_with_compilation_database(self): - buck_args = self.cmd - cmd = [utils.get_cmd_in_bin_dir('InferBuckCompilationDatabase')] - cmd += ['--'] - cmd += buck_args - return subprocess.check_call(cmd) - def capture_without_flavors(self): # Java is a special case, and we run the analysis from here buck_wrapper = bucklib.Wrapper(self.args, self.cmd) diff --git a/infer/lib/python/inferlib/capture/clang-compilation-database.py b/infer/lib/python/inferlib/capture/clang-compilation-database.py deleted file mode 100644 index 54224dbd4..000000000 --- a/infer/lib/python/inferlib/capture/clang-compilation-database.py +++ /dev/null @@ -1,36 +0,0 @@ -import util - -# Copyright (c) 2016 - present Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. - -from inferlib import config, utils -import subprocess - -MODULE_NAME = 'clang-compilation-database' -MODULE_DESCRIPTION = '''Run analysis of code built with the compilation database proided: -clang-compilation-database db.json''' - -# This creates an empty argparser for the module, which provides only -# description/usage information and no arguments. -create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) - - -def gen_instance(*args): - return ClangCompilationDatabase(*args) - - -class ClangCompilationDatabase: - def __init__(self, args, cmd): - self.args = args - self.cmd = cmd - - def capture(self): - args = self.cmd - cmd = [utils.get_cmd_in_bin_dir('InferBuckCompilationDatabase')] - cmd += ['--clang-compilation-database', args[1]] - print(cmd) - return subprocess.check_call(cmd) diff --git a/infer/src/Makefile b/infer/src/Makefile index cd4286ed8..eb0e04872 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -85,11 +85,6 @@ UNIT_SOURCES = unit INFERUNIT_MAIN = $(UNIT_SOURCES)/inferunit -#### Infer integration declarations #### -INTEGRATION_SOURCES = integration - -BUCK_COMPILATION_DATABASE_MAIN = $(INTEGRATION_SOURCES)/BuckCompilationDatabase - #### Java declarations #### JAVA_OCAMLBUILD_OPTIONS = -pkgs javalib,ptrees,sawja @@ -162,7 +157,6 @@ INFER_BASE_TARGETS = \ INFER_ALL_TARGETS = $(INFER_BASE_TARGETS) \ $(INFERJAVA_MAIN).native \ $(INFERCLANG_MAIN).native \ - $(BUCK_COMPILATION_DATABASE_MAIN).native # configure-aware ocamlbuild commands and targets OCAMLBUILD_CONFIG = $(OCAMLBUILD_BASE) @@ -175,7 +169,6 @@ DEPENDENCIES += java endif ifeq ($(BUILD_C_ANALYZERS),yes) INFER_CONFIG_TARGETS += $(INFERCLANG_MAIN).native -INFER_CONFIG_TARGETS += $(BUCK_COMPILATION_DATABASE_MAIN).native DEPENDENCIES += clang endif @@ -197,7 +190,6 @@ endif ifeq ($(BUILD_C_ANALYZERS),yes) $(COPY) $(INFER_BUILD_DIR)/$(INFERCLANG_MAIN).native $(INFERCLANG_BIN) cd $(INFER_BUILD_DIR) && $(LN_S) -f InferClang InferClang++ && cd - - $(COPY) $(INFER_BUILD_DIR)/$(BUCK_COMPILATION_DATABASE_MAIN).native $(INFER_BUCK_COMPILATION_DATABASE_BIN) endif ifeq ($(ENABLE_OCAML_ANNOT),yes) rsync -a --include '*/' --include '*.annot' --exclude '*' $(INFER_BUILD_DIR)/ $(ANNOT_DIR)/ @@ -246,7 +238,7 @@ rei: %.rei : %.mli refmt -assume-explicit-arity -heuristics-file unary.txt -parse ml -print re $< > $*.rei -roots:=Infer InferAnalyzeExe InferClang JMain InferPrintExe BuckCompilationDatabase StatsAggregator +roots:=Infer InferAnalyzeExe InferClang JMain InferPrintExe StatsAggregator clusters:=base clang java IR src_dirs:=$(shell find * -type d) @@ -380,7 +372,7 @@ endif $(REMOVE) checkers/stacktree_{j,t}.ml{,i} $(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) - $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(INFER_BUCK_COMPILATION_DATABASE_BIN) + $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(REMOVE) $(CLANG_ATDGEN_STUBS) $(REMOVE) $(INFER_CLANG_FCP_MIRRORED_FILES) $(REMOVE) mod_dep.dot diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index e82db8bcc..a6796120e 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -90,58 +90,74 @@ let run_command cmd_list after_wait = exit exit_code ) -let capture build_cmd = function - | build_mode -> - let in_buck_mode = build_mode = Buck in - let infer_py = Config.lib_dir // "python" // "infer.py" in - run_command ( - infer_py :: - Config.anon_args @ - (match Config.analyzer with None -> [] | Some a -> - ["--analyzer"; - IList.assoc (=) a (IList.map (fun (n,a) -> (a,n)) Config.string_to_analyzer)]) @ - (match Config.blacklist with - | Some s when in_buck_mode -> ["--blacklist-regex"; s] - | _ -> []) @ - (if not Config.create_harness then [] else - ["--android-harness"]) @ - (if not Config.buck then [] else - ["--buck"]) @ - (match Config.java_jar_compiler with None -> [] | Some p -> - ["--java-jar-compiler"; p]) @ - (match IList.rev Config.buck_build_args with - | args when in_buck_mode -> - IList.map (fun arg -> ["--Xbuck"; "'" ^ arg ^ "'"]) args |> IList.flatten - | _ -> []) @ - (if not Config.continue_capture then [] else - ["--continue"]) @ - (if not Config.debug_mode then [] else - ["--debug"]) @ - (if not Config.debug_exceptions then [] else - ["--debug-exceptions"]) @ - (if Config.filtering then [] else - ["--no-filtering"]) @ - (if not Config.flavors || not in_buck_mode then [] else - ["--use-flavors"]) @ - (if Option.is_none Config.use_compilation_database || not in_buck_mode then [] else - ["--use-compilation-database"]) @ - "-j" :: (string_of_int Config.jobs) :: - "-l" :: (string_of_float Config.load_average) :: - (if not Config.pmd_xml then [] else - ["--pmd-xml"]) @ - ["--project-root"; Config.project_root] @ - (if not Config.reactive_mode then [] else - ["--reactive"]) @ - "--out" :: Config.results_dir :: - (match Config.xcode_developer_dir with None -> [] | Some d -> - ["--xcode-developer-dir"; d]) @ - (if Config.rest = [] then [] else - ("--" :: build_cmd)) - ) (fun exit_code -> - if exit_code = Config.infer_py_argparse_error_exit_code then - (* swallow infer.py argument parsing error *) - Config.print_usage_exit () - ) +let capture build_cmd build_mode = + let analyze_cmd = "analyze" in + let is_analyze_cmd cmd = + match cmd with + | [cmd] when cmd = analyze_cmd -> true + | _ -> false in + let build_cmd = + match build_mode with + | Buck when Option.is_some Config.use_compilation_database -> + let json_cdb = CaptureCompilationDatabase.get_compilation_database_files_buck () in + CaptureCompilationDatabase.capture_files_in_database json_cdb; + [analyze_cmd] + | ClangCompilationDatabase -> + (match Config.rest with + | arg::_ -> CaptureCompilationDatabase.capture_files_in_database [arg] + | _ -> failwith("Errror parsing arguments. Please, pass the compilation \ + database json file as in \ + infer -- clang-compilation-database file.json.")); + [analyze_cmd] + | _ -> build_cmd in + let in_buck_mode = build_mode = Buck in + let infer_py = Config.lib_dir // "python" // "infer.py" in + run_command ( + infer_py :: + Config.anon_args @ + (match Config.analyzer with None -> [] | Some a -> + ["--analyzer"; + IList.assoc (=) a (IList.map (fun (n,a) -> (a,n)) Config.string_to_analyzer)]) @ + (match Config.blacklist with + | Some s when in_buck_mode && not (is_analyze_cmd build_cmd) -> ["--blacklist-regex"; s] + | _ -> []) @ + (if not Config.create_harness then [] else + ["--android-harness"]) @ + (if not Config.buck then [] else + ["--buck"]) @ + (match Config.java_jar_compiler with None -> [] | Some p -> + ["--java-jar-compiler"; p]) @ + (match IList.rev Config.buck_build_args with + | args when in_buck_mode -> + IList.map (fun arg -> ["--Xbuck"; "'" ^ arg ^ "'"]) args |> IList.flatten + | _ -> []) @ + (if not Config.continue_capture then [] else + ["--continue"]) @ + (if not Config.debug_mode then [] else + ["--debug"]) @ + (if not Config.debug_exceptions then [] else + ["--debug-exceptions"]) @ + (if Config.filtering then [] else + ["--no-filtering"]) @ + (if not Config.flavors || not in_buck_mode || is_analyze_cmd build_cmd then [] else + ["--use-flavors"]) @ + "-j" :: (string_of_int Config.jobs) :: + "-l" :: (string_of_float Config.load_average) :: + (if not Config.pmd_xml then [] else + ["--pmd-xml"]) @ + ["--project-root"; Config.project_root] @ + (if not Config.reactive_mode then [] else + ["--reactive"]) @ + "--out" :: Config.results_dir :: + (match Config.xcode_developer_dir with None -> [] | Some d -> + ["--xcode-developer-dir"; d]) @ + (if Config.rest = [] then [] else + ("--" :: build_cmd)) + ) (fun exit_code -> + if exit_code = Config.infer_py_argparse_error_exit_code then + (* swallow infer.py argument parsing error *) + Config.print_usage_exit () + ) let analyze = function | Buck when Config.use_compilation_database = None -> diff --git a/infer/src/base/CommandLineOption.ml b/infer/src/base/CommandLineOption.ml index af266faf6..ea2a49ae6 100644 --- a/infer/src/base/CommandLineOption.ml +++ b/infer/src/base/CommandLineOption.ml @@ -17,12 +17,11 @@ module YBU = Yojson.Basic.Util (** Each command line option may appear in the --help list of any executable, these tags are used to specify which executables for which an option will be documented. *) -type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel +type exe = Analyze | Clang | Interactive | Java | Print | Toplevel (** Association list of executable (base)names to their [exe]s. *) let exes = [ - ("InferBuckCompilationDatabase", BuckCompilationDatabase); ("InferAnalyze", Analyze); ("InferClang", Clang); ("InferJava", Java); @@ -594,7 +593,7 @@ let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file current_exe e let exe_name = Sys.executable_name in let should_parse_cl_args = match current_exe with | Clang | Interactive -> false - | Analyze | BuckCompilationDatabase | Java | Print | Toplevel -> true in + | Analyze | Java | Print | Toplevel -> true in let env_cl_args = if should_parse_cl_args then prepend_to_argv env_args else env_args in diff --git a/infer/src/base/CommandLineOption.mli b/infer/src/base/CommandLineOption.mli index 4c0887a69..76ffb9f53 100644 --- a/infer/src/base/CommandLineOption.mli +++ b/infer/src/base/CommandLineOption.mli @@ -11,7 +11,7 @@ open! Utils -type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel +type exe = Analyze | Clang | Interactive | Java | Print | Toplevel (** Association list of executable (base)names to their [exe]s. *) val exes : (string * exe) list diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index d8cc82644..0df7f8cb3 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -476,7 +476,7 @@ let anon_args = and rest = CLOpt.mk_rest - ~exes:CLOpt.[Toplevel;BuckCompilationDatabase] + ~exes:CLOpt.[Toplevel] "Stop argument processing, use remaining arguments as a build command" and abs_struct = @@ -598,7 +598,7 @@ and buck = and buck_build_args = CLOpt.mk_string_list ~long:"Xbuck" - ~exes:CLOpt.[Toplevel;BuckCompilationDatabase] + ~exes:CLOpt.[Toplevel] "Pass values as command-line arguments to invocations of `buck build` (Buck flavors only)" and buck_out = @@ -678,11 +678,6 @@ and clang_biniou_file = CLOpt.mk_path_opt ~long:"clang-biniou-file" ~exes:CLOpt.[Clang] ~meta:"file" "Specify a file containing the AST of the program, in biniou format" -and clang_compilation_database = - CLOpt.mk_path_opt ~long:"clang-compilation-database" - ~exes:CLOpt.[BuckCompilationDatabase] ~meta:"file" - "Specify a json file containing a clang compilation database to be used for the analysis" - and clang_frontend_action = CLOpt.mk_symbol_opt ~long:"clang-frontend-action" ~exes:CLOpt.[Clang] @@ -1295,11 +1290,6 @@ let exe_usage (exe : CLOpt.exe) = "Usage: InferAnalyze [options]\n\ Analyze the files captured in the project results directory, which can be specified with \ the --results-dir option." - | BuckCompilationDatabase -> - "Usage: BuckCompilationDatabase --Xbuck //target \n\ - Runs buck with the flavor compilation-database or uber-compilation-database. It then \n\ - reads the compilation database emited in json and runs the capture in parallel for \n\ - those commands" | Clang -> "Usage: internal script to capture compilation commands from clang and clang++. \n\ You shouldn't need to call this directly." @@ -1411,7 +1401,6 @@ and check_duplicate_symbols = !check_duplicate_symbols and checkers = !checkers and checkers_repeated_calls = !checkers_repeated_calls and clang_biniou_file = !clang_biniou_file -and clang_compilation_database = !clang_compilation_database and clang_include_to_override = !clang_include_to_override and cluster_cmdline = !cluster and continue_capture = !continue diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index ebacb5c7a..ded2b606c 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -156,7 +156,6 @@ val checkers : bool val checkers_enabled : bool val checkers_repeated_calls : bool val clang_biniou_file : string option -val clang_compilation_database : string option val clang_frontend_action_string : string val clang_frontend_do_capture : bool val clang_frontend_do_lint : bool @@ -309,7 +308,6 @@ val run_with_abs_val_equal_zero : ('a -> 'b) -> 'a -> 'b val allow_leak : bool ref - (** Command Line Interface Documentation *) val print_usage_exit : unit -> 'a diff --git a/infer/src/base/Logging.ml b/infer/src/base/Logging.ml index 740ecbb37..b5aab6942 100644 --- a/infer/src/base/Logging.ml +++ b/infer/src/base/Logging.ml @@ -19,7 +19,6 @@ module F = Format let log_dir_of_current_exe (current_exe : CommandLineOption.exe) = match current_exe with | Analyze -> "analyze" - | BuckCompilationDatabase -> "buck_compilation_database" | Clang -> "clang" | Interactive -> "interactive" | Java -> "java" @@ -39,7 +38,7 @@ let set_log_file_identifier (current_exe : CommandLineOption.exe) string_opt = match current_exe with | Analyze | Clang -> Config.debug_mode || Config.stats_mode - | BuckCompilationDatabase -> true + | Toplevel -> true | _ -> false in if should_setup_log_files then ( let name_prefix = (match string_opt with diff --git a/infer/src/integration/BuckCompilationDatabase.ml b/infer/src/clang/CaptureCompilationDatabase.ml similarity index 93% rename from infer/src/integration/BuckCompilationDatabase.ml rename to infer/src/clang/CaptureCompilationDatabase.ml index 793469eeb..dd12f15ef 100644 --- a/infer/src/integration/BuckCompilationDatabase.ml +++ b/infer/src/clang/CaptureCompilationDatabase.ml @@ -125,7 +125,7 @@ let should_add_file_to_cdb changed_files file_path = | None -> true (** Computes the compilation database files. *) -let get_compilation_database_files () = +let get_compilation_database_files_buck () = let cmd = IList.rev_append Config.rest (IList.rev Config.buck_build_args) in match cmd with | buck :: build :: args -> @@ -155,21 +155,11 @@ let get_compilation_database_files () = let cmd = String.concat " " cmd in Process.print_error_and_exit "Incorrect buck command: %s. Please use buck build " cmd -let () = +let capture_files_in_database db_json_files = let changed_files = read_files_to_compile () in - let db_json_files = - match Config.clang_compilation_database with - | Some file -> [file] - | None -> - if Option.is_some Config.use_compilation_database then - get_compilation_database_files () - else failwith( - "Either the option clang_compilation_database or the option \ - use_compilation_database should be passed to this module ") in let compilation_database = CompilationDatabase.empty () in IList.iter (CompilationDatabase.decode_json_file compilation_database (should_add_file_to_cdb changed_files)) db_json_files; - create_dir (Config.results_dir // Config.clang_build_output_dir_name); run_compilation_database compilation_database diff --git a/infer/src/clang/CaptureCompilationDatabase.mli b/infer/src/clang/CaptureCompilationDatabase.mli new file mode 100644 index 000000000..c57711010 --- /dev/null +++ b/infer/src/clang/CaptureCompilationDatabase.mli @@ -0,0 +1,18 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +(** capture_files_in_database file runs the capture of the files for which + we have compilation commands in the database. If the option changed-files-index + is passed, we only capture the files there *) +val capture_files_in_database : string list -> unit + +(** Gets the compilation database files that contain the compilation given by the + buck command. It will be the compilation of the passed targets only or also + the dependencies according to the flag --use-compilation-database deps | no-deps *) +val get_compilation_database_files_buck : unit -> string list