From c53bfe3159db961cfcb68fda31911765f9a0c5c7 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Wed, 12 Oct 2016 03:38:01 -0700 Subject: [PATCH] [clang] rewrite plugin attachment logic in OCaml Summary: This changes executions of the former InferClang into a function call. In particular, it can be called several times per execution. The new InferClang must be called as if it was clang, and knows how to run clang with our plugin to get the AST of the source file. Reviewed By: akotulski Differential Revision: D3981017 fbshipit-source-id: 7af6490 --- .../attach_plugin_and_run_clang_frontend.sh | 157 --------------- infer/lib/python/inferlib/capture/util.py | 23 +-- infer/lib/wrappers/clang | 3 +- infer/lib/xcode_wrappers/clang | 5 +- infer/src/Makefile | 10 +- infer/src/backend/infer.ml | 6 - infer/src/base/CommandLineOption.ml | 46 +++-- infer/src/base/CommandLineOption.mli | 6 +- infer/src/base/Config.ml | 34 ++-- infer/src/base/Config.mli | 8 +- infer/src/base/Logging.ml | 4 +- infer/src/base/Utils.mli | 2 + infer/src/clang/Capture.re | 180 ++++++++++++++++++ infer/src/clang/Capture.rei | 10 + infer/src/clang/ClangCommand.re | 153 +++++++++++++++ infer/src/clang/ClangCommand.rei | 55 ++++++ .../InferClang.re} | 139 +++++--------- infer/src/clang/cFrontend_checkers_main.ml | 4 +- infer/src/clang/cFrontend_config.ml | 19 +- infer/src/clang/cFrontend_config.mli | 11 +- infer/src/clang/cFrontend_decl.ml | 5 +- infer/src/clang/cFrontend_utils.ml | 4 +- infer/src/clang/cFrontend_utils.mli | 4 +- infer/src/clang/cLocation.ml | 11 +- infer/src/clang/cLocation.mli | 2 +- infer/src/clang/cMain.ml | 94 --------- infer/src/clang/cMain.mli | 16 -- infer/src/clang/cMethod_signature.ml | 2 +- infer/src/clang/cMethod_signature.mli | 4 +- infer/src/clang/cMethod_trans.ml | 12 +- infer/src/clang/cTrans.ml | 2 +- infer/src/clang/cTrans_models.mli | 2 +- infer/tests/codetoanalyze/cpp/errors/Makefile | 4 - 33 files changed, 557 insertions(+), 480 deletions(-) delete mode 100755 infer/lib/clang_wrappers/attach_plugin_and_run_clang_frontend.sh create mode 100644 infer/src/clang/Capture.re create mode 100644 infer/src/clang/Capture.rei create mode 100644 infer/src/clang/ClangCommand.re create mode 100644 infer/src/clang/ClangCommand.rei rename infer/src/{scripts/InferClangWrapper.re => clang/InferClang.re} (54%) delete mode 100644 infer/src/clang/cMain.ml delete mode 100644 infer/src/clang/cMain.mli diff --git a/infer/lib/clang_wrappers/attach_plugin_and_run_clang_frontend.sh b/infer/lib/clang_wrappers/attach_plugin_and_run_clang_frontend.sh deleted file mode 100755 index f00473fd7..000000000 --- a/infer/lib/clang_wrappers/attach_plugin_and_run_clang_frontend.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2016 - present Facebook, Inc. -# All rights reserved. -# -# This source code is licensed under the BSD style license found in the -# LICENSE file in the root directory of this source tree. An additional grant -# of patent rights can be found in the PATENTS file in the same directory. - -# Given the arguments a `clang -cc1 ...` command, attaches a plugin to -# the clang command, then run our own clang with all the arguments -# (passing through filter_args_and_run_fcp_clang.sh) and pipe the -# output to InferClang. - -#### Configuration #### -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -BIN_DIR="${SCRIPT_DIR}/../../bin" -ETC_DIR="${SCRIPT_DIR}/../../etc" - -# path to the wrapped clang compiler to invoke -CLANG_COMPILER="${SCRIPT_DIR}/filter_args_and_run_fcp_clang" -# extension of the file containing the clang cmd intercepted -CMD_FILE_EXT=".sh" -# path of the plugin to load in clang -PLUGIN_PATH="${SCRIPT_DIR}/../../../facebook-clang-plugins/libtooling/build/FacebookClangPlugin.dylib" -# name of the plugin to use -PLUGIN_NAME="BiniouASTExporter" -# output directory of the plugin -RESULTS_DIR="$FCP_RESULTS_DIR" -# this skips the creation of .o files -SYNTAX_ONLY="$FCP_RUN_SYNTAX_ONLY" -# extra arguments to pass during the execution of the infer frontend -INFER_FRONTEND_ARGS=($FCP_INFER_FRONTEND_ARGS) -# this fails the execution of clang if the frontend fails -REPORT_FRONTEND_FAILURE="$FCP_REPORT_FRONTEND_FAILURE" -# enable debug mode (to get more data saved to disk for future inspections) -DEBUG_MODE="$FCP_DEBUG_MODE" -# specify where is located Apple's clang -APPLE_CLANG="$FCP_APPLE_CLANG" -# whether to amend include search path with C++ model headers -INFER_CXX_MODELS="$FCP_INFER_CXX_MODELS" - -# invariants that this script expects -if [ -z "$RESULTS_DIR" ]; then - echo '$FCP_RESULTS_DIR with the name of the output directory not provided.' 1>&2 - exit 1 -fi -if [ "$1" != "-cc1" ]; then - echo "$0 expects to be run with -cc1" 1>&2 - exit 1 -fi - -# we know the first argument is "-cc1" -shift - - -# Functions -function get_option_argument { - # retrieves the value passed to an argument of a clang command - OPT="$1" - shift - while [ -n "$1" ] && [ "$1" != "$OPT" ]; do shift; done - echo "$2" -} - -function has_flag { - # return if the given flag is part of the given command or not - local FLAG="$1" - shift - while [ -n "$1" ] && [ "$1" != "$FLAG" ]; do shift; done - [ -n "$1" ]; -} - -# Main -INPUT_ARGUMENTS=("$@") - -# -cc1 has to be the first argument or clang will think it runs in driver mode -CLANG_CMD=("${CLANG_COMPILER}${XX}" "-cc1") - -# It's important to place this option before other -isystem options. -if [ -n "$INFER_CXX_MODELS" ]; then - CLANG_CMD+=("-isystem" "${SCRIPT_DIR}/../../models/cpp/include") -fi - -# (t7400979) this is a workaround to avoid that clang crashes when the -fmodules flag -# and the YojsonASTExporter plugin are used. Since the -plugin argument disables -# the generation of .o files, we invoke apple clang again to generate the expected -# artifacts. This will keep xcodebuild plus all the sub-steps happy. -if [ -n "$APPLE_CLANG" ]; then - ADD_PLUGIN_FLAG="-plugin" -else - ADD_PLUGIN_FLAG="-add-plugin" -fi - -CLANG_CMD+=( - "-load" - "${PLUGIN_PATH}" - "$ADD_PLUGIN_FLAG" - "${PLUGIN_NAME}" - "-plugin-arg-${PLUGIN_NAME}" - "-" - "-plugin-arg-${PLUGIN_NAME}" - "PREPEND_CURRENT_DIR=1") - -# add the remaining arguments -CLANG_CMD+=("$@") - -# the source file is at the end of the command, match it with the wanted extensions -SOURCE_FILENAME="${INPUT_ARGUMENTS[${#INPUT_ARGUMENTS[@]} - 1]}" - -if ! [[ "$SOURCE_FILENAME" = /* ]]; then - SOURCE_FILENAME="$(pwd)/$SOURCE_FILENAME" -fi - -# add fsyntax-only to the end of arg list to override previous options -if [ -n "$SYNTAX_ONLY" ]; then - CLANG_CMD+=("-fsyntax-only") -fi - -LANGUAGE=$(get_option_argument "-x" "${INPUT_ARGUMENTS[@]}") -if [ -n "$LANGUAGE" ]; then INFER_FRONTEND_ARGS+=("-x" "$LANGUAGE"); fi -if has_flag "-fobjc-arc" "${INPUT_ARGUMENTS[@]}"; then - INFER_FRONTEND_ARGS+=("-fobjc-arc"); -fi - -INFER_FRONTEND_CMD=( - "${BIN_DIR}/InferClang" - "-c" "$SOURCE_FILENAME" - "-results_dir" "$RESULTS_DIR" - "${INFER_FRONTEND_ARGS[@]}") - -if [ -n "$DEBUG_MODE" ]; then - OBJECT_FILENAME="$(get_option_argument "-o" "${INPUT_ARGUMENTS[@]}")" - # Emit the clang command with the extra args piped to InferClang - echo "${CLANG_CMD[@]} " \ - "| tee ${OBJECT_FILENAME}.biniou " \ - "| ${INFER_FRONTEND_CMD[@]}" \ - > "${OBJECT_FILENAME}${CMD_FILE_EXT}" - echo "bdump -x -d ${ETC_DIR}/clang_ast.dict -w '!!DUMMY!!' ${OBJECT_FILENAME}.biniou " \ - "> ${OBJECT_FILENAME}.bdump" \ - >> "${OBJECT_FILENAME}${CMD_FILE_EXT}" -fi - -# run clang and pipe its output to InferClang, or flush it in case the latter crashes -"${CLANG_CMD[@]}" | \ - ("${INFER_FRONTEND_CMD[@]}" || \ - { EC=$?; cat > /dev/null; exit $EC; }) 2>&1 -STATUSES=("${PIPESTATUS[@]}") -STATUS="${STATUSES[0]}" -INFER_STATUS="${STATUSES[1]}" - -# if clang fails, then fail, otherwise, fail with the frontend's exitcode if required -if [ "$STATUS" == 0 ] && [ -n "$REPORT_FRONTEND_FAILURE" ]; then - STATUS="$INFER_STATUS" -fi - -exit $STATUS diff --git a/infer/lib/python/inferlib/capture/util.py b/infer/lib/python/inferlib/capture/util.py index 7789fc31b..caa35791d 100644 --- a/infer/lib/python/inferlib/capture/util.py +++ b/infer/lib/python/inferlib/capture/util.py @@ -129,26 +129,5 @@ def get_clang_frontend_envvars(args): frontend_args = [] env_vars['INFER_RESULTS_DIR'] = args.infer_out - if args.headers: - frontend_args.append('-headers') - if args.models_mode: - frontend_args.append('-models_mode') - if args.project_root: - frontend_args += ['-project_root', args.project_root] - if args.testing_mode: - frontend_args.append('-testing_mode') - if args.cxx: - frontend_args.append('-cxx-experimental') - env_vars['FCP_INFER_CXX_MODELS'] = '1' - if args.frontend_debug: - frontend_args += ['-debug'] - env_vars['FCP_DEBUG_MODE'] = '1' - if args.frontend_stats: - frontend_args += ['-stats'] - env_vars['FCP_DEBUG_MODE'] = '1' - if args.no_failures_allowed: - env_vars['FCP_REPORT_FRONTEND_FAILURE'] = '1' - - # export an env variable with all the arguments to pass to InferClang - env_vars['FCP_INFER_FRONTEND_ARGS'] = ' '.join(frontend_args) + return env_vars diff --git a/infer/lib/wrappers/clang b/infer/lib/wrappers/clang index afee181ee..110ce6e73 100755 --- a/infer/lib/wrappers/clang +++ b/infer/lib/wrappers/clang @@ -13,7 +13,7 @@ fi if [ "${0%++}" != "$0" ]; then INFER_XX="++"; fi export INFER_XX -FRONTEND_COMMAND=("$SCRIPT_DIR/../clang_wrappers/InferClangWrapper" "$@") +FRONTEND_COMMAND=("$SCRIPT_DIR/../../bin/InferClang" "$@") HOST_COMPILER_COMMAND=("$SCRIPT_DIR/../clang_wrappers/filter_args_and_run_fcp_clang$INFER_XX" "$@") if [ -n "$INFER_COMPILER_WRAPPER_IN_RECURSION" ]; then @@ -22,7 +22,6 @@ if [ -n "$INFER_COMPILER_WRAPPER_IN_RECURSION" ]; then fi else export INFER_COMPILER_WRAPPER_IN_RECURSION="Y" - export FCP_RESULTS_DIR="$INFER_RESULTS_DIR"; "${FRONTEND_COMMAND[@]}" fi diff --git a/infer/lib/xcode_wrappers/clang b/infer/lib/xcode_wrappers/clang index 803d806fa..dfd610e07 100755 --- a/infer/lib/xcode_wrappers/clang +++ b/infer/lib/xcode_wrappers/clang @@ -6,8 +6,7 @@ CLANG_WRAPPERS_PATH="${SCRIPT_PATH}/../clang_wrappers" if [ "${0%++}" != "$0" ]; then INFER_XX="++"; fi export INFER_XX -export FCP_CLANG_COMPILER="${CLANG_WRAPPERS_PATH%/}/filter_args_and_run_fcp_clang$INFER_XX"; -export FCP_RESULTS_DIR="${INFER_RESULTS_DIR}"; +FCP_CLANG_COMPILER="${CLANG_WRAPPERS_PATH%/}/filter_args_and_run_fcp_clang$INFER_XX"; if [ -z "$INFER_RESULTS_DIR" ]; then # this redirects to the compiler without adding any FCP flag @@ -17,4 +16,4 @@ if [ -z "$INFER_RESULTS_DIR" ]; then exit $? fi -"${CLANG_WRAPPERS_PATH%/}/InferClangWrapper" "$@" -fno-cxx-modules +"${CLANG_WRAPPERS_PATH%/}/InferClang" "$@" -fno-cxx-modules diff --git a/infer/src/Makefile b/infer/src/Makefile index 406b4bf65..078eabfc8 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -89,7 +89,7 @@ INFERJAVA_MAIN = $(JAVA_SOURCES)/jMain #### Clang declarations #### CLANG_SOURCES = clang -INFERCLANG_MAIN = $(CLANG_SOURCES)/cMain +INFERCLANG_MAIN = $(CLANG_SOURCES)/InferClang FCP_CLANG_OCAML_BUILD_DIR = $(FCP_CLANG_OCAML_DIR)/build @@ -118,9 +118,6 @@ SCRIPT_SOURCES = scripts CHECKCOPYRIGHT_BIN = $(SCRIPT_DIR)/checkCopyright CHECKCOPYRIGHT_MAIN = $(SCRIPT_SOURCES)/checkCopyright -CLANGWRAPPER_BIN = $(LIB_DIR)/clang_wrappers/InferClangWrapper -CLANGWRAPPER_MAIN = $(SCRIPT_SOURCES)/InferClangWrapper - STATSAGGREGATOR_BIN = $(BIN_DIR)/InferStatsAggregator STATSAGGREGATOR_MAIN = $(SCRIPT_SOURCES)/StatsAggregator @@ -165,7 +162,7 @@ INFER_CONFIG_TARGETS += $(INFERJAVA_MAIN).native DEPENDENCIES += java endif ifeq ($(BUILD_C_ANALYZERS),yes) -INFER_CONFIG_TARGETS += $(INFERCLANG_MAIN).native $(CLANGWRAPPER_MAIN).native +INFER_CONFIG_TARGETS += $(INFERCLANG_MAIN).native INFER_CONFIG_TARGETS += $(BUCK_COMPILATION_DATABASE_MAIN).native DEPENDENCIES += clang endif @@ -187,7 +184,6 @@ ifeq ($(BUILD_JAVA_ANALYZERS),yes) endif ifeq ($(BUILD_C_ANALYZERS),yes) $(COPY) $(INFER_BUILD_DIR)/$(INFERCLANG_MAIN).native $(INFERCLANG_BIN) - $(COPY) $(INFER_BUILD_DIR)/$(CLANGWRAPPER_MAIN).native $(CLANGWRAPPER_BIN) $(COPY) $(INFER_BUILD_DIR)/$(BUCK_COMPILATION_DATABASE_MAIN).native $(INFER_BUCK_COMPILATION_DATABASE_BIN) endif ifeq ($(ENABLE_OCAML_ANNOT),yes) @@ -218,7 +214,7 @@ test_build: init $(STACKTREE_ATDGEN_STUBS) $(INFERPRINT_ATDGEN_STUBS) $(CLANG_AT -cflags -warn-error,$(OCAML_FATAL_WARNINGS) \ $(INFER_ALL_TARGETS:.native=.byte) -roots:=Infer InferAnalyze CMain JMain InferPrint BuckCompilationDatabase +roots:=Infer InferAnalyze InferClang CMain JMain InferPrint BuckCompilationDatabase clusters:=base clang java IR src_dirs:=$(shell find * -type d) diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index c85501040..2d87f8b58 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -16,12 +16,6 @@ let set_env_for_clang_wrapper () = | Some dir -> Unix.putenv "FCP_CLANG_INCLUDE_TO_REPLACE" dir | None -> () ); - if Config.cxx_experimental then - Unix.putenv "FCP_INFER_CXX_MODELS" "1" ; - if Config.debug_mode || Config.frontend_stats then - Unix.putenv "FCP_DEBUG_MODE" "1" ; - if not Config.failures_allowed then - Unix.putenv "FCP_REPORT_FRONTEND_FAILURE" "1" ; () (** as the Config.fail_on_bug flag mandates, exit with error when an issue is reported *) diff --git a/infer/src/base/CommandLineOption.ml b/infer/src/base/CommandLineOption.ml index b08451b7c..7f47e2655 100644 --- a/infer/src/base/CommandLineOption.ml +++ b/infer/src/base/CommandLineOption.ml @@ -17,15 +17,14 @@ module YBU = Yojson.Basic.Util (** Each command line option may appear in the --help list of any executable, these tags are used to specify which executables for which an option will be documented. *) -type exe = Analyze | BuckCompilationDatabase | Clang | ClangWrapper | Interactive | Java | Print - | StatsAggregator | Toplevel +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator + | Toplevel let exes = [ ("InferBuckCompilationDatabase", BuckCompilationDatabase); ("InferAnalyze", Analyze); ("InferClang", Clang); - ("InferClangWrapper", ClangWrapper); ("InferJava", Java); ("InferPrint", Print); ("InferStatsAggregator", StatsAggregator); @@ -234,6 +233,11 @@ let mk ?(deprecated=[]) ?(exes=[]) ) deprecated ; variable +(* arguments passed to Arg.parse_argv_dynamic, susceptible to be modified on the fly when parsing *) +let args_to_parse : string array ref = ref (Array.of_list []) +(* reference used by Arg.parse_argv_dynamic to track the index of the argument being parsed *) +let arg_being_parsed : int ref = ref 0 + type 'a t = ?deprecated:string list -> long:Arg.key -> ?short:Arg.key -> ?exes:exe list -> ?meta:string -> Arg.doc -> @@ -332,6 +336,23 @@ let mk_string ~default ?(f=fun s -> s) ?(deprecated=[]) ~long ?short ?exes ?(met ~decode_json:(string_json_decoder ~long) ~mk_spec:(fun set -> Arg.String set) +let mk_path ~default ?(deprecated=[]) ~long ?short ?exes ?(meta="") doc = + mk ~deprecated ~long ?short ~default ?exes ~meta doc + ~default_to_string:(fun s -> s) + ~mk_setter:(fun var str -> + if Filename.is_relative str then ( + (* Replace relative paths with absolute ones on the fly in the args being parsed. This + assumes that [!arg_being_parsed] points at the option name position in + [!args_to_parse], as is the case e.g. when calling [Arg.parse_argv_dynamic + ~current:arg_being_parsed !args_to_parse ...]. *) + let abs_path = Sys.getcwd () // str in + var := abs_path; + (!args_to_parse).(!arg_being_parsed + 1) <- abs_path; + ) else + var := str) + ~decode_json:(string_json_decoder ~long) + ~mk_spec:(fun set -> Arg.String set) + let mk_string_opt ?default ?(f=fun s -> s) ?(deprecated=[]) ~long ?short ?exes ?(meta="") doc = let default_to_string = function Some s -> s | None -> "" in let f s = Some (f s) in @@ -562,9 +583,8 @@ let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file env_var exe_u (* end transitional support for INFERCLANG_ARGS *) let exe_name = Sys.executable_name in let should_parse_cl_args = match current_exe with - | ClangWrapper | Interactive -> false - | Analyze | BuckCompilationDatabase | Clang | Java | Print | StatsAggregator - | Toplevel -> true in + | Clang | Interactive -> false + | Analyze | BuckCompilationDatabase | Java | Print | StatsAggregator | Toplevel -> true in let env_cl_args = if should_parse_cl_args then prepend_to_argv env_args else env_args in @@ -574,22 +594,26 @@ let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file env_var exe_u let json_args = decode_inferconfig_to_argv path in (* read .inferconfig first, as both env vars and command-line options overwrite it *) json_args @ env_cl_args in - let argv = Array.of_list (exe_name :: all_args) in - let current = ref 0 in + args_to_parse := Array.of_list (exe_name :: all_args); + arg_being_parsed := 0; (* tests if msg indicates an unknown option, as opposed to a known option with bad argument *) let is_unknown msg = let prefix = exe_name ^ ": unknown option" in prefix = (String.sub msg 0 (String.length prefix)) in let rec parse_loop () = try - Arg.parse_argv_dynamic ~current argv curr_speclist !anon_fun usage_msg + Arg.parse_argv_dynamic ~current:arg_being_parsed !args_to_parse curr_speclist !anon_fun + usage_msg with | Arg.Bad _ when incomplete -> parse_loop () - | Arg.Bad msg when accept_unknown && is_unknown msg -> !anon_fun argv.(!current) ; parse_loop () + | Arg.Bad msg when accept_unknown && is_unknown msg -> + !anon_fun !args_to_parse.(!arg_being_parsed); + parse_loop () | Arg.Bad usage_msg -> Pervasives.prerr_string usage_msg; exit 2 | Arg.Help usage_msg -> Pervasives.print_string usage_msg; exit 0 in parse_loop (); if not incomplete then - Unix.putenv env_var (encode_argv_to_env (prefix_before_rest all_args)) ; + Unix.putenv env_var + (encode_argv_to_env (prefix_before_rest (IList.tl (Array.to_list !args_to_parse)))) ; curr_usage diff --git a/infer/src/base/CommandLineOption.mli b/infer/src/base/CommandLineOption.mli index bec6fd5fa..5ab265af5 100644 --- a/infer/src/base/CommandLineOption.mli +++ b/infer/src/base/CommandLineOption.mli @@ -11,8 +11,8 @@ open! Utils -type exe = Analyze | BuckCompilationDatabase | Clang | ClangWrapper | Interactive | Java | Print - | StatsAggregator | Toplevel +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator + | Toplevel val current_exe : exe @@ -66,6 +66,8 @@ val mk_float : default:float -> float ref t val mk_string : default:string -> ?f:(string -> string) -> string ref t +val mk_path : default:string -> string ref t + val mk_string_opt : ?default:string -> ?f:(string -> string) -> string option ref t (** [mk_string_list] defines a [string list ref], initialized to [[]] unless overridden by diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index 4ec922ee8..d1ccc2d1c 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -32,8 +32,6 @@ let clang_frontend_action_symbols = [ ("lint_and_capture", `Lint_and_capture); ] -type clang_lang = C | CPP | OBJC | OBJCPP - type language = Clang | Java let string_of_language = function @@ -245,6 +243,9 @@ let bin_dir = let lib_dir = bin_dir // Filename.parent_dir_name // "lib" +let etc_dir = + bin_dir // Filename.parent_dir_name // "etc" + (** Path to lib/specs to retrieve the default models *) let models_dir = lib_dir // specs_dir_name @@ -409,8 +410,8 @@ let inferconfig_home = and project_root = CLOpt.mk_string_opt ~deprecated:["project_root"; "-project_root"] ~long:"project-root" ~short:"pr" - ?default:CLOpt.(match current_exe with Print | Toplevel | StatsAggregator -> - Some (Sys.getcwd ()) | _ -> None) + ?default:CLOpt.(match current_exe with Print | Toplevel | StatsAggregator | Clang -> + Some init_work_dir | _ -> None) ~f:resolve ~exes:CLOpt.[Analyze;Clang;Java;Print;Toplevel] ~meta:"dir" "Specify the root directory of the project" @@ -629,6 +630,10 @@ and calls_csv = ~exes:CLOpt.[Print] ~meta:"file" "Write individual calls in csv format to a file" +and clang_biniou_file = + CLOpt.mk_string_opt ~long:"clang-biniou-file" ~exes:CLOpt.[Clang] ~meta:"file" + "Specify a file containing the AST of the program, in biniou format" + and changed_files_index = CLOpt.mk_string_opt ~long:"changed-files-index" ~exes:CLOpt.[Toplevel] ~meta:"file" "Specify the file containing the list of files from which reactive analysis should start" @@ -650,12 +655,6 @@ and clang_include_to_override = location of internal compiler headers. This option should specify the path to those headers \ so that infer can use its own clang internal headers instead." -(* Default is objc, since it's the default for clang (at least in Mac OS) *) -and clang_lang = - CLOpt.mk_symbol ~long:"clang-lang" ~short:"x" ~default:OBJC - "Specify language for clang frontend" - ~symbols:[("c", C); ("objective-c", OBJC); ("c++", CPP); ("objective-c++", OBJCPP)] - and _ = CLOpt.mk_string_opt ~deprecated:["classpath"] ~long:"classpath" ~meta:"path" "Specify where to find user class files and annotation processors" @@ -970,7 +969,7 @@ and reports_include_ml_loc = "Include the location in the Infer source code from where reports are generated" and results_dir = - CLOpt.mk_string ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o" + CLOpt.mk_path ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o" ~default:(init_work_dir // "infer-out") ~exes:CLOpt.[Analyze;Clang;Java;Print;StatsAggregator] ~meta:"dir" "Write results and internal files in the specified directory" @@ -998,12 +997,6 @@ and skip_translation_headers = ~exes:CLOpt.[Clang] ~meta:"path prefix" "Ignore headers whose path matches the given prefix" -(** File to translate *) -and source_file = - (* clang-plugin normalizes filenames *) - CLOpt.mk_string_opt ~long:"source-file" ~short:"c" ~f:filename_to_absolute - ~meta:"file" "" - and source_file_copy = CLOpt.mk_string_opt ~deprecated:["source_file_copy"] ~long:"source-file-copy" ~meta:"source_file" "Print the path of the copy of source_file in the results directory" @@ -1229,9 +1222,6 @@ let exe_usage (exe : CLOpt.exe) = reads the compilation database emited in json and runs the capture in parallel for \n\ those commands" | Clang -> - "Usage: InferClang -c -ast --results-dir [options] \n\ - Translate the given files using clang into infer internal representation for later analysis." - | ClangWrapper -> "Usage: internal script to capture compilation commands from clang and clang++. \n\ You shouldn't need to call this directly." | Interactive -> @@ -1392,7 +1382,7 @@ and checkers = !checkers (** should the checkers be run? *) and checkers_enabled = not (!eradicate || !crashcontext || !quandary) - +and clang_biniou_file = !clang_biniou_file and clang_frontend_do_capture, clang_frontend_do_lint = match !clang_frontend_action with | Some `Lint -> false, true (* no capture, lint *) @@ -1405,7 +1395,6 @@ and clang_frontend_do_capture, clang_frontend_do_lint = | _ -> true, true (* capture, lint *) and clang_include_to_override = !clang_include_to_override -and clang_lang = !clang_lang and cluster_cmdline = !cluster and continue_capture = !continue and copy_propagation = !copy_propagation @@ -1475,7 +1464,6 @@ and show_progress_bar = !progress_bar and skip_analysis_in_path = !skip_analysis_in_path and skip_clang_analysis_in_path = !skip_clang_analysis_in_path and skip_translation_headers = !skip_translation_headers -and source_file = !source_file and source_file_copy = !source_file_copy and spec_abs_level = !spec_abs_level and specs_library = !specs_library diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index 6c011b5df..45e26154e 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -22,8 +22,6 @@ type analyzer = Capture | Compile | Infer | Eradicate | Checkers | Tracing val string_to_analyzer : (string * analyzer) list -type clang_lang = C | CPP | OBJC | OBJCPP - type language = Clang | Java val string_of_language : language -> string @@ -63,6 +61,7 @@ val anonymous_block_prefix : string val assign : string val attributes_dir_name : string val backend_stats_dir_name : string +val bin_dir : string val bound_error_allowed_in_procedure_call : bool val buck_generated_folder : string val buck_infer_deps_file_name : string @@ -75,6 +74,7 @@ val csl_analysis : bool val default_failure_name : string val default_in_zip_results_dir : string val dotty_output : string +val etc_dir : string val fail_on_issue_exit_code : int val filter_buckets : bool val frontend_stats_dir_name : string @@ -84,6 +84,7 @@ val incremental_procs : bool val infer_py_argparse_error_exit_code : int val initial_analysis_time : float val ivar_attributes : string +val lib_dir : string val lint_issues_dir_name : string val load_average : float val log_analysis_crash : string @@ -168,11 +169,11 @@ val calls_csv : outfile option val check_duplicate_symbols : bool val checkers : bool val checkers_enabled : bool +val clang_biniou_file : string option val clang_frontend_action_string : string val clang_frontend_do_capture : bool val clang_frontend_do_lint : bool val clang_include_to_override : string option -val clang_lang : clang_lang val cluster_cmdline : string option val continue_capture : bool val copy_propagation : bool @@ -241,7 +242,6 @@ val show_progress_bar : bool val skip_analysis_in_path : string list val skip_clang_analysis_in_path : string list val skip_translation_headers : string list -val source_file : string option val source_file_copy : string option val spec_abs_level : int val specs_library : string list diff --git a/infer/src/base/Logging.ml b/infer/src/base/Logging.ml index 5a67a2bae..98dbeca0c 100644 --- a/infer/src/base/Logging.ml +++ b/infer/src/base/Logging.ml @@ -21,7 +21,6 @@ let log_dir_of_current_exe = | Analyze -> "analyze" | BuckCompilationDatabase -> "buck_compilation_database" | Clang -> "clang" - | ClangWrapper -> "clang_wrapper" | Interactive -> "interactive" | Java -> "java" | Print -> "print" @@ -40,8 +39,7 @@ let set_log_file_identifier string_opt = let should_setup_log_files = match CommandLineOption.current_exe with | Analyze - | Clang - | ClangWrapper -> Config.debug_mode || Config.stats_mode + | Clang -> Config.debug_mode || Config.stats_mode | BuckCompilationDatabase -> true | _ -> false in if should_setup_log_files then ( diff --git a/infer/src/base/Utils.mli b/infer/src/base/Utils.mli index db814585d..bc10fb7b5 100644 --- a/infer/src/base/Utils.mli +++ b/infer/src/base/Utils.mli @@ -282,6 +282,8 @@ val read_optional_json_file : string -> (Yojson.Basic.json, string) result val write_json_to_file : string -> Yojson.Basic.json -> unit +val consume_in : in_channel -> unit + val with_process_in: string -> (in_channel -> 'a) -> ('a * Unix.process_status) val with_process_full: string -> (in_channel -> 'a) -> (in_channel -> 'b) -> diff --git a/infer/src/clang/Capture.re b/infer/src/clang/Capture.re new file mode 100644 index 000000000..b9724cc1e --- /dev/null +++ b/infer/src/clang/Capture.re @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +open! Utils; + + +/** this fails the execution of clang if the frontend fails */ +let report_frontend_failure = not Config.failures_allowed; + + +/** enable debug mode (to get more data saved to disk for future inspections) */ +let debug_mode = Config.debug_mode || Config.frontend_stats; + +let buffer_len = 262143; + +let catch_biniou_buffer_errors f x => + try (f x) { + | Invalid_argument "Bi_inbuf.refill_from_channel" => + Logging.err "WARNING: biniou buffer too short, skipping the file@\n"; + assert false + }; + +/* This function reads the json file in fname, validates it, and encoded in the AST data structure + defined in Clang_ast_t. */ +let validate_decl_from_file fname => + catch_biniou_buffer_errors (Ag_util.Biniou.from_file len::buffer_len Clang_ast_b.read_decl) fname; + +let validate_decl_from_channel chan => + catch_biniou_buffer_errors + (Ag_util.Biniou.from_channel len::buffer_len Clang_ast_b.read_decl) chan; + +let register_perf_stats_report source_file => { + let stats_dir = Filename.concat Config.results_dir Config.frontend_stats_dir_name; + let abbrev_source_file = DB.source_file_encoding source_file; + let stats_file = Config.perf_stats_prefix ^ "_" ^ abbrev_source_file ^ ".json"; + create_dir Config.results_dir; + create_dir stats_dir; + PerfStats.register_report_at_exit (Filename.concat stats_dir stats_file) +}; + +let init_global_state_for_capture_and_linters source_file => { + Logging.set_log_file_identifier (Some (Filename.basename (DB.source_file_to_string source_file))); + register_perf_stats_report source_file; + Config.curr_language := Config.Clang; + DB.Results_dir.init source_file; + Clang_ast_main.reset_cache (); + CFrontend_config.reset_global_state () +}; + +let run_clang_frontend trans_unit_ctx ast_source => { + let init_time = Unix.gettimeofday (); + let print_elapsed () => { + let elapsed = Unix.gettimeofday () -. init_time; + Logging.out "Elapsed: %07.3f seconds.@\n" elapsed + }; + let (ast_filename, ast_decl) = + switch ast_source { + | `File path => (path, validate_decl_from_file path) + | `Pipe chan => ( + "stdin of " ^ DB.source_file_to_string trans_unit_ctx.CFrontend_config.source_file, + validate_decl_from_channel chan + ) + }; + let (decl_index, stmt_index, type_index, ivar_to_property_index) = Clang_ast_main.index_node_pointers ast_decl; + CFrontend_config.pointer_decl_index := decl_index; + CFrontend_config.pointer_stmt_index := stmt_index; + CFrontend_config.pointer_type_index := type_index; + CFrontend_config.ivar_to_property_index := ivar_to_property_index; + CFrontend_config.json := ast_filename; + Logging.out "Clang frontend action is %s@\n" Config.clang_frontend_action_string; + Logging.out + "Start %s of AST from %s@\n" Config.clang_frontend_action_string !CFrontend_config.json; + if Config.clang_frontend_do_lint { + CFrontend_checkers_main.do_frontend_checks trans_unit_ctx ast_decl + }; + if Config.clang_frontend_do_capture { + CFrontend.do_source_file trans_unit_ctx ast_decl + }; + Logging.out "End translation AST file %s... OK!@\n" !CFrontend_config.json; + print_elapsed () +}; + +let run_clang clang_command read => + switch (with_process_in clang_command read) { + | (res, Unix.WEXITED 0) => res + | (_, Unix.WEXITED n) => + /* exit with the same error code as clang in case of compilation failure */ + exit n + | _ => exit 1 + }; + +let run_plugin_and_frontend frontend clang_args => { + let clang_command = ClangCommand.command_to_run (ClangCommand.with_plugin_args clang_args); + if debug_mode { + /* -cc1 clang commands always set -o explicitly */ + let object_filename = Option.get (ClangCommand.value_of_option clang_args "-o"); + /* Emit the clang command with the extra args piped to InferClang */ + let frontend_script_fname = Printf.sprintf "%s.sh" object_filename; + let debug_script_out = open_out frontend_script_fname; + let debug_script_fmt = Format.formatter_of_out_channel debug_script_out; + let biniou_fname = Printf.sprintf "%s.biniou" object_filename; + Format.fprintf debug_script_fmt "%s \\@\n > %s@\n" clang_command biniou_fname; + let infer_clang_options = + String.concat + "^" + ( + ( + try [Unix.getenv "INFER_ARGS"] { + | Not_found => [] + } + ) + @ [ + "--clang-biniou-file", + biniou_fname + ] + ); + Format.fprintf + debug_script_fmt + "INFER_ARGS=\"%s\" %s@\n" + infer_clang_options + (ClangCommand.with_exec Sys.executable_name clang_args |> ClangCommand.command_to_run); + Format.fprintf + debug_script_fmt + "bdump -x -d \"%s/clang_ast.dict\" -w '!!DUMMY!!' %s \\@\n > %s.bdump" + Config.etc_dir + biniou_fname + object_filename; + close_out debug_script_out + }; + run_clang clang_command frontend +}; + +let capture clang_args => { + let source_path = { + let argv = ClangCommand.get_argv clang_args; + /* the source file is always the last argument -cc1 clang commands */ + filename_to_absolute argv.(Array.length argv - 1) + }; + Logging.out "@\n*** Beginning capture of file %s ***@\n" source_path; + if (CLocation.is_file_blacklisted source_path) { + Logging.out "@\n Skip the analysis of source file %s@\n@\n" source_path; + /* We still need to run clang, but we don't have to attach the plugin. */ + run_clang (ClangCommand.command_to_run clang_args) consume_in + } else { + let source_file = CLocation.source_file_from_path source_path; + init_global_state_for_capture_and_linters source_file; + let trans_unit_ctx = { + let clang_langs = + CFrontend_config.[("c", C), ("objective-c", ObjC), ("c++", CPP), ("objective-c++", ObjCPP)]; + let lang = + switch (ClangCommand.value_of_option clang_args "-x") { + | Some lang_opt when IList.mem_assoc string_equal lang_opt clang_langs => + IList.assoc string_equal lang_opt clang_langs + | _ => assert false + }; + {CFrontend_config.source_file: source_file, lang} + }; + Config.arc_mode := ClangCommand.has_flag clang_args "-fobjc-arc"; + try ( + switch Config.clang_biniou_file { + | Some fname => run_clang_frontend trans_unit_ctx (`File fname) + | None => + run_plugin_and_frontend + (fun chan_in => run_clang_frontend trans_unit_ctx (`Pipe chan_in)) clang_args + } + ) { + | exc => + if report_frontend_failure { + raise exc + } + }; + () + } +}; diff --git a/infer/src/clang/Capture.rei b/infer/src/clang/Capture.rei new file mode 100644 index 000000000..1a6565667 --- /dev/null +++ b/infer/src/clang/Capture.rei @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +let capture: ClangCommand.args => unit; diff --git a/infer/src/clang/ClangCommand.re b/infer/src/clang/ClangCommand.re new file mode 100644 index 000000000..268472364 --- /dev/null +++ b/infer/src/clang/ClangCommand.re @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +open! Utils; + +type quoting_style = | DoubleQuotes | SingleQuotes; + +type args = {exec: string, argv: list string, quoting_style: quoting_style}; + +type t = | Assembly of args | CC1 of args | ClangError of string | NonCCCommand of args; + + +/** path of the plugin to load in clang */ +let plugin_path = + Config.bin_dir /\/ + Filename.parent_dir_name /\/ + Filename.parent_dir_name /\/ + "facebook-clang-plugins" /\/ + "libtooling" /\/ + "build" /\/ + "FacebookClangPlugin.dylib"; + +let test_env_var var => + switch (Sys.getenv var) { + | "1" => true + | _ => false + | exception Not_found => false + }; + + +/** name of the plugin to use */ +let plugin_name = "BiniouASTExporter"; + + +/** this skips the creation of .o files */ +let syntax_only = test_env_var "FCP_RUN_SYNTAX_ONLY"; + + +/** specify where is located Apple's clang */ +let apple_clang = test_env_var "FCP_APPLE_CLANG"; + + +/** whether to amend include search path with C++ model headers */ +let infer_cxx_models = Config.cxx_experimental; + +let value_of_argv_option argv opt_name => + IList.fold_left + ( + fun (prev_arg, result) arg => { + let result' = + if (Option.is_some result) { + result + } else if (string_equal opt_name prev_arg) { + Some arg + } else { + None + }; + (arg, result') + } + ) + ("", None) + argv |> snd; + +let value_of_option {argv} => value_of_argv_option argv; + +let has_flag {argv} flag => IList.exists (string_equal flag) argv; + +let mk quoting_style argv => { + let argv_list = Array.to_list argv; + let is_assembly = + /* whether language is set to "assembler" or "assembler-with-cpp" */ + { + let assembly_language = + switch (value_of_argv_option argv_list "-x") { + | Some lang => string_is_prefix "assembler" lang + | _ => false + }; + /* Detect -cc1as or assembly language commands. -cc1as is always the first argument if + present. */ + string_equal argv.(1) "-cc1as" || assembly_language + }; + let args = {exec: argv.(0), argv: IList.tl argv_list, quoting_style}; + if is_assembly { + Assembly args + } else if (argv.(1) == "-cc1") { + CC1 + /* -cc1 is always the first argument if present. */ + args + } else { + NonCCCommand args + } +}; + +let command_to_run {exec, argv, quoting_style} => { + let quote = + switch quoting_style { + | DoubleQuotes => (fun s => "\"" ^ s ^ "\"") + | SingleQuotes => (fun s => "'" ^ s ^ "'") + }; + Printf.sprintf "'%s' %s" exec (IList.map quote argv |> String.concat " ") +}; + +let with_exec exec args => {...args, exec}; + +let with_plugin_args args => { + let cons a b => [a, ...b]; + let do_if cond action x => + if cond { + action x + } else { + x + }; + let rev_args_before = + [] |> + /* -cc1 has to be the first argument or clang will think it runs in driver mode */ + cons "-cc1" |> + /* It's important to place this option before other -isystem options. */ + do_if infer_cxx_models (IList.rev_append ["-isystem", Config.cpp_models_dir]) |> + IList.rev_append [ + "-load", + plugin_path, + /* (t7400979) this is a workaround to avoid that clang crashes when the -fmodules flag and the + YojsonASTExporter plugin are used. Since the -plugin argument disables the generation of .o + files, we invoke apple clang again to generate the expected artifacts. This will keep + xcodebuild plus all the sub-steps happy. */ + if apple_clang { + "-plugin" + } else { + "-add-plugin" + }, + plugin_name, + "-plugin-arg-" ^ plugin_name, + "-", + "-plugin-arg-" ^ plugin_name, + "PREPEND_CURRENT_DIR=1" + ]; + let args_after = [] |> do_if syntax_only (cons "-fsyntax-only"); + {...args, argv: IList.rev_append rev_args_before (args.argv @ args_after)} +}; + +let prepend_arg arg clang_args => {...clang_args, argv: [arg, ...clang_args.argv]}; + +let prepend_args args clang_args => {...clang_args, argv: args @ clang_args.argv}; + +let append_args args clang_args => {...clang_args, argv: clang_args.argv @ args}; + +let get_argv {exec, argv} => Array.of_list [exec, ...argv]; diff --git a/infer/src/clang/ClangCommand.rei b/infer/src/clang/ClangCommand.rei new file mode 100644 index 000000000..3915cbc4d --- /dev/null +++ b/infer/src/clang/ClangCommand.rei @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +type args; + +type t = + | Assembly of args + /** a normalized clang command that runs the assembler */ + | CC1 of args + /** a -cc1 clang command */ + | ClangError of string + | NonCCCommand of args /** other commands (as, ld, ...) */; + +type quoting_style = + | DoubleQuotes /** the arguments are ready to be enclosed in "double quotes" */ + | SingleQuotes /** the arguments are ready to be enclodes in 'single quotes' */; + + +/** [mk qs argv] finds the type of command depending on its arguments [argv]. The quoting style of + the arguments have to be provided, so that the command may be run later on. */ +let mk: quoting_style => array string => t; + + +/** change an args object into a string ready to be passed to a shell to be executed */ +let command_to_run: args => string; + + +/** whether the command has this flag set in its arguments */ +let has_flag: args => string => bool; + + +/** the value passed to an option in the arguments of a command */ +let value_of_option: args => string => option string; + + +/** add the arguments needed to attach the facebook-clang-plugins plugin */ +let with_plugin_args: args => args; + +let prepend_arg: string => args => args; + +let prepend_args: list string => args => args; + +let append_args: list string => args => args; + +let get_argv: args => array string; + + +/** updates the executable to be run */ +let with_exec: string => args => args; diff --git a/infer/src/scripts/InferClangWrapper.re b/infer/src/clang/InferClang.re similarity index 54% rename from infer/src/scripts/InferClangWrapper.re rename to infer/src/clang/InferClang.re index cf7e2d24b..39e88fde3 100644 --- a/infer/src/scripts/InferClangWrapper.re +++ b/infer/src/clang/InferClang.re @@ -12,74 +12,16 @@ open! Utils; -/** where we are */ -let exec_dir = Filename.dirname @@ Sys.executable_name; - -let xx_suffix = - try (Sys.getenv "INFER_XX") { - | Not_found => "" - }; - - -/** script to run our own clang */ -let infer_clang_bin = exec_dir /\/ ("filter_args_and_run_fcp_clang" ^ xx_suffix); - - -/** script to attach the plugin to clang -cc1 commands and run InferClang */ -let clang_cc1_capture = exec_dir /\/ "attach_plugin_and_run_clang_frontend.sh"; - - -/** path to Apple's clang */ -let apple_clang = - try (Some (Sys.getenv "FCP_APPLE_CLANG" ^ xx_suffix)) { - | Not_found => None - }; - -let typed_clang_invocation args => { - let is_assembly = - /* whether language is set to "assembler" or "assembler-with-cpp" */ - { - let assembly_language = - Array.fold_left - ( - fun (prev_arg, b) arg => { - let b' = b || string_equal "-x" prev_arg && string_is_prefix "assembler" arg; - (arg, b') - } - ) - ("", false) - args |> snd; - /* Detect -cc1as or assembly language commands. -cc1as is always the first argument if - present. */ - string_equal args.(1) "-cc1as" || assembly_language - }; - if is_assembly { - `Assembly args - } else if (args.(1) == "-cc1") { - `CC1 - /* -cc1 is always the first argument if present. */ - args - } else { - `NonCCCommand args - } -}; - -let commands_or_errors = - /* commands generated by `clang -### ...` start with ' "/absolute/path/to/binary"' */ - Str.regexp " \"/\\|clang\\(\\|++\\): error:"; - - /** Given a list of arguments for clang [args], return a list of new commands to run according to - the results of `clang -### [args]`. */ -let normalize args => - switch (typed_clang_invocation args) { - | `CC1 args => - Logging.out "InferClangWrapper got toplevel -cc1 command@\n"; - [`CC1 args] - | `NonCCCommand args => - let clang_hashhashhash = - String.concat - " " (IList.map Filename.quote [infer_clang_bin, "-###", ...Array.to_list args |> IList.tl]); + the results of `clang -### [args]`. Assembly commands (eg, clang -cc1as ...) are filtered out, + although the type cannot reflect that fact. */ +let normalize (args: array string) :list ClangCommand.t => + switch (ClangCommand.mk ClangCommand.SingleQuotes args) { + | CC1 args => + Logging.out "InferClang got toplevel -cc1 command@\n"; + [ClangCommand.CC1 args] + | NonCCCommand args => + let clang_hashhashhash = ClangCommand.prepend_arg "-###" args |> ClangCommand.command_to_run; Logging.out "clang -### invocation: %s@\n" clang_hashhashhash; let normalized_commands = ref []; let one_line line => @@ -89,10 +31,13 @@ let normalize args => /* split by whitespace */ Str.split (Str.regexp_string "\" \"") |> Array.of_list |> - typed_clang_invocation + ClangCommand.mk ClangCommand.DoubleQuotes } else { - `ClangError line + ClangCommand.ClangError line }; + let commands_or_errors = + /* commands generated by `clang -### ...` start with ' "/absolute/path/to/binary"' */ + Str.regexp " \"/\\|clang\\(\\|++\\): error:"; let consume_input i => try ( while true { @@ -114,34 +59,37 @@ let normalize args => IList.filter ( fun - | `Assembly asm_cmd => { - Logging.out - "Skipping assembly command %s@\n" (String.concat " " @@ Array.to_list asm_cmd); + | ClangCommand.Assembly asm_cmd => { + Logging.out "Skipping assembly command %s@\n" (ClangCommand.command_to_run asm_cmd); false } | _ => true ) !normalized_commands - | `Assembly _ => + | Assembly _ => /* discard assembly commands -- see above */ - Logging.out "InferClangWrapper got toplevel assembly command@\n"; + Logging.out "InferClang got toplevel assembly command@\n"; [] + | ClangError _ => + /* we cannot possibly get this from the command-line... */ + assert false }; -let execute_clang_command clang_cmd => +let execute_clang_command (clang_cmd: ClangCommand.t) => switch clang_cmd { - | `CC1 args => + | CC1 args => /* this command compiles some code; replace the invocation of clang with our own clang and plugin */ - args.(0) = clang_cc1_capture; - Logging.out "Executing -cc1 command: %s@\n" (String.concat " " @@ Array.to_list args); - Process.create_process_and_wait args - | `ClangError error => + /* reset logging, otherwise we might print into the logs of the previous file that was compiled */ + Logging.set_log_file_identifier None; + Logging.out "Capturing -cc1 command: %s@\n" (ClangCommand.command_to_run args); + Capture.capture args + | ClangError error => /* An error in the output of `clang -### ...`. Outputs the error and fail. This is because `clang -###` pretty much never fails, but warns of failures on stderr instead. */ Logging.err "%s" error; exit 1 - | `Assembly args => + | Assembly args => /* We shouldn't get any assembly command at this point */ ( if Config.debug_mode { @@ -150,19 +98,23 @@ let execute_clang_command clang_cmd => Logging.err } ) - "WARNING: unexpected assembly command: %s@\n" (String.concat " " @@ Array.to_list args) - | `NonCCCommand args => + "WARNING: unexpected assembly command: %s@\n" (ClangCommand.command_to_run args) + | NonCCCommand args => /* Non-compilation (eg, linking) command. Run the command as-is. It will not get captured further since `clang -### ...` will only output commands that invoke binaries using their absolute paths. */ - if Config.debug_mode { - Logging.out "Executing raw command: %s@\n" (String.concat " " @@ Array.to_list args) - }; - Process.create_process_and_wait args + Logging.out "Executing raw command: %s@\n" (ClangCommand.command_to_run args); + Process.create_process_and_wait (ClangCommand.get_argv args) }; let () = { - let args = Sys.argv; + let xx_suffix = + try (Sys.getenv "INFER_XX") { + | Not_found => "" + }; + let args = Array.copy Sys.argv; + /* make sure we don't call ourselves recursively */ + args.(0) = CFrontend_config.clang_bin xx_suffix; let commands = normalize args; if (commands == []) { /* No command to execute after -###, let's execute the original command @@ -173,9 +125,8 @@ let () = { - the user tries to run `infer -- clang -c file_that_does_not_exist.c`. In this case, this will fail with the appropriate error message from clang instead of silently analyzing 0 files. */ - args.(0) = infer_clang_bin; Logging.out - "WARNING: `clang -### ` returned\n an empty set of commands to run and no error. Will run the original command directly:@\n\n %s@\n" + "WARNING: `clang -### ` returned an empty set of commands to run and no error. Will run the original command directly:@\n %s@\n" (String.concat " " @@ Array.to_list args); Process.create_process_and_wait args } else { @@ -183,10 +134,10 @@ let () = { }; /* xcodebuild projects may require the object files to be generated by the Apple compiler, eg to generate precompiled headers compatible with Apple's clang. */ - switch apple_clang { - | None => () - | Some bin => - args.(0) = bin; + switch (Sys.getenv "FCP_APPLE_CLANG") { + | bin => + args.(0) = bin ^ xx_suffix; Process.create_process_and_wait args + | exception Not_found => () } }; diff --git a/infer/src/clang/cFrontend_checkers_main.ml b/infer/src/clang/cFrontend_checkers_main.ml index c835d6a99..6c6fb0651 100644 --- a/infer/src/clang/cFrontend_checkers_main.ml +++ b/infer/src/clang/cFrontend_checkers_main.ml @@ -97,6 +97,6 @@ let do_frontend_checks trans_unit_ctx ast = Logging.out "End linting file %s@\n" (DB.source_file_to_string source_file) | _ -> assert false (* NOTE: Assumes that an AST always starts with a TranslationUnitDecl *) with - | Assert_failure (file, line, column) -> + | Assert_failure (file, line, column) as exn -> Logging.err "Fatal error: exception Assert_failure(%s, %d, %d)@\n%!" file line column; - exit 1 + raise exn diff --git a/infer/src/clang/cFrontend_config.ml b/infer/src/clang/cFrontend_config.ml index f38aa16dd..d46ef46ae 100644 --- a/infer/src/clang/cFrontend_config.ml +++ b/infer/src/clang/cFrontend_config.ml @@ -11,8 +11,10 @@ open! Utils (** Module that contains constants and global state used in the frontend *) +type clang_lang = C | CPP | ObjC | ObjCPP + type translation_unit_context = { - lang : Config.clang_lang; + lang : clang_lang; source_file : DB.source_file } @@ -35,6 +37,9 @@ let cf_bridging_retain = "CFBridgingRetain" let cf_non_null_alloc ="__cf_non_null_alloc" let ckcomponent_cl = "CKComponent" let ckcomponentcontroller_cl = "CKComponentController" + +(** script to run our own clang *) +let clang_bin xx = Config.lib_dir // "clang_wrappers" // "filter_args_and_run_fcp_clang" ^ xx let class_method = "class" let class_type = "Class" let copy = "copy" @@ -87,7 +92,19 @@ let enum_map = ref Clang_ast_main.PointerMap.empty let global_translation_unit_decls : Clang_ast_t.decl list ref = ref [] let ivar_to_property_index = ref Clang_ast_main.PointerMap.empty let json = ref "" +let log_out = ref Format.std_formatter let pointer_decl_index = ref Clang_ast_main.PointerMap.empty let pointer_stmt_index = ref Clang_ast_main.PointerMap.empty let pointer_type_index = ref Clang_ast_main.PointerMap.empty let sil_types_map = ref Clang_ast_types.TypePointerMap.empty + +let reset_global_state () = + enum_map := Clang_ast_main.PointerMap.empty; + global_translation_unit_decls := []; + ivar_to_property_index := Clang_ast_main.PointerMap.empty; + json := ""; + log_out := Format.std_formatter; + pointer_decl_index := Clang_ast_main.PointerMap.empty; + pointer_stmt_index := Clang_ast_main.PointerMap.empty; + pointer_type_index := Clang_ast_main.PointerMap.empty; + sil_types_map := Clang_ast_types.TypePointerMap.empty; diff --git a/infer/src/clang/cFrontend_config.mli b/infer/src/clang/cFrontend_config.mli index 40d5067a4..2176f0b03 100644 --- a/infer/src/clang/cFrontend_config.mli +++ b/infer/src/clang/cFrontend_config.mli @@ -11,8 +11,10 @@ open! Utils (** Module that contains constants and global state used in the frontend *) +type clang_lang = C | CPP | ObjC | ObjCPP + type translation_unit_context = { - lang : Config.clang_lang; + lang : clang_lang; source_file : DB.source_file } @@ -35,6 +37,10 @@ val cf_bridging_retain : string val cf_non_null_alloc : string val ckcomponent_cl : string val ckcomponentcontroller_cl : string + +(** Script to run our own clang. The argument is expected to be either "" or "++". *) +val clang_bin : string -> string + val class_method : string val class_type : string val copy : string @@ -89,6 +95,7 @@ val enum_map : (Clang_ast_t.pointer option * Exp.t option) Clang_ast_main.Pointe val global_translation_unit_decls : Clang_ast_t.decl list ref val ivar_to_property_index : Clang_ast_t.decl Clang_ast_main.PointerMap.t ref val json : string ref +val log_out : Format.formatter ref val pointer_decl_index : Clang_ast_t.decl Clang_ast_main.PointerMap.t ref val pointer_stmt_index : Clang_ast_t.stmt Clang_ast_main.PointerMap.t ref @@ -99,3 +106,5 @@ val pointer_type_index : Clang_ast_t.c_type Clang_ast_main.PointerMap.t ref (** Map from type pointers (clang pointers and types created later by frontend) to sil types Populated during frontend execution when new type is found *) val sil_types_map : (Typ.t Clang_ast_types.TypePointerMap.t) ref + +val reset_global_state : unit -> unit diff --git a/infer/src/clang/cFrontend_decl.ml b/infer/src/clang/cFrontend_decl.ml index 92dc4ab7d..7ff1771eb 100644 --- a/infer/src/clang/cFrontend_decl.ml +++ b/infer/src/clang/cFrontend_decl.ml @@ -214,10 +214,9 @@ struct let class_decl = Ast_utils.get_decl parent_ptr in (match class_decl with | Some (CXXRecordDecl _) - | Some (ClassTemplateSpecializationDecl _) -> + | Some (ClassTemplateSpecializationDecl _) when Config.cxx_experimental -> let curr_class = CContext.ContextClsDeclPtr parent_ptr in - if Config.cxx_experimental then - process_methods trans_unit_ctx tenv cg cfg curr_class [dec] + process_methods trans_unit_ctx tenv cg cfg curr_class [dec] | Some dec -> Logging.out "Methods of %s skipped\n" (Ast_utils.string_of_decl dec) | None -> ()) diff --git a/infer/src/clang/cFrontend_utils.ml b/infer/src/clang/cFrontend_utils.ml index eaa45e4c7..5b9c58374 100644 --- a/infer/src/clang/cFrontend_utils.ml +++ b/infer/src/clang/cFrontend_utils.ml @@ -639,11 +639,11 @@ struct let is_cpp_translation translation_unit_context = let lang = translation_unit_context.CFrontend_config.lang in - lang = Config.CPP || lang = Config.OBJCPP + lang = CFrontend_config.CPP || lang = CFrontend_config.ObjCPP let is_objc_extension translation_unit_context = let lang = translation_unit_context.CFrontend_config.lang in - lang = Config.OBJC || lang = Config.OBJCPP + lang = CFrontend_config.ObjC || lang = CFrontend_config.ObjCPP let rec get_mangled_method_name function_decl_info method_decl_info = (* For virtual methods return mangled name of the method from most base class diff --git a/infer/src/clang/cFrontend_utils.mli b/infer/src/clang/cFrontend_utils.mli index b2dac8a18..401c7ce53 100644 --- a/infer/src/clang/cFrontend_utils.mli +++ b/infer/src/clang/cFrontend_utils.mli @@ -243,10 +243,10 @@ sig val mk_sil_var : Clang_ast_t.named_decl_info -> var_info option -> Procname.t -> Procname.t -> Pvar.t - (** true if Config.clang_lang is C++ or ObjC++ *) + (** true if the current language is C++ or ObjC++ *) val is_cpp_translation : CFrontend_config.translation_unit_context -> bool - (** true if Config.clang_lang is ObjC or ObjC++ *) + (** true if the current language is ObjC or ObjC++ *) val is_objc_extension : CFrontend_config.translation_unit_context -> bool end diff --git a/infer/src/clang/cLocation.ml b/infer/src/clang/cLocation.ml index f74c7b2a7..c737ea169 100644 --- a/infer/src/clang/cLocation.ml +++ b/infer/src/clang/cLocation.ml @@ -17,10 +17,9 @@ let curr_file = ref DB.source_file_empty let source_file_from_path path = if Filename.is_relative path then - (Logging.err_debug + (failwithf "ERROR: Path %s is relative. Please pass an absolute path in the -c argument.@." - path; - exit 1); + path); match Config.project_root with | Some root -> (try @@ -146,9 +145,3 @@ let get_sil_location stmt_info context = let sloc = choose_sloc sloc1 sloc2 in clang_to_sil_location context.CContext.translation_unit_context sloc (Some (CContext.get_procdesc context)) - -let check_source_file source_file = - if is_file_blacklisted source_file then - (Logging.out "%s" - ("\n Skip the analysis of source file" ^ source_file ^ "\n\n"); - exit(0)); diff --git a/infer/src/clang/cLocation.mli b/infer/src/clang/cLocation.mli index a7195efff..560152d8a 100644 --- a/infer/src/clang/cLocation.mli +++ b/infer/src/clang/cLocation.mli @@ -29,7 +29,7 @@ val should_do_frontend_check : CFrontend_config.translation_unit_context -> val update_curr_file : CFrontend_config.translation_unit_context -> Clang_ast_t.decl_info -> unit -val check_source_file : string -> unit +val is_file_blacklisted : string -> bool val source_file_from_path : string -> DB.source_file diff --git a/infer/src/clang/cMain.ml b/infer/src/clang/cMain.ml deleted file mode 100644 index 518a750af..000000000 --- a/infer/src/clang/cMain.ml +++ /dev/null @@ -1,94 +0,0 @@ -(* - * Copyright (c) 2013 - present Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - *) - -open! Utils - -(* Take as input an ast file and a C or ObjectiveC file such that the ast file - corresponds to the compilation of the C file with clang. - Parse the ast file into a data structure and translates it into a cfg. *) - -module L = Logging - -let buffer_len = 262143 - -(* This function reads the json file in fname, validates it, and encoded in the AST data structure*) -(* defined in Clang_ast_t. *) -let validate_decl_from_file fname = - try - Ag_util.Biniou.from_file ~len:buffer_len Clang_ast_b.read_decl fname - with (Invalid_argument "Bi_inbuf.refill_from_channel") -> - Logging.out "WARNING: biniou buffer too short, skipping the file\n"; - assert false - -let validate_decl_from_stdin () = - try - Ag_util.Biniou.from_channel ~len:buffer_len Clang_ast_b.read_decl stdin - with (Invalid_argument "Bi_inbuf.refill_from_channel") -> - Logging.out "WARNING: biniou buffer too short, skipping the file\n"; - assert false - -let register_perf_stats_report source_file = - let stats_dir = Filename.concat Config.results_dir Config.frontend_stats_dir_name in - let abbrev_source_file = DB.source_file_encoding source_file in - let stats_file = Config.perf_stats_prefix ^ "_" ^ abbrev_source_file ^ ".json" in - create_dir Config.results_dir ; - create_dir stats_dir ; - PerfStats.register_report_at_exit (Filename.concat stats_dir stats_file) - -let init_global_state_for_capture_and_linters source_file = - register_perf_stats_report source_file; - DB.Results_dir.init source_file - -let do_run source_file ast_path = - let init_time = Unix.gettimeofday () in - let print_elapsed () = - let elapsed = Unix.gettimeofday () -. init_time in - Logging.out "Elapsed: %07.3f seconds.\n" elapsed in - try - let ast_filename, ast_decl = - match ast_path with - | Some path -> - path, validate_decl_from_file path - | None -> - "stdin of " ^ DB.source_file_to_string source_file, validate_decl_from_stdin () in - let decl_index, stmt_index, type_index, ivar_to_property_index = - Clang_ast_main.index_node_pointers ast_decl in - CFrontend_config.pointer_decl_index := decl_index; - CFrontend_config.pointer_stmt_index := stmt_index; - CFrontend_config.pointer_type_index := type_index; - CFrontend_config.ivar_to_property_index := ivar_to_property_index; - CFrontend_config.json := ast_filename; - Logging.out "Clang frontend action is %s\n" Config.clang_frontend_action_string; - Logging.out "Start %s of AST from %s\n" Config.clang_frontend_action_string - !CFrontend_config.json; - init_global_state_for_capture_and_linters source_file; - let translation_unit_context = CFrontend_config.{source_file; lang=Config.clang_lang} in - if Config.clang_frontend_do_lint then - CFrontend_checkers_main.do_frontend_checks translation_unit_context ast_decl; - if Config.clang_frontend_do_capture then - CFrontend.do_source_file translation_unit_context ast_decl; - Logging.out "End translation AST file %s... OK!@\n" !CFrontend_config.json; - print_elapsed (); - with - (Yojson.Json_error s) as exc -> - Logging.err_debug "%s\n" s; - print_elapsed (); - raise exc - -let () = - let source_file = - (match Config.source_file with - | Some path -> - Logging.set_log_file_identifier (Some (Filename.basename path)); - CLocation.check_source_file path; - CLocation.source_file_from_path path - | None -> - Logging.err_debug "Incorrect command line arguments\n"; - Config.print_usage_exit ()) in - do_run source_file Config.ast_file diff --git a/infer/src/clang/cMain.mli b/infer/src/clang/cMain.mli deleted file mode 100644 index 56af51a4f..000000000 --- a/infer/src/clang/cMain.mli +++ /dev/null @@ -1,16 +0,0 @@ -(* - * Copyright (c) 2013 - present Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - *) - -open! Utils - -(** Main module of InferClang. Take as input AST files produced by clang during compilation and - their corresponding C/C++/ObjectiveC source files. - - Parse the arguments, parse and validate the input AST into a data structure and translates it - into a cfg. *) diff --git a/infer/src/clang/cMethod_signature.ml b/infer/src/clang/cMethod_signature.ml index e7ea3aadc..5cdfd3b3f 100644 --- a/infer/src/clang/cMethod_signature.ml +++ b/infer/src/clang/cMethod_signature.ml @@ -21,7 +21,7 @@ type method_signature = { loc : Clang_ast_t.source_range; is_instance : bool; is_cpp_virtual : bool; - language : Config.clang_lang; + language : CFrontend_config.clang_lang; pointer_to_parent : Clang_ast_t.pointer option; pointer_to_property_opt : Clang_ast_t.pointer option; (* If set then method is a getter/setter *) return_param_typ : Typ.t option; diff --git a/infer/src/clang/cMethod_signature.mli b/infer/src/clang/cMethod_signature.mli index 6d8a95e6e..b1893ab75 100644 --- a/infer/src/clang/cMethod_signature.mli +++ b/infer/src/clang/cMethod_signature.mli @@ -31,7 +31,7 @@ val ms_is_instance : method_signature -> bool val ms_is_cpp_virtual : method_signature -> bool -val ms_get_lang : method_signature -> Config.clang_lang +val ms_get_lang : method_signature -> CFrontend_config.clang_lang val ms_get_pointer_to_parent : method_signature -> Clang_ast_t.pointer option @@ -45,7 +45,7 @@ val ms_is_setter : method_signature -> bool val make_ms : Procname.t -> (Mangled.t * Clang_ast_t.qual_type) list -> Clang_ast_t.type_ptr -> Clang_ast_t.attribute list -> Clang_ast_t.source_range -> bool -> ?is_cpp_virtual:bool - -> Config.clang_lang -> Clang_ast_t.pointer option -> Clang_ast_t.pointer option + -> CFrontend_config.clang_lang -> Clang_ast_t.pointer option -> Clang_ast_t.pointer option -> Typ.t option -> method_signature val replace_name_ms : method_signature -> Procname.t -> method_signature diff --git a/infer/src/clang/cMethod_trans.ml b/infer/src/clang/cMethod_trans.ml index 87ebc9c5d..05947012d 100644 --- a/infer/src/clang/cMethod_trans.ml +++ b/infer/src/clang/cMethod_trans.ml @@ -92,9 +92,9 @@ let get_param_decls function_method_decl_info = let get_language trans_unit_ctx function_method_decl_info = match function_method_decl_info with | Func_decl_info (_, _) -> trans_unit_ctx.CFrontend_config.lang - | Cpp_Meth_decl_info _ -> Config.CPP - | ObjC_Meth_decl_info _ -> Config.OBJC - | Block_decl_info _ -> Config.OBJC + | Cpp_Meth_decl_info _ -> CFrontend_config.CPP + | ObjC_Meth_decl_info _ -> CFrontend_config.ObjC + | Block_decl_info _ -> CFrontend_config.ObjC let is_cpp_virtual function_method_decl_info = match function_method_decl_info with @@ -300,9 +300,9 @@ let get_formal_parameters tenv ms = | (mangled, {Clang_ast_t.qt_type_ptr}):: pl' -> let should_add_pointer name ms = let is_objc_self = name = CFrontend_config.self && - CMethod_signature.ms_get_lang ms = Config.OBJC in + CMethod_signature.ms_get_lang ms = CFrontend_config.ObjC in let is_cxx_this = name = CFrontend_config.this && - CMethod_signature.ms_get_lang ms = Config.CPP in + CMethod_signature.ms_get_lang ms = CFrontend_config.CPP in (is_objc_self && CMethod_signature.ms_is_instance ms) || is_cxx_this in let tp = if should_add_pointer (Mangled.to_string mangled) ms then (Ast_expressions.create_pointer_type qt_type_ptr) @@ -386,7 +386,7 @@ let create_local_procdesc trans_unit_ctx cfg tenv ms fbody captured is_objc_inst let method_annotation = sil_method_annotation_of_args (CMethod_signature.ms_get_args ms) in let is_cpp_inst_method = CMethod_signature.ms_is_instance ms - && CMethod_signature.ms_get_lang ms = Config.CPP in + && CMethod_signature.ms_get_lang ms = CFrontend_config.CPP in let create_new_procdesc () = let formals = get_formal_parameters tenv ms in let captured_mangled = IList.map (fun (var, t) -> (Pvar.get_name var), t) captured in diff --git a/infer/src/clang/cTrans.ml b/infer/src/clang/cTrans.ml index 7dc639339..afd8b225d 100644 --- a/infer/src/clang/cTrans.ml +++ b/infer/src/clang/cTrans.ml @@ -48,7 +48,7 @@ struct | Procname.ObjC_Cpp objc_cpp -> let class_name = Procname.objc_cpp_get_class_name objc_cpp in CTrans_models.get_predefined_model_method_signature class_name selector - General_utils.mk_procname_from_objc_method Config.OBJC + General_utils.mk_procname_from_objc_method CFrontend_config.ObjC | _ -> None in match predefined_ms_opt, ms_opt with diff --git a/infer/src/clang/cTrans_models.mli b/infer/src/clang/cTrans_models.mli index 6c408b45b..1a335b463 100644 --- a/infer/src/clang/cTrans_models.mli +++ b/infer/src/clang/cTrans_models.mli @@ -41,6 +41,6 @@ val is_cf_retain_release : Procname.t -> bool val get_predefined_model_method_signature : string -> string -> (string -> string -> Procname.objc_cpp_method_kind -> Procname.t) -> - Config.clang_lang -> CMethod_signature.method_signature option + CFrontend_config.clang_lang -> CMethod_signature.method_signature option val is_dispatch_function_name : string -> (string * int) option diff --git a/infer/tests/codetoanalyze/cpp/errors/Makefile b/infer/tests/codetoanalyze/cpp/errors/Makefile index 3ee736ad7..41493b7cb 100644 --- a/infer/tests/codetoanalyze/cpp/errors/Makefile +++ b/infer/tests/codetoanalyze/cpp/errors/Makefile @@ -15,11 +15,9 @@ FILES = \ include_header/include_templ.cpp \ memory_leaks/*.cpp \ models/*.cpp \ - nestedoperators/*.cpp \ npe/*.cpp \ numeric/*.cpp \ overwrite_attribute/*.cpp \ - reference/*.cpp \ resource_leaks/*.cpp \ shared/attributes/*.cpp \ shared/conditional/*.cpp \ @@ -53,8 +51,6 @@ FILES = \ shared/types/typeid_expr.cpp \ smart_ptr/*.cpp \ subtyping/*.cpp \ - templates/*.cpp \ - types/*.cpp \ vector/*.cpp \ compile: