From c93bbbbbc56b17c398c8ec80b7c801b69dc2af19 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Wed, 2 Nov 2016 03:52:23 -0700 Subject: [PATCH] [clang] run assembly commands with the fcp clang Summary: The build system may expect the assembly commands to run. The only issue with assembly commands is that we shouldn't attach the plugin to them. This diff also moves the logic of what to capture to the `Capture.capture` function to be able to reuse code from Capture. This makes sense because the Capture module is the one with the knowledge of what to actually capture or not. Reviewed By: akotulski Differential Revision: D4096019 fbshipit-source-id: 7fc99e1 --- infer/src/clang/Capture.re | 28 +++- infer/src/clang/Capture.rei | 2 +- infer/src/clang/ClangCommand.re | 141 +++++++++-------- infer/src/clang/ClangCommand.rei | 49 +++--- infer/src/clang/ClangWrapper.re | 144 +++++++----------- .../build_systems/build_integration_tests.py | 21 --- .../expected_outputs/cc1_report.json | 7 - 7 files changed, 170 insertions(+), 222 deletions(-) delete mode 100644 infer/tests/build_systems/expected_outputs/cc1_report.json diff --git a/infer/src/clang/Capture.re b/infer/src/clang/Capture.re index cf469ada3..42c8a60f6 100644 --- a/infer/src/clang/Capture.re +++ b/infer/src/clang/Capture.re @@ -138,9 +138,9 @@ let run_plugin_and_frontend frontend clang_args => { run_clang clang_command frontend }; -let capture clang_args => { +let cc1_capture clang_cmd => { let source_path = { - let orig_argv = ClangCommand.get_orig_argv clang_args; + let orig_argv = ClangCommand.get_orig_argv clang_cmd; /* the source file is always the last argument of the original -cc1 clang command */ filename_to_absolute orig_argv.(Array.length orig_argv - 1) }; @@ -148,7 +148,7 @@ let capture clang_args => { if (Config.analyzer == Some Config.Compile || CLocation.is_file_blacklisted source_path) { Logging.out "@\n Skip the analysis of source file %s@\n@\n" source_path; /* We still need to run clang, but we don't have to attach the plugin. */ - run_clang (ClangCommand.command_to_run clang_args) consume_in + run_clang (ClangCommand.command_to_run clang_cmd) consume_in } else { let source_file = CLocation.source_file_from_path source_path; init_global_state_for_capture_and_linters source_file; @@ -161,20 +161,20 @@ let capture clang_args => { ("objective-c++", ObjCPP) ]; let lang = - switch (ClangCommand.value_of_option clang_args "-x") { + switch (ClangCommand.value_of_option clang_cmd "-x") { | Some lang_opt when IList.mem_assoc string_equal lang_opt clang_langs => IList.assoc string_equal lang_opt clang_langs | _ => assert false }; {CFrontend_config.source_file: source_file, lang} }; - Config.arc_mode := ClangCommand.has_flag clang_args "-fobjc-arc"; + Config.arc_mode := ClangCommand.has_flag clang_cmd "-fobjc-arc"; try ( switch Config.clang_biniou_file { | Some fname => run_clang_frontend trans_unit_ctx (`File fname) | None => run_plugin_and_frontend - (fun chan_in => run_clang_frontend trans_unit_ctx (`Pipe chan_in)) clang_args + (fun chan_in => run_clang_frontend trans_unit_ctx (`Pipe chan_in)) clang_cmd } ) { | exc => @@ -182,6 +182,22 @@ let capture clang_args => { raise exc } }; + /* reset logging to stop capturing log output into the source file's log */ + Logging.set_log_file_identifier CommandLineOption.Clang None; () } }; + +let capture clang_cmd => + if (ClangCommand.can_attach_ast_exporter clang_cmd) { + /* this command compiles some code; replace the invocation of clang with our own clang and + plugin */ + cc1_capture clang_cmd + } else { + /* Non-compilation (eg, linking) command. Run the command as-is. It will not get captured + further since `clang -### ...` will only output commands that invoke binaries using their + absolute paths. */ + let command_to_run = ClangCommand.command_to_run clang_cmd; + Logging.out "Running non-cc command without capture: %s@\n" command_to_run; + run_clang command_to_run consume_in + }; diff --git a/infer/src/clang/Capture.rei b/infer/src/clang/Capture.rei index 2e1c165f2..a6986d710 100644 --- a/infer/src/clang/Capture.rei +++ b/infer/src/clang/Capture.rei @@ -6,4 +6,4 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ -let capture: ClangCommand.args => unit; +let capture: ClangCommand.t => unit; diff --git a/infer/src/clang/ClangCommand.re b/infer/src/clang/ClangCommand.re index 25cbe696f..c00ce5143 100644 --- a/infer/src/clang/ClangCommand.re +++ b/infer/src/clang/ClangCommand.re @@ -8,22 +8,13 @@ */ open! Utils; -type args = { +type t = { exec: string, argv: list string, orig_argv: list string, quoting_style: ClangQuotes.style }; -type t = - | Assembly args - /** a normalized clang command that runs the assembler */ - | CC1 args - /** a -cc1 clang command */ - | ClangError string - | ClangWarning string - | NonCCCommand args /** other commands (as, ld, ...) */; - let fcp_dir = Config.bin_dir /\/ Filename.parent_dir_name /\/ Filename.parent_dir_name /\/ "facebook-clang-plugins"; @@ -61,10 +52,35 @@ let value_of_option {orig_argv} => value_of_argv_option orig_argv; let has_flag {orig_argv} flag => IList.exists (string_equal flag) orig_argv; +let can_attach_ast_exporter cmd => + has_flag cmd "-cc1" && ( + switch (value_of_option cmd "-x") { + | None => + Logging.stderr "malformed -cc1 command has no \"-x\" flag!"; + false + | Some lang when string_is_prefix "assembler" lang => false + | Some _ => true + } + ); + +let argv_cons a b => [a, ...b]; + +let argv_do_if cond action x => + if cond { + action x + } else { + x + }; + +let file_arg_cmd_sanitizer cmd => { + let file = ClangQuotes.mk_arg_file "clang_command_" cmd.quoting_style cmd.argv; + {...cmd, argv: [Format.sprintf "@%s" file]} +}; + /* Work around various path or library issues occurring when one tries to substitute Apple's version of clang with a different version. Also mitigate version discrepancies in clang's fatal warnings. */ -let mk_clang_compat_args args => { +let clang_cc1_cmd_sanitizer cmd => { /* command line options not supported by the opensource compiler or the plugins */ let flags_blacklist = ["-fembed-bitcode-marker", "-fno-canonical-system-headers"]; let replace_option_arg option arg => @@ -82,83 +98,66 @@ let mk_clang_compat_args args => { } else { arg }; - let post_args = { - let global_defines_h = Config.lib_dir /\/ "clang_wrappers" /\/ "global_defines.h"; - [ - "-Wno-everything", - /* Never error on warnings. Clang is often more strict than Apple's version. These arguments - are appended at the end to override previous opposite settings. How it's done: suppress - all the warnings, since there are no warnings, compiler can't elevate them to error - level. */ - "-include", - global_defines_h - ] - }; - let rec filter_unsupported_args_and_swap_includes (prev, res) => + let post_args_rev = + [] |> IList.rev_append ["-include", Config.lib_dir /\/ "clang_wrappers" /\/ "global_defines.h"] |> + argv_do_if (has_flag cmd "-fmodules") (argv_cons "-fno-cxx-modules") |> + /* Never error on warnings. Clang is often more strict than Apple's version. These arguments + are appended at the end to override previous opposite settings. How it's done: suppress + all the warnings, since there are no warnings, compiler can't elevate them to error + level. */ + argv_cons "-Wno-everything"; + let rec filter_unsupported_args_and_swap_includes (prev, res_rev) => fun - | [] => IList.rev (IList.rev post_args @ res) + | [] => + /* return non-reversed list */ + IList.rev (post_args_rev @ res_rev) | [flag, ...tl] when IList.mem string_equal flag flags_blacklist => - filter_unsupported_args_and_swap_includes (flag, res) tl + filter_unsupported_args_and_swap_includes (flag, res_rev) tl | [arg, ...tl] => { - let res' = [replace_option_arg prev arg, ...res]; - filter_unsupported_args_and_swap_includes (arg, res') tl + let res_rev' = [replace_option_arg prev arg, ...res_rev]; + filter_unsupported_args_and_swap_includes (arg, res_rev') tl }; - let clang_arguments = filter_unsupported_args_and_swap_includes ("", []) args.argv; - let file = ClangQuotes.mk_arg_file "clang_args_" args.quoting_style clang_arguments; - {...args, argv: [Format.sprintf "@%s" file]} + let clang_arguments = filter_unsupported_args_and_swap_includes ("", []) cmd.argv; + file_arg_cmd_sanitizer {...cmd, argv: clang_arguments} }; let mk quoting_style argv => { let argv_list = Array.to_list argv; - let is_assembly = { - /* whether language is set to "assembler" or "assembler-with-cpp" */ - let assembly_language = - switch (value_of_argv_option argv_list "-x") { - | Some lang => string_is_prefix "assembler" lang - | _ => false - }; - /* Detect -cc1as or assembly language commands. -cc1as is always the first argument if - present. */ - string_equal argv.(1) "-cc1as" || assembly_language - }; - let args = - switch argv_list { - | [exec, ...argv_no_exec] => {exec, orig_argv: argv_no_exec, argv: argv_no_exec, quoting_style} - | [] => failwith "argv cannot be an empty list" - }; - if is_assembly { - Assembly args - } else if (argv.(1) == "-cc1") { - CC1 - /* -cc1 is always the first argument if present. */ - args - } else { - NonCCCommand args + switch argv_list { + | [exec, ...argv_no_exec] => {exec, orig_argv: argv_no_exec, argv: argv_no_exec, quoting_style} + | [] => failwith "argv cannot be an empty list" } }; -let command_to_run args => { - let {exec, argv, quoting_style} = mk_clang_compat_args args; - Printf.sprintf - "'%s' %s" exec (IList.map (ClangQuotes.quote quoting_style) argv |> String.concat " ") +let command_to_run cmd => { + let mk_cmd normalizer => { + let {exec, argv, quoting_style} = normalizer cmd; + Printf.sprintf + "'%s' %s" exec (IList.map (ClangQuotes.quote quoting_style) argv |> String.concat " ") + }; + if (can_attach_ast_exporter cmd) { + mk_cmd clang_cc1_cmd_sanitizer + } else if ( + string_is_prefix "clang" (Filename.basename cmd.exec) + ) { + /* `clang` supports argument files and the commands can be longer than the maximum length of the + command line, so put arguments in a file */ + mk_cmd file_arg_cmd_sanitizer + } else { + /* other commands such as `ld` do not support argument files */ + mk_cmd (fun x => x) + } }; let with_exec exec args => {...args, exec}; let with_plugin_args args => { - let cons a b => [a, ...b]; - let do_if cond action x => - if cond { - action x - } else { - x - }; - let rev_args_before = + let args_before_rev = [] |> /* -cc1 has to be the first argument or clang will think it runs in driver mode */ - cons "-cc1" |> + argv_cons "-cc1" |> /* It's important to place this option before other -isystem options. */ - do_if infer_cxx_models (IList.rev_append ["-isystem", Config.cpp_models_dir]) |> + argv_do_if infer_cxx_models (IList.rev_append ["-isystem", Config.cpp_models_dir]) |> IList.rev_append [ "-load", plugin_path, @@ -177,8 +176,8 @@ let with_plugin_args args => { "-plugin-arg-" ^ plugin_name, "PREPEND_CURRENT_DIR=1" ]; - let args_after = [] |> do_if Config.fcp_syntax_only (cons "-fsyntax-only"); - {...args, argv: IList.rev_append rev_args_before (args.argv @ args_after)} + let args_after_rev = [] |> argv_do_if Config.fcp_syntax_only (argv_cons "-fsyntax-only"); + {...args, argv: IList.rev_append args_before_rev (args.argv @ IList.rev args_after_rev)} }; let prepend_arg arg clang_args => {...clang_args, argv: [arg, ...clang_args.argv]}; diff --git a/infer/src/clang/ClangCommand.rei b/infer/src/clang/ClangCommand.rei index c262dfa1a..7e3769a21 100644 --- a/infer/src/clang/ClangCommand.rei +++ b/infer/src/clang/ClangCommand.rei @@ -6,46 +6,45 @@ * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. */ -type args; - -type t = - | Assembly args - /** a normalized clang command that runs the assembler */ - | CC1 args - /** a -cc1 clang command */ - | ClangError string - | ClangWarning string - | NonCCCommand args /** other commands (as, ld, ...) */; +type t; /** [mk qs argv] finds the type of command depending on its arguments [argv]. The quoting style of - the arguments have to be provided, so that the command may be run later on. */ + the arguments have to be provided, so that the command may be run later on. Beware that this + doesn't look inside argument files. This can be used to create a "clang -### ..." command on + which to call [command_to_run], but other functions from the module will not work as expected + unless the command has been normalized by "clang -### ...". */ let mk: ClangQuotes.style => array string => t; -/** change an args object into a string ready to be passed to a shell to be executed */ -let command_to_run: args => string; +/** Make a command into a string ready to be passed to a shell to be executed. Fine to call with + clang driver commands. */ +let command_to_run: t => string; + + +/** Whether the command has this flag set in its arguments. Must be called on normalized commands. */ +let has_flag: t => string => bool; -/** whether the command has this flag set in its arguments */ -let has_flag: args => string => bool; +/** The value passed to an option in the arguments of a command. Must be called on normalized commands. */ +let value_of_option: t => string => option string; -/** the value passed to an option in the arguments of a command */ -let value_of_option: args => string => option string; +/** Whether the command is suitable for attaching the AST exporter. Must be called on normalized commands. */ +let can_attach_ast_exporter: t => bool; -/** add the arguments needed to attach the facebook-clang-plugins plugin */ -let with_plugin_args: args => args; +/** Add the arguments needed to attach the facebook-clang-plugins plugin. Must be called on normalized commands. */ +let with_plugin_args: t => t; -let prepend_arg: string => args => args; +let prepend_arg: string => t => t; -let prepend_args: list string => args => args; +let prepend_args: list string => t => t; -let append_args: list string => args => args; +let append_args: list string => t => t; -let get_orig_argv: args => array string; +let get_orig_argv: t => array string; -/** updates the executable to be run */ -let with_exec: string => args => args; +/** update the executable to be run */ +let with_exec: string => t => t; diff --git a/infer/src/clang/ClangWrapper.re b/infer/src/clang/ClangWrapper.re index 8380f5a59..652ca2859 100644 --- a/infer/src/clang/ClangWrapper.re +++ b/infer/src/clang/ClangWrapper.re @@ -11,105 +11,67 @@ commands by our own clang with our plugin attached for each source file. */ open! Utils; +type action_item = + | Command ClangCommand.t + | ClangError string + | ClangWarning string; + /** Given a list of arguments for clang [args], return a list of new commands to run according to - the results of `clang -### [args]`. Assembly commands (eg, clang -cc1as ...) are filtered out, - although the type cannot reflect that fact. */ -let normalize (args: array string) :list ClangCommand.t => - switch (ClangCommand.mk ClangQuotes.SingleQuotes args) { - | CC1 args => - Logging.out "InferClang got toplevel -cc1 command@\n"; - [ClangCommand.CC1 args] - | NonCCCommand args => - let args' = ClangCommand.append_args ["-fno-cxx-modules"] args; - let clang_hashhashhash = - Printf.sprintf - "%s 2>&1" (ClangCommand.prepend_arg "-###" args' |> ClangCommand.command_to_run); - Logging.out "clang -### invocation: %s@\n" clang_hashhashhash; - let normalized_commands = ref []; - let one_line line => - if (string_is_prefix " \"" line) { + the results of `clang -### [args]`. */ +let normalize (args: array string) :list action_item => { + let cmd = ClangCommand.mk ClangQuotes.SingleQuotes args; + let clang_hashhashhash = + Printf.sprintf "%s 2>&1" (ClangCommand.prepend_arg "-###" cmd |> ClangCommand.command_to_run); + Logging.out "clang -### invocation: %s@\n" clang_hashhashhash; + let normalized_commands = ref []; + let one_line line => + if (string_is_prefix " \"" line) { + let cmd = /* massage line to remove edge-cases for splitting */ "\"" ^ line ^ " \"" |> /* split by whitespace */ Str.split (Str.regexp_string "\" \"") |> Array.of_list |> - ClangCommand.mk ClangQuotes.EscapedDoubleQuotes - } else if ( - Str.string_match (Str.regexp "clang[^ :]*: warning: ") line 0 - ) { - ClangCommand.ClangWarning line - } else { - ClangCommand.ClangError line - }; - let commands_or_errors = - /* commands generated by `clang -### ...` start with ' "/absolute/path/to/binary"' */ - Str.regexp " \"/\\|clang[^ :]*: \\(error\\|warning\\): "; - let consume_input i => - try ( - while true { - let line = input_line i; - /* keep only commands and errors */ - if (Str.string_match commands_or_errors line 0) { - normalized_commands := [one_line line, ...!normalized_commands] - } + ClangCommand.mk ClangQuotes.EscapedDoubleQuotes; + Command cmd + } else if ( + Str.string_match (Str.regexp "clang[^ :]*: warning: ") line 0 + ) { + ClangWarning line + } else { + ClangError line + }; + let commands_or_errors = + /* commands generated by `clang -### ...` start with ' "/absolute/path/to/binary"' */ + Str.regexp " \"/\\|clang[^ :]*: \\(error\\|warning\\): "; + let consume_input i => + try ( + while true { + let line = input_line i; + /* keep only commands and errors */ + if (Str.string_match commands_or_errors line 0) { + normalized_commands := [one_line line, ...!normalized_commands] } - ) { - | End_of_file => () - }; - /* collect stdout and stderr output together (in reverse order) */ - with_process_in clang_hashhashhash consume_input |> ignore; - normalized_commands := IList.rev !normalized_commands; - /* Discard assembly commands. This may make the list of commands empty, in which case we'll run - the original clang command. We could be smarter about this and try to execute the assembly - commands with our own clang. */ - IList.filter - ( - fun - | ClangCommand.Assembly asm_cmd => { - Logging.out "Skipping assembly command %s@\n" (ClangCommand.command_to_run asm_cmd); - false - } - | _ => true - ) - !normalized_commands - | Assembly _ => - /* discard assembly commands -- see above */ - Logging.out "InferClang got toplevel assembly command@\n"; - [] - | ClangError _ - | ClangWarning _ => - /* we cannot possibly get this from the command-line... */ - assert false - }; + } + ) { + | End_of_file => () + }; + /* collect stdout and stderr output together (in reverse order) */ + with_process_in clang_hashhashhash consume_input |> ignore; + normalized_commands := IList.rev !normalized_commands; + !normalized_commands +}; -let execute_clang_command (clang_cmd: ClangCommand.t) => { - /* reset logging, otherwise we might print into the logs of the previous file that was compiled */ - Logging.set_log_file_identifier CommandLineOption.Clang None; - switch clang_cmd { - | CC1 args => - /* this command compiles some code; replace the invocation of clang with our own clang and - plugin */ - Logging.out "Capturing -cc1 command: %s@\n" (ClangCommand.command_to_run args); - Capture.capture args - | ClangError error => - /* An error in the output of `clang -### ...`. Outputs the error and fail. This is because - `clang -###` pretty much never fails, but warns of failures on stderr instead. */ - Logging.err "%s" error; - exit 1 +let exec_action_item = + fun + | ClangError error => { + /* An error in the output of `clang -### ...`. Outputs the error and fail. This is because + `clang -###` pretty much never fails, but warns of failures on stderr instead. */ + Logging.err "%s" error; + exit 1 + } | ClangWarning warning => Logging.err "%s@\n" warning - | Assembly args => - /* We shouldn't get any assembly command at this point */ - (if Config.debug_mode {failwithf} else {Logging.err}) - "WARNING: unexpected assembly command: %s@\n" (ClangCommand.command_to_run args) - | NonCCCommand args => - /* Non-compilation (eg, linking) command. Run the command as-is. It will not get captured - further since `clang -### ...` will only output commands that invoke binaries using their - absolute paths. */ - let argv = ClangCommand.get_orig_argv args; - Logging.out "Executing raw command: %s@\n" (String.concat " " (Array.to_list argv)); - Process.create_process_and_wait argv - } -}; + | Command clang_cmd => Capture.capture clang_cmd; let exe args xx_suffix => { /* make sure args.(0) points to clang in facebook-clang-plugins */ @@ -126,7 +88,7 @@ let exe args xx_suffix => { true | None => false }; - IList.iter execute_clang_command commands; + IList.iter exec_action_item commands; if (commands == [] || should_run_original_command) { if (commands == []) { /* No command to execute after -###, let's execute the original command diff --git a/infer/tests/build_systems/build_integration_tests.py b/infer/tests/build_systems/build_integration_tests.py index aefa68282..ec75da74f 100755 --- a/infer/tests/build_systems/build_integration_tests.py +++ b/infer/tests/build_systems/build_integration_tests.py @@ -63,7 +63,6 @@ ALL_TESTS = [ 'ant', 'assembly', 'buck', - 'cc1', 'cmake', 'componentkit', 'delete', @@ -477,26 +476,6 @@ class BuildIntegrationTest(unittest.TestCase): 'infer_args': reactive_args}, {'compile': ['analyze']}]) - def test_clang_cc1(self): - def preprocess(): - hashhashhash = subprocess.check_output( - [CLANG_BIN, '-###', '-c', 'hello.c'], - # `clang -### -c hello.c` prints on stderr - stderr=subprocess.STDOUT) - # pick the line containing the compilation command, which - # should be the only one to include "-cc1" - cc1_line = filter(lambda s: s.find('"-cc1"') != -1, - hashhashhash.splitlines())[0] - # [cc1_line] usually looks like ' "/foo/clang" "bar" "baz"'. - # return ['clang', 'bar', 'baz'] - cmd = [s.strip('"') for s in cc1_line.strip().split('" "')] - cmd[0] = 'clang' - return [{'compile': cmd}] - test('cc1', 'clang -cc1', - CODETOANALYZE_DIR, - [], - preprocess=preprocess) - def test_clang_assembly(self): test('assembly', 'compile with assembly code', CODETOANALYZE_DIR, [{'compile': ['clang', '-x', 'c', '-c', 'hello.c', '-x', diff --git a/infer/tests/build_systems/expected_outputs/cc1_report.json b/infer/tests/build_systems/expected_outputs/cc1_report.json deleted file mode 100644 index a6ed10eff..000000000 --- a/infer/tests/build_systems/expected_outputs/cc1_report.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "bug_type": "NULL_DEREFERENCE", - "file": "hello.c", - "procedure": "test" - } -] \ No newline at end of file