[clang] run assembly commands with the fcp clang

Summary:
The build system may expect the assembly commands to run. The only issue with
assembly commands is that we shouldn't attach the plugin to them.

This diff also moves the logic of what to capture to the `Capture.capture`
function to be able to reuse code from Capture. This makes sense because the
Capture module is the one with the knowledge of what to actually capture or
not.

Reviewed By: akotulski

Differential Revision: D4096019

fbshipit-source-id: 7fc99e1

infer/src/clang/Capture.re

@@ -138,9 +138,9 @@ let run_plugin_and_frontend frontend clang_args => {
   run_clang clang_command frontend
 };
 
-let capture clang_args => {
+let cc1_capture clang_cmd => {
   let source_path = {
-    let orig_argv = ClangCommand.get_orig_argv clang_args;
+    let orig_argv = ClangCommand.get_orig_argv clang_cmd;
     /* the source file is always the last argument of the original -cc1 clang command */
     filename_to_absolute orig_argv.(Array.length orig_argv - 1)
   };
@@ -148,7 +148,7 @@ let capture clang_args => {
   if (Config.analyzer == Some Config.Compile || CLocation.is_file_blacklisted source_path) {
     Logging.out "@\n Skip the analysis of source file %s@\n@\n" source_path;
     /* We still need to run clang, but we don't have to attach the plugin. */
-    run_clang (ClangCommand.command_to_run clang_args) consume_in
+    run_clang (ClangCommand.command_to_run clang_cmd) consume_in
   } else {
     let source_file = CLocation.source_file_from_path source_path;
     init_global_state_for_capture_and_linters source_file;
@@ -161,20 +161,20 @@ let capture clang_args => {
           ("objective-c++", ObjCPP)
         ];
       let lang =
-        switch (ClangCommand.value_of_option clang_args "-x") {
+        switch (ClangCommand.value_of_option clang_cmd "-x") {
         | Some lang_opt when IList.mem_assoc string_equal lang_opt clang_langs =>
           IList.assoc string_equal lang_opt clang_langs
         | _ => assert false
         };
       {CFrontend_config.source_file: source_file, lang}
     };
-    Config.arc_mode := ClangCommand.has_flag clang_args "-fobjc-arc";
+    Config.arc_mode := ClangCommand.has_flag clang_cmd "-fobjc-arc";
     try (
       switch Config.clang_biniou_file {
       | Some fname => run_clang_frontend trans_unit_ctx (`File fname)
       | None =>
         run_plugin_and_frontend
-          (fun chan_in => run_clang_frontend trans_unit_ctx (`Pipe chan_in)) clang_args
+          (fun chan_in => run_clang_frontend trans_unit_ctx (`Pipe chan_in)) clang_cmd
       }
     ) {
     | exc =>
@@ -182,6 +182,22 @@ let capture clang_args => {
         raise exc
       }
     };
+    /* reset logging to stop capturing log output into the source file's log */
+    Logging.set_log_file_identifier CommandLineOption.Clang None;
     ()
   }
 };
+
+let capture clang_cmd =>
+  if (ClangCommand.can_attach_ast_exporter clang_cmd) {
+    /* this command compiles some code; replace the invocation of clang with our own clang and
+       plugin */
+    cc1_capture clang_cmd
+  } else {
+    /* Non-compilation (eg, linking) command. Run the command as-is. It will not get captured
+       further since `clang -### ...` will only output commands that invoke binaries using their
+       absolute paths. */
+    let command_to_run = ClangCommand.command_to_run clang_cmd;
+    Logging.out "Running non-cc command without capture: %s@\n" command_to_run;
+    run_clang command_to_run consume_in
+  };

infer/src/clang/Capture.rei

@@ -6,4 +6,4 @@
  * LICENSE file in the root directory of this source tree. An additional grant
  * of patent rights can be found in the PATENTS file in the same directory.
  */
-let capture: ClangCommand.args => unit;
+let capture: ClangCommand.t => unit;

infer/src/clang/ClangCommand.re

@@ -8,22 +8,13 @@
  */
 open! Utils;
 
-type args = {
+type t = {
   exec: string,
   argv: list string,
   orig_argv: list string,
   quoting_style: ClangQuotes.style
 };
 
-type t =
-  | Assembly args
-  /** a normalized clang command that runs the assembler */
-  | CC1 args
-  /** a -cc1 clang command */
-  | ClangError string
-  | ClangWarning string
-  | NonCCCommand args /** other commands (as, ld, ...) */;
-
 let fcp_dir =
   Config.bin_dir /\/ Filename.parent_dir_name /\/ Filename.parent_dir_name /\/ "facebook-clang-plugins";
 
@@ -61,10 +52,35 @@ let value_of_option {orig_argv} => value_of_argv_option orig_argv;
 
 let has_flag {orig_argv} flag => IList.exists (string_equal flag) orig_argv;
 
+let can_attach_ast_exporter cmd =>
+  has_flag cmd "-cc1" && (
+    switch (value_of_option cmd "-x") {
+    | None =>
+      Logging.stderr "malformed -cc1 command has no \"-x\" flag!";
+      false
+    | Some lang when string_is_prefix "assembler" lang => false
+    | Some _ => true
+    }
+  );
+
+let argv_cons a b => [a, ...b];
+
+let argv_do_if cond action x =>
+  if cond {
+    action x
+  } else {
+    x
+  };
+
+let file_arg_cmd_sanitizer cmd => {
+  let file = ClangQuotes.mk_arg_file "clang_command_" cmd.quoting_style cmd.argv;
+  {...cmd, argv: [Format.sprintf "@%s" file]}
+};
+
 /* Work around various path or library issues occurring when one tries to substitute Apple's version
    of clang with a different version. Also mitigate version discrepancies in clang's
    fatal warnings. */
-let mk_clang_compat_args args => {
+let clang_cc1_cmd_sanitizer cmd => {
   /* command line options not supported by the opensource compiler or the plugins */
   let flags_blacklist = ["-fembed-bitcode-marker", "-fno-canonical-system-headers"];
   let replace_option_arg option arg =>
@@ -82,83 +98,66 @@ let mk_clang_compat_args args => {
     } else {
       arg
     };
-  let post_args = {
+  let post_args_rev =
-    let global_defines_h = Config.lib_dir /\/ "clang_wrappers" /\/ "global_defines.h";
+    [] |> IList.rev_append ["-include", Config.lib_dir /\/ "clang_wrappers" /\/ "global_defines.h"] |>
-    [
+    argv_do_if (has_flag cmd "-fmodules") (argv_cons "-fno-cxx-modules") |>
-      "-Wno-everything",
     /* Never error on warnings. Clang is often more strict than Apple's version.  These arguments
        are appended at the end to override previous opposite settings.  How it's done: suppress
        all the warnings, since there are no warnings, compiler can't elevate them to error
        level. */
-      "-include",
+    argv_cons "-Wno-everything";
-      global_defines_h
+  let rec filter_unsupported_args_and_swap_includes (prev, res_rev) =>
-    ]
-  };
-  let rec filter_unsupported_args_and_swap_includes (prev, res) =>
     fun
-    | [] => IList.rev (IList.rev post_args @ res)
+    | [] =>
+      /* return non-reversed list */
+      IList.rev (post_args_rev @ res_rev)
     | [flag, ...tl] when IList.mem string_equal flag flags_blacklist =>
-      filter_unsupported_args_and_swap_includes (flag, res) tl
+      filter_unsupported_args_and_swap_includes (flag, res_rev) tl
     | [arg, ...tl] => {
-        let res' = [replace_option_arg prev arg, ...res];
+        let res_rev' = [replace_option_arg prev arg, ...res_rev];
-        filter_unsupported_args_and_swap_includes (arg, res') tl
+        filter_unsupported_args_and_swap_includes (arg, res_rev') tl
       };
-  let clang_arguments = filter_unsupported_args_and_swap_includes ("", []) args.argv;
+  let clang_arguments = filter_unsupported_args_and_swap_includes ("", []) cmd.argv;
-  let file = ClangQuotes.mk_arg_file "clang_args_" args.quoting_style clang_arguments;
+  file_arg_cmd_sanitizer {...cmd, argv: clang_arguments}
-  {...args, argv: [Format.sprintf "@%s" file]}
 };
 
 let mk quoting_style argv => {
   let argv_list = Array.to_list argv;
-  let is_assembly = {
-    /* whether language is set to "assembler" or "assembler-with-cpp" */
-    let assembly_language =
-      switch (value_of_argv_option argv_list "-x") {
-      | Some lang => string_is_prefix "assembler" lang
-      | _ => false
-      };
-    /* Detect -cc1as or assembly language commands. -cc1as is always the first argument if
-       present. */
-    string_equal argv.(1) "-cc1as" || assembly_language
-  };
-  let args =
   switch argv_list {
   | [exec, ...argv_no_exec] => {exec, orig_argv: argv_no_exec, argv: argv_no_exec, quoting_style}
   | [] => failwith "argv cannot be an empty list"
-    };
-  if is_assembly {
-    Assembly args
-  } else if (argv.(1) == "-cc1") {
-    CC1
-      /* -cc1 is always the first argument if present. */
-      args
-  } else {
-    NonCCCommand args
   }
 };
 
-let command_to_run args => {
+let command_to_run cmd => {
-  let {exec, argv, quoting_style} = mk_clang_compat_args args;
+  let mk_cmd normalizer => {
+    let {exec, argv, quoting_style} = normalizer cmd;
     Printf.sprintf
       "'%s' %s" exec (IList.map (ClangQuotes.quote quoting_style) argv |> String.concat " ")
   };
+  if (can_attach_ast_exporter cmd) {
+    mk_cmd clang_cc1_cmd_sanitizer
+  } else if (
+    string_is_prefix "clang" (Filename.basename cmd.exec)
+  ) {
+    /* `clang` supports argument files and the commands can be longer than the maximum length of the
+       command line, so put arguments in a file */
+    mk_cmd file_arg_cmd_sanitizer
+  } else {
+    /* other commands such as `ld` do not support argument files */
+    mk_cmd (fun x => x)
+  }
+};
 
 let with_exec exec args => {...args, exec};
 
 let with_plugin_args args => {
-  let cons a b => [a, ...b];
+  let args_before_rev =
-  let do_if cond action x =>
-    if cond {
-      action x
-    } else {
-      x
-    };
-  let rev_args_before =
     [] |>
     /* -cc1 has to be the first argument or clang will think it runs in driver mode */
-    cons "-cc1" |>
+    argv_cons "-cc1" |>
     /* It's important to place this option before other -isystem options. */
-    do_if infer_cxx_models (IList.rev_append ["-isystem", Config.cpp_models_dir]) |>
+    argv_do_if infer_cxx_models (IList.rev_append ["-isystem", Config.cpp_models_dir]) |>
     IList.rev_append [
       "-load",
       plugin_path,
@@ -177,8 +176,8 @@ let with_plugin_args args => {
       "-plugin-arg-" ^ plugin_name,
       "PREPEND_CURRENT_DIR=1"
     ];
-  let args_after = [] |> do_if Config.fcp_syntax_only (cons "-fsyntax-only");
+  let args_after_rev = [] |> argv_do_if Config.fcp_syntax_only (argv_cons "-fsyntax-only");
-  {...args, argv: IList.rev_append rev_args_before (args.argv @ args_after)}
+  {...args, argv: IList.rev_append args_before_rev (args.argv @ IList.rev args_after_rev)}
 };
 
 let prepend_arg arg clang_args => {...clang_args, argv: [arg, ...clang_args.argv]};

infer/src/clang/ClangCommand.rei

@@ -6,46 +6,45 @@
  * LICENSE file in the root directory of this source tree. An additional grant
  * of patent rights can be found in the PATENTS file in the same directory.
  */
-type args;
+type t;
-
-type t =
-  | Assembly args
-  /** a normalized clang command that runs the assembler */
-  | CC1 args
-  /** a -cc1 clang command */
-  | ClangError string
-  | ClangWarning string
-  | NonCCCommand args /** other commands (as, ld, ...) */;
 
 
 /** [mk qs argv] finds the type of command depending on its arguments [argv]. The quoting style of
-    the arguments have to be provided, so that the command may be run later on. */
+    the arguments have to be provided, so that the command may be run later on. Beware that this
+    doesn't look inside argument files. This can be used to create a "clang -### ..." command on
+    which to call [command_to_run], but other functions from the module will not work as expected
+    unless the command has been normalized by "clang -### ...". */
 let mk: ClangQuotes.style => array string => t;
 
 
-/** change an args object into a string ready to be passed to a shell to be executed */
+/** Make a command into a string ready to be passed to a shell to be executed. Fine to call with
-let command_to_run: args => string;
+    clang driver commands. */
+let command_to_run: t => string;
+
+
+/** Whether the command has this flag set in its arguments. Must be called on normalized commands. */
+let has_flag: t => string => bool;
 
 
-/** whether the command has this flag set in its arguments */
+/** The value passed to an option in the arguments of a command. Must be called on normalized commands. */
-let has_flag: args => string => bool;
+let value_of_option: t => string => option string;
 
 
-/** the value passed to an option in the arguments of a command */
+/** Whether the command is suitable for attaching the AST exporter. Must be called on normalized commands. */
-let value_of_option: args => string => option string;
+let can_attach_ast_exporter: t => bool;
 
 
-/** add the arguments needed to attach the facebook-clang-plugins plugin */
+/** Add the arguments needed to attach the facebook-clang-plugins plugin. Must be called on normalized commands. */
-let with_plugin_args: args => args;
+let with_plugin_args: t => t;
 
-let prepend_arg: string => args => args;
+let prepend_arg: string => t => t;
 
-let prepend_args: list string => args => args;
+let prepend_args: list string => t => t;
 
-let append_args: list string => args => args;
+let append_args: list string => t => t;
 
-let get_orig_argv: args => array string;
+let get_orig_argv: t => array string;
 
 
-/** updates the executable to be run */
+/** update the executable to be run */
-let with_exec: string => args => args;
+let with_exec: string => t => t;

infer/src/clang/ClangWrapper.re

@@ -11,35 +11,35 @@
     commands by our own clang with our plugin attached for each source file. */
 open! Utils;
 
+type action_item =
+  | Command ClangCommand.t
+  | ClangError string
+  | ClangWarning string;
+
 
 /** Given a list of arguments for clang [args], return a list of new commands to run according to
-    the results of `clang -### [args]`. Assembly commands (eg, clang -cc1as ...) are filtered out,
+    the results of `clang -### [args]`. */
-    although the type cannot reflect that fact. */
+let normalize (args: array string) :list action_item => {
-let normalize (args: array string) :list ClangCommand.t =>
+  let cmd = ClangCommand.mk ClangQuotes.SingleQuotes args;
-  switch (ClangCommand.mk ClangQuotes.SingleQuotes args) {
-  | CC1 args =>
-    Logging.out "InferClang got toplevel -cc1 command@\n";
-    [ClangCommand.CC1 args]
-  | NonCCCommand args =>
-    let args' = ClangCommand.append_args ["-fno-cxx-modules"] args;
   let clang_hashhashhash =
-      Printf.sprintf
+    Printf.sprintf "%s 2>&1" (ClangCommand.prepend_arg "-###" cmd |> ClangCommand.command_to_run);
-        "%s 2>&1" (ClangCommand.prepend_arg "-###" args' |> ClangCommand.command_to_run);
   Logging.out "clang -### invocation: %s@\n" clang_hashhashhash;
   let normalized_commands = ref [];
   let one_line line =>
     if (string_is_prefix " \"" line) {
+      let cmd =
         /* massage line to remove edge-cases for splitting */
         "\"" ^ line ^ " \"" |>
         /* split by whitespace */
         Str.split (Str.regexp_string "\" \"") |> Array.of_list |>
-        ClangCommand.mk ClangQuotes.EscapedDoubleQuotes
+        ClangCommand.mk ClangQuotes.EscapedDoubleQuotes;
+      Command cmd
     } else if (
       Str.string_match (Str.regexp "clang[^ :]*: warning: ") line 0
     ) {
-        ClangCommand.ClangWarning line
+      ClangWarning line
     } else {
-        ClangCommand.ClangError line
+      ClangError line
     };
   let commands_or_errors =
     /* commands generated by `clang -### ...` start with ' "/absolute/path/to/binary"' */
@@ -59,57 +59,19 @@ let normalize (args: array string) :list ClangCommand.t =>
   /* collect stdout and stderr output together (in reverse order) */
   with_process_in clang_hashhashhash consume_input |> ignore;
   normalized_commands := IList.rev !normalized_commands;
-    /* Discard assembly commands. This may make the list of commands empty, in which case we'll run
-       the original clang command. We could be smarter about this and try to execute the assembly
-       commands with our own clang. */
-    IList.filter
-      (
-        fun
-        | ClangCommand.Assembly asm_cmd => {
-            Logging.out "Skipping assembly command %s@\n" (ClangCommand.command_to_run asm_cmd);
-            false
-          }
-        | _ => true
-      )
   !normalized_commands
-  | Assembly _ =>
-    /* discard assembly commands -- see above */
-    Logging.out "InferClang got toplevel assembly command@\n";
-    []
-  | ClangError _
-  | ClangWarning _ =>
-    /* we cannot possibly get this from the command-line... */
-    assert false
 };
 
-let execute_clang_command (clang_cmd: ClangCommand.t) => {
+let exec_action_item =
-  /* reset logging, otherwise we might print into the logs of the previous file that was compiled */
+  fun
-  Logging.set_log_file_identifier CommandLineOption.Clang None;
+  | ClangError error => {
-  switch clang_cmd {
-  | CC1 args =>
-    /* this command compiles some code; replace the invocation of clang with our own clang and
-       plugin */
-    Logging.out "Capturing -cc1 command: %s@\n" (ClangCommand.command_to_run args);
-    Capture.capture args
-  | ClangError error =>
       /* An error in the output of `clang -### ...`. Outputs the error and fail. This is because
          `clang -###` pretty much never fails, but warns of failures on stderr instead. */
       Logging.err "%s" error;
       exit 1
-  | ClangWarning warning => Logging.err "%s@\n" warning
-  | Assembly args =>
-    /* We shouldn't get any assembly command at this point */
-    (if Config.debug_mode {failwithf} else {Logging.err})
-      "WARNING: unexpected assembly command: %s@\n" (ClangCommand.command_to_run args)
-  | NonCCCommand args =>
-    /* Non-compilation (eg, linking) command. Run the command as-is. It will not get captured
-       further since `clang -### ...` will only output commands that invoke binaries using their
-       absolute paths. */
-    let argv = ClangCommand.get_orig_argv args;
-    Logging.out "Executing raw command: %s@\n" (String.concat " " (Array.to_list argv));
-    Process.create_process_and_wait argv
     }
-};
+  | ClangWarning warning => Logging.err "%s@\n" warning
+  | Command clang_cmd => Capture.capture clang_cmd;
 
 let exe args xx_suffix => {
   /* make sure args.(0) points to clang in facebook-clang-plugins */
@@ -126,7 +88,7 @@ let exe args xx_suffix => {
       true
     | None => false
     };
-  IList.iter execute_clang_command commands;
+  IList.iter exec_action_item commands;
   if (commands == [] || should_run_original_command) {
     if (commands == []) {
       /* No command to execute after -###, let's execute the original command

infer/tests/build_systems/build_integration_tests.py

@@ -63,7 +63,6 @@ ALL_TESTS = [
     'ant',
     'assembly',
     'buck',
-    'cc1',
     'cmake',
     'componentkit',
     'delete',
@@ -477,26 +476,6 @@ class BuildIntegrationTest(unittest.TestCase):
                'infer_args': reactive_args},
               {'compile': ['analyze']}])
 
-    def test_clang_cc1(self):
-        def preprocess():
-            hashhashhash = subprocess.check_output(
-                [CLANG_BIN, '-###', '-c', 'hello.c'],
-                # `clang -### -c hello.c` prints on stderr
-                stderr=subprocess.STDOUT)
-            # pick the line containing the compilation command, which
-            # should be the only one to include "-cc1"
-            cc1_line = filter(lambda s: s.find('"-cc1"') != -1,
-                              hashhashhash.splitlines())[0]
-            # [cc1_line] usually looks like ' "/foo/clang" "bar" "baz"'.
-            # return ['clang', 'bar', 'baz']
-            cmd = [s.strip('"') for s in cc1_line.strip().split('" "')]
-            cmd[0] = 'clang'
-            return [{'compile': cmd}]
-        test('cc1', 'clang -cc1',
-             CODETOANALYZE_DIR,
-             [],
-             preprocess=preprocess)
-
     def test_clang_assembly(self):
         test('assembly', 'compile with assembly code', CODETOANALYZE_DIR,
              [{'compile': ['clang', '-x', 'c', '-c', 'hello.c', '-x',

infer/tests/build_systems/expected_outputs/cc1_report.json

@@ -1,7 +0,0 @@
-[
-  {
-    "bug_type": "NULL_DEREFERENCE",
-    "file": "hello.c",
-    "procedure": "test"
-  }
-]