From e48cc3a370c4f9766c23d4d14f148fbcfc5e917e Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Fri, 23 Mar 2018 09:50:10 -0700 Subject: [PATCH] [capture] store compilation db arguments one per line Summary: Infer reads the arguments passed to clang, in particular to filter out some incompatible clang command-line options. But, infer only understands arguments in arg files if they are presented one per line. That's usually the case, but infer itself stores clang arguments from compilation databases all on one line. This breaks filtering. This changes the type of compilation database items to a list of arguments, so that they can be stored one per line when possible. Also does some cleanup/renamings, and remove trailing `_` from temp file names, eg: clang_command_.tmp.2b2602.txt -> clang_command.tmp.2b2602.txt Also, do not escape arguments from arg files when printing them in logs (that was useless and ugly). Reviewed By: mbouaziz Differential Revision: D7365907 fbshipit-source-id: 5a3fe70 --- infer/src/base/CommandLineOption.ml | 2 +- infer/src/base/Escape.ml | 20 ++++++++++++++++ infer/src/base/Escape.mli | 3 +++ infer/src/base/Pp.ml | 3 +-- infer/src/base/Utils.ml | 23 +------------------ infer/src/base/Utils.mli | 2 -- infer/src/integration/Buck.ml | 2 +- .../integration/CaptureCompilationDatabase.ml | 19 +++++++-------- infer/src/integration/ClangQuotes.ml | 8 +++++-- infer/src/integration/CompilationDatabase.ml | 14 +++++------ infer/src/integration/CompilationDatabase.mli | 10 +++++++- infer/src/integration/Javac.ml | 4 ++-- 12 files changed, 61 insertions(+), 49 deletions(-) diff --git a/infer/src/base/CommandLineOption.ml b/infer/src/base/CommandLineOption.ml index 4f8f96a01..a9784d425 100644 --- a/infer/src/base/CommandLineOption.ml +++ b/infer/src/base/CommandLineOption.ml @@ -927,7 +927,7 @@ let parse ?config_file ~usage action initial_command = environment contributes to the length of the command to be run. If the environment + CLI is too big, running any command will fail with a cryptic "exit code 127" error. Use an argfile to prevent this from happening *) - let file = Filename.temp_file "args_" "" in + let file = Filename.temp_file "args" "" in Out_channel.with_file file ~f:(fun oc -> Out_channel.output_lines oc argv_to_export) ; if not !keep_args_file then Utils.unlink_file_on_exit file ; "@" ^ file ) diff --git a/infer/src/base/Escape.ml b/infer/src/base/Escape.ml index 1e130758e..3503f5700 100644 --- a/infer/src/base/Escape.ml +++ b/infer/src/base/Escape.ml @@ -12,6 +12,8 @@ open! IStd (** Escape a string for use in a CSV or XML file: replace reserved characters with escape sequences *) +module F = Format + (** apply a map function for escape sequences *) let escape_map map_fun s = let needs_escape = String.exists ~f:(fun c -> Option.is_some (map_fun c)) s in @@ -134,3 +136,21 @@ let escape_double_quotes s = escape_map (function '"' -> Some "\\\"" | _ -> None let escape_in_single_quotes s = Printf.sprintf "'%s'" (escape_map (function '\'' -> Some "'\\''" | _ -> None) s) + + +let escape_shell = + let no_quote_needed = Str.regexp "^[A-Za-z0-9-_%/:,.]+$" in + let easy_single_quotable = Str.regexp "^[^']+$" in + let easy_double_quotable = Str.regexp "^[^$`\\!]+$" in + function + | "" -> + "''" + | arg -> + if Str.string_match no_quote_needed arg 0 then arg + else if Str.string_match easy_single_quotable arg 0 then F.sprintf "'%s'" arg + else if Str.string_match easy_double_quotable arg 0 then + escape_double_quotes arg |> F.sprintf "\"%s\"" + else + (* ends on-going single quote, output single quote inside double quotes, then open a new + single quote *) + escape_map (function '\'' -> Some "'\"'\"'" | _ -> None) arg |> F.sprintf "'%s'" diff --git a/infer/src/base/Escape.mli b/infer/src/base/Escape.mli index 0d13f5c40..a8eafba34 100644 --- a/infer/src/base/Escape.mli +++ b/infer/src/base/Escape.mli @@ -37,3 +37,6 @@ val escape_double_quotes : string -> string val escape_in_single_quotes : string -> string (** put the string inside single quotes and escape the single quotes within that string *) + +val escape_shell : string -> string +(** escape the string so it can be passed to the shell without remorse *) diff --git a/infer/src/base/Pp.ml b/infer/src/base/Pp.ml index 31f75bebd..a2ff12a35 100644 --- a/infer/src/base/Pp.ml +++ b/infer/src/base/Pp.ml @@ -137,8 +137,7 @@ let to_string ~f fmt x = string fmt (f x) let cli_args fmt args = let pp_args fmt args = F.fprintf fmt "@[ " ; - seq ~sep:"" ~print_env:{text with break_lines= true} string fmt - (List.map args ~f:Escape.escape_in_single_quotes) ; + seq ~sep:"" ~print_env:{text with break_lines= true} string fmt args ; F.fprintf fmt "@]@\n" in let rec pp_argfile_args in_argfiles fmt args = diff --git a/infer/src/base/Utils.ml b/infer/src/base/Utils.ml index b0bf641fa..4f7defdb0 100644 --- a/infer/src/base/Utils.ml +++ b/infer/src/base/Utils.ml @@ -208,29 +208,8 @@ let with_process_in command read = do_finally_swallow_timeout ~f ~finally -let shell_escape_command = - let no_quote_needed = Str.regexp "^[A-Za-z0-9-_%/:,.]+$" in - let easy_single_quotable = Str.regexp "^[^']+$" in - let easy_double_quotable = Str.regexp "^[^$`\\!]+$" in - let escape = function - | "" -> - "''" - | arg -> - if Str.string_match no_quote_needed arg 0 then arg - else if Str.string_match easy_single_quotable arg 0 then F.sprintf "'%s'" arg - else if Str.string_match easy_double_quotable arg 0 then - arg |> Escape.escape_double_quotes |> F.sprintf "\"%s\"" - else - (* ends on-going single quote, output single quote inside double quotes, then open a new single - quote *) - arg |> Escape.escape_map (function '\'' -> Some "'\"'\"'" | _ -> None) - |> F.sprintf "'%s'" - in - fun cmd -> List.map ~f:escape cmd |> String.concat ~sep:" " - - let with_process_lines ~(debug: ('a, F.formatter, unit) format -> 'a) ~cmd ~tmp_prefix ~f = - let shell_cmd = shell_escape_command cmd in + let shell_cmd = List.map ~f:Escape.escape_shell cmd |> String.concat ~sep:" " in let verbose_err_file = Filename.temp_file tmp_prefix ".err" in let shell_cmd_redirected = Printf.sprintf "%s 2>'%s'" shell_cmd verbose_err_file in debug "Trying to execute: %s@\n%!" shell_cmd_redirected ; diff --git a/infer/src/base/Utils.mli b/infer/src/base/Utils.mli index a0b01e9ec..88b3f84e5 100644 --- a/infer/src/base/Utils.mli +++ b/infer/src/base/Utils.mli @@ -66,8 +66,6 @@ val echo_in : In_channel.t -> unit val with_process_in : string -> (In_channel.t -> 'a) -> 'a * Unix.Exit_or_signal.t -val shell_escape_command : string list -> string - val with_process_lines : debug:((string -> unit, Format.formatter, unit) format -> string -> unit) -> cmd:string list -> tmp_prefix:string -> f:(string list -> 'res) -> 'res diff --git a/infer/src/integration/Buck.ml b/infer/src/integration/Buck.ml index 6a0071f77..6549a9e76 100644 --- a/infer/src/integration/Buck.ml +++ b/infer/src/integration/Buck.ml @@ -259,7 +259,7 @@ let rec exceed_length ~max = function let store_args_in_file args = if exceed_length ~max:max_command_line_length args then ( - let file = Filename.temp_file "buck_targets_" ".txt" in + let file = Filename.temp_file "buck_targets" ".txt" in let write_args outc = Out_channel.output_string outc (String.concat ~sep:"\n" args) in let () = Utils.with_file_out file ~f:write_args in L.(debug Capture Quiet) "Buck targets options stored in file '%s'@\n" file ; diff --git a/infer/src/integration/CaptureCompilationDatabase.ml b/infer/src/integration/CaptureCompilationDatabase.ml index 5e5847641..769663b2b 100644 --- a/infer/src/integration/CaptureCompilationDatabase.ml +++ b/infer/src/integration/CaptureCompilationDatabase.ml @@ -12,17 +12,18 @@ module F = Format module CLOpt = CommandLineOption module L = Logging -type cmd = {cwd: string; prog: string; args: string} - let create_cmd (compilation_data: CompilationDatabase.compilation_data) = - let swap_command cmd = + let swap_executable cmd = if String.is_suffix ~suffix:"++" cmd then Config.wrappers_dir ^/ "clang++" else Config.wrappers_dir ^/ "clang" in let arg_file = - ClangQuotes.mk_arg_file "cdb_clang_args_" ClangQuotes.EscapedNoQuotes [compilation_data.args] + ClangQuotes.mk_arg_file "cdb_clang_args" ClangQuotes.EscapedNoQuotes + compilation_data.escaped_arguments in - {cwd= compilation_data.dir; prog= swap_command compilation_data.command; args= arg_file} + { CompilationDatabase.directory= compilation_data.directory + ; executable= swap_executable compilation_data.executable + ; escaped_arguments= ["@" ^ arg_file; "-fsyntax-only"] } (* A sentinel is a file which indicates that a failure occurred in another infer process. @@ -33,7 +34,7 @@ let sentinel_exists sentinel_opt = Option.value_map ~default:false sentinel_opt ~f:file_exists -let invoke_cmd ~fail_sentinel cmd = +let invoke_cmd ~fail_sentinel (cmd: CompilationDatabase.compilation_data) = let create_sentinel_if_needed () = let create_empty_file fname = Utils.with_file_out ~f:(fun _ -> ()) fname in Option.iter fail_sentinel ~f:create_empty_file @@ -42,9 +43,9 @@ let invoke_cmd ~fail_sentinel cmd = else try let pid = - let prog = cmd.prog in - let argv = [prog; "@" ^ cmd.args; "-fsyntax-only"] in - Spawn.(spawn ~cwd:(Path cmd.cwd) ~prog ~argv ()) + let open Spawn in + spawn ~cwd:(Path cmd.directory) ~prog:cmd.executable + ~argv:(cmd.executable :: cmd.escaped_arguments) () in match Unix.waitpid (Pid.of_int pid) with | Ok () -> diff --git a/infer/src/integration/ClangQuotes.ml b/infer/src/integration/ClangQuotes.ml index 60d05147f..e3b7be3b4 100644 --- a/infer/src/integration/ClangQuotes.ml +++ b/infer/src/integration/ClangQuotes.ml @@ -32,9 +32,13 @@ let quote style = let mk_arg_file prefix style args = let file = Filename.temp_file prefix ".txt" in let write_args outc = - List.map ~f:(quote style) args |> String.concat ~sep:" " |> Out_channel.output_string outc + List.iter + ~f:(fun arg -> + quote style arg |> Out_channel.output_string outc ; + Out_channel.newline outc ) + args in - Utils.with_file_out file ~f:write_args |> ignore ; + Utils.with_file_out file ~f:write_args ; L.(debug Capture Medium) "Clang options stored in file %s@\n" file ; if not Config.debug_mode then Utils.unlink_file_on_exit file ; file diff --git a/infer/src/integration/CompilationDatabase.ml b/infer/src/integration/CompilationDatabase.ml index 81c29be02..f352b7987 100644 --- a/infer/src/integration/CompilationDatabase.ml +++ b/infer/src/integration/CompilationDatabase.ml @@ -10,7 +10,7 @@ open! IStd module L = Logging -type compilation_data = {dir: string; command: string; args: string} +type compilation_data = {directory: string; executable: string; escaped_arguments: string list} type t = compilation_data SourceFile.Map.t ref @@ -27,7 +27,7 @@ let parse_command_and_arguments command_and_arguments = let index = Str.search_forward regexp command_and_arguments 0 in let command = Str.string_before command_and_arguments (index + 1) in let arguments = Str.string_after command_and_arguments (index + 1) in - (command, arguments) + (command, [arguments]) (** Parse the compilation database json file into the compilationDatabase @@ -91,7 +91,7 @@ let decode_json_file (database: t) json_format = "the value of the \"arguments\" field is an empty list in command %s" (Yojson.Basic.to_string json) | cmd :: args -> - command := Some (cmd, Utils.shell_escape_command args) ) + command := Some (cmd, List.map ~f:Escape.escape_shell args) ) | "arguments", json -> exit_format_error "the value of the \"arguments\" field is not a list; found '%s' instead" @@ -104,7 +104,7 @@ let decode_json_file (database: t) json_format = match json with | `Assoc fields -> List.iter ~f:one_field fields ; - let dir = + let directory = match !directory with | Some directory -> directory @@ -120,7 +120,7 @@ let decode_json_file (database: t) json_format = exit_format_error "no \"file\" entry found in command %s" (Yojson.Basic.to_string json) in - let command, args = + let executable, escaped_arguments = match !command with | Some x -> x @@ -128,8 +128,8 @@ let decode_json_file (database: t) json_format = exit_format_error "no \"command\" or \"arguments\" entry found in command %s" (Yojson.Basic.to_string json) in - let compilation_data = {dir; command; args} in - let abs_file = if Filename.is_relative file then dir ^/ file else file in + let compilation_data = {directory; executable; escaped_arguments} in + let abs_file = if Filename.is_relative file then directory ^/ file else file in let source_file = SourceFile.from_abs_path abs_file in database := SourceFile.Map.add source_file compilation_data !database | _ -> diff --git a/infer/src/integration/CompilationDatabase.mli b/infer/src/integration/CompilationDatabase.mli index 9b330dbcc..0e3d464e0 100644 --- a/infer/src/integration/CompilationDatabase.mli +++ b/infer/src/integration/CompilationDatabase.mli @@ -11,7 +11,15 @@ open! IStd type t -type compilation_data = {dir: string; command: string; args: string} +type compilation_data = + { directory: string + ; executable: string + ; escaped_arguments: string list + (** argument list, where each argument is already escaped for the shell. This is because in + some cases the argument list contains arguments that are actually themselves a list of + arguments, for instance because the compilation database only contains a "command" + entry. *) + } val filter_compilation_data : t -> f:(SourceFile.t -> bool) -> compilation_data list diff --git a/infer/src/integration/Javac.ml b/infer/src/integration/Javac.ml index 5cec61cde..e3b46241f 100644 --- a/infer/src/integration/Javac.ml +++ b/infer/src/integration/Javac.ml @@ -44,9 +44,9 @@ let compile compiler build_prog build_args = let cli_file_args = cli_args @ ["@" ^ args_file] in let args = prog_args @ cli_file_args in L.(debug Capture Quiet) "Current working directory: '%s'@." (Sys.getcwd ()) ; - let verbose_out_file = Filename.temp_file "javac_" ".out" in + let verbose_out_file = Filename.temp_file "javac" ".out" in let try_run cmd error_k = - let shell_cmd = Utils.shell_escape_command cmd in + let shell_cmd = List.map ~f:Escape.escape_shell cmd |> String.concat ~sep:" " in let shell_cmd_redirected = Printf.sprintf "%s 2>'%s'" shell_cmd verbose_out_file in L.(debug Capture Quiet) "Trying to execute: %s@." shell_cmd_redirected ; match Utils.with_process_in shell_cmd_redirected In_channel.input_all with