[buck] Unify buck command line arguments recognition, buck query invocation, and support target patterns and aliases

Summary:
:
Make both buck capture and compilation database handle buck command line arguments and invoke buck query the same way.
Plus allow:
- target patterns `//some/dir:` and `//some/dir/...`. However since `//some/dir:#flavor` and `//some/dir/...#flavor` are not supported, they need to be expanded before adding the infer flavor.
- target aliases (defined in `.buckconfig`)
- shortcuts `//some/dir` rewritten to `//some/dir:dir`
- relative path `some/dir:name` rewritten to `//some/dir:name`

Reviewed By: jvillard

Differential Revision: D5321087

fbshipit-source-id: 48876d4
master
Mehdi Bouaziz 7 years ago committed by Facebook Github Bot
parent 6c9cee700b
commit a2f69050ac

@ -129,3 +129,5 @@ let escape_filename s =
in
escape_map map s
let escape_double_quotes s = escape_map (function '"' -> Some "\\\"" | _ -> None) s

@ -31,3 +31,6 @@ val escape_url : string -> string
val escape_filename : string -> string
(** escape a string to be used as a file name *)
val escape_double_quotes : string -> string
(** replaces double-quote with backslash double-quote *)

@ -221,13 +221,46 @@ let with_process_in command read =
do_finally_swallow_timeout ~f ~finally
let shell_escape_command cmd =
let escape arg =
let shell_escape_command =
let no_quote_needed = Str.regexp "^[A-Za-z0-9-_%/:,.]+$" in
let easy_single_quotable = Str.regexp "^[^']+$" in
let easy_double_quotable = Str.regexp "^[^$`\\!]+$" in
let escape = function
| "" ->
"''"
| arg ->
if Str.string_match no_quote_needed arg 0 then arg
else if Str.string_match easy_single_quotable arg 0 then F.sprintf "'%s'" arg
else if Str.string_match easy_double_quotable arg 0 then arg |> Escape.escape_double_quotes
|> F.sprintf "\"%s\""
else
(* ends on-going single quote, output single quote inside double quotes, then open a new single
quote *)
Escape.escape_map (function '\'' -> Some "'\"'\"'" | _ -> None) arg |> Printf.sprintf "'%s'"
arg |> Escape.escape_map (function '\'' -> Some "'\"'\"'" | _ -> None)
|> F.sprintf "'%s'"
in
List.map ~f:escape cmd |> String.concat ~sep:" "
fun cmd -> List.map ~f:escape cmd |> String.concat ~sep:" "
let with_process_lines ~(debug: ('a, F.formatter, unit) format -> 'a) ~cmd ~tmp_prefix ~f =
let shell_cmd = shell_escape_command cmd in
let verbose_err_file = Filename.temp_file tmp_prefix ".err" in
let shell_cmd_redirected = Printf.sprintf "%s 2>'%s'" shell_cmd verbose_err_file in
debug "Trying to execute: %s@\n%!" shell_cmd_redirected ;
let input_lines chan = In_channel.input_lines ~fix_win_eol:true chan in
let res = with_process_in shell_cmd_redirected input_lines in
let verbose_errlog = with_file_in verbose_err_file ~f:In_channel.input_all in
if not (String.equal verbose_errlog "") then
debug "@\nlog:@\n<<<<<<@\n%s@\n>>>>>>@\n%!" verbose_errlog ;
match res with
| lines, Ok () ->
f lines
| lines, (Error _ as err) ->
let output = String.concat ~sep:"\n" lines in
L.(die ExternalError)
"*** Failed to execute: %s@\n*** Command: %s@\n*** Output:@\n%s@."
(Unix.Exit_or_signal.to_string_hum err)
shell_cmd output
(** Create a directory if it does not exist already. *)

@ -67,6 +67,12 @@ val with_process_in : string -> (In_channel.t -> 'a) -> 'a * Unix.Exit_or_signal
val shell_escape_command : string list -> string
val with_process_lines :
debug:((string -> unit, Format.formatter, unit) format -> string -> unit) -> cmd:string list
-> tmp_prefix:string -> f:(string list -> 'res) -> 'res
(** Runs the command [cmd] and calls [f] on the output lines. Uses [debug] to print debug
information, and [tmp_prefix] as a prefix for temporary files. *)
val create_dir : string -> unit
(** create a directory if it does not exist already *)

@ -8,11 +8,13 @@
*)
open! IStd
module F = Format
module L = Logging
type target = {name: string; flavors: string list}
module Target = struct
type t = {name: string; flavors: string list}
let target_of_string target =
let of_string target =
match String.split target ~on:'#' with
| [name; flavors_string] ->
let flavors = String.split flavors_string ~on:',' in
@ -23,81 +25,157 @@ let target_of_string target =
L.(die ExternalError) "cannot parse target %s" target
let string_of_target {name; flavors} =
let pp_string fmt s = Format.fprintf fmt "%s" s in
Format.asprintf "%s#%a" name (Pp.comma_seq pp_string) flavors
let to_string {name; flavors} = F.asprintf "%s#%a" name (Pp.comma_seq Pp.string) flavors
let is_target_string =
let target_regexp = Str.regexp "[^/]*//[^/]+.*:.*" in
fun s -> Str.string_match target_regexp s 0
let no_targets_found_error_and_exit buck_cmd =
Process.print_error_and_exit
"No targets found in Buck command %s.@\nOnly fully qualified Buck targets are supported. In particular, aliases are not allowed.@."
(String.concat ~sep:" " buck_cmd)
let add_flavor_to_target target =
let add flavor =
let add_flavor_internal target flavor =
if List.mem ~equal:String.equal target.flavors flavor then
(* there's already an infer flavor associated to the target, do nothing *)
target
else {target with flavors= flavor :: target.flavors}
in
let add_flavor ~extra_flavors target =
let target = List.fold_left ~f:add_flavor_internal ~init:target extra_flavors in
match (Config.buck_compilation_database, Config.analyzer) with
| Some _, _ ->
add "compilation-database"
add_flavor_internal target "compilation-database"
| None, CompileOnly ->
target
| None, (BiAbduction | CaptureOnly | Checkers | Linters) ->
add "infer-capture-all"
add_flavor_internal target "infer-capture-all"
| None, Crashcontext ->
L.(die UserError)
"Analyzer %s is Java-only; not supported with Buck flavors"
(Config.string_of_analyzer Config.analyzer)
end
let add_flavors_to_buck_command build_cmd =
let add_infer_if_target s (cmd, found_one_target) =
if not (is_target_string s) then (s :: cmd, found_one_target)
else (string_of_target (add_flavor_to_target (target_of_string s)) :: cmd, true)
in
let cmd', found_one_target =
List.fold_right build_cmd ~f:add_infer_if_target ~init:([], false)
in
if not found_one_target then no_targets_found_error_and_exit build_cmd ;
cmd'
let get_dependency_targets_and_add_flavors targets ~depth =
let build_deps_string targets =
List.map targets ~f:(fun target ->
match depth with
| None (* full depth *) ->
Printf.sprintf "deps('%s')" target
| Some n ->
Printf.sprintf "deps('%s', %d)" target n )
|> String.concat ~sep:" union "
let parse_target_string =
let alias_target_regexp = Str.regexp "^[^/:]+\\(#.*\\)?$" in
let pattern_target_regexp = Str.regexp "^[^/]*//\\(\\.\\.\\.\\|.*\\(:\\|/\\.\\.\\.\\)\\)$" in
let normal_target_regexp = Str.regexp "^[^/]*//[^/].*:.+$" in
let noname_target_regexp = Str.regexp "^[^/]*//.*$" in
let parse_with_retry s ~retry =
(* do not consider --buck-options as targets *)
if String.equal s "" || Char.equal s.[0] '-' || Char.equal s.[0] '@' then `NotATarget s
else if Str.string_match alias_target_regexp s 0 then `AliasTarget s
else if Str.string_match pattern_target_regexp s 0 then `PatternTarget s
else if Str.string_match normal_target_regexp s 0 then `NormalTarget s
else if Str.string_match noname_target_regexp s 0 then
let name = String.split s ~on:'/' |> List.last_exn in
`NormalTarget (F.sprintf "%s:%s" s name)
else retry s
in
let buck_query =
[ "buck"
; "query"
; "\"kind('(apple_binary|apple_library|apple_test|cxx_binary|cxx_library|cxx_test)', "
^ build_deps_string targets ^ ")\"" ]
fun s ->
parse_with_retry s ~retry:(fun s ->
parse_with_retry ("//" ^ s) ~retry:(fun s ->
L.(die InternalError) "Do not know how to parse buck command line argument '%s'" s ) )
module Query = struct
type expr =
| Deps of {depth: int option; expr: expr}
| Kind of {pattern: string; expr: expr}
| Set of string list
| Target of string
| Union of expr list
exception NotATarget
let quote_if_needed =
let no_quote_needed_regexp = Str.regexp "^[a-zA-Z0-9/:_*][a-zA-Z0-9/:.-_*]*$" in
fun s ->
if Str.string_match no_quote_needed_regexp s 0 then s
else s |> Escape.escape_double_quotes |> F.sprintf "\"%s\""
let target string = Target (quote_if_needed string)
let kind ~pattern expr = Kind {pattern= quote_if_needed pattern; expr}
let deps ?depth expr = Deps {depth; expr}
let set exprs =
match List.rev_map exprs ~f:(function Target t -> t | _ -> raise NotATarget) with
| targets ->
Set targets
| exception NotATarget ->
Union exprs
let rec pp fmt = function
| Target s ->
Pp.string fmt s
| Kind {pattern; expr} ->
F.fprintf fmt "kind(%s, %a)" pattern pp expr
| Deps {depth= None; expr} ->
F.fprintf fmt "deps(%a)" pp expr (* full depth *)
| Deps {depth= Some depth; expr} ->
F.fprintf fmt "deps(%a, %d)" pp expr depth
| Set sl ->
F.fprintf fmt "set(%a)" (Pp.seq Pp.string) sl
| Union exprs ->
Pp.seq ~sep:" + " pp fmt exprs
let exec expr =
let query = F.asprintf "%a" pp expr in
let cmd = ["buck"; "query"; query] in
let tmp_prefix = "buck_query_" in
let debug = L.(debug Capture Medium) in
Utils.with_process_lines ~debug ~cmd ~tmp_prefix ~f:Fn.id
end
let accepted_buck_commands = ["build"]
let parameters_with_argument =
["--build-report"; "--config"; "-j"; "--num-threads"; "--out"; "-v"; "--verbose"]
let accepted_buck_kinds_pattern = "(apple|cxx)_(binary|library|test)"
let max_command_line_length = 50
let die_if_empty f = function [] -> f L.(die UserError) | l -> l
let resolve_pattern_targets ~filter_kind ~dep_depth targets =
targets |> List.rev_map ~f:Query.target |> Query.set
|> (match dep_depth with None -> Fn.id | Some depth -> Query.deps ?depth)
|> (if filter_kind then Query.kind ~pattern:accepted_buck_kinds_pattern else Fn.id) |> Query.exec
|> die_if_empty (fun die -> die "*** buck query returned no targets.")
let resolve_alias_targets aliases =
let debug = L.(debug Capture Medium) in
(* we could use buck query to resolve aliases but buck targets --resolve-alias is faster *)
let cmd = "buck" :: "targets" :: "--resolve-alias" :: aliases in
let tmp_prefix = "buck_targets_" in
let on_result_lines =
die_if_empty (fun die ->
die "*** No alias found for: '%a'." (Pp.seq ~sep:"', '" Pp.string) aliases )
in
let buck_query_cmd = String.concat buck_query ~sep:" " in
Logging.(debug Linters Quiet) "*** Executing command:@\n*** %s@." buck_query_cmd ;
let output, exit_or_signal = Utils.with_process_in buck_query_cmd In_channel.input_lines in
match exit_or_signal with
| Error _ as status ->
Logging.(die ExternalError)
"*** command failed:@\n*** %s@\n*** %s@." buck_query_cmd
(Unix.Exit_or_signal.to_string_hum status)
| Ok () ->
List.map output ~f:(fun name ->
string_of_target (add_flavor_to_target {name; flavors= Config.append_buck_flavors}) )
Utils.with_process_lines ~debug ~cmd ~tmp_prefix ~f:on_result_lines
type parsed_args =
{ rev_not_targets': string list
; normal_targets: string list
; alias_targets: string list
; pattern_targets: string list }
let empty_parsed_args =
{rev_not_targets'= []; normal_targets= []; alias_targets= []; pattern_targets= []}
let split_buck_command buck_cmd =
match buck_cmd with
| command :: args when List.mem ~equal:String.equal accepted_buck_commands command ->
(command, args)
| _ ->
L.(die UserError)
"ERROR: cannot parse buck command `%a`. Expected %a." (Pp.seq Pp.string) buck_cmd
(Pp.seq ~sep:" or " Pp.string) accepted_buck_commands
(** Given a list of arguments return the extended list of arguments where
@ -120,12 +198,72 @@ let inline_argument_files buck_args =
List.concat_map ~f:expand_buck_arg buck_args
let store_targets_in_file buck_targets =
type flavored_arguments = {command: string; rev_not_targets: string list; targets: string list}
let add_flavors_to_buck_arguments ~filter_kind ~dep_depth ~extra_flavors original_buck_args =
let expanded_buck_args = inline_argument_files original_buck_args in
let command, args = split_buck_command expanded_buck_args in
let rec parse_cmd_args parsed_args = function
| [] ->
parsed_args
| param :: arg :: args when List.mem ~equal:String.equal parameters_with_argument param ->
parse_cmd_args
{parsed_args with rev_not_targets'= arg :: param :: parsed_args.rev_not_targets'} args
| target :: args ->
let parsed_args =
match parse_target_string target with
| `NotATarget s ->
{parsed_args with rev_not_targets'= s :: parsed_args.rev_not_targets'}
| `NormalTarget t ->
{parsed_args with normal_targets= t :: parsed_args.normal_targets}
| `AliasTarget a ->
{parsed_args with alias_targets= a :: parsed_args.alias_targets}
| `PatternTarget p ->
{parsed_args with pattern_targets= p :: parsed_args.pattern_targets}
in
parse_cmd_args parsed_args args
in
let parsed_args = parse_cmd_args empty_parsed_args args in
let targets =
match (filter_kind, dep_depth, parsed_args) with
| false, None, {pattern_targets= []; alias_targets= []; normal_targets} ->
normal_targets
| false, None, {pattern_targets= []; alias_targets; normal_targets} ->
alias_targets |> resolve_alias_targets |> List.rev_append normal_targets
| _, _, {pattern_targets; alias_targets; normal_targets} ->
pattern_targets |> List.rev_append alias_targets |> List.rev_append normal_targets
|> resolve_pattern_targets ~filter_kind ~dep_depth
in
match targets with
| [] ->
L.(die UserError)
"ERROR: no targets found in Buck command `%a`." (Pp.seq Pp.string) original_buck_args
| _ ->
let rev_not_targets = parsed_args.rev_not_targets' in
let targets =
List.rev_map targets ~f:(fun t ->
Target.(t |> of_string |> add_flavor ~extra_flavors |> to_string) )
in
{command; rev_not_targets; targets}
let rec exceed_length ~max = function
| _ when max < 0 ->
true
| [] ->
false
| h :: t ->
exceed_length ~max:(max - String.length h) t
let store_args_in_file args =
if exceed_length ~max:max_command_line_length args then
let file = Filename.temp_file "buck_targets_" ".txt" in
let write_args outc = Out_channel.output_string outc (String.concat ~sep:"\n" buck_targets) in
Utils.with_file_out file ~f:write_args |> ignore ;
let write_args outc = Out_channel.output_string outc (String.concat ~sep:"\n" args) in
let () = Utils.with_file_out file ~f:write_args in
L.(debug Capture Quiet) "Buck targets options stored in file '%s'@\n" file ;
Printf.sprintf "@%s" file
[Printf.sprintf "@%s" file]
else args
let filter_compatible subcommand args =

@ -9,30 +9,20 @@
open! IStd
val is_target_string : string -> bool
(** is this a Buck target string, eg //foo/bar:baz or boo//foo/bar:baz *)
type flavored_arguments = {command: string; rev_not_targets: string list; targets: string list}
val no_targets_found_error_and_exit : string list -> unit
(** prints an error that no Buck targets were identified in the given list, and exits *)
val add_flavors_to_buck_command : string list -> string list
(** Add infer flavors to the targets in the given buck command, depending on the infer analyzer. For
val add_flavors_to_buck_arguments :
filter_kind:bool -> dep_depth:int option option -> extra_flavors:string list -> string list
-> flavored_arguments
(** Add infer flavors to the targets in the given buck arguments, depending on the infer analyzer. For
instance, in capture mode, the buck command:
buck build //foo/bar:baz#some,flavor
build //foo/bar:baz#some,flavor
becomes:
buck build //foo/bar:baz#infer-capture-all,some,flavor
build //foo/bar:baz#infer-capture-all,some,flavor
*)
val get_dependency_targets_and_add_flavors : string list -> depth:int option -> string list
(** Runs buck query to get the dependency targets of the given targets
[get_dependency_targets args] = targets with dependent targets, other args *)
val inline_argument_files : string list -> string list
(** Given a list of arguments to buck, return the extended list of arguments where
the args in a file have been extracted *)
val store_targets_in_file : string list -> string
(** Given a list of buck targets, stores them in a file and returns the file name *)
val store_args_in_file : string list -> string list
(** Given a list of arguments, stores them in a file if needed and returns the new command line *)
val filter_compatible : [> `Targets] -> string list -> string list
(** keep only the options compatible with the given Buck subcommand *)

@ -90,45 +90,36 @@ let run_compilation_database compilation_database should_capture_file =
(** Computes the compilation database files. *)
let get_compilation_database_files_buck ~prog ~args =
let all_buck_args = Buck.inline_argument_files args in
let targets, no_targets = List.partition_tf ~f:Buck.is_target_string all_buck_args in
let targets =
match Config.buck_compilation_database with
| Some Deps depth ->
Buck.get_dependency_targets_and_add_flavors targets ~depth
| _ ->
Buck.add_flavors_to_buck_command targets
let dep_depth =
match Config.buck_compilation_database with Some Deps depth -> Some depth | _ -> None
in
match
Buck.add_flavors_to_buck_arguments ~filter_kind:true ~dep_depth
~extra_flavors:Config.append_buck_flavors args
with
| {command= "build" as command; rev_not_targets; targets} ->
let targets_args = Buck.store_args_in_file targets in
let build_args =
command
:: List.rev_append rev_not_targets
("--config" :: "*//cxx.pch_enabled=false" :: targets_args)
in
match no_targets with
| "build" :: no_targets_no_build
-> (
let targets_in_file = Buck.store_targets_in_file targets in
let build_args = no_targets @ ["--config"; "*//cxx.pch_enabled=false"; targets_in_file] in
Logging.(debug Linters Quiet)
"Processed buck command is : 'buck %s'@\n"
(String.concat ~sep:" " build_args) ;
"Processed buck command is: 'buck %a'@\n" (Pp.seq Pp.string) build_args ;
Process.create_process_and_wait ~prog ~args:build_args ;
let buck_targets_shell =
Buck.filter_compatible `Targets no_targets_no_build
|> List.append [prog; "targets"; "--show-output"; targets_in_file]
|> Utils.shell_escape_command
in
let output, exit_or_signal =
Utils.with_process_in buck_targets_shell In_channel.input_lines
prog
:: "targets"
:: List.rev_append
(Buck.filter_compatible `Targets rev_not_targets)
("--show-output" :: targets_args)
in
match exit_or_signal with
| Error _ as status ->
L.(die ExternalError)
"*** command failed:@\n*** %s@\n*** %s@." buck_targets_shell
(Unix.Exit_or_signal.to_string_hum status)
| Ok () ->
match output with
let on_target_lines = function
| [] ->
L.external_error "There are no files to process, exiting@." ;
L.exit 0
L.(die ExternalError) "There are no files to process, exiting"
| lines ->
L.(debug Capture Quiet)
"Reading compilation database from:@\n%s@\n" (String.concat ~sep:"\n" lines) ;
"Reading compilation database from:@\n%a@\n" (Pp.seq ~sep:"\n" Pp.string) lines ;
(* this assumes that flavors do not contain spaces *)
let split_regex = Str.regexp "#[^ ]* " in
let scan_output compilation_database_files line =
@ -139,11 +130,14 @@ let get_compilation_database_files_buck ~prog ~args =
L.(die ExternalError)
"Failed to parse `buck targets --show-output ...` line of output:@\n%s" line
in
List.fold ~f:scan_output ~init:[] lines )
List.fold ~f:scan_output ~init:[] lines
in
Utils.with_process_lines
~debug:L.(debug Capture Quiet)
~cmd:buck_targets_shell ~tmp_prefix:"buck_targets_" ~f:on_target_lines
| _ ->
let cmd = String.concat ~sep:" " (prog :: args) in
Process.print_error_and_exit "Incorrect buck command: %s. Please use buck build <targets>"
cmd
Process.print_error_and_exit "Incorrect buck command: %s %a. Please use buck build <targets>"
prog (Pp.seq Pp.string) args
(** Compute the compilation database files. *)

@ -287,16 +287,15 @@ let capture ~changed_files mode =
(Option.to_list (Sys.getenv CLOpt.args_env_var) @ ["--buck"])
in
Unix.putenv ~key:CLOpt.args_env_var ~data:infer_args_with_buck ;
let all_buck_args = Buck.inline_argument_files build_cmd in
let targets, no_targets =
List.partition_tf ~f:Buck.is_target_string all_buck_args
let prog, buck_args = IList.uncons_exn build_cmd in
let {Buck.command; rev_not_targets; targets} =
Buck.add_flavors_to_buck_arguments ~filter_kind:false ~dep_depth:None
~extra_flavors:[] buck_args
in
let targets_with_flavor = Buck.add_flavors_to_buck_command targets in
let targets_in_file = Buck.store_targets_in_file targets_with_flavor in
let updated_buck_cmd = no_targets @ [targets_in_file] in
let all_args = List.rev_append rev_not_targets targets in
let updated_buck_cmd = prog :: command :: Buck.store_args_in_file all_args in
Logging.(debug Capture Quiet)
"Processed buck command '%s'@\n"
(String.concat ~sep:" " updated_buck_cmd) ;
"Processed buck command '%a'@\n" (Pp.seq Pp.string) updated_buck_cmd ;
updated_buck_cmd
else build_cmd ) )
in

@ -100,3 +100,5 @@ let to_string f l =
let rec aux l = match l with [] -> "" | [s] -> f s | s :: rest -> f s ^ ", " ^ aux rest in
"[" ^ aux l ^ "]"
let uncons_exn = function [] -> failwith "uncons_exn" | hd :: tl -> (hd, tl)

@ -27,3 +27,6 @@ val inter : ('a -> 'a -> int) -> 'a list -> 'a list -> 'a list
(** [inter cmp xs ys] are the elements in both [xs] and [ys], sorted according to [cmp]. *)
val to_string : ('a -> string) -> 'a list -> string
val uncons_exn : 'a list -> 'a * 'a list
(** deconstruct a list, like hd_exn and tl_exn *)

@ -24,4 +24,4 @@ infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
NO_BUCKD=1 \
$(INFER_BIN) -a $(ANALYZER) --stats $(INFER_OPTIONS) -o $(CURDIR)/$(@D) \
--buck-compilation-database no-deps \
-- $(BUCK) build --no-cache '//clang_compilation_database:Hel lo#x86_64' @clang_compilation_database/buck_target_hello_test.txt)
-- $(BUCK) build --no-cache '//clang_compilation_database:Hel lo#default' @clang_compilation_database/buck_target_hello_test.txt)

Loading…
Cancel
Save