From 733a29e44b9cc3ffab367fa305890de036eda06f Mon Sep 17 00:00:00 2001 From: Nikos Gorogiannis Date: Mon, 10 Jun 2019 05:55:37 -0700 Subject: [PATCH] [buckjava2] Move genrule capture integration logic from shell to OCaml Summary: Move genrule capture integration logic from shell to OCaml. Also, stop relying on side-effects of buck compilation for constructing the infer-deps.txt file used for merging. Now this is obtained by passing `--show-output` to buck, which spits out the `buck-out` output paths to the targets we asked to build. Reviewed By: ezgicicek Differential Revision: D15715608 fbshipit-source-id: 8fa896ba6 --- infer/src/base/Config.mli | 2 + infer/src/integration/Buck.ml | 48 ++++++++++++--- infer/src/integration/Buck.mli | 8 +++ infer/src/integration/BuckGenrule.ml | 59 +++++++++++++++++++ infer/src/integration/BuckGenrule.mli | 11 ++++ infer/src/integration/Driver.ml | 12 +++- infer/src/integration/Driver.mli | 1 + infer/tests/build_systems/genrulecapture/DEFS | 8 +-- .../build_systems/genrulecapture/Makefile | 6 +- scripts/genrule_run.sh | 57 ------------------ 10 files changed, 137 insertions(+), 75 deletions(-) create mode 100644 infer/src/integration/BuckGenrule.ml create mode 100644 infer/src/integration/BuckGenrule.mli delete mode 100755 scripts/genrule_run.sh diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index 34aa73217..47d0b5c32 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -89,6 +89,8 @@ val etc_dir : string val events_dir_name : string +val exe_basename : string + val fail_on_issue_exit_code : int val frontend_stats_dir_name : string diff --git a/infer/src/integration/Buck.ml b/infer/src/integration/Buck.ml index 58b4f41a0..ecb0af298 100644 --- a/infer/src/integration/Buck.ml +++ b/infer/src/integration/Buck.ml @@ -72,6 +72,7 @@ module Query = struct | Set of string list | Target of string | Union of expr list + | Labelfilter of {label: string; expr: expr} exception NotATarget @@ -96,6 +97,8 @@ module Query = struct Union exprs + let label_filter ~label expr = Labelfilter {label; expr} + let rec pp fmt = function | Target s -> F.pp_print_string fmt s @@ -109,11 +112,15 @@ module Query = struct F.fprintf fmt "set(%a)" (Pp.seq F.pp_print_string) sl | Union exprs -> Pp.seq ~sep:" + " pp fmt exprs + | Labelfilter {label; expr} -> + F.fprintf fmt "attrfilter(labels, %s, %a)" label pp expr - let exec expr = + let exec ?(buck_config = []) expr = let query = F.asprintf "%a" pp expr in - let cmd = ["buck"; "query"] @ List.rev_append Config.buck_build_args_no_inline [query] in + let cmd = + ("buck" :: "query" :: buck_config) @ List.rev_append Config.buck_build_args_no_inline [query] + in let tmp_prefix = "buck_query_" in let debug = L.(debug Capture Medium) in Utils.with_process_lines ~debug ~cmd ~tmp_prefix ~f:Fn.id @@ -142,6 +149,7 @@ let parameters_with_argument = let get_accepted_buck_kinds_pattern () = if Option.is_some Config.buck_compilation_database then "^(apple|cxx)_(binary|library|test)$" + else if Config.genrule_master_mode then "^(java|android)_library$" else "^(apple|cxx)_(binary|library)$" @@ -149,11 +157,31 @@ let max_command_line_length = 50 let die_if_empty f = function [] -> f L.(die UserError) | l -> l +let buck_config = + lazy + ( if Config.genrule_master_mode then + [ "infer.project_root=" ^ Utils.realpath Config.project_root + ; "infer.infer_out=" ^ Utils.realpath Config.results_dir + ; "infer.version=" ^ Version.versionString + ; "infer.infer_bin=" ^ Config.bin_dir ^/ Config.exe_basename + ; "infer.mode=capture" ] + |> List.fold ~init:[] ~f:(fun acc f -> "--config" :: f :: acc) + else [] ) + + +(** for genrule_master_mode, this is the label expected on the capture genrules *) +let infer_enabled_label = "infer_enabled" + +(** for genrule_master_mode, this is the target name suffix for the capture genrules *) +let genrule_suffix = "_infer" + let resolve_pattern_targets ~filter_kind ~dep_depth targets = targets |> List.rev_map ~f:Query.target |> Query.set |> (match dep_depth with None -> Fn.id | Some depth -> Query.deps ?depth) |> (if filter_kind then Query.kind ~pattern:(get_accepted_buck_kinds_pattern ()) else Fn.id) - |> Query.exec + |> (if Config.genrule_master_mode then Query.label_filter ~label:infer_enabled_label else Fn.id) + |> Query.exec ~buck_config:(Lazy.force buck_config) + |> (if Config.genrule_master_mode then List.rev_map ~f:(fun s -> s ^ genrule_suffix) else Fn.id) |> die_if_empty (fun die -> die "*** buck query returned no targets.") @@ -210,9 +238,7 @@ let inline_argument_files buck_args = List.concat_map ~f:expand_buck_arg buck_args -type flavored_arguments = {command: string; rev_not_targets: string list; targets: string list} - -let add_flavors_to_buck_arguments ~filter_kind ~dep_depth ~extra_flavors original_buck_args = +let parse_command_and_targets ~filter_kind ~dep_depth original_buck_args = let expanded_buck_args = inline_argument_files original_buck_args in let command, args = split_buck_command expanded_buck_args in let buck_targets_blacklist_regexp = @@ -259,13 +285,21 @@ let add_flavors_to_buck_arguments ~filter_kind ~dep_depth ~extra_flavors origina ~f:(fun re -> List.filter ~f:(fun tgt -> not (Str.string_match re tgt 0)) targets) buck_targets_blacklist_regexp in + (command, parsed_args.rev_not_targets', targets) + + +type flavored_arguments = {command: string; rev_not_targets: string list; targets: string list} + +let add_flavors_to_buck_arguments ~filter_kind ~dep_depth ~extra_flavors original_buck_args = + let command, rev_not_targets, targets = + parse_command_and_targets ~filter_kind ~dep_depth original_buck_args + in match targets with | [] -> L.(die UserError) "ERROR: no targets found in Buck command `%a`." (Pp.seq F.pp_print_string) original_buck_args | _ -> - let rev_not_targets = parsed_args.rev_not_targets' in let targets = List.rev_map targets ~f:(fun t -> Target.(t |> of_string |> add_flavor ~extra_flavors |> to_string) ) diff --git a/infer/src/integration/Buck.mli b/infer/src/integration/Buck.mli index 57bd4d3d8..3616a7e04 100644 --- a/infer/src/integration/Buck.mli +++ b/infer/src/integration/Buck.mli @@ -7,6 +7,14 @@ open! IStd +val buck_config : string list Lazy.t + +val parse_command_and_targets : + filter_kind:[< `Yes | `No | `Auto] + -> dep_depth:int option option + -> string list + -> string * string list * string list + type flavored_arguments = {command: string; rev_not_targets: string list; targets: string list} val add_flavors_to_buck_arguments : diff --git a/infer/src/integration/BuckGenrule.ml b/infer/src/integration/BuckGenrule.ml new file mode 100644 index 000000000..f86f621d0 --- /dev/null +++ b/infer/src/integration/BuckGenrule.ml @@ -0,0 +1,59 @@ +(* + * Copyright (c) 2019-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd +module F = Format +module L = Logging + +let write_infer_deps infile = + let out_line out_channel line = + match String.split ~on:' ' line with + | [target; target_output_path] -> + Printf.fprintf out_channel "%s\t-\t%s\n" target (Config.project_root ^/ target_output_path) + | _ -> + L.die ExternalError "Couldn't parse buck target output: %s@." line + in + let infer_deps = Config.(results_dir ^/ buck_infer_deps_file_name) in + Utils.with_file_out infer_deps ~f:(fun out_channel -> + Utils.with_file_in infile ~f:(In_channel.iter_lines ~f:(out_line out_channel)) ) + + +let run_buck_capture cmd = + let buck_output_file = Filename.temp_file "buck_output" ".log" in + let shell_cmd = List.map ~f:Escape.escape_shell cmd |> String.concat ~sep:" " in + let shell_cmd_redirected = Printf.sprintf "%s >'%s'" shell_cmd buck_output_file in + match Utils.with_process_in shell_cmd_redirected Utils.consume_in |> snd with + | Ok () -> + write_infer_deps buck_output_file ; Unix.unlink buck_output_file + | Error _ as err -> + L.(die ExternalError) + "*** Buck genrule capture failed to execute: %s@\n***@." + (Unix.Exit_or_signal.to_string_hum err) + + +let capture build_cmd = + let prog, buck_cmd = (List.hd_exn build_cmd, List.tl_exn build_cmd) in + L.progress "Querying buck for genrule capture targets...@." ; + let time0 = Mtime_clock.counter () in + let command, args, targets = + Buck.parse_command_and_targets ~filter_kind:`Yes ~dep_depth:(Some None) buck_cmd + in + L.progress "Found %d genrule capture targets in %a.@." (List.length targets) Mtime.Span.pp + (Mtime_clock.count time0) ; + let all_args = List.rev_append args targets in + let updated_buck_cmd = + (* make buck tell us where in buck-out are the capture directories for merging *) + (prog :: command :: "--show-output" :: Lazy.force Buck.buck_config) + @ List.rev_append Config.buck_build_args_no_inline (Buck.store_args_in_file all_args) + in + L.(debug Capture Quiet) + "Processed buck command '%a'@." (Pp.seq F.pp_print_string) updated_buck_cmd ; + let time0 = Mtime_clock.counter () in + run_buck_capture updated_buck_cmd ; + L.progress "Genrule capture took %a.@." Mtime.Span.pp (Mtime_clock.count time0) ; + RunState.set_merge_capture true ; + RunState.store () diff --git a/infer/src/integration/BuckGenrule.mli b/infer/src/integration/BuckGenrule.mli new file mode 100644 index 000000000..dc945700a --- /dev/null +++ b/infer/src/integration/BuckGenrule.mli @@ -0,0 +1,11 @@ +(* + * Copyright (c) 2019-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd + +val capture : string list -> unit +(** do genrule capture with the given buck command line *) diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index fb1748238..da3dede43 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -17,6 +17,7 @@ module F = Format type mode = | Analyze | BuckGenrule of string + | BuckGenruleMaster of string list | BuckCompilationDB of string * string list | Clang of Clang.compiler * string * string list | ClangCompilationDB of [`Escaped of string | `Raw of string] list @@ -33,6 +34,8 @@ let pp_mode fmt = function F.fprintf fmt "Analyze driver mode" | BuckGenrule prog -> F.fprintf fmt "BuckGenRule driver mode:@\nprog = '%s'" prog + | BuckGenruleMaster build_cmd -> + F.fprintf fmt "BuckGenrule driver mode:@\nbuild command = %a" Pp.cli_args build_cmd | BuckCompilationDB (prog, args) -> F.fprintf fmt "BuckCompilationDB driver mode:@\nprog = '%s'@\nargs = %a" prog Pp.cli_args args @@ -200,6 +203,9 @@ let capture ~changed_files = function | BuckGenrule path -> L.progress "Capturing for Buck genrule compatibility...@." ; JMain.from_arguments path + | BuckGenruleMaster build_cmd -> + L.progress "Capturing for BuckGenruleMaster integration...@." ; + BuckGenrule.capture build_cmd | Clang (compiler, prog, args) -> if CLOpt.is_originator then L.progress "Capturing in make/cc mode...@." ; Clang.capture compiler ~prog ~args @@ -399,9 +405,7 @@ let analyze_and_report ?suppress_console_report ~changed_files mode = | PythonCapture (BBuck, _) when Config.flavors && InferCommand.equal Run Config.command -> (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *) true - | Analyze when Config.genrule_master_mode -> - true - | Analyze -> + | Analyze | BuckGenruleMaster _ -> RunState.get_merge_capture () | _ -> (* else rely on the command line value *) Config.merge @@ -523,6 +527,8 @@ let mode_of_build_command build_cmd = "WARNING: the linters require --buck-compilation-database to be set.@ Alternatively, \ set --no-linters to disable them and this warning.@." ; PythonCapture (BBuck, build_cmd) + | BBuck when Config.genrule_master_mode -> + BuckGenruleMaster build_cmd | (BAnt | BBuck | BGradle | BNdk | BXcode) as build_system -> PythonCapture (build_system, build_cmd) ) diff --git a/infer/src/integration/Driver.mli b/infer/src/integration/Driver.mli index 6e5dc2ab8..cc9f1e8d5 100644 --- a/infer/src/integration/Driver.mli +++ b/infer/src/integration/Driver.mli @@ -14,6 +14,7 @@ open! IStd type mode = | Analyze | BuckGenrule of string + | BuckGenruleMaster of string list | BuckCompilationDB of string * string list | Clang of Clang.compiler * string * string list | ClangCompilationDB of [`Escaped of string | `Raw of string] list diff --git a/infer/tests/build_systems/genrulecapture/DEFS b/infer/tests/build_systems/genrulecapture/DEFS index 67cad39bd..c59d9afd2 100644 --- a/infer/tests/build_systems/genrulecapture/DEFS +++ b/infer/tests/build_systems/genrulecapture/DEFS @@ -43,17 +43,15 @@ def _infer_capture_genrule( "echo {} >> {}".format(arg, args_file) for arg in args ] + [ - " ".join([_get_infer_bin(), "@" + args_file]), - # need to use a cross-platform kind of flock to avoid fragmentation - 'echo -e "_\\t_\\t$OUT" >> {}'.format(_get_infer_deps()) + " ".join([_get_infer_bin(), "@" + args_file]) ] genrule( - name = name + "_infer_capture", + name = name + "_infer", srcs = srcs, cmd = " && ".join(subcommands), out = "infer_out", - labels = ["infer_capture_genrule"], + labels = ["infer_genrule"], ) diff --git a/infer/tests/build_systems/genrulecapture/Makefile b/infer/tests/build_systems/genrulecapture/Makefile index 1b45af772..b1666eee2 100644 --- a/infer/tests/build_systems/genrulecapture/Makefile +++ b/infer/tests/build_systems/genrulecapture/Makefile @@ -8,17 +8,17 @@ ROOT_DIR = $(TESTS_DIR)/../.. ANALYSE = $(ROOT_DIR)/scripts/genrule_run.sh BUCK_TARGET = //module2:module2 -INFER_OUT = $(shell pwd)/infer-out CLEAN_EXTRA = buck-out SOURCES = $(shell find . -name '*.java') INFERPRINT_OPTIONS = --issues-tests -INFER_OPTIONS = --debug-exceptions +INFER_OPTIONS = --genrule-master-mode --debug-exceptions include $(TESTS_DIR)/infer.make $(INFER_OUT)/report.json: $(MAKEFILE_LIST) $(SOURCES) $(QUIET) $(REMOVE_DIR) buck-out && \ $(call silent_on_success,Testing genrule capture integration in $(TEST_REL_DIR),\ - $(ANALYSE) $(BUCK_TARGET) $(INFER_OUT) $(INFER_OPTIONS)) + $(INFER_BIN) --results-dir $(@D) $(INFER_OPTIONS) -- \ + buck build --no-cache $(BUCK_TARGET)) diff --git a/scripts/genrule_run.sh b/scripts/genrule_run.sh deleted file mode 100755 index e6dfda3f1..000000000 --- a/scripts/genrule_run.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2019-present, Facebook, Inc. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -set -e -set -x - -BUCK=buck -GENRULE_SUFFIX="_infer_capture" -BUCK_KIND_PATTERN="^(java|android)_library$" -INFER_BIN="${INFER_BIN:-infer}" - -INFER_VERSION=$(${INFER_BIN} --version | head -1 | cut -f3 -d' ') - -ROOT_TARGET="${1?Must specify a root target.}" -shift -QUERY="kind('${BUCK_KIND_PATTERN}', deps('${ROOT_TARGET}'))" -QUERY="attrfilter(labels, infer_enabled, ${QUERY})" - -INFER_OUT="${1?Must specify an infer out location.}" -if [[ "$INFER_OUT" != /* ]] ; then - echo "Must use absolute path for infer out location." - exit 1 -fi -shift - -BUCK_CONFIG="--config infer.project_root=${PWD}" -BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_out=${INFER_OUT}" -BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_bin=${INFER_BIN}" -BUCK_CONFIG="${BUCK_CONFIG} --config infer.enabled=True" -BUCK_CONFIG="${BUCK_CONFIG} --config infer.version=${INFER_VERSION}" - -# prepare infer-out, mainly for runstate -$INFER_BIN -o "${INFER_OUT}" > /dev/null 2>&1 - -TARGET_FILE=$(mktemp) -trap "{ rm -f $TARGET_FILE; }" EXIT - -echo "Running buck query." -$BUCK query ${BUCK_CONFIG} "${QUERY}" | sed "s/\$/${GENRULE_SUFFIX}/" > "${TARGET_FILE}" - -if [ -s "${TARGET_FILE}" ] -then - echo "Found $(wc -l < ${TARGET_FILE}) targets." -else - echo "Zero targets found!" - exit 1 -fi - -echo 'Running genrule capture under buck.' -$BUCK build --no-cache ${BUCK_CONFIG} "@${TARGET_FILE}" - -echo 'Running merge and analysis.' -$INFER_BIN analyze --genrule-master-mode -o "${INFER_OUT}" "$@"