[buck] java & clang combined capture

Summary:
A buck integration for capturing simultaneously clang and java targets.
Just like the java-specific `JavaGenruleCapture` integration, it relies on
dummy targets that depend on the flavoured clang versions.

For example, a `cxx_library` target named `//clang:hello` will have an associated target
called `//clang:hello_infer` that depends on `//clang:hello#infer-capture-all`,
and whose output is a text file containing the output path of the dependency.

Reviewed By: jvillard

Differential Revision: D21620458

fbshipit-source-id: 23919387b
master
Nikos Gorogiannis 5 years ago committed by Facebook GitHub Bot
parent 090ce4edb9
commit 2bbd25087c

@ -126,6 +126,10 @@ BUCK OPTIONS
Activates: Buck integration for clang-based targets
(C/C++/Objective-C/Objective-C++). (Conversely: --no-buck-clang)
--buck-combined
Activates: Buck integration for clang-based and Java targets.
(Conversely: --no-buck-combined)
--buck-compilation-database { no-deps | deps }
Buck integration using the compilation database, with or without
dependencies. Only includes clang targets, as per Buck's

@ -143,6 +143,10 @@ OPTIONS
(C/C++/Objective-C/Objective-C++). (Conversely: --no-buck-clang)
See also infer-capture(1).
--buck-combined
Activates: Buck integration for clang-based and Java targets.
(Conversely: --no-buck-combined) See also infer-capture(1).
--buck-compilation-database { no-deps | deps }
Buck integration using the compilation database, with or without
dependencies. Only includes clang targets, as per Buck's

@ -143,6 +143,10 @@ OPTIONS
(C/C++/Objective-C/Objective-C++). (Conversely: --no-buck-clang)
See also infer-capture(1).
--buck-combined
Activates: Buck integration for clang-based and Java targets.
(Conversely: --no-buck-combined) See also infer-capture(1).
--buck-compilation-database { no-deps | deps }
Buck integration using the compilation database, with or without
dependencies. Only includes clang targets, as per Buck's

@ -20,10 +20,14 @@ let pp_clang_compilation_db_deps fmt = function
F.pp_print_string fmt "DepsAllDepths"
type t = ClangFlavors | ClangCompilationDB of clang_compilation_db_deps | JavaGenruleMaster
let is_java_genrule_master = function
| JavaGenruleMaster ->
type t =
| CombinedGenrule
| ClangFlavors
| ClangCompilationDB of clang_compilation_db_deps
| JavaGenruleMaster
let is_java_genrule_master_or_combined = function
| JavaGenruleMaster | CombinedGenrule ->
true
| ClangFlavors | ClangCompilationDB _ ->
false
@ -32,12 +36,12 @@ let is_java_genrule_master = function
let is_clang_compilation_db = function
| ClangCompilationDB _ ->
true
| ClangFlavors | JavaGenruleMaster ->
| ClangFlavors | JavaGenruleMaster | CombinedGenrule ->
false
let is_clang_flavors = function
| ClangFlavors ->
true
| ClangCompilationDB _ | JavaGenruleMaster ->
| ClangCompilationDB _ | JavaGenruleMaster | CombinedGenrule ->
false

@ -13,9 +13,13 @@ type clang_compilation_db_deps = NoDependencies | DepsUpToDepth of int | DepsAll
val pp_clang_compilation_db_deps : F.formatter -> clang_compilation_db_deps -> unit
type t = ClangFlavors | ClangCompilationDB of clang_compilation_db_deps | JavaGenruleMaster
type t =
| CombinedGenrule
| ClangFlavors
| ClangCompilationDB of clang_compilation_db_deps
| JavaGenruleMaster
val is_java_genrule_master : t -> bool
val is_java_genrule_master_or_combined : t -> bool
val is_clang_compilation_db : t -> bool

@ -727,6 +727,11 @@ and buck_mode =
clang targets, as per Buck's $(i,#compilation-database) flavor."
~symbols:[("no-deps", `NoDeps); ("deps", `DepsTmp)]
|> ignore ;
CLOpt.mk_bool ~long:"buck-combined"
~in_help:InferCommand.[(Capture, manual_buck)]
~f:(set_mode `CombinedGenrule)
"Buck integration for clang-based and Java targets."
|> ignore ;
buck_mode
@ -2590,6 +2595,8 @@ and buck_mode : BuckMode.t option =
Some (ClangCompilationDB DepsAllDepths)
| `ClangCompilationDB `DepsTmp, Some depth ->
Some (ClangCompilationDB (DepsUpToDepth depth))
| `CombinedGenrule, _ ->
Some CombinedGenrule
and buck_targets_blacklist = !buck_targets_blacklist

@ -81,7 +81,7 @@ module Target = struct
match (mode, command) with
| ClangCompilationDB _, _ ->
add_flavor_internal target "compilation-database"
| ClangFlavors, Compile | JavaGenruleMaster, _ ->
| ClangFlavors, Compile | JavaGenruleMaster, _ | CombinedGenrule, _ ->
target
| ClangFlavors, _ ->
add_flavor_internal target "infer-capture-all"
@ -191,12 +191,14 @@ let parameters_with_argument =
let get_accepted_buck_kinds_pattern (mode : BuckMode.t) =
match mode with
| CombinedGenrule ->
"^(android|apple|cxx|java)_(binary|library)$"
| ClangCompilationDB _ ->
"^(apple|cxx)_(binary|library|test)$"
| JavaGenruleMaster ->
"^(java|android)_library$"
| ClangFlavors ->
"^(apple|cxx)_(binary|library)$"
| JavaGenruleMaster ->
"^(java|android)_library$"
let max_command_line_length = 50
@ -209,15 +211,14 @@ let infer_enabled_label = "infer_enabled"
(** for genrule_master_mode, this is the target name suffix for the capture genrules *)
let genrule_suffix = "_infer"
let config buck_mode =
let config =
let clang_path =
List.fold ["clang"; "install"; "bin"; "clang"] ~init:Config.fcp_dir ~f:Filename.concat
in
let args =
match (buck_mode : BuckMode.t) with
| JavaGenruleMaster ->
let get_java_genrule_config () =
["infer.version=" ^ Version.versionString; "infer.mode=capture"]
| ClangFlavors ->
in
let get_flavors_config () =
[ "client.id=infer.clang"
; Printf.sprintf "*//infer.infer_bin=%s" Config.bin_dir
; Printf.sprintf "*//infer.clang_compiler=%s" clang_path
@ -236,6 +237,16 @@ let config buck_mode =
else
[ Printf.sprintf "*//infer.blacklist_regex=(%s)"
(String.concat ~sep:")|(" Config.buck_blacklist) ]
in
fun buck_mode ->
let args =
match (buck_mode : BuckMode.t) with
| JavaGenruleMaster ->
get_java_genrule_config ()
| ClangFlavors ->
get_flavors_config ()
| CombinedGenrule ->
get_java_genrule_config () @ get_flavors_config ()
| ClangCompilationDB _ ->
[]
in
@ -247,17 +258,18 @@ let resolve_pattern_targets (buck_mode : BuckMode.t) ~filter_kind targets =
|> ( match buck_mode with
| ClangFlavors | ClangCompilationDB NoDependencies ->
Fn.id
| JavaGenruleMaster | ClangCompilationDB DepsAllDepths ->
| CombinedGenrule | ClangCompilationDB DepsAllDepths | JavaGenruleMaster ->
Query.deps None
| ClangCompilationDB (DepsUpToDepth depth) ->
Query.deps (Some depth) )
|> (if filter_kind then Query.kind ~pattern:(get_accepted_buck_kinds_pattern buck_mode) else Fn.id)
|> ( if BuckMode.is_java_genrule_master buck_mode then
|> ( if BuckMode.is_java_genrule_master_or_combined buck_mode then
Query.label_filter ~label:infer_enabled_label
else Fn.id )
|> Query.exec ~buck_config:(config buck_mode)
|>
if BuckMode.is_java_genrule_master buck_mode then List.rev_map ~f:(fun s -> s ^ genrule_suffix)
if BuckMode.is_java_genrule_master_or_combined buck_mode then
List.rev_map ~f:(fun s -> s ^ genrule_suffix)
else Fn.id
@ -346,11 +358,12 @@ let parse_command_and_targets (buck_mode : BuckMode.t) ~filter_kind original_buc
let targets =
match (filter_kind, buck_mode, parsed_args) with
| ( (`No | `Auto)
, (ClangFlavors | JavaGenruleMaster)
, (ClangFlavors | JavaGenruleMaster | CombinedGenrule)
, {pattern_targets= []; alias_targets= []; normal_targets} ) ->
normal_targets
| `No, (ClangFlavors | JavaGenruleMaster), {pattern_targets= []; alias_targets; normal_targets}
->
| ( `No
, (ClangFlavors | JavaGenruleMaster | CombinedGenrule)
, {pattern_targets= []; alias_targets; normal_targets} ) ->
alias_targets |> resolve_alias_targets |> List.rev_append normal_targets
| (`Yes | `No | `Auto), _, {pattern_targets; alias_targets; normal_targets} ->
let filter_kind = match filter_kind with `No -> false | `Yes | `Auto -> true in

@ -42,7 +42,7 @@ module L = Logging
]
*)
(** Read the build report json file buck produced, and parse into a sorted list of pairs
(** Read the build report json file buck produced, and parse into a list of pairs
[(target, output-path)]. NB contrary to what buck documentation says, the output path is always
present even when the target is locally cached. *)
let read_and_parse_report build_report =
@ -78,9 +78,56 @@ let read_and_parse_report build_report =
| _ ->
None
in
Yojson.Basic.from_file build_report
|> get_json_field "results" |> Option.bind ~f:parse_results
|> Option.map ~f:(List.stable_sort ~compare:[%compare: string * string])
Yojson.Basic.from_file build_report |> get_json_field "results" |> Option.bind ~f:parse_results
(** Function for processing paths in a buck build report and generating an [infer-deps.txt] file.
Given a pair [(buck_target, output_path)],
- if [output_path] contains a capture DB, then generate the appropriate deps line;
- if [output_path] contains an [infer-deps.txt] file, expand and inline it;
- if [output_path] is a dummy target used in the combined genrule integration for clang targets,
read its contents, parse them as an output directory path and apply the above two tests to
that *)
let expand_target acc (target, target_path) =
let expand_dir acc (target, target_path) =
(* invariant: [target_path] is absolute *)
let db_file = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDB in
match Sys.file_exists db_file with
| `Yes ->
(* there is a capture DB at this path, so terminate expansion and generate deps line *)
let line = Printf.sprintf "%s\t-\t%s" target target_path in
line :: acc
| `No | `Unknown -> (
(* no capture DB was found, so look for, and inline, an [infer-deps.txt] file *)
let infer_deps = ResultsDirEntryName.get_path ~results_dir:target_path BuckDependencies in
match Sys.file_exists infer_deps with
| `Yes ->
Utils.with_file_in infer_deps
~f:(In_channel.fold_lines ~init:acc ~f:(fun acc line -> line :: acc))
| `No | `Unknown ->
L.internal_error "No capture DB or infer-deps file in %s@." target_path ;
acc )
in
let target_path =
if Filename.is_absolute target_path then target_path else Config.project_root ^/ target_path
in
match Sys.is_directory target_path with
| `Yes ->
(* output path is directory, so should contain either a capture DB or an [infer-deps.txt] file *)
expand_dir acc (target, target_path)
| `No | `Unknown -> (
(* output path is not a directory, so assume it's an intermediate genrule output containing the
output path of the underlying capture target *)
match Utils.read_file target_path with
| Ok [new_target_path] ->
expand_dir acc (target, new_target_path)
| Ok _ ->
L.internal_error "Couldn't parse intermediate deps file %s@." target_path ;
acc
| Error error ->
L.internal_error "Error %s@\nCouldn't read intermediate deps file %s@." error target_path ;
acc )
let infer_deps_of_build_report build_report =
@ -88,12 +135,13 @@ let infer_deps_of_build_report build_report =
| None ->
L.die InternalError "Couldn't parse buck build report: %s@." build_report
| Some target_path_list ->
let out_line out_channel (target, target_output_path) =
Printf.fprintf out_channel "%s\t-\t%s\n" target (Config.project_root ^/ target_output_path)
let infer_deps_lines =
List.fold target_path_list ~init:[] ~f:expand_target
|> List.dedup_and_sort ~compare:String.compare
in
let infer_deps = ResultsDir.get_path BuckDependencies in
Utils.with_file_out infer_deps ~f:(fun out_channel ->
List.iter target_path_list ~f:(out_line out_channel) )
Out_channel.output_lines out_channel infer_deps_lines )
let run_buck_capture cmd =
@ -106,12 +154,12 @@ let run_buck_capture cmd =
Buck.wrap_buck_call ~extend_env ~label:"build" cmd |> ignore
let capture build_cmd =
let capture buck_mode build_cmd =
let prog, buck_cmd = (List.hd_exn build_cmd, List.tl_exn build_cmd) in
L.progress "Querying buck for genrule capture targets...@." ;
let time0 = Mtime_clock.counter () in
let command, args, targets =
Buck.parse_command_and_targets JavaGenruleMaster ~filter_kind:`Yes buck_cmd
Buck.parse_command_and_targets buck_mode ~filter_kind:`Yes buck_cmd
in
L.progress "Found %d genrule capture targets in %a.@." (List.length targets) Mtime.Span.pp
(Mtime_clock.count time0) ;
@ -121,7 +169,7 @@ let capture build_cmd =
in
let updated_buck_cmd =
(* make buck tell us where in buck-out are the capture directories for merging *)
(prog :: command :: "--build-report" :: build_report_file :: Buck.config JavaGenruleMaster)
(prog :: command :: "--build-report" :: build_report_file :: Buck.config buck_mode)
@ List.rev_append Config.buck_build_args_no_inline (Buck.store_args_in_file all_args)
in
L.(debug Capture Quiet)

@ -7,5 +7,5 @@
open! IStd
val capture : string list -> unit
val capture : BuckMode.t -> string list -> unit
(** do genrule capture with the given buck command line *)

@ -18,6 +18,7 @@ type mode =
| Analyze
| Ant of {prog: string; args: string list}
| BuckClangFlavor of {build_cmd: string list}
| BuckCombinedGenrule of {build_cmd: string list}
| BuckCompilationDB of {deps: BuckMode.clang_compilation_db_deps; prog: string; args: string list}
| BuckGenrule of {prog: string}
| BuckGenruleMaster of {build_cmd: string list}
@ -39,6 +40,8 @@ let pp_mode fmt = function
F.fprintf fmt "Ant driver mode:@\nprog = '%s'@\nargs = %a" prog Pp.cli_args args
| BuckClangFlavor {build_cmd} ->
F.fprintf fmt "BuckClangFlavor driver mode: build_cmd = %a" Pp.cli_args build_cmd
| BuckCombinedGenrule {build_cmd} ->
F.fprintf fmt "BuckCombinedGenrule driver mode: build_cmd = %a" Pp.cli_args build_cmd
| BuckCompilationDB {deps; prog; args} ->
F.fprintf fmt "BuckCompilationDB driver mode:@\nprog = '%s'@\nargs = %a@\ndeps = %a" prog
Pp.cli_args args BuckMode.pp_clang_compilation_db_deps deps
@ -119,6 +122,9 @@ let capture ~changed_files = function
| BuckClangFlavor {build_cmd} ->
L.progress "Capturing in buck mode...@." ;
BuckFlavors.capture build_cmd
| BuckCombinedGenrule {build_cmd} ->
L.progress "Capturing in buck combined genrule mode...@." ;
BuckGenrule.capture CombinedGenrule build_cmd
| BuckCompilationDB {deps; prog; args} ->
L.progress "Capturing using Buck's compilation database...@." ;
let json_cdb =
@ -130,7 +136,7 @@ let capture ~changed_files = function
JMain.from_arguments prog
| BuckGenruleMaster {build_cmd} ->
L.progress "Capturing for BuckGenruleMaster integration...@." ;
BuckGenrule.capture build_cmd
BuckGenrule.capture JavaGenruleMaster build_cmd
| Clang {compiler; prog; args} ->
if CLOpt.is_originator then L.progress "Capturing in make/cc mode...@." ;
Clang.capture compiler ~prog ~args
@ -246,7 +252,7 @@ let analyze_and_report ?suppress_console_report ~changed_files mode =
&& InferCommand.equal Run Config.command ->
(* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *)
true
| Analyze | BuckGenruleMaster _ ->
| Analyze | BuckGenruleMaster _ | BuckCombinedGenrule _ ->
ResultsDir.RunState.get_merge_capture ()
| _ ->
false
@ -285,6 +291,8 @@ let assert_supported_mode required_analyzer requested_mode_string =
match required_analyzer with
| `Clang ->
Version.clang_enabled
| `ClangJava ->
Version.clang_enabled && Version.java_enabled
| `Java ->
Version.java_enabled
| `Xcode ->
@ -295,6 +303,8 @@ let assert_supported_mode required_analyzer requested_mode_string =
match required_analyzer with
| `Clang ->
"clang"
| `ClangJava ->
"clang & java"
| `Java ->
"java"
| `Xcode ->
@ -326,6 +336,8 @@ let assert_supported_build_system build_system =
match Config.buck_mode with
| None ->
error_no_buck_mode_specified ()
| Some CombinedGenrule ->
(`ClangJava, "buck combined genrule")
| Some ClangFlavors ->
(`Clang, "buck with flavors")
| Some (ClangCompilationDB _) ->
@ -357,6 +369,8 @@ let mode_of_build_command build_cmd (buck_mode : BuckMode.t option) =
Ant {prog; args}
| BBuck, None ->
error_no_buck_mode_specified ()
| BBuck, Some CombinedGenrule ->
BuckCombinedGenrule {build_cmd}
| BBuck, Some (ClangCompilationDB deps) ->
BuckCompilationDB {deps; prog; args= List.append args (List.rev Config.buck_build_args)}
| BBuck, Some ClangFlavors when Config.is_checker_enabled Linters ->

@ -15,6 +15,7 @@ type mode =
| Analyze
| Ant of {prog: string; args: string list}
| BuckClangFlavor of {build_cmd: string list}
| BuckCombinedGenrule of {build_cmd: string list}
| BuckCompilationDB of {deps: BuckMode.clang_compilation_db_deps; prog: string; args: string list}
| BuckGenrule of {prog: string}
| BuckGenruleMaster of {build_cmd: string list}

@ -0,0 +1,2 @@
[buildfile]
includes = //DEFS

@ -0,0 +1,91 @@
import os
original_cxx_library = cxx_library
original_java_library = java_library
infer_clang_flavor = "infer-capture-all"
def _cxx_infer_capture_genrule(name):
cmd = "echo $(location :{}#{}) > $OUT".format(name, infer_clang_flavor)
genrule(
name = name + "_infer",
out = "dummy_out",
cmd = cmd,
)
def cxx_library(name, **kwargs):
_cxx_infer_capture_genrule(name)
new_kwargs = dict(kwargs, labels=kwargs.get("labels", []) + ["infer_enabled"])
original_cxx_library(
name=name,
**new_kwargs
)
def _get_project_root():
return "\$(git rev-parse --show-toplevel)/infer/tests/build_systems/buck_combined"
def _infer_capture_genrule(
name,
srcs
):
args = [
"--jobs",
"1",
"--genrule-mode",
"--quiet",
"--no-progress-bar",
"--results-dir",
"$OUT",
"--sourcepath",
"$SRCDIR",
"--project-root",
_get_project_root(),
"--classpath",
"$(classpath :{})".format(name),
"--generated-classes",
"$(location :{})".format(name),
"capture",
]
args_file = os.path.join("$TMP", "args.txt")
subcommands = [
"echo {} >> {}".format(arg, args_file)
for arg in args
] + [
"infer @" + args_file
]
genrule(
name = name + "_infer",
srcs = srcs,
cmd = " && ".join(subcommands),
out = "infer_out",
labels = ["infer_genrule"],
)
def _make_infer_capture_genrule(name, kwargs):
java_sources = [
f
for f in kwargs.get("srcs", [])
if f.endswith(".java")
]
if java_sources != []:
_infer_capture_genrule(name, java_sources)
kwargs["labels"] = kwargs.get("labels", []) + ["infer_enabled"]
return kwargs
def java_library(name, **kwargs):
new_kwargs = _make_infer_capture_genrule(name, kwargs)
original_java_library(
name=name,
**new_kwargs
)

@ -0,0 +1,26 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
TESTS_DIR = ../..
ROOT_DIR = $(TESTS_DIR)/../..
BUCK_TARGET = //clang:hello //java:hello
SOURCES = clang/hello.c clang/hello2.c
OBJECTS = buck-out/gen/clang/hello\#compile-hello.c.o1f717d69,default/hello.c.o
INFER_OPTIONS = --report-custom-error --developer-mode --no-linters --buck-combined
INFERPRINT_OPTIONS = --issues-tests
CLEAN_EXTRA = buck-out
include $(TESTS_DIR)/infer.make
$(OBJECTS): $(SOURCES)
$(QUIET)$(call silent_on_success,Compiling Buck flavors tests,\
$(BUCK) build --no-cache $(BUCK_TARGET))
infer-out/report.json: $(CLANG_DEPS) $(SOURCES) $(MAKEFILE_LIST)
$(QUIET)$(REMOVE_DIR) buck-out && \
$(call silent_on_success,Testing infer-run Buck combined genrule integration,\
$(INFER_BIN) $(INFER_OPTIONS) run --results-dir $(CURDIR)/infer-out -- \
$(BUCK) build --no-cache $(BUCK_TARGET))

@ -0,0 +1,6 @@
cxx_library(
name = 'hello',
srcs = [
'hello.c', 'hello2.c',
],
)

@ -0,0 +1,13 @@
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <stdlib.h>
void test() {
int* s = NULL;
*s = 42;
}

@ -0,0 +1,13 @@
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <stdlib.h>
void test2() {
int* s = NULL;
*s = 42;
}

@ -0,0 +1,3 @@
clang/hello.c, test, 2, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure test()]
clang/hello2.c, test2, 2, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure test2()]
java/NullDeref.java, NullDeref.callToStringWithNullBad():java.lang.String, 1, NULL_DEREFERENCE, no_bucket, ERROR, [start of procedure callToStringWithNullBad(),start of procedure myToString(...),Skipping toString(): unknown method]

@ -0,0 +1,6 @@
java_library(
name='hello',
srcs=glob(["*.java"]),
deps=[],
visibility=['PUBLIC'],
)

@ -0,0 +1,16 @@
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
public class NullDeref {
private static String myToString(Object foo) {
return foo.toString();
}
public String callToStringWithNullBad() {
return myToString(null);
}
}
Loading…
Cancel
Save