diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index af16111fd..b9a827020 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -108,7 +108,9 @@ let assign = "<\"Assign\">" with a direct array access where an error is produced and the analysis continues *) let bound_error_allowed_in_procedure_call = true -let buck_out_gen = "buck-out" ^/ "gen" +let buck_out = "buck-out" + +let buck_out_gen = buck_out ^/ "gen" let buck_results_dir_name = "infer" diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index 5c98799a0..262dffe77 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -189,6 +189,8 @@ val buck_merge_all_deps : bool val buck_mode : BuckMode.t option +val buck_out : string + val buck_out_gen : string val buck_targets_block_list : string list diff --git a/infer/src/integration/BuckBuildReport.ml b/infer/src/integration/BuckBuildReport.ml new file mode 100644 index 000000000..1263204b4 --- /dev/null +++ b/infer/src/integration/BuckBuildReport.ml @@ -0,0 +1,142 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd +module L = Logging + +(* example build report json output + [ + { + "success" : true, + "results" : { + "//annotations:annotations_infer" : { + "success" : true, + "type" : "BUILT_LOCALLY", + "output" : "buck-out/gen/annotations/annotations_infer/infer_out" + }, + "//module2:module2_infer" : { + "success" : true, + "type" : "BUILT_LOCALLY", + "output" : "buck-out/gen/module2/module2_infer/infer_out" + }, + "//module1:module1_infer" : { + "success" : true, + "type" : "BUILT_LOCALLY", + "output" : "buck-out/gen/module1/module1_infer/infer_out" + }, + "//module3:module1_infer" : { + "success" : "SUCCESS", + "type" : "BUILT_LOCALLY", + "outputs" : { + "DEFAULT" : [ "buck-out/gen/module1/module3_infer/infer_out" ] + } + } + }, + "failures" : { } + }% + ] +*) + +(** Read the build report json file buck produced, and parse into a list of pairs + [(target, output-path)]. NB contrary to what buck documentation says, the output path is always + present even when the target is locally cached. *) +let read_and_parse_report build_report = + let get_json_field fieldname = function + | `Assoc fields -> + List.Assoc.find fields ~equal:String.equal fieldname + | _ -> + None + in + let parse_target (target, json) = + let path_opt = + match get_json_field "output" json with + | Some (`String str) -> + Some str + | _ -> ( + match get_json_field "outputs" json |> Option.bind ~f:(get_json_field "DEFAULT") with + | Some (`List [`String str]) -> + Some str + | _ -> + None ) + in + match path_opt with + | None -> + L.internal_error "Could not parse target json: %s@." (Yojson.Basic.to_string json) ; + None + | Some path -> + Some (target, path) + in + let parse_results = function + | `Assoc results -> + (* NB this will simply skip unparseable targets *) + List.filter_map results ~f:parse_target |> Option.some + | _ -> + None + in + Yojson.Basic.from_file build_report |> get_json_field "results" |> Option.bind ~f:parse_results + + +(** Function for processing paths in a buck build report and generating an [infer-deps.txt] file. + Given a pair [(buck_target, output_path)], + + - if [output_path] contains a capture DB, then generate the appropriate deps line; + - if [output_path] contains an [infer-deps.txt] file, expand and inline it; + - if [output_path] is a dummy target used in the combined genrule integration for clang targets, + read its contents, parse them as an output directory path and apply the above two tests to + that *) +let expand_target acc (target, target_path) = + let inline acc path = + Utils.with_file_in path ~f:(In_channel.fold_lines ~init:acc ~f:(fun acc line -> line :: acc)) + in + let expand_dir acc (target, target_path) = + (* invariant: [target_path] is absolute *) + let db_file = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDB in + if ISys.file_exists db_file then + (* we found a capture DB so add this as a target line *) + Printf.sprintf "%s\t-\t%s" target target_path :: acc + else + let infer_deps_file = + ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies + in + if ISys.file_exists infer_deps_file then + (* we found an [infer_deps.txt] file so inline in *) + inline acc infer_deps_file + else ( + (* don't know what to do with this directory *) + L.internal_error "Didn't find capture DB or infer-deps file in path %s.@\n" target_path ; + acc ) + in + let target_path = + if Filename.is_absolute target_path then target_path else Config.project_root ^/ target_path + in + match Sys.is_directory target_path with + | `Yes -> + expand_dir acc (target, target_path) + | _ when String.is_suffix target_path ~suffix:ResultsDirEntryName.buck_infer_deps_file_name -> + (* direct path of an [infer-deps.txt] file, inline *) + inline acc target_path + | _ -> ( + (* assume path is an intermediate genrule output containing the + output path of the underlying capture target *) + match Utils.read_file target_path with + | Ok [new_target_path] -> + expand_dir acc (target, new_target_path) + | Ok _ -> + L.internal_error "Couldn't parse intermediate deps file %s@." target_path ; + acc + | Error error -> + L.internal_error "Error %s@\nCouldn't read intermediate deps file %s@." error target_path ; + acc ) + + +let parse_infer_deps ~build_report_file = + match read_and_parse_report build_report_file with + | None -> + L.die InternalError "Couldn't parse buck build report: %s@." build_report_file + | Some target_path_list -> + List.fold target_path_list ~init:[] ~f:expand_target + |> List.dedup_and_sort ~compare:String.compare diff --git a/infer/src/integration/BuckBuildReport.mli b/infer/src/integration/BuckBuildReport.mli new file mode 100644 index 000000000..240381764 --- /dev/null +++ b/infer/src/integration/BuckBuildReport.mli @@ -0,0 +1,12 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd + +val parse_infer_deps : build_report_file:string -> string list +(** parse a JSON build report by buck and return all capture DBs found in the [infer_deps.txt] + format *) diff --git a/infer/src/integration/BuckFlavors.ml b/infer/src/integration/BuckFlavors.ml index e30ed082d..322602e52 100644 --- a/infer/src/integration/BuckFlavors.ml +++ b/infer/src/integration/BuckFlavors.ml @@ -23,8 +23,8 @@ let add_flavors_to_buck_arguments buck_mode ~extra_flavors original_buck_args = {command; rev_not_targets; targets} -let capture_buck_args () = - ("--show-output" :: (if Config.keep_going then ["--keep-going"] else [])) +let capture_buck_args build_report_file = + ("--build-report" :: build_report_file :: (if Config.keep_going then ["--keep-going"] else [])) @ (match Config.load_average with Some l -> ["-L"; Float.to_string l] | None -> []) @ Buck.config ClangFlavors @ Config.buck_build_args @@ -36,68 +36,19 @@ let run_buck_build prog buck_build_args = ~f:(fun acc arg -> Printf.sprintf "%s%c%s" acc CommandLineOption.env_var_sep arg) in let extend_env = [(CommandLineOption.args_env_var, infer_args)] in - let lines = Buck.wrap_buck_call ~extend_env ~label:"build" (prog :: buck_build_args) in - (* Process a line of buck stdout output, in this case the result of '--show-output' - These paths (may) contain a 'infer-deps.txt' file, which we will later merge - *) - let process_buck_line acc line = - L.debug Capture Verbose "BUCK OUT: %s@." line ; - match String.lsplit2 ~on:' ' line with - | Some (_, infer_deps_path) -> - let full_path = Config.project_root ^/ infer_deps_path in - let dirname = Filename.dirname full_path in - let get_path results_dir = ResultsDirEntryName.get_path ~results_dir CaptureDependencies in - (* Buck can either give the full path to infer-deps.txt ... *) - if ISys.file_exists (get_path dirname) then get_path dirname :: acc - (* ... or a folder which contains infer-deps.txt *) - else if ISys.file_exists (get_path full_path) then get_path full_path :: acc - else acc - | _ -> - L.internal_error "Couldn't parse buck target output: %s@\n" line ; - acc - in - List.fold lines ~init:[] ~f:process_buck_line + Buck.wrap_buck_call ~extend_env ~label:"build" (prog :: buck_build_args) |> ignore -let merge_deps_files depsfiles = - let buck_out = Config.project_root ^/ Config.buck_out_gen in - let depslines, depsfiles = - match depsfiles with - | [] when Config.keep_going || Config.buck_merge_all_deps -> - let infouts = - Utils.fold_folders ~init:[] ~path:buck_out ~f:(fun acc dir -> - if - String.is_substring dir ~substring:"infer-out" - && ISys.file_exists (ResultsDirEntryName.get_path ~results_dir:dir CaptureDB) - then Printf.sprintf "\t\t%s" dir :: acc - else acc ) - in - (infouts, []) - | [] when Config.buck_merge_all_deps -> - let files = - Utils.find_files ~path:buck_out ~extension:ResultsDirEntryName.buck_infer_deps_file_name - in - ([], files) - | _ -> - ([], depsfiles) - in - depslines - @ List.fold depsfiles ~init:[] ~f:(fun acc file -> - List.rev_append acc (Utils.with_file_in file ~f:In_channel.input_lines) ) +let get_all_infer_deps_under_buck_out () = + Utils.fold_folders ~init:[] ~path:(Config.project_root ^/ Config.buck_out) ~f:(fun acc dir -> + if + String.is_substring dir ~substring:"infer-out" + && ISys.file_exists (ResultsDirEntryName.get_path ~results_dir:dir CaptureDB) + then Printf.sprintf "\t\t%s" dir :: acc + else acc ) |> List.dedup_and_sort ~compare:String.compare -let clang_flavor_capture ~prog ~buck_build_cmd = - if Config.keep_going && not Config.continue_capture then - Process.create_process_and_wait ~prog ~args:["clean"] ; - let depsfiles = run_buck_build prog (buck_build_cmd @ capture_buck_args ()) in - let deplines = merge_deps_files depsfiles in - let infer_out_depsfile = ResultsDir.get_path CaptureDependencies in - Utils.with_file_out infer_out_depsfile ~f:(fun out_chan -> - Out_channel.output_lines out_chan deplines ) ; - () - - let capture build_cmd = let prog, buck_args = (List.hd_exn build_cmd, List.tl_exn build_cmd) in (* let children infer processes know that they are inside Buck *) @@ -121,4 +72,16 @@ let capture build_cmd = updated_buck_cmd ; let prog, buck_build_cmd = (prog, updated_buck_cmd) in ResultsDir.RunState.set_merge_capture true ; - clang_flavor_capture ~prog ~buck_build_cmd ) + if Config.keep_going && not Config.continue_capture then + Process.create_process_and_wait ~prog ~args:["clean"] ; + let build_report_file = + Filename.temp_file ~in_dir:(ResultsDir.get_path Temporary) "buck_build_report" ".json" + in + run_buck_build prog (buck_build_cmd @ capture_buck_args build_report_file) ; + let infer_deps_lines = + if Config.buck_merge_all_deps then get_all_infer_deps_under_buck_out () + else BuckBuildReport.parse_infer_deps ~build_report_file + in + let infer_deps = ResultsDir.get_path CaptureDependencies in + Utils.with_file_out infer_deps ~f:(fun out_channel -> + Out_channel.output_lines out_channel infer_deps_lines ) ) diff --git a/infer/src/integration/BuckJavaFlavor.ml b/infer/src/integration/BuckJavaFlavor.ml index 418182ea6..312ef1752 100644 --- a/infer/src/integration/BuckJavaFlavor.ml +++ b/infer/src/integration/BuckJavaFlavor.ml @@ -9,139 +9,6 @@ open! IStd module F = Format module L = Logging -(* example build report json output -[ - { - "success" : true, - "results" : { - "//annotations:annotations_infer" : { - "success" : true, - "type" : "BUILT_LOCALLY", - "output" : "buck-out/gen/annotations/annotations_infer/infer_out" - }, - "//module2:module2_infer" : { - "success" : true, - "type" : "BUILT_LOCALLY", - "output" : "buck-out/gen/module2/module2_infer/infer_out" - }, - "//module1:module1_infer" : { - "success" : true, - "type" : "BUILT_LOCALLY", - "output" : "buck-out/gen/module1/module1_infer/infer_out" - }, - "//module3:module1_infer" : { - "success" : "SUCCESS", - "type" : "BUILT_LOCALLY", - "outputs" : { - "DEFAULT" : [ "buck-out/gen/module1/module3_infer/infer_out" ] - } - } - }, - "failures" : { } -}% -] -*) - -(** Read the build report json file buck produced, and parse into a list of pairs - [(target, output-path)]. NB contrary to what buck documentation says, the output path is always - present even when the target is locally cached. *) -let read_and_parse_report build_report = - let get_json_field fieldname = function - | `Assoc fields -> - List.Assoc.find fields ~equal:String.equal fieldname - | _ -> - None - in - let parse_target (target, json) = - let path_opt = - match get_json_field "output" json with - | Some (`String str) -> - Some str - | _ -> ( - match get_json_field "outputs" json |> Option.bind ~f:(get_json_field "DEFAULT") with - | Some (`List [`String str]) -> - Some str - | _ -> - None ) - in - match path_opt with - | None -> - L.internal_error "Could not parse target json: %s@." (Yojson.Basic.to_string json) ; - None - | Some path -> - Some (target, path) - in - let parse_results = function - | `Assoc results -> - (* NB this will simply skip unparseable targets *) - List.filter_map results ~f:parse_target |> Option.some - | _ -> - None - in - Yojson.Basic.from_file build_report |> get_json_field "results" |> Option.bind ~f:parse_results - - -(** Function for processing paths in a buck build report and generating an [infer-deps.txt] file. - Given a pair [(buck_target, output_path)], - - - if [output_path] contains a capture DB, then generate the appropriate deps line; - - if [output_path] contains an [infer-deps.txt] file, expand and inline it; - - if [output_path] is a dummy target used in the combined genrule integration for clang targets, - read its contents, parse them as an output directory path and apply the above two tests to - that *) -let expand_target acc (target, target_path) = - let expand_dir acc (target, target_path) = - (* invariant: [target_path] is absolute *) - let db_file = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDB in - if ISys.file_exists db_file then - (* there is a capture DB at this path, so terminate expansion and generate deps line *) - let line = Printf.sprintf "%s\t-\t%s" target target_path in - line :: acc - else - (* no capture DB was found, so look for, and inline, an [infer-deps.txt] file *) - let infer_deps = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies in - if ISys.file_exists infer_deps then - Utils.with_file_in infer_deps - ~f:(In_channel.fold_lines ~init:acc ~f:(fun acc line -> line :: acc)) - else ( - L.internal_error "No capture DB or infer-deps file in %s@." target_path ; - acc ) - in - let target_path = - if Filename.is_absolute target_path then target_path else Config.project_root ^/ target_path - in - match Sys.is_directory target_path with - | `Yes -> - (* output path is directory, so should contain either a capture DB or an [infer-deps.txt] file *) - expand_dir acc (target, target_path) - | `No | `Unknown -> ( - (* output path is not a directory, so assume it's an intermediate genrule output containing the - output path of the underlying capture target *) - match Utils.read_file target_path with - | Ok [new_target_path] -> - expand_dir acc (target, new_target_path) - | Ok _ -> - L.internal_error "Couldn't parse intermediate deps file %s@." target_path ; - acc - | Error error -> - L.internal_error "Error %s@\nCouldn't read intermediate deps file %s@." error target_path ; - acc ) - - -let infer_deps_of_build_report build_report = - match read_and_parse_report build_report with - | None -> - L.die InternalError "Couldn't parse buck build report: %s@." build_report - | Some target_path_list -> - let infer_deps_lines = - List.fold target_path_list ~init:[] ~f:expand_target - |> List.dedup_and_sort ~compare:String.compare - in - let infer_deps = ResultsDir.get_path CaptureDependencies in - Utils.with_file_out infer_deps ~f:(fun out_channel -> - Out_channel.output_lines out_channel infer_deps_lines ) - - let capture build_cmd = let prog, buck_cmd = (List.hd_exn build_cmd, List.tl_exn build_cmd) in L.progress "Querying buck for java flavor capture targets...@." ; @@ -167,7 +34,10 @@ let capture build_cmd = else let time0 = Mtime_clock.counter () in Buck.wrap_buck_call ~label:"build" updated_buck_cmd |> ignore ; - infer_deps_of_build_report build_report_file ; + let infer_deps_lines = BuckBuildReport.parse_infer_deps ~build_report_file in + let infer_deps = ResultsDir.get_path CaptureDependencies in + Utils.with_file_out infer_deps ~f:(fun out_channel -> + Out_channel.output_lines out_channel infer_deps_lines ) ; L.progress "Java flavor capture took %a.@." Mtime.Span.pp (Mtime_clock.count time0) ; ResultsDir.RunState.set_merge_capture true ; ()