[clang/buck] use build report instead of stdout

Summary: Use the JSON build report to figure out where each build output is instead of parsing the output of buck.

Reviewed By: da319

Differential Revision: D29935374

fbshipit-source-id: f08f68889

infer/src/base/Config.ml

@@ -108,7 +108,9 @@ let assign = "<\"Assign\">"
     with a direct array access where an error is produced and the analysis continues *)
 let bound_error_allowed_in_procedure_call = true
 
-let buck_out_gen = "buck-out" ^/ "gen"
+let buck_out = "buck-out"
+
+let buck_out_gen = buck_out ^/ "gen"
 
 let buck_results_dir_name = "infer"
 

infer/src/base/Config.mli

@@ -189,6 +189,8 @@ val buck_merge_all_deps : bool
 
 val buck_mode : BuckMode.t option
 
+val buck_out : string
+
 val buck_out_gen : string
 
 val buck_targets_block_list : string list

infer/src/integration/BuckBuildReport.ml

@@ -0,0 +1,142 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+module L = Logging
+
+(* example build report json output
+   [
+     {
+     "success" : true,
+     "results" : {
+       "//annotations:annotations_infer" : {
+         "success" : true,
+         "type" : "BUILT_LOCALLY",
+         "output" : "buck-out/gen/annotations/annotations_infer/infer_out"
+       },
+       "//module2:module2_infer" : {
+         "success" : true,
+         "type" : "BUILT_LOCALLY",
+         "output" : "buck-out/gen/module2/module2_infer/infer_out"
+       },
+       "//module1:module1_infer" : {
+         "success" : true,
+         "type" : "BUILT_LOCALLY",
+         "output" : "buck-out/gen/module1/module1_infer/infer_out"
+       },
+       "//module3:module1_infer" : {
+         "success" : "SUCCESS",
+         "type" : "BUILT_LOCALLY",
+         "outputs" : {
+           "DEFAULT" : [ "buck-out/gen/module1/module3_infer/infer_out" ]
+         }
+       }
+     },
+     "failures" : { }
+   }%
+   ]
+*)
+
+(** Read the build report json file buck produced, and parse into a list of pairs
+    [(target, output-path)]. NB contrary to what buck documentation says, the output path is always
+    present even when the target is locally cached. *)
+let read_and_parse_report build_report =
+  let get_json_field fieldname = function
+    | `Assoc fields ->
+        List.Assoc.find fields ~equal:String.equal fieldname
+    | _ ->
+        None
+  in
+  let parse_target (target, json) =
+    let path_opt =
+      match get_json_field "output" json with
+      | Some (`String str) ->
+          Some str
+      | _ -> (
+        match get_json_field "outputs" json |> Option.bind ~f:(get_json_field "DEFAULT") with
+        | Some (`List [`String str]) ->
+            Some str
+        | _ ->
+            None )
+    in
+    match path_opt with
+    | None ->
+        L.internal_error "Could not parse target json: %s@." (Yojson.Basic.to_string json) ;
+        None
+    | Some path ->
+        Some (target, path)
+  in
+  let parse_results = function
+    | `Assoc results ->
+        (* NB this will simply skip unparseable targets *)
+        List.filter_map results ~f:parse_target |> Option.some
+    | _ ->
+        None
+  in
+  Yojson.Basic.from_file build_report |> get_json_field "results" |> Option.bind ~f:parse_results
+
+
+(** Function for processing paths in a buck build report and generating an [infer-deps.txt] file.
+    Given a pair [(buck_target, output_path)],
+
+    - if [output_path] contains a capture DB, then generate the appropriate deps line;
+    - if [output_path] contains an [infer-deps.txt] file, expand and inline it;
+    - if [output_path] is a dummy target used in the combined genrule integration for clang targets,
+      read its contents, parse them as an output directory path and apply the above two tests to
+      that *)
+let expand_target acc (target, target_path) =
+  let inline acc path =
+    Utils.with_file_in path ~f:(In_channel.fold_lines ~init:acc ~f:(fun acc line -> line :: acc))
+  in
+  let expand_dir acc (target, target_path) =
+    (* invariant: [target_path] is absolute *)
+    let db_file = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDB in
+    if ISys.file_exists db_file then
+      (* we found a capture DB so add this as a target line *)
+      Printf.sprintf "%s\t-\t%s" target target_path :: acc
+    else
+      let infer_deps_file =
+        ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies
+      in
+      if ISys.file_exists infer_deps_file then
+        (* we found an [infer_deps.txt] file so inline in *)
+        inline acc infer_deps_file
+      else (
+        (* don't know what to do with this directory *)
+        L.internal_error "Didn't find capture DB or infer-deps file in path %s.@\n" target_path ;
+        acc )
+  in
+  let target_path =
+    if Filename.is_absolute target_path then target_path else Config.project_root ^/ target_path
+  in
+  match Sys.is_directory target_path with
+  | `Yes ->
+      expand_dir acc (target, target_path)
+  | _ when String.is_suffix target_path ~suffix:ResultsDirEntryName.buck_infer_deps_file_name ->
+      (* direct path of an [infer-deps.txt] file, inline *)
+      inline acc target_path
+  | _ -> (
+    (* assume path is an intermediate genrule output containing the
+       output path of the underlying capture target *)
+    match Utils.read_file target_path with
+    | Ok [new_target_path] ->
+        expand_dir acc (target, new_target_path)
+    | Ok _ ->
+        L.internal_error "Couldn't parse intermediate deps file %s@." target_path ;
+        acc
+    | Error error ->
+        L.internal_error "Error %s@\nCouldn't read intermediate deps file %s@." error target_path ;
+        acc )
+
+
+let parse_infer_deps ~build_report_file =
+  match read_and_parse_report build_report_file with
+  | None ->
+      L.die InternalError "Couldn't parse buck build report: %s@." build_report_file
+  | Some target_path_list ->
+      List.fold target_path_list ~init:[] ~f:expand_target
+      |> List.dedup_and_sort ~compare:String.compare

infer/src/integration/BuckBuildReport.mli

@@ -0,0 +1,12 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+val parse_infer_deps : build_report_file:string -> string list
+(** parse a JSON build report by buck and return all capture DBs found in the [infer_deps.txt]
+    format *)

infer/src/integration/BuckFlavors.ml

@@ -23,8 +23,8 @@ let add_flavors_to_buck_arguments buck_mode ~extra_flavors original_buck_args =
   {command; rev_not_targets; targets}
 
 
-let capture_buck_args () =
+let capture_buck_args build_report_file =
-  ("--show-output" :: (if Config.keep_going then ["--keep-going"] else []))
+  ("--build-report" :: build_report_file :: (if Config.keep_going then ["--keep-going"] else []))
   @ (match Config.load_average with Some l -> ["-L"; Float.to_string l] | None -> [])
   @ Buck.config ClangFlavors @ Config.buck_build_args
 
@@ -36,68 +36,19 @@ let run_buck_build prog buck_build_args =
       ~f:(fun acc arg -> Printf.sprintf "%s%c%s" acc CommandLineOption.env_var_sep arg)
   in
   let extend_env = [(CommandLineOption.args_env_var, infer_args)] in
-  let lines = Buck.wrap_buck_call ~extend_env ~label:"build" (prog :: buck_build_args) in
+  Buck.wrap_buck_call ~extend_env ~label:"build" (prog :: buck_build_args) |> ignore
-  (* Process a line of buck stdout output, in this case the result of '--show-output'
-     These paths (may) contain a 'infer-deps.txt' file, which we will later merge
-  *)
-  let process_buck_line acc line =
-    L.debug Capture Verbose "BUCK OUT: %s@." line ;
-    match String.lsplit2 ~on:' ' line with
-    | Some (_, infer_deps_path) ->
-        let full_path = Config.project_root ^/ infer_deps_path in
-        let dirname = Filename.dirname full_path in
-        let get_path results_dir = ResultsDirEntryName.get_path ~results_dir CaptureDependencies in
-        (* Buck can either give the full path to infer-deps.txt ... *)
-        if ISys.file_exists (get_path dirname) then get_path dirname :: acc
-          (* ... or a folder which contains infer-deps.txt *)
-        else if ISys.file_exists (get_path full_path) then get_path full_path :: acc
-        else acc
-    | _ ->
-        L.internal_error "Couldn't parse buck target output: %s@\n" line ;
-        acc
-  in
-  List.fold lines ~init:[] ~f:process_buck_line
 
 
-let merge_deps_files depsfiles =
+let get_all_infer_deps_under_buck_out () =
-  let buck_out = Config.project_root ^/ Config.buck_out_gen in
+  Utils.fold_folders ~init:[] ~path:(Config.project_root ^/ Config.buck_out) ~f:(fun acc dir ->
-  let depslines, depsfiles =
+      if
-    match depsfiles with
+        String.is_substring dir ~substring:"infer-out"
-    | [] when Config.keep_going || Config.buck_merge_all_deps ->
+        && ISys.file_exists (ResultsDirEntryName.get_path ~results_dir:dir CaptureDB)
-        let infouts =
+      then Printf.sprintf "\t\t%s" dir :: acc
-          Utils.fold_folders ~init:[] ~path:buck_out ~f:(fun acc dir ->
+      else acc )
-              if
-                String.is_substring dir ~substring:"infer-out"
-                && ISys.file_exists (ResultsDirEntryName.get_path ~results_dir:dir CaptureDB)
-              then Printf.sprintf "\t\t%s" dir :: acc
-              else acc )
-        in
-        (infouts, [])
-    | [] when Config.buck_merge_all_deps ->
-        let files =
-          Utils.find_files ~path:buck_out ~extension:ResultsDirEntryName.buck_infer_deps_file_name
-        in
-        ([], files)
-    | _ ->
-        ([], depsfiles)
-  in
-  depslines
-  @ List.fold depsfiles ~init:[] ~f:(fun acc file ->
-        List.rev_append acc (Utils.with_file_in file ~f:In_channel.input_lines) )
   |> List.dedup_and_sort ~compare:String.compare
 
 
-let clang_flavor_capture ~prog ~buck_build_cmd =
-  if Config.keep_going && not Config.continue_capture then
-    Process.create_process_and_wait ~prog ~args:["clean"] ;
-  let depsfiles = run_buck_build prog (buck_build_cmd @ capture_buck_args ()) in
-  let deplines = merge_deps_files depsfiles in
-  let infer_out_depsfile = ResultsDir.get_path CaptureDependencies in
-  Utils.with_file_out infer_out_depsfile ~f:(fun out_chan ->
-      Out_channel.output_lines out_chan deplines ) ;
-  ()
-
-
 let capture build_cmd =
   let prog, buck_args = (List.hd_exn build_cmd, List.tl_exn build_cmd) in
   (* let children infer processes know that they are inside Buck *)
@@ -121,4 +72,16 @@ let capture build_cmd =
       updated_buck_cmd ;
     let prog, buck_build_cmd = (prog, updated_buck_cmd) in
     ResultsDir.RunState.set_merge_capture true ;
-    clang_flavor_capture ~prog ~buck_build_cmd )
+    if Config.keep_going && not Config.continue_capture then
+      Process.create_process_and_wait ~prog ~args:["clean"] ;
+    let build_report_file =
+      Filename.temp_file ~in_dir:(ResultsDir.get_path Temporary) "buck_build_report" ".json"
+    in
+    run_buck_build prog (buck_build_cmd @ capture_buck_args build_report_file) ;
+    let infer_deps_lines =
+      if Config.buck_merge_all_deps then get_all_infer_deps_under_buck_out ()
+      else BuckBuildReport.parse_infer_deps ~build_report_file
+    in
+    let infer_deps = ResultsDir.get_path CaptureDependencies in
+    Utils.with_file_out infer_deps ~f:(fun out_channel ->
+        Out_channel.output_lines out_channel infer_deps_lines ) )

infer/src/integration/BuckJavaFlavor.ml

@@ -9,139 +9,6 @@ open! IStd
 module F = Format
 module L = Logging
 
-(* example build report json output
-[
-  {
-  "success" : true,
-  "results" : {
-    "//annotations:annotations_infer" : {
-      "success" : true,
-      "type" : "BUILT_LOCALLY",
-      "output" : "buck-out/gen/annotations/annotations_infer/infer_out"
-    },
-    "//module2:module2_infer" : {
-      "success" : true,
-      "type" : "BUILT_LOCALLY",
-      "output" : "buck-out/gen/module2/module2_infer/infer_out"
-    },
-    "//module1:module1_infer" : {
-      "success" : true,
-      "type" : "BUILT_LOCALLY",
-      "output" : "buck-out/gen/module1/module1_infer/infer_out"
-    },
-    "//module3:module1_infer" : {
-      "success" : "SUCCESS",
-      "type" : "BUILT_LOCALLY",
-      "outputs" : {
-        "DEFAULT" : [ "buck-out/gen/module1/module3_infer/infer_out" ]
-      }
-    }
-  },
-  "failures" : { }
-}%
-]
-*)
-
-(** Read the build report json file buck produced, and parse into a list of pairs
-    [(target, output-path)]. NB contrary to what buck documentation says, the output path is always
-    present even when the target is locally cached. *)
-let read_and_parse_report build_report =
-  let get_json_field fieldname = function
-    | `Assoc fields ->
-        List.Assoc.find fields ~equal:String.equal fieldname
-    | _ ->
-        None
-  in
-  let parse_target (target, json) =
-    let path_opt =
-      match get_json_field "output" json with
-      | Some (`String str) ->
-          Some str
-      | _ -> (
-        match get_json_field "outputs" json |> Option.bind ~f:(get_json_field "DEFAULT") with
-        | Some (`List [`String str]) ->
-            Some str
-        | _ ->
-            None )
-    in
-    match path_opt with
-    | None ->
-        L.internal_error "Could not parse target json: %s@." (Yojson.Basic.to_string json) ;
-        None
-    | Some path ->
-        Some (target, path)
-  in
-  let parse_results = function
-    | `Assoc results ->
-        (* NB this will simply skip unparseable targets *)
-        List.filter_map results ~f:parse_target |> Option.some
-    | _ ->
-        None
-  in
-  Yojson.Basic.from_file build_report |> get_json_field "results" |> Option.bind ~f:parse_results
-
-
-(** Function for processing paths in a buck build report and generating an [infer-deps.txt] file.
-    Given a pair [(buck_target, output_path)],
-
-    - if [output_path] contains a capture DB, then generate the appropriate deps line;
-    - if [output_path] contains an [infer-deps.txt] file, expand and inline it;
-    - if [output_path] is a dummy target used in the combined genrule integration for clang targets,
-      read its contents, parse them as an output directory path and apply the above two tests to
-      that *)
-let expand_target acc (target, target_path) =
-  let expand_dir acc (target, target_path) =
-    (* invariant: [target_path] is absolute *)
-    let db_file = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDB in
-    if ISys.file_exists db_file then
-      (* there is a capture DB at this path, so terminate expansion and generate deps line *)
-      let line = Printf.sprintf "%s\t-\t%s" target target_path in
-      line :: acc
-    else
-      (* no capture DB was found, so look for, and inline, an [infer-deps.txt] file *)
-      let infer_deps = ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies in
-      if ISys.file_exists infer_deps then
-        Utils.with_file_in infer_deps
-          ~f:(In_channel.fold_lines ~init:acc ~f:(fun acc line -> line :: acc))
-      else (
-        L.internal_error "No capture DB or infer-deps file in %s@." target_path ;
-        acc )
-  in
-  let target_path =
-    if Filename.is_absolute target_path then target_path else Config.project_root ^/ target_path
-  in
-  match Sys.is_directory target_path with
-  | `Yes ->
-      (* output path is directory, so should contain either a capture DB or an [infer-deps.txt] file *)
-      expand_dir acc (target, target_path)
-  | `No | `Unknown -> (
-    (* output path is not a directory, so assume it's an intermediate genrule output containing the
-       output path of the underlying capture target *)
-    match Utils.read_file target_path with
-    | Ok [new_target_path] ->
-        expand_dir acc (target, new_target_path)
-    | Ok _ ->
-        L.internal_error "Couldn't parse intermediate deps file %s@." target_path ;
-        acc
-    | Error error ->
-        L.internal_error "Error %s@\nCouldn't read intermediate deps file %s@." error target_path ;
-        acc )
-
-
-let infer_deps_of_build_report build_report =
-  match read_and_parse_report build_report with
-  | None ->
-      L.die InternalError "Couldn't parse buck build report: %s@." build_report
-  | Some target_path_list ->
-      let infer_deps_lines =
-        List.fold target_path_list ~init:[] ~f:expand_target
-        |> List.dedup_and_sort ~compare:String.compare
-      in
-      let infer_deps = ResultsDir.get_path CaptureDependencies in
-      Utils.with_file_out infer_deps ~f:(fun out_channel ->
-          Out_channel.output_lines out_channel infer_deps_lines )
-
-
 let capture build_cmd =
   let prog, buck_cmd = (List.hd_exn build_cmd, List.tl_exn build_cmd) in
   L.progress "Querying buck for java flavor capture targets...@." ;
@@ -167,7 +34,10 @@ let capture build_cmd =
   else
     let time0 = Mtime_clock.counter () in
     Buck.wrap_buck_call ~label:"build" updated_buck_cmd |> ignore ;
-    infer_deps_of_build_report build_report_file ;
+    let infer_deps_lines = BuckBuildReport.parse_infer_deps ~build_report_file in
+    let infer_deps = ResultsDir.get_path CaptureDependencies in
+    Utils.with_file_out infer_deps ~f:(fun out_channel ->
+        Out_channel.output_lines out_channel infer_deps_lines ) ;
     L.progress "Java flavor capture took %a.@." Mtime.Span.pp (Mtime_clock.count time0) ;
     ResultsDir.RunState.set_merge_capture true ;
     ()