[gradle] parallelize capture

Summary: Gradle produces a number of compilation units which are currently captured sequentially.  This diff parallelizes this step.

Reviewed By: jvillard

Differential Revision: D23930978

fbshipit-source-id: d71c22ba3

infer/src/backend/mergeCapture.ml

@@ -46,7 +46,7 @@ let merge_json_results infer_out_src json_entry =
 
 let merge_all_json_results merge_results results_json_str =
   L.progress "Merging %s files...@." results_json_str ;
-  let infer_deps_file = ResultsDir.get_path BuckDependencies in
+  let infer_deps_file = ResultsDir.get_path CaptureDependencies in
   Utils.iter_infer_deps ~project_root:Config.project_root ~f:merge_results infer_deps_file ;
   L.progress "Done merging %s files@." results_json_str
 
@@ -61,7 +61,7 @@ let merge_changed_functions () =
 let merge_captured_targets () =
   let time0 = Mtime_clock.counter () in
   L.progress "Merging captured Buck targets...@\n%!" ;
-  let infer_deps_file = ResultsDir.get_path BuckDependencies in
+  let infer_deps_file = ResultsDir.get_path CaptureDependencies in
   DBWriter.merge ~infer_deps_file ;
   ScubaLogging.execute_with_time_logging "merge_captured_tenvs" (fun () ->
       merge_global_tenvs infer_deps_file ) ;

infer/src/base/ResultsDirEntryName.ml

@@ -10,8 +10,8 @@ open! IStd
 let buck_infer_deps_file_name = "infer-deps.txt"
 
 type id =
-  | BuckDependencies
   | CaptureDB
+  | CaptureDependencies
   | ChangedFunctions
   | Debug
   | Differential
@@ -52,7 +52,7 @@ type t =
             e.g., a distributed Buck cache. *) }
 
 let of_id = function
-  | BuckDependencies ->
+  | CaptureDependencies ->
       { rel_path= buck_infer_deps_file_name
       ; kind= File
       ; before_incremental_analysis= Delete

infer/src/base/ResultsDirEntryName.mli

@@ -10,8 +10,8 @@ open! IStd
     directory you probably want to use {!ResultsDir.Entry} instead of this module. *)
 
 type id =
-  | BuckDependencies  (** list of Buck directories with infer-out/ directories *)
   | CaptureDB  (** the capture database *)
+  | CaptureDependencies  (** list of infer-out/ directories that contain capture artefacts *)
   | ChangedFunctions  (** results of the clang test determinator *)
   | Debug  (** directory containing debug data *)
   | Differential  (** contains the results of [infer reportdiff] *)

infer/src/integration/BuckFlavors.ml

@@ -47,7 +47,7 @@ let run_buck_build prog buck_build_args =
         let filename =
           ResultsDirEntryName.get_path
             ~results_dir:(Config.project_root ^/ target_path)
-            BuckDependencies
+            CaptureDependencies
         in
         if PolyVariantEqual.(Sys.file_exists filename = `Yes) then filename :: acc else acc
     | _ ->
@@ -92,7 +92,7 @@ let clang_flavor_capture ~prog ~buck_build_cmd =
     Process.create_process_and_wait ~prog ~args:["clean"] ;
   let depsfiles = run_buck_build prog (buck_build_cmd @ capture_buck_args ()) in
   let deplines = merge_deps_files depsfiles in
-  let infer_out_depsfile = ResultsDir.get_path BuckDependencies in
+  let infer_out_depsfile = ResultsDir.get_path CaptureDependencies in
   Utils.with_file_out infer_out_depsfile ~f:(fun out_chan ->
       Out_channel.output_lines out_chan deplines ) ;
   ()

infer/src/integration/BuckGenrule.ml

@@ -100,7 +100,9 @@ let expand_target acc (target, target_path) =
         line :: acc
     | `No | `Unknown -> (
         (* no capture DB was found, so look for, and inline, an [infer-deps.txt] file *)
-        let infer_deps = ResultsDirEntryName.get_path ~results_dir:target_path BuckDependencies in
+        let infer_deps =
+          ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies
+        in
         match Sys.file_exists infer_deps with
         | `Yes ->
             Utils.with_file_in infer_deps
@@ -139,7 +141,7 @@ let infer_deps_of_build_report build_report =
         List.fold target_path_list ~init:[] ~f:expand_target
         |> List.dedup_and_sort ~compare:String.compare
       in
-      let infer_deps = ResultsDir.get_path BuckDependencies in
+      let infer_deps = ResultsDir.get_path CaptureDependencies in
       Utils.with_file_out infer_deps ~f:(fun out_channel ->
           Out_channel.output_lines out_channel infer_deps_lines )
 

infer/src/integration/Driver.ml

@@ -264,7 +264,7 @@ let analyze_and_report ?suppress_console_report ~changed_files mode =
            && InferCommand.equal Run Config.command ->
         (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *)
         true
-    | Analyze | BuckGenruleMaster _ | BuckCombinedGenrule _ | BuckJavaFlavor _ ->
+    | Analyze | BuckGenruleMaster _ | BuckCombinedGenrule _ | BuckJavaFlavor _ | Gradle _ ->
         ResultsDir.RunState.get_merge_capture ()
     | _ ->
         false

infer/src/integration/Gradle.ml

@@ -37,38 +37,21 @@ let parse_gradle_line ~line =
   {files= concat_st res.files res.file_st; opts= res.opts @ res.opt_st}
 
 
-let normalize path = if String.is_substring path ~substring:" " then "\"" ^ path ^ "\"" else path
+let process_gradle_output_line =
-
+  let capture_output_template = ResultsDir.get_path Temporary ^/ "capture_target" in
-let capture ~prog ~args =
+  fun ((seen, target_dirs) as acc) line ->
-  let java_version =
+    String.substr_index line ~pattern:arg_start_pattern
-    Process.create_process_and_wait_with_output ~prog:"java" ~args:["-version"] ReadStderr
+    |> Option.value_map ~default:acc ~f:(fun pos ->
-  in
-  let javac_version =
-    Process.create_process_and_wait_with_output ~prog:"javac" ~args:["-version"] ReadStderr
-  in
-  let gradle_version =
-    Process.create_process_and_wait_with_output ~prog ~args:["--version"] ReadStdout
-  in
-  L.environment_info "%s %s %s@." java_version javac_version gradle_version ;
-  let process_gradle_line seen line =
-    match String.substr_index line ~pattern:arg_start_pattern with
-    | Some pos ->
            let content = String.drop_prefix line (pos + String.length arg_start_pattern) in
            L.debug Capture Verbose "Processing: %s@." content ;
-        if String.Set.mem seen content then seen
+           if String.Set.mem seen content then acc
            else
              let javac_data = parse_gradle_line ~line:content in
-          let tmpfile, oc =
+             let out_dir = Unix.mkdtemp capture_output_template in
-            Core.Filename.open_temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_files" ""
+             (String.Set.add seen content, (out_dir, javac_data) :: target_dirs) )
-          in
+
-          List.iter javac_data.files ~f:(fun file ->
+
-              Out_channel.output_string oc (normalize file ^ "\n") ) ;
+let run_gradle ~prog ~args =
-          Out_channel.close oc ;
-          Javac.call_infer_javac_capture ~javac_args:(("@" ^ tmpfile) :: javac_data.opts) ;
-          String.Set.add seen content
-    | None ->
-        seen
-  in
   let gradle_output_file =
     Filename.temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_output" ".log"
   in
@@ -77,11 +60,73 @@ let capture ~prog ~args =
     |> String.concat ~sep:" "
     |> fun cmd -> Printf.sprintf "%s >'%s'" cmd gradle_output_file
   in
-  L.progress "[GRADLE] %s@." shell_cmd ;
+  L.progress "[GRADLE] %s@\n" shell_cmd ;
+  let time = Mtime_clock.counter () in
   Process.create_process_and_wait ~prog:"sh" ~args:["-c"; shell_cmd] ;
+  L.progress "[GRADLE] running gradle took %a@\n" Mtime.Span.pp (Mtime_clock.count time) ;
   match Utils.read_file gradle_output_file with
   | Ok lines ->
-      let processed = List.fold lines ~init:String.Set.empty ~f:process_gradle_line in
+      List.fold lines ~init:(String.Set.empty, []) ~f:process_gradle_output_line
-      L.progress "[GRADLE] processed %d lines" @@ String.Set.length processed
   | Error _ ->
-      L.die ExternalError "*** failed to read gradle output: %s" gradle_output_file
+      L.die ExternalError "*** failed to read gradle output: %s@\n" gradle_output_file
+
+
+let capture_gradle_target (out_dir, (javac_data : javac_data)) =
+  let tmpfile, oc =
+    Core.Filename.open_temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_files" ""
+  in
+  List.iter javac_data.files ~f:(fun file ->
+      Out_channel.output_string oc (Escape.escape_shell file) ;
+      Out_channel.newline oc ) ;
+  Out_channel.close oc ;
+  let prog = Config.bin_dir ^/ "infer" in
+  let args =
+    "capture" :: "-j" :: "1" :: "-o" :: out_dir :: "--" :: "javac" :: ("@" ^ tmpfile)
+    :: List.filter_map javac_data.opts ~f:(fun arg ->
+           if String.equal "-Werror" arg then None
+           else if String.is_substring arg ~substring:"-g:" then Some "-g"
+           else Some arg )
+  in
+  L.debug Capture Verbose "%s %s@." prog (String.concat ~sep:" " args) ;
+  Process.create_process_and_wait ~prog ~args ;
+  None
+
+
+let run_infer_capture target_data =
+  Tasks.Runner.create ~jobs:Config.jobs
+    ~child_prologue:(fun () -> ())
+    ~f:capture_gradle_target
+    ~child_epilogue:(fun () -> ())
+    ~tasks:(fun () -> ProcessPool.TaskGenerator.of_list target_data)
+  |> Tasks.Runner.run |> ignore
+
+
+let write_rev_infer_deps rev_target_data =
+  ResultsDir.get_path CaptureDependencies
+  |> Utils.with_file_out ~f:(fun out_channel ->
+         List.rev_map rev_target_data ~f:(fun (out_dir, _) -> Printf.sprintf "-\t-\t%s" out_dir)
+         |> Out_channel.output_lines out_channel )
+
+
+let log_environment_info ~prog =
+  let java_version =
+    Process.create_process_and_wait_with_output ~prog:"java" ~args:["-version"] ReadStderr
+  in
+  let javac_version =
+    Process.create_process_and_wait_with_output ~prog:"javac" ~args:["-version"] ReadStderr
+  in
+  let gradle_version =
+    Process.create_process_and_wait_with_output ~prog ~args:["--version"] ReadStdout
+  in
+  L.environment_info "%s %s %s@\n" java_version javac_version gradle_version
+
+
+let capture ~prog ~args =
+  log_environment_info ~prog ;
+  let processed, rev_target_data = run_gradle ~prog ~args in
+  let time = Mtime_clock.counter () in
+  run_infer_capture rev_target_data ;
+  write_rev_infer_deps rev_target_data ;
+  ResultsDir.RunState.set_merge_capture true ;
+  L.debug Capture Quiet "[GRADLE] infer processed %d lines in %a@\n" (String.Set.length processed)
+    Mtime.Span.pp (Mtime_clock.count time)