From d2fc917ef94e6b53c4bf998f06538ec913877619 Mon Sep 17 00:00:00 2001 From: Nikos Gorogiannis Date: Fri, 2 Oct 2020 01:00:19 -0700 Subject: [PATCH] [gradle] parallelize capture Summary: Gradle produces a number of compilation units which are currently captured sequentially. This diff parallelizes this step. Reviewed By: jvillard Differential Revision: D23930978 fbshipit-source-id: d71c22ba3 --- infer/src/backend/mergeCapture.ml | 4 +- infer/src/base/ResultsDirEntryName.ml | 4 +- infer/src/base/ResultsDirEntryName.mli | 2 +- infer/src/integration/BuckFlavors.ml | 4 +- infer/src/integration/BuckGenrule.ml | 6 +- infer/src/integration/Driver.ml | 2 +- infer/src/integration/Gradle.ml | 115 +++++++++++++++++-------- 7 files changed, 92 insertions(+), 45 deletions(-) diff --git a/infer/src/backend/mergeCapture.ml b/infer/src/backend/mergeCapture.ml index 88b0bfbad..4dc57335b 100644 --- a/infer/src/backend/mergeCapture.ml +++ b/infer/src/backend/mergeCapture.ml @@ -46,7 +46,7 @@ let merge_json_results infer_out_src json_entry = let merge_all_json_results merge_results results_json_str = L.progress "Merging %s files...@." results_json_str ; - let infer_deps_file = ResultsDir.get_path BuckDependencies in + let infer_deps_file = ResultsDir.get_path CaptureDependencies in Utils.iter_infer_deps ~project_root:Config.project_root ~f:merge_results infer_deps_file ; L.progress "Done merging %s files@." results_json_str @@ -61,7 +61,7 @@ let merge_changed_functions () = let merge_captured_targets () = let time0 = Mtime_clock.counter () in L.progress "Merging captured Buck targets...@\n%!" ; - let infer_deps_file = ResultsDir.get_path BuckDependencies in + let infer_deps_file = ResultsDir.get_path CaptureDependencies in DBWriter.merge ~infer_deps_file ; ScubaLogging.execute_with_time_logging "merge_captured_tenvs" (fun () -> merge_global_tenvs infer_deps_file ) ; diff --git a/infer/src/base/ResultsDirEntryName.ml b/infer/src/base/ResultsDirEntryName.ml index d337fc3af..9196a765b 100644 --- a/infer/src/base/ResultsDirEntryName.ml +++ b/infer/src/base/ResultsDirEntryName.ml @@ -10,8 +10,8 @@ open! IStd let buck_infer_deps_file_name = "infer-deps.txt" type id = - | BuckDependencies | CaptureDB + | CaptureDependencies | ChangedFunctions | Debug | Differential @@ -52,7 +52,7 @@ type t = e.g., a distributed Buck cache. *) } let of_id = function - | BuckDependencies -> + | CaptureDependencies -> { rel_path= buck_infer_deps_file_name ; kind= File ; before_incremental_analysis= Delete diff --git a/infer/src/base/ResultsDirEntryName.mli b/infer/src/base/ResultsDirEntryName.mli index ebe85ecf5..8305ba28f 100644 --- a/infer/src/base/ResultsDirEntryName.mli +++ b/infer/src/base/ResultsDirEntryName.mli @@ -10,8 +10,8 @@ open! IStd directory you probably want to use {!ResultsDir.Entry} instead of this module. *) type id = - | BuckDependencies (** list of Buck directories with infer-out/ directories *) | CaptureDB (** the capture database *) + | CaptureDependencies (** list of infer-out/ directories that contain capture artefacts *) | ChangedFunctions (** results of the clang test determinator *) | Debug (** directory containing debug data *) | Differential (** contains the results of [infer reportdiff] *) diff --git a/infer/src/integration/BuckFlavors.ml b/infer/src/integration/BuckFlavors.ml index d9affd587..19eaeee96 100644 --- a/infer/src/integration/BuckFlavors.ml +++ b/infer/src/integration/BuckFlavors.ml @@ -47,7 +47,7 @@ let run_buck_build prog buck_build_args = let filename = ResultsDirEntryName.get_path ~results_dir:(Config.project_root ^/ target_path) - BuckDependencies + CaptureDependencies in if PolyVariantEqual.(Sys.file_exists filename = `Yes) then filename :: acc else acc | _ -> @@ -92,7 +92,7 @@ let clang_flavor_capture ~prog ~buck_build_cmd = Process.create_process_and_wait ~prog ~args:["clean"] ; let depsfiles = run_buck_build prog (buck_build_cmd @ capture_buck_args ()) in let deplines = merge_deps_files depsfiles in - let infer_out_depsfile = ResultsDir.get_path BuckDependencies in + let infer_out_depsfile = ResultsDir.get_path CaptureDependencies in Utils.with_file_out infer_out_depsfile ~f:(fun out_chan -> Out_channel.output_lines out_chan deplines ) ; () diff --git a/infer/src/integration/BuckGenrule.ml b/infer/src/integration/BuckGenrule.ml index 6373baea3..c89fe7a86 100644 --- a/infer/src/integration/BuckGenrule.ml +++ b/infer/src/integration/BuckGenrule.ml @@ -100,7 +100,9 @@ let expand_target acc (target, target_path) = line :: acc | `No | `Unknown -> ( (* no capture DB was found, so look for, and inline, an [infer-deps.txt] file *) - let infer_deps = ResultsDirEntryName.get_path ~results_dir:target_path BuckDependencies in + let infer_deps = + ResultsDirEntryName.get_path ~results_dir:target_path CaptureDependencies + in match Sys.file_exists infer_deps with | `Yes -> Utils.with_file_in infer_deps @@ -139,7 +141,7 @@ let infer_deps_of_build_report build_report = List.fold target_path_list ~init:[] ~f:expand_target |> List.dedup_and_sort ~compare:String.compare in - let infer_deps = ResultsDir.get_path BuckDependencies in + let infer_deps = ResultsDir.get_path CaptureDependencies in Utils.with_file_out infer_deps ~f:(fun out_channel -> Out_channel.output_lines out_channel infer_deps_lines ) diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index c3808f0cf..73606eca7 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -264,7 +264,7 @@ let analyze_and_report ?suppress_console_report ~changed_files mode = && InferCommand.equal Run Config.command -> (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *) true - | Analyze | BuckGenruleMaster _ | BuckCombinedGenrule _ | BuckJavaFlavor _ -> + | Analyze | BuckGenruleMaster _ | BuckCombinedGenrule _ | BuckJavaFlavor _ | Gradle _ -> ResultsDir.RunState.get_merge_capture () | _ -> false diff --git a/infer/src/integration/Gradle.ml b/infer/src/integration/Gradle.ml index 814097bc6..667cca5ef 100644 --- a/infer/src/integration/Gradle.ml +++ b/infer/src/integration/Gradle.ml @@ -37,38 +37,21 @@ let parse_gradle_line ~line = {files= concat_st res.files res.file_st; opts= res.opts @ res.opt_st} -let normalize path = if String.is_substring path ~substring:" " then "\"" ^ path ^ "\"" else path +let process_gradle_output_line = + let capture_output_template = ResultsDir.get_path Temporary ^/ "capture_target" in + fun ((seen, target_dirs) as acc) line -> + String.substr_index line ~pattern:arg_start_pattern + |> Option.value_map ~default:acc ~f:(fun pos -> + let content = String.drop_prefix line (pos + String.length arg_start_pattern) in + L.debug Capture Verbose "Processing: %s@." content ; + if String.Set.mem seen content then acc + else + let javac_data = parse_gradle_line ~line:content in + let out_dir = Unix.mkdtemp capture_output_template in + (String.Set.add seen content, (out_dir, javac_data) :: target_dirs) ) -let capture ~prog ~args = - let java_version = - Process.create_process_and_wait_with_output ~prog:"java" ~args:["-version"] ReadStderr - in - let javac_version = - Process.create_process_and_wait_with_output ~prog:"javac" ~args:["-version"] ReadStderr - in - let gradle_version = - Process.create_process_and_wait_with_output ~prog ~args:["--version"] ReadStdout - in - L.environment_info "%s %s %s@." java_version javac_version gradle_version ; - let process_gradle_line seen line = - match String.substr_index line ~pattern:arg_start_pattern with - | Some pos -> - let content = String.drop_prefix line (pos + String.length arg_start_pattern) in - L.debug Capture Verbose "Processing: %s@." content ; - if String.Set.mem seen content then seen - else - let javac_data = parse_gradle_line ~line:content in - let tmpfile, oc = - Core.Filename.open_temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_files" "" - in - List.iter javac_data.files ~f:(fun file -> - Out_channel.output_string oc (normalize file ^ "\n") ) ; - Out_channel.close oc ; - Javac.call_infer_javac_capture ~javac_args:(("@" ^ tmpfile) :: javac_data.opts) ; - String.Set.add seen content - | None -> - seen - in + +let run_gradle ~prog ~args = let gradle_output_file = Filename.temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_output" ".log" in @@ -77,11 +60,73 @@ let capture ~prog ~args = |> String.concat ~sep:" " |> fun cmd -> Printf.sprintf "%s >'%s'" cmd gradle_output_file in - L.progress "[GRADLE] %s@." shell_cmd ; + L.progress "[GRADLE] %s@\n" shell_cmd ; + let time = Mtime_clock.counter () in Process.create_process_and_wait ~prog:"sh" ~args:["-c"; shell_cmd] ; + L.progress "[GRADLE] running gradle took %a@\n" Mtime.Span.pp (Mtime_clock.count time) ; match Utils.read_file gradle_output_file with | Ok lines -> - let processed = List.fold lines ~init:String.Set.empty ~f:process_gradle_line in - L.progress "[GRADLE] processed %d lines" @@ String.Set.length processed + List.fold lines ~init:(String.Set.empty, []) ~f:process_gradle_output_line | Error _ -> - L.die ExternalError "*** failed to read gradle output: %s" gradle_output_file + L.die ExternalError "*** failed to read gradle output: %s@\n" gradle_output_file + + +let capture_gradle_target (out_dir, (javac_data : javac_data)) = + let tmpfile, oc = + Core.Filename.open_temp_file ~in_dir:(ResultsDir.get_path Temporary) "gradle_files" "" + in + List.iter javac_data.files ~f:(fun file -> + Out_channel.output_string oc (Escape.escape_shell file) ; + Out_channel.newline oc ) ; + Out_channel.close oc ; + let prog = Config.bin_dir ^/ "infer" in + let args = + "capture" :: "-j" :: "1" :: "-o" :: out_dir :: "--" :: "javac" :: ("@" ^ tmpfile) + :: List.filter_map javac_data.opts ~f:(fun arg -> + if String.equal "-Werror" arg then None + else if String.is_substring arg ~substring:"-g:" then Some "-g" + else Some arg ) + in + L.debug Capture Verbose "%s %s@." prog (String.concat ~sep:" " args) ; + Process.create_process_and_wait ~prog ~args ; + None + + +let run_infer_capture target_data = + Tasks.Runner.create ~jobs:Config.jobs + ~child_prologue:(fun () -> ()) + ~f:capture_gradle_target + ~child_epilogue:(fun () -> ()) + ~tasks:(fun () -> ProcessPool.TaskGenerator.of_list target_data) + |> Tasks.Runner.run |> ignore + + +let write_rev_infer_deps rev_target_data = + ResultsDir.get_path CaptureDependencies + |> Utils.with_file_out ~f:(fun out_channel -> + List.rev_map rev_target_data ~f:(fun (out_dir, _) -> Printf.sprintf "-\t-\t%s" out_dir) + |> Out_channel.output_lines out_channel ) + + +let log_environment_info ~prog = + let java_version = + Process.create_process_and_wait_with_output ~prog:"java" ~args:["-version"] ReadStderr + in + let javac_version = + Process.create_process_and_wait_with_output ~prog:"javac" ~args:["-version"] ReadStderr + in + let gradle_version = + Process.create_process_and_wait_with_output ~prog ~args:["--version"] ReadStdout + in + L.environment_info "%s %s %s@\n" java_version javac_version gradle_version + + +let capture ~prog ~args = + log_environment_info ~prog ; + let processed, rev_target_data = run_gradle ~prog ~args in + let time = Mtime_clock.counter () in + run_infer_capture rev_target_data ; + write_rev_infer_deps rev_target_data ; + ResultsDir.RunState.set_merge_capture true ; + L.debug Capture Quiet "[GRADLE] infer processed %d lines in %a@\n" (String.Set.length processed) + Mtime.Span.pp (Mtime_clock.count time)