From b7bc95e1f93ab7877d154128d58e731fcda35911 Mon Sep 17 00:00:00 2001 From: Cristiano Calcagno Date: Mon, 21 Mar 2016 03:36:36 -0700 Subject: [PATCH] Add option -merge_captured to merge the result of capture for different targets. Summary:public The option -merge_captured specifies that this is merging the results of capture using the buck integration. The file specifying the targets is read from `infer-out/infer-deps.txt`. Each dependency specifies a path in buck-out for one target: where the results directory after capture is. The option triggers a merge of the results directories into infer-out. The merge consists in making a virtual copy, where each file in a target in `buck-out` gets virtually copied into infer-out by making one symbolic link per file. There is a mechanism to detect when the capture of a target has already been copied: when each source file already exists at the destination. There's also an option `-modified_targets modified_targets.txt`. If a target is listed in `modified_targets.txt`, this will force a new creation of links for that target, whether those links exist or not. Reviewed By: martinoluca Differential Revision: D3070318 fb-gh-sync-id: 6d2e7a5 shipit-source-id: 6d2e7a5 --- infer/src/backend/DB.ml | 4 +- infer/src/backend/DB.mli | 5 +- infer/src/backend/inferanalyze.ml | 47 ++++++---- infer/src/backend/mergeCapture.ml | 148 ++++++++++++++++++++++++++++++ infer/src/backend/procname.ml | 6 -- 5 files changed, 182 insertions(+), 28 deletions(-) create mode 100644 infer/src/backend/mergeCapture.ml diff --git a/infer/src/backend/DB.ml b/infer/src/backend/DB.ml index b1729aad0..692ce951b 100644 --- a/infer/src/backend/DB.ml +++ b/infer/src/backend/DB.ml @@ -183,8 +183,8 @@ module FilenameMap = Map.Make( end) (** Return the time when a file was last modified. The file must exist. *) -let file_modified_time fname = - let stat = Unix.stat fname in +let file_modified_time ?(symlink=false) fname = + let stat = (if symlink then Unix.lstat else Unix.stat) fname in stat.Unix.st_mtime (** Create a directory if it does not exist already. *) diff --git a/infer/src/backend/DB.mli b/infer/src/backend/DB.mli index 9b1ad05f8..35b0ad81a 100644 --- a/infer/src/backend/DB.mli +++ b/infer/src/backend/DB.mli @@ -26,7 +26,10 @@ val filename_concat : filename -> string -> filename val filename_add_suffix : filename -> string -> filename val file_exists : filename -> bool val file_remove : filename -> unit -val file_modified_time : filename -> float (** Return the time when a file was last modified. The file must exist. *) + +(** Return the time when a file was last modified. The file must exist. *) +val file_modified_time : ?symlink:bool -> filename -> float + (** Return whether filename was updated after analysis started. File doesn't have to exist *) val file_was_updated_after_start : filename -> bool diff --git a/infer/src/backend/inferanalyze.ml b/infer/src/backend/inferanalyze.ml index 56bff3e77..59b980c0d 100644 --- a/infer/src/backend/inferanalyze.ml +++ b/infer/src/backend/inferanalyze.ml @@ -126,6 +126,11 @@ let arg_desc = let desc = reserved_arg_desc @ [ + "-allow_specs_cleanup", + Arg.Unit (fun () -> allow_specs_cleanup := true), + None, + "Allow to remove existing specs before running analysis when it's not incremental" + ; "-analysis_stops", Arg.Set Config.analysis_stops, None, @@ -159,11 +164,31 @@ let arg_desc = None, " activate the eradicate checker for java annotations" ; + "-merge_captured", + Arg.Unit MergeCapture.merge_captured_targets, + None, + "merge the captured results directories specified in the dependency file" + ; "-makefile", Arg.Set_string makefile_cmdline, Some "file", "create a makefile to perform the analysis" ; + "-modified_targets", + Arg.String (fun file -> MergeCapture.modified_file file), + Some "file", + "read the file of buck targets modified since the last analysis" + ; + "-optimistic_cast", + Arg.Set Config.optimistic_cast, + None, + "allow cast of undefined values" + ; + "-print_buckets", + Arg.Unit (fun() -> Config.show_buckets := true; Config.show_ml_buckets := true), + None, + "Add buckets to issue descriptions, useful when developing infer" + ; "-seconds_per_iteration", Arg.Set_float seconds_per_iteration, Some "n", @@ -174,36 +199,21 @@ let arg_desc = None, "use the multirange subtyping domain" ; - "-optimistic_cast", - Arg.Set Config.optimistic_cast, - None, - "allow cast of undefined values" - ; "-symops_per_iteration", Arg.Set_int symops_per_iteration, Some "n", "set the number of symbolic operations per iteration (default n=" ^ (string_of_int !symops_per_iteration) ^ ")" ; - "-type_size", - Arg.Set Config.type_size, - None, - "consider the size of types during analysis" - ; "-tracing", Arg.Unit (fun () -> Config.report_runtime_exceptions := true), None, "Report error traces for runtime exceptions (Only for Java)" ; - "-allow_specs_cleanup", - Arg.Unit (fun () -> allow_specs_cleanup := true), - None, - "Allow to remove existing specs before running analysis when it's not incremental" - ; - "-print_buckets", - Arg.Unit (fun() -> Config.show_buckets := true; Config.show_ml_buckets := true), + "-type_size", + Arg.Set Config.type_size, None, - "Add buckets to issue descriptions, useful when developing infer" + "consider the size of types during analysis" ; ] in Arg.create_options_desc false @@ -372,4 +382,3 @@ let () = end; output_json_makefile_stats clusters; finish_logging () - diff --git a/infer/src/backend/mergeCapture.ml b/infer/src/backend/mergeCapture.ml new file mode 100644 index 000000000..858237574 --- /dev/null +++ b/infer/src/backend/mergeCapture.ml @@ -0,0 +1,148 @@ +(* + * Copyright (c) 2015 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +module L = Logging +module F = Format + +(** Module to merge the results of capture for different buck targets. *) + +(** Flag to control whether the timestamp of symbolic links + is used to determine whether a captured directory needs to be merged. *) +let check_timestamp_of_symlinks = false + +let buck_out = ref (Filename.concat (Sys.getcwd ()) "buck-out") + +let infer_deps () = Filename.concat !Config.results_dir "infer-deps.txt" + +let modified_targets = ref StringSet.empty + +let modified_file file = match Utils.read_file file with + | Some targets -> + modified_targets := + IList.fold_left (fun s target -> StringSet.add target s) StringSet.empty targets + | None -> + () + +let debug = 0 + +type stats = + { + mutable files_linked: int; + mutable targets_merged: int; + } + +let empty_stats () = + { + files_linked = 0; + targets_merged = 0; + } + +(** Create symbolic links recursively from the destination to the source. + Replicate the structure of the source directory in the destination, + with files replaced by links to the source. *) +let rec slink ~stats src dst = + if debug >=3 + then L.stderr "slink src:%s dst:%s@." src dst; + if Sys.is_directory src + then + begin + if not (Sys.file_exists dst) + then Unix.mkdir dst 0o700; + let items = Sys.readdir src in + Array.iter + (fun item -> slink ~stats (Filename.concat src item) (Filename.concat dst item)) + items + end + else + begin + if Sys.file_exists dst then Sys.remove dst; + Unix.symlink src dst; + stats.files_linked <- stats.files_linked + 1; + end + +(** Determine if the destination should link to the source. + To check if it was linked before, check if all the captured source files + from the source are also in the destination. + And for each of the files inside (.cfg, .cg, etc), check that the destinations + of symbolic links were not modified after the links themselves. *) +let should_link ~target ~target_results_dir ~stats infer_out_src infer_out_dst = + let num_captured_files = ref 0 in + let symlink_up_to_date file = + let filename = DB.filename_from_string file in + let time_orig = DB.file_modified_time ~symlink:false filename in + let time_link = DB.file_modified_time ~symlink:true filename in + if debug >= 2 then + L.stderr "file:%s time_orig:%f time_link:%f@." + file time_orig time_link; + time_link >= time_orig in + let symlinks_up_to_date captured_file = + if Sys.is_directory captured_file then + let contents = Array.to_list (Sys.readdir captured_file) in + IList.for_all + (fun file -> + let file_path = Filename.concat captured_file file in + Sys.file_exists file_path && + (not check_timestamp_of_symlinks || symlink_up_to_date file_path)) + contents + else true in + let check_file captured_file = + Sys.file_exists captured_file && + symlinks_up_to_date captured_file in + let was_copied () = + let captured_src = Filename.concat infer_out_src Config.captured_dir_name in + let captured_dst = Filename.concat infer_out_dst Config.captured_dir_name in + if Sys.file_exists captured_src && Sys.is_directory captured_src + then + begin + let captured_files = Array.to_list (Sys.readdir captured_src) in + num_captured_files := IList.length captured_files; + IList.for_all + (fun file -> + check_file (Filename.concat captured_dst file)) + captured_files + end + else + true in + let was_modified () = + StringSet.mem target !modified_targets in + let r = + was_modified () || + not (was_copied ()) in + if r then stats.targets_merged <- stats.targets_merged + 1; + if debug >= 2 + then L.stderr "lnk:%s:%d %s@." (if r then "T" else "F") !num_captured_files target_results_dir + else if debug >= 1 && r + then L.stderr "%s@."target_results_dir; + r + +(** should_link needs to know whether the source file has changed, + and to determine whether the destination has never been copied. + In both cases, perform the link. *) +let process_merge_file deps_file = + let infer_out_dst = !Config.results_dir in + let stats = empty_stats () in + let process_line line = + match Str.split_delim (Str.regexp (Str.quote "\t")) line with + | target :: _ :: target_results_dir :: _ -> + let infer_out_src = Filename.concat (Filename.dirname !buck_out) target_results_dir in + if should_link ~target ~target_results_dir ~stats infer_out_src infer_out_dst + then slink ~stats infer_out_src infer_out_dst + | _ -> + () in + Option.may + (fun lines -> IList.iter process_line lines) + (read_file deps_file); + L.stdout "Captured results merged.@."; + L.stdout "Targets merged: %d@." stats.targets_merged; + L.stdout "Files linked: %d@." stats.files_linked + + +let merge_captured_targets () = + process_merge_file (infer_deps ()); + exit 0 diff --git a/infer/src/backend/procname.ml b/infer/src/backend/procname.ml index a6b90aa46..ccf23c721 100644 --- a/infer/src/backend/procname.ml +++ b/infer/src/backend/procname.ml @@ -507,9 +507,3 @@ module Set = Set.Make(struct (** Pretty print a set of proc names *) let pp_set fmt set = Set.iter (fun pname -> F.fprintf fmt "%a " pp pname) set - -(* -(** Return path components of a java class name *) -let java_get_class_components proc_name = - Str.split (Str.regexp (Str.quote ".")) (java_get_class proc_name) -*)