diff --git a/infer/src/backend/DB.ml b/infer/src/backend/DB.ml index 43df75a93..f77c4fb08 100644 --- a/infer/src/backend/DB.ml +++ b/infer/src/backend/DB.ml @@ -209,6 +209,17 @@ let filename_create_dir fname = if not (Sys.file_exists dirname) then create_dir dirname + +let rec create_path path = + try + Unix.mkdir path 0o700 + with + | Unix.Unix_error (Unix.EEXIST, _, _) -> () + | Unix.Unix_error (Unix.ENOENT, _, _) -> + create_path (Filename.dirname path); + create_path path + + let read_whole_file fd = let stats = Unix.fstat fd in let size = stats.Unix.st_size in @@ -328,6 +339,12 @@ module Results_dir = struct Unix.openfile full_fname [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC] 0o777 end +(** origin of a analysis artifact: current results dir, a spec library, or models *) +type origin = + | Res_dir + | Spec_lib + | Models + let global_tenv_fname () = let basename = Config.global_tenv_filename in filename_concat (captured_dir ()) basename diff --git a/infer/src/backend/DB.mli b/infer/src/backend/DB.mli index b0fc63a91..2a01a468d 100644 --- a/infer/src/backend/DB.mli +++ b/infer/src/backend/DB.mli @@ -63,6 +63,12 @@ module Results_dir : sig val create_file : path_kind -> path -> Unix.file_descr end +(** origin of a analysis artifact: current results dir, a spec library, or models *) +type origin = + | Res_dir + | Spec_lib + | Models + (** {2 Source Files} *) type source_file @@ -145,6 +151,10 @@ val find_source_dirs : unit -> source_dir list (** create a directory if it does not exist already *) val create_dir : string -> unit +(** [make_directory path] will create a directory [path] creating all the + sub directories if non-existing *) +val create_path : string -> unit + (** Read a file using a lock to allow write attempts in parallel. *) val read_file_with_lock : string -> string -> bytes option diff --git a/infer/src/backend/config.ml b/infer/src/backend/config.ml index e95982d8f..a1c776233 100644 --- a/infer/src/backend/config.ml +++ b/infer/src/backend/config.ml @@ -72,9 +72,10 @@ let models_dir = let string_crc_hex32 s = Digest.to_hex (Digest.string s) +let infer_cache : string option ref = ref None + module JarCache = struct - let infer_cache : string option ref = ref None let mkdir s = try @@ -108,7 +109,7 @@ end type zip_library = { zip_filename: string; - zip_channel: Zip.in_file; + zip_channel: Zip.in_file Lazy.t; models: bool; } @@ -116,22 +117,26 @@ let zip_filename zip_library = zip_library.zip_filename let zip_channel zip_library = - zip_library.zip_channel + Lazy.force zip_library.zip_channel + +let is_models zip_library = + zip_library.models (** list of the zip files to search for specs files *) let zip_libraries : zip_library list ref = ref [] let zip_models : zip_library list ref = ref [] +let use_jar_cache = from_env_variable "INFER_USE_JAR_CACHE" + let add_zip_library zip_filename = - if !JarCache.infer_cache != None - then + if !infer_cache != None && use_jar_cache then JarCache.handle_jar zip_filename else (* The order matters, the jar files should be added following the order *) (* specs files should be searched in them *) let zip_library = { zip_filename = zip_filename; - zip_channel = Zip.open_in zip_filename; + zip_channel = lazy (Zip.open_in zip_filename); models = false } in zip_libraries := zip_library :: !zip_libraries @@ -139,7 +144,7 @@ let add_zip_library zip_filename = let add_models zip_filename = let zip_library = { zip_filename = zip_filename; - zip_channel = Zip.open_in zip_filename; + zip_channel = lazy (Zip.open_in zip_filename); models = true } in zip_models := zip_library :: !zip_models diff --git a/infer/src/backend/specs.ml b/infer/src/backend/specs.ml index 95c1e1e49..ff48a0678 100644 --- a/infer/src/backend/specs.ml +++ b/infer/src/backend/specs.ml @@ -334,13 +334,7 @@ type summary = attributes : ProcAttributes.t; (** Attributes of the procedure *) } -(** origin of a summary: current results dir, a spec library, or models *) -type origin = - | Res_dir - | Spec_lib - | Models - -type spec_tbl = (summary * origin) Procname.Hash.t +type spec_tbl = (summary * DB.origin) Procname.Hash.t let spec_tbl: spec_tbl = Procname.Hash.create 128 @@ -505,13 +499,13 @@ let summary_compact sh summary = let set_summary_origin proc_name summary origin = Procname.Hash.replace spec_tbl proc_name (summary, origin) -let add_summary_origin (proc_name : Procname.t) (summary: summary) (origin: origin) : unit = +let add_summary_origin (proc_name : Procname.t) (summary: summary) (origin: DB.origin) : unit = L.out "Adding summary for %a@\n@[ %a@]@." Procname.pp proc_name (pp_summary pe_text false) summary; set_summary_origin proc_name summary origin (** Add the summary to the table for the given function *) let add_summary (proc_name : Procname.t) (summary: summary) : unit = - add_summary_origin proc_name summary Res_dir + add_summary_origin proc_name summary DB.Res_dir let specs_filename pname = let pname_file = Procname.to_filename pname in @@ -560,22 +554,6 @@ let store_summary pname (summ: summary) = let load_summary specs_file = Serialization.from_file summary_serializer specs_file -(** Load procedure summary from the given zip file *) -(* TODO: instead of always going through the same list for zip files for every proc_name, *) -(* create beforehand a map from specs filenames to zip filenames, so that looking up the specs for a given procedure is fast *) -let load_summary_from_zip zip_specs_path zip_channel = - let found_summary = - try - let entry = Zip.find_entry zip_channel zip_specs_path in - begin - match Serialization.from_string summary_serializer (Zip.read_entry zip_channel entry) with - | Some summ -> Some summ - | None -> - L.err "Could not load specs datastructure from %s@." zip_specs_path; - None - end - with Not_found -> None in - found_summary (** Load procedure summary for the given procedure name and update spec table *) let load_summary_to_spec_table proc_name = @@ -585,7 +563,7 @@ let load_summary_to_spec_table proc_name = let load_summary_models models_dir = match load_summary models_dir with | None -> false - | Some summ -> add summ Models in + | Some summ -> add summ DB.Models in let rec load_summary_libs = function (* try to load the summary from a list of libs *) | [] -> false | spec_path :: spec_paths -> @@ -593,31 +571,20 @@ let load_summary_to_spec_table proc_name = | None -> load_summary_libs spec_paths | Some summ -> add summ Spec_lib) in - let rec load_summary_ziplibs zip_libraries = (* try to load the summary from a list of zip libraries *) - let zip_specs_filename = specs_filename proc_name in - let zip_specs_path = - let root = Filename.concat Config.default_in_zip_results_dir Config.specs_dir_name in - Filename.concat root zip_specs_filename in - match zip_libraries with - | [] -> false - | zip_library:: zip_libraries -> - begin - match load_summary_from_zip zip_specs_path (Config.zip_channel zip_library) with - | None -> load_summary_ziplibs zip_libraries - | Some summ -> - let origin = if zip_library.Config.models then Models else Spec_lib in - add summ origin - end in + let load_summary_ziplibs zip_specs_filename = + let zip_specs_path = Filename.concat Config.specs_dir_name zip_specs_filename in + match ZipLib.load summary_serializer zip_specs_path with + | None -> false + | Some (summary, origin) -> add summary origin in let default_spec_dir = res_dir_specs_filename proc_name in match load_summary default_spec_dir with | None -> (* search on models, libzips, and libs *) - if load_summary_models (specs_models_filename proc_name) then true - else if load_summary_ziplibs (Config.get_zip_libraries ()) then true - else load_summary_libs (specs_library_filenames proc_name) - + load_summary_models (specs_models_filename proc_name) || + load_summary_ziplibs (specs_filename proc_name) || + load_summary_libs (specs_library_filenames proc_name) | Some summ -> - add summ Res_dir + add summ DB.Res_dir let rec get_summary_origin proc_name = try @@ -687,7 +654,7 @@ let pdesc_resolve_attributes proc_desc = let get_origin proc_name = match get_summary_origin proc_name with | Some (_, origin) -> origin - | None -> Res_dir + | None -> DB.Res_dir let summary_exists proc_name = match get_summary proc_name with @@ -806,7 +773,7 @@ let init_summary { proc_attributes with ProcAttributes.proc_flags = proc_flags; }; } in - Procname.Hash.replace spec_tbl proc_attributes.ProcAttributes.proc_name (summary, Res_dir) + Procname.Hash.replace spec_tbl proc_attributes.ProcAttributes.proc_name (summary, DB.Res_dir) (** Reset a summary rebuilding the dependents and preserving the proc attributes if present. *) let reset_summary call_graph proc_name attributes_opt = diff --git a/infer/src/backend/specs.mli b/infer/src/backend/specs.mli index 967fedd4c..ad74141dc 100644 --- a/infer/src/backend/specs.mli +++ b/infer/src/backend/specs.mli @@ -147,12 +147,6 @@ type summary = attributes : ProcAttributes.t; (** Attributes of the procedure *) } -(** origin of a summary: current results dir, a spec library, or models *) -type origin = - | Res_dir - | Spec_lib - | Models - (** Add the summary to the table for the given function *) val add_summary : Procname.t -> summary -> unit @@ -184,7 +178,7 @@ val get_flag : Procname.t -> string -> string option val get_phase : Procname.t -> phase (** Return the origin of the spec file *) -val get_origin: Procname.t -> origin +val get_origin: Procname.t -> DB.origin (** Return the signature of a procedure declaration as a string *) val get_signature : summary -> string diff --git a/infer/src/backend/tabulation.ml b/infer/src/backend/tabulation.ml index c96cbe066..989d6ac87 100644 --- a/infer/src/backend/tabulation.ml +++ b/infer/src/backend/tabulation.ml @@ -664,7 +664,7 @@ let prop_set_exn pname prop se_exn = (** Include a subtrace for a procedure call if the callee is not a model. *) let include_subtrace callee_pname = - Specs.get_origin callee_pname <> Specs.Models + Specs.get_origin callee_pname <> DB.Models (** combine the spec's post with a splitting and actual precondition *) let combine diff --git a/infer/src/backend/utils.ml b/infer/src/backend/utils.ml index e774b84be..798796563 100644 --- a/infer/src/backend/utils.ml +++ b/infer/src/backend/utils.ml @@ -520,7 +520,7 @@ let base_arg_desc = "root directory of the project"; "-infer_cache", - Arg.String (fun s -> Config.JarCache.infer_cache := Some (filename_to_absolute s)), + Arg.String (fun s -> Config.infer_cache := Some (filename_to_absolute s)), Some "dir", "Select a directory to contain the infer cache"; diff --git a/infer/src/backend/zipLib.ml b/infer/src/backend/zipLib.ml new file mode 100644 index 000000000..04abafdda --- /dev/null +++ b/infer/src/backend/zipLib.ml @@ -0,0 +1,65 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +open! Utils + +module L = Logging + + +let get_cache_dir infer_cache zip_filename = + let basename = Filename.basename zip_filename in + let key = basename ^ string_crc_hex32 zip_filename in + Filename.concat infer_cache key + +let load_from_cache serializer zip_path cache_dir zip_library = + let absolute_path = Filename.concat cache_dir zip_path in + let deserialize = Serialization.from_file serializer in + let extract to_path = + if not (Sys.file_exists to_path) then + begin + DB.create_path (Filename.dirname to_path); + let zip_channel = Config.zip_channel zip_library in + let entry = Zip.find_entry zip_channel zip_path in + Zip.copy_entry_to_file zip_channel entry to_path + end; + DB.filename_from_string to_path in + match deserialize (extract absolute_path) with + | Some data when Config.is_models zip_library -> Some (data, DB.Models) + | Some data -> Some (data, DB.Spec_lib) + | None -> None + | exception Not_found -> None + +let load_from_zip serializer zip_path zip_library = + let zip_channel = Config.zip_channel zip_library in + let deserialize = Serialization.from_string serializer in + match deserialize (Zip.read_entry zip_channel (Zip.find_entry zip_channel zip_path)) with + | Some data when Config.is_models zip_library -> Some (data, DB.Models) + | Some data -> Some (data, DB.Spec_lib) + | None -> None + | exception Not_found -> None + +let load_data serializer path zip_library = + let zip_path = Filename.concat Config.default_in_zip_results_dir path in + match !Config.infer_cache with + | None -> + load_from_zip serializer zip_path zip_library + | Some infer_cache -> + let cache_dir = get_cache_dir infer_cache (Config.zip_filename zip_library) in + load_from_cache serializer zip_path cache_dir zip_library + +(* Search path in the list of zip libraries and use a cache directory to save already + deserialized data *) +let load serializer path = + let rec loop = function + | [] -> None + | zip_library :: other_libraries -> + let opt = load_data serializer path zip_library in + if Option.is_some opt then opt + else loop other_libraries in + loop (Config.get_zip_libraries ()) diff --git a/infer/src/backend/zipLib.mli b/infer/src/backend/zipLib.mli new file mode 100644 index 000000000..c2441c57c --- /dev/null +++ b/infer/src/backend/zipLib.mli @@ -0,0 +1,13 @@ +(* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + *) + +(** [load serializer path] searches for the file at the given path in the zip libraries. + If Config.infer_cache is set, already deserialized data will be saved there and [path] + will be searched from the cache first. *) +val load : 'a Serialization.serializer -> string -> ('a * DB.origin) option