diff --git a/infer/src/clang/Capture.ml b/infer/src/clang/Capture.ml index e1c600527..145c622a7 100644 --- a/infer/src/clang/Capture.ml +++ b/infer/src/clang/Capture.ml @@ -99,14 +99,11 @@ let run_clang_frontend ast_source = L.(debug Capture Medium) "Start %s of AST from %a@\n" Config.clang_frontend_action_string pp_ast_filename ast_source ; if Config.linters then AL.do_frontend_checks trans_unit_ctx ast_decl ; - if Config.export_changed_functions then ( + ( if Config.export_changed_functions then let source_file = trans_unit_ctx.CFrontend_config.source_file in - let process_ast_fn = - AstToRangeMap.process_ast trans_unit_ctx ast_decl (Tenv.create ()) source_file - in - TestDeterminator.test_to_run_clang source_file ~process_ast_fn - ~changed_lines_file:Config.modified_lines ~test_samples_file:None ; - TestDeterminator.emit_relevant_methods () ) ; + let clang_range_map = AstToRangeMap.process_ast ast_decl source_file in + TestDeterminator.compute_and_emit_relevant_methods ~clang_range_map ~source_file + ~changed_lines_file:Config.modified_lines ) ; if Config.capture then CFrontend.do_source_file trans_unit_ctx ast_decl ; L.(debug Capture Medium) "End %s of AST file %a... OK!@\n" Config.clang_frontend_action_string pp_ast_filename diff --git a/infer/src/clang/astToRangeMap.ml b/infer/src/clang/astToRangeMap.ml index f5154ce06..0ad93e2f0 100644 --- a/infer/src/clang/astToRangeMap.ml +++ b/infer/src/clang/astToRangeMap.ml @@ -5,21 +5,10 @@ * LICENSE file in the root directory of this source tree. *) open! IStd -module F = Format -module L = Logging -let process_ast trans_unit_ctx ast tenv default_source_file f = +let process_ast ast default_source_file = let open Clang_ast_t in - let mk_key decl = CType_decl.CProcname.from_decl ~tenv decl in - let call_f decl range = - CFrontend_errors.protect trans_unit_ctx - ~recover:(fun () -> ()) - ~pp_context:(fun f () -> - F.fprintf f "%a: processing %s" SourceFile.pp default_source_file - (Clang_ast_j.string_of_decl decl) ) - ~f:(fun () -> f (mk_key decl) range) - in - let rec extract_location decl = + let rec extract_location ast_range decl = match decl with | ObjCMethodDecl (di, _, _) | CXXConversionDecl (di, _, _, _, _) @@ -28,27 +17,23 @@ let process_ast trans_unit_ctx ast tenv default_source_file f = | CXXConstructorDecl (di, _, _, _, _) | CXXDestructorDecl (di, _, _, _, _) -> let range = CLocation.location_of_decl_info default_source_file di in - call_f decl range + Typ.Procname.Map.add (CType_decl.CProcname.from_decl decl) range ast_range | _ -> ( match Clang_ast_proj.get_decl_context_tuple decl with | Some (decls, _) -> - List.iter decls ~f:extract_location + List.fold decls ~f:extract_location ~init:ast_range | None -> - L.(debug Capture Verbose) - "@\n Found %s.., skipping" - (Clang_ast_proj.get_decl_kind_string decl) ) + ast_range ) in match ast with | TranslationUnitDecl (_, decl_list, _, _) -> - CFrontend_config.global_translation_unit_decls := decl_list ; - L.(debug Capture Verbose) "@\n Processing AST...@\n" ; - List.iter decl_list ~f:(fun d -> - let info = Clang_ast_proj.get_decl_tuple d in + List.fold decl_list ~init:Typ.Procname.Map.empty ~f:(fun map decl -> + let info = Clang_ast_proj.get_decl_tuple decl in let source_range = info.di_source_range in if CLocation.should_translate_lib default_source_file source_range `DeclTraversal ~translate_when_used:true - then extract_location d ) ; - L.(debug Capture Verbose) "@\n Finished processing AST.@\n" + then extract_location map decl + else map ) | _ -> assert false diff --git a/infer/src/clang/astToRangeMap.mli b/infer/src/clang/astToRangeMap.mli index ad4d92fb5..9ab416d0c 100644 --- a/infer/src/clang/astToRangeMap.mli +++ b/infer/src/clang/astToRangeMap.mli @@ -6,10 +6,4 @@ *) open! IStd -val process_ast : - CFrontend_config.translation_unit_context - -> Clang_ast_t.decl - -> Tenv.t - -> SourceFile.t - -> (Typ.Procname.t -> Location.t * Location.t -> unit) - -> unit +val process_ast : Clang_ast_t.decl -> SourceFile.t -> (Location.t * Location.t) Typ.Procname.Map.t diff --git a/infer/src/infer.ml b/infer/src/infer.ml index 5fa703075..d2bd71373 100644 --- a/infer/src/infer.ml +++ b/infer/src/infer.ml @@ -147,10 +147,10 @@ let () = if Config.debug_mode && CLOpt.is_originator then ( L.progress "Logs in %s@." (Config.results_dir ^/ Config.log_file) ; L.progress "Execution ID %Ld@." Config.execution_id ) ; - ( if Config.test_determinator then ( - TestDeterminator.test_to_run_java ~changed_lines_file:Config.modified_lines - ~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info ; - TestDeterminator.emit_tests_to_run () ) + ( if Config.test_determinator then + TestDeterminator.compute_and_emit_test_to_run ~changed_lines_file:Config.modified_lines + ~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info + ?clang_range_map:None ?source_file:None else match Config.command with | Analyze -> diff --git a/infer/src/integration/testDeterminator.ml b/infer/src/integration/testDeterminator.ml index 6f312e9b6..5749ddb78 100644 --- a/infer/src/integration/testDeterminator.ml +++ b/infer/src/integration/testDeterminator.ml @@ -14,29 +14,15 @@ module YB = Yojson.Basic (* a flag used to make the method search signature sensitive *) let use_method_signature = false -module RangeMap = Caml.Map.Make (struct - type t = Typ.Procname.t - - let compare = Typ.Procname.compare -end) - -let initialized_test_determinator = ref false - -let is_test_determinator_init () = !initialized_test_determinator +module RangeMap = Typ.Procname.Map module MethodRangeMap = struct - type t = (Location.t * Location.t) RangeMap.t - - let map : t ref = ref RangeMap.empty - - let method_range_map () = !map - let split_class_method_name qualified_method_name = String.rsplit2_exn qualified_method_name ~on:'.' - let create_java_method_range_map code_graph_file' = - match code_graph_file' with + let create_java_method_range_map code_graph_file_opt = + match code_graph_file_opt with | Some code_graph_file -> let open Java_method_decl_j in let json_string = @@ -48,112 +34,85 @@ module MethodRangeMap = struct L.die UserError "Could not read file %s" code_graph_file in let method_decls = java_method_decls_of_string json_string in - let map' = - List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl -> - let start_location = - { Location.line= decl.start_line - ; col= -1 - ; file= SourceFile.create ~warn_on_error:false decl.source_file } - in - let end_location = - { Location.line= decl.end_line - ; col= -1 - ; file= SourceFile.create ~warn_on_error:false decl.source_file } - in - let range = (start_location, end_location) in - let classname, methodname = split_class_method_name decl.method_name in - match decl.signature with - | Some signature -> - let signature = - if use_method_signature then signature - else - (* When we should not use the signature we use 'void ()' *) - JPS.JNI.void_method_with_no_arguments - in - let key = JPS.create_procname ~classname ~methodname ~signature in - RangeMap.add key range acc - | None -> - acc ) - in - map := map' + List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl -> + let start_location = + { Location.line= decl.start_line + ; col= -1 + ; file= SourceFile.create ~warn_on_error:false decl.source_file } + in + let end_location = + { Location.line= decl.end_line + ; col= -1 + ; file= SourceFile.create ~warn_on_error:false decl.source_file } + in + let range = (start_location, end_location) in + let classname, methodname = split_class_method_name decl.method_name in + match decl.signature with + | Some signature -> + let signature = + if use_method_signature then signature + else + (* When we should not use the signature we use 'void ()' *) + JPS.JNI.void_method_with_no_arguments + in + let key = JPS.create_procname ~classname ~methodname ~signature in + RangeMap.add key range acc + | None -> + acc ) | _ -> L.die UserError "Missing method declaration info argument" - - - let create_clang_method_range_map process_ast_fn = - process_ast_fn (fun key range -> map := RangeMap.add key range !map) - - - let pp_map fmt () = - let pp_map' fmt m = - RangeMap.iter - (fun key range -> - F.fprintf fmt "@\n %a --> (%a,%a)" Typ.Procname.pp key Location.pp (fst range) - Location.pp (snd range) ) - m - in - F.fprintf fmt "@\n--- Method Range Map ---%a@\n--- End Method Range Map --- @\n" pp_map' !map end module DiffLines = struct (* This is a map file name |--> {set of changed line } *) - let map : int list String.Map.t ref = ref String.Map.empty - - let changed_lines_map () = !map (* Read the file containing info on changed lines and populate the map *) - let init_changed_lines_map changed_lines_file' = - match changed_lines_file' with + let create_changed_lines_map changed_lines_file_opt = + match changed_lines_file_opt with | Some changed_lines_file -> ( - L.(debug TestDeterminator Medium) - "Initializing changed lines map from file '%s'..." changed_lines_file ; - match Utils.read_file changed_lines_file with - | Ok cl_list -> - let changed_lines = - List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item -> - let fname, cl = String.rsplit2_exn ~on:':' cl_item in - String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) ) - in - L.(debug TestDeterminator Medium) "done@\n" ; - map := changed_lines - | Error _ -> - L.die UserError "Could not read file %s" changed_lines_file ) - | _ -> + match Utils.read_file changed_lines_file with + | Ok cl_list -> + List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item -> + let fname, cl = String.rsplit2_exn ~on:':' cl_item in + String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) ) + | Error _ -> + L.die UserError "Could not read file %s" changed_lines_file ) + | None -> L.die UserError "Missing modified lines argument" - let print_changed_lines () = - L.(debug TestDeterminator Quiet) "--- Changed Lines Map ---@\n" ; - String.Map.iteri !map ~f:(fun ~key ~data -> - L.(debug TestDeterminator Quiet) - "%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data ) + [@@@warning "-32"] + + let pp_changed_lines fmt map = + F.fprintf fmt "--- Changed Lines Map ---@\n" ; + String.Map.iteri map ~f:(fun ~key ~data -> + F.fprintf fmt "%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data ) end +[@@@warning "-32"] + let pp_profiler_sample_set fmt s = F.fprintf fmt " (set size = %i) " (JPS.ProfilerSample.cardinal s) ; JPS.ProfilerSample.iter (fun m -> F.fprintf fmt "@\n %a " Typ.Procname.pp m) s module TestSample = struct - let labeled_test_samples = ref [] - - let test_sample () = !labeled_test_samples - - let init_test_sample test_samples_file' = - match test_samples_file' with + let read_test_sample test_samples_file_opt = + match test_samples_file_opt with | Some test_samples_file -> L.(debug TestDeterminator Medium) "Reading Profiler Samples File '%s'....@\n" test_samples_file ; - let ts = JPS.from_json_file test_samples_file ~use_signature:use_method_signature in - labeled_test_samples := ts - | _ -> + JPS.from_json_file test_samples_file ~use_signature:use_method_signature + | None -> L.die UserError "Missing profiler samples argument" - let pp_map fmt () = - List.iter !labeled_test_samples ~f:(fun (label, profiler_samples) -> + [@@@warning "-32"] + + let pp_map fmt labeled_test_samples = + List.iter labeled_test_samples ~f:(fun (label, profiler_samples) -> F.fprintf fmt "=== Samples for %s ===@\n%a@\n=== End Samples for %s ===@\n" label pp_profiler_sample_set profiler_samples label ) end @@ -161,147 +120,81 @@ end let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line let affected_methods method_range_map file_changed_lines changed_lines = - L.(debug TestDeterminator Medium) "Looking for affected methods in file '%s' " file_changed_lines ; RangeMap.fold (fun key ((l1, _) as range) acc -> let method_file = SourceFile.to_string l1.Location.file in if String.equal method_file file_changed_lines && List.exists ~f:(fun l -> in_range l range) changed_lines - then ( - L.(debug TestDeterminator Medium) - "Adding '%a' in affected methods...@\n" Typ.Procname.pp key ; - JPS.ProfilerSample.add key acc ) + then JPS.ProfilerSample.add key acc else acc ) method_range_map JPS.ProfilerSample.empty let compute_affected_methods_java changed_lines_map method_range_map = - let affected_methods = - String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty - ~f:(fun ~key:file_changed_lines ~data acc -> - let am = affected_methods method_range_map file_changed_lines data in - JPS.ProfilerSample.union am acc ) - in - L.(debug TestDeterminator Quiet) - "== Affected Methods ==%a@\n== End Affected Methods ==@\n" pp_profiler_sample_set - affected_methods ; - affected_methods + String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty + ~f:(fun ~key:file_changed_lines ~data acc -> + let am = affected_methods method_range_map file_changed_lines data in + JPS.ProfilerSample.union am acc ) -let compute_affected_methods_clang source_file changed_lines_map method_range_map = +let compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map = let fname = SourceFile.to_rel_path source_file in - L.(debug TestDeterminator Medium) "Looking for file %s in changed-line map..." fname ; match String.Map.find changed_lines_map fname with | Some changed_lines -> - L.(debug TestDeterminator Medium) "found!@\n" ; - let affected_methods = affected_methods method_range_map fname changed_lines in - L.(debug TestDeterminator Medium) - "@\n@\n== Resulting Affected Methods ==%a@\n== End Affected Methods ==@\n\n" - pp_profiler_sample_set affected_methods ; - affected_methods + affected_methods clang_range_map fname changed_lines | None -> - L.(debug TestDeterminator Medium) - "%s not found in changed-line map. Nothing else to do for it.@\n" fname ; JPS.ProfilerSample.empty -let relevant_tests = ref [] - -(* Methods modified in a diff *) -let relevant_methods = ref JPS.ProfilerSample.empty - -let _get_relevant_test_to_run () = !relevant_tests - -let emit_tests_to_run () = - let json = `List (List.map ~f:(fun t -> `String t) !relevant_tests) in - let outpath = Config.results_dir ^/ Config.test_determinator_output in - YB.to_file outpath json ; - L.(debug TestDeterminator Medium) - "Tests to run: [%a]@\n" - (Pp.seq ~sep:", " F.pp_print_string) - !relevant_tests - - -let emit_relevant_methods () = +let emit_relevant_methods relevant_methods = let cleaned_methods = List.dedup_and_sort ~compare:String.compare - (List.map (JPS.ProfilerSample.elements !relevant_methods) ~f:Typ.Procname.to_string) + (List.map (JPS.ProfilerSample.elements relevant_methods) ~f:Typ.Procname.to_string) in let json = `List (List.map ~f:(fun t -> `String t) cleaned_methods) in let outpath = Config.results_dir ^/ Config.export_changed_functions_output in - YB.to_file outpath json ; - L.(debug TestDeterminator Medium) - "Methods modified in this Diff: %a@\n" - (Pp.seq ~sep:", " F.pp_print_string) - cleaned_methods - - -let init_clang process_ast_fn changed_lines_file test_samples_file = - DiffLines.init_changed_lines_map changed_lines_file ; - DiffLines.print_changed_lines () ; - MethodRangeMap.create_clang_method_range_map process_ast_fn ; - L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ; - match test_samples_file with - | Some _ -> - TestSample.init_test_sample test_samples_file - | _ -> - () ; - L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ; - initialized_test_determinator := true - - -let init_java changed_lines_file test_samples_file code_graph_file = - DiffLines.init_changed_lines_map changed_lines_file ; - DiffLines.print_changed_lines () ; - MethodRangeMap.create_java_method_range_map code_graph_file ; - L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ; - TestSample.init_test_sample test_samples_file ; - L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ; - initialized_test_determinator := true + YB.to_file outpath json -(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) -let test_to_run_clang source_file ~process_ast_fn ~changed_lines_file ~test_samples_file = - L.(debug TestDeterminator Quiet) - "****** Start Test Determinator for %s *****@\n" - (SourceFile.to_string source_file) ; - if is_test_determinator_init () then () - else init_clang process_ast_fn changed_lines_file test_samples_file ; - let affected_methods = - compute_affected_methods_clang source_file (DiffLines.changed_lines_map ()) - (MethodRangeMap.method_range_map ()) +let compute_and_emit_relevant_methods ~clang_range_map ~source_file ~changed_lines_file = + let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in + let relevant_methods = + compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map in - relevant_methods := JPS.ProfilerSample.union affected_methods !relevant_methods ; - let test_to_run = - if JPS.ProfilerSample.is_empty affected_methods then [] - else - List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) -> - let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in - if JPS.ProfilerSample.is_empty intersection then acc else label :: acc ) - in - relevant_tests := List.append test_to_run !relevant_tests + emit_relevant_methods relevant_methods -let test_to_run_java ~changed_lines_file ~test_samples_file ~code_graph_file = - L.(debug TestDeterminator Quiet) "***** Start Java Test Determinator *****@\n" ; - if is_test_determinator_init () then () - else init_java changed_lines_file test_samples_file code_graph_file ; +(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) +let test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file + ~test_samples_file = + let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in let affected_methods = - compute_affected_methods_java (DiffLines.changed_lines_map ()) - (MethodRangeMap.method_range_map ()) + match (clang_range_map, source_file) with + | Some clang_range_map, Some source_file -> + compute_affected_methods_clang ~source_file ~clang_range_map ~changed_lines_map + | _ (* Java case *) -> + let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in + compute_affected_methods_java changed_lines_map method_range in - let test_to_run = - if JPS.ProfilerSample.is_empty affected_methods then [] - else - List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) -> - let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in - if JPS.ProfilerSample.is_empty intersection then acc - else ( - L.(debug TestDeterminator Quiet) - "Choosing test '%s' because of [%a]@\n" label - (Pp.seq Typ.Procname.pp ~sep:", ") - (JPS.ProfilerSample.elements intersection) ; - label :: acc ) ) + let profiler_samples = TestSample.read_test_sample test_samples_file in + if JPS.ProfilerSample.is_empty affected_methods then [] + else + List.fold profiler_samples ~init:[] ~f:(fun acc (label, profiler_samples) -> + let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in + if JPS.ProfilerSample.is_empty intersection then acc else label :: acc ) + + +let emit_tests_to_run relevant_tests = + let json = `List (List.map ~f:(fun t -> `String t) relevant_tests) in + let outpath = Config.results_dir ^/ Config.test_determinator_output in + YB.to_file outpath json + + +let compute_and_emit_test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file + ~test_samples_file = + let relevant_tests = + test_to_run ?clang_range_map ~code_graph_file ?source_file ~changed_lines_file + ~test_samples_file in - relevant_tests := List.append test_to_run !relevant_tests + emit_tests_to_run relevant_tests diff --git a/infer/src/integration/testDeterminator.mli b/infer/src/integration/testDeterminator.mli index 7184ffda7..c35e57b68 100644 --- a/infer/src/integration/testDeterminator.mli +++ b/infer/src/integration/testDeterminator.mli @@ -7,21 +7,16 @@ open! IStd -val test_to_run_java : - changed_lines_file:string option - -> test_samples_file:string option +val compute_and_emit_test_to_run : + ?clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t + -> ?source_file:SourceFile.t -> code_graph_file:string option - -> unit - -val test_to_run_clang : - SourceFile.t - -> process_ast_fn:((Typ.Procname.t -> Location.t * Location.t -> unit) -> unit) -> changed_lines_file:string option -> test_samples_file:string option -> unit -val emit_tests_to_run : unit -> unit - -val emit_relevant_methods : unit -> unit - -val _get_relevant_test_to_run : unit -> string list +val compute_and_emit_relevant_methods : + clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t + -> source_file:SourceFile.t + -> changed_lines_file:string option + -> unit