From e2725029f20f7d1114966fed3a2cb5868acffd14 Mon Sep 17 00:00:00 2001 From: Dulma Churchill Date: Thu, 17 Oct 2019 02:06:07 -0700 Subject: [PATCH] [test determinator] Parse the clang profiler samples and matches c functions Summary: - Adds ATD file to parse the clang profiler samples - Procnames don't help us here because we want to use mangled names, not the version of names that Infer needs, so passing in the RangeMap also ClangProc that just include the names and mangled names. - First matching of c functions to have something in place to add a test, matching further method kinds to be done in next diff. Reviewed By: skcho Differential Revision: D17877071 fbshipit-source-id: b31d651a7 --- infer/src/Makefile | 4 +- infer/src/atd/clang_profiler_samples.atd | 18 ++++ infer/src/base/ClangProc.ml | 14 +++ infer/src/base/ClangProc.mli | 14 +++ infer/src/clang/astToRangeMap.ml | 37 +++++++- infer/src/clang/astToRangeMap.mli | 5 +- .../src/test_determinator/testDeterminator.ml | 95 +++++++++++++++---- .../test_determinator/testDeterminator.mli | 6 +- 8 files changed, 167 insertions(+), 26 deletions(-) create mode 100644 infer/src/atd/clang_profiler_samples.atd create mode 100644 infer/src/base/ClangProc.ml create mode 100644 infer/src/base/ClangProc.mli diff --git a/infer/src/Makefile b/infer/src/Makefile index 621df3f83..eaa959132 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -19,7 +19,7 @@ INFER_MAIN = infer #### Checkers declarations #### -INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples +INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples atd/clang_profiler_samples INFER_ATDGEN_TYPES = j t INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd) INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli) @@ -305,7 +305,7 @@ clean: $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(REMOVE) $(BIN_DIR)/llvm_sil $(REMOVE) $(INFER_CREATE_TRACEVIEW_LINKS_BIN) - $(REMOVE) atd/*_{j,t,v}.ml{,i} atd/clang_* + $(REMOVE) atd/*_{j,t,v}.ml{,i} atd/clang_ast_* $(REMOVE) mod_dep.dot $(REMOVE) mod_dep.pdf diff --git a/infer/src/atd/clang_profiler_samples.atd b/infer/src/atd/clang_profiler_samples.atd new file mode 100644 index 000000000..6f63cfdad --- /dev/null +++ b/infer/src/atd/clang_profiler_samples.atd @@ -0,0 +1,18 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +type native_symbol = { + name : string; + ?mangled_name : string option; +} + +type profiler_sample = { + test : string; + native_symbols : native_symbol list; +} + +type profiler_samples = profiler_sample list diff --git a/infer/src/base/ClangProc.ml b/infer/src/base/ClangProc.ml new file mode 100644 index 000000000..67514f279 --- /dev/null +++ b/infer/src/base/ClangProc.ml @@ -0,0 +1,14 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd + +type t = + | CFunction of {name: string; mangled_name: string option} + | CppMethod of {mangled_name: string} + | ObjcMethod of {mangled_name: string} + | ObjcBlock of {mangled_name: string} diff --git a/infer/src/base/ClangProc.mli b/infer/src/base/ClangProc.mli new file mode 100644 index 000000000..67514f279 --- /dev/null +++ b/infer/src/base/ClangProc.mli @@ -0,0 +1,14 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd + +type t = + | CFunction of {name: string; mangled_name: string option} + | CppMethod of {mangled_name: string} + | ObjcMethod of {mangled_name: string} + | ObjcBlock of {mangled_name: string} diff --git a/infer/src/clang/astToRangeMap.ml b/infer/src/clang/astToRangeMap.ml index 0ad93e2f0..54f07736f 100644 --- a/infer/src/clang/astToRangeMap.ml +++ b/infer/src/clang/astToRangeMap.ml @@ -6,6 +6,36 @@ *) open! IStd +(* Builds a clang procedure, following the format required to match with profiler samples: +C Functions: name, mangled name optional +ObjC Methods: mangled_name +ObjC Blocks: mangled_name +C++ methods: mangled_name (For us mangled name is optional, but if it is not there then we can't match the method) *) +let clang_proc_of_decl decl = + let open Clang_ast_t in + match decl with + | ObjCMethodDecl (_, _, omdi) -> + Some (ClangProc.ObjcMethod {mangled_name= omdi.Clang_ast_t.omdi_mangled_name}) + | BlockDecl (_, bdi) -> + Some (ClangProc.ObjcBlock {mangled_name= bdi.Clang_ast_t.bdi_mangled_name}) + | FunctionDecl (_, named_decl_info, _, fdi) -> + Some + (ClangProc.CFunction + { name= named_decl_info.Clang_ast_t.ni_name + ; mangled_name= fdi.Clang_ast_t.fdi_mangled_name }) + | CXXConversionDecl (_, _, _, fdi, _) + | CXXMethodDecl (_, _, _, fdi, _) + | CXXConstructorDecl (_, _, _, fdi, _) + | CXXDestructorDecl (_, _, _, fdi, _) -> ( + match fdi.Clang_ast_t.fdi_mangled_name with + | Some mangled_name -> + Some (ClangProc.CppMethod {mangled_name}) + | None -> + None ) + | _ -> + None + + let process_ast ast default_source_file = let open Clang_ast_t in let rec extract_location ast_range decl = @@ -15,9 +45,12 @@ let process_ast ast default_source_file = | FunctionDecl (di, _, _, _) | CXXMethodDecl (di, _, _, _, _) | CXXConstructorDecl (di, _, _, _, _) - | CXXDestructorDecl (di, _, _, _, _) -> + | CXXDestructorDecl (di, _, _, _, _) + | BlockDecl (di, _) -> let range = CLocation.location_of_decl_info default_source_file di in - Typ.Procname.Map.add (CType_decl.CProcname.from_decl decl) range ast_range + let procname = CType_decl.CProcname.from_decl decl in + let clang_proc = clang_proc_of_decl decl in + Typ.Procname.Map.add procname (range, clang_proc) ast_range | _ -> ( match Clang_ast_proj.get_decl_context_tuple decl with | Some (decls, _) -> diff --git a/infer/src/clang/astToRangeMap.mli b/infer/src/clang/astToRangeMap.mli index 9ab416d0c..d3c7bcc09 100644 --- a/infer/src/clang/astToRangeMap.mli +++ b/infer/src/clang/astToRangeMap.mli @@ -6,4 +6,7 @@ *) open! IStd -val process_ast : Clang_ast_t.decl -> SourceFile.t -> (Location.t * Location.t) Typ.Procname.Map.t +val process_ast : + Clang_ast_t.decl + -> SourceFile.t + -> ((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t diff --git a/infer/src/test_determinator/testDeterminator.ml b/infer/src/test_determinator/testDeterminator.ml index cb69ec48a..b47e3eb15 100644 --- a/infer/src/test_determinator/testDeterminator.ml +++ b/infer/src/test_determinator/testDeterminator.ml @@ -49,7 +49,7 @@ module MethodRangeMap = struct let key = JProcname.create_procname ~use_signature ~classname ~methodname ~signature in - Typ.Procname.Map.add key range acc + Typ.Procname.Map.add key (range, ()) acc | None -> acc ) | _ -> @@ -92,7 +92,7 @@ let pp_profiler_sample_set fmt s = module TestSample = struct - let read_test_sample test_samples_file_opt = + let read_java_test_sample test_samples_file_opt = match test_samples_file_opt with | Some test_samples_file -> L.(debug TestDeterminator Medium) @@ -102,6 +102,15 @@ module TestSample = struct L.die UserError "Missing profiler samples argument" + let read_clang_test_sample test_samples_file_opt = + match test_samples_file_opt with + | Some test_samples_file -> + Atdgen_runtime.Util.Json.from_file Clang_profiler_samples_j.read_profiler_samples + test_samples_file + | None -> + L.die UserError "Missing profiler samples argument" + + [@@@warning "-32"] let pp_map fmt labeled_test_samples = @@ -112,14 +121,18 @@ end let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line +let is_file_in_changed_lines file_changed_lines changed_lines range = + let l1, _ = range in + let method_file = SourceFile.to_string l1.Location.file in + String.equal method_file file_changed_lines + && List.exists ~f:(fun l -> in_range l range) changed_lines + + let affected_methods method_range_map file_changed_lines changed_lines = Typ.Procname.Map.fold - (fun key ((l1, _) as range) acc -> - let method_file = SourceFile.to_string l1.Location.file in - if - String.equal method_file file_changed_lines - && List.exists ~f:(fun l -> in_range l range) changed_lines - then Typ.Procname.Set.add key acc + (fun key (range, _) acc -> + if is_file_in_changed_lines file_changed_lines changed_lines range then + Typ.Procname.Set.add key acc else acc ) method_range_map Typ.Procname.Set.empty @@ -140,6 +153,18 @@ let compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_ Typ.Procname.Set.empty +let compute_affected_proc_names_clang ~clang_range_map ~source_file ~changed_lines_map = + let fname = SourceFile.to_rel_path source_file in + match String.Map.find changed_lines_map fname with + | Some changed_lines -> + Typ.Procname.Map.fold + (fun _ (range, clang_proc) acc -> + if is_file_in_changed_lines fname changed_lines range then clang_proc :: acc else acc ) + clang_range_map [] + | None -> + [] + + let emit_relevant_methods relevant_methods = let cleaned_methods = List.dedup_and_sort ~compare:String.compare @@ -160,20 +185,14 @@ let compute_and_emit_relevant_methods ~clang_range_map ~source_file = (* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) -let test_to_run ?clang_range_map ?source_file () = +let java_test_to_run () = let test_samples_file = Config.profiler_samples in let code_graph_file = Config.method_decls_info in let changed_lines_file = Config.modified_lines in let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in - let affected_methods = - match (clang_range_map, source_file) with - | Some clang_range_map, Some source_file -> - compute_affected_methods_clang ~source_file ~clang_range_map ~changed_lines_map - | _ (* Java case *) -> - let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in - compute_affected_methods_java changed_lines_map method_range - in - let profiler_samples = TestSample.read_test_sample test_samples_file in + let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in + let affected_methods = compute_affected_methods_java changed_lines_map method_range in + let profiler_samples = TestSample.read_java_test_sample test_samples_file in if Typ.Procname.Set.is_empty affected_methods then [] else List.fold profiler_samples ~init:[] ~f:(fun acc (label, profiler_samples) -> @@ -181,6 +200,38 @@ let test_to_run ?clang_range_map ?source_file () = if Typ.Procname.Set.is_empty intersection then acc else label :: acc ) +let match_profiler_samples_affected_methods native_symbols affected_methods = + let match_samples_method affected_method = + let match_sample_method affected_method native_symbol = + match affected_method with + | Some (ClangProc.CFunction {name}) -> + String.equal name native_symbol.Clang_profiler_samples_t.name + | _ -> + false + (* TODO: deal with mangled names, other method kinds *) + in + List.exists ~f:(match_sample_method affected_method) native_symbols + in + List.exists ~f:match_samples_method affected_methods + + +(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) +let clang_test_to_run ~clang_range_map ~source_file () = + let test_samples_file = Config.profiler_samples in + let changed_lines_file = Config.modified_lines in + let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in + let affected_methods = + compute_affected_proc_names_clang ~source_file ~clang_range_map ~changed_lines_map + in + let profiler_samples = TestSample.read_clang_test_sample test_samples_file in + if List.is_empty affected_methods then [] + else + List.fold profiler_samples ~init:[] + ~f:(fun acc ({test; native_symbols} : Clang_profiler_samples_t.profiler_sample) -> + if match_profiler_samples_affected_methods native_symbols affected_methods then test :: acc + else acc ) + + let emit_tests_to_run relevant_tests = let json = `List (List.map ~f:(fun t -> `String t) relevant_tests) in let outpath = Config.results_dir ^/ Config.test_determinator_output in @@ -188,5 +239,11 @@ let emit_tests_to_run relevant_tests = let compute_and_emit_test_to_run ?clang_range_map ?source_file () = - let relevant_tests = test_to_run ?clang_range_map ?source_file () in + let relevant_tests = + match (clang_range_map, source_file) with + | Some clang_range_map, Some source_file -> + clang_test_to_run ~clang_range_map ~source_file () + | _ -> + java_test_to_run () + in emit_tests_to_run relevant_tests diff --git a/infer/src/test_determinator/testDeterminator.mli b/infer/src/test_determinator/testDeterminator.mli index 00e2c7f41..be1f7a3af 100644 --- a/infer/src/test_determinator/testDeterminator.mli +++ b/infer/src/test_determinator/testDeterminator.mli @@ -8,10 +8,12 @@ open! IStd val compute_and_emit_test_to_run : - ?clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t + ?clang_range_map:((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t -> ?source_file:SourceFile.t -> unit -> unit val compute_and_emit_relevant_methods : - clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t -> source_file:SourceFile.t -> unit + clang_range_map:((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t + -> source_file:SourceFile.t + -> unit