[test determinator] Parse the clang profiler samples and matches c functions

Summary:
- Adds ATD file to parse the clang profiler samples
- Procnames don't help us here because we want to use mangled names, not the version of names that Infer needs, so passing in the RangeMap also ClangProc that just include the names and mangled names.
- First matching of c functions to have something in place to add a test, matching further method kinds to be done in next diff.

Reviewed By: skcho

Differential Revision: D17877071

fbshipit-source-id: b31d651a7

infer/src/Makefile

@@ -19,7 +19,7 @@ INFER_MAIN = infer
 
 #### Checkers declarations ####
 
-INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples
+INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples atd/clang_profiler_samples
 INFER_ATDGEN_TYPES = j t
 INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd)
 INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli)
@@ -305,7 +305,7 @@ clean:
 	  $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN)
 	$(REMOVE) $(BIN_DIR)/llvm_sil
 	$(REMOVE) $(INFER_CREATE_TRACEVIEW_LINKS_BIN)
-	$(REMOVE) atd/*_{j,t,v}.ml{,i} atd/clang_*
+	$(REMOVE) atd/*_{j,t,v}.ml{,i} atd/clang_ast_*
 	$(REMOVE) mod_dep.dot
 	$(REMOVE) mod_dep.pdf
 

infer/src/atd/clang_profiler_samples.atd

@@ -0,0 +1,18 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+type native_symbol = {
+    name : string;
+    ?mangled_name : string option;
+}
+
+type profiler_sample = {
+  test : string;
+  native_symbols : native_symbol list;
+}
+
+type profiler_samples = profiler_sample list

infer/src/base/ClangProc.ml

@@ -0,0 +1,14 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+type t =
+  | CFunction of {name: string; mangled_name: string option}
+  | CppMethod of {mangled_name: string}
+  | ObjcMethod of {mangled_name: string}
+  | ObjcBlock of {mangled_name: string}

infer/src/base/ClangProc.mli

@@ -0,0 +1,14 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+type t =
+  | CFunction of {name: string; mangled_name: string option}
+  | CppMethod of {mangled_name: string}
+  | ObjcMethod of {mangled_name: string}
+  | ObjcBlock of {mangled_name: string}

infer/src/clang/astToRangeMap.ml

@@ -6,6 +6,36 @@
  *)
 open! IStd
 
+(* Builds a clang procedure, following the format required to match with profiler samples: 
+C Functions: name, mangled name optional 
+ObjC Methods: mangled_name  
+ObjC Blocks: mangled_name 
+C++ methods: mangled_name (For us mangled name is optional, but if it is not there then we can't match the method) *)
+let clang_proc_of_decl decl =
+  let open Clang_ast_t in
+  match decl with
+  | ObjCMethodDecl (_, _, omdi) ->
+      Some (ClangProc.ObjcMethod {mangled_name= omdi.Clang_ast_t.omdi_mangled_name})
+  | BlockDecl (_, bdi) ->
+      Some (ClangProc.ObjcBlock {mangled_name= bdi.Clang_ast_t.bdi_mangled_name})
+  | FunctionDecl (_, named_decl_info, _, fdi) ->
+      Some
+        (ClangProc.CFunction
+           { name= named_decl_info.Clang_ast_t.ni_name
+           ; mangled_name= fdi.Clang_ast_t.fdi_mangled_name })
+  | CXXConversionDecl (_, _, _, fdi, _)
+  | CXXMethodDecl (_, _, _, fdi, _)
+  | CXXConstructorDecl (_, _, _, fdi, _)
+  | CXXDestructorDecl (_, _, _, fdi, _) -> (
+    match fdi.Clang_ast_t.fdi_mangled_name with
+    | Some mangled_name ->
+        Some (ClangProc.CppMethod {mangled_name})
+    | None ->
+        None )
+  | _ ->
+      None
+
+
 let process_ast ast default_source_file =
   let open Clang_ast_t in
   let rec extract_location ast_range decl =
@@ -15,9 +45,12 @@ let process_ast ast default_source_file =
     | FunctionDecl (di, _, _, _)
     | CXXMethodDecl (di, _, _, _, _)
     | CXXConstructorDecl (di, _, _, _, _)
-    | CXXDestructorDecl (di, _, _, _, _) ->
+    | CXXDestructorDecl (di, _, _, _, _)
+    | BlockDecl (di, _) ->
         let range = CLocation.location_of_decl_info default_source_file di in
-        Typ.Procname.Map.add (CType_decl.CProcname.from_decl decl) range ast_range
+        let procname = CType_decl.CProcname.from_decl decl in
+        let clang_proc = clang_proc_of_decl decl in
+        Typ.Procname.Map.add procname (range, clang_proc) ast_range
     | _ -> (
       match Clang_ast_proj.get_decl_context_tuple decl with
       | Some (decls, _) ->

infer/src/clang/astToRangeMap.mli

@@ -6,4 +6,7 @@
  *)
 open! IStd
 
-val process_ast : Clang_ast_t.decl -> SourceFile.t -> (Location.t * Location.t) Typ.Procname.Map.t
+val process_ast :
+     Clang_ast_t.decl
+  -> SourceFile.t
+  -> ((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t

infer/src/test_determinator/testDeterminator.ml

@@ -49,7 +49,7 @@ module MethodRangeMap = struct
                 let key =
                   JProcname.create_procname ~use_signature ~classname ~methodname ~signature
                 in
-                Typ.Procname.Map.add key range acc
+                Typ.Procname.Map.add key (range, ()) acc
             | None ->
                 acc )
     | _ ->
@@ -92,7 +92,7 @@ let pp_profiler_sample_set fmt s =
 
 
 module TestSample = struct
-  let read_test_sample test_samples_file_opt =
+  let read_java_test_sample test_samples_file_opt =
     match test_samples_file_opt with
     | Some test_samples_file ->
         L.(debug TestDeterminator Medium)
@@ -102,6 +102,15 @@ module TestSample = struct
         L.die UserError "Missing profiler samples argument"
 
 
+  let read_clang_test_sample test_samples_file_opt =
+    match test_samples_file_opt with
+    | Some test_samples_file ->
+        Atdgen_runtime.Util.Json.from_file Clang_profiler_samples_j.read_profiler_samples
+          test_samples_file
+    | None ->
+        L.die UserError "Missing profiler samples argument"
+
+
   [@@@warning "-32"]
 
   let pp_map fmt labeled_test_samples =
@@ -112,14 +121,18 @@ end
 
 let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line
 
+let is_file_in_changed_lines file_changed_lines changed_lines range =
+  let l1, _ = range in
+  let method_file = SourceFile.to_string l1.Location.file in
+  String.equal method_file file_changed_lines
+  && List.exists ~f:(fun l -> in_range l range) changed_lines
+
+
 let affected_methods method_range_map file_changed_lines changed_lines =
   Typ.Procname.Map.fold
-    (fun key ((l1, _) as range) acc ->
-      let method_file = SourceFile.to_string l1.Location.file in
-      if
-        String.equal method_file file_changed_lines
-        && List.exists ~f:(fun l -> in_range l range) changed_lines
-      then Typ.Procname.Set.add key acc
+    (fun key (range, _) acc ->
+      if is_file_in_changed_lines file_changed_lines changed_lines range then
+        Typ.Procname.Set.add key acc
       else acc )
     method_range_map Typ.Procname.Set.empty
 
@@ -140,6 +153,18 @@ let compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_
       Typ.Procname.Set.empty
 
 
+let compute_affected_proc_names_clang ~clang_range_map ~source_file ~changed_lines_map =
+  let fname = SourceFile.to_rel_path source_file in
+  match String.Map.find changed_lines_map fname with
+  | Some changed_lines ->
+      Typ.Procname.Map.fold
+        (fun _ (range, clang_proc) acc ->
+          if is_file_in_changed_lines fname changed_lines range then clang_proc :: acc else acc )
+        clang_range_map []
+  | None ->
+      []
+
+
 let emit_relevant_methods relevant_methods =
   let cleaned_methods =
     List.dedup_and_sort ~compare:String.compare
@@ -160,20 +185,14 @@ let compute_and_emit_relevant_methods ~clang_range_map ~source_file =
 
 
 (* test_to_run = { n | Affected_Method /\ ts_n != 0 } *)
-let test_to_run ?clang_range_map ?source_file () =
+let java_test_to_run () =
   let test_samples_file = Config.profiler_samples in
   let code_graph_file = Config.method_decls_info in
   let changed_lines_file = Config.modified_lines in
   let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
-  let affected_methods =
-    match (clang_range_map, source_file) with
-    | Some clang_range_map, Some source_file ->
-        compute_affected_methods_clang ~source_file ~clang_range_map ~changed_lines_map
-    | _ (* Java case *) ->
-        let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in
-        compute_affected_methods_java changed_lines_map method_range
-  in
-  let profiler_samples = TestSample.read_test_sample test_samples_file in
+  let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in
+  let affected_methods = compute_affected_methods_java changed_lines_map method_range in
+  let profiler_samples = TestSample.read_java_test_sample test_samples_file in
   if Typ.Procname.Set.is_empty affected_methods then []
   else
     List.fold profiler_samples ~init:[] ~f:(fun acc (label, profiler_samples) ->
@@ -181,6 +200,38 @@ let test_to_run ?clang_range_map ?source_file () =
         if Typ.Procname.Set.is_empty intersection then acc else label :: acc )
 
 
+let match_profiler_samples_affected_methods native_symbols affected_methods =
+  let match_samples_method affected_method =
+    let match_sample_method affected_method native_symbol =
+      match affected_method with
+      | Some (ClangProc.CFunction {name}) ->
+          String.equal name native_symbol.Clang_profiler_samples_t.name
+      | _ ->
+          false
+      (* TODO: deal with mangled names, other method kinds *)
+    in
+    List.exists ~f:(match_sample_method affected_method) native_symbols
+  in
+  List.exists ~f:match_samples_method affected_methods
+
+
+(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *)
+let clang_test_to_run ~clang_range_map ~source_file () =
+  let test_samples_file = Config.profiler_samples in
+  let changed_lines_file = Config.modified_lines in
+  let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
+  let affected_methods =
+    compute_affected_proc_names_clang ~source_file ~clang_range_map ~changed_lines_map
+  in
+  let profiler_samples = TestSample.read_clang_test_sample test_samples_file in
+  if List.is_empty affected_methods then []
+  else
+    List.fold profiler_samples ~init:[]
+      ~f:(fun acc ({test; native_symbols} : Clang_profiler_samples_t.profiler_sample) ->
+        if match_profiler_samples_affected_methods native_symbols affected_methods then test :: acc
+        else acc )
+
+
 let emit_tests_to_run relevant_tests =
   let json = `List (List.map ~f:(fun t -> `String t) relevant_tests) in
   let outpath = Config.results_dir ^/ Config.test_determinator_output in
@@ -188,5 +239,11 @@ let emit_tests_to_run relevant_tests =
 
 
 let compute_and_emit_test_to_run ?clang_range_map ?source_file () =
-  let relevant_tests = test_to_run ?clang_range_map ?source_file () in
+  let relevant_tests =
+    match (clang_range_map, source_file) with
+    | Some clang_range_map, Some source_file ->
+        clang_test_to_run ~clang_range_map ~source_file ()
+    | _ ->
+        java_test_to_run ()
+  in
   emit_tests_to_run relevant_tests

infer/src/test_determinator/testDeterminator.mli

@@ -8,10 +8,12 @@
 open! IStd
 
 val compute_and_emit_test_to_run :
-     ?clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t
+     ?clang_range_map:((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t
   -> ?source_file:SourceFile.t
   -> unit
   -> unit
 
 val compute_and_emit_relevant_methods :
-  clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t -> source_file:SourceFile.t -> unit
+     clang_range_map:((Location.t * Location.t) * ClangProc.t option) Typ.Procname.Map.t
+  -> source_file:SourceFile.t
+  -> unit