[test determinator] Refactor the code to make it more functional

Reviewed By: ngorogiannis

Differential Revision: D17479742

fbshipit-source-id: 1603374a0
master
Dulma Churchill 5 years ago committed by Facebook Github Bot
parent 16819fa1a4
commit b76ab1f8b9

@ -99,14 +99,11 @@ let run_clang_frontend ast_source =
L.(debug Capture Medium)
"Start %s of AST from %a@\n" Config.clang_frontend_action_string pp_ast_filename ast_source ;
if Config.linters then AL.do_frontend_checks trans_unit_ctx ast_decl ;
if Config.export_changed_functions then (
( if Config.export_changed_functions then
let source_file = trans_unit_ctx.CFrontend_config.source_file in
let process_ast_fn =
AstToRangeMap.process_ast trans_unit_ctx ast_decl (Tenv.create ()) source_file
in
TestDeterminator.test_to_run_clang source_file ~process_ast_fn
~changed_lines_file:Config.modified_lines ~test_samples_file:None ;
TestDeterminator.emit_relevant_methods () ) ;
let clang_range_map = AstToRangeMap.process_ast ast_decl source_file in
TestDeterminator.compute_and_emit_relevant_methods ~clang_range_map ~source_file
~changed_lines_file:Config.modified_lines ) ;
if Config.capture then CFrontend.do_source_file trans_unit_ctx ast_decl ;
L.(debug Capture Medium)
"End %s of AST file %a... OK!@\n" Config.clang_frontend_action_string pp_ast_filename

@ -5,21 +5,10 @@
* LICENSE file in the root directory of this source tree.
*)
open! IStd
module F = Format
module L = Logging
let process_ast trans_unit_ctx ast tenv default_source_file f =
let process_ast ast default_source_file =
let open Clang_ast_t in
let mk_key decl = CType_decl.CProcname.from_decl ~tenv decl in
let call_f decl range =
CFrontend_errors.protect trans_unit_ctx
~recover:(fun () -> ())
~pp_context:(fun f () ->
F.fprintf f "%a: processing %s" SourceFile.pp default_source_file
(Clang_ast_j.string_of_decl decl) )
~f:(fun () -> f (mk_key decl) range)
in
let rec extract_location decl =
let rec extract_location ast_range decl =
match decl with
| ObjCMethodDecl (di, _, _)
| CXXConversionDecl (di, _, _, _, _)
@ -28,27 +17,23 @@ let process_ast trans_unit_ctx ast tenv default_source_file f =
| CXXConstructorDecl (di, _, _, _, _)
| CXXDestructorDecl (di, _, _, _, _) ->
let range = CLocation.location_of_decl_info default_source_file di in
call_f decl range
Typ.Procname.Map.add (CType_decl.CProcname.from_decl decl) range ast_range
| _ -> (
match Clang_ast_proj.get_decl_context_tuple decl with
| Some (decls, _) ->
List.iter decls ~f:extract_location
List.fold decls ~f:extract_location ~init:ast_range
| None ->
L.(debug Capture Verbose)
"@\n Found %s.., skipping"
(Clang_ast_proj.get_decl_kind_string decl) )
ast_range )
in
match ast with
| TranslationUnitDecl (_, decl_list, _, _) ->
CFrontend_config.global_translation_unit_decls := decl_list ;
L.(debug Capture Verbose) "@\n Processing AST...@\n" ;
List.iter decl_list ~f:(fun d ->
let info = Clang_ast_proj.get_decl_tuple d in
List.fold decl_list ~init:Typ.Procname.Map.empty ~f:(fun map decl ->
let info = Clang_ast_proj.get_decl_tuple decl in
let source_range = info.di_source_range in
if
CLocation.should_translate_lib default_source_file source_range `DeclTraversal
~translate_when_used:true
then extract_location d ) ;
L.(debug Capture Verbose) "@\n Finished processing AST.@\n"
then extract_location map decl
else map )
| _ ->
assert false

@ -6,10 +6,4 @@
*)
open! IStd
val process_ast :
CFrontend_config.translation_unit_context
-> Clang_ast_t.decl
-> Tenv.t
-> SourceFile.t
-> (Typ.Procname.t -> Location.t * Location.t -> unit)
-> unit
val process_ast : Clang_ast_t.decl -> SourceFile.t -> (Location.t * Location.t) Typ.Procname.Map.t

@ -147,10 +147,10 @@ let () =
if Config.debug_mode && CLOpt.is_originator then (
L.progress "Logs in %s@." (Config.results_dir ^/ Config.log_file) ;
L.progress "Execution ID %Ld@." Config.execution_id ) ;
( if Config.test_determinator then (
TestDeterminator.test_to_run_java ~changed_lines_file:Config.modified_lines
~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info ;
TestDeterminator.emit_tests_to_run () )
( if Config.test_determinator then
TestDeterminator.compute_and_emit_test_to_run ~changed_lines_file:Config.modified_lines
~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info
?clang_range_map:None ?source_file:None
else
match Config.command with
| Analyze ->

@ -14,29 +14,15 @@ module YB = Yojson.Basic
(* a flag used to make the method search signature sensitive *)
let use_method_signature = false
module RangeMap = Caml.Map.Make (struct
type t = Typ.Procname.t
let compare = Typ.Procname.compare
end)
let initialized_test_determinator = ref false
let is_test_determinator_init () = !initialized_test_determinator
module RangeMap = Typ.Procname.Map
module MethodRangeMap = struct
type t = (Location.t * Location.t) RangeMap.t
let map : t ref = ref RangeMap.empty
let method_range_map () = !map
let split_class_method_name qualified_method_name =
String.rsplit2_exn qualified_method_name ~on:'.'
let create_java_method_range_map code_graph_file' =
match code_graph_file' with
let create_java_method_range_map code_graph_file_opt =
match code_graph_file_opt with
| Some code_graph_file ->
let open Java_method_decl_j in
let json_string =
@ -48,112 +34,85 @@ module MethodRangeMap = struct
L.die UserError "Could not read file %s" code_graph_file
in
let method_decls = java_method_decls_of_string json_string in
let map' =
List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl ->
let start_location =
{ Location.line= decl.start_line
; col= -1
; file= SourceFile.create ~warn_on_error:false decl.source_file }
in
let end_location =
{ Location.line= decl.end_line
; col= -1
; file= SourceFile.create ~warn_on_error:false decl.source_file }
in
let range = (start_location, end_location) in
let classname, methodname = split_class_method_name decl.method_name in
match decl.signature with
| Some signature ->
let signature =
if use_method_signature then signature
else
(* When we should not use the signature we use 'void ()' *)
JPS.JNI.void_method_with_no_arguments
in
let key = JPS.create_procname ~classname ~methodname ~signature in
RangeMap.add key range acc
| None ->
acc )
in
map := map'
List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl ->
let start_location =
{ Location.line= decl.start_line
; col= -1
; file= SourceFile.create ~warn_on_error:false decl.source_file }
in
let end_location =
{ Location.line= decl.end_line
; col= -1
; file= SourceFile.create ~warn_on_error:false decl.source_file }
in
let range = (start_location, end_location) in
let classname, methodname = split_class_method_name decl.method_name in
match decl.signature with
| Some signature ->
let signature =
if use_method_signature then signature
else
(* When we should not use the signature we use 'void ()' *)
JPS.JNI.void_method_with_no_arguments
in
let key = JPS.create_procname ~classname ~methodname ~signature in
RangeMap.add key range acc
| None ->
acc )
| _ ->
L.die UserError "Missing method declaration info argument"
let create_clang_method_range_map process_ast_fn =
process_ast_fn (fun key range -> map := RangeMap.add key range !map)
let pp_map fmt () =
let pp_map' fmt m =
RangeMap.iter
(fun key range ->
F.fprintf fmt "@\n %a --> (%a,%a)" Typ.Procname.pp key Location.pp (fst range)
Location.pp (snd range) )
m
in
F.fprintf fmt "@\n--- Method Range Map ---%a@\n--- End Method Range Map --- @\n" pp_map' !map
end
module DiffLines = struct
(* This is a map
file name |--> {set of changed line }
*)
let map : int list String.Map.t ref = ref String.Map.empty
let changed_lines_map () = !map
(* Read the file containing info on changed lines and populate the map *)
let init_changed_lines_map changed_lines_file' =
match changed_lines_file' with
let create_changed_lines_map changed_lines_file_opt =
match changed_lines_file_opt with
| Some changed_lines_file -> (
L.(debug TestDeterminator Medium)
"Initializing changed lines map from file '%s'..." changed_lines_file ;
match Utils.read_file changed_lines_file with
| Ok cl_list ->
let changed_lines =
List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item ->
let fname, cl = String.rsplit2_exn ~on:':' cl_item in
String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) )
in
L.(debug TestDeterminator Medium) "done@\n" ;
map := changed_lines
| Error _ ->
L.die UserError "Could not read file %s" changed_lines_file )
| _ ->
match Utils.read_file changed_lines_file with
| Ok cl_list ->
List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item ->
let fname, cl = String.rsplit2_exn ~on:':' cl_item in
String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) )
| Error _ ->
L.die UserError "Could not read file %s" changed_lines_file )
| None ->
L.die UserError "Missing modified lines argument"
let print_changed_lines () =
L.(debug TestDeterminator Quiet) "--- Changed Lines Map ---@\n" ;
String.Map.iteri !map ~f:(fun ~key ~data ->
L.(debug TestDeterminator Quiet)
"%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data )
[@@@warning "-32"]
let pp_changed_lines fmt map =
F.fprintf fmt "--- Changed Lines Map ---@\n" ;
String.Map.iteri map ~f:(fun ~key ~data ->
F.fprintf fmt "%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data )
end
[@@@warning "-32"]
let pp_profiler_sample_set fmt s =
F.fprintf fmt " (set size = %i) " (JPS.ProfilerSample.cardinal s) ;
JPS.ProfilerSample.iter (fun m -> F.fprintf fmt "@\n <Method:> %a " Typ.Procname.pp m) s
module TestSample = struct
let labeled_test_samples = ref []
let test_sample () = !labeled_test_samples
let init_test_sample test_samples_file' =
match test_samples_file' with
let read_test_sample test_samples_file_opt =
match test_samples_file_opt with
| Some test_samples_file ->
L.(debug TestDeterminator Medium)
"Reading Profiler Samples File '%s'....@\n" test_samples_file ;
let ts = JPS.from_json_file test_samples_file ~use_signature:use_method_signature in
labeled_test_samples := ts
| _ ->
JPS.from_json_file test_samples_file ~use_signature:use_method_signature
| None ->
L.die UserError "Missing profiler samples argument"
let pp_map fmt () =
List.iter !labeled_test_samples ~f:(fun (label, profiler_samples) ->
[@@@warning "-32"]
let pp_map fmt labeled_test_samples =
List.iter labeled_test_samples ~f:(fun (label, profiler_samples) ->
F.fprintf fmt "=== Samples for %s ===@\n%a@\n=== End Samples for %s ===@\n" label
pp_profiler_sample_set profiler_samples label )
end
@ -161,147 +120,81 @@ end
let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line
let affected_methods method_range_map file_changed_lines changed_lines =
L.(debug TestDeterminator Medium) "Looking for affected methods in file '%s' " file_changed_lines ;
RangeMap.fold
(fun key ((l1, _) as range) acc ->
let method_file = SourceFile.to_string l1.Location.file in
if
String.equal method_file file_changed_lines
&& List.exists ~f:(fun l -> in_range l range) changed_lines
then (
L.(debug TestDeterminator Medium)
"Adding '%a' in affected methods...@\n" Typ.Procname.pp key ;
JPS.ProfilerSample.add key acc )
then JPS.ProfilerSample.add key acc
else acc )
method_range_map JPS.ProfilerSample.empty
let compute_affected_methods_java changed_lines_map method_range_map =
let affected_methods =
String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty
~f:(fun ~key:file_changed_lines ~data acc ->
let am = affected_methods method_range_map file_changed_lines data in
JPS.ProfilerSample.union am acc )
in
L.(debug TestDeterminator Quiet)
"== Affected Methods ==%a@\n== End Affected Methods ==@\n" pp_profiler_sample_set
affected_methods ;
affected_methods
String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty
~f:(fun ~key:file_changed_lines ~data acc ->
let am = affected_methods method_range_map file_changed_lines data in
JPS.ProfilerSample.union am acc )
let compute_affected_methods_clang source_file changed_lines_map method_range_map =
let compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map =
let fname = SourceFile.to_rel_path source_file in
L.(debug TestDeterminator Medium) "Looking for file %s in changed-line map..." fname ;
match String.Map.find changed_lines_map fname with
| Some changed_lines ->
L.(debug TestDeterminator Medium) "found!@\n" ;
let affected_methods = affected_methods method_range_map fname changed_lines in
L.(debug TestDeterminator Medium)
"@\n@\n== Resulting Affected Methods ==%a@\n== End Affected Methods ==@\n\n"
pp_profiler_sample_set affected_methods ;
affected_methods
affected_methods clang_range_map fname changed_lines
| None ->
L.(debug TestDeterminator Medium)
"%s not found in changed-line map. Nothing else to do for it.@\n" fname ;
JPS.ProfilerSample.empty
let relevant_tests = ref []
(* Methods modified in a diff *)
let relevant_methods = ref JPS.ProfilerSample.empty
let _get_relevant_test_to_run () = !relevant_tests
let emit_tests_to_run () =
let json = `List (List.map ~f:(fun t -> `String t) !relevant_tests) in
let outpath = Config.results_dir ^/ Config.test_determinator_output in
YB.to_file outpath json ;
L.(debug TestDeterminator Medium)
"Tests to run: [%a]@\n"
(Pp.seq ~sep:", " F.pp_print_string)
!relevant_tests
let emit_relevant_methods () =
let emit_relevant_methods relevant_methods =
let cleaned_methods =
List.dedup_and_sort ~compare:String.compare
(List.map (JPS.ProfilerSample.elements !relevant_methods) ~f:Typ.Procname.to_string)
(List.map (JPS.ProfilerSample.elements relevant_methods) ~f:Typ.Procname.to_string)
in
let json = `List (List.map ~f:(fun t -> `String t) cleaned_methods) in
let outpath = Config.results_dir ^/ Config.export_changed_functions_output in
YB.to_file outpath json ;
L.(debug TestDeterminator Medium)
"Methods modified in this Diff: %a@\n"
(Pp.seq ~sep:", " F.pp_print_string)
cleaned_methods
let init_clang process_ast_fn changed_lines_file test_samples_file =
DiffLines.init_changed_lines_map changed_lines_file ;
DiffLines.print_changed_lines () ;
MethodRangeMap.create_clang_method_range_map process_ast_fn ;
L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ;
match test_samples_file with
| Some _ ->
TestSample.init_test_sample test_samples_file
| _ ->
() ;
L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ;
initialized_test_determinator := true
let init_java changed_lines_file test_samples_file code_graph_file =
DiffLines.init_changed_lines_map changed_lines_file ;
DiffLines.print_changed_lines () ;
MethodRangeMap.create_java_method_range_map code_graph_file ;
L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ;
TestSample.init_test_sample test_samples_file ;
L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ;
initialized_test_determinator := true
YB.to_file outpath json
(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *)
let test_to_run_clang source_file ~process_ast_fn ~changed_lines_file ~test_samples_file =
L.(debug TestDeterminator Quiet)
"****** Start Test Determinator for %s *****@\n"
(SourceFile.to_string source_file) ;
if is_test_determinator_init () then ()
else init_clang process_ast_fn changed_lines_file test_samples_file ;
let affected_methods =
compute_affected_methods_clang source_file (DiffLines.changed_lines_map ())
(MethodRangeMap.method_range_map ())
let compute_and_emit_relevant_methods ~clang_range_map ~source_file ~changed_lines_file =
let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
let relevant_methods =
compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map
in
relevant_methods := JPS.ProfilerSample.union affected_methods !relevant_methods ;
let test_to_run =
if JPS.ProfilerSample.is_empty affected_methods then []
else
List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) ->
let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in
if JPS.ProfilerSample.is_empty intersection then acc else label :: acc )
in
relevant_tests := List.append test_to_run !relevant_tests
emit_relevant_methods relevant_methods
let test_to_run_java ~changed_lines_file ~test_samples_file ~code_graph_file =
L.(debug TestDeterminator Quiet) "***** Start Java Test Determinator *****@\n" ;
if is_test_determinator_init () then ()
else init_java changed_lines_file test_samples_file code_graph_file ;
(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *)
let test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file
~test_samples_file =
let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
let affected_methods =
compute_affected_methods_java (DiffLines.changed_lines_map ())
(MethodRangeMap.method_range_map ())
match (clang_range_map, source_file) with
| Some clang_range_map, Some source_file ->
compute_affected_methods_clang ~source_file ~clang_range_map ~changed_lines_map
| _ (* Java case *) ->
let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in
compute_affected_methods_java changed_lines_map method_range
in
let test_to_run =
if JPS.ProfilerSample.is_empty affected_methods then []
else
List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) ->
let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in
if JPS.ProfilerSample.is_empty intersection then acc
else (
L.(debug TestDeterminator Quiet)
"Choosing test '%s' because of [%a]@\n" label
(Pp.seq Typ.Procname.pp ~sep:", ")
(JPS.ProfilerSample.elements intersection) ;
label :: acc ) )
let profiler_samples = TestSample.read_test_sample test_samples_file in
if JPS.ProfilerSample.is_empty affected_methods then []
else
List.fold profiler_samples ~init:[] ~f:(fun acc (label, profiler_samples) ->
let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in
if JPS.ProfilerSample.is_empty intersection then acc else label :: acc )
let emit_tests_to_run relevant_tests =
let json = `List (List.map ~f:(fun t -> `String t) relevant_tests) in
let outpath = Config.results_dir ^/ Config.test_determinator_output in
YB.to_file outpath json
let compute_and_emit_test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file
~test_samples_file =
let relevant_tests =
test_to_run ?clang_range_map ~code_graph_file ?source_file ~changed_lines_file
~test_samples_file
in
relevant_tests := List.append test_to_run !relevant_tests
emit_tests_to_run relevant_tests

@ -7,21 +7,16 @@
open! IStd
val test_to_run_java :
changed_lines_file:string option
-> test_samples_file:string option
val compute_and_emit_test_to_run :
?clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t
-> ?source_file:SourceFile.t
-> code_graph_file:string option
-> unit
val test_to_run_clang :
SourceFile.t
-> process_ast_fn:((Typ.Procname.t -> Location.t * Location.t -> unit) -> unit)
-> changed_lines_file:string option
-> test_samples_file:string option
-> unit
val emit_tests_to_run : unit -> unit
val emit_relevant_methods : unit -> unit
val _get_relevant_test_to_run : unit -> string list
val compute_and_emit_relevant_methods :
clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t
-> source_file:SourceFile.t
-> changed_lines_file:string option
-> unit

Loading…
Cancel
Save