[test determinator] Refactor the code to make it more functional

Reviewed By: ngorogiannis

Differential Revision: D17479742

fbshipit-source-id: 1603374a0
master
Dulma Churchill 5 years ago committed by Facebook Github Bot
parent 16819fa1a4
commit b76ab1f8b9

@ -99,14 +99,11 @@ let run_clang_frontend ast_source =
L.(debug Capture Medium) L.(debug Capture Medium)
"Start %s of AST from %a@\n" Config.clang_frontend_action_string pp_ast_filename ast_source ; "Start %s of AST from %a@\n" Config.clang_frontend_action_string pp_ast_filename ast_source ;
if Config.linters then AL.do_frontend_checks trans_unit_ctx ast_decl ; if Config.linters then AL.do_frontend_checks trans_unit_ctx ast_decl ;
if Config.export_changed_functions then ( ( if Config.export_changed_functions then
let source_file = trans_unit_ctx.CFrontend_config.source_file in let source_file = trans_unit_ctx.CFrontend_config.source_file in
let process_ast_fn = let clang_range_map = AstToRangeMap.process_ast ast_decl source_file in
AstToRangeMap.process_ast trans_unit_ctx ast_decl (Tenv.create ()) source_file TestDeterminator.compute_and_emit_relevant_methods ~clang_range_map ~source_file
in ~changed_lines_file:Config.modified_lines ) ;
TestDeterminator.test_to_run_clang source_file ~process_ast_fn
~changed_lines_file:Config.modified_lines ~test_samples_file:None ;
TestDeterminator.emit_relevant_methods () ) ;
if Config.capture then CFrontend.do_source_file trans_unit_ctx ast_decl ; if Config.capture then CFrontend.do_source_file trans_unit_ctx ast_decl ;
L.(debug Capture Medium) L.(debug Capture Medium)
"End %s of AST file %a... OK!@\n" Config.clang_frontend_action_string pp_ast_filename "End %s of AST file %a... OK!@\n" Config.clang_frontend_action_string pp_ast_filename

@ -5,21 +5,10 @@
* LICENSE file in the root directory of this source tree. * LICENSE file in the root directory of this source tree.
*) *)
open! IStd open! IStd
module F = Format
module L = Logging
let process_ast trans_unit_ctx ast tenv default_source_file f = let process_ast ast default_source_file =
let open Clang_ast_t in let open Clang_ast_t in
let mk_key decl = CType_decl.CProcname.from_decl ~tenv decl in let rec extract_location ast_range decl =
let call_f decl range =
CFrontend_errors.protect trans_unit_ctx
~recover:(fun () -> ())
~pp_context:(fun f () ->
F.fprintf f "%a: processing %s" SourceFile.pp default_source_file
(Clang_ast_j.string_of_decl decl) )
~f:(fun () -> f (mk_key decl) range)
in
let rec extract_location decl =
match decl with match decl with
| ObjCMethodDecl (di, _, _) | ObjCMethodDecl (di, _, _)
| CXXConversionDecl (di, _, _, _, _) | CXXConversionDecl (di, _, _, _, _)
@ -28,27 +17,23 @@ let process_ast trans_unit_ctx ast tenv default_source_file f =
| CXXConstructorDecl (di, _, _, _, _) | CXXConstructorDecl (di, _, _, _, _)
| CXXDestructorDecl (di, _, _, _, _) -> | CXXDestructorDecl (di, _, _, _, _) ->
let range = CLocation.location_of_decl_info default_source_file di in let range = CLocation.location_of_decl_info default_source_file di in
call_f decl range Typ.Procname.Map.add (CType_decl.CProcname.from_decl decl) range ast_range
| _ -> ( | _ -> (
match Clang_ast_proj.get_decl_context_tuple decl with match Clang_ast_proj.get_decl_context_tuple decl with
| Some (decls, _) -> | Some (decls, _) ->
List.iter decls ~f:extract_location List.fold decls ~f:extract_location ~init:ast_range
| None -> | None ->
L.(debug Capture Verbose) ast_range )
"@\n Found %s.., skipping"
(Clang_ast_proj.get_decl_kind_string decl) )
in in
match ast with match ast with
| TranslationUnitDecl (_, decl_list, _, _) -> | TranslationUnitDecl (_, decl_list, _, _) ->
CFrontend_config.global_translation_unit_decls := decl_list ; List.fold decl_list ~init:Typ.Procname.Map.empty ~f:(fun map decl ->
L.(debug Capture Verbose) "@\n Processing AST...@\n" ; let info = Clang_ast_proj.get_decl_tuple decl in
List.iter decl_list ~f:(fun d ->
let info = Clang_ast_proj.get_decl_tuple d in
let source_range = info.di_source_range in let source_range = info.di_source_range in
if if
CLocation.should_translate_lib default_source_file source_range `DeclTraversal CLocation.should_translate_lib default_source_file source_range `DeclTraversal
~translate_when_used:true ~translate_when_used:true
then extract_location d ) ; then extract_location map decl
L.(debug Capture Verbose) "@\n Finished processing AST.@\n" else map )
| _ -> | _ ->
assert false assert false

@ -6,10 +6,4 @@
*) *)
open! IStd open! IStd
val process_ast : val process_ast : Clang_ast_t.decl -> SourceFile.t -> (Location.t * Location.t) Typ.Procname.Map.t
CFrontend_config.translation_unit_context
-> Clang_ast_t.decl
-> Tenv.t
-> SourceFile.t
-> (Typ.Procname.t -> Location.t * Location.t -> unit)
-> unit

@ -147,10 +147,10 @@ let () =
if Config.debug_mode && CLOpt.is_originator then ( if Config.debug_mode && CLOpt.is_originator then (
L.progress "Logs in %s@." (Config.results_dir ^/ Config.log_file) ; L.progress "Logs in %s@." (Config.results_dir ^/ Config.log_file) ;
L.progress "Execution ID %Ld@." Config.execution_id ) ; L.progress "Execution ID %Ld@." Config.execution_id ) ;
( if Config.test_determinator then ( ( if Config.test_determinator then
TestDeterminator.test_to_run_java ~changed_lines_file:Config.modified_lines TestDeterminator.compute_and_emit_test_to_run ~changed_lines_file:Config.modified_lines
~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info ; ~test_samples_file:Config.profiler_samples ~code_graph_file:Config.method_decls_info
TestDeterminator.emit_tests_to_run () ) ?clang_range_map:None ?source_file:None
else else
match Config.command with match Config.command with
| Analyze -> | Analyze ->

@ -14,29 +14,15 @@ module YB = Yojson.Basic
(* a flag used to make the method search signature sensitive *) (* a flag used to make the method search signature sensitive *)
let use_method_signature = false let use_method_signature = false
module RangeMap = Caml.Map.Make (struct module RangeMap = Typ.Procname.Map
type t = Typ.Procname.t
let compare = Typ.Procname.compare
end)
let initialized_test_determinator = ref false
let is_test_determinator_init () = !initialized_test_determinator
module MethodRangeMap = struct module MethodRangeMap = struct
type t = (Location.t * Location.t) RangeMap.t
let map : t ref = ref RangeMap.empty
let method_range_map () = !map
let split_class_method_name qualified_method_name = let split_class_method_name qualified_method_name =
String.rsplit2_exn qualified_method_name ~on:'.' String.rsplit2_exn qualified_method_name ~on:'.'
let create_java_method_range_map code_graph_file' = let create_java_method_range_map code_graph_file_opt =
match code_graph_file' with match code_graph_file_opt with
| Some code_graph_file -> | Some code_graph_file ->
let open Java_method_decl_j in let open Java_method_decl_j in
let json_string = let json_string =
@ -48,112 +34,85 @@ module MethodRangeMap = struct
L.die UserError "Could not read file %s" code_graph_file L.die UserError "Could not read file %s" code_graph_file
in in
let method_decls = java_method_decls_of_string json_string in let method_decls = java_method_decls_of_string json_string in
let map' = List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl ->
List.fold method_decls ~init:RangeMap.empty ~f:(fun acc decl -> let start_location =
let start_location = { Location.line= decl.start_line
{ Location.line= decl.start_line ; col= -1
; col= -1 ; file= SourceFile.create ~warn_on_error:false decl.source_file }
; file= SourceFile.create ~warn_on_error:false decl.source_file } in
in let end_location =
let end_location = { Location.line= decl.end_line
{ Location.line= decl.end_line ; col= -1
; col= -1 ; file= SourceFile.create ~warn_on_error:false decl.source_file }
; file= SourceFile.create ~warn_on_error:false decl.source_file } in
in let range = (start_location, end_location) in
let range = (start_location, end_location) in let classname, methodname = split_class_method_name decl.method_name in
let classname, methodname = split_class_method_name decl.method_name in match decl.signature with
match decl.signature with | Some signature ->
| Some signature -> let signature =
let signature = if use_method_signature then signature
if use_method_signature then signature else
else (* When we should not use the signature we use 'void ()' *)
(* When we should not use the signature we use 'void ()' *) JPS.JNI.void_method_with_no_arguments
JPS.JNI.void_method_with_no_arguments in
in let key = JPS.create_procname ~classname ~methodname ~signature in
let key = JPS.create_procname ~classname ~methodname ~signature in RangeMap.add key range acc
RangeMap.add key range acc | None ->
| None -> acc )
acc )
in
map := map'
| _ -> | _ ->
L.die UserError "Missing method declaration info argument" L.die UserError "Missing method declaration info argument"
let create_clang_method_range_map process_ast_fn =
process_ast_fn (fun key range -> map := RangeMap.add key range !map)
let pp_map fmt () =
let pp_map' fmt m =
RangeMap.iter
(fun key range ->
F.fprintf fmt "@\n %a --> (%a,%a)" Typ.Procname.pp key Location.pp (fst range)
Location.pp (snd range) )
m
in
F.fprintf fmt "@\n--- Method Range Map ---%a@\n--- End Method Range Map --- @\n" pp_map' !map
end end
module DiffLines = struct module DiffLines = struct
(* This is a map (* This is a map
file name |--> {set of changed line } file name |--> {set of changed line }
*) *)
let map : int list String.Map.t ref = ref String.Map.empty
let changed_lines_map () = !map
(* Read the file containing info on changed lines and populate the map *) (* Read the file containing info on changed lines and populate the map *)
let init_changed_lines_map changed_lines_file' = let create_changed_lines_map changed_lines_file_opt =
match changed_lines_file' with match changed_lines_file_opt with
| Some changed_lines_file -> ( | Some changed_lines_file -> (
L.(debug TestDeterminator Medium) match Utils.read_file changed_lines_file with
"Initializing changed lines map from file '%s'..." changed_lines_file ; | Ok cl_list ->
match Utils.read_file changed_lines_file with List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item ->
| Ok cl_list -> let fname, cl = String.rsplit2_exn ~on:':' cl_item in
let changed_lines = String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) )
List.fold cl_list ~init:String.Map.empty ~f:(fun acc cl_item -> | Error _ ->
let fname, cl = String.rsplit2_exn ~on:':' cl_item in L.die UserError "Could not read file %s" changed_lines_file )
String.Map.set acc ~key:fname ~data:(FileDiff.parse_unix_diff cl) ) | None ->
in
L.(debug TestDeterminator Medium) "done@\n" ;
map := changed_lines
| Error _ ->
L.die UserError "Could not read file %s" changed_lines_file )
| _ ->
L.die UserError "Missing modified lines argument" L.die UserError "Missing modified lines argument"
let print_changed_lines () = [@@@warning "-32"]
L.(debug TestDeterminator Quiet) "--- Changed Lines Map ---@\n" ;
String.Map.iteri !map ~f:(fun ~key ~data -> let pp_changed_lines fmt map =
L.(debug TestDeterminator Quiet) F.fprintf fmt "--- Changed Lines Map ---@\n" ;
"%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data ) String.Map.iteri map ~f:(fun ~key ~data ->
F.fprintf fmt "%s --> [%a]@\n" key (Pp.seq ~sep:", " F.pp_print_int) data )
end end
[@@@warning "-32"]
let pp_profiler_sample_set fmt s = let pp_profiler_sample_set fmt s =
F.fprintf fmt " (set size = %i) " (JPS.ProfilerSample.cardinal s) ; F.fprintf fmt " (set size = %i) " (JPS.ProfilerSample.cardinal s) ;
JPS.ProfilerSample.iter (fun m -> F.fprintf fmt "@\n <Method:> %a " Typ.Procname.pp m) s JPS.ProfilerSample.iter (fun m -> F.fprintf fmt "@\n <Method:> %a " Typ.Procname.pp m) s
module TestSample = struct module TestSample = struct
let labeled_test_samples = ref [] let read_test_sample test_samples_file_opt =
match test_samples_file_opt with
let test_sample () = !labeled_test_samples
let init_test_sample test_samples_file' =
match test_samples_file' with
| Some test_samples_file -> | Some test_samples_file ->
L.(debug TestDeterminator Medium) L.(debug TestDeterminator Medium)
"Reading Profiler Samples File '%s'....@\n" test_samples_file ; "Reading Profiler Samples File '%s'....@\n" test_samples_file ;
let ts = JPS.from_json_file test_samples_file ~use_signature:use_method_signature in JPS.from_json_file test_samples_file ~use_signature:use_method_signature
labeled_test_samples := ts | None ->
| _ ->
L.die UserError "Missing profiler samples argument" L.die UserError "Missing profiler samples argument"
let pp_map fmt () = [@@@warning "-32"]
List.iter !labeled_test_samples ~f:(fun (label, profiler_samples) ->
let pp_map fmt labeled_test_samples =
List.iter labeled_test_samples ~f:(fun (label, profiler_samples) ->
F.fprintf fmt "=== Samples for %s ===@\n%a@\n=== End Samples for %s ===@\n" label F.fprintf fmt "=== Samples for %s ===@\n%a@\n=== End Samples for %s ===@\n" label
pp_profiler_sample_set profiler_samples label ) pp_profiler_sample_set profiler_samples label )
end end
@ -161,147 +120,81 @@ end
let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line let in_range l range = l >= (fst range).Location.line && l <= (snd range).Location.line
let affected_methods method_range_map file_changed_lines changed_lines = let affected_methods method_range_map file_changed_lines changed_lines =
L.(debug TestDeterminator Medium) "Looking for affected methods in file '%s' " file_changed_lines ;
RangeMap.fold RangeMap.fold
(fun key ((l1, _) as range) acc -> (fun key ((l1, _) as range) acc ->
let method_file = SourceFile.to_string l1.Location.file in let method_file = SourceFile.to_string l1.Location.file in
if if
String.equal method_file file_changed_lines String.equal method_file file_changed_lines
&& List.exists ~f:(fun l -> in_range l range) changed_lines && List.exists ~f:(fun l -> in_range l range) changed_lines
then ( then JPS.ProfilerSample.add key acc
L.(debug TestDeterminator Medium)
"Adding '%a' in affected methods...@\n" Typ.Procname.pp key ;
JPS.ProfilerSample.add key acc )
else acc ) else acc )
method_range_map JPS.ProfilerSample.empty method_range_map JPS.ProfilerSample.empty
let compute_affected_methods_java changed_lines_map method_range_map = let compute_affected_methods_java changed_lines_map method_range_map =
let affected_methods = String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty
String.Map.fold changed_lines_map ~init:JPS.ProfilerSample.empty ~f:(fun ~key:file_changed_lines ~data acc ->
~f:(fun ~key:file_changed_lines ~data acc -> let am = affected_methods method_range_map file_changed_lines data in
let am = affected_methods method_range_map file_changed_lines data in JPS.ProfilerSample.union am acc )
JPS.ProfilerSample.union am acc )
in
L.(debug TestDeterminator Quiet)
"== Affected Methods ==%a@\n== End Affected Methods ==@\n" pp_profiler_sample_set
affected_methods ;
affected_methods
let compute_affected_methods_clang source_file changed_lines_map method_range_map = let compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map =
let fname = SourceFile.to_rel_path source_file in let fname = SourceFile.to_rel_path source_file in
L.(debug TestDeterminator Medium) "Looking for file %s in changed-line map..." fname ;
match String.Map.find changed_lines_map fname with match String.Map.find changed_lines_map fname with
| Some changed_lines -> | Some changed_lines ->
L.(debug TestDeterminator Medium) "found!@\n" ; affected_methods clang_range_map fname changed_lines
let affected_methods = affected_methods method_range_map fname changed_lines in
L.(debug TestDeterminator Medium)
"@\n@\n== Resulting Affected Methods ==%a@\n== End Affected Methods ==@\n\n"
pp_profiler_sample_set affected_methods ;
affected_methods
| None -> | None ->
L.(debug TestDeterminator Medium)
"%s not found in changed-line map. Nothing else to do for it.@\n" fname ;
JPS.ProfilerSample.empty JPS.ProfilerSample.empty
let relevant_tests = ref [] let emit_relevant_methods relevant_methods =
(* Methods modified in a diff *)
let relevant_methods = ref JPS.ProfilerSample.empty
let _get_relevant_test_to_run () = !relevant_tests
let emit_tests_to_run () =
let json = `List (List.map ~f:(fun t -> `String t) !relevant_tests) in
let outpath = Config.results_dir ^/ Config.test_determinator_output in
YB.to_file outpath json ;
L.(debug TestDeterminator Medium)
"Tests to run: [%a]@\n"
(Pp.seq ~sep:", " F.pp_print_string)
!relevant_tests
let emit_relevant_methods () =
let cleaned_methods = let cleaned_methods =
List.dedup_and_sort ~compare:String.compare List.dedup_and_sort ~compare:String.compare
(List.map (JPS.ProfilerSample.elements !relevant_methods) ~f:Typ.Procname.to_string) (List.map (JPS.ProfilerSample.elements relevant_methods) ~f:Typ.Procname.to_string)
in in
let json = `List (List.map ~f:(fun t -> `String t) cleaned_methods) in let json = `List (List.map ~f:(fun t -> `String t) cleaned_methods) in
let outpath = Config.results_dir ^/ Config.export_changed_functions_output in let outpath = Config.results_dir ^/ Config.export_changed_functions_output in
YB.to_file outpath json ; YB.to_file outpath json
L.(debug TestDeterminator Medium)
"Methods modified in this Diff: %a@\n"
(Pp.seq ~sep:", " F.pp_print_string)
cleaned_methods
let init_clang process_ast_fn changed_lines_file test_samples_file =
DiffLines.init_changed_lines_map changed_lines_file ;
DiffLines.print_changed_lines () ;
MethodRangeMap.create_clang_method_range_map process_ast_fn ;
L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ;
match test_samples_file with
| Some _ ->
TestSample.init_test_sample test_samples_file
| _ ->
() ;
L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ;
initialized_test_determinator := true
let init_java changed_lines_file test_samples_file code_graph_file =
DiffLines.init_changed_lines_map changed_lines_file ;
DiffLines.print_changed_lines () ;
MethodRangeMap.create_java_method_range_map code_graph_file ;
L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ;
TestSample.init_test_sample test_samples_file ;
L.(debug TestDeterminator Medium) "%a@\n" TestSample.pp_map () ;
initialized_test_determinator := true
(* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) let compute_and_emit_relevant_methods ~clang_range_map ~source_file ~changed_lines_file =
let test_to_run_clang source_file ~process_ast_fn ~changed_lines_file ~test_samples_file = let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
L.(debug TestDeterminator Quiet) let relevant_methods =
"****** Start Test Determinator for %s *****@\n" compute_affected_methods_clang ~clang_range_map ~source_file ~changed_lines_map
(SourceFile.to_string source_file) ;
if is_test_determinator_init () then ()
else init_clang process_ast_fn changed_lines_file test_samples_file ;
let affected_methods =
compute_affected_methods_clang source_file (DiffLines.changed_lines_map ())
(MethodRangeMap.method_range_map ())
in in
relevant_methods := JPS.ProfilerSample.union affected_methods !relevant_methods ; emit_relevant_methods relevant_methods
let test_to_run =
if JPS.ProfilerSample.is_empty affected_methods then []
else
List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) ->
let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in
if JPS.ProfilerSample.is_empty intersection then acc else label :: acc )
in
relevant_tests := List.append test_to_run !relevant_tests
let test_to_run_java ~changed_lines_file ~test_samples_file ~code_graph_file = (* test_to_run = { n | Affected_Method /\ ts_n != 0 } *)
L.(debug TestDeterminator Quiet) "***** Start Java Test Determinator *****@\n" ; let test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file
if is_test_determinator_init () then () ~test_samples_file =
else init_java changed_lines_file test_samples_file code_graph_file ; let changed_lines_map = DiffLines.create_changed_lines_map changed_lines_file in
let affected_methods = let affected_methods =
compute_affected_methods_java (DiffLines.changed_lines_map ()) match (clang_range_map, source_file) with
(MethodRangeMap.method_range_map ()) | Some clang_range_map, Some source_file ->
compute_affected_methods_clang ~source_file ~clang_range_map ~changed_lines_map
| _ (* Java case *) ->
let method_range = MethodRangeMap.create_java_method_range_map code_graph_file in
compute_affected_methods_java changed_lines_map method_range
in in
let test_to_run = let profiler_samples = TestSample.read_test_sample test_samples_file in
if JPS.ProfilerSample.is_empty affected_methods then [] if JPS.ProfilerSample.is_empty affected_methods then []
else else
List.fold (TestSample.test_sample ()) ~init:[] ~f:(fun acc (label, profiler_samples) -> List.fold profiler_samples ~init:[] ~f:(fun acc (label, profiler_samples) ->
let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in let intersection = JPS.ProfilerSample.inter affected_methods profiler_samples in
if JPS.ProfilerSample.is_empty intersection then acc if JPS.ProfilerSample.is_empty intersection then acc else label :: acc )
else (
L.(debug TestDeterminator Quiet)
"Choosing test '%s' because of [%a]@\n" label let emit_tests_to_run relevant_tests =
(Pp.seq Typ.Procname.pp ~sep:", ") let json = `List (List.map ~f:(fun t -> `String t) relevant_tests) in
(JPS.ProfilerSample.elements intersection) ; let outpath = Config.results_dir ^/ Config.test_determinator_output in
label :: acc ) ) YB.to_file outpath json
let compute_and_emit_test_to_run ?clang_range_map ?source_file ~code_graph_file ~changed_lines_file
~test_samples_file =
let relevant_tests =
test_to_run ?clang_range_map ~code_graph_file ?source_file ~changed_lines_file
~test_samples_file
in in
relevant_tests := List.append test_to_run !relevant_tests emit_tests_to_run relevant_tests

@ -7,21 +7,16 @@
open! IStd open! IStd
val test_to_run_java : val compute_and_emit_test_to_run :
changed_lines_file:string option ?clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t
-> test_samples_file:string option -> ?source_file:SourceFile.t
-> code_graph_file:string option -> code_graph_file:string option
-> unit
val test_to_run_clang :
SourceFile.t
-> process_ast_fn:((Typ.Procname.t -> Location.t * Location.t -> unit) -> unit)
-> changed_lines_file:string option -> changed_lines_file:string option
-> test_samples_file:string option -> test_samples_file:string option
-> unit -> unit
val emit_tests_to_run : unit -> unit val compute_and_emit_relevant_methods :
clang_range_map:(Location.t * Location.t) Typ.Procname.Map.t
val emit_relevant_methods : unit -> unit -> source_file:SourceFile.t
-> changed_lines_file:string option
val _get_relevant_test_to_run : unit -> string list -> unit

Loading…
Cancel
Save