From 124036ea0b31692b1a3a6251adcc441cb2dd6da4 Mon Sep 17 00:00:00 2001 From: Martin Trojer Date: Thu, 11 Jul 2019 06:14:38 -0700 Subject: [PATCH] New faster version of Diff/Test-Determinator Reviewed By: jvillard Differential Revision: D15876508 fbshipit-source-id: f5d407025 --- .gitignore | 4 +- Makefile | 4 +- infer/man/man1/infer-full.txt | 16 +++- infer/src/IR/Attributes.ml | 9 +-- infer/src/backend/InferPrint.ml | 4 - infer/src/backend/mergeCapture.ml | 7 ++ infer/src/backend/mergeCapture.mli | 2 + infer/src/base/Config.ml | 25 ++++-- infer/src/base/Config.mli | 6 +- infer/src/base/MergeResults.ml | 26 ++++++- infer/src/base/MergeResults.mli | 4 + infer/src/base/ResultsDatabase.ml | 1 - infer/src/clang/Capture.ml | 6 ++ infer/src/clang/astToRangeMap.ml | 45 +++++++++++ infer/src/clang/astToRangeMap.mli | 14 ++++ infer/src/clang/cFrontend.ml | 3 - infer/src/clang/cLocation.ml | 7 ++ infer/src/clang/cLocation.mli | 2 + infer/src/integration/Driver.ml | 6 +- infer/src/integration/testDeterminator.ml | 76 +++++-------------- infer/src/integration/testDeterminator.mli | 8 +- .../.buckconfig | 0 .../.buckversion | 0 .../Makefile | 10 +-- .../changed_functions.json.exp} | 0 .../src/BUCK | 0 .../src/a.cpp | 0 .../src/b.cpp | 0 .../src/mod-a.cpp | 0 .../src/mod-b.cpp | 0 .../diff_determinator.json.mod1.exp | 1 - .../Makefile | 15 ++-- .../changed_functions.json.mod1.exp | 1 + .../changed_functions.json.mod2.exp} | 0 .../mod1-A.cpp | 0 .../mod2-A.cpp | 0 .../orig-A.cpp | 0 37 files changed, 199 insertions(+), 103 deletions(-) create mode 100644 infer/src/clang/astToRangeMap.ml create mode 100644 infer/src/clang/astToRangeMap.mli rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/.buckconfig (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/.buckversion (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/Makefile (72%) rename infer/tests/build_systems/{buck_test_determinator/diff_determinator.json.exp => buck_export_changed_functions/changed_functions.json.exp} (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/src/BUCK (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/src/a.cpp (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/src/b.cpp (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/src/mod-a.cpp (100%) rename infer/tests/build_systems/{buck_test_determinator => buck_export_changed_functions}/src/mod-b.cpp (100%) delete mode 100644 infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod1.exp rename infer/tests/build_systems/{clang_test_determinator => export_changed_functions}/Makefile (63%) create mode 100644 infer/tests/build_systems/export_changed_functions/changed_functions.json.mod1.exp rename infer/tests/build_systems/{clang_test_determinator/diff_determinator.json.mod2.exp => export_changed_functions/changed_functions.json.mod2.exp} (100%) rename infer/tests/build_systems/{clang_test_determinator => export_changed_functions}/mod1-A.cpp (100%) rename infer/tests/build_systems/{clang_test_determinator => export_changed_functions}/mod2-A.cpp (100%) rename infer/tests/build_systems/{clang_test_determinator => export_changed_functions}/orig-A.cpp (100%) diff --git a/.gitignore b/.gitignore index ae38add62..cebd23c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,10 +26,10 @@ duplicates.txt *.ast.biniou /infer/tests/build_systems/buck_flavors_deterministic/capture_hash-*.sha /infer/tests/build_systems/buck_flavors_diff/src/hello.c -/infer/tests/build_systems/buck_test_determinator/diff.mod.test +/infer/tests/build_systems/buck_export_changed_functions/diff.mod.test /infer/tests/build_systems/clang_compilation_db_escaped/compile_commands.json /infer/tests/build_systems/clang_compilation_db_relpath/compile_commands.json -/infer/tests/build_systems/clang_test_determinator/*.test +/infer/tests/build_systems/export_changed_functions/*.test /infer/tests/build_systems/clang_with_MD_flag/hello.d /infer/tests/build_systems/codetoanalyze/mvn/**/target/ /infer/tests/build_systems/codetoanalyze/ndk-build/hello_app/libs/ diff --git a/Makefile b/Makefile index b8e81107f..fa0ab7985 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,8 @@ BUILD_SYSTEMS_TESTS += \ run_hidden_linters \ tracebugs \ utf8_in_procname \ - clang_test_determinator \ - buck_test_determinator \ + export_changed_functions \ + buck_export_changed_functions \ DIRECT_TESTS += \ c_biabduction \ diff --git a/infer/man/man1/infer-full.txt b/infer/man/man1/infer-full.txt index 577f7d3bc..32ed38d6c 100644 --- a/infer/man/man1/infer-full.txt +++ b/infer/man/man1/infer-full.txt @@ -1326,6 +1326,15 @@ INTERNAL OPTIONS --exit-node-bias nodes nearest the exit node are analyzed first + --export-changed-functions + Activates: Make infer outout changed functions, similar to + test-determinator. It is used together with the --modified-lines. + (Conversely: --no-export-changed-functions) + + --export-changed-functions-output path + Name of file for export-changed-functions results (default: + changed_functions.json) + --external-java-packages-reset Set --external-java-packages to the empty list. @@ -1712,10 +1721,9 @@ INTERNAL OPTIONS which speficy the relevant arguments. (Conversely: --no-test-determinator) - --test-determinator-clang - Activates: Run infer in Test Determinator mode for clang. It is - used together with the --modified-lines. (Conversely: - --no-test-determinator-clang) + --test-determinator-output path + Name of file for test-determinator results (default: + test_determinator.json) --test-filtering Activates: List all the files Infer can report on (should be diff --git a/infer/src/IR/Attributes.ml b/infer/src/IR/Attributes.ml index 725dbe64a..102e7c23d 100644 --- a/infer/src/IR/Attributes.ml +++ b/infer/src/IR/Attributes.ml @@ -52,7 +52,7 @@ let replace_statement = ResultsDatabase.register_statement {| INSERT OR REPLACE INTO procedures -SELECT :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees, :modified_flag +SELECT :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees FROM ( SELECT NULL FROM ( @@ -64,7 +64,7 @@ FROM ( OR (attr_kind = :akind AND source_file <= :sfile) )|} -let replace pname pname_blob akind loc_file attr_blob proc_desc callees modified_flag = +let replace pname pname_blob akind loc_file attr_blob proc_desc callees = ResultsDatabase.with_registered_statement replace_statement ~f:(fun db replace_stmt -> Sqlite3.bind replace_stmt 1 (* :pname *) pname_blob |> SqliteUtils.check_result_code db ~log:"replace bind pname" ; @@ -81,9 +81,6 @@ let replace pname pname_blob akind loc_file attr_blob proc_desc callees modified |> SqliteUtils.check_result_code db ~log:"replace bind cfg" ; Sqlite3.bind replace_stmt 7 (* :callees *) (Typ.Procname.SQLiteList.serialize callees) |> SqliteUtils.check_result_code db ~log:"replace bind callees" ; - Sqlite3.bind replace_stmt 8 - (* :modified_flag *) (Sqlite3.Data.INT (Int64.of_int (if modified_flag then 1 else 0))) - |> SqliteUtils.check_result_code db ~log:"replace bind modified_flag" ; SqliteUtils.result_unit db ~finalize:false ~log:"Attributes.replace" replace_stmt ) @@ -140,7 +137,7 @@ let store ~proc_desc (attr : ProcAttributes.t) = (ProcAttributes.SQLite.serialize attr) proc_desc (Option.map proc_desc ~f:Procdesc.get_static_callees |> Option.value ~default:[]) - false + let load_defined pname = Typ.Procname.SQLite.serialize pname |> find ~defined:true diff --git a/infer/src/backend/InferPrint.ml b/infer/src/backend/InferPrint.ml index e99085752..ad974f738 100644 --- a/infer/src/backend/InferPrint.ml +++ b/infer/src/backend/InferPrint.ml @@ -1235,10 +1235,6 @@ let register_perf_stats_report () = let main ~report_json = - if Config.test_determinator_clang then ( - TestDeterminator.persist_relevant_method_in_db () ; - TestDeterminator.emit_tests_to_run () ; - TestDeterminator.emit_relevant_methods () ) ; let issue_formats = init_issues_format_list report_json in let formats_by_report_kind = let costs_report_format_kind = diff --git a/infer/src/backend/mergeCapture.ml b/infer/src/backend/mergeCapture.ml index d806dffcb..cf1b80695 100644 --- a/infer/src/backend/mergeCapture.ml +++ b/infer/src/backend/mergeCapture.ml @@ -25,6 +25,13 @@ let merge_global_tenvs infer_deps_file = L.progress "Merging type environments took %a@." Mtime.Span.pp (Mtime_clock.count time0) +let merge_changed_functions () = + L.progress "Merging changed functions files...@." ; + let infer_deps_file = Config.(results_dir ^/ buck_infer_deps_file_name) in + MergeResults.merge_buck_changed_functions infer_deps_file ; + L.progress "Done merging changed functions files@." + + let merge_captured_targets () = let time0 = Mtime_clock.counter () in L.progress "Merging captured Buck targets...@\n%!" ; diff --git a/infer/src/backend/mergeCapture.mli b/infer/src/backend/mergeCapture.mli index 739abfa35..c2b67953f 100644 --- a/infer/src/backend/mergeCapture.mli +++ b/infer/src/backend/mergeCapture.mli @@ -8,3 +8,5 @@ open! IStd val merge_captured_targets : unit -> unit + +val merge_changed_functions : unit -> unit diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index e58c95eec..e973d1808 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -2230,10 +2230,21 @@ and test_determinator = $(b,--test-profiler) flags, which speficy the relevant arguments." -and test_determinator_clang = - CLOpt.mk_bool ~long:"test-determinator-clang" ~default:false - "Run infer in Test Determinator mode for clang. It is used together with the \ - $(b,--modified-lines)." +and test_determinator_output = + CLOpt.mk_path ~long:"test-determinator-output" ~default:"test_determinator.json" + "Name of file for test-determinator results" + + +and export_changed_functions = + CLOpt.mk_bool ~deprecated:["test-determinator-clang"] ~long:"export-changed-functions" + ~default:false + "Make infer outout changed functions, similar to test-determinator. It is used together with \ + the $(b,--modified-lines)." + + +and export_changed_functions_output = + CLOpt.mk_path ~long:"export-changed-functions-output" ~default:"changed_functions.json" + "Name of file for export-changed-functions results" and test_filtering = @@ -3119,7 +3130,11 @@ and keep_going = !keep_going and test_determinator = !test_determinator -and test_determinator_clang = !test_determinator_clang +and test_determinator_output = !test_determinator_output + +and export_changed_functions = !export_changed_functions + +and export_changed_functions_output = !export_changed_functions_output and test_filtering = !test_filtering diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index c4e8f64b0..24c3c2dcb 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -670,7 +670,11 @@ val symops_per_iteration : int option val test_determinator : bool -val test_determinator_clang : bool +val test_determinator_output : string + +val export_changed_functions : bool + +val export_changed_functions_output : string val test_filtering : bool diff --git a/infer/src/base/MergeResults.ml b/infer/src/base/MergeResults.ml index a7084e385..da23a982c 100644 --- a/infer/src/base/MergeResults.ml +++ b/infer/src/base/MergeResults.ml @@ -6,6 +6,8 @@ *) open! IStd module L = Logging +module YB = Yojson.Basic +module YBU = Yojson.Basic.Util let merge_procedures_table ~db_file = let db = ResultsDatabase.get_database () in @@ -16,7 +18,7 @@ let merge_procedures_table ~db_file = Sqlite3.exec db {| INSERT OR REPLACE INTO procedures -SELECT sub.proc_name, sub.proc_name_hum, sub.attr_kind, sub.source_file, sub.proc_attributes, sub.cfg, sub.callees, sub.modified_flag +SELECT sub.proc_name, sub.proc_name_hum, sub.attr_kind, sub.source_file, sub.proc_attributes, sub.cfg, sub.callees FROM ( attached.procedures AS sub LEFT OUTER JOIN procedures AS main @@ -42,7 +44,7 @@ let merge_source_files_table ~db_file = ~log:(Printf.sprintf "copying source_files of database '%s'" db_file) -let merge infer_out_src = +let merge_dbs infer_out_src = let db_file = infer_out_src ^/ ResultsDatabase.database_filename in let main_db = ResultsDatabase.get_database () in Sqlite3.exec main_db (Printf.sprintf "ATTACH '%s' AS attached" db_file) @@ -54,6 +56,21 @@ let merge infer_out_src = () +let merge_changed_functions_json infer_out_src = + let main_changed_fs_file = Config.results_dir ^/ Config.export_changed_functions_output in + let changed_fs_file = infer_out_src ^/ Config.export_changed_functions_output in + let main_json = try YB.from_file main_changed_fs_file |> YBU.to_list with Sys_error _ -> [] in + let changed_json = try YB.from_file changed_fs_file |> YBU.to_list with Sys_error _ -> [] in + let all_fs = + `List + (List.dedup_and_sort + ~compare:(fun s1 s2 -> + match (s1, s2) with `String s1, `String s2 -> String.compare s1 s2 | _ -> 0 ) + (List.append main_json changed_json)) + in + YB.to_file main_changed_fs_file all_fs + + let iter_infer_deps infer_deps_file ~f = let one_line line = match String.split ~on:'\t' line with @@ -74,4 +91,7 @@ let iter_infer_deps infer_deps_file ~f = L.internal_error "Couldn't read deps file '%s': %s" infer_deps_file error -let merge_buck_flavors_results infer_deps_file = iter_infer_deps infer_deps_file ~f:merge +let merge_buck_flavors_results infer_deps_file = iter_infer_deps infer_deps_file ~f:merge_dbs + +let merge_buck_changed_functions infer_deps_file = + iter_infer_deps infer_deps_file ~f:merge_changed_functions_json diff --git a/infer/src/base/MergeResults.mli b/infer/src/base/MergeResults.mli index 3c89f546b..e64dfbe6e 100644 --- a/infer/src/base/MergeResults.mli +++ b/infer/src/base/MergeResults.mli @@ -11,4 +11,8 @@ val merge_buck_flavors_results : string -> unit (** Merge the results from sub-invocations of infer inside buck-out/. Takes as argument the infer_deps file. *) +val merge_buck_changed_functions : string -> unit +(** Merge the changed functions from sub-invocations of infer inside buck-out/. Takes as argument the + infer_deps file. *) + val iter_infer_deps : string -> f:(string -> unit) -> unit diff --git a/infer/src/base/ResultsDatabase.ml b/infer/src/base/ResultsDatabase.ml index 57822b39e..c8b5afcd4 100644 --- a/infer/src/base/ResultsDatabase.ml +++ b/infer/src/base/ResultsDatabase.ml @@ -21,7 +21,6 @@ let procedures_schema = , proc_attributes BLOB NOT NULL , cfg BLOB , callees BLOB NOT NULL - , modified_flag INT NOT NULL DEFAULT 0 )|} diff --git a/infer/src/clang/Capture.ml b/infer/src/clang/Capture.ml index 7977e3289..ce24bb5bf 100644 --- a/infer/src/clang/Capture.ml +++ b/infer/src/clang/Capture.ml @@ -99,6 +99,12 @@ let run_clang_frontend ast_source = L.(debug Capture Medium) "Start %s of AST from %a@\n" Config.clang_frontend_action_string pp_ast_filename ast_source ; if Config.linters then CFrontend_checkers_main.do_frontend_checks trans_unit_ctx ast_decl ; + if Config.export_changed_functions then ( + let source_file = trans_unit_ctx.CFrontend_config.source_file in + let process_fn = AstToRangeMap.process_ast ast_decl (Tenv.create ()) source_file in + TestDeterminator.test_to_run_clang source_file ~process_ast_fn:process_fn + ~changed_lines_file:Config.modified_lines ~test_samples_file:None ; + TestDeterminator.emit_relevant_methods () ) ; if Config.capture then CFrontend.do_source_file trans_unit_ctx ast_decl ; L.(debug Capture Medium) "End %s of AST file %a... OK!@\n" Config.clang_frontend_action_string pp_ast_filename diff --git a/infer/src/clang/astToRangeMap.ml b/infer/src/clang/astToRangeMap.ml new file mode 100644 index 000000000..8db30fbfe --- /dev/null +++ b/infer/src/clang/astToRangeMap.ml @@ -0,0 +1,45 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) +open! IStd +module L = Logging + +let process_ast ast tenv default_source_file f = + let open Clang_ast_t in + let mk_key decl = CType_decl.CProcname.from_decl ~tenv decl in + let rec extract_location decl = + match decl with + | ObjCMethodDecl (di, _, _) + | CXXConversionDecl (di, _, _, _, _) + | FunctionDecl (di, _, _, _) + | CXXMethodDecl (di, _, _, _, _) + | CXXConstructorDecl (di, _, _, _, _) + | CXXDestructorDecl (di, _, _, _, _) -> + let range = CLocation.location_of_decl_info default_source_file di in + f (mk_key decl) range + | _ -> ( + match Clang_ast_proj.get_decl_context_tuple decl with + | Some (decls, _) -> + List.iter decls ~f:extract_location + | None -> + L.(debug Capture Verbose) + "@\n Found %s.., skipping" + (Clang_ast_proj.get_decl_kind_string decl) ) + in + match ast with + | TranslationUnitDecl (_, decl_list, _, _) -> + CFrontend_config.global_translation_unit_decls := decl_list ; + L.(debug Capture Verbose) "@\n Processing AST...@\n" ; + List.iter decl_list ~f:(fun d -> + let info = Clang_ast_proj.get_decl_tuple d in + let source_range = info.di_source_range in + if + CLocation.should_translate_lib default_source_file source_range `DeclTraversal + ~translate_when_used:true + then extract_location d ) ; + L.(debug Capture Verbose) "@\n Finished processing AST.@\n" + | _ -> + assert false diff --git a/infer/src/clang/astToRangeMap.mli b/infer/src/clang/astToRangeMap.mli new file mode 100644 index 000000000..0823bd8f6 --- /dev/null +++ b/infer/src/clang/astToRangeMap.mli @@ -0,0 +1,14 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) +open! IStd + +val process_ast : + Clang_ast_t.decl + -> Tenv.t + -> SourceFile.t + -> (Typ.Procname.t -> Location.t * Location.t -> unit) + -> unit diff --git a/infer/src/clang/cFrontend.ml b/infer/src/clang/cFrontend.ml index 545059643..26631c440 100644 --- a/infer/src/clang/cFrontend.ml +++ b/infer/src/clang/cFrontend.ml @@ -63,9 +63,6 @@ let do_source_file (translation_unit_context : CFrontend_config.translation_unit || Option.is_some Config.icfg_dotty_outfile then Dotty.print_icfg_dotty source_file cfg ; L.(debug Capture Verbose) "Stored on disk:@[%a@]@." Cfg.pp_proc_signatures cfg ; - if Config.test_determinator_clang then ( - TestDeterminator.test_to_run_clang source_file cfg Config.modified_lines None ; - TestDeterminator.persist_relevant_method_in_db () ) ; let procedures_translated_summary = EventLogger.ProceduresTranslatedSummary { procedures_translated_total= !CFrontend_config.procedures_attempted diff --git a/infer/src/clang/cLocation.ml b/infer/src/clang/cLocation.ml index 49ff22fc5..946fce613 100644 --- a/infer/src/clang/cLocation.ml +++ b/infer/src/clang/cLocation.ml @@ -99,3 +99,10 @@ let location_of_source_range ?(pick_location = `Start) default_source_file sourc let location_of_stmt_info default_source_file stmt_info = location_of_source_range default_source_file stmt_info.Clang_ast_t.si_source_range + + +let location_of_decl_info default_source_file decl_info = + ( location_of_source_range ~pick_location:`Start default_source_file + decl_info.Clang_ast_t.di_source_range + , location_of_source_range ~pick_location:`End default_source_file + decl_info.Clang_ast_t.di_source_range ) diff --git a/infer/src/clang/cLocation.mli b/infer/src/clang/cLocation.mli index 096df5a83..e9ecb2fea 100644 --- a/infer/src/clang/cLocation.mli +++ b/infer/src/clang/cLocation.mli @@ -27,3 +27,5 @@ val location_of_source_range : (** picks the start of the source range by default *) val location_of_stmt_info : SourceFile.t -> Clang_ast_t.stmt_info -> Location.t + +val location_of_decl_info : SourceFile.t -> Clang_ast_t.decl_info -> Location.t * Location.t diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index f841bfa54..20f9ea25d 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -109,7 +109,10 @@ let clean_results_dir () = (not (List.exists ~f:(String.equal (Filename.basename name)) - [Config.report_json; Config.costs_report_json])) + [ Config.report_json + ; Config.costs_report_json + ; Config.test_determinator_output + ; Config.export_changed_functions_output ])) && ( List.mem ~equal:String.equal files_to_delete (Filename.basename name) || List.exists ~f:(Filename.check_suffix name) suffixes_to_delete ) in @@ -418,6 +421,7 @@ let analyze_and_report ?suppress_console_report ~changed_files mode = (* else rely on the command line value *) Config.merge in if should_merge then ( + if Config.export_changed_functions then MergeCapture.merge_changed_functions () ; MergeCapture.merge_captured_targets () ; RunState.set_merge_capture false ; RunState.store () ) ; diff --git a/infer/src/integration/testDeterminator.ml b/infer/src/integration/testDeterminator.ml index 776378e01..fe80a446b 100644 --- a/infer/src/integration/testDeterminator.ml +++ b/infer/src/integration/testDeterminator.ml @@ -9,36 +9,7 @@ open! IStd module L = Logging module F = Format module JPS = JavaProfilerSamples - -let load_all_statement = - ResultsDatabase.register_statement - "SELECT DISTINCT proc_name_hum FROM procedures WHERE modified_flag = 1 ORDER BY proc_name_hum" - - -let load_all () = - ResultsDatabase.with_registered_statement load_all_statement ~f:(fun db stmt -> - Container.to_list stmt - ~fold: - (SqliteUtils.result_fold_single_column_rows ~finalize:false db - ~log:"modified_flag.load_all") - |> List.map ~f:Sqlite3.Data.to_string ) - - -let store_statement = - ResultsDatabase.register_statement - "UPDATE procedures SET modified_flag = :i WHERE proc_name = :k" - - -let store pname modified = - let modified' = Sqlite3.Data.INT (Int64.of_int (if modified then 1 else 0)) in - ResultsDatabase.with_registered_statement store_statement ~f:(fun db stmt -> - Typ.Procname.SQLite.serialize pname - |> Sqlite3.bind stmt 2 - |> SqliteUtils.check_result_code db ~log:"store bind proc name" ; - modified' |> Sqlite3.bind stmt 1 - |> SqliteUtils.check_result_code db ~log:"store bind modified flag " ; - SqliteUtils.result_unit db ~finalize:false ~log:"store modified flag " stmt ) - +module YB = Yojson.Basic (* a flag used to make the method search signature sensitive *) let use_method_signature = false @@ -54,7 +25,9 @@ let initialized_test_determinator = ref false let is_test_determinator_init () = !initialized_test_determinator module MethodRangeMap = struct - let map : (Location.t * Location.t) RangeMap.t ref = ref RangeMap.empty + type t = (Location.t * Location.t) RangeMap.t + + let map : t ref = ref RangeMap.empty let method_range_map () = !map @@ -107,15 +80,8 @@ module MethodRangeMap = struct L.die UserError "Missing method declaration info argument" - let create_clang_method_range_map cfg = - let update_method_range_map pdesc = - let start_node = Procdesc.get_start_node pdesc in - let exit_node = Procdesc.get_exit_node pdesc in - let range = (Procdesc.Node.get_loc start_node, Procdesc.Node.get_loc exit_node) in - let key = Procdesc.get_proc_name pdesc in - map := RangeMap.add key range !map - in - Typ.Procname.Hash.iter (fun _ pdesc -> update_method_range_map pdesc) cfg + let create_clang_method_range_map process_ast_fn = + process_ast_fn (fun key range -> map := RangeMap.add key range !map) let pp_map fmt () = @@ -249,33 +215,32 @@ let _get_relevant_test_to_run () = !relevant_tests let emit_tests_to_run () = let json = `List (List.map ~f:(fun t -> `String t) !relevant_tests) in - let outpath = Config.results_dir ^/ "test_determinator.json" in - Yojson.Basic.to_file outpath json ; + let outpath = Config.results_dir ^/ Config.test_determinator_output in + YB.to_file outpath json ; L.(debug TestDeterminator Medium) "Tests to run: [%a]@\n" (Pp.seq ~sep:", " F.pp_print_string) !relevant_tests -let persist_relevant_method_in_db () = - JPS.ProfilerSample.iter (fun pname -> store pname true) !relevant_methods - - let emit_relevant_methods () = - let rel_methods = load_all () in - let json = `List (List.map ~f:(fun t -> `String t) rel_methods) in - let outpath = Config.results_dir ^/ "diff_determinator.json" in - Yojson.Basic.to_file outpath json ; + let cleaned_methods = + List.dedup_and_sort ~compare:String.compare + (List.map (JPS.ProfilerSample.elements !relevant_methods) ~f:Typ.Procname.to_string) + in + let json = `List (List.map ~f:(fun t -> `String t) cleaned_methods) in + let outpath = Config.results_dir ^/ Config.export_changed_functions_output in + YB.to_file outpath json ; L.(debug TestDeterminator Medium) "Methods modified in this Diff: %a@\n" (Pp.seq ~sep:", " F.pp_print_string) - rel_methods + cleaned_methods -let init_clang cfg changed_lines_file test_samples_file = +let init_clang process_ast_fn changed_lines_file test_samples_file = DiffLines.init_changed_lines_map changed_lines_file ; DiffLines.print_changed_lines () ; - MethodRangeMap.create_clang_method_range_map cfg ; + MethodRangeMap.create_clang_method_range_map process_ast_fn ; L.(debug TestDeterminator Medium) "%a@\n" MethodRangeMap.pp_map () ; match test_samples_file with | Some _ -> @@ -297,11 +262,12 @@ let init_java changed_lines_file test_samples_file code_graph_file = (* test_to_run = { n | Affected_Method /\ ts_n != 0 } *) -let test_to_run_clang source_file cfg changed_lines_file test_samples_file = +let test_to_run_clang source_file ~process_ast_fn ~changed_lines_file ~test_samples_file = L.(debug TestDeterminator Quiet) "****** Start Test Determinator for %s *****@\n" (SourceFile.to_string source_file) ; - if is_test_determinator_init () then () else init_clang cfg changed_lines_file test_samples_file ; + if is_test_determinator_init () then () + else init_clang process_ast_fn changed_lines_file test_samples_file ; let affected_methods = compute_affected_methods_clang source_file (DiffLines.changed_lines_map ()) (MethodRangeMap.method_range_map ()) diff --git a/infer/src/integration/testDeterminator.mli b/infer/src/integration/testDeterminator.mli index 9abbe2dbf..b534af1be 100644 --- a/infer/src/integration/testDeterminator.mli +++ b/infer/src/integration/testDeterminator.mli @@ -10,12 +10,14 @@ open! IStd val test_to_run_java : string option -> string option -> string option -> unit val test_to_run_clang : - SourceFile.t -> Procdesc.t Typ.Procname.Hash.t -> string option -> string option -> unit + SourceFile.t + -> process_ast_fn:((Typ.Procname.t -> Location.t * Location.t -> unit) -> unit) + -> changed_lines_file:string option + -> test_samples_file:string option + -> unit val emit_tests_to_run : unit -> unit -val persist_relevant_method_in_db : unit -> unit - val emit_relevant_methods : unit -> unit val _get_relevant_test_to_run : unit -> string list diff --git a/infer/tests/build_systems/buck_test_determinator/.buckconfig b/infer/tests/build_systems/buck_export_changed_functions/.buckconfig similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/.buckconfig rename to infer/tests/build_systems/buck_export_changed_functions/.buckconfig diff --git a/infer/tests/build_systems/buck_test_determinator/.buckversion b/infer/tests/build_systems/buck_export_changed_functions/.buckversion similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/.buckversion rename to infer/tests/build_systems/buck_export_changed_functions/.buckversion diff --git a/infer/tests/build_systems/buck_test_determinator/Makefile b/infer/tests/build_systems/buck_export_changed_functions/Makefile similarity index 72% rename from infer/tests/build_systems/buck_test_determinator/Makefile rename to infer/tests/build_systems/buck_export_changed_functions/Makefile index 48904694d..f1e06fc8e 100644 --- a/infer/tests/build_systems/buck_test_determinator/Makefile +++ b/infer/tests/build_systems/buck_export_changed_functions/Makefile @@ -9,9 +9,9 @@ include $(TESTS_DIR)/base.make A_CPP = src/a.cpp B_CPP = src/b.cpp BUCK_TARGET = //src:test -TEST_DETERMINATOR_RESULT = infer-out/diff_determinator.json +TEST_DETERMINATOR_RESULT = infer-out/changed_functions.json DIFF_OUTPUT = diff.mod.test -INFER_OPTIONS = --flavors --no-linters --test-determinator-clang --modified-lines $(DIFF_OUTPUT) --project-root $(TESTS_DIR) +INFER_OPTIONS = --flavors --linters --no-capture --export-changed-functions --modified-lines $(DIFF_OUTPUT) --project-root $(TESTS_DIR) $(DIFF_OUTPUT): $(QUIET)echo -n '$(A_CPP):' > diff.mod.test @@ -24,16 +24,16 @@ $(DIFF_OUTPUT): $(QUIET)echo >> diff.mod.test $(TEST_DETERMINATOR_RESULT): $(DIFF_OUTPUT) - $(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes,\ + $(QUIET)$(call silent_on_success,Testing export-changed-functions with set of changes,\ $(INFER_BIN) $(INFER_OPTIONS) -- buck build --no-cache $(BUCK_TARGET)) .PHONY: test test: $(TEST_DETERMINATOR_RESULT) - $(QUIET)$(call check_no_diff,diff_determinator.json.exp,$(TEST_DETERMINATOR_RESULT)) + $(QUIET)$(call check_no_diff,changed_functions.json.exp,$(TEST_DETERMINATOR_RESULT)) .PHONY: replace replace: $(TEST_DETERMINATOR_RESULT) - $(COPY) $(TEST_DETERMINATOR_RESULT) diff_determinator.json.exp + $(COPY) $(TEST_DETERMINATOR_RESULT) changed_functions.json.exp .PHONY: clean clean: diff --git a/infer/tests/build_systems/buck_test_determinator/diff_determinator.json.exp b/infer/tests/build_systems/buck_export_changed_functions/changed_functions.json.exp similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/diff_determinator.json.exp rename to infer/tests/build_systems/buck_export_changed_functions/changed_functions.json.exp diff --git a/infer/tests/build_systems/buck_test_determinator/src/BUCK b/infer/tests/build_systems/buck_export_changed_functions/src/BUCK similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/src/BUCK rename to infer/tests/build_systems/buck_export_changed_functions/src/BUCK diff --git a/infer/tests/build_systems/buck_test_determinator/src/a.cpp b/infer/tests/build_systems/buck_export_changed_functions/src/a.cpp similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/src/a.cpp rename to infer/tests/build_systems/buck_export_changed_functions/src/a.cpp diff --git a/infer/tests/build_systems/buck_test_determinator/src/b.cpp b/infer/tests/build_systems/buck_export_changed_functions/src/b.cpp similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/src/b.cpp rename to infer/tests/build_systems/buck_export_changed_functions/src/b.cpp diff --git a/infer/tests/build_systems/buck_test_determinator/src/mod-a.cpp b/infer/tests/build_systems/buck_export_changed_functions/src/mod-a.cpp similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/src/mod-a.cpp rename to infer/tests/build_systems/buck_export_changed_functions/src/mod-a.cpp diff --git a/infer/tests/build_systems/buck_test_determinator/src/mod-b.cpp b/infer/tests/build_systems/buck_export_changed_functions/src/mod-b.cpp similarity index 100% rename from infer/tests/build_systems/buck_test_determinator/src/mod-b.cpp rename to infer/tests/build_systems/buck_export_changed_functions/src/mod-b.cpp diff --git a/infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod1.exp b/infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod1.exp deleted file mode 100644 index dad09161e..000000000 --- a/infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod1.exp +++ /dev/null @@ -1 +0,0 @@ -["Shapes::Cube::area","Shapes::Cube::sort","Shapes::Cube::sort::lambda_A.cpp:24:25::","Shapes::Cube::sort::lambda_A.cpp:24:25::operator()"] \ No newline at end of file diff --git a/infer/tests/build_systems/clang_test_determinator/Makefile b/infer/tests/build_systems/export_changed_functions/Makefile similarity index 63% rename from infer/tests/build_systems/clang_test_determinator/Makefile rename to infer/tests/build_systems/export_changed_functions/Makefile index 66255666e..550d290b3 100644 --- a/infer/tests/build_systems/clang_test_determinator/Makefile +++ b/infer/tests/build_systems/export_changed_functions/Makefile @@ -11,6 +11,7 @@ include $(TESTS_DIR)/base.make A_CPP = A.cpp TEST_DETERMINATOR_RESULT = infer-out-mod2/diff_determinator.json DIFF_OUTPUT = diff.mod2.test +INFER_OPTIONS = --export-changed-functions --linters --no-capture --buck-compilation-database 'no-deps' default: $(TEST_DETERMINATOR_RESULT) @@ -23,23 +24,23 @@ $(DIFF_OUTPUT): orig-$(A_CPP) mod2-$(A_CPP) || [ $$? = 1 ]) >> diff.mod2.test $(TEST_DETERMINATOR_RESULT): $(DIFF_OUTPUT) - $(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes in mod1,\ + $(QUIET)$(call silent_on_success,Testing export-changed-functions with set of changes in mod1,\ cp mod1-$(A_CPP) $(A_CPP);\ - $(INFER_BIN) -o infer-out-mod1 --test-determinator-clang --modified-lines diff.mod1.test -- clang -c $(A_CPP)) + $(INFER_BIN) -o infer-out-mod1 $(INFER_OPTIONS) --modified-lines diff.mod1.test -- clang -c $(A_CPP)) $(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes in mod2,\ cp mod2-$(A_CPP) $(A_CPP);\ - $(INFER_BIN) -o infer-out-mod2 --test-determinator-clang --modified-lines diff.mod2.test -- clang -c $(A_CPP)) + $(INFER_BIN) -o infer-out-mod2 $(INFER_OPTIONS) --modified-lines diff.mod2.test -- clang -c $(A_CPP)) $(QUIET) rm $(A_CPP) .PHONY: test test: $(TEST_DETERMINATOR_RESULT) - $(QUIET)$(call check_no_diff,diff_determinator.json.mod1.exp,infer-out-mod1/diff_determinator.json) - $(QUIET)$(call check_no_diff,diff_determinator.json.mod2.exp,infer-out-mod2/diff_determinator.json) + $(QUIET)$(call check_no_diff,changed_functions.json.mod1.exp,infer-out-mod1/changed_functions.json) + $(QUIET)$(call check_no_diff,changed_functions.json.mod2.exp,infer-out-mod2/changed_functions.json) .PHONY: replace replace: $(TEST_DETERMINATOR_RESULT) - $(COPY) infer-out-mod1/diff_determinator.json diff_determinator.json.mod1.exp - $(COPY) infer-out-mod2/diff_determinator.json diff_determinator.json.mod2.exp + $(COPY) infer-out-mod1/changed_functions.json changed_functions.json.mod1.exp + $(COPY) infer-out-mod2/changed_functions.json changed_functions.json.mod2.exp .PHONY: clean clean: diff --git a/infer/tests/build_systems/export_changed_functions/changed_functions.json.mod1.exp b/infer/tests/build_systems/export_changed_functions/changed_functions.json.mod1.exp new file mode 100644 index 000000000..ea2689dce --- /dev/null +++ b/infer/tests/build_systems/export_changed_functions/changed_functions.json.mod1.exp @@ -0,0 +1 @@ +["Shapes::Cube::area","Shapes::Cube::sort"] \ No newline at end of file diff --git a/infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod2.exp b/infer/tests/build_systems/export_changed_functions/changed_functions.json.mod2.exp similarity index 100% rename from infer/tests/build_systems/clang_test_determinator/diff_determinator.json.mod2.exp rename to infer/tests/build_systems/export_changed_functions/changed_functions.json.mod2.exp diff --git a/infer/tests/build_systems/clang_test_determinator/mod1-A.cpp b/infer/tests/build_systems/export_changed_functions/mod1-A.cpp similarity index 100% rename from infer/tests/build_systems/clang_test_determinator/mod1-A.cpp rename to infer/tests/build_systems/export_changed_functions/mod1-A.cpp diff --git a/infer/tests/build_systems/clang_test_determinator/mod2-A.cpp b/infer/tests/build_systems/export_changed_functions/mod2-A.cpp similarity index 100% rename from infer/tests/build_systems/clang_test_determinator/mod2-A.cpp rename to infer/tests/build_systems/export_changed_functions/mod2-A.cpp diff --git a/infer/tests/build_systems/clang_test_determinator/orig-A.cpp b/infer/tests/build_systems/export_changed_functions/orig-A.cpp similarity index 100% rename from infer/tests/build_systems/clang_test_determinator/orig-A.cpp rename to infer/tests/build_systems/export_changed_functions/orig-A.cpp