Using DB to store modified functions

Reviewed By: jvillard

Differential Revision: D15181951

fbshipit-source-id: 96be170c9

.gitignore

@@ -26,6 +26,7 @@ duplicates.txt
 *.ast.biniou
 /infer/tests/build_systems/buck_flavors_deterministic/capture_hash-*.sha
 /infer/tests/build_systems/buck_flavors_diff/src/hello.c
+/infer/tests/build_systems/buck_test_determinator/diff.mod.test
 /infer/tests/build_systems/clang_compilation_db_escaped/compile_commands.json
 /infer/tests/build_systems/clang_compilation_db_relpath/compile_commands.json
 /infer/tests/build_systems/clang_test_determinator/*.test

Makefile

@@ -45,7 +45,8 @@ BUILD_SYSTEMS_TESTS += \
   run_hidden_linters \
   tracebugs \
   utf8_in_procname \
-	clang_test_determinator \
+  clang_test_determinator \
+  buck_test_determinator \
 
 DIRECT_TESTS += \
   c_biabduction \

infer/src/IR/Attributes.ml

@@ -52,7 +52,7 @@ let replace_statement =
   ResultsDatabase.register_statement
     {|
 INSERT OR REPLACE INTO procedures
-SELECT :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees
+SELECT :pname, :proc_name_hum, :akind, :sfile, :pattr, :cfg, :callees, :modified_flag
 FROM (
   SELECT NULL
   FROM (
@@ -64,7 +64,7 @@ FROM (
         OR (attr_kind = :akind AND source_file <= :sfile) )|}
 
 
-let replace pname pname_blob akind loc_file attr_blob proc_desc callees =
+let replace pname pname_blob akind loc_file attr_blob proc_desc callees modified_flag =
   ResultsDatabase.with_registered_statement replace_statement ~f:(fun db replace_stmt ->
       Sqlite3.bind replace_stmt 1 (* :pname *) pname_blob
       |> SqliteUtils.check_result_code db ~log:"replace bind pname" ;
@@ -81,6 +81,9 @@ let replace pname pname_blob akind loc_file attr_blob proc_desc callees =
       |> SqliteUtils.check_result_code db ~log:"replace bind cfg" ;
       Sqlite3.bind replace_stmt 7 (* :callees *) (Typ.Procname.SQLiteList.serialize callees)
       |> SqliteUtils.check_result_code db ~log:"replace bind callees" ;
+      Sqlite3.bind replace_stmt 8
+        (* :modified_flag *) (Sqlite3.Data.INT (Int64.of_int (if modified_flag then 1 else 0)))
+      |> SqliteUtils.check_result_code db ~log:"replace bind modified_flag" ;
       SqliteUtils.result_unit db ~finalize:false ~log:"Attributes.replace" replace_stmt )
 
 
@@ -137,6 +140,7 @@ let store ~proc_desc (attr : ProcAttributes.t) =
       (ProcAttributes.SQLite.serialize attr)
       proc_desc
       (Option.map proc_desc ~f:Procdesc.get_static_callees |> Option.value ~default:[])
+      false
 
 
 let load_defined pname = Typ.Procname.SQLite.serialize pname |> find ~defined:true

infer/src/backend/InferPrint.ml

@@ -1235,6 +1235,7 @@ let register_perf_stats_report () =
 
 let main ~report_json =
   if Config.test_determinator_clang then (
+    TestDeterminator.persist_relevant_method_in_db () ;
     TestDeterminator.emit_tests_to_run () ;
     TestDeterminator.emit_relevant_methods () ) ;
   let issue_formats = init_issues_format_list report_json in

infer/src/base/MergeResults.ml

@@ -16,7 +16,7 @@ let merge_procedures_table ~db_file =
   Sqlite3.exec db
     {|
 INSERT OR REPLACE INTO procedures
-SELECT sub.proc_name, sub.proc_name_hum, sub.attr_kind, sub.source_file, sub.proc_attributes, sub.cfg, sub.callees
+SELECT sub.proc_name, sub.proc_name_hum, sub.attr_kind, sub.source_file, sub.proc_attributes, sub.cfg, sub.callees, sub.modified_flag
 FROM (
   attached.procedures AS sub
   LEFT OUTER JOIN procedures AS main

infer/src/base/ResultsDatabase.ml

@@ -21,6 +21,7 @@ let procedures_schema =
   , proc_attributes BLOB NOT NULL
   , cfg BLOB
   , callees BLOB NOT NULL
+  , modified_flag INT NOT NULL DEFAULT 0
   )|}
 
 

infer/src/clang/cFrontend.ml

@@ -52,8 +52,6 @@ let do_source_file (translation_unit_context : CFrontend_config.translation_unit
   let cfg = compute_icfg translation_unit_context tenv ast in
   L.(debug Capture Verbose)
     "@\n End building call/cfg graph for '%a'.@\n" SourceFile.pp source_file ;
-  if Config.test_determinator_clang then
-    TestDeterminator.test_to_run_clang source_file cfg Config.modified_lines None ;
   (* This part below is a boilerplate in every frontends. *)
   (* This could be moved in the cfg_infer module *)
   NullabilityPreanalysis.analysis cfg tenv ;
@@ -64,6 +62,9 @@ let do_source_file (translation_unit_context : CFrontend_config.translation_unit
     || Option.is_some Config.icfg_dotty_outfile
   then Dotty.print_icfg_dotty source_file cfg ;
   L.(debug Capture Verbose) "Stored on disk:@[<v>%a@]@." Cfg.pp_proc_signatures cfg ;
+  if Config.test_determinator_clang then (
+    TestDeterminator.test_to_run_clang source_file cfg Config.modified_lines None ;
+    TestDeterminator.persist_relevant_method_in_db () ) ;
   let procedures_translated_summary =
     EventLogger.ProceduresTranslatedSummary
       { procedures_translated_total= !CFrontend_config.procedures_attempted

infer/src/integration/testDeterminator.ml

@@ -10,6 +10,36 @@ module L = Logging
 module F = Format
 module JPS = JavaProfilerSamples
 
+let load_all_statement =
+  ResultsDatabase.register_statement
+    "SELECT DISTINCT proc_name_hum FROM procedures WHERE modified_flag = 1 ORDER BY proc_name_hum"
+
+
+let load_all () =
+  ResultsDatabase.with_registered_statement load_all_statement ~f:(fun db stmt ->
+      Container.to_list stmt
+        ~fold:
+          (SqliteUtils.result_fold_single_column_rows ~finalize:false db
+             ~log:"modified_flag.load_all")
+      |> List.map ~f:Sqlite3.Data.to_string )
+
+
+let store_statement =
+  ResultsDatabase.register_statement
+    "UPDATE procedures SET modified_flag = :i WHERE proc_name = :k"
+
+
+let store pname modified =
+  let modified' = Sqlite3.Data.INT (Int64.of_int (if modified then 1 else 0)) in
+  ResultsDatabase.with_registered_statement store_statement ~f:(fun db stmt ->
+      Typ.Procname.SQLite.serialize pname
+      |> Sqlite3.bind stmt 2
+      |> SqliteUtils.check_result_code db ~log:"store bind proc name" ;
+      modified' |> Sqlite3.bind stmt 1
+      |> SqliteUtils.check_result_code db ~log:"store bind modified flag " ;
+      SqliteUtils.result_unit db ~finalize:false ~log:"store modified flag " stmt )
+
+
 (* a flag used to make the method search signature sensitive *)
 let use_method_signature = false
 
@@ -213,7 +243,7 @@ let compute_affected_methods_clang source_file changed_lines_map method_range_ma
 let relevant_tests = ref []
 
 (* Methods modified in a diff *)
-let relevant_methods = ref []
+let relevant_methods = ref JPS.ProfilerSample.empty
 
 let _get_relevant_test_to_run () = !relevant_tests
 
@@ -221,17 +251,25 @@ let emit_tests_to_run () =
   let json = `List (List.map ~f:(fun t -> `String t) !relevant_tests) in
   let outpath = Config.results_dir ^/ "test_determinator.json" in
   Yojson.Basic.to_file outpath json ;
-  L.progress "Tests to run: [%a]@\n" (Pp.seq ~sep:", " F.pp_print_string) !relevant_tests
+  L.(debug TestDeterminator Medium)
+    "Tests to run: [%a]@\n"
+    (Pp.seq ~sep:", " F.pp_print_string)
+    !relevant_tests
+
+
+let persist_relevant_method_in_db () =
+  JPS.ProfilerSample.iter (fun pname -> store pname true) !relevant_methods
 
 
 let emit_relevant_methods () =
-  let methods = List.dedup_and_sort ~compare:String.compare !relevant_methods in
-  let json = `List (List.map ~f:(fun t -> `String t) methods) in
+  let rel_methods = load_all () in
+  let json = `List (List.map ~f:(fun t -> `String t) rel_methods) in
   let outpath = Config.results_dir ^/ "diff_determinator.json" in
   Yojson.Basic.to_file outpath json ;
-  L.progress "Methods modified in this Diff: [%a]@\n"
+  L.(debug TestDeterminator Medium)
+    "Methods modified in this Diff: %a@\n"
     (Pp.seq ~sep:", " F.pp_print_string)
-    !relevant_methods
+    rel_methods
 
 
 let init_clang cfg changed_lines_file test_samples_file =
@@ -268,10 +306,7 @@ let test_to_run_clang source_file cfg changed_lines_file test_samples_file =
     compute_affected_methods_clang source_file (DiffLines.changed_lines_map ())
       (MethodRangeMap.method_range_map ())
   in
-  let affected_methods_list =
-    JPS.ProfilerSample.fold (fun m acc -> Typ.Procname.to_string m :: acc) affected_methods []
-  in
-  relevant_methods := List.append affected_methods_list !relevant_methods ;
+  relevant_methods := JPS.ProfilerSample.union affected_methods !relevant_methods ;
   let test_to_run =
     if JPS.ProfilerSample.is_empty affected_methods then []
     else

infer/src/integration/testDeterminator.mli

@@ -14,6 +14,8 @@ val test_to_run_clang :
 
 val emit_tests_to_run : unit -> unit
 
+val persist_relevant_method_in_db : unit -> unit
+
 val emit_relevant_methods : unit -> unit
 
 val _get_relevant_test_to_run : unit -> string list

infer/tests/build_systems/buck_test_determinator/.buckversion

@@ -0,0 +1 @@
+../../../../.buckversion

infer/tests/build_systems/buck_test_determinator/Makefile

@@ -0,0 +1,40 @@
+# Copyright (c) 2019-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+TESTS_DIR = ../..
+include $(TESTS_DIR)/base.make
+
+A_CPP = src/a.cpp
+B_CPP = src/b.cpp
+BUCK_TARGET = //src:test
+TEST_DETERMINATOR_RESULT = infer-out/diff_determinator.json
+DIFF_OUTPUT = diff.mod.test
+INFER_OPTIONS = --flavors --no-linters --test-determinator-clang --modified-lines $(DIFF_OUTPUT) --project-root $(TESTS_DIR)
+
+$(DIFF_OUTPUT):
+	$(QUIET)echo -n '$(A_CPP):' > diff.mod.test
+	$(QUIET)(diff -N --unchanged-line-format="U" --old-line-format="O" --new-line-format="N" \
+		$(A_CPP) src/mod-a.cpp || [ $$? = 1 ]) >> diff.mod.test
+	$(QUIET)echo >> diff.mod.test
+	$(QUIET)echo -n '$(B_CPP):' >> diff.mod.test
+	$(QUIET)(diff -N --unchanged-line-format="U" --old-line-format="O" --new-line-format="N" \
+		$(B_CPP) src/mod-b.cpp || [ $$? = 1 ]) >> diff.mod.test
+	$(QUIET)echo >> diff.mod.test
+
+$(TEST_DETERMINATOR_RESULT): $(DIFF_OUTPUT)
+	$(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes,\
+		$(INFER_BIN) $(INFER_OPTIONS) -- buck build --no-cache $(BUCK_TARGET))
+
+.PHONY: test
+test: $(TEST_DETERMINATOR_RESULT)
+	$(QUIET)$(call check_no_diff,diff_determinator.json.exp,$(TEST_DETERMINATOR_RESULT))
+
+.PHONY: replace
+replace: $(TEST_DETERMINATOR_RESULT)
+	$(COPY) $(TEST_DETERMINATOR_RESULT) diff_determinator.json.exp
+
+.PHONY: clean
+clean:
+	$(REMOVE_DIR) *.test infer-out* buck-out*

infer/tests/build_systems/buck_test_determinator/diff_determinator.json.exp

@@ -0,0 +1 @@
+["B::B","test"]

infer/tests/build_systems/buck_test_determinator/src/BUCK

@@ -0,0 +1,6 @@
+cxx_library(
+  name = 'test',
+  srcs = [
+    'a.cpp', 'b.cpp',
+  ],
+)

infer/tests/build_systems/buck_test_determinator/src/a.cpp

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2019-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <stdlib.h>
+
+void test() {
+  int* s = NULL;
+  *s = 42;
+}

infer/tests/build_systems/buck_test_determinator/src/b.cpp

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2019-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+class B {
+  int val;
+
+ public:
+  B(int v) { val = v; }
+};

infer/tests/build_systems/buck_test_determinator/src/mod-a.cpp

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2019-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <stdlib.h>
+
+/* comment */
+void test() {
+  int* s = NULL;
+  *s = 42 + 1;
+}

infer/tests/build_systems/buck_test_determinator/src/mod-b.cpp

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2019-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+class B {
+  int val;
+
+ public:
+  B(int v) { val = v + 42; }
+};

infer/tests/build_systems/clang_test_determinator/Makefile

@@ -23,7 +23,7 @@ $(DIFF_OUTPUT):
 		orig-$(A_CPP) mod2-$(A_CPP) || [ $$? = 1 ]) >> diff.mod2.test
 
 $(TEST_DETERMINATOR_RESULT): $(DIFF_OUTPUT)
-	$(QUIET)$(call silent_on_success,Testing test-determinator with set of changes in mod1,\
+	$(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes in mod1,\
 		cp mod1-$(A_CPP) $(A_CPP);\
 		$(INFER_BIN) -o infer-out-mod1 --test-determinator-clang --modified-lines diff.mod1.test -- clang -c $(A_CPP))
 	$(QUIET)$(call silent_on_success,Testing test-determinator-clang with set of changes in mod2,\