From 4ec36904135cc34675205730a926519d895ff2d0 Mon Sep 17 00:00:00 2001
From: Jules Villard <jul@fb.com>
Date: Wed, 22 Mar 2017 11:35:48 -0700
Subject: [PATCH] [tests] add a `replace` target to all tests

Summary:
Most of our tests work by comparing the output of some command to a baseline
expected output. It's often needed to update that baseline.

Previously, that was crudely done by attempting to move every foo.exp.test file
to foo.exp. This does not work terribly well, in particular because
foo.exp.test might be stale.

Instead, add a `replace` target to every test that knows how to update the
baseline. This allows custom behaviours too, eg in the differential tests.

Most of the tests include base.make or differential.make, so add a replace target
there. A few tests are completely custom, add a replace target to them too.

Reviewed By: jeremydubreil

Differential Revision: D4754279

fbshipit-source-id: ec34273
---
 Makefile                                      | 32 ++++++++-----------
 infer/tests/base.make                         |  4 +++
 .../build_systems/fail_on_issue/Makefile      |  4 +++
 infer/tests/build_systems/j1/Makefile         |  4 +--
 .../tests/build_systems/utf8_in_pwd/Makefile  | 12 ++++---
 .../codetoanalyze/java/crashcontext/Makefile  | 10 ++++--
 infer/tests/differential.make                 |  6 ++++
 7 files changed, 45 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile
index 75101db2e..482465f00 100644
--- a/Makefile
+++ b/Makefile
@@ -165,16 +165,11 @@ endif
 ocaml_unit_test: test_build
 	$(call silent_on_success,$(BUILD_DIR)/test/infer/unit/inferunit.byte)
 
-DIRECT_TESTS_REPLACE = $(patsubst %_frontend,%_frontend_replace,$(filter %_frontend,$(DIRECT_TESTS)))
-
 define silence_make
   ($(1) 2> >(grep -v "warning: \(ignoring old\|overriding\) \(commands\|recipe\) for target") \
   ; exit $${PIPESTATUS[0]})
 endef
 
-.PHONY: frontend_replace
-frontend_replace: $(DIRECT_TESTS_REPLACE)
-
 .PHONY: $(DIRECT_TESTS:%=direct_%_test)
 $(DIRECT_TESTS:%=direct_%_test): infer
 	@$(call silence_make,\
@@ -196,6 +191,13 @@ $(DIRECT_TESTS:%=direct_%_clean):
 	  $(INFER_DIR)/tests/codetoanalyze/$(shell printf $@ | cut -f 2 -d _)/$(shell printf $@ | cut -f 3 -d _) \
 	  clean)
 
+.PHONY: $(DIRECT_TESTS:%=direct_%_replace)
+$(DIRECT_TESTS:%=direct_%_replace): infer
+	@$(call silence_make,\
+	$(MAKE) -C \
+	  $(INFER_DIR)/tests/codetoanalyze/$(shell printf $@ | cut -f 2 -d _)/$(shell printf $@ | cut -f 3 -d _) \
+	  replace)
+
 .PHONY: direct_tests
 direct_tests: $(DIRECT_TESTS:%=direct_%_test)
 
@@ -207,9 +209,6 @@ build_genrule_print: build_buck_print
 build_waf_test: build_make_test
 build_waf_print: build_make_print
 
-.PHONY: print_direct_tests
-print_direct_tests: $(DIRECT_TESTS:%=direct_%_print)
-
 .PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 $(BUILD_SYSTEMS_TESTS:%=build_%_test): infer
 	@$(call silence_make,\
@@ -225,14 +224,16 @@ $(BUILD_SYSTEMS_TESTS:%=build_%_clean):
 	@$(call silence_make,\
 	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_clean,%,$@) clean)
 
+.PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_replace)
+$(BUILD_SYSTEMS_TESTS:%=build_%_replace): infer
+	@$(call silence_make,\
+	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_replace,%,$@) replace)
+
 .PHONY: build_systems_tests
 build_systems_tests: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 
-.PHONY: print_build_systems_tests
-print_build_systems_tests: $(BUILD_SYSTEMS_TESTS:%=build_%_print)
-
 .PHONY: endtoend_test
-endtoend_test: print_direct_tests print_build_systems_tests
+endtoend_test: $(BUILD_SYSTEMS_TESTS:%=build_%_print) $(DIRECT_TESTS:%=direct_%_print)
 # pre-compute all the results first so that the test failures show up near the end of the output
 	$(MAKE) direct_tests build_systems_tests
 
@@ -295,12 +296,7 @@ endif
 quick-test: test_build ocaml_unit_test
 
 .PHONY: test-replace
-test-replace:
-	@$(MAKE) -k endtoend_test || true
-	@for file in $$(find $(INFER_DIR)/tests -name "*.exp.test"); do \
-	    mv -f $$file $$(dirname $$file)/$$(basename -s .exp.test $$file).exp; done
-	@for file in $$(find $(INFER_DIR)/tests -name "*.test.dot"); do \
-	    mv -f $$file $$(dirname $$file)/$$(basename -s .test.dot $$file).dot; done
+test-replace: $(BUILD_SYSTEMS_TESTS:%=build_%_replace) $(DIRECT_TESTS:%=direct_%_replace)
 
 .PHONY: uninstall
 uninstall:
diff --git a/infer/tests/base.make b/infer/tests/base.make
index bd21932c2..9d45ea930 100644
--- a/infer/tests/base.make
+++ b/infer/tests/base.make
@@ -33,6 +33,10 @@ test: issues.exp.test$(TEST_SUFFIX)
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test$(TEST_SUFFIX)
 
+.PHONY: print
+replace: issues.exp.test$(TEST_SUFFIX)
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) infer-out$(TEST_SUFFIX) \
diff --git a/infer/tests/build_systems/fail_on_issue/Makefile b/infer/tests/build_systems/fail_on_issue/Makefile
index a20ec747a..3697bc4cd 100644
--- a/infer/tests/build_systems/fail_on_issue/Makefile
+++ b/infer/tests/build_systems/fail_on_issue/Makefile
@@ -43,6 +43,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	rm -rf issues.exp.test infer-out $(OBJECTS)
diff --git a/infer/tests/build_systems/j1/Makefile b/infer/tests/build_systems/j1/Makefile
index 7141223fe..3eae62a52 100644
--- a/infer/tests/build_systems/j1/Makefile
+++ b/infer/tests/build_systems/j1/Makefile
@@ -7,6 +7,6 @@
 
 default: compile
 
-.PHONY: print test clean
-print test clean:
+.PHONY: print replace test clean
+print replace test clean:
 	INFER_ARGS=-j^1 $(MAKE) -C ../../codetoanalyze/cpp/checkers TEST_SUFFIX=-j1 $@
diff --git a/infer/tests/build_systems/utf8_in_pwd/Makefile b/infer/tests/build_systems/utf8_in_pwd/Makefile
index 001cf5bb9..61146395e 100644
--- a/infer/tests/build_systems/utf8_in_pwd/Makefile
+++ b/infer/tests/build_systems/utf8_in_pwd/Makefile
@@ -93,11 +93,11 @@ infer-out-make/report.json: $(CLANG_DEPS) $(MAKE_DIR)
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- \
 	    make clean all)
 
-issues-%.exp.test.noreplace: infer-out-%/report.json $(INFERPRINT_BIN)
+issues-%.exp.test: infer-out-%/report.json $(INFERPRINT_BIN)
 	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
 
-issues.exp.test: issues-cmake.exp.test.noreplace issues-gradle.exp.test.noreplace \
-                 issues-javac.exp.test.noreplace issues-make.exp.test.noreplace
+issues.exp.test: issues-cmake.exp.test issues-gradle.exp.test issues-javac.exp.test \
+                 issues-make.exp.test
 	cat $^ > $@
 
 .PHONY: print
@@ -108,6 +108,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
-	rm -rf $(UTF8_DIR) issues.exp.test* infer-out-*
+	rm -rf $(UTF8_DIR) issues*.exp.test* infer-out-*
diff --git a/infer/tests/codetoanalyze/java/crashcontext/Makefile b/infer/tests/codetoanalyze/java/crashcontext/Makefile
index 7b57e82b6..d8610f694 100644
--- a/infer/tests/codetoanalyze/java/crashcontext/Makefile
+++ b/infer/tests/codetoanalyze/java/crashcontext/Makefile
@@ -15,14 +15,14 @@ ANALYZER = crashcontext
 
 SOURCES = $(wildcard *.java)
 OBJECTS = $(patsubst %.java,%.class,$(SOURCES))
-EXP_TESTs = $(sort $(patsubst %.java,%.exp.test.noreplace,$(SOURCES)))
+EXP_TESTs = $(sort $(patsubst %.java,issues-%.exp.test,$(SOURCES)))
 INFER_OUTs = $(patsubst %.java,infer-out-%,$(SOURCES))
 
 $(OBJECTS): $(SOURCES)
 	$(JAVAC) -cp $(CLASSPATH) $(SOURCES)
 
 # analyze a single source file and generate the test results for it
-%.exp.test.noreplace: $(JAVA_DEPS) %.stacktrace.json %.java
+issues-%.exp.test: $(JAVA_DEPS) %.stacktrace.json %.java
 	$(call silent_on_success,\
 	  $(INFER_BIN) -a $(ANALYZER) -o infer-out-$* --stacktrace $*.stacktrace.json \
 	  -- $(JAVAC) -cp $(CLASSPATH) $*.java)
@@ -31,7 +31,7 @@ $(OBJECTS): $(SOURCES)
 
 # combine the test results for all the source files
 issues.exp.test: $(EXP_TESTs)
-	cat $(EXP_TESTs) > issues.exp.test
+	cat $^ > issues.exp.test
 
 default: compile
 
@@ -49,6 +49,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) $(INFER_OUTs) $(OBJECTS) $(EXP_TESTs)
diff --git a/infer/tests/differential.make b/infer/tests/differential.make
index 5fade819f..cce910c18 100644
--- a/infer/tests/differential.make
+++ b/infer/tests/differential.make
@@ -59,6 +59,12 @@ test: print
 	diff -u fixed.exp fixed.exp.test
 	diff -u preexisting.exp preexisting.exp.test
 
+.PHONY: replace
+replace: $(EXPECTED_TEST_OUTPUT)
+	cp introduced.exp.test introduced.exp
+	cp fixed.exp.test fixed.exp
+	cp preexisting.exp.test preexisting.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) *.exp.test $(INFER_OUT) $(CURRENT_DIR) $(PREVIOUS_DIR) \