[tests] add a `replace` target to all tests

Summary:
Most of our tests work by comparing the output of some command to a baseline
expected output. It's often needed to update that baseline.

Previously, that was crudely done by attempting to move every foo.exp.test file
to foo.exp. This does not work terribly well, in particular because
foo.exp.test might be stale.

Instead, add a `replace` target to every test that knows how to update the
baseline. This allows custom behaviours too, eg in the differential tests.

Most of the tests include base.make or differential.make, so add a replace target
there. A few tests are completely custom, add a replace target to them too.

Reviewed By: jeremydubreil

Differential Revision: D4754279

fbshipit-source-id: ec34273

Makefile

@@ -165,16 +165,11 @@ endif
 ocaml_unit_test: test_build
 	$(call silent_on_success,$(BUILD_DIR)/test/infer/unit/inferunit.byte)
 
-DIRECT_TESTS_REPLACE = $(patsubst %_frontend,%_frontend_replace,$(filter %_frontend,$(DIRECT_TESTS)))
-
 define silence_make
   ($(1) 2> >(grep -v "warning: \(ignoring old\|overriding\) \(commands\|recipe\) for target") \
   ; exit $${PIPESTATUS[0]})
 endef
 
-.PHONY: frontend_replace
-frontend_replace: $(DIRECT_TESTS_REPLACE)
-
 .PHONY: $(DIRECT_TESTS:%=direct_%_test)
 $(DIRECT_TESTS:%=direct_%_test): infer
 	@$(call silence_make,\
@@ -196,6 +191,13 @@ $(DIRECT_TESTS:%=direct_%_clean):
 	  $(INFER_DIR)/tests/codetoanalyze/$(shell printf $@ | cut -f 2 -d _)/$(shell printf $@ | cut -f 3 -d _) \
 	  clean)
 
+.PHONY: $(DIRECT_TESTS:%=direct_%_replace)
+$(DIRECT_TESTS:%=direct_%_replace): infer
+	@$(call silence_make,\
+	$(MAKE) -C \
+	  $(INFER_DIR)/tests/codetoanalyze/$(shell printf $@ | cut -f 2 -d _)/$(shell printf $@ | cut -f 3 -d _) \
+	  replace)
+
 .PHONY: direct_tests
 direct_tests: $(DIRECT_TESTS:%=direct_%_test)
 
@@ -207,9 +209,6 @@ build_genrule_print: build_buck_print
 build_waf_test: build_make_test
 build_waf_print: build_make_print
 
-.PHONY: print_direct_tests
-print_direct_tests: $(DIRECT_TESTS:%=direct_%_print)
-
 .PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 $(BUILD_SYSTEMS_TESTS:%=build_%_test): infer
 	@$(call silence_make,\
@@ -225,14 +224,16 @@ $(BUILD_SYSTEMS_TESTS:%=build_%_clean):
 	@$(call silence_make,\
 	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_clean,%,$@) clean)
 
+.PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_replace)
+$(BUILD_SYSTEMS_TESTS:%=build_%_replace): infer
+	@$(call silence_make,\
+	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_replace,%,$@) replace)
+
 .PHONY: build_systems_tests
 build_systems_tests: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 
-.PHONY: print_build_systems_tests
-print_build_systems_tests: $(BUILD_SYSTEMS_TESTS:%=build_%_print)
-
 .PHONY: endtoend_test
-endtoend_test: print_direct_tests print_build_systems_tests
+endtoend_test: $(BUILD_SYSTEMS_TESTS:%=build_%_print) $(DIRECT_TESTS:%=direct_%_print)
 # pre-compute all the results first so that the test failures show up near the end of the output
 	$(MAKE) direct_tests build_systems_tests
 
@@ -295,12 +296,7 @@ endif
 quick-test: test_build ocaml_unit_test
 
 .PHONY: test-replace
-test-replace:
+test-replace: $(BUILD_SYSTEMS_TESTS:%=build_%_replace) $(DIRECT_TESTS:%=direct_%_replace)
-	@$(MAKE) -k endtoend_test || true
-	@for file in $$(find $(INFER_DIR)/tests -name "*.exp.test"); do \
-	    mv -f $$file $$(dirname $$file)/$$(basename -s .exp.test $$file).exp; done
-	@for file in $$(find $(INFER_DIR)/tests -name "*.test.dot"); do \
-	    mv -f $$file $$(dirname $$file)/$$(basename -s .test.dot $$file).dot; done
 
 .PHONY: uninstall
 uninstall:

infer/tests/base.make

@@ -33,6 +33,10 @@ test: issues.exp.test$(TEST_SUFFIX)
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test$(TEST_SUFFIX)
 
+.PHONY: print
+replace: issues.exp.test$(TEST_SUFFIX)
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) infer-out$(TEST_SUFFIX) \

infer/tests/build_systems/fail_on_issue/Makefile

@@ -43,6 +43,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	rm -rf issues.exp.test infer-out $(OBJECTS)

infer/tests/build_systems/j1/Makefile

@@ -7,6 +7,6 @@
 
 default: compile
 
-.PHONY: print test clean
+.PHONY: print replace test clean
-print test clean:
+print replace test clean:
 	INFER_ARGS=-j^1 $(MAKE) -C ../../codetoanalyze/cpp/checkers TEST_SUFFIX=-j1 $@

infer/tests/build_systems/utf8_in_pwd/Makefile

@@ -93,11 +93,11 @@ infer-out-make/report.json: $(CLANG_DEPS) $(MAKE_DIR)
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- \
 	    make clean all)
 
-issues-%.exp.test.noreplace: infer-out-%/report.json $(INFERPRINT_BIN)
+issues-%.exp.test: infer-out-%/report.json $(INFERPRINT_BIN)
 	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
 
-issues.exp.test: issues-cmake.exp.test.noreplace issues-gradle.exp.test.noreplace \
+issues.exp.test: issues-cmake.exp.test issues-gradle.exp.test issues-javac.exp.test \
-                 issues-javac.exp.test.noreplace issues-make.exp.test.noreplace
+                 issues-make.exp.test
 	cat $^ > $@
 
 .PHONY: print
@@ -108,6 +108,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
-	rm -rf $(UTF8_DIR) issues.exp.test* infer-out-*
+	rm -rf $(UTF8_DIR) issues*.exp.test* infer-out-*

infer/tests/codetoanalyze/java/crashcontext/Makefile

@@ -15,14 +15,14 @@ ANALYZER = crashcontext
 
 SOURCES = $(wildcard *.java)
 OBJECTS = $(patsubst %.java,%.class,$(SOURCES))
-EXP_TESTs = $(sort $(patsubst %.java,%.exp.test.noreplace,$(SOURCES)))
+EXP_TESTs = $(sort $(patsubst %.java,issues-%.exp.test,$(SOURCES)))
 INFER_OUTs = $(patsubst %.java,infer-out-%,$(SOURCES))
 
 $(OBJECTS): $(SOURCES)
 	$(JAVAC) -cp $(CLASSPATH) $(SOURCES)
 
 # analyze a single source file and generate the test results for it
-%.exp.test.noreplace: $(JAVA_DEPS) %.stacktrace.json %.java
+issues-%.exp.test: $(JAVA_DEPS) %.stacktrace.json %.java
 	$(call silent_on_success,\
 	  $(INFER_BIN) -a $(ANALYZER) -o infer-out-$* --stacktrace $*.stacktrace.json \
 	  -- $(JAVAC) -cp $(CLASSPATH) $*.java)
@@ -31,7 +31,7 @@ $(OBJECTS): $(SOURCES)
 
 # combine the test results for all the source files
 issues.exp.test: $(EXP_TESTs)
-	cat $(EXP_TESTs) > issues.exp.test
+	cat $^ > issues.exp.test
 
 default: compile
 
@@ -49,6 +49,10 @@ test: issues.exp.test
 	@cd $(TESTS_DIR) && \
 	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
 
+.PHONY: replace
+replace: issues.exp.test
+	cp $< issues.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) $(INFER_OUTs) $(OBJECTS) $(EXP_TESTs)

infer/tests/differential.make

@@ -59,6 +59,12 @@ test: print
 	diff -u fixed.exp fixed.exp.test
 	diff -u preexisting.exp preexisting.exp.test
 
+.PHONY: replace
+replace: $(EXPECTED_TEST_OUTPUT)
+	cp introduced.exp.test introduced.exp
+	cp fixed.exp.test fixed.exp
+	cp preexisting.exp.test preexisting.exp
+
 .PHONY: clean
 clean:
 	$(REMOVE_DIR) *.exp.test $(INFER_OUT) $(CURRENT_DIR) $(PREVIOUS_DIR) \