From 3ab6c43fb61ed42c930e0e0fcedabcb4a39e67a1 Mon Sep 17 00:00:00 2001
From: Jules Villard <jul@fb.com>
Date: Tue, 28 Mar 2017 10:03:50 -0700
Subject: [PATCH] [tests] awesomize make output

Summary:
Hide the output of all the toplevel `make` actions and print messages instead, as if we had a better build system than Makefiles. When a command fails, print its stdout and stderr and prefix it with bold red letters. Also display timing information.

```
$ make -j 4 test
[17:39:18][27290] Facebook setup...
[17:39:18][27363] Building checkCopyright...
[17:39:18][27443] Validating facebook/...
[      0s][27363] SUCCESS Building checkCopyright
[      0s][27290] SUCCESS Facebook setup
[17:39:18][27863] Building clang plugin...
[      0s][27443] SUCCESS Validating facebook/
[      0s][27863] SUCCESS Building clang plugin
[17:39:18][27898] Building clang plugin OCaml interface...
[      0s][27898] SUCCESS Building clang plugin OCaml interface
[17:39:18][27974] Testing Infer builds without warnings...
[17:39:18][28033] Building native Infer...
[     12s][27974] SUCCESS Testing Infer builds without warnings
[17:39:30][28998] Testing Infer toplevel builds...
[      6s][28998] SUCCESS Testing Infer toplevel builds
[17:39:36][31417] Running OCaml unit tests...
[17:39:36][31456] Testing infer OCaml REPL...
[      6s][31417] SUCCESS Running OCaml unit tests
[      9s][31456] SUCCESS Testing infer OCaml REPL
[     92s][28033] SUCCESS Building native Infer
[17:40:50][ 2170] Building Java annotations...
[      0s][ 2170] SUCCESS Building Java annotations
[17:40:50][ 2186] Building Infer models...
[     11s][ 2186] SUCCESS Building Infer models
[17:41:01][ 2803] Running build assembly test...
[17:41:01][ 2861] Running build ck analytics test...
[17:41:01][ 2941] Running build ck imports test...
[17:41:01][ 3035] Running build clang compilation db escaped test...
[      1s][ 3035] SUCCESS Running build clang compilation db escaped test
[17:41:02][ 3139] Running build clang compilation db relpath test...
[      3s][ 2861] SUCCESS Running build ck analytics test
[17:41:04][ 3296] Running build clang multiple files test...
[*ERROR**][2803] *** ERROR Running build assembly test
[*ERROR**][2803] *** command:    ( make INTERACTIVE=1 -C /home/jul/infer/infer/tests/build_systems/assembly test 2> >(grep -v warning: (ignoring old|overriding) (commands|recipe) for target) ; exit 0)
[*ERROR**][2803] *** CWD: /home/jul/infer
[*ERROR**][2803] *** stdout:
[*ERROR**][2803] make[1]: Entering directory '/home/jul/infer/infer/tests/build_systems/assembly'
[*ERROR**][2803] *** inner Testing infer/clang in build_systems/assembly
[*ERROR**][2803] *** inner command:  /home/jul/infer/infer/bin/infer --results-dir infer-out --dump-duplicate-symbols --report-custom-error --developer-mode --project-root ../codetoanalyze --no-failures-allowed -a infer -- clang -c ../codetoanalyze/example.S ../codetoanalyze/hello.c
[*ERROR**][2803] *** inner CWD: /home/jul/infer/infer/tests/build_systems/assembly
[*ERROR**][2803] Capturing in make/cc mode...
[*ERROR**][2803] Starting analysis...
[*ERROR**][2803]
[*ERROR**][2803] legend:
[*ERROR**][2803]   "F" analyzing a file
[*ERROR**][2803]   "." analyzing a procedure
[*ERROR**][2803]
[*ERROR**][2803] Found 1 (out of 1) source files to be analyzed in /home/jul/infer/infer/tests/build_systems/assembly/infer-out
[*ERROR**][2803]
[*ERROR**][2803]
[*ERROR**][2803]   No issues found
[*ERROR**][2803]
[*ERROR**][2803] ../../infer.make:30: recipe for target 'test' failed
[*ERROR**][2803] make[1]: Leaving directory '/home/jul/infer/infer/tests/build_systems/assembly'
[*ERROR**][2803] F.--- build_systems/assembly/issues.exp        2017-03-22 16:25:44.583854270 +0000
[*ERROR**][2803] +++ build_systems/assembly/issues.exp.test     2017-03-28 17:41:04.743678254 +0100
[*ERROR**][2803] @@ -1 +0,0 @@
[*ERROR**][2803] -hello.c, test, 2, NULL_DEREFERENCE, [start of procedure test()]
[*ERROR**][2803]
[*ERROR**][2803] Test output (build_systems/assembly/issues.exp.test) differs from expected test output build_systems/assembly/issues.exp
[*ERROR**][2803] Run the following command to replace the expected test output with the new output:
[*ERROR**][2803]
[*ERROR**][2803]   make -C build_systems/assembly replace
[*ERROR**][2803]
[*ERROR**][2803] make[1]: *** [test] Error 1
[*ERROR**][2803] *** stderr:
Makefile:230: recipe for target 'build_assembly_test' failed
make: *** [build_assembly_test] Error 1
make: *** Waiting for unfinished jobs....
[...]
```

Reviewed By: jberdine

Differential Revision: D4781857

fbshipit-source-id: cbce26d
---
 .gitignore                                    |   1 +
 Makefile                                      | 123 +++++++++++-------
 Makefile.autoconf.in                          |   1 +
 Makefile.config                               |  71 +++++++++-
 infer/models/c/Makefile                       |   4 +-
 infer/models/cpp/Makefile                     |   4 +-
 infer/models/java/Makefile                    |   4 +-
 infer/models/objc/Makefile                    |   2 +-
 infer/tests/base.make                         |  47 +++----
 infer/tests/build_systems/ant/Makefile        |   4 +-
 .../build_systems/buck-clang-db/Makefile      |   6 +-
 infer/tests/build_systems/buck/Makefile       |  10 +-
 .../clang_compilation_db/Makefile             |  30 +++--
 .../clang_compilation_db_escaped/Makefile     |  10 +-
 .../clang_compilation_db_relpath/Makefile     |  10 +-
 .../build_systems/clang_translation/Makefile  |  15 ++-
 infer/tests/build_systems/cmake/Makefile      |  12 +-
 .../build_systems/codetoanalyze/make/Makefile |   2 +-
 .../build_systems/delete_results_dir/Makefile |   8 +-
 .../Makefile                                  |  10 +-
 .../Makefile                                  |  10 +-
 .../Makefile                                  |  12 +-
 .../Makefile                                  |  10 +-
 .../build_systems/fail_on_issue/Makefile      |  13 +-
 infer/tests/build_systems/genrule/Makefile    |  14 +-
 infer/tests/build_systems/gradle/Makefile     |   6 +-
 infer/tests/build_systems/j1/Makefile         |   5 +-
 infer/tests/build_systems/make/Makefile       |   4 +-
 infer/tests/build_systems/mvn/Makefile        |  16 +--
 infer/tests/build_systems/ndk_build/Makefile  |   6 +-
 infer/tests/build_systems/reactive/Makefile   |  10 +-
 .../build_systems/utf8_in_procname/Makefile   |  20 +--
 .../tests/build_systems/utf8_in_pwd/Makefile  |  62 +++++----
 infer/tests/build_systems/waf/Makefile        |   6 +-
 infer/tests/build_systems/xcodebuild/Makefile |   9 +-
 infer/tests/clang-frontend.make               |   6 +-
 infer/tests/clang.make                        |   6 +-
 .../codetoanalyze/java/crashcontext/Makefile  |  12 +-
 .../tests/codetoanalyze/objc/errors/Makefile  |  20 +--
 infer/tests/differential.make                 |  20 +--
 infer/tests/infer.make                        |  40 ++++++
 infer/tests/javac.make                        |   4 +-
 42 files changed, 405 insertions(+), 280 deletions(-)
 create mode 100644 infer/tests/infer.make

diff --git a/.gitignore b/.gitignore
index 5c4dac4f2..78f8e0a1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@
 
 # generated by build and tests
 /_build
+/_build_logs
 /infer/_build
 /infer/tests/codetoanalyze/java/*/codetoanalyze
 /infer-deps-*
diff --git a/Makefile b/Makefile
index 95b68fa08..39cfece4b 100644
--- a/Makefile
+++ b/Makefile
@@ -90,37 +90,60 @@ endif
 all: infer
 
 configure: configure.ac $(wildcard m4/*.m4)
+#	rerun ./autogen.sh in case of failure as the failure may be due to needing to rerun
+#	./configure
+	$(QUIET)($(call silent_on_success,Generate ./configure,./autogen.sh)) || \
 	./autogen.sh
 
 Makefile.autoconf: configure Makefile.autoconf.in
 #	rerun ./configure with the flags that were used last time it was run (if available)
+#	retry in case of failure as the failure may be due to needing to rerun ./configure
+	$(QUIET)($(call silent_on_success,Running\
+	./configure $(shell ./config.status --config || true),\
+	./configure $(shell ./config.status --config || true))) || \
 	./configure $(shell ./config.status --config || true)
 
+.PHONY: fb-setup
+fb-setup:
+	$(QUIET)$(call silent_on_success,Facebook setup,\
+	$(MAKE) -C facebook setup)
+
 .PHONY: src_build
-src_build: $(MAKEFILE_LIST)
-ifeq ($(IS_FACEBOOK_TREE),yes)
-	$(QUIET)$(MAKE) -C facebook setup
-endif
-	$(QUIET)$(MAKE) -C $(SRC_DIR) infer
+src_build:
+	$(QUIET)$(call silent_on_success,Building native Infer,\
+	$(MAKE) -C $(SRC_DIR) infer)
 
 .PHONY: byte
 byte:
+	$(QUIET)$(call silent_on_success,Building byte Infer,\
+	$(MAKE) -C $(SRC_DIR) byte)
+
+.PHONY: test_build
+test_build:
+	$(QUIET)$(call silent_on_success,Testing Infer builds without warnings,\
+	$(MAKE) -C $(SRC_DIR) TEST=1 byte_no_install)
+#	byte_no_install builds most of what toplevel needs, so it's more efficient to run the
+#	toplevel build straight after it rather than in parallel. Note that both targets build files
+#	that the other doesn't.
+	$(QUIET)$(call silent_on_success,Testing Infer toplevel builds,\
+	$(MAKE) -C $(SRC_DIR) TEST=1 toplevel)
+
 ifeq ($(IS_FACEBOOK_TREE),yes)
-	$(QUIET)$(MAKE) -C facebook setup
+byte src_build test_build: fb-setup
 endif
-	$(QUIET)$(MAKE) -C $(SRC_DIR) byte
 
 ifeq ($(BUILD_C_ANALYZERS),yes)
-src_build: clang_plugin
-byte: clang_plugin
+byte src_build test_build: clang_plugin
 endif
 
 .PHONY: infer
 infer: src_build
 ifeq ($(BUILD_JAVA_ANALYZERS),yes)
-	$(QUIET)$(MAKE) -C $(ANNOTATIONS_DIR)
+	$(QUIET)$(call silent_on_success,Building Java annotations,\
+	$(MAKE) -C $(ANNOTATIONS_DIR))
 endif
-	$(QUIET)$(MAKE) -C $(MODELS_DIR) all
+	$(QUIET)$(call silent_on_success,Building Infer models,\
+	$(MAKE) -C $(MODELS_DIR) all)
 
 .PHONY: clang_setup
 clang_setup:
@@ -133,37 +156,29 @@ clang_setup:
 .PHONY: clang_plugin
 clang_plugin: clang_setup
 ifeq ($(IS_RELEASE_TREE),no)
-	$(QUIET)$(MAKE) -C $(FCP_DIR)/libtooling all \
+	$(QUIET)$(call silent_on_success,Building clang plugin,\
+	$(MAKE) -C $(FCP_DIR)/libtooling all \
 	  CC=$(CC) CXX=$(CXX) \
 	  CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
 	  CPP="$(CPP)" LDFLAGS="$(LDFLAGS)" LIBS="$(LIBS)" \
 	  LOCAL_CLANG=$(CLANG_PREFIX)/bin/clang \
 	  CLANG_PREFIX=$(CLANG_PREFIX) \
-	  CLANG_INCLUDES=$(CLANG_INCLUDES)
-	$(QUIET)$(MAKE) -C $(FCP_DIR)/clang-ocaml all \
+	  CLANG_INCLUDES=$(CLANG_INCLUDES))
+	$(QUIET)$(call silent_on_success,Building clang plugin OCaml interface,\
+	$(MAKE) -C $(FCP_DIR)/clang-ocaml all \
           build/clang_ast_proj.ml build/clang_ast_proj.mli \
 	  CC=$(CC) CXX=$(CXX) \
 	  CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
 	  CPP="$(CPP)" LDFLAGS="$(LDFLAGS)" LIBS="$(LIBS)" \
 	  LOCAL_CLANG=$(CLANG_PREFIX)/bin/clang \
 	  CLANG_PREFIX=$(CLANG_PREFIX) \
-	  CLANG_INCLUDES=$(CLANG_INCLUDES)
+	  CLANG_INCLUDES=$(CLANG_INCLUDES))
 endif
 
-.PHONY: test_build
-test_build: clang_plugin
-ifeq ($(IS_FACEBOOK_TREE),yes)
-	$(QUIET)$(MAKE) -C facebook setup
-endif
-	$(QUIET)$(MAKE) -C $(SRC_DIR) TEST=1 byte_no_install
-#	byte_no_install builds most of what toplevel needs, so it's more efficient to run the
-#	toplevel build straight after it rather than in parallel. Note that both targets build files
-#	that the other doesn't.
-	$(QUIET)$(MAKE) -C $(SRC_DIR) TEST=1 toplevel
-
 .PHONY: ocaml_unit_test
 ocaml_unit_test: test_build
-	$(call silent_on_success,$(BUILD_DIR)/test/infer/unit/inferunit.byte)
+	$(QUIET)$(call silent_on_success,Running OCaml unit tests,\
+	$(BUILD_DIR)/test/infer/unit/inferunit.byte)
 
 define silence_make
   ($(1) 2> >(grep -v "warning: \(ignoring old\|overriding\) \(commands\|recipe\) for target") \
@@ -172,10 +187,11 @@ endef
 
 .PHONY: $(DIRECT_TESTS:%=direct_%_test)
 $(DIRECT_TESTS:%=direct_%_test): infer
-	$(QUIET)$(call silence_make,\
+	$(QUIET)$(call silent_on_success,Running $(subst _, ,$@),\
+	$(call silence_make,\
 	$(MAKE) -C \
 	  $(INFER_DIR)/tests/codetoanalyze/$(shell printf $@ | cut -f 2 -d _)/$(shell printf $@ | cut -f 3 -d _) \
-	  test)
+	  test))
 
 .PHONY: $(DIRECT_TESTS:%=direct_%_print)
 $(DIRECT_TESTS:%=direct_%_print): infer
@@ -211,8 +227,9 @@ build_waf_print: build_make_print
 
 .PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 $(BUILD_SYSTEMS_TESTS:%=build_%_test): infer
-	$(QUIET)$(call silence_make,\
-	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_test,%,$@) test)
+	$(QUIET)$(call silent_on_success,Running $(subst _, ,$@),\
+	$(call silence_make,\
+	$(MAKE) -C $(INFER_DIR)/tests/build_systems/$(patsubst build_%_test,%,$@) test))
 
 .PHONY: $(BUILD_SYSTEMS_TESTS:%=build_%_print)
 $(BUILD_SYSTEMS_TESTS:%=build_%_print): infer
@@ -233,31 +250,33 @@ $(BUILD_SYSTEMS_TESTS:%=build_%_replace): infer
 build_systems_tests: $(BUILD_SYSTEMS_TESTS:%=build_%_test)
 
 .PHONY: endtoend_test
-endtoend_test: $(BUILD_SYSTEMS_TESTS:%=build_%_print) $(DIRECT_TESTS:%=direct_%_print)
-# pre-compute all the results first so that the test failures show up near the end of the output
-	$(MAKE) direct_tests build_systems_tests
+endtoend_test: $(BUILD_SYSTEMS_TESTS:%=build_%_test) $(DIRECT_TESTS:%=direct_%_test)
 
 .PHONY: inferTraceBugs_test
 inferTraceBugs_test: infer
 ifeq ($(BUILD_JAVA_ANALYZERS),yes)
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: running infer,\
 	$(INFER_BIN) -o __test-infer-out__ -- \
-	  $(JAVAC) $(EXAMPLES_DIR)/Hello.java \
-	   > /dev/null
+	  $(JAVAC) $(EXAMPLES_DIR)/Hello.java)
 else
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: running infer,\
 	$(INFER_BIN) -o __test-infer-out__ -- \
-	  clang -c $(EXAMPLES_DIR)/hello.c \
-	   > /dev/null
+	  clang -c $(EXAMPLES_DIR)/hello.c)
 endif
-	$(QUIET)rm -f Hello.class
+	$(QUIET)$(REMOVE) Hello.class
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: --max-level=max,\
 	$(PYTHON_DIR)/inferTraceBugs -o __test-infer-out__ \
-	  --select 0 --max-level max > /dev/null
+	  --select 0 --max-level max)
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: --max-level=0,\
 	$(PYTHON_DIR)/inferTraceBugs -o __test-infer-out__ \
-	  --select 0 --max-level 0 > /dev/null
+	  --select 0 --max-level 0)
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: --max-level=max --no-source,\
 	$(PYTHON_DIR)/inferTraceBugs -o __test-infer-out__ \
-	  --select 0 --max-level max --no-source > /dev/null
+	  --select 0 --max-level max --no-source)
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs: --only-show,\
 	$(PYTHON_DIR)/inferTraceBugs -o __test-infer-out__ \
-	  --only-show > /dev/null
-	$(QUIET)rm -fr __test-infer-out__
+	  --only-show)
+	$(QUIET)$(REMOVE_DIR) __test-infer-out__
 
 .PHONY: check_missing_mli
 check_missing_mli:
@@ -270,26 +289,29 @@ toplevel: clang_plugin
 
 .PHONY: inferScriptMode_test
 inferScriptMode_test: test_build
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer OCaml REPL,\
 	 INFER_REPL_BINARY=ocaml $(SCRIPT_DIR)/infer_repl $(INFER_DIR)/tests/repl/infer_batch_script.ml)
 
 .PHONY: checkCopyright
 checkCopyright:
-	$(QUIET)$(MAKE) -C $(SRC_DIR) checkCopyright
+	$(QUIET)$(call silent_on_success,Building checkCopyright,\
+	$(MAKE) -C $(SRC_DIR) checkCopyright)
 
 .PHONY: validate-skel
 validate-skel:
 ifeq ($(IS_FACEBOOK_TREE),yes)
-	$(QUIET)$(MAKE) -C facebook validate
+	$(QUIET)$(call silent_on_success,Validating facebook/,\
+	$(MAKE) -C facebook validate)
 endif
 
 
 .PHONY: test
 test: test_build ocaml_unit_test endtoend_test inferTraceBugs_test inferScriptMode_test \
       checkCopyright validate-skel
-	$(QUIET)$(MAKE) -C $(SRC_DIR) mod_dep.dot
+	$(QUIET)$(call silent_on_success,Building Infer source dependency graph,\
+	$(MAKE) -C $(SRC_DIR) mod_dep.dot)
 ifeq (,$(findstring s,$(MAKEFLAGS)))
-	$(QUIET)echo "ALL TESTS PASSED"
+	$(QUIET)echo "$(TERM_INFO)ALL TESTS PASSED$(TERM_RESET)"
 endif
 
 .PHONY: quick-test
@@ -432,7 +454,8 @@ ifeq ($(IS_FACEBOOK_TREE),yes)
 	$(QUIET)$(MAKE) -C facebook clean
 endif
 	$(QUIET)$(MAKE) -C $(DEPENDENCIES_DIR)/ocamldot clean
-	find $(INFER_DIR)/tests -name '*.o' -or -name '*.o.sh' -delete
+	find $(INFER_DIR)/tests \( -name '*.o' -o -name '*.o.sh' \) -delete
+	$(REMOVE_DIR) _build_logs
 
 .PHONY: conf-clean
 conf-clean: clean
diff --git a/Makefile.autoconf.in b/Makefile.autoconf.in
index 31e405a9b..3514d3043 100644
--- a/Makefile.autoconf.in
+++ b/Makefile.autoconf.in
@@ -55,6 +55,7 @@ OCAMLOPT = @OCAMLOPT@
 prefix = @prefix@
 PYTHON_lxml = @PYTHON_lxml@
 REBUILD = @REBUILD@
+SHASUM = @SHASUM@
 USER_JAVA_HOME = @USER_JAVA_HOME@
 XCODE_SELECT = @XCODE_SELECT@
 XCPRETTY = @XCPRETTY@
diff --git a/Makefile.config b/Makefile.config
index 0bea1959b..fb62a4005 100644
--- a/Makefile.config
+++ b/Makefile.config
@@ -97,10 +97,75 @@ ifneq ($(XCODE_SELECT),no)
 CLANG_DEPS += $(OBJC_MODELS_FILE)
 endif
 
-define silent_on_success
-  $(1) >/dev/null 2>/dev/null || ($(1) && exit 1)
-endef
+INTERACTIVE = $(shell [ -t 0 ] && echo 1)
+# remove "jobserver-fds" because it contains "s"...
+SILENT = $(findstring s,$(subst jobserver-fds,,$(MAKEFLAGS)))
+
+ifeq (1,$(INTERACTIVE))
+TERM_ERROR = $(shell printf '\e[31;1m')
+TERM_INFO = $(shell printf '\e[;1m')
+TERM_SUCCESS = $(shell printf '\e[;2m')
+TERM_RESET = $(shell printf '\e[0m')
+endif
 
 ifneq ($(VERBOSE),1)
+# quiet
 QUIET = @
 endif
+
+MAKE := $(MAKE) INTERACTIVE=$(INTERACTIVE)
+
+# 99999 PIDs ought to be enough for anybody, but check if pid_max can be found just in case
+MAX_PID_SIZE = \
+  $(shell PID_MAX=$$(cat /proc/sys/kernel/pid_max 2>/dev/null); echo $${\#PID_MAX} || echo 5)
+
+# Arguments:
+#   $(1) is a string describing the command
+#   $(2) is the command to run
+#
+# Run and time the command and redirect its stdout and stderr to files. Display info about the
+# command only in case of error. Try to be as helpful as possible in the error case.
+#
+# The PID of the process is used in the names of the output files, and as a prefix for each error
+# message so that it's possible to piece error messages together even when they are interleaved with
+# other messages from concurrent `make` processes.
+#
+# Detect if we are already wrapped inside a silent_on_success call and try not to clutter the output
+# too much in that case.
+define silent_on_success
+  if [ "$$INSIDE_SILENT_ON_SUCCESS" = 1 ]; then \
+    echo "*** inner $(1)"; \
+    echo "*** inner command: $(2)"; \
+    echo "*** inner CWD: $(CURDIR)"; \
+    ($(2)); \
+    exit $$?; \
+  fi; \
+  export INSIDE_SILENT_ON_SUCCESS=1; \
+  HASH="$$$$"; \
+  UNIX_START_DATE=$$(date +"%s"); \
+  HUMAN_START_DATE=$$(date +"%H:%M:%S"); \
+  if [ -z $(SILENT) ]; then \
+    printf '[%s][%$(MAX_PID_SIZE)s] $(TERM_INFO)$(1)...$(TERM_RESET)\n' \
+      "$$HUMAN_START_DATE" "$$HASH"; \
+  fi; \
+  $(MKDIR_P) $(ABSOLUTE_ROOT_DIR)/_build_logs; \
+  ($(2)) 1>$(ABSOLUTE_ROOT_DIR)/_build_logs/cmd-$$HASH.out \
+         2>$(ABSOLUTE_ROOT_DIR)/_build_logs/cmd-$$HASH.err; \
+  ERRCODE=$$?; \
+  if [ $$ERRCODE != 0 ]; then \
+    echo "$(TERM_ERROR)[*ERROR**][$$HASH] *** ERROR $(1)$(TERM_RESET)" >&2; \
+    echo "$(TERM_ERROR)[*ERROR**][$$HASH] *** command: $(2)$(TERM_RESET)" >&2; \
+    echo "$(TERM_ERROR)[*ERROR**][$$HASH] *** CWD: $(CURDIR)$(TERM_RESET)" >&2; \
+    echo "$(TERM_ERROR)[*ERROR**][$$HASH] *** stdout:$(TERM_RESET)" >&2; \
+    sed -e "s/^\(.*\)$$/$(TERM_ERROR)[*ERROR**][$$HASH]$(TERM_RESET) \1/" \
+      $(ABSOLUTE_ROOT_DIR)/_build_logs/cmd-$$HASH.out; >&2; \
+    echo "$(TERM_ERROR)[*ERROR**][$$HASH] *** stderr:$(TERM_RESET)" >&2; \
+    sed -e "s/^\(.*\)$$/$(TERM_ERROR)[*ERROR**][$$HASH]$(TERM_RESET) \1/" \
+      $(ABSOLUTE_ROOT_DIR)/_build_logs/cmd-$$HASH.err; >&2; \
+    exit 1; \
+  elif [ -z $(SILENT) ]; then \
+    UNIX_END_DATE=$$(date +"%s"); \
+    printf '[%7ss][%$(MAX_PID_SIZE)s] $(TERM_SUCCESS)SUCCESS $(1)$(TERM_RESET)\n' \
+      "$$(($$UNIX_END_DATE - $$UNIX_START_DATE))" "$$HASH"; \
+  fi
+endef
diff --git a/infer/models/c/Makefile b/infer/models/c/Makefile
index fc69233c0..865f276e7 100644
--- a/infer/models/c/Makefile
+++ b/infer/models/c/Makefile
@@ -14,8 +14,8 @@ INFER_RESULTS = out/report.json
 all: install
 
 $(INFER_RESULTS): $(C_MODELS_SOURCES) $(CLANG_DEPS_NO_MODELS)
-# make clean in src/ to recompile all the models
-	$(call silent_on_success,\
+#	make clean in src/ to recompile all the models
+	$(QUIET)$(call silent_on_success,Building C models,\
 	  $(INFER_BIN) -o $(@D) --models-mode --no-failures-allowed -- $(MAKE) -C src clean all)
 
 $(C_MODELS_FILE): $(INFER_RESULTS)
diff --git a/infer/models/cpp/Makefile b/infer/models/cpp/Makefile
index 7df469609..3c45adf2b 100644
--- a/infer/models/cpp/Makefile
+++ b/infer/models/cpp/Makefile
@@ -15,8 +15,8 @@ INFER_RESULTS = out/report.json
 all: install
 
 $(INFER_RESULTS): $(CPP_MODELS_SOURCES) $(C_MODELS_SOURCES) $(CLANG_DEPS_NO_MODELS)
-# make clean in src/ to recompile all the models
-	$(call silent_on_success,\
+#	make clean in src/ to recompile all the models
+	$(QUIET)$(call silent_on_success,Building C++ models,\
 	  $(INFER_BIN) -o $(@D) --models-mode --no-failures-allowed -- $(MAKE) -C src clean all)
 
 $(CPP_MODELS_FILE): $(INFER_RESULTS)
diff --git a/infer/models/java/Makefile b/infer/models/java/Makefile
index f45e9a9cc..be3ef4f28 100644
--- a/infer/models/java/Makefile
+++ b/infer/models/java/Makefile
@@ -29,9 +29,9 @@ compile:
 
 $(INFER_REPORT): $(JAVA_DEPS_NO_MODELS) $(JAVA_SOURCES)
 	$(QUIET)rm -fr $(MODELS_OUT)
-	mkdir -p $(MODELS_OUT)
+	$(QUIET)$(MKDIR_P) $(MODELS_OUT)
 	$(QUIET)rm -f $(JAVA_MODELS_JAR)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Building Java models,\
 	  $(INFER_BIN) --jobs 1 --results-dir $(INFER_RESULTS_DIR) --models-mode -- \
 	  $(JAVAC) -bootclasspath $(ANDROID_JAR) -d $(MODELS_OUT) -classpath $(MODELS_CLASSPATH) \
 	    $(JAVA_SOURCES))
diff --git a/infer/models/objc/Makefile b/infer/models/objc/Makefile
index 57665d414..3f3e5fb52 100644
--- a/infer/models/objc/Makefile
+++ b/infer/models/objc/Makefile
@@ -15,7 +15,7 @@ all: install
 
 $(INFER_RESULTS): $(OBJC_MODELS_SOURCES) $(CLANG_DEPS_NO_MODELS)
 # make clean in src/ to recompile all the models
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Building Objective-C models,\
 	  $(INFER_BIN) -o $(@D) --models-mode --no-failures-allowed -- $(MAKE) -C src clean all)
 
 $(OBJC_MODELS_FILE): $(INFER_RESULTS)
diff --git a/infer/tests/base.make b/infer/tests/base.make
index ccee491e2..0a02d23ee 100644
--- a/infer/tests/base.make
+++ b/infer/tests/base.make
@@ -6,38 +6,27 @@
 # of patent rights can be found in the PATENTS file in the same directory.
 
 ROOT_DIR = $(TESTS_DIR)/../..
+include $(ROOT_DIR)/Makefile.config
 
 # The relative path from infer/tests/ to the directory containing the current Makefile. This is
 # computed in a hacky way and might not always be a relative path, so only use this for cosmetic
 # reasons.
 TEST_REL_DIR = $(patsubst $(abspath $(TESTS_DIR))/%,%,$(abspath $(CURDIR)))
 
-include $(ROOT_DIR)/Makefile.config
-
-default: compile
-
-issues.exp.test$(TEST_SUFFIX): infer-out$(TEST_SUFFIX)/report.json $(INFERPRINT_BIN)
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
-
-.PHONY: compile
-compile: $(OBJECTS)
-
-.PHONY: analyze
-analyze: infer-out$(TEST_SUFFIX)/report.json
-
-.PHONY: print
-print: issues.exp.test$(TEST_SUFFIX)
-
-.PHONY: test
-test: issues.exp.test$(TEST_SUFFIX)
-	$(QUIET)cd $(TESTS_DIR) && \
-	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test$(TEST_SUFFIX)
-
-.PHONY: print
-replace: issues.exp.test$(TEST_SUFFIX)
-	cp $< issues.exp
-
-.PHONY: clean
-clean:
-	$(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) infer-out$(TEST_SUFFIX) \
-	  $(OBJECTS) $(CLEAN_EXTRA)
+define check_no_duplicates
+  grep "DUPLICATE_SYMBOLS" $(1); test $$? -ne 0 || \
+  (echo '$(TEST_ERROR)Duplicate symbols found in $(CURDIR).' \
+   'Please make sure all the function names in all the source test files are different.$(TEST_RESET)';\
+   exit 1)
+endef
+
+define check_no_diff
+  diff -u $(1) $(2) >&2 || \
+  (printf '\n' >&2; \
+   printf '$(TERM_ERROR)Test output ($(2)) differs from expected test output $(1)$(TERM_RESET)\n' >&2; \
+   printf '$(TERM_ERROR)Run the following command to replace the expected test output with the new output:$(TERM_RESET)\n' >&2; \
+   printf '\n' >&2; \
+   printf '$(TERM_ERROR)  make -C $(TEST_REL_DIR) replace\n$(TERM_RESET)' >&2; \
+   printf '\n' >&2; \
+   exit 1)
+endef
diff --git a/infer/tests/build_systems/ant/Makefile b/infer/tests/build_systems/ant/Makefile
index c71e1599e..4b5096a17 100644
--- a/infer/tests/build_systems/ant/Makefile
+++ b/infer/tests/build_systems/ant/Makefile
@@ -13,13 +13,13 @@ OBJECTS = $(patsubst %.java,ant_out/%.class,$(SOURCES))
 INFERPRINT_OPTIONS = --issues-tests
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(OBJECTS): $(SOURCES)
 	$(ANT)
 
 infer-out/report.json: $(JAVA_DEPS) $(SOURCES)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing ant integration,\
 	  $(INFER_BIN) -a $(ANALYZER) --project-root $(TESTS_DIR) --inferconfig-home . -- $(ANT))
 
 clean:
diff --git a/infer/tests/build_systems/buck-clang-db/Makefile b/infer/tests/build_systems/buck-clang-db/Makefile
index 3b9c60d77..6ee40e821 100644
--- a/infer/tests/build_systems/buck-clang-db/Makefile
+++ b/infer/tests/build_systems/buck-clang-db/Makefile
@@ -16,11 +16,11 @@ OBJECTS = '$(SOURCE_DIR)/buck-out/gen/clang_compilation_database/__Hel lo\#compi
 INFERPRINT_OPTIONS = --issues-tests
 CLEAN_EXTRA = $(SOURCE_DIR)/buck-out
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
-	cd $(SOURCE_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(SOURCE_DIR) && \
+	$(call silent_on_success,Testing Buck Clang compilation database integration,\
 	  NO_BUCKD=1 \
 	  $(INFER_BIN) -a $(ANALYZER) --stats $(INFER_OPTIONS) -o $(CURDIR)/$(@D) \
 	  --buck-compilation-database no-deps \
diff --git a/infer/tests/build_systems/buck/Makefile b/infer/tests/build_systems/buck/Makefile
index f1544130e..887cc7150 100644
--- a/infer/tests/build_systems/buck/Makefile
+++ b/infer/tests/build_systems/buck/Makefile
@@ -16,18 +16,18 @@ INFERPRINT_OPTIONS = --project-root $(ROOT_DIR) --issues-tests
 CLEAN_EXTRA = $(ROOT_DIR)/buck-out
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(OBJECTS): $(JAVA_SOURCE_FILES)
-	cd $(ROOT_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(ROOT_DIR) && \
+	$(call silent_on_success,Compiling Buck Java tests,\
 	INFER_BIN=$(INFER_BIN) NO_BUCKD=1 \
 	$(BUCK) build --no-cache $(BUCK_TARGET))
 
 infer-out/report.json: $(JAVA_DEPS) $(JAVA_SOURCE_FILES)
-	cd $(ROOT_DIR) && \
+	$(QUIET)cd $(ROOT_DIR) && \
 	$(REMOVE_DIR) buck-out && \
-	$(call silent_on_success,\
+	$(call silent_on_success,Testing Buck Java integration,\
 	INFER_BIN=$(INFER_BIN) NO_BUCKD=1 \
 	$(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/infer-out -- \
 	  $(BUCK) build --no-cache $(BUCK_TARGET))
diff --git a/infer/tests/build_systems/clang_compilation_db/Makefile b/infer/tests/build_systems/clang_compilation_db/Makefile
index 115126f94..8feb4008b 100644
--- a/infer/tests/build_systems/clang_compilation_db/Makefile
+++ b/infer/tests/build_systems/clang_compilation_db/Makefile
@@ -17,36 +17,38 @@ SOURCES = $(CMAKE_DIR)/hello.cpp
 OBJECTS = $(CMAKE_BUILD_DIR)/compile_commands.json
 INFERPRINT_OPTIONS = --issues-tests
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(CMAKE_BUILD_DIR):
-	$(MKDIR_P) $@
+	$(QUIET)$(MKDIR_P) $@
 
 $(CMAKE_BUILD_DIR)/compile_commands.json: $(SOURCES) $(CMAKE_DIR)/CMakeLists.txt $(CMAKE_BUILD_DIR)
-	cd $(CMAKE_BUILD_DIR) && \
-	$(CMAKE) -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ..
+	$(QUIET)cd $(CMAKE_BUILD_DIR) && \
+	$(call silent_on_success,Running cmake to generate Makefiles,\
+	$(CMAKE) -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ..)
 
 infer-out/report.json:
 #	make this target empty - since this test defines issues.exp.test directly
 
 infer-out-with-index/report.json: $(CMAKE_BUILD_DIR)/compile_commands.json $(CLANG_DEPS) $(SOURCES)
-	$(call silent_on_success,\
-	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o infer-out-with-index --changed-files-index $(CMAKE_DIR)/index.txt --compilation-database $<)
+	$(QUIET)$(call silent_on_success,Testing Clang compilation database with index integration,\
+	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o $(@D) \
+	    --changed-files-index $(CMAKE_DIR)/index.txt --compilation-database $<)
 
 infer-out-no-index/report.json: $(CMAKE_BUILD_DIR)/compile_commands.json $(CLANG_DEPS) $(SOURCES)
-	$(call silent_on_success,\
-	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o infer-out-no-index --compilation-database $<)
+	$(QUIET)$(call silent_on_success,Testing Clang compilation database integration,\
+	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o $(@D) --compilation-database $<)
 
 infer-out-reactive-capture/report.json: $(CMAKE_BUILD_DIR)/compile_commands.json $(CLANG_DEPS) $(SOURCES)
-	$(call silent_on_success,\
-	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o infer-out-reactive-capture --reactive-capture \
+	$(QUIET)$(call silent_on_success,Testing Clang compilation database reactive capture integration,\
+	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -o $(@D) --reactive-capture \
 	  --changed-files-index $(CMAKE_DIR)/index.txt --compilation-database $<)
 
 issues.exp.test: infer-out-with-index/report.json infer-out-no-index/report.json infer-out-reactive-capture/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.with-index \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.with-index \
 		--from-json-report infer-out-with-index/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.no-index \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.no-index \
 	  --from-json-report infer-out-no-index/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.reactive-capture \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.reactive-capture \
 	  --from-json-report infer-out-reactive-capture/report.json
-	cat $@.with-index $@.no-index $@.reactive-capture > $@
+	$(QUIET)cat $@.with-index $@.no-index $@.reactive-capture > $@
diff --git a/infer/tests/build_systems/clang_compilation_db_escaped/Makefile b/infer/tests/build_systems/clang_compilation_db_escaped/Makefile
index 2c1aa37e7..f25ec1684 100644
--- a/infer/tests/build_systems/clang_compilation_db_escaped/Makefile
+++ b/infer/tests/build_systems/clang_compilation_db_escaped/Makefile
@@ -14,19 +14,19 @@ OBJECTS = $(SOURCES:.c=.o)
 CLEAN_EXTRA = compile_commands.json
 INFERPRINT_OPTIONS = --issues-tests
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 ../codetoanalyze/path\ with\ spaces/hel\ lo.c: ../codetoanalyze/path_with_spaces/hel_lo.c
 #	make does not want to interpret "$(@D)" in the right way here...
-	$(MKDIR_P) ../codetoanalyze/path\ with\ spaces/
+	$(QUIET)$(MKDIR_P) ../codetoanalyze/path\ with\ spaces/
 	$(QUIET)cp "$<" "$@"
 	$(QUIET)touch "$@"
 
 compile_commands.json: compile_commands.json.in
-	sed -e 's#%pwd%#$(CURDIR)#g' $< > $@ || $(REMOVE) $@
+	$(QUIET)sed -e 's#%pwd%#$(CURDIR)#g' $< > $@ || $(REMOVE) $@
 
 infer-out/report.json: compile_commands.json $(SOURCES) Makefile \
                        ../codetoanalyze/path\ with\ spaces/hel\ lo.c
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Clang compilation database escaped integration,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --project-root $(TESTS_DIR) \
-      --compilation-database-escaped $<)
+	  --compilation-database-escaped $<)
diff --git a/infer/tests/build_systems/clang_compilation_db_relpath/Makefile b/infer/tests/build_systems/clang_compilation_db_relpath/Makefile
index 8509009b9..f8167349d 100644
--- a/infer/tests/build_systems/clang_compilation_db_relpath/Makefile
+++ b/infer/tests/build_systems/clang_compilation_db_relpath/Makefile
@@ -16,19 +16,19 @@ OBJECTS = $(SOURCES:.c=.o)
 CLEAN_EXTRA = compile_commands.json ../codetoanalyze/path\ with\ spaces
 INFERPRINT_OPTIONS = --issues-tests
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 ../codetoanalyze/path\ with\ spaces/hel\ lo.c: ../codetoanalyze/path_with_spaces/hel_lo.c
 #	make does not want to interpret "$(@D)" in the right way here...
-	$(MKDIR_P) ../codetoanalyze/path\ with\ spaces/
+	$(QUIET)$(MKDIR_P) ../codetoanalyze/path\ with\ spaces/
 	$(QUIET)cp "$<" "$@"
 	$(QUIET)touch "$@"
 
 compile_commands.json: compile_commands.json.in
-	sed -e 's#%pwd%#$(CURDIR)#g' $< > $@ || $(REMOVE) $@
+	$(QUIET)sed -e 's#%pwd%#$(CURDIR)#g' $< > $@ || $(REMOVE) $@
 
 infer-out/report.json: compile_commands.json $(SOURCES) Makefile \
                        ../codetoanalyze/path\ with\ spaces/hel\ lo.c
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Clang compilation database with relative paths integration,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --project-root $(TESTS_DIR) \
-      --compilation-database $<)
+	  --compilation-database $<)
diff --git a/infer/tests/build_systems/clang_translation/Makefile b/infer/tests/build_systems/clang_translation/Makefile
index 73c3bc631..3d25a7961 100644
--- a/infer/tests/build_systems/clang_translation/Makefile
+++ b/infer/tests/build_systems/clang_translation/Makefile
@@ -29,15 +29,15 @@ INFER_OPTIONS = -a capture --frontend-tests --skip-translation-headers exclude_d
 capture: $(DOT_FILES)
 
 $(ROOT)/main.cpp.test.dot $(SYM_ROOT)/main_symlink.cpp.test.dot: $(SOURCES) $(CLANG_DEPS)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing clang translation with symlinks with project root,\
 	$(INFER_BIN) $(INFER_OPTIONS) --results-dir infer-out-$(basename $(@F:.test.dot=)) \
 	  --project-root $(ROOT) --icfg-dotty-outfile $@ -- \
 	  clang $(CLANG_OPTIONS) $(@:.test.dot=))
 
 $(ROOT)/main_default_root.cpp.test.dot $(SYM_ROOT)/main_default_symlink.cpp.test.dot: \
   $(CLANG_DEPS) $(SOURCES)
-	cd $(@D) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(@D) && \
+	$(call silent_on_success,Testing clang translation with symlinks with CWD=project root,\
 	$(INFER_BIN) $(INFER_OPTIONS) --results-dir $(CURDIR)/infer-out-$(basename $(@F:.test.dot=)) \
 	  --icfg-dotty-outfile $(@F) -- \
 	  clang $(CLANG_OPTIONS) $(@F:.test.dot=))
@@ -48,6 +48,9 @@ test: test_extra
 
 # all dot files should be exactly the same - if they aren't there is something wrong
 test_extra: $(DOT_FILES)
-	diff -u $(ROOT)/main.cpp.test.dot $(SYM_ROOT)/main_symlink.cpp.test.dot
-	diff -u $(ROOT)/main.cpp.test.dot $(ROOT)/main_default_root.cpp.test.dot
-	diff -u $(ROOT)/main.cpp.test.dot $(SYM_ROOT)/main_default_symlink.cpp.test.dot
+	$(QUIET)$(call check_no_diff,$(ROOT)/main.cpp.test.dot,\
+	                         $(SYM_ROOT)/main_symlink.cpp.test.dot)
+	$(QUIET)$(call check_no_diff,$(ROOT)/main.cpp.test.dot,\
+	                             $(ROOT)/main_default_root.cpp.test.dot)
+	$(QUIET)$(call check_no_diff,$(ROOT)/main.cpp.test.dot,\
+	                         $(SYM_ROOT)/main_default_symlink.cpp.test.dot)
diff --git a/infer/tests/build_systems/cmake/Makefile b/infer/tests/build_systems/cmake/Makefile
index 03795651b..f844c9c32 100644
--- a/infer/tests/build_systems/cmake/Makefile
+++ b/infer/tests/build_systems/cmake/Makefile
@@ -18,7 +18,7 @@ SOURCES = $(CMAKE_DIR)/hello.c
 OBJECTS = $(CMAKE_BUILD_DIR)/CMakeFiles/Hello.dir/hello.c.o
 INFERPRINT_OPTIONS = --issues-tests
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(CMAKE_BUILD_DIR)/Makefile: $(CMAKE_DIR)/CMakeLists.txt
 	$(QUIET)$(MKDIR_P) $(CMAKE_BUILD_DIR)
@@ -31,12 +31,12 @@ $(OBJECTS): $(SOURCES) $(CMAKE_BUILD_DIR)/Makefile
 
 infer-out/report.json: $(SOURCES) $(CMAKE_DIR)/CMakeLists.txt $(CLANG_DEPS)
 #	nuke the build dir and recreate everything to avoid stale compilation/analysis results
-	$(QUIET)$(REMOVE_DIR) $(CMAKE_ANALYZE_DIR) && \
-	$(MKDIR_P) $(CMAKE_ANALYZE_DIR) && \
-	cd $(CMAKE_ANALYZE_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)$(REMOVE_DIR) $(CMAKE_ANALYZE_DIR)
+	$(QUIET)$(MKDIR_P) $(CMAKE_ANALYZE_DIR)
+	$(QUIET)cd $(CMAKE_ANALYZE_DIR) && \
+	$(call silent_on_success,Testing CMake integration: Generate Makefiles,\
 	  $(INFER_BIN) -a compile $(INFER_OPTIONS) -- cmake ..)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing CMake integration: Run Makefiles,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) -- $(MAKE) -C $(CMAKE_ANALYZE_DIR))
 #	cmake makes all paths absolute
 	$(QUIET)sed -i -e 's#$(abspath $(TESTS_DIR))/##g' $@
diff --git a/infer/tests/build_systems/codetoanalyze/make/Makefile b/infer/tests/build_systems/codetoanalyze/make/Makefile
index caaed29ce..087da7b66 100644
--- a/infer/tests/build_systems/codetoanalyze/make/Makefile
+++ b/infer/tests/build_systems/codetoanalyze/make/Makefile
@@ -14,4 +14,4 @@ all: $(OBJECTS)
 	$(CC) -c $<
 
 clean:
-	$(QUIET)rm -rf $(OBJECTS)
+	rm -rf $(OBJECTS)
diff --git a/infer/tests/build_systems/delete_results_dir/Makefile b/infer/tests/build_systems/delete_results_dir/Makefile
index 08ec65135..a050a6991 100644
--- a/infer/tests/build_systems/delete_results_dir/Makefile
+++ b/infer/tests/build_systems/delete_results_dir/Makefile
@@ -23,9 +23,9 @@ SOURCES = $(CODETOANALYZE_DIR)/hello.c $(CODETOANALYZE_DIR)/hello2.c
 include $(TESTS_DIR)/clang.make
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing that infer deletes the results directory: part 1/2,\
 	  $(INFER_BIN) --dump-duplicate-symbols $(INFER_OPTIONS) -a $(ANALYZER) -- clang $(CLANG_OPTIONS) $(CODETOANALYZE_DIR)/hello.c)
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
-	$(call silent_on_success,\
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
+	$(QUIET)$(call silent_on_success,Testing that infer deletes the results directory: part 2/2,\
 	  $(INFER_BIN) --dump-duplicate-symbols $(INFER_OPTIONS) -a $(ANALYZER) -- clang $(CLANG_OPTIONS) $(CODETOANALYZE_DIR)/hello2.c)
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
diff --git a/infer/tests/build_systems/differential_resolve_infer_eradicate_conflict/Makefile b/infer/tests/build_systems/differential_resolve_infer_eradicate_conflict/Makefile
index 208f74b00..845244056 100644
--- a/infer/tests/build_systems/differential_resolve_infer_eradicate_conflict/Makefile
+++ b/infer/tests/build_systems/differential_resolve_infer_eradicate_conflict/Makefile
@@ -14,13 +14,13 @@ CLEAN_EXTRA = src/com/example/Diff*.java src/Diff*.java *.class com/
 include ../../differential.make
 
 $(CURRENT_REPORT): .inferconfig
-	cp src/com/example/DiffExample.java.current src/com/example/DiffExample.java
-	cp src/DiffExampleTwo.java.current src/DiffExampleTwo.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/com/example/DiffExample.java.current src/com/example/DiffExample.java
+	$(QUIET)$(COPY) src/DiffExampleTwo.java.current src/DiffExampleTwo.java
+	$(QUIET)$(call silent_on_success,Testing Differential resolves Infer/Eradicate conflicts: current,\
 	$(INFER_BIN) \
 		-o $(CURRENT_DIR) -- $(JAVAC) src/com/example/DiffExample.java src/DiffExampleTwo.java)
 
 $(PREVIOUS_REPORT): .inferconfig
-	cp src/com/example/DiffExample.java.previous src/com/example/DiffExample.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/com/example/DiffExample.java.previous src/com/example/DiffExample.java
+	$(QUIET)$(call silent_on_success,Testing Differential resolves Infer/Eradicate conflicts: previous,\
 	$(INFER_BIN) -o $(PREVIOUS_DIR) -- $(JAVAC) src/com/example/DiffExample.java)
diff --git a/infer/tests/build_systems/differential_skip_anonymous_class_renamings/Makefile b/infer/tests/build_systems/differential_skip_anonymous_class_renamings/Makefile
index f731f9647..77b3782c4 100644
--- a/infer/tests/build_systems/differential_skip_anonymous_class_renamings/Makefile
+++ b/infer/tests/build_systems/differential_skip_anonymous_class_renamings/Makefile
@@ -13,9 +13,11 @@ CLEAN_EXTRA = src/Diff*.java *.class
 include ../../differential.make
 
 $(CURRENT_REPORT):
-	cp src/DiffExample.java.current src/DiffExample.java
-	$(call silent_on_success, $(INFER_BIN) -o $(CURRENT_DIR) -- $(JAVAC) src/*.java)
+	$(QUIET)$(COPY) src/DiffExample.java.current src/DiffExample.java
+	$(QUIET)$(call silent_on_success,Testing Differential skips anon class renamings: current,\
+	$(INFER_BIN) -o $(CURRENT_DIR) -- $(JAVAC) src/*.java)
 
 $(PREVIOUS_REPORT):
-	cp src/DiffExample.java.previous src/DiffExample.java
-	$(call silent_on_success, $(INFER_BIN) -o $(PREVIOUS_DIR) -- $(JAVAC) src/*.java)
+	$(QUIET)$(COPY) src/DiffExample.java.previous src/DiffExample.java
+	$(QUIET)$(call silent_on_success,Testing Differential skips anon class renamings: previous,\
+	$(INFER_BIN) -o $(PREVIOUS_DIR) -- $(JAVAC) src/*.java)
diff --git a/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames/Makefile b/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames/Makefile
index 27f5b6948..c047604c7 100644
--- a/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames/Makefile
+++ b/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames/Makefile
@@ -13,12 +13,14 @@ CLEAN_EXTRA = src/Diff*.java src/Diff*.java *.class
 include ../../differential.make
 
 $(CURRENT_REPORT):
-	cp src/DiffExample.java.current src/DiffExample.java
-	cp src/DiffExampleTwo.java.current src/DiffExampleTwo.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/DiffExample.java.current src/DiffExample.java
+	$(QUIET)$(COPY) src/DiffExampleTwo.java.current src/DiffExampleTwo.java
+	$(QUIET)$(call silent_on_success,Testing\
+	Differential skips duplicated types on filenames: current,\
 	$(INFER_BIN) -o $(CURRENT_DIR) -- $(JAVAC) src/DiffExample.java src/DiffExampleTwo.java)
 
 $(PREVIOUS_REPORT):
-	cp src/DiffExample.java.previous src/DiffExample.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/DiffExample.java.previous src/DiffExample.java
+	$(QUIET)$(call silent_on_success,Testing\
+	Differential skips duplicated types on filenames: previous,\
 	$(INFER_BIN) -o $(PREVIOUS_DIR) -- $(JAVAC) src/DiffExample.java)
diff --git a/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames_with_renamings/Makefile b/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames_with_renamings/Makefile
index a07d177e2..018e1c054 100644
--- a/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames_with_renamings/Makefile
+++ b/infer/tests/build_systems/differential_skip_duplicated_types_on_filenames_with_renamings/Makefile
@@ -14,11 +14,13 @@ CLEAN_EXTRA = src/Diff*.java *.class
 include ../../differential.make
 
 $(CURRENT_REPORT):
-	cp src/DiffExampleRenamed.java.current src/DiffExampleRenamed.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/DiffExampleRenamed.java.current src/DiffExampleRenamed.java
+	$(QUIET)$(call silent_on_success,Testing\
+	Differential skips duplicated types on renamed files: current,\
 	$(INFER_BIN) -o $(CURRENT_DIR) -- $(JAVAC) src/DiffExampleRenamed.java)
 
 $(PREVIOUS_REPORT):
-	cp src/DiffExample.java.previous src/DiffExample.java
-	$(call silent_on_success,\
+	$(QUIET)$(COPY) src/DiffExample.java.previous src/DiffExample.java
+	$(QUIET)$(call silent_on_success,Testing\
+	Differential skips duplicated types on renamed files: previous,\
 	$(INFER_BIN) -o $(PREVIOUS_DIR) -- $(JAVAC) src/DiffExample.java)
diff --git a/infer/tests/build_systems/fail_on_issue/Makefile b/infer/tests/build_systems/fail_on_issue/Makefile
index 178b08bff..d4cd63fcd 100644
--- a/infer/tests/build_systems/fail_on_issue/Makefile
+++ b/infer/tests/build_systems/fail_on_issue/Makefile
@@ -7,6 +7,8 @@
 
 TESTS_DIR = ../..
 
+include $(TESTS_DIR)/base.make
+
 ANALYZER = infer
 
 CODETOANALYZE_DIR = ../codetoanalyze
@@ -18,17 +20,10 @@ INFERPRINT_OPTIONS = --issues-tests
 SOURCES = $(CODETOANALYZE_DIR)/hello.c
 OBJECTS = $(foreach source,$(SOURCES),$(basename $(source)).o)
 
-ROOT_DIR = $(TESTS_DIR)/../..
-
-include $(ROOT_DIR)/Makefile.config
-
-# see base.make
-TEST_REL_DIR = $(patsubst $(abspath $(TESTS_DIR))/%,%,$(abspath $(CURDIR)))
-
 default: compile
 
 issues.exp.test: $(CLANG_DEPS) $(SOURCES)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing Infer fails on issue,\
 	  ($(INFER_BIN) --fail-on-issue -- clang $(CLANG_OPTIONS) $(SOURCES); \
 	   echo "infer exit code: $$?" > $@))
 
@@ -41,7 +36,7 @@ print: issues.exp.test
 .PHONY: test
 test: issues.exp.test
 	$(QUIET)cd $(TESTS_DIR) && \
-	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
+	$(call check_no_diff,$(TEST_REL_DIR)/issues.exp,$(TEST_REL_DIR)/issues.exp.test)
 
 .PHONY: replace
 replace: issues.exp.test
diff --git a/infer/tests/build_systems/genrule/Makefile b/infer/tests/build_systems/genrule/Makefile
index 86e56fd64..538ab4a5c 100644
--- a/infer/tests/build_systems/genrule/Makefile
+++ b/infer/tests/build_systems/genrule/Makefile
@@ -19,11 +19,11 @@ INFERPRINT_OPTIONS = --project-root $(ROOT_DIR) --issues-tests
 CLEAN_EXTRA = $(ROOT_DIR)/buck-out/genruletest
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(OBJECTS): $(JAVA_SOURCE_FILES)
-	cd $(ROOT_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(ROOT_DIR) && \
+	$(call silent_on_success,Compiling Buck genrule for Java sources,\
 	NO_BUCKD=1 $(BUCK) build --no-cache $(BUCK_TARGET))
 
 .PHONY: genrule
@@ -31,14 +31,14 @@ genrule: $(JSON_REPORT)
 
 
 $(JSON_REPORT): $(JAVA_DEPS) $(JAVA_SOURCE_FILES)
-	cd $(ROOT_DIR) && \
+	$(QUIET)cd $(ROOT_DIR) && \
 	$(REMOVE_DIR) buck-out && \
-	$(call silent_on_success,\
+	$(call silent_on_success,Testing Buck genrule for Java integration,\
 	INFER_BIN="$(INFER_BIN)" NO_BUCKD=1 GENERATE_INFER_GENRULES=1 \
 	$(BUCK) build --no-cache $(INFER_TARGET))
-	touch $@
+	$(QUIET)touch $@
 
 infer-out/report.json: $(JSON_REPORT)
-	$(MKDIR_P) infer-out
+	$(QUIET)$(MKDIR_P) infer-out
 #	the report contains absolute paths
 	$(QUIET)sed -e 's#$(abspath $(TESTS_DIR))/##g' $< > $@
diff --git a/infer/tests/build_systems/gradle/Makefile b/infer/tests/build_systems/gradle/Makefile
index 4ea92eed4..006ea4405 100644
--- a/infer/tests/build_systems/gradle/Makefile
+++ b/infer/tests/build_systems/gradle/Makefile
@@ -16,11 +16,11 @@ OBJECTS =
 INFERPRINT_OPTIONS = --issues-tests
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 infer-out/report.json: $(JAVA_DEPS) $(SOURCES)
 #	mock version of gradle
-	PATH=$(CURDIR)/../mock:"$$PATH"; \
+	$(QUIET)PATH=$(CURDIR)/../mock:"$$PATH"; \
 	cd $(SOURCES_DIR) && \
-	$(call silent_on_success,\
+	$(call silent_on_success,Testing gradle Java integration,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/infer-out -- gradle build)
diff --git a/infer/tests/build_systems/j1/Makefile b/infer/tests/build_systems/j1/Makefile
index 3eae62a52..6cbac2f01 100644
--- a/infer/tests/build_systems/j1/Makefile
+++ b/infer/tests/build_systems/j1/Makefile
@@ -5,8 +5,11 @@
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
 
+ROOT_DIR = ../../../..
+include $(ROOT_DIR)/Makefile.config
+
 default: compile
 
 .PHONY: print replace test clean
 print replace test clean:
-	INFER_ARGS=-j^1 $(MAKE) -C ../../codetoanalyze/cpp/checkers TEST_SUFFIX=-j1 $@
+	$(QUIET)INFER_ARGS=-j^1 $(MAKE) -C ../../codetoanalyze/cpp/checkers TEST_SUFFIX=-j1 $@
diff --git a/infer/tests/build_systems/make/Makefile b/infer/tests/build_systems/make/Makefile
index 6178d851f..99a400158 100644
--- a/infer/tests/build_systems/make/Makefile
+++ b/infer/tests/build_systems/make/Makefile
@@ -17,7 +17,7 @@ SOURCES = $(wildcard ../codetoanalyze/make/*.c)
 include $(TESTS_DIR)/clang.make
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES) $(HEADERS)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing make clang integration,\
 	  $(INFER_BIN) --dump-duplicate-symbols --project-root $(TESTS_DIR) -a $(ANALYZER) -- \
 	    make -C ../codetoanalyze/make clean all)
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
diff --git a/infer/tests/build_systems/mvn/Makefile b/infer/tests/build_systems/mvn/Makefile
index 2667b1726..b7ff77b84 100644
--- a/infer/tests/build_systems/mvn/Makefile
+++ b/infer/tests/build_systems/mvn/Makefile
@@ -19,15 +19,15 @@ CLEAN_EXTRA = \
   $(MVN_DIRS:%=issues-%.exp.test)
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 infer-out/report.json:
-	$(MKDIR_P) $(@D)
-	touch $@
+	$(QUIET)$(MKDIR_P) $(@D)
+	$(QUIET)touch $@
 
 infer-out-%/report.json: $(JAVA_DEPS) $(SOURCES)
-	cd ../codetoanalyze/mvn/$* && \
-	$(call silent_on_success,\
+	$(QUIET)cd ../codetoanalyze/mvn/$* && \
+	$(call silent_on_success,Testing mvn Java integration: $*,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) \
 	    --project-root $(CURDIR)/$(TESTS_DIR) -- \
 	    $(MVN) clean compile)
@@ -39,13 +39,13 @@ infer-out-simple_app/report.json: infer-out-app_with_infer_profile/report.json
 infer-out-app_with_infer_profile/report.json: infer-out-app_with_profiles/report.json
 
 issues-%.exp.test: infer-out-%/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
 
 issues.exp.test: $(foreach mvndir,$(MVN_DIRS),issues-$(mvndir).exp.test)
 #	erase the contents of the file
-	: > $@
+	$(QUIET): > $@
 #	remember the file name so it's easier to know which bug is from where
-	for mvndir in $(MVN_DIRS); do \
+	$(QUIET)for mvndir in $(MVN_DIRS); do \
 	  echo "-- $$mvndir" >> $@; \
 	  cat issues-$$mvndir.exp.test >> $@; \
 	done
diff --git a/infer/tests/build_systems/ndk_build/Makefile b/infer/tests/build_systems/ndk_build/Makefile
index aeb4fc53c..e04a29848 100644
--- a/infer/tests/build_systems/ndk_build/Makefile
+++ b/infer/tests/build_systems/ndk_build/Makefile
@@ -15,14 +15,14 @@ OBJECTS = $(PROJECT_ROOT)/hello.o
 INFERPRINT_OPTIONS = --issues-tests
 CLEAN_EXTRA = libs obj
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 $(OBJECTS): $(SOURCES)
 	cd $(PROJECT_ROOT) && \
 	$(NDKBUILD) -B NDK_LIBS_OUT=./libs NDK_OUT=./obj
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
-	cd $(PROJECT_ROOT) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(PROJECT_ROOT) && \
+	$(call silent_on_success,Testing ndk-build clang integration,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/infer-out -- \
 	    $(NDKBUILD) -B NDK_LIBS_OUT=./libs NDK_OUT=./obj)
diff --git a/infer/tests/build_systems/reactive/Makefile b/infer/tests/build_systems/reactive/Makefile
index e22d0716f..ca1a32470 100644
--- a/infer/tests/build_systems/reactive/Makefile
+++ b/infer/tests/build_systems/reactive/Makefile
@@ -21,15 +21,15 @@ SOURCES = $(SOURCES1) $(SOURCES2) $(SOURCES3)
 include $(TESTS_DIR)/clang.make
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES) $(HEADERS)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing reactive mode: capture 1/3,\
 	  $(INFER_BIN) $(INFER_OPTIONS) -a capture -- \
 	    clang $(CLANG_OPTIONS) $(SOURCES1))
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing reactive mode: capture 2/3,\
 	  $(INFER_BIN) $(INFER_OPTIONS) -a capture --reactive --continue -- \
 	    clang $(CLANG_OPTIONS) $(SOURCES2))
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing reactive mode: capture 3/3,\
 	  $(INFER_BIN) $(INFER_OPTIONS) -a capture --reactive --continue -- \
 	    clang $(CLANG_OPTIONS) $(SOURCES3))
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing reactive mode: analysis,\
 	  $(INFER_BIN) --dump-duplicate-symbols $(INFER_OPTIONS) -a $(ANALYZER) -- analyze)
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
diff --git a/infer/tests/build_systems/utf8_in_procname/Makefile b/infer/tests/build_systems/utf8_in_procname/Makefile
index 951e1f8e9..142c4afd6 100644
--- a/infer/tests/build_systems/utf8_in_procname/Makefile
+++ b/infer/tests/build_systems/utf8_in_procname/Makefile
@@ -19,16 +19,18 @@ include $(TESTS_DIR)/clang.make
 
 infer-out/report.json: $(CLANG_DEPS) $(INFERTRACEBUGS_BIN) $(SOURCES) $(HEADERS) $(MAKEFILE_LIST)
 #	set non-utf8-supporting locale
-	LC_ALL=C; \
-	$(call silent_on_success,\
+	$(QUIET)LC_ALL=C; \
+	$(call silent_on_success,Testing Infer is immune to UTF-8 in procnames,\
 	  $(INFER_BIN) --dump-duplicate-symbols $(INFER_OPTIONS) -a $(ANALYZER) -- clang $(CLANG_OPTIONS) $(SOURCES))
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
 #	make sure inferTraceBugs is immune to UTF-8
-	$(call silent_on_success, $(INFERTRACEBUGS_BIN) --max-level max --select 0)
-	$(call silent_on_success, $(INFERTRACEBUGS_BIN) --html)
-	[ -f infer-out/report.html/index.html ]
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs is immune to UTF-8 in procnames,\
+	$(INFERTRACEBUGS_BIN) --max-level max --select 0)
+	$(QUIET)$(call silent_on_success,Testing inferTraceBugs --html is immune to UTF-8 in procnames,\
+	$(INFERTRACEBUGS_BIN) --html)
+	$(QUIET)[ -f infer-out/report.html/index.html ]
 #	run again to check that infer manages to delete the results directory
-	LC_ALL=C; \
-	$(call silent_on_success,\
+	$(QUIET)LC_ALL=C; \
+	$(call silent_on_success,Testing Infer rerun is immune to UTF-8 in procnames,\
 	  $(INFER_BIN) --dump-duplicate-symbols $(INFER_OPTIONS) -a $(ANALYZER) -- clang $(CLANG_OPTIONS) $(SOURCES))
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
diff --git a/infer/tests/build_systems/utf8_in_pwd/Makefile b/infer/tests/build_systems/utf8_in_pwd/Makefile
index a1ca16b59..c19a6adae 100644
--- a/infer/tests/build_systems/utf8_in_pwd/Makefile
+++ b/infer/tests/build_systems/utf8_in_pwd/Makefile
@@ -10,13 +10,9 @@
 # or 'analyze' target.
 
 TESTS_DIR = ../..
-ROOT_DIR = $(TESTS_DIR)/../..
 
 include $(TESTS_DIR)/java.make
-include $(ROOT_DIR)/Makefile.config
-
-# see base.make
-TEST_REL_DIR = $(patsubst $(abspath $(TESTS_DIR))/%,%,$(abspath $(CURDIR)))
+include $(TESTS_DIR)/base.make
 
 UTF8_DIR = $(shell printf '../codetoanalyze/utf8_\u03B9n_pwd')
 
@@ -44,61 +40,63 @@ INFERPRINT_OPTIONS = --issues-tests
 default: print
 
 $(UTF8_DIR):
-	$(MKDIR_P) $@
+	$(QUIET)$(MKDIR_P) $@
 
 $(CMAKE_DIR): $(CMAKE_SOURCES) $(UTF8_DIR)
-	rsync -aL --exclude='_build*' $(CMAKE_SOURCES_DIR)/ $@/
-	touch $@
+	$(QUIET)rsync -aL --exclude='_build*' $(CMAKE_SOURCES_DIR)/ $@/
+	$(QUIET)touch $@
 
 $(GRADLE_DIR): $(GRADLE_SOURCES) $(UTF8_DIR)
-	rsync -a $(GRADLE_SOURCES_DIR)/ $@/
-	touch $@
+	$(QUIET)rsync -a $(GRADLE_SOURCES_DIR)/ $@/
+	$(QUIET)touch $@
 
 $(JAVAC_DIR): $(JAVAC_SOURCES) $(UTF8_DIR)
-	$(MKDIR_P) $(JAVAC_DIR)
-	rsync -a $(JAVAC_SOURCES) $@
-	touch $@
+	$(QUIET)$(MKDIR_P) $(JAVAC_DIR)
+	$(QUIET)rsync -a $(JAVAC_SOURCES) $@
+	$(QUIET)touch $@
 
 $(MAKE_DIR): $(MAKE_SOURCES) $(UTF8_DIR)
-	rsync -a $(MAKE_SOURCES_DIR)/ $@/
-	touch $@
+	$(QUIET)rsync -a $(MAKE_SOURCES_DIR)/ $@/
+	$(QUIET)touch $@
 
 infer-out-cmake/report.json: $(CMAKE_DIR) $(CMAKE_SOURCES_DIR)/CMakeLists.txt $(CLANG_DEPS)
-	$(REMOVE_DIR) $(CMAKE_ANALYZE_DIR) && \
-	$(MKDIR_P) $(CMAKE_ANALYZE_DIR) && \
-	cd $(CMAKE_ANALYZE_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)$(REMOVE_DIR) $(CMAKE_ANALYZE_DIR)
+	$(QUIET)$(MKDIR_P) $(CMAKE_ANALYZE_DIR)
+	$(QUIET)cd $(CMAKE_ANALYZE_DIR) && \
+	$(call silent_on_success,Testing\
+	  Infer/CMake is immune to UTF-8 in PWD: Makefile generation,\
 	  $(INFER_BIN) -a compile --project-root $(CMAKE_DIR) --results-dir $(CURDIR)/$(@D) -- cmake ..)
-	cd $(CMAKE_ANALYZE_DIR) && \
-	$(call silent_on_success,\
-	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- $(MAKE))
-	sed -i -e 's#$(abspath $(CMAKE_DIR))/##g' $@
+	$(QUIET)cd $(CMAKE_ANALYZE_DIR) && \
+	$(call silent_on_success,Testing\
+	  Infer/CMake is immune to UTF-8 in PWD: Makefile execution,\
+	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- make)
+	$(QUIET)sed -i -e 's#$(abspath $(CMAKE_DIR))/##g' $@
 
 infer-out-gradle/report.json: $(JAVA_DEPS) $(GRADLE_DIR)
 #	mock version of gradle
-	PATH=$(CURDIR)/../mock:"$$PATH"; \
+	$(QUIET)PATH=$(CURDIR)/../mock:"$$PATH"; \
 	cd $(GRADLE_DIR) && \
-	$(call silent_on_success,\
+	$(call silent_on_success,Testing Infer/gradle is immune to UTF-8 in PWD,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- gradle build)
 
 infer-out-javac/report.json: $(JAVA_DEPS) $(JAVAC_DIR)
-	cd $(JAVAC_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(JAVAC_DIR) && \
+	$(call silent_on_success,Testing Infer/javac is immune to UTF-8 in PWD,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- \
 	    $(JAVAC) -cp $(CLASSPATH) $(foreach source,$(JAVAC_SOURCES),$(notdir $(source))))
 
 infer-out-make/report.json: $(CLANG_DEPS) $(MAKE_DIR)
-	cd $(MAKE_DIR) && \
-	$(call silent_on_success,\
+	$(QUIET)cd $(MAKE_DIR) && \
+	$(call silent_on_success,Testing Infer/make is immune to UTF-8 in PWD,\
 	  $(INFER_BIN) -a $(ANALYZER) --results-dir $(CURDIR)/$(@D) -- \
 	    make clean all)
 
 issues-%.exp.test: infer-out-%/report.json $(INFERPRINT_BIN)
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
 
 issues.exp.test: issues-cmake.exp.test issues-gradle.exp.test issues-javac.exp.test \
                  issues-make.exp.test
-	cat $^ > $@
+	$(QUIET)cat $^ > $@
 
 .PHONY: print
 print: issues.exp.test
@@ -106,7 +104,7 @@ print: issues.exp.test
 .PHONY: test
 test: issues.exp.test
 	$(QUIET)cd $(TESTS_DIR) && \
-	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
+	$(call check_no_diff,$(TEST_REL_DIR)/issues.exp,$(TEST_REL_DIR)/issues.exp.test)
 
 .PHONY: replace
 replace: issues.exp.test
diff --git a/infer/tests/build_systems/waf/Makefile b/infer/tests/build_systems/waf/Makefile
index 4bda6ec3a..aaf016c9a 100644
--- a/infer/tests/build_systems/waf/Makefile
+++ b/infer/tests/build_systems/waf/Makefile
@@ -18,8 +18,8 @@ SOURCES = $(wildcard ../codetoanalyze/make/*.c)
 include $(TESTS_DIR)/clang.make
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
-	$(MAKE) -C ../codetoanalyze/make clean
-	cd ../codetoanalyze/make && \
-	$(call silent_on_success,\
+	$(QUIET)$(MAKE) -C ../codetoanalyze/make clean
+	$(QUIET)cd ../codetoanalyze/make && \
+	$(call silent_on_success,Testing Waf clang integration,\
 	  $(INFER_BIN) --dump-duplicate-symbols --results-dir $(CUR_DIR)/$(@D) -a $(ANALYZER) -- \
 	    ./waf)
diff --git a/infer/tests/build_systems/xcodebuild/Makefile b/infer/tests/build_systems/xcodebuild/Makefile
index 6384c1645..7a37aec44 100644
--- a/infer/tests/build_systems/xcodebuild/Makefile
+++ b/infer/tests/build_systems/xcodebuild/Makefile
@@ -13,7 +13,7 @@ SOURCES = $(wildcard $(APP_DIR)/simple_app/*.m)
 OBJECTS = $(APP_DIR)/app_built
 INFERPRINT_OPTIONS = --issues-tests
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 XCODEBUILD = xcodebuild -verbose -target simple_app -configuration Release -sdk iphonesimulator
 
@@ -23,10 +23,11 @@ $(OBJECTS): $(SOURCES)
 	touch app_built
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES)
-	cd $(APP_DIR) && \
+	$(QUIET)cd $(APP_DIR) && \
 	$(XCODEBUILD) clean && \
-	$(call silent_on_success,\
-	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --project-root $(TESTS_DIR) --results-dir $(CURDIR)/$(@D) -- $(XCODEBUILD))
+	$(call silent_on_success,Testing xcodebuild default integration,\
+	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --project-root $(TESTS_DIR) \
+	    --results-dir $(CURDIR)/$(@D) -- $(XCODEBUILD))
 
 clean:
 	cd $(APP_DIR) && \
diff --git a/infer/tests/clang-frontend.make b/infer/tests/clang-frontend.make
index 29b860a28..c28ffe4f6 100644
--- a/infer/tests/clang-frontend.make
+++ b/infer/tests/clang-frontend.make
@@ -5,11 +5,9 @@
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
 
-ROOT_DIR = $(TESTS_DIR)/../..
-include $(ROOT_DIR)/Makefile.config
-
 OBJECTS = $(foreach source,$(SOURCES),$(basename $(source)).o)
 
+include $(TESTS_DIR)/base.make
 include $(TESTS_DIR)/clang-base.make
 
 ONE_SOURCE = $(lastword $(SOURCES))
@@ -19,7 +17,7 @@ default: compile
 compile: $(OBJECTS)
 
 $(ONE_SOURCE).test.dot: $(CLANG_DEPS) $(SOURCES) $(HEADERS)
-	$(call silent_on_success, \
+	$(QUIET)$(call silent_on_success,Testing the infer/clang frontend in $(TEST_REL_DIR),\
 	  $(INFER_BIN) -a capture --frontend-tests --project-root $(TESTS_DIR) $(INFER_OPTIONS) -- \
 	    clang $(CLANG_OPTIONS) $(SOURCES))
 
diff --git a/infer/tests/clang.make b/infer/tests/clang.make
index 4912c1f05..86d1c8ff5 100644
--- a/infer/tests/clang.make
+++ b/infer/tests/clang.make
@@ -11,12 +11,12 @@ CLEAN_EXTRA += duplicates.txt
 
 OBJECTS = $(foreach source,$(SOURCES),$(basename $(source)).o)
 
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 include $(TESTS_DIR)/clang-base.make
 
 infer-out$(TEST_SUFFIX)/report.json: $(CLANG_DEPS) $(SOURCES) $(HEADERS) $(TESTS_DIR)/.inferconfig
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer/clang$(ANALYZER_STRING) in $(TEST_REL_DIR),\
 	  $(INFER_BIN) --results-dir $(@D) --dump-duplicate-symbols \
 	    $(INFER_OPTIONS) -a $(ANALYZER) -- \
 	    clang $(CLANG_OPTIONS) $(SOURCES))
-	grep "DUPLICATE_SYMBOLS" infer-out$(TEST_SUFFIX)/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out$(TEST_SUFFIX)/duplicates.txt)
diff --git a/infer/tests/codetoanalyze/java/crashcontext/Makefile b/infer/tests/codetoanalyze/java/crashcontext/Makefile
index 5398d6846..3ce4442bd 100644
--- a/infer/tests/codetoanalyze/java/crashcontext/Makefile
+++ b/infer/tests/codetoanalyze/java/crashcontext/Makefile
@@ -7,9 +7,7 @@
 
 TESTS_DIR = ../../..
 include $(TESTS_DIR)/java.make
-
-# see base.make
-TEST_REL_DIR = $(patsubst $(abspath $(TESTS_DIR))/%,%,$(abspath $(CURDIR)))
+include $(TESTS_DIR)/base.make
 
 ANALYZER = crashcontext
 
@@ -23,15 +21,15 @@ $(OBJECTS): $(SOURCES)
 
 # analyze a single source file and generate the test results for it
 issues-%.exp.test: $(JAVA_DEPS) %.stacktrace.json %.java
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing crashcontext: $*,\
 	  $(INFER_BIN) -a $(ANALYZER) -o infer-out-$* --stacktrace $*.stacktrace.json \
 	  -- $(JAVAC) -cp $(CLASSPATH) $*.java)
 #	add a newline at the end of the json when creating the exp.test
-	$(COPY) infer-out-$*/crashcontext/crashcontext.json $@ && echo >> $@
+	$(QUIET)$(COPY) infer-out-$*/crashcontext/crashcontext.json $@ && echo >> $@
 
 # combine the test results for all the source files
 issues.exp.test: $(EXP_TESTs)
-	cat $^ > issues.exp.test
+	$(QUIET)cat $^ > $@
 
 default: compile
 
@@ -47,7 +45,7 @@ print: issues.exp.test
 .PHONY: test
 test: issues.exp.test
 	$(QUIET)cd $(TESTS_DIR) && \
-	diff -u $(TEST_REL_DIR)/issues.exp $(TEST_REL_DIR)/issues.exp.test
+	$(call check_no_diff,$(TEST_REL_DIR)/issues.exp,$(TEST_REL_DIR)/issues.exp.test)
 
 .PHONY: replace
 replace: issues.exp.test
diff --git a/infer/tests/codetoanalyze/objc/errors/Makefile b/infer/tests/codetoanalyze/objc/errors/Makefile
index 4e9f7d982..79ca230ae 100644
--- a/infer/tests/codetoanalyze/objc/errors/Makefile
+++ b/infer/tests/codetoanalyze/objc/errors/Makefile
@@ -115,28 +115,28 @@ $(OBJECTS_BASE) $(OBJECTS_BUCKETS_ALL): $(SOURCES_BASE) $(SOURCES_BUCKET_ALL)
 	clang $(CLANG_OPTIONS) $*
 
 infer-out-all/report.json: $(CLANG_DEPS) $(SOURCES_BUCKET_ALL)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer/Objective-C with all memleak buckets,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --ml-buckets all -o infer-out-all -- \
 	  clang $(CLANG_OPTIONS) $(SOURCES_BUCKET_ALL))
-	grep "DUPLICATE_SYMBOLS" infer-out-all/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out-all/duplicates.txt)
 
 infer-out-arc/report.json: $(CLANG_DEPS) $(SOURCES_ARC)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer/Objective-C with arc memleak buckets,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --ml-buckets cf -o infer-out-arc -- \
 	  clang $(CLANG_OPTIONS) -fobjc-arc $(SOURCES_ARC))
-	grep "DUPLICATE_SYMBOLS" infer-out-arc/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out-arc/duplicates.txt)
 
 infer-out/report.json: $(CLANG_DEPS) $(SOURCES_DEFAULT)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer/Objective-C with CF memleak buckets,\
 	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --ml-buckets cf -o infer-out -- \
 	  clang $(CLANG_OPTIONS) $(SOURCES_DEFAULT))
-	grep "DUPLICATE_SYMBOLS" infer-out/duplicates.txt; test $$? -ne 0
+	$(QUIET)$(call check_no_duplicates,infer-out/duplicates.txt)
 
 issues.exp.test: infer-out-all/report.json infer-out-arc/report.json infer-out/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.all \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.all \
 	  --from-json-report infer-out-all/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.arc \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.arc \
 	  --from-json-report infer-out-arc/report.json
-	$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.default \
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@.default \
 	  --from-json-report infer-out/report.json
-	cat $@.all $@.arc $@.default > $@
+	$(QUIET)cat $@.all $@.arc $@.default > $@
diff --git a/infer/tests/differential.make b/infer/tests/differential.make
index cce910c18..c41fb0ace 100644
--- a/infer/tests/differential.make
+++ b/infer/tests/differential.make
@@ -8,8 +8,7 @@
 # Targets that must be defined: CURRENT_REPORT and PREVIOUS_REPORT
 # Optional variables: DIFFERENTIAL_ARGS, CLEAN_EXTRA
 
-ROOT_DIR = $(TESTS_DIR)/../..
-include $(ROOT_DIR)/Makefile.config
+include $(TESTS_DIR)/base.make
 
 INFER_OUT = infer-out
 DIFFERENTIAL_REPORT = $(INFER_OUT)/differential/introduced.json
@@ -32,20 +31,21 @@ $(PREVIOUS_REPORT): $(CURRENT_REPORT)
 analyze: $(CURRENT_REPORT) $(PREVIOUS_REPORT)
 
 $(DIFFERENTIAL_REPORT): $(CURRENT_REPORT) $(PREVIOUS_REPORT)
-	$(INFER_BIN) -o $(INFER_OUT) --project-root $(CURDIR) --diff \
+	$(QUIET)$(call silent_on_success,Computing results difference in $(TEST_REL_DIR),\
+	  $(INFER_BIN) -o $(INFER_OUT) --project-root $(CURDIR) --diff \
 		--report-current $(CURRENT_REPORT) --report-previous $(PREVIOUS_REPORT) \
-		$(DIFFERENTIAL_ARGS)
+		$(DIFFERENTIAL_ARGS))
 
 $(EXPECTED_TEST_OUTPUT): $(DIFFERENTIAL_REPORT) $(INFERPRINT_BIN)
-	$(INFERPRINT_BIN) \
+	$(QUIET)$(INFERPRINT_BIN) \
 		--issues-fields $(INFERPRINT_ISSUES_FIELDS) \
 		--from-json-report $(INFER_OUT)/differential/introduced.json \
 		--issues-tests introduced.exp.test
-	$(INFERPRINT_BIN) \
+	$(QUIET)$(INFERPRINT_BIN) \
 		--issues-fields $(INFERPRINT_ISSUES_FIELDS) \
 		--from-json-report $(INFER_OUT)/differential/fixed.json \
 		--issues-tests fixed.exp.test
-	$(INFERPRINT_BIN) \
+	$(QUIET)$(INFERPRINT_BIN) \
 		--issues-fields $(INFERPRINT_ISSUES_FIELDS) \
 		--from-json-report $(INFER_OUT)/differential/preexisting.json \
 		--issues-tests preexisting.exp.test
@@ -55,9 +55,9 @@ print: $(EXPECTED_TEST_OUTPUT)
 
 .PHONY: test
 test: print
-	diff -u introduced.exp introduced.exp.test
-	diff -u fixed.exp fixed.exp.test
-	diff -u preexisting.exp preexisting.exp.test
+	$(QUIET)$(call check_no_diff,introduced.exp,introduced.exp.test)
+	$(QUIET)$(call check_no_diff,fixed.exp,fixed.exp.test)
+	$(QUIET)$(call check_no_diff,preexisting.exp,preexisting.exp.test)
 
 .PHONY: replace
 replace: $(EXPECTED_TEST_OUTPUT)
diff --git a/infer/tests/infer.make b/infer/tests/infer.make
new file mode 100644
index 000000000..35e9b9ae3
--- /dev/null
+++ b/infer/tests/infer.make
@@ -0,0 +1,40 @@
+# Copyright (c) 2016 - present Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+include $(TESTS_DIR)/base.make
+
+# useful to print non-default analyzer
+ANALYZER_STRING=$(shell if [ -n $(ANALYZER) ] && [ $(ANALYZER) != infer ]; then \
+  printf ' ($(ANALYZER))'; fi)
+
+default: compile
+
+issues.exp.test$(TEST_SUFFIX): infer-out$(TEST_SUFFIX)/report.json $(INFERPRINT_BIN)
+	$(QUIET)$(INFERPRINT_BIN) -q -a $(ANALYZER) $(INFERPRINT_OPTIONS) $@ --from-json-report $<
+
+.PHONY: compile
+compile: $(OBJECTS)
+
+.PHONY: analyze
+analyze: infer-out$(TEST_SUFFIX)/report.json
+
+.PHONY: print
+print: issues.exp.test$(TEST_SUFFIX)
+
+.PHONY: test
+test: issues.exp.test$(TEST_SUFFIX)
+	$(QUIET)cd $(TESTS_DIR) && \
+	$(call check_no_diff,$(TEST_REL_DIR)/issues.exp,$(TEST_REL_DIR)/issues.exp.test$(TEST_SUFFIX))
+
+.PHONY: print
+replace: issues.exp.test$(TEST_SUFFIX)
+	cp $< issues.exp
+
+.PHONY: clean
+clean:
+	$(REMOVE_DIR) codetoanalyze com issues.exp.test$(TEST_SUFFIX) infer-out$(TEST_SUFFIX) \
+	  $(OBJECTS) $(CLEAN_EXTRA)
diff --git a/infer/tests/javac.make b/infer/tests/javac.make
index 3f4007023..fb5d42d97 100644
--- a/infer/tests/javac.make
+++ b/infer/tests/javac.make
@@ -15,7 +15,7 @@
 OBJECTS = $(patsubst %.java,%.class,$(SOURCES))
 
 include $(TESTS_DIR)/java.make
-include $(TESTS_DIR)/base.make
+include $(TESTS_DIR)/infer.make
 
 PROJECT_ROOT ?= $(TESTS_DIR)
 
@@ -23,6 +23,6 @@ $(OBJECTS): $(SOURCES)
 	$(JAVAC) -cp $(CLASSPATH) $(SOURCES)
 
 infer-out/report.json: $(JAVA_DEPS) $(SOURCES)
-	$(call silent_on_success,\
+	$(QUIET)$(call silent_on_success,Testing infer/java$(ANALYZER_STRING) in $(TEST_REL_DIR),\
 	  $(INFER_BIN) -a $(ANALYZER) --inferconfig-home . --project-root $(PROJECT_ROOT) $(INFER_OPTIONS) -- \
 	    $(JAVAC) -cp $(CLASSPATH) $(SOURCES))