[sledge] Revise test scripts

Summary:
This diff adapts the test scripts to the new sledge CLI, and reworks
them to enable checking changes with respect to a baseline. In
particular, now
```
make -C test
```
has exit code 0 if the current test results match the expected ones,
and otherwise prints the diff. Also,
```
make -C test promote
```
promotes the current test results to the new baseline.

Reviewed By: kren1

Differential Revision: D15706573

fbshipit-source-id: 0cbf3231e

sledge/.gitignore

@@ -12,4 +12,5 @@
 /test/*/*.bc
 /test/*/*.bc.err
 /test/*/*.bc.out
+/test/report.current
 _build

sledge/test/Makefile

@@ -4,8 +4,10 @@
 # LICENSE file in the root directory of this source tree.
 
 # additional arguments to pass to clang
-OPT_ARGS?=-Os
+CLANG_ARGS?=-O0
-CLANG_ARGS?=-g $(OPT_ARGS)
+
+# executable to test
+SLEDGE_EXE=$(CURDIR)/../_build/dev/src/sledge.exe
 
 # additional arguments to pass to sledge
 SLEDGE_ARGS?=
@@ -13,83 +15,66 @@ SLEDGE_ARGS?=
 # limits for each test run
 TIMEOUT?=30
 MEMOUT?=4096
-MEMOUTk=$(shell echo $$(( $(MEMOUT) * 1024 )))
 
-# executable to test
+SLEDGE=./wrap.sh $(TIMEOUT) $(MEMOUT) $(SLEDGE_EXE)
-SLEDGE_EXE=$(CURDIR)/../_build/dev/src/sledge.exe
 
-MODEL_DIR=$(CURDIR)/../model
+DIFF?=diff
 
 # configure the non-host llvm and clang
-export PATH := $(CURDIR)/../_opam/llvm/bin:$(PATH)
+export PATH := $(CURDIR)/../llvm/_install/sledge/bin:$(PATH)
 
-# code to test
+default: test
-SrcCs:=$(shell find -L * -not -path 'llvm/*' -name '*.c')
-SrcCPPs:=$(shell find -L * -not -path 'llvm/*' -name '*.cpp')
-SrcLLs:=$(shell find -L * -name '*.ll')
-SrcBCs:=$(shell find -L * -name '*.bc')
 
-GenBCs:=$(patsubst %.c,%.bc,$(SrcCs)) $(patsubst %.cpp,%.bc,$(SrcCPPs))
+# all analyze tests
+translate:
+	@find -L llvm -name '*.ll' -or -name '*.bc' \
+	 | parallel --bar $(SLEDGE) llvm translate $(SLEDGE_ARGS)
 
-TestBCs:=$(GenBCs) $(SrcBCs)
+_translate-report-raw:
-TestLLs:=$(SrcLLs)
+	@find -L llvm -name '*.out' \
-Tests:=$(TestBCs) $(TestLLs)
+	 | xargs grep "RESULT:" \
-Outs:=$(patsubst %.bc,%.bc.out,$(TestBCs)) $(patsubst %.ll,%.ll.out,$(TestLLs))
+	 | sed 's/\.out:/:	/' | sort | sort -s -t':' -k 3,4
 
-default: test
+# report all results
+translate-report-full:
+	@$(MAKE) --silent _translate-report-raw | column -ts$$'\t'
 
-$(MODEL_DIR)/cxxabi.bc : $(MODEL_DIR)/cxxabi.cpp
+# report all errors
-	@(cd $(dir $@) && clang $(CLANG_ARGS) -I../llvm/projects/libcxxabi/include -I../llvm/projects/libcxxabi/src -c -emit-llvm cxxabi.cpp)
+translate-report-errors:
+	@$(MAKE) --silent _translate-report-raw \
+	 | grep -E -v "RESULT: (Success|Invalid input)" \
+	 | column -ts$$'\t'
+
+# report errors
+translate-report:
+	@$(MAKE) --silent _translate-report-raw \
+	 | grep -E -v "RESULT: Unimplemented: (coroutines|landingpad of type other than {i8\*, i32}|windows exception handling|non-integral pointer types|types with undetermined size):" \
+	 | grep -E -v "RESULT: (Success|Invalid input)" \
+	 | column -ts$$'\t'
 
 # compile c to llvm bitcode
 %.bc : %.c
-	@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).c -o - | opt $(OPT_ARGS) -o $(notdir $*).bc)
+	@(cd $(dir $*) && clang -g -c -emit-llvm $(CLANG_ARGS) $(notdir $*).c -o $(notdir $*).bc)
-
-#  $(MODEL_DIR)/cxxabi.bc
-#	@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).c -o - | llvm-link $(MODEL_DIR)/cxxabi.bc - | opt $(OPT_ARGS) -o $(notdir $*).bc)
 
 # compile c++ to llvm bitcode
 %.bc : %.cpp
-	@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).cpp -o - | opt $(OPT_ARGS) -o $(notdir $*).bc)
+	@(cd $(dir $*) && clang -g -c -emit-llvm $(CLANG_ARGS) $(notdir $*).cpp -o $(notdir $*).bc)
-
+
-#  $(MODEL_DIR)/cxxabi.bc
+# code to test analyze
-#	@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).cpp -o - | llvm-link $(MODEL_DIR)/cxxabi.bc - | opt $(OPT_ARGS) -o $(notdir $*).bc)
+AnalyzeCs:=$(shell find * -not -path 'llvm/*' -name '*.c')
-
+AnalyzeCPPs:=$(shell find * -not -path 'llvm/*' -name '*.cpp')
-# disassemble bitcode to llvm assembly
+
-%.ll : %.bc
+AnalyzeBCs:=$(patsubst %.c,%.bc,$(AnalyzeCs)) $(patsubst %.cpp,%.bc,$(AnalyzeCPPs))
-	@(cd $(dir $*) && llvm-dis -show-annotations -o $(notdir $*).ll $(notdir $*).bc)
+AnalyzeLLs:=$(shell find * -not -path 'llvm/*' -name '*.ll')
-
+
-# analyze one test input
+AnalyzeTests:=$(AnalyzeBCs) $(AnalyzeLLs)
-define analyze
+
-  @bash -c ' \
+# compile all c/c++ to bc
-    if test -e $(1).$(2); then \
+compile: $(AnalyzeBCs)
-    cd $(dir $(1)); \
+
-    status=$$( \
+# all analyze tests
-        ( ulimit -t $(TIMEOUT) -v $(MEMOUTk) \
+analyze: compile
-        ; $(SLEDGE_EXE) $(SLEDGE_ARGS) $(notdir $(1).$(2)) 1> $(notdir $(1).$(2)).out 2> $(notdir $(1).$(2)).err ) \
+	@parallel --bar $(SLEDGE) llvm analyze $(SLEDGE_ARGS) ::: $(AnalyzeTests)
-    )$$?; \
-    case $$status in \
-      ( 0 | 2 ) ;; \
-      ( 137 | 152 ) echo -e "RESULT: TIMEOUT" >> $(notdir $(1).$(2)).out ;; \
-      ( * ) echo -e "\nRESULT: Internal error: "$$status >> $(notdir $(1).$(2)).out ;; \
-    esac ; \
-    exit $$status ; \
-    fi; '
-endef
-
-# analyze ll
-%.ll.out : %.ll
-	$(call analyze,$*,ll)
-
-# analyze bc
-%.bc.out : %.bc
-	$(call analyze,$*,bc)
-
-# compile all c to bc
-compile: $(GenBCs)
-
-# run all tests
-test: $(Outs)
 
 # run all tests and generate code coverage information
 BISECT_DIR=$(CURDIR)/../_coverage/out
@@ -100,66 +85,48 @@ coverage:
 	@find $(BISECT_DIR) -type f | xargs bisect-ppx-report -I ../_build/coverage/ -text ../_coverage/summary.txt -html ../_coverage/
 	@echo "open ../_coverage/index.html"
 
-# translate all tests
+_analyze-report-raw:
-translate:
+	@find * -not -path 'llvm/*' -name '*.out' \
-	$(MAKE) SLEDGE_ARGS='$(SLEDGE_ARGS) --compile-only' test
-
-# report all results
-full-report:
-	@find -L * -name '*.out' \
 	 | xargs grep "RESULT:" \
-	 | sed 's/.out:/:	/' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4  | uniq
+	 | sed 's/\.out:/:	/' | sort | sort -s -t':' -k 3,4
 
-# report all errors
+# report all results
-report-errors:
+analyze-report-full:
-	@find -L * -name '*.out' \
+	@$(MAKE) --silent _analyze-report-raw | column -ts$$'\t'
-	 | xargs grep "RESULT:" \
-	 | grep -E -v "RESULT: (Success|Invalid input)" \
-	 | sed 's/.out:/:	/' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4 | uniq
 
-# report errors
+# list tests with zero or multiple RESULT lines
-report:
+report-invalid-results:
-	@find -L * -name '*.out' \
+	@find -L * -name '*.out' -exec grep -H -c "RESULT:" {} \; \
-	 | xargs grep "RESULT:" \
+	 | grep -v ":1$"
-	 | grep -E -v "RESULT: Unimplemented: (landingpad of type other than {i8\*, i32}|windows exception handling|non-integral pointer types|types with undetermined size):" \
-	 | grep -E -v "RESULT: (Success|Invalid input)" \
-	 | sed 's/.out:/:	/' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4 | uniq
 
 # report warnings
 warnings:
-	@find -L * -name '*.out' | xargs grep -h "Warning:" | sort | uniq
+	@find -L * -name '*.out' | xargs grep -h "Warning:" | sort
+
+# run tests and check against expected results
+test:
+	-@$(MAKE) --silent --keep-going clean analyze translate 2>/dev/null
+	@$(MAKE) --silent _analyze-report-raw > report.current
+	@$(MAKE) --silent _translate-report-raw >> report.current
+	@$(DIFF) report.expected report.current
+
+# set current results as new expected results
+promote:
+	@cp report.current report.expected
 
 # remove generated bitcode files
-cleanBC:
+cleanbc:
-	@rm -f $(GenBCs)
+	@rm -f $(AnalyzeBCs)
 
 # remove result files
 cleanout:
 	@find -L * -name "*.out" -or -name '*.err' \
 	 | xargs rm -f
 
-# remove result files for TIMEOUTs
+clean: cleanbc cleanout
-cleanTO:
-	@find -L * -name "*.out" -or -name '*.err' \
-	 | xargs grep -l TIMEOUT \
-	 | xargs rm -f
-
-# remove result files for MEMOUTs
-cleanMO:
-	@find -L * -name "*.out" -or -name '*.err' \
-	 | xargs grep -l MEMOUT \
-	 | xargs rm -f
-
-# remove result files for Internal Errors
-cleanIE:
-	@find -L * -name "*.out" -or -name '*.err' \
-	 | xargs grep -l -E "RESULT: (Internal error|Unimplemented|Unknown error)" \
-	 | xargs rm -f
-
-clean: cleanBC cleanout
 
 fmt:
-	clang-format -i $(SrcCs) $(SrcCPPs)
+	clang-format -i $(AnalyzeCs) $(AnalyzeCPPs)
 
 # print any variable for Makefile debugging
 print-%:

sledge/test/exec/global_vars.c

@@ -22,7 +22,7 @@ int an_int = 0;
 int c() { return an_int; }
 
 int main() {
-  char* c1 = a_string;
+  const char* c1 = a_string;
 
   return c();
 }

sledge/test/wrap.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright (c) 2019-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# wrap execution of sledge with time and memory limits
+# and add RESULT status to output in case of an unexpected exit
+
+# usage: wrap.sh <timeout(sec)> <memout(MB)> <command> <testdir/testname>
+
+set -u
+
+timeout=$1
+memout=$(( $2*1024 ))
+command=${@: 3: $#-3}
+test=${@: -1}
+
+testdir=$(dirname $test)
+testname=$(basename $test)
+
+cd $testdir
+(
+  ulimit -t $timeout -v $memout
+  $command $testname 1> $testname.out 2> $testname.err
+)
+status=$?
+case $status in
+  ( 139 ) echo -e "RESULT: SEGFAULT" >> $testname.out ;;
+  ( 127 ) echo -e "RESULT: MEMOUT" >> $testname.out ;;
+  ( 137 | 152 ) echo -e "RESULT: TIMEOUT" >> $testname.out ;;
+  ( * ) ;;
+esac
+if ! grep -q 'RESULT:' $testname.out; then
+  echo -e "RESULT: Internal error: "$status >> $testname.out
+fi
+exit $status