pghs975uc
/
infer_clone
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[sledge] Revise test scripts

Browse Source 
Summary:
This diff adapts the test scripts to the new sledge CLI, and reworks
them to enable checking changes with respect to a baseline. In
particular, now
```
make -C test
```
has exit code 0 if the current test results match the expected ones,
and otherwise prints the diff. Also,
```
make -C test promote
```
promotes the current test results to the new baseline.

Reviewed By: kren1

Differential Revision: D15706573

fbshipit-source-id: 0cbf3231e
master
Josh Berdine 6 years ago committed by Facebook Github Bot
parent f119154a41
commit caef28f49e
5 changed files with 18848 additions and 109 deletions
Show Stats Download Patch File Download Diff File

1
sledge/.gitignore vendored
Unescape View File

@ -12,4 +12,5 @@
/test/*/*.bc
/test/*/*.bc.err
/test/*/*.bc.out
/test/report.current
_build

183
sledge/test/Makefile
Unescape View File

@ -4,8 +4,10 @@
# LICENSE file in the root directory of this source tree.
# additional arguments to pass to clang
OPT_ARGS?=-Os
CLANG_ARGS?=-g $(OPT_ARGS)
CLANG_ARGS?=-O0
# executable to test
SLEDGE_EXE=$(CURDIR)/../_build/dev/src/sledge.exe
# additional arguments to pass to sledge
SLEDGE_ARGS?=
@ -13,83 +15,66 @@ SLEDGE_ARGS?=
# limits for each test run
TIMEOUT?=30
MEMOUT?=4096
MEMOUTk=$(shell echo $$(( $(MEMOUT) * 1024 )))
# executable to test
SLEDGE_EXE=$(CURDIR)/../_build/dev/src/sledge.exe
SLEDGE=./wrap.sh $(TIMEOUT) $(MEMOUT) $(SLEDGE_EXE)
MODEL_DIR=$(CURDIR)/../model
DIFF?=diff
# configure the non-host llvm and clang
export PATH := $(CURDIR)/../_opam/llvm/bin:$(PATH)
export PATH := $(CURDIR)/../llvm/_install/sledge/bin:$(PATH)
# code to test
SrcCs:=$(shell find -L * -not -path 'llvm/*' -name '*.c')
SrcCPPs:=$(shell find -L * -not -path 'llvm/*' -name '*.cpp')
SrcLLs:=$(shell find -L * -name '*.ll')
SrcBCs:=$(shell find -L * -name '*.bc')
default: test
GenBCs:=$(patsubst %.c,%.bc,$(SrcCs)) $(patsubst %.cpp,%.bc,$(SrcCPPs))
# all analyze tests
translate:
@find -L llvm -name '*.ll' -or -name '*.bc' \
| parallel --bar $(SLEDGE) llvm translate $(SLEDGE_ARGS)
TestBCs:=$(GenBCs) $(SrcBCs)
TestLLs:=$(SrcLLs)
Tests:=$(TestBCs) $(TestLLs)
Outs:=$(patsubst %.bc,%.bc.out,$(TestBCs)) $(patsubst %.ll,%.ll.out,$(TestLLs))
_translate-report-raw:
@find -L llvm -name '*.out' \
| xargs grep "RESULT:" \
| sed 's/\.out:/: /' | sort | sort -s -t':' -k 3,4
default: test
# report all results
translate-report-full:
@$(MAKE) --silent _translate-report-raw | column -ts$$'\t'
$(MODEL_DIR)/cxxabi.bc : $(MODEL_DIR)/cxxabi.cpp
@(cd $(dir $@) && clang $(CLANG_ARGS) -I../llvm/projects/libcxxabi/include -I../llvm/projects/libcxxabi/src -c -emit-llvm cxxabi.cpp)
# report all errors
translate-report-errors:
@$(MAKE) --silent _translate-report-raw \
| grep -E -v "RESULT: (Success|Invalid input)" \
| column -ts$$'\t'
# report errors
translate-report:
@$(MAKE) --silent _translate-report-raw \
| grep -E -v "RESULT: Unimplemented: (coroutines|landingpad of type other than {i8\*, i32}|windows exception handling|non-integral pointer types|types with undetermined size):" \
| grep -E -v "RESULT: (Success|Invalid input)" \
| column -ts$$'\t'
# compile c to llvm bitcode
%.bc : %.c
@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).c -o - | opt $(OPT_ARGS) -o $(notdir $*).bc)
# $(MODEL_DIR)/cxxabi.bc
# @(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).c -o - | llvm-link $(MODEL_DIR)/cxxabi.bc - | opt $(OPT_ARGS) -o $(notdir $*).bc)
@(cd $(dir $*) && clang -g -c -emit-llvm $(CLANG_ARGS) $(notdir $*).c -o $(notdir $*).bc)
# compile c++ to llvm bitcode
%.bc : %.cpp
@(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).cpp -o - | opt $(OPT_ARGS) -o $(notdir $*).bc)
# $(MODEL_DIR)/cxxabi.bc
# @(cd $(dir $*) && clang $(CLANG_ARGS) -c -emit-llvm $(notdir $*).cpp -o - | llvm-link $(MODEL_DIR)/cxxabi.bc - | opt $(OPT_ARGS) -o $(notdir $*).bc)
# disassemble bitcode to llvm assembly
%.ll : %.bc
@(cd $(dir $*) && llvm-dis -show-annotations -o $(notdir $*).ll $(notdir $*).bc)
# analyze one test input
define analyze
@bash -c ' \
if test -e $(1).$(2); then \
cd $(dir $(1)); \
status=$$( \
( ulimit -t $(TIMEOUT) -v $(MEMOUTk) \
; $(SLEDGE_EXE) $(SLEDGE_ARGS) $(notdir $(1).$(2)) 1> $(notdir $(1).$(2)).out 2> $(notdir $(1).$(2)).err ) \
)$$?; \
case $$status in \
( 0 | 2 ) ;; \
( 137 | 152 ) echo -e "RESULT: TIMEOUT" >> $(notdir $(1).$(2)).out ;; \
( * ) echo -e "\nRESULT: Internal error: "$$status >> $(notdir $(1).$(2)).out ;; \
esac ; \
exit $$status ; \
fi; '
endef
# analyze ll
%.ll.out : %.ll
$(call analyze,$*,ll)
# analyze bc
%.bc.out : %.bc
$(call analyze,$*,bc)
# compile all c to bc
compile: $(GenBCs)
# run all tests
test: $(Outs)
@(cd $(dir $*) && clang -g -c -emit-llvm $(CLANG_ARGS) $(notdir $*).cpp -o $(notdir $*).bc)
# code to test analyze
AnalyzeCs:=$(shell find * -not -path 'llvm/*' -name '*.c')
AnalyzeCPPs:=$(shell find * -not -path 'llvm/*' -name '*.cpp')
AnalyzeBCs:=$(patsubst %.c,%.bc,$(AnalyzeCs)) $(patsubst %.cpp,%.bc,$(AnalyzeCPPs))
AnalyzeLLs:=$(shell find * -not -path 'llvm/*' -name '*.ll')
AnalyzeTests:=$(AnalyzeBCs) $(AnalyzeLLs)
# compile all c/c++ to bc
compile: $(AnalyzeBCs)
# all analyze tests
analyze: compile
@parallel --bar $(SLEDGE) llvm analyze $(SLEDGE_ARGS) ::: $(AnalyzeTests)
# run all tests and generate code coverage information
BISECT_DIR=$(CURDIR)/../_coverage/out
@ -100,66 +85,48 @@ coverage:
@find $(BISECT_DIR) -type f | xargs bisect-ppx-report -I ../_build/coverage/ -text ../_coverage/summary.txt -html ../_coverage/
@echo "open ../_coverage/index.html"
# translate all tests
translate:
$(MAKE) SLEDGE_ARGS='$(SLEDGE_ARGS) --compile-only' test
# report all results
full-report:
@find -L * -name '*.out' \
_analyze-report-raw:
@find * -not -path 'llvm/*' -name '*.out' \
| xargs grep "RESULT:" \
| sed 's/.out:/: /' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4 | uniq
| sed 's/\.out:/: /' | sort | sort -s -t':' -k 3,4
# report all errors
report-errors:
@find -L * -name '*.out' \
| xargs grep "RESULT:" \
| grep -E -v "RESULT: (Success|Invalid input)" \
| sed 's/.out:/: /' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4 | uniq
# report all results
analyze-report-full:
@$(MAKE) --silent _analyze-report-raw | column -ts$$'\t'
# report errors
report:
@find -L * -name '*.out' \
| xargs grep "RESULT:" \
| grep -E -v "RESULT: Unimplemented: (landingpad of type other than {i8\*, i32}|windows exception handling|non-integral pointer types|types with undetermined size):" \
| grep -E -v "RESULT: (Success|Invalid input)" \
| sed 's/.out:/: /' | column -ts$$'\t' | sort | sort -s -t':' -k 3,4 | uniq
# list tests with zero or multiple RESULT lines
report-invalid-results:
@find -L * -name '*.out' -exec grep -H -c "RESULT:" {} \; \
| grep -v ":1$"
# report warnings
warnings:
@find -L * -name '*.out' | xargs grep -h "Warning:" | sort | uniq
@find -L * -name '*.out' | xargs grep -h "Warning:" | sort
# run tests and check against expected results
test:
-@$(MAKE) --silent --keep-going clean analyze translate 2>/dev/null
@$(MAKE) --silent _analyze-report-raw > report.current
@$(MAKE) --silent _translate-report-raw >> report.current
@$(DIFF) report.expected report.current
# set current results as new expected results
promote:
@cp report.current report.expected
# remove generated bitcode files
cleanBC:
@rm -f $(GenBCs)
cleanbc:
@rm -f $(AnalyzeBCs)
# remove result files
cleanout:
@find -L * -name "*.out" -or -name '*.err' \
| xargs rm -f
# remove result files for TIMEOUTs
cleanTO:
@find -L * -name "*.out" -or -name '*.err' \
| xargs grep -l TIMEOUT \
| xargs rm -f
# remove result files for MEMOUTs
cleanMO:
@find -L * -name "*.out" -or -name '*.err' \
| xargs grep -l MEMOUT \
| xargs rm -f
# remove result files for Internal Errors
cleanIE:
@find -L * -name "*.out" -or -name '*.err' \
| xargs grep -l -E "RESULT: (Internal error|Unimplemented|Unknown error)" \
| xargs rm -f
clean: cleanBC cleanout
clean: cleanbc cleanout
fmt:
clang-format -i $(SrcCs) $(SrcCPPs)
clang-format -i $(AnalyzeCs) $(AnalyzeCPPs)
# print any variable for Makefile debugging
print-%:

2
sledge/test/exec/global_vars.c
Unescape View File

@ -22,7 +22,7 @@ int an_int = 0;
int c() { return an_int; }
int main() {
char* c1 = a_string;
const char* c1 = a_string;
return c();
}

18733
sledge/test/report.expected
View File

File diff suppressed because it is too large Load Diff

38
sledge/test/wrap.sh
Unescape View File

@ -0,0 +1,38 @@
#!/bin/bash
# Copyright (c) 2019-present, Facebook, Inc.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# wrap execution of sledge with time and memory limits
# and add RESULT status to output in case of an unexpected exit
# usage: wrap.sh <timeout(sec)> <memout(MB)> <command> <testdir/testname>
set -u
timeout=$1
memout=$(( $2*1024 ))
command=${@: 3: $#-3}
test=${@: -1}
testdir=$(dirname $test)
testname=$(basename $test)
cd $testdir
(
ulimit -t $timeout -v $memout
$command $testname 1> $testname.out 2> $testname.err
)
status=$?
case $status in
( 139 ) echo -e "RESULT: SEGFAULT" >> $testname.out ;;
( 127 ) echo -e "RESULT: MEMOUT" >> $testname.out ;;
( 137 | 152 ) echo -e "RESULT: TIMEOUT" >> $testname.out ;;
( * ) ;;
esac
if ! grep -q 'RESULT:' $testname.out; then
echo -e "RESULT: Internal error: "$status >> $testname.out
fi
exit $status
Write Preview
Loading…
Cancel
Save