From d6379fb7bea2c97c9bc4627e5c487ac4fea3fe94 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Mon, 24 Aug 2015 14:15:39 -0100 Subject: [PATCH] [clang] replace yojson frontend with biniou frontend Summary: Use the new clang plugin that outputs biniou instead of Yojson. This binary format is more compact, which makes the frontend a little faster (5 to 10%). @update-submodule: facebook-clang-plugins --- .gitignore | 5 ++++ compile-fcp.sh | 11 +++------ infer/etc/.keep | 0 infer/lib/clang/clang_general_wrapper | 17 +++++++------ infer/src/Makefile | 35 ++++++++++++++++++--------- infer/src/clang/cAstProcessor.mli | 4 +-- infer/src/clang/cMain.ml | 4 +-- 7 files changed, 46 insertions(+), 30 deletions(-) create mode 100644 infer/etc/.keep diff --git a/.gitignore b/.gitignore index 38c0e8439..4b97e6051 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,8 @@ org.eclipse.core.resources.prefs # Directories and files generated by Infer *.o.astlog *.o.sh +*.o.bdump +*.o.biniou infer-sources.tar.gz infer-osx-*.tar.xz infer-linux64-*.tar.xz @@ -65,6 +67,7 @@ buck-out/ /infer/lib/specs/objc_models /infer/lib/specs/clean_models /scripts/checkCopyright +/infer/etc/clang_ast.dict #atdgen stubs /infer/src/backend/jsonbug_* @@ -78,6 +81,8 @@ buck-out/ /infer/_build-infer/ +/infer/src/clang/clang_ast_b.ml +/infer/src/clang/clang_ast_b.mli /infer/src/clang/clang_ast_j.ml /infer/src/clang/clang_ast_j.mli /infer/src/clang/clang_ast_proj.ml diff --git a/compile-fcp.sh b/compile-fcp.sh index b4a383744..382c63658 100755 --- a/compile-fcp.sh +++ b/compile-fcp.sh @@ -11,9 +11,6 @@ set -e set -x # This script installs the facebook-clang-plugins -# -# TODO (t5939566): ADD INSTRUCTIONS ON HOW TO CUSTOMIZE THE ENVVARS FOR -# THE INSTALLATION OF THE PLUGINS. INFER_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" PLUGIN_DIR="$INFER_ROOT/facebook-clang-plugins" @@ -32,9 +29,9 @@ if [ "$platform" == 'Linux' ]; then export SDKPATH="" export PATH="$PLUGIN_DIR/clang/bin:$PATH" [ -z "$CC" ] && export CC="$PLUGIN_DIR/clang/bin/clang" + make_vars+=" CC=$CC" [ -z "$CXX" ] && export CXX="$PLUGIN_DIR/clang/bin/clang++" - [ -z "$CFLAGS" ] && export CFLAGS="-std=c++11 -fPIC" - [ -z "$LDFLAGS" ] && export LDFLAGS="-shared" + make_vars+=" CXX=$CXX" [ -z "$CLANG_PREFIX" ] && export CLANG_PREFIX="$PLUGIN_DIR/clang" [ -z "$LLVM_INCLUDES" ] && export LLVM_INCLUDES="$PLUGIN_DIR/clang/include" [ -z "$CLANG_INCLUDES" ] && export CLANG_INCLUDES="$LLVM_INCLUDES $CLANG_PREFIX/include" @@ -43,7 +40,7 @@ fi # compile make clean make -C clang-ocaml clean -make +make $make_vars make -C clang-ocaml all build/clang_ast_proj.ml build/clang_ast_proj.mli popd @@ -51,6 +48,6 @@ popd echo "int main() { return 0; }" | \ $CLANG_EXEC -o /dev/null -x c \ -Xclang -load -Xclang $PLUGIN_DIR/libtooling/build/FacebookClangPlugin.dylib \ - -Xclang -plugin -Xclang YojsonASTExporter -c - > /dev/null \ + -Xclang -plugin -Xclang BiniouASTExporter -c - > /dev/null \ || { echo "$CLANG_EXEC and the facebook-clang-plugins are not working."; echo "Check you're using the right revision of clang, then retry"; exit 1; } diff --git a/infer/etc/.keep b/infer/etc/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/infer/lib/clang/clang_general_wrapper b/infer/lib/clang/clang_general_wrapper index db409b2aa..4020b58ba 100755 --- a/infer/lib/clang/clang_general_wrapper +++ b/infer/lib/clang/clang_general_wrapper @@ -6,6 +6,7 @@ PARENT=$(dirname "$0") SCRIPT_DIR=$(cd "$PARENT" && pwd) SCRIPT_DIR="${SCRIPT_DIR%/}" BIN_DIR="${SCRIPT_DIR}/../../bin" +ETC_DIR="${SCRIPT_DIR}/../../etc" #### Configuration #### # path to the wrapped clang compiler to invoke @@ -18,7 +19,7 @@ INFERCLANG_LOG_FILE_EXT=".astlog" CLANG_PLUGIN_REL_PATH="facebook-clang-plugins/libtooling/build/FacebookClangPlugin.dylib" PLUGIN_PATH="${SCRIPT_DIR}/../../../${CLANG_PLUGIN_REL_PATH}" # name of the plugin to use -PLUGIN_NAME="YojsonASTExporter" +PLUGIN_NAME="BiniouASTExporter" # output directory of the plugin RESULTS_DIR="${FCP_RESULTS_DIR}" # space-separated list of source file extensions to compile, where the plugin is needed @@ -129,12 +130,6 @@ then if [ -n "$SYNTAX_ONLY" ]; then EXTRA_ARGS+=("-fsyntax-only") fi - if [ -z "$DEBUG_MODE" ]; then - # uglify json - EXTRA_ARGS+=( - "-Xclang" "-plugin-arg-${PLUGIN_NAME}" - "-Xclang" "PRETTIFY_JSON=0") - fi unset IFS # using always the original clang command for several reasons: @@ -163,7 +158,13 @@ if [ -n "$ATTACH_PLUGIN" ]; then if [ -n "$DEBUG_MODE" ]; then # Emit the clang command with the extra args piped to InferClang - echo "${CLANG_CMD[@]} | tee ${OBJECT_FILENAME}.yjson | ${INFERCLANG_CMD[@]}" > "${OBJECT_FILENAME}${CMD_FILE_EXT}" + echo "${CLANG_CMD[@]} " \ + "| tee ${OBJECT_FILENAME}.biniou " \ + "| ${INFERCLANG_CMD[@]}" \ + > "${OBJECT_FILENAME}${CMD_FILE_EXT}" + echo "bdump -x -d ${ETC_DIR}/clang_ast.dict -w '!!DUMMY!!' ${OBJECT_FILENAME}.biniou " \ + "> ${OBJECT_FILENAME}.bdump" \ + >> "${OBJECT_FILENAME}${CMD_FILE_EXT}" # Emit the InferClang cmd used to run the frontend INFERCLANG_LOG_FILE="${OBJECT_FILENAME}${INFERCLANG_LOG_FILE_EXT}" echo "${INFERCLANG_CMD[@]}" > "$INFERCLANG_LOG_FILE" diff --git a/infer/src/Makefile b/infer/src/Makefile index e5779d55c..ccc1cf236 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -16,7 +16,7 @@ OCAML_INCLUDE_DIR = $(shell ocamlc -where) #### ATDGEN declarations #### -ATDGEN_SUFFIXES = _t.ml _t.mli _j.ml _j.mli _v.ml _v.mli +ATDGEN = atdgen ATDGEN_INCLUDE_DIR = $(shell ocamlfind query atdgen) BINIOU_INCLUDE_DIR = $(shell ocamlfind query biniou) @@ -24,8 +24,8 @@ YOJSON_INCLUDE_DIR = $(shell ocamlfind query yojson) EASYFORMAT_INCLUDE_DIR = $(shell ocamlfind query easy-format) ATDGEN_INCLUDES = -I,$(EASYFORMAT_INCLUDE_DIR),-I,$(BINIOU_INCLUDE_DIR),-I,$(YOJSON_INCLUDE_DIR),-I,$(ATDGEN_INCLUDE_DIR) -ATDGEN_LIBS = biniou atdgen -ATDGEN_MODS = easy_format yojson ag_oj_run ag_util +ATDGEN_LIBS = atdgen +ATDGEN_MODS = easy_format bi_util bi_share bi_outbuf bi_inbuf bi_vint bi_io yojson ag_oj_run ag_ob_run ag_util ATDGEN_OPTIONS = -cflags -annot -lflags $(ATDGEN_INCLUDES) -cflags $(ATDGEN_INCLUDES) $(addprefix -lib ,$(ATDGEN_LIBS)) $(addprefix -mod ,$(ATDGEN_MODS)) #### Global declarations #### @@ -38,6 +38,7 @@ BUILDDIR = ../_build-infer ANNOTDIR = $(ROOT)/infer/src/_build BINDIR = $(ROOT)/infer/bin SCRIPTDIR = $(ROOT)/scripts +ETC_DIR = $(ROOT)/infer/etc ifneq ($(wildcard $(BUILDDIR)/sanitize.sh),) SANITIZE_SCRIPT = $(BUILDDIR)/sanitize.sh @@ -63,7 +64,8 @@ TYPEPROP_BINARY = $(BINDIR)/Typeprop INFERPRINT_ATDGEN_STUB_BASE = $(BACKEND_SOURCES)/jsonbug INFERPRINT_ATDGEN_STUB_ATD = $(INFERPRINT_ATDGEN_STUB_BASE).atd -INFERPRINT_ATDGEN_STUBS = $(addprefix $(INFERPRINT_ATDGEN_STUB_BASE), $(ATDGEN_SUFFIXES)) +INFERPRINT_ATDGEN_SUFFIXES = _t.ml _t.mli _j.ml _j.mli _v.ml _v.mli +INFERPRINT_ATDGEN_STUBS = $(addprefix $(INFERPRINT_ATDGEN_STUB_BASE), $(INFERPRINT_ATDGEN_SUFFIXES)) INFERPRINT_MAIN = $(BACKEND_SOURCES)/inferprint INFERPRINT_BINARY = $(BINDIR)/InferPrint @@ -100,12 +102,15 @@ CLANG_OCAML_BUILD = $(CLANG_OCAML_ROOT)/build CLANG_AST_BASE_NAME = clang_ast CLANG_ATDGEN_STUB_BASE = $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME) CLANG_ATDGEN_STUB_ATD = $(CLANG_OCAML_BUILD)/$(CLANG_AST_BASE_NAME).atd -CLANG_ATDGEN_STUBS = $(addprefix $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME), $(ATDGEN_SUFFIXES)) +CLANG_ATDGEN_SUFFIXES = _t.ml _t.mli _b.ml _b.mli _j.ml _j.mli _v.ml _v.mli +CLANG_ATDGEN_STUBS = $(addprefix $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME), $(CLANG_ATDGEN_SUFFIXES)) INFER_CLANG_AST_PROJ = $(addprefix $(CLANG_SOURCES)/, clang_ast_proj.ml clang_ast_proj.mli) FCP_CLANG_AST_PROJ = $(addprefix $(CLANG_OCAML_BUILD)/, clang_ast_proj.ml clang_ast_proj.mli) FCP_CLANG_AST_MAIN = $(addprefix $(CLANG_OCAML_ROOT)/, clang_ast_visit.ml clang_ast_main.ml) +CLANG_BINIOU_DICT = $(ETC_DIR)/clang_ast.dict + #### LLVM declarations #### LLVM_SOURCES = llvm @@ -139,7 +144,7 @@ all: java clang llvm checkCopyright java: build_java annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERJAVA_BINARY) -clang: build_clang annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERCLANG_BINARY) +clang: build_clang annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERCLANG_BINARY) $(CLANG_BINIOU_DICT) llvm: build_llvm annotations $(INFERLLVM_BINARY) @@ -161,20 +166,28 @@ annotations: rsync -r --delete --exclude=*.ml* --exclude=*.o --exclude=*.cm* --exclude=*.native $(BUILDDIR)/* $(ANNOTDIR) $(INFERPRINT_ATDGEN_STUBS): $(INFERPRINT_ATDGEN_STUB_ATD) - atdgen -t $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE) - atdgen -j $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE) + $(ATDGEN) -t $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE) + $(ATDGEN) -j $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE) $(CLANG_ATDGEN_STUBS) $(INFER_CLANG_AST_PROJ): $(CLANG_ATDGEN_STUB_ATD) $(CLANG_PLUGIN_BINARIES) $(FCP_CLANG_AST_PROJ) $(FCP_CLANG_AST_MAIN) # rebuild the artifacts of the AST files whenever they're upated in FCP # also copy the ast_proj files whenever they are updated - atdgen -rec -t $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) - atdgen -rec -j $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) - atdgen -rec -v $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) + $(ATDGEN) -rec -t $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) + $(ATDGEN) -rec -b $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) + $(ATDGEN) -rec -j $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) + $(ATDGEN) -rec -v $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE) $(COPY) $(CLANG_OCAML_BUILD)/clang_ast_proj.ml $(CLANG_SOURCES) $(COPY) $(CLANG_OCAML_BUILD)/clang_ast_proj.mli $(CLANG_SOURCES) $(COPY) $(CLANG_OCAML_ROOT)/clang_ast_visit.ml $(CLANG_SOURCES) $(COPY) $(CLANG_OCAML_ROOT)/clang_ast_main.ml $(CLANG_SOURCES) +$(CLANG_BINIOU_DICT): $(CLANG_ATDGEN_STUB_ATD) +# overapproximation of the words we need in the biniou dictionary +# the long litany of symbols is [:punct:] minus "_-'" + tr -s '[*!"#\$%&\(\)\+,\\\.\/:;<=>\?@\[\\\\]^`\{|\}~[:space:]]' '\n' \ + < $(CLANG_ATDGEN_STUB_ATD) \ + | sort | uniq \ + > $(CLANG_BINIOU_DICT) init: sanitize version $(BUILDDIR) diff --git a/infer/src/clang/cAstProcessor.mli b/infer/src/clang/cAstProcessor.mli index eb6c7af0c..f0719083a 100644 --- a/infer/src/clang/cAstProcessor.mli +++ b/infer/src/clang/cAstProcessor.mli @@ -12,7 +12,7 @@ w.r.t. the previous one. This module processes the AST and makes locations explicit. *) (** Pretty print an AST. *) -val pp_ast_decl : Format.formatter -> Clang_ast_j.decl -> unit +val pp_ast_decl : Format.formatter -> Clang_ast_t.decl -> unit (** Preprocess the AST to make locations explicit. *) -val preprocess_ast_decl : Clang_ast_j.decl -> Clang_ast_j.decl +val preprocess_ast_decl : Clang_ast_t.decl -> Clang_ast_t.decl diff --git a/infer/src/clang/cMain.ml b/infer/src/clang/cMain.ml index b0b141e10..eb885deb1 100644 --- a/infer/src/clang/cMain.ml +++ b/infer/src/clang/cMain.ml @@ -90,10 +90,10 @@ let () = (* This function reads the json file in fname, validates it, and encoded in the AST data structure*) (* defined in Clang_ast_t. *) let validate_decl_from_file fname = - Ag_util.Json.from_file Clang_ast_j.read_decl fname + Ag_util.Biniou.from_file Clang_ast_b.read_decl fname let validate_decl_from_stdin () = - Ag_util.Json.from_channel Clang_ast_j.read_decl stdin + Ag_util.Biniou.from_channel Clang_ast_b.read_decl stdin let do_run source_path ast_path = try