[clang] replace yojson frontend with biniou frontend

Summary:
Use the new clang plugin that outputs biniou instead of Yojson. This binary format is more compact, which makes the frontend a little faster (5 to 10%).

@update-submodule: facebook-clang-plugins

.gitignore

@@ -41,6 +41,8 @@ org.eclipse.core.resources.prefs
 # Directories and files generated by Infer
 *.o.astlog
 *.o.sh
+*.o.bdump
+*.o.biniou
 infer-sources.tar.gz
 infer-osx-*.tar.xz
 infer-linux64-*.tar.xz
@@ -65,6 +67,7 @@ buck-out/
 /infer/lib/specs/objc_models
 /infer/lib/specs/clean_models
 /scripts/checkCopyright
+/infer/etc/clang_ast.dict
 
 #atdgen stubs
 /infer/src/backend/jsonbug_*
@@ -78,6 +81,8 @@ buck-out/
 
 /infer/_build-infer/
 
+/infer/src/clang/clang_ast_b.ml
+/infer/src/clang/clang_ast_b.mli
 /infer/src/clang/clang_ast_j.ml
 /infer/src/clang/clang_ast_j.mli
 /infer/src/clang/clang_ast_proj.ml

compile-fcp.sh

@@ -11,9 +11,6 @@ set -e
 set -x
 
 # This script installs the facebook-clang-plugins
-#
-# TODO (t5939566): ADD INSTRUCTIONS ON HOW TO CUSTOMIZE THE ENVVARS FOR
-# THE INSTALLATION OF THE PLUGINS.
 
 INFER_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 PLUGIN_DIR="$INFER_ROOT/facebook-clang-plugins"
@@ -32,9 +29,9 @@ if [ "$platform" == 'Linux' ]; then
     export SDKPATH=""
     export PATH="$PLUGIN_DIR/clang/bin:$PATH"
     [ -z "$CC" ] && export CC="$PLUGIN_DIR/clang/bin/clang"
+    make_vars+=" CC=$CC"
     [ -z "$CXX" ] && export CXX="$PLUGIN_DIR/clang/bin/clang++"
-    [ -z "$CFLAGS" ] && export CFLAGS="-std=c++11 -fPIC"
-    [ -z "$LDFLAGS" ] && export LDFLAGS="-shared"
+    make_vars+=" CXX=$CXX"
     [ -z "$CLANG_PREFIX" ] && export CLANG_PREFIX="$PLUGIN_DIR/clang"
     [ -z "$LLVM_INCLUDES" ] && export LLVM_INCLUDES="$PLUGIN_DIR/clang/include"
     [ -z "$CLANG_INCLUDES" ] && export CLANG_INCLUDES="$LLVM_INCLUDES $CLANG_PREFIX/include"
@@ -43,7 +40,7 @@ fi
 # compile
 make clean
 make -C clang-ocaml clean
-make
+make $make_vars
 make -C clang-ocaml all build/clang_ast_proj.ml build/clang_ast_proj.mli
 popd
 
@@ -51,6 +48,6 @@ popd
 echo "int main() { return 0; }" | \
     $CLANG_EXEC -o /dev/null -x c \
         -Xclang -load -Xclang $PLUGIN_DIR/libtooling/build/FacebookClangPlugin.dylib \
-        -Xclang -plugin -Xclang YojsonASTExporter -c - > /dev/null \
+        -Xclang -plugin -Xclang BiniouASTExporter -c - > /dev/null \
 || { echo "$CLANG_EXEC and the facebook-clang-plugins are not working.";
      echo "Check you're using the right revision of clang, then retry"; exit 1; }

infer/lib/clang/clang_general_wrapper

@@ -6,6 +6,7 @@ PARENT=$(dirname "$0")
 SCRIPT_DIR=$(cd "$PARENT" && pwd)
 SCRIPT_DIR="${SCRIPT_DIR%/}"
 BIN_DIR="${SCRIPT_DIR}/../../bin"
+ETC_DIR="${SCRIPT_DIR}/../../etc"
 
 #### Configuration ####
 # path to the wrapped clang compiler to invoke
@@ -18,7 +19,7 @@ INFERCLANG_LOG_FILE_EXT=".astlog"
 CLANG_PLUGIN_REL_PATH="facebook-clang-plugins/libtooling/build/FacebookClangPlugin.dylib"
 PLUGIN_PATH="${SCRIPT_DIR}/../../../${CLANG_PLUGIN_REL_PATH}"
 # name of the plugin to use
-PLUGIN_NAME="YojsonASTExporter"
+PLUGIN_NAME="BiniouASTExporter"
 # output directory of the plugin
 RESULTS_DIR="${FCP_RESULTS_DIR}"
 # space-separated list of source file extensions to compile, where the plugin is needed
@@ -129,12 +130,6 @@ then
         if [ -n "$SYNTAX_ONLY" ]; then
             EXTRA_ARGS+=("-fsyntax-only")
         fi
-        if [ -z "$DEBUG_MODE" ]; then
-            # uglify json
-            EXTRA_ARGS+=(
-                "-Xclang" "-plugin-arg-${PLUGIN_NAME}"
-                "-Xclang" "PRETTIFY_JSON=0")
-        fi
         unset IFS
 
         # using always the original clang command for several reasons:
@@ -163,7 +158,13 @@ if [ -n "$ATTACH_PLUGIN" ]; then
 
     if [ -n "$DEBUG_MODE" ]; then
         # Emit the clang command with the extra args piped to InferClang
-        echo "${CLANG_CMD[@]} | tee ${OBJECT_FILENAME}.yjson | ${INFERCLANG_CMD[@]}" > "${OBJECT_FILENAME}${CMD_FILE_EXT}"
+        echo "${CLANG_CMD[@]} " \
+             "| tee ${OBJECT_FILENAME}.biniou " \
+             "| ${INFERCLANG_CMD[@]}" \
+             > "${OBJECT_FILENAME}${CMD_FILE_EXT}"
+        echo "bdump -x -d ${ETC_DIR}/clang_ast.dict -w '!!DUMMY!!' ${OBJECT_FILENAME}.biniou " \
+             "> ${OBJECT_FILENAME}.bdump" \
+             >> "${OBJECT_FILENAME}${CMD_FILE_EXT}"
         # Emit the InferClang cmd used to run the frontend
         INFERCLANG_LOG_FILE="${OBJECT_FILENAME}${INFERCLANG_LOG_FILE_EXT}"
         echo "${INFERCLANG_CMD[@]}" > "$INFERCLANG_LOG_FILE"

infer/src/Makefile

@@ -16,7 +16,7 @@ OCAML_INCLUDE_DIR = $(shell ocamlc -where)
 
 #### ATDGEN declarations ####
 
-ATDGEN_SUFFIXES = _t.ml _t.mli _j.ml _j.mli _v.ml _v.mli
+ATDGEN = atdgen
 
 ATDGEN_INCLUDE_DIR = $(shell ocamlfind query atdgen)
 BINIOU_INCLUDE_DIR = $(shell ocamlfind query biniou)
@@ -24,8 +24,8 @@ YOJSON_INCLUDE_DIR = $(shell ocamlfind query yojson)
 EASYFORMAT_INCLUDE_DIR = $(shell ocamlfind query easy-format)
 
 ATDGEN_INCLUDES = -I,$(EASYFORMAT_INCLUDE_DIR),-I,$(BINIOU_INCLUDE_DIR),-I,$(YOJSON_INCLUDE_DIR),-I,$(ATDGEN_INCLUDE_DIR)
-ATDGEN_LIBS = biniou atdgen
-ATDGEN_MODS = easy_format yojson ag_oj_run ag_util
+ATDGEN_LIBS = atdgen
+ATDGEN_MODS = easy_format bi_util bi_share bi_outbuf bi_inbuf bi_vint bi_io yojson ag_oj_run ag_ob_run ag_util
 ATDGEN_OPTIONS = -cflags -annot -lflags $(ATDGEN_INCLUDES) -cflags $(ATDGEN_INCLUDES) $(addprefix -lib ,$(ATDGEN_LIBS)) $(addprefix -mod ,$(ATDGEN_MODS))
 
 #### Global declarations ####
@@ -38,6 +38,7 @@ BUILDDIR = ../_build-infer
 ANNOTDIR = $(ROOT)/infer/src/_build
 BINDIR = $(ROOT)/infer/bin
 SCRIPTDIR = $(ROOT)/scripts
+ETC_DIR = $(ROOT)/infer/etc
 
 ifneq ($(wildcard $(BUILDDIR)/sanitize.sh),)
 	SANITIZE_SCRIPT = $(BUILDDIR)/sanitize.sh
@@ -63,7 +64,8 @@ TYPEPROP_BINARY = $(BINDIR)/Typeprop
 
 INFERPRINT_ATDGEN_STUB_BASE = $(BACKEND_SOURCES)/jsonbug
 INFERPRINT_ATDGEN_STUB_ATD = $(INFERPRINT_ATDGEN_STUB_BASE).atd
-INFERPRINT_ATDGEN_STUBS = $(addprefix $(INFERPRINT_ATDGEN_STUB_BASE), $(ATDGEN_SUFFIXES))
+INFERPRINT_ATDGEN_SUFFIXES = _t.ml _t.mli _j.ml _j.mli _v.ml _v.mli
+INFERPRINT_ATDGEN_STUBS = $(addprefix $(INFERPRINT_ATDGEN_STUB_BASE), $(INFERPRINT_ATDGEN_SUFFIXES))
 
 INFERPRINT_MAIN = $(BACKEND_SOURCES)/inferprint
 INFERPRINT_BINARY = $(BINDIR)/InferPrint
@@ -100,12 +102,15 @@ CLANG_OCAML_BUILD = $(CLANG_OCAML_ROOT)/build
 CLANG_AST_BASE_NAME = clang_ast
 CLANG_ATDGEN_STUB_BASE = $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME)
 CLANG_ATDGEN_STUB_ATD = $(CLANG_OCAML_BUILD)/$(CLANG_AST_BASE_NAME).atd
-CLANG_ATDGEN_STUBS = $(addprefix $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME), $(ATDGEN_SUFFIXES))
+CLANG_ATDGEN_SUFFIXES = _t.ml _t.mli _b.ml _b.mli _j.ml _j.mli _v.ml _v.mli
+CLANG_ATDGEN_STUBS = $(addprefix $(CLANG_SOURCES)/$(CLANG_AST_BASE_NAME), $(CLANG_ATDGEN_SUFFIXES))
 
 INFER_CLANG_AST_PROJ = $(addprefix $(CLANG_SOURCES)/, clang_ast_proj.ml clang_ast_proj.mli)
 FCP_CLANG_AST_PROJ = $(addprefix $(CLANG_OCAML_BUILD)/, clang_ast_proj.ml clang_ast_proj.mli)
 FCP_CLANG_AST_MAIN = $(addprefix $(CLANG_OCAML_ROOT)/, clang_ast_visit.ml clang_ast_main.ml)
 
+CLANG_BINIOU_DICT = $(ETC_DIR)/clang_ast.dict
+
 #### LLVM declarations ####
 
 LLVM_SOURCES = llvm
@@ -139,7 +144,7 @@ all: java clang llvm checkCopyright
 
 java: build_java annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERJAVA_BINARY)
 
-clang: build_clang annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERCLANG_BINARY)
+clang: build_clang annotations $(INFERANALYZE_BINARY) $(INFERPRINT_BINARY) $(INFERCLANG_BINARY) $(CLANG_BINIOU_DICT)
 
 llvm: build_llvm annotations $(INFERLLVM_BINARY)
 
@@ -161,20 +166,28 @@ annotations:
 	rsync -r --delete --exclude=*.ml* --exclude=*.o --exclude=*.cm* --exclude=*.native $(BUILDDIR)/* $(ANNOTDIR)
 
 $(INFERPRINT_ATDGEN_STUBS): $(INFERPRINT_ATDGEN_STUB_ATD)
-	atdgen -t $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE)
-	atdgen -j $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE)
+	$(ATDGEN) -t $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE)
+	$(ATDGEN) -j $(INFERPRINT_ATDGEN_STUB_ATD) -o $(INFERPRINT_ATDGEN_STUB_BASE)
 
 $(CLANG_ATDGEN_STUBS) $(INFER_CLANG_AST_PROJ): $(CLANG_ATDGEN_STUB_ATD) $(CLANG_PLUGIN_BINARIES) $(FCP_CLANG_AST_PROJ) $(FCP_CLANG_AST_MAIN)
 # rebuild the artifacts of the AST files whenever they're upated in FCP
 # also copy the ast_proj files whenever they are updated
-	atdgen -rec -t $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
-	atdgen -rec -j $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
-	atdgen -rec -v $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
+	$(ATDGEN) -rec -t $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
+	$(ATDGEN) -rec -b $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
+	$(ATDGEN) -rec -j $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
+	$(ATDGEN) -rec -v $(CLANG_ATDGEN_STUB_ATD) -o $(CLANG_ATDGEN_STUB_BASE)
 	$(COPY) $(CLANG_OCAML_BUILD)/clang_ast_proj.ml $(CLANG_SOURCES)
 	$(COPY) $(CLANG_OCAML_BUILD)/clang_ast_proj.mli $(CLANG_SOURCES)
 	$(COPY) $(CLANG_OCAML_ROOT)/clang_ast_visit.ml $(CLANG_SOURCES)
 	$(COPY) $(CLANG_OCAML_ROOT)/clang_ast_main.ml $(CLANG_SOURCES)
 
+$(CLANG_BINIOU_DICT): $(CLANG_ATDGEN_STUB_ATD)
+# overapproximation of the words we need in the biniou dictionary
+# the long litany of symbols is [:punct:] minus "_-'"
+	tr -s '[*!"#\$%&\(\)\+,\\\.\/:;<=>\?@\[\\\\]^`\{|\}~[:space:]]' '\n' \
+	  < $(CLANG_ATDGEN_STUB_ATD) \
+	  | sort | uniq  \
+	  > $(CLANG_BINIOU_DICT)
 
 init: sanitize version $(BUILDDIR)
 

infer/src/clang/cAstProcessor.mli

@@ -12,7 +12,7 @@
     w.r.t. the previous one. This module processes the AST and makes locations explicit. *)
 
 (** Pretty print an AST. *)
-val pp_ast_decl : Format.formatter -> Clang_ast_j.decl -> unit
+val pp_ast_decl : Format.formatter -> Clang_ast_t.decl -> unit
 
 (** Preprocess the AST to make locations explicit. *)
-val preprocess_ast_decl : Clang_ast_j.decl -> Clang_ast_j.decl
+val preprocess_ast_decl : Clang_ast_t.decl -> Clang_ast_t.decl

infer/src/clang/cMain.ml

@@ -90,10 +90,10 @@ let () =
 (* This function reads the json file in fname, validates it, and encoded in the AST data structure*)
 (* defined in Clang_ast_t.  *)
 let validate_decl_from_file fname =
-  Ag_util.Json.from_file Clang_ast_j.read_decl fname
+  Ag_util.Biniou.from_file Clang_ast_b.read_decl fname
 
 let validate_decl_from_stdin () =
-  Ag_util.Json.from_channel Clang_ast_j.read_decl stdin
+  Ag_util.Biniou.from_channel Clang_ast_b.read_decl stdin
 
 let do_run source_path ast_path =
   try