[jbuilder] turn I{Std,List} and atd-generated files into libraries

Summary:
This simplifies the jbuild files: no need to list these files explicitly
anymore, nor to exclude them explicitly from the main `InferModules` library
(due to their different compilation flags).

Isolate common parts into jbuild.common do `cat`-based code inclusion into
jbuild files to factorize code.

Reviewed By: jberdine

Differential Revision: D5678328

fbshipit-source-id: 6d7d925

.gitignore

@@ -122,15 +122,13 @@ buck-out/
 /infer/etc/clang_ast.dict
 /infer/src/toplevel.mlpack
 
-#atdgen stubs
+# atdgen generated files
-/infer/src/backend/jsonbug_j.ml
+/infer/src/atd/*_j.ml
-/infer/src/backend/jsonbug_j.mli
+/infer/src/atd/*_j.mli
-/infer/src/backend/jsonbug_t.ml
+/infer/src/atd/*_t.ml
-/infer/src/backend/jsonbug_t.mli
+/infer/src/atd/*_t.mli
-/infer/src/checkers/stacktree_j.ml
+/infer/src/atd/clang_*.ml
-/infer/src/checkers/stacktree_j.mli
+/infer/src/atd/clang_*.mli
-/infer/src/checkers/stacktree_t.ml
-/infer/src/checkers/stacktree_t.mli
 
 # intelliJ files
 /infer/src/backend/.projectSettings
@@ -153,6 +151,9 @@ infer/src/.project
 
 # jbuilder
 /infer/src/_build
+/infer/src/jbuild.common
 /infer/src/jbuild
+/infer/src/atd/jbuild
+/infer/src/istd/jbuild
+/infer/src/scripts/jbuild
 /infer/src/jbuild-workspace
-/infer/src/scripts/.merlin

CONTRIBUTING.md

@@ -71,7 +71,7 @@ us a line at cla@fb.com. Thanks!
 
 ### OCaml
 
-- The module IStd (infer/src/base/IStd.ml) is automatically opened in every file. Beware that this
+- The module IStd (infer/src/istd/IStd.ml) is automatically opened in every file. Beware that this
   can cause weird errors such as:
 ```
 $ pwd
@@ -85,10 +85,6 @@ Error: This variant expression is expected to have type 'a list
        The constructor true does not belong to type list
 ```
 
-  If your new module cannot compile with `IStd`, for instance because it's generated code, modify
-  the line in infer/src/Makefile that adds `-open IStd` so that your module is excluded (see how
-  it's done for other such modules, eg IStd.ml).
-
 - All modules open `IStd` using `open! IStd`. This is to make that fact more explicit (there's also
   the compilation flag mentioned above), and also it helps merlin find the right types. In
   particular this also opens `Core.Std`.

Makefile

@@ -309,7 +309,7 @@ toplevel: src_build_common
 	$(MAKE) -C $(SRC_DIR) toplevel)
 
 .PHONY: checkCopyright
-checkCopyright:
+checkCopyright: src_build_common
 	$(QUIET)$(call silent_on_success,Building checkCopyright,\
 	$(MAKE) -C $(SRC_DIR) checkCopyright)
 

configure.ac

@@ -18,7 +18,7 @@ AC_INIT([Infer],
         [0.13.0],
         [https://github.com/facebook/infer/issues/])
 
-AC_CONFIG_SRCDIR([infer/src/base/IStd.ml])
+AC_CONFIG_SRCDIR([infer/src/istd/IStd.ml])
 
 # WARNING: keep in sync with above
 INFER_MAJOR=0

infer/src/.merlin

@@ -1,4 +1,6 @@
-B _build/default/**
+B _build/default
+B _build/default/atd
+B _build/default/istd
 PKG ANSITerminal
 PKG atdgen
 PKG cmdliner
@@ -16,5 +18,5 @@ PKG yojson
 PKG zip
 FLG -principal -safe-string -short-paths -strict-formats -strict-sequence
 FLG -w +a-4-9-40-41-42-44-45-48-60
-FLG -open InferBaseStdlib -open InferGenerated -open InferModules
+FLG -open InferStdlib -open InferGenerated -open InferModules -open IStd
 S **

infer/src/Makefile

@@ -28,13 +28,13 @@ INFER_MAIN = infer
 
 #### Checkers declarations ####
 
-STACKTREE_ATDGEN_STUB_BASE = checkers/stacktree
+STACKTREE_ATDGEN_STUB_BASE = atd/stacktree
 STACKTREE_ATDGEN_STUB_ATD = $(STACKTREE_ATDGEN_STUB_BASE).atd
 STACKTREE_ATDGEN_STUBS = $(addprefix $(STACKTREE_ATDGEN_STUB_BASE), $(ATDGEN_SUFFIXES))
 
 #### InferPrint declarations ####
 
-INFERPRINT_ATDGEN_STUB_BASE = backend/jsonbug
+INFERPRINT_ATDGEN_STUB_BASE = atd/jsonbug
 INFERPRINT_ATDGEN_STUB_ATD = $(INFERPRINT_ATDGEN_STUB_BASE).atd
 INFERPRINT_ATDGEN_STUBS = $(addprefix $(INFERPRINT_ATDGEN_STUB_BASE), $(ATDGEN_SUFFIXES))
 
@@ -58,7 +58,7 @@ JAVA_SOURCES = java
 
 CLANG_SOURCES = clang
 
-CLANG_PLUGIN_MIRROR = clang_plugin
+CLANG_PLUGIN_MIRROR = atd
 
 FCP_CLANG_OCAML_BUILD_DIR = $(FCP_CLANG_OCAML_DIR)/build
 
@@ -116,7 +116,9 @@ OCAML_SOURCES = \
 .PHONY: all
 all: infer
 
-$(INFER_BUILD_DIR)/.ppx/ppx_compare/ppx.exe: jbuild jbuild-workspace $(OCAML_GENERATED_SOURCES)
+$(INFER_BUILD_DIR)/.ppx/ppx_compare/ppx.exe: \
+  jbuild atd/jbuild istd/jbuild scripts/jbuild jbuild-workspace \
+  $(OCAML_GENERATED_SOURCES) $(MAKEFILE_LIST)
 #      some voodoo to make jbuilder tolerate being run in parallel: force jbuilder to build its
 #      jbuild files and some files that have been seen to race otherwise
 	jbuilder build \
@@ -215,7 +217,7 @@ toplevel: $(INFER_BUILD_DIR)/infertop.bc
 .PHONY: checkCopyright
 checkCopyright: $(CHECKCOPYRIGHT_BIN)
 
-$(CHECKCOPYRIGHT_BIN): scripts/$(CHECKCOPYRIGHT_MAIN).ml $(MAKEFILE_LIST)
+$(CHECKCOPYRIGHT_BIN): $(SRC_BUILD_COMMON) $(MAKEFILE_LIST)
 	jbuilder build $(INFER_BUILD_DIR)/scripts/$(CHECKCOPYRIGHT_MAIN).exe
 	$(INSTALL_PROGRAM) $(INFER_BUILD_DIR)/scripts/$(CHECKCOPYRIGHT_MAIN).exe $(CHECKCOPYRIGHT_BIN)
 
@@ -264,7 +266,7 @@ $(CLANG_BINIOU_DICT): $(CLANG_ATDGEN_STUB_ATD)
 	  | sort | uniq  \
 	  > $@
 
-jbuild jbuild-workspace base/Version.ml: $(MAKEFILE_LIST)
+jbuild.common jbuild-workspace base/Version.ml: $(MAKEFILE_LIST)
 	TMPFILE=$$(mktemp $@.tmp.XXXX); \
 	INFER_GIT_COMMIT=$$(git --work-tree=$(ROOT_DIR) --git-dir=$(ROOT_DIR)/.git rev-parse --short HEAD || printf "unknown"); \
 	INFER_GIT_BRANCH=$$(git --work-tree=$(ROOT_DIR) --git-dir=$(ROOT_DIR)/.git rev-parse --abbrev-ref HEAD || printf "unknown"); \
@@ -285,26 +287,32 @@ jbuild jbuild-workspace base/Version.ml: $(MAKEFILE_LIST)
 	  $@.in > "$$TMPFILE"; \
 	cat "$$TMPFILE" > $@; \
 	$(REMOVE) "$$TMPFILE"
-jbuild: jbuild.in
+jbuild.common: jbuild.common.in
 jbuild-workspace: jbuild-workspace.in
 base/Version.ml: base/Version.ml.in
 
+jbuild atd/jbuild istd/jbuild scripts/jbuild: jbuild.common
+	$(QUIET)cat $+ > $@
+
+jbuild: jbuild.in
+atd/jbuild: atd/jbuild.in
+istd/jbuild: istd/jbuild.in
+scripts/jbuild: scripts/jbuild.in
+
 .PHONY: clean
 clean:
 	$(REMOVE) $(INFER_TARGET)
 	$(REMOVE) toplevel.mlpack
 	$(REMOVE_DIR) $(BUILD_DIR)
 	$(REMOVE) $(ETC_DIR)/clang_ast.dict
-	$(REMOVE) base/Version.ml jbuild jbuild-workspace
+	$(REMOVE) base/Version.ml jbuild atd/jbuild istd/jbuild jbuild-workspace
 	$(REMOVE) base/Version.ml.tmp.* jbuild.tmp.* jbuild-workspace.tmp.*
-	$(REMOVE) backend/jsonbug_{j,t}.ml{,i}
-	$(REMOVE) checkers/stacktree_{j,t}.ml{,i}
 #	be a bit more aggressive than needed with what we remove here so that stale binaries that
 #	only existed in previous versions get removed as well
 	$(REMOVE) $(BIN_DIR)/Infer* $(BIN_DIR)/infer-* $(INFER_BIN){,.bc,.exe} $(INFER_BIN_ALIASES) \
 	  $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN)
 	$(REMOVE) $(INFER_CREATE_TRACEVIEW_LINKS_BIN)
-	$(REMOVE) $(CLANG_PLUGIN_MIRROR)/*
+	$(REMOVE) atd/*_{j,t,v}.ml{,i} atd/clang_*
 	$(REMOVE) mod_dep.dot
 	$(REMOVE) mod_dep.pdf
 

infer/src/atd/.merlin

@@ -0,0 +1,10 @@
+B ../_build/default/atd
+FLG -open InferGenerated -g -short-paths -safe-string -principal -strict-formats -strict-sequence -bin-annot -w +3+5+6+8+10+11+12+18+19+20+21+23+26+29+27+32+33+34+35+37+38+39+50+52+57-4-9-40-41-42-45-48 -w -27-32-34-35-39
+PKG atd
+PKG atdgen
+PKG biniou
+PKG easy-format
+PKG str
+PKG unix
+PKG yojson
+S .

infer/src/atd/jbuild.in

@@ -0,0 +1,14 @@
+(* -*- tuareg -*- *)
+(* NOTE: prepend jbuild.common to this file! *)
+
+let cflags = common_cflags @ ["-w"; "-27-32-34-35-39"]
+
+;; Format.sprintf {|
+(library
+ ((name InferGenerated)
+  (flags (%s))
+  (libraries (atdgen))
+ ))
+|}
+     (String.concat " " cflags)
+   |> Jbuild_plugin.V1.send

infer/src/istd/.merlin

@@ -0,0 +1,19 @@
+B ../_build/default/istd
+PKG ANSITerminal
+PKG atdgen
+PKG cmdliner
+PKG core
+PKG javalib
+PKG oUnit
+PKG parmap
+PKG ppx_compare
+PKG ptrees
+PKG sawja
+PKG str
+PKG unix
+PKG xmlm
+PKG yojson
+PKG zip
+FLG -principal -safe-string -short-paths -strict-formats -strict-sequence
+FLG -w +a-4-9-40-41-42-44-45-48-60
+S .

infer/src/istd/jbuild.in

@@ -0,0 +1,12 @@
+(* -*- tuareg -*- *)
+(* NOTE: prepend jbuild.common to this file! *)
+
+;; Format.sprintf {|
+(library
+ ((name InferStdlib)
+  (flags (%s))
+  (libraries (%s))
+ ))
+|}
+     (String.concat " " common_cflags) (String.concat " " common_libraries)
+   |> Jbuild_plugin.V1.send

infer/src/jbuild.common.in

@@ -0,0 +1,49 @@
+(* -*- tuareg -*- *)
+(* use strings so that it looks like OCaml even before substituting, e.g. to use ocamlformat *)
+
+let is_yes = String.equal "yes"
+
+let clang = is_yes "@BUILD_C_ANALYZERS@"
+
+let java = is_yes "@BUILD_JAVA_ANALYZERS@"
+
+let facebook = is_yes "@IS_FACEBOOK_TREE@"
+
+let extra_cflags = if "@EXTRA_CFLAGS" = "" then [] else ["@EXTRA_CFLAGS@"]
+
+let common_cflags =
+  let fatal_warnings = "+3+5+6+8+10+11+12+18+19+20+21+23+26+29+27+32+33+34+35+37+38+39+50+52+57" in
+  let warnings = fatal_warnings ^ "-4-9-40-41-42-45-48" in
+  let common_flags =
+    [ "-g"
+    ; "-short-paths"
+    ; "-safe-string"
+    ; "-principal"
+    ; "-strict-formats"
+    ; "-strict-sequence"
+    ; "-bin-annot"
+    ; "-w"
+    ; warnings ]
+  in
+  match Jbuild_plugin.V1.context with
+  | "test"
+   -> "-warn-error" :: fatal_warnings :: common_flags
+  | "default"
+   -> common_flags
+  | ctx
+   -> invalid_arg ("unknown context: " ^ ctx)
+
+let common_libraries =
+  (if java then ["javalib"; "ptrees"; "sawja"] else [])
+  @ [ "ANSITerminal"
+    ; "atdgen"
+    ; "cmdliner"
+    ; "core"
+    ; "extlib"
+    ; "oUnit"
+    ; "parmap"
+    ; "str"
+    ; "unix"
+    ; "xmlm"
+    ; "yojson"
+    ; "zip" ]

infer/src/jbuild.in

@@ -1,15 +1,5 @@
 (* -*- tuareg -*- *)
-(* use strings so that it looks like OCaml even before substituting, e.g. to use ocamlformat *)
+(* NOTE: prepend jbuild.common to this file! *)
-
-let is_yes = String.equal "yes"
-
-let clang = is_yes "@BUILD_C_ANALYZERS@"
-
-let java = is_yes "@BUILD_JAVA_ANALYZERS@"
-
-let facebook = is_yes "@IS_FACEBOOK_TREE@"
-
-let extra_cflags = if "@EXTRA_CFLAGS" = "" then [] else ["@EXTRA_CFLAGS@"]
 
 let ( ^/ ) = Filename.concat
 
@@ -26,7 +16,7 @@ let sources =
            (* do not use the symlinks in src/facebook/: jbuilder will not detect that the files have changed if they are hidden behind a symlink *)
            "../../facebook/skel/infer/src/facebook"
        else "opensource" )
-       :: ( ( if clang then ["clang"; "clang_plugin"; ("unit" ^/ "clang")]
+       :: ( ( if clang then ["clang"; ("unit" ^/ "clang")]
             else ["clang_stubs"; ("unit" ^/ "clang_stubs")] )
           @ [ (if java then "java" else "java_stubs")
             ; "absint"
@@ -55,76 +45,13 @@ let sources =
   in
   List.iter one_dir src_dirs ; !files
 
-let common_cflags =
-  let fatal_warnings = "+3+5+6+8+10+11+12+18+19+20+21+23+26+29+27+32+33+34+35+37+38+39+50+52+57" in
-  let warnings = fatal_warnings ^ "-4-9-40-41-42-45-48" in
-  let common_flags =
-    [ "-g"
-    ; "-short-paths"
-    ; "-safe-string"
-    ; "-principal"
-    ; "-strict-formats"
-    ; "-strict-sequence"
-    ; "-bin-annot"
-    ; "-w"
-    ; warnings ]
-  in
-  match Jbuild_plugin.V1.context with
-  | "test"
-   -> "-warn-error" :: fatal_warnings :: common_flags
-  | "default"
-   -> common_flags
-  | ctx
-   -> invalid_arg ("unknown context: " ^ ctx)
-
-(** Make a library that includes just IStd and IList. These files have different compilation flags
-    than infer (they don't include IStd!). *)
-let inferBaseStdlib = "InferBaseStdlib"
-
-let iStd = "IStd"
-
-let inferBaseStdlib_modules = ["IList"; iStd]
-
-let inferBaseStdlib_cflags = common_cflags
-
-(** Generated code (eg, from atdgen) is also compiled using different flags, so pack it in a
-    separated library. *)
-let inferGenerated = "InferGenerated"
-
-let inferGenerated_modules =
-  ( if clang then
-      [ "Clang_ast_b"
-      ; "Clang_ast_j"
-      ; "Clang_ast_t"
-      ; "Clang_ast_v"
-      ; "Clang_ast_proj"
-      ; "Clang_ast_types"
-      ; "Clang_ast_visit" ]
-  else [] )
-  @ ["Jsonbug_j"; "Jsonbug_t"; "Stacktree_j"; "Stacktree_t"]
-
-let inferGenerated_cflags = common_cflags @ ["-w"; "-27-32-34-35-39"]
-
 let infer_binaries =
   ["infer"; "inferunit"] @ if facebook then ["InferCreateTraceViewLinks"] else []
 
 let infer_cflags =
-  common_cflags @ ["-open"; inferBaseStdlib; "-open"; iStd; "-open"; inferGenerated]
+  common_cflags @ ["-open"; "InferStdlib"; "-open"; "IStd"; "-open"; "InferGenerated"]
 
-let common_libraries =
+let infer_libraries = "InferStdlib" :: "InferGenerated" :: common_libraries
-  (if java then ["javalib"; "ptrees"; "sawja"] else [])
-  @ [ "ANSITerminal"
-    ; "atdgen"
-    ; "cmdliner"
-    ; "core"
-    ; "extlib"
-    ; "oUnit"
-    ; "parmap"
-    ; "str"
-    ; "unix"
-    ; "xmlm"
-    ; "yojson"
-    ; "zip" ]
 
 (** Return the best copy action for a given [source] file. That is, when possibly, use "copy#" so
     that jump-to-defition and compiler errors go to the right original source file, otherwise do a
@@ -143,39 +70,16 @@ let stanzas =
   else [] )
   @ [ Format.sprintf
         {|
-(library
- ((name %s)
-  (flags (%s))
-  (modules (%s))
-  (libraries (atdgen))
- ))
-|}
-        inferGenerated (String.concat " " inferGenerated_cflags)
-        (String.concat " " inferGenerated_modules)
-    ; Format.sprintf
-        {|
-(library
- ((name %s)
-  (flags (%s))
-  (modules (%s))
-  (libraries (%s))
- ))
-|}
-        inferBaseStdlib (String.concat " " inferBaseStdlib_cflags)
-        (String.concat " " inferBaseStdlib_modules) (String.concat " " common_libraries)
-    ; Format.sprintf
-        {|
 (library
  ((name InferModules)
   (flags (%s))
-  (libraries (%s %s %s))
+  (libraries (%s))
-  (modules (:standard \ %s infertop %s %s))
+  (modules (:standard \ %s infertop))
   (preprocess (pps (ppx_compare)))
  ))
 |}
-        (String.concat " " infer_cflags) (String.concat " " common_libraries) inferBaseStdlib
+        (String.concat " " infer_cflags) (String.concat " " infer_libraries)
-        inferGenerated (String.concat " " infer_binaries)
+        (String.concat " " infer_binaries)
-        (String.concat " " inferBaseStdlib_modules) (String.concat " " inferGenerated_modules)
     ; Format.sprintf
         {|
 (executables

infer/src/scripts/.merlin

@@ -0,0 +1,6 @@
+B ../_build/default/scripts
+PKG core
+PKG str
+FLG -principal -safe-string -short-paths -strict-formats -strict-sequence
+FLG -w +a-4-9-40-41-42-44-45-48-60
+S .

infer/src/scripts/jbuild

@@ -1,7 +0,0 @@
-(jbuild_version 1)
-
-(executable
- ((name checkCopyright)
-  (flags (:standard -g -short-paths -safe-string -principal -strict-formats -strict-sequence -bin-annot -w +3+5+6+8+10+11+12+18+19+20+21+23+26+29+27+33+34+35+37+38+39+50+52+57-4-9-32-40-41-42-45-48))
-  (libraries (core str))
- ))

infer/src/scripts/jbuild.in

@@ -0,0 +1,11 @@
+(* NOTE: prepend jbuild.common to this file! *)
+;; Format.sprintf
+     {|
+(executable
+ ((name checkCopyright)
+  (flags (%s))
+  (libraries (core str))
+ ))
+|}
+     (String.concat " " common_cflags)
+   |> Jbuild_plugin.V1.send

infer/tests/build_systems/infertop/Makefile

@@ -10,7 +10,8 @@ ROOT_DIR = $(TESTS_DIR)/../..
 
 include $(TESTS_DIR)/base.make
 
-toplevel.exp.test: $(BUILD_DIR)/test/infertop.bc
+toplevel.exp.test: $(BUILD_DIR)/test/infertop.bc $(SCRIPT_DIR)/infer_repl \
+                   $(SCRIPT_DIR)/toplevel_init $(INFER_DIR)/tests/repl/infer_batch_script.mltop
 	$(QUIET)$(call silent_on_success,Testing infer OCaml REPL,\
 	TOPLEVEL_DIR=$(BUILD_DIR)/test $(SCRIPT_DIR)/infer_repl \
 	  $(INFER_DIR)/tests/repl/infer_batch_script.mltop > $@)

scripts/infer_repl

@@ -21,5 +21,8 @@ cd "$SCRIPT_DIR"/../infer
 "$TOPLEVEL_DIR"/infertop.bc \
   -init "$SCRIPT_DIR"/toplevel_init \
   -I "$TOPLEVEL_DIR" \
+  -I "$TOPLEVEL_DIR"/atd \
+  -I "$TOPLEVEL_DIR"/istd \
+  -I "$TOPLEVEL_DIR"/scripts \
   -I "$SCRIPT_DIR" \
   "$@"

scripts/toplevel_init

@@ -12,5 +12,6 @@
 #require "xmlm";;
 #require "ANSITerminal";;
 
+open InferStdlib;;
+open InferGenerated;;
 open InferModules;;
-open InferBaseStdlib;;