[build] Generate sources from atd using dune (but not FCP atds yet)

Summary:
Next step in moving logic from make to dune. Since dune understands
build rules with multiple targets we don't need to introduce
artificial pipelining as in make. Rules are pretty straightforward,
albeit somewhat verbose.

The tricky part here was adjusting deadcode detection.

Reviewed By: jvillard

Differential Revision: D20322605

fbshipit-source-id: 688e5f96f

Makefile

@@ -575,6 +575,9 @@ mod_dep: src_build_common
 	$(QUIET)$(call silent_on_success,Building Infer source dependency graph,\
 	$(MAKE) -C $(SRC_DIR) mod_dep.dot)
 
+# `test_build` and `src_build` (which is a dependency of `endtoend_test`) should not be run in
+# parallel since they build infer with different profiles (and therefore conflict). Therefore,
+# `test_build` is in the dependency, and `endtoend_test` in the recipe.
 .PHONY: config_tests
 config_tests: test_build ocaml_unit_test validate-skel mod_dep
 	$(MAKE) endtoend_test checkCopyright

infer/src/Makefile

@@ -22,14 +22,6 @@ INFER_CREATE_TRACEVIEW_LINKS_MODULE = InferCreateTraceViewLinks
 INFER_CREATE_TRACEVIEW_LINKS_MAIN = $(INFER_CREATE_TRACEVIEW_LINKS_MODULE)
 CHECKCOPYRIGHT_MAIN = checkCopyright
 
-INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples atd/clang_profiler_samples
-INFER_ATDGEN_TYPES = j t
-INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd)
-INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli)
-INFER_ATDGEN_STUBS = $(foreach base,$(INFER_ATDGEN_STUB_BASES),\
-	$(addprefix $(base),$(INFER_ATDGEN_SUFFIXES)))
-
-
 FACEBOOK_DIR = facebook
 
 #### Clang declarations ####
@@ -66,8 +58,7 @@ ifeq ($(IS_FACEBOOK_TREE),yes)
 INFER_CONFIG_TARGETS += $(INFER_CREATE_TRACEVIEW_LINKS_MAIN).exe
 endif
 
-OCAML_GENERATED_SOURCES = \
+OCAML_GENERATED_SOURCES = base/Version.ml
-  base/Version.ml $(INFER_ATDGEN_STUBS)
 
 ifeq ($(BUILD_C_ANALYZERS),yes)
 OCAML_GENERATED_SOURCES += $(CLANG_ATDGEN_STUBS) $(CLANG_PLUGIN_MIRRORED_FILES)
@@ -198,11 +189,6 @@ $(2)_$(3).mli: $(1)
 $(2)_$(3).ml: $(2)_$(3).mli
 endef
 
-$(foreach atd_type,$(INFER_ATDGEN_TYPES),\
-  $(foreach base,$(INFER_ATDGEN_STUB_BASES),\
-    $(eval \
-        $(call gen_atdgen_rules,$(base).atd,$(base),$(atd_type)))))
-
 # rebuild the artifacts of the AST files whenever they're upated in FCP
 $(foreach atd_type,$(CLANG_ATDGEN_TYPES),\
     $(eval \

infer/src/atd/dune

@@ -15,3 +15,84 @@
 (documentation
  (package InferGenerated)
  (mld_files index))
+
+; Rules below are boilerplatey, but this is benign and very easy to
+; understand boilerplate.
+
+; ATD for jsonbug
+(rule
+ (targets jsonbug_j.ml
+          jsonbug_j.mli)
+ (deps    jsonbug.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets jsonbug_t.ml
+          jsonbug_t.mli)
+ (deps    jsonbug.atd)
+ (action  (run atdgen -t %{deps})))
+
+; ATD for runstate
+(rule
+ (targets runstate_j.ml
+          runstate_j.mli)
+ (deps    runstate.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets runstate_t.ml
+          runstate_t.mli)
+ (deps    runstate.atd)
+ (action  (run atdgen -t %{deps})))
+
+; ATD for java_method_decl
+(rule
+ (targets java_method_decl_j.ml
+          java_method_decl_j.mli)
+ (deps    java_method_decl.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets java_method_decl_t.ml
+          java_method_decl_t.mli)
+ (deps    java_method_decl.atd)
+ (action  (run atdgen -t %{deps})))
+
+; ATD for perf_profiler
+(rule
+ (targets perf_profiler_j.ml
+          perf_profiler_j.mli)
+ (deps    perf_profiler.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets perf_profiler_t.ml
+          perf_profiler_t.mli)
+ (deps    perf_profiler.atd)
+ (action  (run atdgen -t %{deps})))
+
+; ATD for java_profiler_samples
+(rule
+ (targets java_profiler_samples_j.ml
+          java_profiler_samples_j.mli)
+ (deps    java_profiler_samples.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets java_profiler_samples_t.ml
+          java_profiler_samples_t.mli)
+ (deps    java_profiler_samples.atd)
+ (action  (run atdgen -t %{deps})))
+
+; ATD for clang_profiler_samples
+(rule
+ (targets clang_profiler_samples_j.ml
+          clang_profiler_samples_j.mli)
+ (deps    clang_profiler_samples.atd)
+ (action  (run atdgen -j -j-std %{deps})))
+
+(rule
+ (targets clang_profiler_samples_t.ml
+          clang_profiler_samples_t.mli)
+ (deps    clang_profiler_samples.atd)
+ (action  (run atdgen -t %{deps})))

infer/src/deadcode/Makefile

@@ -29,6 +29,7 @@ ROOT_DIR = ../../..
 include $(ROOT_DIR)/Makefile.config
 
 INFER_BUILD_DIR = $(INFER_DIR)/_build/default/src
+DEADCODE_DIR = $(INFER_DIR)/src/deadcode
 
 ALL_INFER_IN_ONE_FILE_ML = all_infer_in_one_file.ml
 ALL_ML_FILES = all_ml_files
@@ -38,23 +39,43 @@ ALL_MLI_FILES_COPIED = all_mli_files_copied
 
 default: detect_dead_code
 
-ml_src_files_from_mlly:=$(shell find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml[ly]')
+# ./ is necessary for find to work correctly.
-
+# Note that we run find under _build directory. Since we copy some
-ml_src_files:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml'); do echo $${i\#"../"}; done)
+# sources from subfolders to src/ folder to avoid duplicates we use
-
+# -depth 1 and iteration over main and library folders.
-mli_src_files:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.mli'); do echo $${i\#"../"}; done)
+LIBRARY_FOLDERS = . ./IR ./atd ./base ./c_stubs ./istd ./scripts
-
+INCLUDE_FOLDERS = -I IR -I atd -I base -I c_stubs -I istd -I scripts
-ml_src_files_without_mli:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml'); do [ -f $${i}i ] || echo $${i\#"../"}; done)
+
+ml_src_files:=$(shell \
+	cd $(INFER_BUILD_DIR); \
+	for d in $(LIBRARY_FOLDERS); do \
+	[ -d $$d ] && echo $$(find $$d -depth 1 -regex '\./[a-zA-Z].*\.ml' \( -not -regex '.*\.pp\.ml' \) \
+				| sed 's/^\.\///'); \
+	done)
+mli_src_files:=$(shell \
+	cd $(INFER_BUILD_DIR); \
+	for d in $(LIBRARY_FOLDERS); do \
+	[ -d $$d ] && echo $$(find $$d -depth 1 -regex '\./[a-zA-Z].*\.mli' \( -not -regex '.*\.pp\.mli' \) \
+				| sed 's/^\.\///'); \
+	done)
+ml_src_files_without_mli:=$(shell \
+	cd $(INFER_BUILD_DIR); \
+	for i in $(ml_src_files); do [ -f $${i}i ] || echo $$i; done)
+
+.PHONY: dump_ml dump_mli dump_ml_only
+dump_ml:
+	@echo $(ml_src_files)
+dump_mli:
+	@echo $(mli_src_files)
+dump_ml_only:
+	@echo $(ml_src_files_without_mli)
 
 .PHONY: depend
 depend:
-	cd .. && \
+	cd $(INFER_BUILD_DIR) && \
 	ocamldep -native \
-	  -I IR -I absint -I al -I atd -I backend -I base -I biabduction -I bufferoverrun \
+	  $(INCLUDE_FOLDERS) \
-	  -I c_stubs -I checkers -I cost -I clang -I concurrency -I facebook -I integration -I istd -I java \
+	  $(ml_src_files) $(mli_src_files) > $(DEADCODE_DIR)/.depend
-	  -I labs -I nullsafe -I pulse -I scuba -I quandary -I topl -I unit -I unit/clang -I unit/nullsafe -I deadcode \
-	  -I test_determinator \
-	  $(ml_src_files) $(mli_src_files) > deadcode/.depend
 
 # circular dependency... not sure how to fix properly
 %.cmi: %.cmx
@@ -73,12 +94,12 @@ depend:
 	$(QUIET)echo "  = sig " >> $(ALL_INFER_IN_ONE_FILE_ML)
 #	pre-processor directive to get errors in the original files and not in all_infer_in_one_file.ml
 	$(QUIET)echo '# 1 "$*.mli"' >> $(ALL_INFER_IN_ONE_FILE_ML)
-	cat ../$*.mli >> $(ALL_INFER_IN_ONE_FILE_ML)
+	cat $(INFER_BUILD_DIR)/$*.mli >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "  end end " >> $(ALL_INFER_IN_ONE_FILE_ML)
 #	silence "unused module" warnings for executables
 	$(QUIET)if [ $@ = "infer.cmi" ] \
-	        || [ $@ = "unit/inferunit.cmi" ] \
+	        || [ $@ = "inferunit.cmi" ] \
-	        || [ $@ = "facebook/InferCreateTraceViewLinks.cmi" ] \
+	        || [ $@ = "InferCreateTraceViewLinks.cmi" ] \
 	        ; then \
 	  echo '[@warning "-60"] ' >> $(ALL_INFER_IN_ONE_FILE_ML); \
 	fi
@@ -93,13 +114,13 @@ depend:
 	$(QUIET)echo $(shell basename $*) | $(GNU_SED) -e "s/\b\(.\)/  \u\1/g" >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "  = struct " >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo '# 1 "$*.ml"' >> $(ALL_INFER_IN_ONE_FILE_ML)
-	cat ../$*.ml >> $(ALL_INFER_IN_ONE_FILE_ML)
+	cat $(INFER_BUILD_DIR)/$*.ml >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "  end end" >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "(* END OF MODULE $*.ml *)" >> $(ALL_INFER_IN_ONE_FILE_ML)
 #	silence "unused module" warnings for executables
 	$(QUIET)if [ $@ = "infer.cmi" ] \
-	        || [ $@ = "unit/inferunit.cmi" ] \
+	        || [ $@ = "inferunit.cmi" ] \
-	        || [ $@ = "facebook/InferCreateTraceViewLinks.cmi" ] \
+	        || [ $@ = "InferCreateTraceViewLinks.cmi" ] \
 	        || [ $@ = "scripts/checkCopyright.cmi" ] \
 	        ; then \
 	  echo '[@warning "-60"] ' >> $(ALL_INFER_IN_ONE_FILE_ML);\
@@ -116,7 +137,7 @@ $(ml_src_files_without_mli:.ml=.cmx):
 	$(QUIET)echo $(shell basename $@ .cmx) | $(GNU_SED) -e "s/\b\(.\)/  \u\1/g" >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "  = struct " >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "# 1 \"$$(echo $@ | $(GNU_SED) -e 's/\.cmx$$/.ml/')\"" >> $(ALL_INFER_IN_ONE_FILE_ML)
-	cat ../$$(echo $@ | $(GNU_SED) -e "s/\.cmx$$/.ml/") >> $(ALL_INFER_IN_ONE_FILE_ML)
+	cat $(INFER_BUILD_DIR)/$$(echo $@ | $(GNU_SED) -e "s/\.cmx$$/.ml/") >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "  end " >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo "(* END OF MODULE $@ *)" >> $(ALL_INFER_IN_ONE_FILE_ML)
 	$(QUIET)echo >> $(ALL_INFER_IN_ONE_FILE_ML)
@@ -132,7 +153,7 @@ $(ml_src_files_without_mli:.ml=.cmx):
 # root .cmx to include all the code. Any code not used in the construction of these "root .cmx" will
 # be considered dead.
 .PHONY: flatten_infer
-flatten_infer: infer.cmx unit/inferunit.cmx facebook/InferCreateTraceViewLinks.cmx scripts/checkCopyright.cmx
+flatten_infer: infer.cmx inferunit.cmx InferCreateTraceViewLinks.cmx scripts/checkCopyright.cmx
 	$(QUIET)echo "see results in $(ALL_INFER_IN_ONE_FILE_ML)"
 
 .PHONY: detect_dead_code
@@ -143,19 +164,6 @@ detect_dead_code:
 	touch $(ALL_INFER_IN_ONE_FILE_ML) $(ALL_INFER_IN_ONE_FILE_ML:.ml=.mli)
 #	needed to get dune generated, and the generated code for the lexers and parsers in ../_build
 	$(MAKE) GENERATED_DUNES=deadcode/dune -C .. test
-#	copy generated source files from ../_build
-	for file in $(ml_src_files_from_mlly); do \
-	  set +e; \
-	  [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mly).ml" ] && \
-	    $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mly).ml .; \
-	  [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mly).mli" ] && \
-	    $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mly).mli .; \
-	  [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mll).ml" ] && \
-	    $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mll).ml .; \
-	  [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mll).mli" ] && \
-	    $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mll).mli .; \
-	  set -e; \
-	done
 	$(MAKE) depend
 #	Need to be sequential to avoid getting a garbled file. Need to re-include .depend as it may
 #	have changed. For both of these reasons, run another `make`.