From 2c96491e2343cb0804f1b339e5bb063bc8d47ce0 Mon Sep 17 00:00:00 2001 From: Artem Pianykh Date: Wed, 11 Mar 2020 05:57:20 -0700 Subject: [PATCH] [build] Generate sources from atd using dune (but not FCP atds yet) Summary: Next step in moving logic from make to dune. Since dune understands build rules with multiple targets we don't need to introduce artificial pipelining as in make. Rules are pretty straightforward, albeit somewhat verbose. The tricky part here was adjusting deadcode detection. Reviewed By: jvillard Differential Revision: D20322605 fbshipit-source-id: 688e5f96f --- Makefile | 3 ++ infer/src/Makefile | 16 +------- infer/src/atd/dune | 81 +++++++++++++++++++++++++++++++++++++ infer/src/deadcode/Makefile | 76 ++++++++++++++++++---------------- 4 files changed, 127 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 922554041..22201788c 100644 --- a/Makefile +++ b/Makefile @@ -575,6 +575,9 @@ mod_dep: src_build_common $(QUIET)$(call silent_on_success,Building Infer source dependency graph,\ $(MAKE) -C $(SRC_DIR) mod_dep.dot) +# `test_build` and `src_build` (which is a dependency of `endtoend_test`) should not be run in +# parallel since they build infer with different profiles (and therefore conflict). Therefore, +# `test_build` is in the dependency, and `endtoend_test` in the recipe. .PHONY: config_tests config_tests: test_build ocaml_unit_test validate-skel mod_dep $(MAKE) endtoend_test checkCopyright diff --git a/infer/src/Makefile b/infer/src/Makefile index ad1187a37..5ea5d0708 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -22,14 +22,6 @@ INFER_CREATE_TRACEVIEW_LINKS_MODULE = InferCreateTraceViewLinks INFER_CREATE_TRACEVIEW_LINKS_MAIN = $(INFER_CREATE_TRACEVIEW_LINKS_MODULE) CHECKCOPYRIGHT_MAIN = checkCopyright -INFER_ATDGEN_STUB_BASES = atd/jsonbug atd/runstate atd/java_method_decl atd/perf_profiler atd/java_profiler_samples atd/clang_profiler_samples -INFER_ATDGEN_TYPES = j t -INFER_ATDGEN_STUB_ATDS = $(INFER_ATDGEN_STUB_BASES:.atd) -INFER_ATDGEN_SUFFIXES = $(foreach atd_t,$(INFER_ATDGEN_TYPES),_$(atd_t).ml _$(atd_t).mli) -INFER_ATDGEN_STUBS = $(foreach base,$(INFER_ATDGEN_STUB_BASES),\ - $(addprefix $(base),$(INFER_ATDGEN_SUFFIXES))) - - FACEBOOK_DIR = facebook #### Clang declarations #### @@ -66,8 +58,7 @@ ifeq ($(IS_FACEBOOK_TREE),yes) INFER_CONFIG_TARGETS += $(INFER_CREATE_TRACEVIEW_LINKS_MAIN).exe endif -OCAML_GENERATED_SOURCES = \ - base/Version.ml $(INFER_ATDGEN_STUBS) +OCAML_GENERATED_SOURCES = base/Version.ml ifeq ($(BUILD_C_ANALYZERS),yes) OCAML_GENERATED_SOURCES += $(CLANG_ATDGEN_STUBS) $(CLANG_PLUGIN_MIRRORED_FILES) @@ -198,11 +189,6 @@ $(2)_$(3).mli: $(1) $(2)_$(3).ml: $(2)_$(3).mli endef -$(foreach atd_type,$(INFER_ATDGEN_TYPES),\ - $(foreach base,$(INFER_ATDGEN_STUB_BASES),\ - $(eval \ - $(call gen_atdgen_rules,$(base).atd,$(base),$(atd_type))))) - # rebuild the artifacts of the AST files whenever they're upated in FCP $(foreach atd_type,$(CLANG_ATDGEN_TYPES),\ $(eval \ diff --git a/infer/src/atd/dune b/infer/src/atd/dune index 3de89b88e..06484f4e9 100644 --- a/infer/src/atd/dune +++ b/infer/src/atd/dune @@ -15,3 +15,84 @@ (documentation (package InferGenerated) (mld_files index)) + +; Rules below are boilerplatey, but this is benign and very easy to +; understand boilerplate. + +; ATD for jsonbug +(rule + (targets jsonbug_j.ml + jsonbug_j.mli) + (deps jsonbug.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets jsonbug_t.ml + jsonbug_t.mli) + (deps jsonbug.atd) + (action (run atdgen -t %{deps}))) + +; ATD for runstate +(rule + (targets runstate_j.ml + runstate_j.mli) + (deps runstate.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets runstate_t.ml + runstate_t.mli) + (deps runstate.atd) + (action (run atdgen -t %{deps}))) + +; ATD for java_method_decl +(rule + (targets java_method_decl_j.ml + java_method_decl_j.mli) + (deps java_method_decl.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets java_method_decl_t.ml + java_method_decl_t.mli) + (deps java_method_decl.atd) + (action (run atdgen -t %{deps}))) + +; ATD for perf_profiler +(rule + (targets perf_profiler_j.ml + perf_profiler_j.mli) + (deps perf_profiler.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets perf_profiler_t.ml + perf_profiler_t.mli) + (deps perf_profiler.atd) + (action (run atdgen -t %{deps}))) + +; ATD for java_profiler_samples +(rule + (targets java_profiler_samples_j.ml + java_profiler_samples_j.mli) + (deps java_profiler_samples.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets java_profiler_samples_t.ml + java_profiler_samples_t.mli) + (deps java_profiler_samples.atd) + (action (run atdgen -t %{deps}))) + +; ATD for clang_profiler_samples +(rule + (targets clang_profiler_samples_j.ml + clang_profiler_samples_j.mli) + (deps clang_profiler_samples.atd) + (action (run atdgen -j -j-std %{deps}))) + +(rule + (targets clang_profiler_samples_t.ml + clang_profiler_samples_t.mli) + (deps clang_profiler_samples.atd) + (action (run atdgen -t %{deps}))) diff --git a/infer/src/deadcode/Makefile b/infer/src/deadcode/Makefile index 7684c5521..179cae0aa 100644 --- a/infer/src/deadcode/Makefile +++ b/infer/src/deadcode/Makefile @@ -29,6 +29,7 @@ ROOT_DIR = ../../.. include $(ROOT_DIR)/Makefile.config INFER_BUILD_DIR = $(INFER_DIR)/_build/default/src +DEADCODE_DIR = $(INFER_DIR)/src/deadcode ALL_INFER_IN_ONE_FILE_ML = all_infer_in_one_file.ml ALL_ML_FILES = all_ml_files @@ -38,23 +39,43 @@ ALL_MLI_FILES_COPIED = all_mli_files_copied default: detect_dead_code -ml_src_files_from_mlly:=$(shell find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml[ly]') - -ml_src_files:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml'); do echo $${i\#"../"}; done) - -mli_src_files:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.mli'); do echo $${i\#"../"}; done) - -ml_src_files_without_mli:=$(shell for i in $$(find .. -not -path "../*clang_stubs/*" -not -path "../*java_stubs/*" -not -path "../../_build/*" -regex '\.\./[a-zA-Z].*\.ml'); do [ -f $${i}i ] || echo $${i\#"../"}; done) +# ./ is necessary for find to work correctly. +# Note that we run find under _build directory. Since we copy some +# sources from subfolders to src/ folder to avoid duplicates we use +# -depth 1 and iteration over main and library folders. +LIBRARY_FOLDERS = . ./IR ./atd ./base ./c_stubs ./istd ./scripts +INCLUDE_FOLDERS = -I IR -I atd -I base -I c_stubs -I istd -I scripts + +ml_src_files:=$(shell \ + cd $(INFER_BUILD_DIR); \ + for d in $(LIBRARY_FOLDERS); do \ + [ -d $$d ] && echo $$(find $$d -depth 1 -regex '\./[a-zA-Z].*\.ml' \( -not -regex '.*\.pp\.ml' \) \ + | sed 's/^\.\///'); \ + done) +mli_src_files:=$(shell \ + cd $(INFER_BUILD_DIR); \ + for d in $(LIBRARY_FOLDERS); do \ + [ -d $$d ] && echo $$(find $$d -depth 1 -regex '\./[a-zA-Z].*\.mli' \( -not -regex '.*\.pp\.mli' \) \ + | sed 's/^\.\///'); \ + done) +ml_src_files_without_mli:=$(shell \ + cd $(INFER_BUILD_DIR); \ + for i in $(ml_src_files); do [ -f $${i}i ] || echo $$i; done) + +.PHONY: dump_ml dump_mli dump_ml_only +dump_ml: + @echo $(ml_src_files) +dump_mli: + @echo $(mli_src_files) +dump_ml_only: + @echo $(ml_src_files_without_mli) .PHONY: depend depend: - cd .. && \ + cd $(INFER_BUILD_DIR) && \ ocamldep -native \ - -I IR -I absint -I al -I atd -I backend -I base -I biabduction -I bufferoverrun \ - -I c_stubs -I checkers -I cost -I clang -I concurrency -I facebook -I integration -I istd -I java \ - -I labs -I nullsafe -I pulse -I scuba -I quandary -I topl -I unit -I unit/clang -I unit/nullsafe -I deadcode \ - -I test_determinator \ - $(ml_src_files) $(mli_src_files) > deadcode/.depend + $(INCLUDE_FOLDERS) \ + $(ml_src_files) $(mli_src_files) > $(DEADCODE_DIR)/.depend # circular dependency... not sure how to fix properly %.cmi: %.cmx @@ -73,12 +94,12 @@ depend: $(QUIET)echo " = sig " >> $(ALL_INFER_IN_ONE_FILE_ML) # pre-processor directive to get errors in the original files and not in all_infer_in_one_file.ml $(QUIET)echo '# 1 "$*.mli"' >> $(ALL_INFER_IN_ONE_FILE_ML) - cat ../$*.mli >> $(ALL_INFER_IN_ONE_FILE_ML) + cat $(INFER_BUILD_DIR)/$*.mli >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo " end end " >> $(ALL_INFER_IN_ONE_FILE_ML) # silence "unused module" warnings for executables $(QUIET)if [ $@ = "infer.cmi" ] \ - || [ $@ = "unit/inferunit.cmi" ] \ - || [ $@ = "facebook/InferCreateTraceViewLinks.cmi" ] \ + || [ $@ = "inferunit.cmi" ] \ + || [ $@ = "InferCreateTraceViewLinks.cmi" ] \ ; then \ echo '[@warning "-60"] ' >> $(ALL_INFER_IN_ONE_FILE_ML); \ fi @@ -93,13 +114,13 @@ depend: $(QUIET)echo $(shell basename $*) | $(GNU_SED) -e "s/\b\(.\)/ \u\1/g" >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo " = struct " >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo '# 1 "$*.ml"' >> $(ALL_INFER_IN_ONE_FILE_ML) - cat ../$*.ml >> $(ALL_INFER_IN_ONE_FILE_ML) + cat $(INFER_BUILD_DIR)/$*.ml >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo " end end" >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo "(* END OF MODULE $*.ml *)" >> $(ALL_INFER_IN_ONE_FILE_ML) # silence "unused module" warnings for executables $(QUIET)if [ $@ = "infer.cmi" ] \ - || [ $@ = "unit/inferunit.cmi" ] \ - || [ $@ = "facebook/InferCreateTraceViewLinks.cmi" ] \ + || [ $@ = "inferunit.cmi" ] \ + || [ $@ = "InferCreateTraceViewLinks.cmi" ] \ || [ $@ = "scripts/checkCopyright.cmi" ] \ ; then \ echo '[@warning "-60"] ' >> $(ALL_INFER_IN_ONE_FILE_ML);\ @@ -116,7 +137,7 @@ $(ml_src_files_without_mli:.ml=.cmx): $(QUIET)echo $(shell basename $@ .cmx) | $(GNU_SED) -e "s/\b\(.\)/ \u\1/g" >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo " = struct " >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo "# 1 \"$$(echo $@ | $(GNU_SED) -e 's/\.cmx$$/.ml/')\"" >> $(ALL_INFER_IN_ONE_FILE_ML) - cat ../$$(echo $@ | $(GNU_SED) -e "s/\.cmx$$/.ml/") >> $(ALL_INFER_IN_ONE_FILE_ML) + cat $(INFER_BUILD_DIR)/$$(echo $@ | $(GNU_SED) -e "s/\.cmx$$/.ml/") >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo " end " >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo "(* END OF MODULE $@ *)" >> $(ALL_INFER_IN_ONE_FILE_ML) $(QUIET)echo >> $(ALL_INFER_IN_ONE_FILE_ML) @@ -132,7 +153,7 @@ $(ml_src_files_without_mli:.ml=.cmx): # root .cmx to include all the code. Any code not used in the construction of these "root .cmx" will # be considered dead. .PHONY: flatten_infer -flatten_infer: infer.cmx unit/inferunit.cmx facebook/InferCreateTraceViewLinks.cmx scripts/checkCopyright.cmx +flatten_infer: infer.cmx inferunit.cmx InferCreateTraceViewLinks.cmx scripts/checkCopyright.cmx $(QUIET)echo "see results in $(ALL_INFER_IN_ONE_FILE_ML)" .PHONY: detect_dead_code @@ -143,19 +164,6 @@ detect_dead_code: touch $(ALL_INFER_IN_ONE_FILE_ML) $(ALL_INFER_IN_ONE_FILE_ML:.ml=.mli) # needed to get dune generated, and the generated code for the lexers and parsers in ../_build $(MAKE) GENERATED_DUNES=deadcode/dune -C .. test -# copy generated source files from ../_build - for file in $(ml_src_files_from_mlly); do \ - set +e; \ - [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mly).ml" ] && \ - $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mly).ml .; \ - [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mly).mli" ] && \ - $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mly).mli .; \ - [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mll).ml" ] && \ - $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mll).ml .; \ - [ -f "$(INFER_BUILD_DIR)/$$(basename $$file .mll).mli" ] && \ - $(COPY) $(INFER_BUILD_DIR)/$$(basename $$file .mll).mli .; \ - set -e; \ - done $(MAKE) depend # Need to be sequential to avoid getting a garbled file. Need to re-include .depend as it may # have changed. For both of these reasons, run another `make`.