diff --git a/Makefile b/Makefile index 6bee78a5c..96a8efbe0 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ endif ifneq ($(BUCK),no) -BUILD_SYSTEMS_TESTS += buck genrule buck_javac_jar +BUILD_SYSTEMS_TESTS += buck genrule genrulecapture buck_javac_jar endif ifneq ($(MVN),no) BUILD_SYSTEMS_TESTS += mvn diff --git a/infer/man/man1/infer-full.txt b/infer/man/man1/infer-full.txt index 07f21b272..080448dd6 100644 --- a/infer/man/man1/infer-full.txt +++ b/infer/man/man1/infer-full.txt @@ -1360,6 +1360,11 @@ INTERNAL OPTIONS --generated-classes-reset Cancel the effect of --generated-classes. + --genrule-master-mode + Activates: Make the master Infer process merge capture artefacts + generated by the genrule integration, and report after analysis. + (Conversely: --no-genrule-master-mode) + --genrule-mode Activates: Enable the genrule compatibility mode used for the Buck integration (Conversely: --no-genrule-mode) diff --git a/infer/src/backend/mergeCapture.ml b/infer/src/backend/mergeCapture.ml index a60f4a43d..35daf13fa 100644 --- a/infer/src/backend/mergeCapture.ml +++ b/infer/src/backend/mergeCapture.ml @@ -146,11 +146,25 @@ let process_merge_file deps_file = L.progress "Files linked: %d@\n" stats.files_linked +let merge_global_tenvs infer_deps_file = + let global_tenv = Tenv.create () in + let merge infer_out_src = + let global_tenv_path = + infer_out_src ^/ Config.global_tenv_filename |> DB.filename_from_string + in + Tenv.read global_tenv_path + |> Option.iter ~f:(fun tenv -> Tenv.merge ~src:tenv ~dst:global_tenv) + in + MergeResults.iter_infer_deps infer_deps_file ~f:merge ; + Tenv.store_global global_tenv + + let merge_captured_targets () = let time0 = Mtime_clock.counter () in L.progress "Merging captured Buck targets...@\n%!" ; let infer_deps_file = Config.(results_dir ^/ buck_infer_deps_file_name) in MergeResults.merge_buck_flavors_results infer_deps_file ; + if Config.genrule_master_mode then merge_global_tenvs infer_deps_file ; process_merge_file infer_deps_file ; L.progress "Merging captured Buck targets took %a@\n%!" Mtime.Span.pp (Mtime_clock.count time0) diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index d1a7cd0f5..12998691b 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -1435,6 +1435,12 @@ and generated_classes = "Specify where to load the generated class files" +and genrule_master_mode = + CLOpt.mk_bool ~default:false ~long:"genrule-master-mode" + "Make the master Infer process merge capture artefacts generated by the genrule integration, \ + and report after analysis." + + and genrule_mode = CLOpt.mk_bool ~default:false ~long:"genrule-mode" "Enable the genrule compatibility mode used for the Buck integration" @@ -2799,6 +2805,10 @@ and gen_previous_build_command_script = !gen_previous_build_command_script and generated_classes = !generated_classes +and genrule_master_mode = !genrule_master_mode + +and genrule_mode = !genrule_mode + and get_linter_doc_url = process_linters_doc_url !linters_doc_url and html = !html diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index fdd84df62..9f3eed0bc 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -388,6 +388,10 @@ val gen_previous_build_command_script : string option val generated_classes : string option +val genrule_master_mode : bool + +val genrule_mode : bool + val get_linter_doc_url : linter_id:string -> string option val hoisting_report_only_expensive : bool diff --git a/infer/src/infer.ml b/infer/src/infer.ml index b91e7cccb..24db6cfab 100644 --- a/infer/src/infer.ml +++ b/infer/src/infer.ml @@ -39,7 +39,7 @@ let setup () = (* In Buck mode, delete infer-out directories inside buck-out to start fresh and to avoid getting errors because some of their contents is missing (removed by [Driver.clean_results_dir ()]). *) - buck && flavors) + (buck && flavors) || genrule_mode) || not ( Driver.(equal_mode driver_mode Analyze) || Config.(continue_capture || infer_is_clang || infer_is_javac || reactive_mode) ) diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index 3d1d6bc5e..fb1748238 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -72,7 +72,8 @@ let register_perf_stats_report stats_type = (* Clean up the results dir to select only what's relevant to go in the Buck cache. In particular, get rid of non-deterministic outputs.*) let clean_results_dir () = - if Config.flavors then ResultsDatabase.db_canonicalize () ; + let cache_capture = Config.(flavors || genrule_mode) in + if cache_capture then ResultsDatabase.db_canonicalize () ; (* make sure we are done with the database *) ResultsDatabase.db_close () ; (* In Buck flavors mode we keep all capture data, but in Java mode we keep only the tenv *) @@ -85,7 +86,7 @@ let clean_results_dir () = ; frontend_stats_dir_name ; reporting_stats_dir_name ] in - if flavors then common_list + if cache_capture then common_list else captured_dir_name :: racerd_issues_dir_name :: starvation_issues_dir_name :: common_list in List.mem ~equal:String.equal dirs_to_delete @@ -93,7 +94,7 @@ let clean_results_dir () = let should_delete_file = let files_to_delete = (* we do not need to keep the database in Buck/Java mode *) - (if Config.flavors then [] else [ResultsDatabase.database_filename]) + (if cache_capture then [] else [ResultsDatabase.database_filename]) @ [ Config.log_file ; (* some versions of sqlite do not clean up after themselves *) ResultsDatabase.database_filename ^ "-shm" @@ -398,6 +399,8 @@ let analyze_and_report ?suppress_console_report ~changed_files mode = | PythonCapture (BBuck, _) when Config.flavors && InferCommand.equal Run Config.command -> (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *) true + | Analyze when Config.genrule_master_mode -> + true | Analyze -> RunState.get_merge_capture () | _ -> diff --git a/infer/tests/build_systems/genrulecapture/.buckconfig b/infer/tests/build_systems/genrulecapture/.buckconfig new file mode 100644 index 000000000..250f76f47 --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/.buckconfig @@ -0,0 +1,9 @@ +[buildfile] + includes = //DEFS + +[project] + ignore = .git, .ml, .mli + +[java] + source_level = 8 + target_level = 8 \ No newline at end of file diff --git a/infer/tests/build_systems/genrulecapture/.buckversion b/infer/tests/build_systems/genrulecapture/.buckversion new file mode 120000 index 000000000..0a46cf46a --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/.buckversion @@ -0,0 +1 @@ +../../../../.buckversion \ No newline at end of file diff --git a/infer/tests/build_systems/genrulecapture/DEFS b/infer/tests/build_systems/genrulecapture/DEFS new file mode 100644 index 000000000..67cad39bd --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/DEFS @@ -0,0 +1,88 @@ +import os + +original_java_library = java_library +original_android_library = android_library + +def _get_infer_bin(): + return read_config("infer", "infer_bin") + +def _get_project_root(): + return read_config("infer", "project_root") + +def _get_infer_deps(): + infer_out = read_config("infer", "infer_out") + infer_deps = "{}/infer-deps.txt".format(infer_out) + return infer_deps + +def _infer_capture_genrule( + name, + srcs + ): + + args = [ + "--jobs", + "1", + "--genrule-mode", + "--quiet", + "--no-progress-bar", + "--results-dir", + "$OUT", + "--sourcepath", + "$SRCDIR", + "--project-root", + _get_project_root(), + "--classpath", + "$(classpath :{})".format(name), + "--generated-classes", + "$(location :{})".format(name), + "capture", + ] + + args_file = os.path.join("$TMP", "args.txt") + subcommands = [ + "echo {} >> {}".format(arg, args_file) + for arg in args + ] + [ + " ".join([_get_infer_bin(), "@" + args_file]), + # need to use a cross-platform kind of flock to avoid fragmentation + 'echo -e "_\\t_\\t$OUT" >> {}'.format(_get_infer_deps()) + ] + + genrule( + name = name + "_infer_capture", + srcs = srcs, + cmd = " && ".join(subcommands), + out = "infer_out", + labels = ["infer_capture_genrule"], + ) + + +def _make_infer_capture_genrule(name, kwargs): + java_sources = [ + f + for f in kwargs.get("srcs", []) + if f.endswith(".java") + ] + + if java_sources != []: + _infer_capture_genrule(name, java_sources) + kwargs["labels"] = kwargs.get("labels", []) + ["infer_enabled"] + + return kwargs + + +def java_library(name, **kwargs): + new_kwargs = _make_infer_capture_genrule(name, kwargs) + + original_java_library( + name=name, + **new_kwargs + ) + +def android_library(name, **kwargs): + new_kwargs = _make_infer_capture_genrule(name, kwargs) + + original_android_library( + name=name, + **new_kwargs + ) diff --git a/infer/tests/build_systems/genrulecapture/Makefile b/infer/tests/build_systems/genrulecapture/Makefile new file mode 100644 index 000000000..1b45af772 --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/Makefile @@ -0,0 +1,24 @@ +# Copyright (c) 2016-present, Facebook, Inc. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +TESTS_DIR = ../.. +ROOT_DIR = $(TESTS_DIR)/../.. + +ANALYSE = $(ROOT_DIR)/scripts/genrule_run.sh +BUCK_TARGET = //module2:module2 +INFER_OUT = $(shell pwd)/infer-out +CLEAN_EXTRA = buck-out +SOURCES = $(shell find . -name '*.java') + +INFERPRINT_OPTIONS = --issues-tests +INFER_OPTIONS = --debug-exceptions + +include $(TESTS_DIR)/infer.make + +$(INFER_OUT)/report.json: $(MAKEFILE_LIST) $(SOURCES) + $(QUIET) $(REMOVE_DIR) buck-out && \ + $(call silent_on_success,Testing genrule capture integration in $(TEST_REL_DIR),\ + $(ANALYSE) $(BUCK_TARGET) $(INFER_OUT) $(INFER_OPTIONS)) + diff --git a/infer/tests/build_systems/genrulecapture/annotations/BUCK b/infer/tests/build_systems/genrulecapture/annotations/BUCK new file mode 100644 index 000000000..013a1d88a --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/annotations/BUCK @@ -0,0 +1,7 @@ +java_library( + name='annotations', + srcs=['ThreadSafe.java'], + visibility=[ + 'PUBLIC' + ], +) diff --git a/infer/tests/build_systems/genrulecapture/annotations/ThreadSafe.java b/infer/tests/build_systems/genrulecapture/annotations/ThreadSafe.java new file mode 100644 index 000000000..f1b0094da --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/annotations/ThreadSafe.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2018-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +package genrulecapture.annotations; + +public @interface ThreadSafe {} diff --git a/infer/tests/build_systems/genrulecapture/issues.exp b/infer/tests/build_systems/genrulecapture/issues.exp new file mode 100644 index 000000000..da26ea8e4 --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/issues.exp @@ -0,0 +1,2 @@ +module2/Class2.java, genrulecapture.module2.Class2.get():int, 22, THREAD_SAFETY_VIOLATION, no_bucket, WARNING, [,call to int Class1.get(),access to `this.c.x`,,call to void Class1.set(int),access to `this.c.x`] +module2/Class2.java, genrulecapture.module2.Class2.set(int):void, 18, THREAD_SAFETY_VIOLATION, no_bucket, WARNING, [call to void Class1.set(int),access to `this.c.x`] diff --git a/infer/tests/build_systems/genrulecapture/module1/BUCK b/infer/tests/build_systems/genrulecapture/module1/BUCK new file mode 100644 index 000000000..4ac82ec7d --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module1/BUCK @@ -0,0 +1,10 @@ +java_library( + name='module1', + srcs=glob(["*.java"]), + deps=[ + '//annotations:annotations', + ], + visibility=[ + 'PUBLIC' + ], +) diff --git a/infer/tests/build_systems/genrulecapture/module1/Class1.java b/infer/tests/build_systems/genrulecapture/module1/Class1.java new file mode 100644 index 000000000..b4a36e090 --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module1/Class1.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package genrulecapture.module1; + +public class Class1 { + int x; + + public void set(int d) { + x = d; + } + + public int get() { + return x; + } +} diff --git a/infer/tests/build_systems/genrulecapture/module2/BUCK b/infer/tests/build_systems/genrulecapture/module2/BUCK new file mode 100644 index 000000000..239ec0d9b --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module2/BUCK @@ -0,0 +1,9 @@ +java_library( + name='module2', + srcs=glob(["*.java"]), + deps=[ + '//module1:module1', + '//annotations:annotations', + '//module3:module3', + ] +) diff --git a/infer/tests/build_systems/genrulecapture/module2/Class2.java b/infer/tests/build_systems/genrulecapture/module2/Class2.java new file mode 100644 index 000000000..c48ebc29d --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module2/Class2.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package genrulecapture.module2; + +import genrulecapture.annotations.ThreadSafe; +import genrulecapture.module1.Class1; + +@ThreadSafe +public class Class2 { + Class1 c; + + void set(int x) { + c.set(x); + } + + int get() { + return c.get(); + } +} diff --git a/infer/tests/build_systems/genrulecapture/module3/BUCK b/infer/tests/build_systems/genrulecapture/module3/BUCK new file mode 100644 index 000000000..7d613ba9b --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module3/BUCK @@ -0,0 +1,9 @@ +cxx_library( + name = 'module3', + visibility = [ + 'PUBLIC', + ], + srcs = [ + 'some_c_code.c', + ], +) diff --git a/infer/tests/build_systems/genrulecapture/module3/some_c_code.c b/infer/tests/build_systems/genrulecapture/module3/some_c_code.c new file mode 100644 index 000000000..8ebbe2866 --- /dev/null +++ b/infer/tests/build_systems/genrulecapture/module3/some_c_code.c @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2019-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include + +void some_c_func() { + int* s = NULL; + *s = 42; +} diff --git a/scripts/genrule_run.sh b/scripts/genrule_run.sh new file mode 100755 index 000000000..e6dfda3f1 --- /dev/null +++ b/scripts/genrule_run.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright (c) 2019-present, Facebook, Inc. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +set -e +set -x + +BUCK=buck +GENRULE_SUFFIX="_infer_capture" +BUCK_KIND_PATTERN="^(java|android)_library$" +INFER_BIN="${INFER_BIN:-infer}" + +INFER_VERSION=$(${INFER_BIN} --version | head -1 | cut -f3 -d' ') + +ROOT_TARGET="${1?Must specify a root target.}" +shift +QUERY="kind('${BUCK_KIND_PATTERN}', deps('${ROOT_TARGET}'))" +QUERY="attrfilter(labels, infer_enabled, ${QUERY})" + +INFER_OUT="${1?Must specify an infer out location.}" +if [[ "$INFER_OUT" != /* ]] ; then + echo "Must use absolute path for infer out location." + exit 1 +fi +shift + +BUCK_CONFIG="--config infer.project_root=${PWD}" +BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_out=${INFER_OUT}" +BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_bin=${INFER_BIN}" +BUCK_CONFIG="${BUCK_CONFIG} --config infer.enabled=True" +BUCK_CONFIG="${BUCK_CONFIG} --config infer.version=${INFER_VERSION}" + +# prepare infer-out, mainly for runstate +$INFER_BIN -o "${INFER_OUT}" > /dev/null 2>&1 + +TARGET_FILE=$(mktemp) +trap "{ rm -f $TARGET_FILE; }" EXIT + +echo "Running buck query." +$BUCK query ${BUCK_CONFIG} "${QUERY}" | sed "s/\$/${GENRULE_SUFFIX}/" > "${TARGET_FILE}" + +if [ -s "${TARGET_FILE}" ] +then + echo "Found $(wc -l < ${TARGET_FILE}) targets." +else + echo "Zero targets found!" + exit 1 +fi + +echo 'Running genrule capture under buck.' +$BUCK build --no-cache ${BUCK_CONFIG} "@${TARGET_FILE}" + +echo 'Running merge and analysis.' +$INFER_BIN analyze --genrule-master-mode -o "${INFER_OUT}" "$@"