[buckjava2] integration

Reviewed By: jeremydubreil

Differential Revision: D15518291

fbshipit-source-id: 5cd0d3941

Makefile

@@ -155,7 +155,7 @@ endif
 
 
 ifneq ($(BUCK),no)
-BUILD_SYSTEMS_TESTS += buck genrule buck_javac_jar
+BUILD_SYSTEMS_TESTS += buck genrule genrulecapture buck_javac_jar
 endif
 ifneq ($(MVN),no)
 BUILD_SYSTEMS_TESTS += mvn

infer/man/man1/infer-full.txt

@@ -1360,6 +1360,11 @@ INTERNAL OPTIONS
        --generated-classes-reset
            Cancel the effect of --generated-classes.
 
+       --genrule-master-mode
+           Activates: Make the master Infer process merge capture artefacts
+           generated by the genrule integration, and report after analysis.
+           (Conversely: --no-genrule-master-mode)
+
        --genrule-mode
            Activates: Enable the genrule compatibility mode used for the Buck
            integration (Conversely: --no-genrule-mode)

infer/src/backend/mergeCapture.ml

@@ -146,11 +146,25 @@ let process_merge_file deps_file =
   L.progress "Files linked: %d@\n" stats.files_linked
 
 
+let merge_global_tenvs infer_deps_file =
+  let global_tenv = Tenv.create () in
+  let merge infer_out_src =
+    let global_tenv_path =
+      infer_out_src ^/ Config.global_tenv_filename |> DB.filename_from_string
+    in
+    Tenv.read global_tenv_path
+    |> Option.iter ~f:(fun tenv -> Tenv.merge ~src:tenv ~dst:global_tenv)
+  in
+  MergeResults.iter_infer_deps infer_deps_file ~f:merge ;
+  Tenv.store_global global_tenv
+
+
 let merge_captured_targets () =
   let time0 = Mtime_clock.counter () in
   L.progress "Merging captured Buck targets...@\n%!" ;
   let infer_deps_file = Config.(results_dir ^/ buck_infer_deps_file_name) in
   MergeResults.merge_buck_flavors_results infer_deps_file ;
+  if Config.genrule_master_mode then merge_global_tenvs infer_deps_file ;
   process_merge_file infer_deps_file ;
   L.progress "Merging captured Buck targets took %a@\n%!" Mtime.Span.pp (Mtime_clock.count time0)
 

infer/src/base/Config.ml

@@ -1435,6 +1435,12 @@ and generated_classes =
     "Specify where to load the generated class files"
 
 
+and genrule_master_mode =
+  CLOpt.mk_bool ~default:false ~long:"genrule-master-mode"
+    "Make the master Infer process merge capture artefacts generated by the genrule integration, \
+     and report after analysis."
+
+
 and genrule_mode =
   CLOpt.mk_bool ~default:false ~long:"genrule-mode"
     "Enable the genrule compatibility mode used for the Buck integration"
@@ -2799,6 +2805,10 @@ and gen_previous_build_command_script = !gen_previous_build_command_script
 
 and generated_classes = !generated_classes
 
+and genrule_master_mode = !genrule_master_mode
+
+and genrule_mode = !genrule_mode
+
 and get_linter_doc_url = process_linters_doc_url !linters_doc_url
 
 and html = !html

infer/src/base/Config.mli

@@ -388,6 +388,10 @@ val gen_previous_build_command_script : string option
 
 val generated_classes : string option
 
+val genrule_master_mode : bool
+
+val genrule_mode : bool
+
 val get_linter_doc_url : linter_id:string -> string option
 
 val hoisting_report_only_expensive : bool

infer/src/infer.ml

@@ -39,7 +39,7 @@ let setup () =
           (* In Buck mode, delete infer-out directories inside buck-out to start fresh and to
               avoid getting errors because some of their contents is missing (removed by
               [Driver.clean_results_dir ()]). *)
-          buck && flavors)
+          (buck && flavors) || genrule_mode)
         || not
              ( Driver.(equal_mode driver_mode Analyze)
              || Config.(continue_capture || infer_is_clang || infer_is_javac || reactive_mode) )

infer/src/integration/Driver.ml

@@ -72,7 +72,8 @@ let register_perf_stats_report stats_type =
 (* Clean up the results dir to select only what's relevant to go in the Buck cache. In particular,
    get rid of non-deterministic outputs.*)
 let clean_results_dir () =
-  if Config.flavors then ResultsDatabase.db_canonicalize () ;
+  let cache_capture = Config.(flavors || genrule_mode) in
+  if cache_capture then ResultsDatabase.db_canonicalize () ;
   (* make sure we are done with the database *)
   ResultsDatabase.db_close () ;
   (* In Buck flavors mode we keep all capture data, but in Java mode we keep only the tenv *)
@@ -85,7 +86,7 @@ let clean_results_dir () =
         ; frontend_stats_dir_name
         ; reporting_stats_dir_name ]
       in
-      if flavors then common_list
+      if cache_capture then common_list
       else captured_dir_name :: racerd_issues_dir_name :: starvation_issues_dir_name :: common_list
     in
     List.mem ~equal:String.equal dirs_to_delete
@@ -93,7 +94,7 @@ let clean_results_dir () =
   let should_delete_file =
     let files_to_delete =
       (* we do not need to keep the database in Buck/Java mode *)
-      (if Config.flavors then [] else [ResultsDatabase.database_filename])
+      (if cache_capture then [] else [ResultsDatabase.database_filename])
       @ [ Config.log_file
         ; (* some versions of sqlite do not clean up after themselves *)
           ResultsDatabase.database_filename ^ "-shm"
@@ -398,6 +399,8 @@ let analyze_and_report ?suppress_console_report ~changed_files mode =
     | PythonCapture (BBuck, _) when Config.flavors && InferCommand.equal Run Config.command ->
         (* if doing capture + analysis of buck with flavors, we always need to merge targets before the analysis phase *)
         true
+    | Analyze when Config.genrule_master_mode ->
+        true
     | Analyze ->
         RunState.get_merge_capture ()
     | _ ->

infer/tests/build_systems/genrulecapture/.buckconfig

@@ -0,0 +1,9 @@
+[buildfile]
+  includes = //DEFS
+
+[project]
+  ignore = .git, .ml, .mli
+
+[java]
+  source_level = 8
+  target_level = 8

infer/tests/build_systems/genrulecapture/.buckversion

@@ -0,0 +1 @@
+../../../../.buckversion

infer/tests/build_systems/genrulecapture/DEFS

@@ -0,0 +1,88 @@
+import os
+
+original_java_library = java_library
+original_android_library = android_library
+
+def _get_infer_bin():
+  return read_config("infer", "infer_bin")
+
+def _get_project_root():
+  return read_config("infer", "project_root")
+
+def _get_infer_deps():
+  infer_out = read_config("infer", "infer_out")
+  infer_deps = "{}/infer-deps.txt".format(infer_out)
+  return infer_deps
+
+def _infer_capture_genrule(
+    name,
+    srcs
+  ):
+	
+  args = [
+      "--jobs",
+      "1",
+      "--genrule-mode",
+      "--quiet",
+      "--no-progress-bar",
+      "--results-dir",
+      "$OUT",
+      "--sourcepath",
+      "$SRCDIR",
+      "--project-root",
+      _get_project_root(),
+      "--classpath",
+      "$(classpath :{})".format(name),
+      "--generated-classes",
+      "$(location :{})".format(name),
+      "capture",
+  ]
+  
+  args_file = os.path.join("$TMP", "args.txt")
+  subcommands = [
+    "echo {} >> {}".format(arg, args_file)
+    for arg in args
+  ] + [
+    " ".join([_get_infer_bin(), "@" + args_file]),
+    # need to use a cross-platform kind of flock to avoid fragmentation
+    'echo -e "_\\t_\\t$OUT" >> {}'.format(_get_infer_deps())
+  ]
+	
+  genrule(
+      name = name + "_infer_capture",
+      srcs = srcs,
+      cmd = " && ".join(subcommands),
+      out = "infer_out",
+      labels = ["infer_capture_genrule"],
+  )  
+
+
+def _make_infer_capture_genrule(name, kwargs):
+  java_sources = [
+    f
+    for f in kwargs.get("srcs", [])
+    if f.endswith(".java")
+  ]
+
+  if java_sources != []:
+    _infer_capture_genrule(name, java_sources)
+    kwargs["labels"] = kwargs.get("labels", []) + ["infer_enabled"]
+  
+  return kwargs
+
+
+def java_library(name, **kwargs):
+  new_kwargs = _make_infer_capture_genrule(name, kwargs)
+
+  original_java_library(
+    name=name,
+    **new_kwargs
+  )
+
+def android_library(name, **kwargs):
+  new_kwargs = _make_infer_capture_genrule(name, kwargs)
+
+  original_android_library(
+    name=name,
+    **new_kwargs
+  )

infer/tests/build_systems/genrulecapture/Makefile

@@ -0,0 +1,24 @@
+# Copyright (c) 2016-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+TESTS_DIR = ../..
+ROOT_DIR = $(TESTS_DIR)/../..
+
+ANALYSE = $(ROOT_DIR)/scripts/genrule_run.sh 
+BUCK_TARGET = //module2:module2
+INFER_OUT = $(shell pwd)/infer-out
+CLEAN_EXTRA = buck-out 
+SOURCES = $(shell find . -name '*.java')
+
+INFERPRINT_OPTIONS = --issues-tests
+INFER_OPTIONS = --debug-exceptions
+
+include $(TESTS_DIR)/infer.make
+
+$(INFER_OUT)/report.json: $(MAKEFILE_LIST) $(SOURCES)
+	$(QUIET) $(REMOVE_DIR) buck-out && \
+	$(call silent_on_success,Testing genrule capture integration in $(TEST_REL_DIR),\
+	  $(ANALYSE) $(BUCK_TARGET) $(INFER_OUT) $(INFER_OPTIONS))
+

infer/tests/build_systems/genrulecapture/annotations/BUCK

@@ -0,0 +1,7 @@
+java_library(
+    name='annotations',
+    srcs=['ThreadSafe.java'],
+    visibility=[
+        'PUBLIC'
+    ],
+)

infer/tests/build_systems/genrulecapture/annotations/ThreadSafe.java

@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2018-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+package genrulecapture.annotations;
+
+public @interface ThreadSafe {}

infer/tests/build_systems/genrulecapture/issues.exp

@@ -0,0 +1,2 @@
+module2/Class2.java, genrulecapture.module2.Class2.get():int, 22, THREAD_SAFETY_VIOLATION, no_bucket, WARNING, [<Read trace>,call to int Class1.get(),access to `this.c.x`,<Write trace>,call to void Class1.set(int),access to `this.c.x`]
+module2/Class2.java, genrulecapture.module2.Class2.set(int):void, 18, THREAD_SAFETY_VIOLATION, no_bucket, WARNING, [call to void Class1.set(int),access to `this.c.x`]

infer/tests/build_systems/genrulecapture/module1/BUCK

@@ -0,0 +1,10 @@
+java_library(
+    name='module1',
+    srcs=glob(["*.java"]),
+    deps=[
+        '//annotations:annotations',
+    ],
+    visibility=[
+        'PUBLIC'
+    ],
+)

infer/tests/build_systems/genrulecapture/module1/Class1.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package genrulecapture.module1;
+
+public class Class1 {
+  int x;
+
+  public void set(int d) {
+    x = d;
+  }
+
+  public int get() {
+    return x;
+  }
+}

infer/tests/build_systems/genrulecapture/module2/BUCK

@@ -0,0 +1,9 @@
+java_library(
+    name='module2',
+    srcs=glob(["*.java"]),
+    deps=[
+        '//module1:module1',
+        '//annotations:annotations',
+        '//module3:module3',
+    ]
+)

infer/tests/build_systems/genrulecapture/module2/Class2.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package genrulecapture.module2;
+
+import genrulecapture.annotations.ThreadSafe;
+import genrulecapture.module1.Class1;
+
+@ThreadSafe
+public class Class2 {
+  Class1 c;
+
+  void set(int x) {
+    c.set(x);
+  }
+
+  int get() {
+    return c.get();
+  }
+}

infer/tests/build_systems/genrulecapture/module3/BUCK

@@ -0,0 +1,9 @@
+cxx_library(
+  name = 'module3',
+  visibility = [
+    'PUBLIC',
+  ],
+  srcs = [
+    'some_c_code.c',
+  ],
+)

infer/tests/build_systems/genrulecapture/module3/some_c_code.c

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2019-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <stdlib.h>
+
+void some_c_func() {
+  int* s = NULL;
+  *s = 42;
+}

scripts/genrule_run.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright (c) 2019-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+set -x
+
+BUCK=buck
+GENRULE_SUFFIX="_infer_capture"
+BUCK_KIND_PATTERN="^(java|android)_library$"
+INFER_BIN="${INFER_BIN:-infer}"
+
+INFER_VERSION=$(${INFER_BIN} --version | head -1 | cut -f3 -d' ')
+
+ROOT_TARGET="${1?Must specify a root target.}"
+shift
+QUERY="kind('${BUCK_KIND_PATTERN}', deps('${ROOT_TARGET}'))"
+QUERY="attrfilter(labels, infer_enabled, ${QUERY})"
+
+INFER_OUT="${1?Must specify an infer out location.}"
+if [[ "$INFER_OUT" != /* ]] ; then
+  echo "Must use absolute path for infer out location."
+  exit 1
+fi
+shift
+
+BUCK_CONFIG="--config infer.project_root=${PWD}"
+BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_out=${INFER_OUT}"
+BUCK_CONFIG="${BUCK_CONFIG} --config infer.infer_bin=${INFER_BIN}"
+BUCK_CONFIG="${BUCK_CONFIG} --config infer.enabled=True"
+BUCK_CONFIG="${BUCK_CONFIG} --config infer.version=${INFER_VERSION}"
+
+# prepare infer-out, mainly for runstate
+$INFER_BIN -o "${INFER_OUT}" > /dev/null 2>&1
+
+TARGET_FILE=$(mktemp)
+trap "{ rm -f $TARGET_FILE; }" EXIT
+
+echo "Running buck query."
+$BUCK query ${BUCK_CONFIG} "${QUERY}" | sed "s/\$/${GENRULE_SUFFIX}/" > "${TARGET_FILE}"
+
+if [ -s "${TARGET_FILE}" ]
+then
+  echo "Found $(wc -l < ${TARGET_FILE})  targets."
+else
+  echo "Zero targets found!"
+  exit 1
+fi
+
+echo 'Running genrule capture under buck.'
+$BUCK build --no-cache ${BUCK_CONFIG} "@${TARGET_FILE}"
+
+echo 'Running merge and analysis.'
+$INFER_BIN analyze --genrule-master-mode -o "${INFER_OUT}" "$@"