Crashbot results stitching and end-to-end testing.

Reviewed By: sblackshear

Differential Revision: D3619339

fbshipit-source-id: 46f3cc1

infer/lib/python/inferlib/analyze.py

@@ -30,6 +30,10 @@ csv.field_size_limit(sys.maxsize)
 
 INFER_ANALYZE_BINARY = 'InferAnalyze'
 
+CRASHCONTEXT_METHOD_FIELD = 'method'
+CRASHCONTEXT_LOCATION_FIELD = 'location'
+CRASHCONTEXT_CALLEES_FIELD = 'callees'
+
 
 def get_infer_version():
     try:
@@ -106,6 +110,13 @@ base_group.add_argument('-nf', '--no-filtering', action='store_true',
                         help='''Also show the results from the experimental
                         checks. Warning: some checks may contain many false
                         alarms''')
+base_group.add_argument('-st', '--stacktrace',
+                        dest='stacktrace',
+                        default='',
+                        help='''File containing a JSON encoded stacktrace. For
+                        use with e.g. -a crashcontext. See
+                        tests/codetoanalyze/java/crashcontext/*.json for
+                        examples of the expected format.''')
 base_group.add_argument('--fail-on-bug', action='store_true',
                         help='''Exit with error code %d if Infer found
                         something to report'''
@@ -481,6 +492,55 @@ class AnalyzerWrapper(object):
 
         return exit_status
 
+    def crashcontext_stitch_summaries(self):
+        """Take crashcontext per-method summaries and join them together to
+        produce the final crashcontext.json output file."""
+
+        crashcontext_dir = os.path.join(self.args.infer_out, 'crashcontext')
+        summaries_map_by_frame_id = {}
+        st_json = utils.load_json_from_path(self.args.stacktrace)
+        stacktrace = st_json['stack_trace']
+        k = 1   # Where k is the number of levels of inlined calls.
+        for f in os.listdir(crashcontext_dir):
+            if f.endswith('.json'):
+                path = os.path.join(crashcontext_dir, f)
+                method_summary = utils.load_json_from_path(path)
+                method_signature = method_summary[CRASHCONTEXT_METHOD_FIELD]
+                method_name = method_signature.split('(')[0]
+                src_path = method_summary[CRASHCONTEXT_LOCATION_FIELD]['file']
+                line = method_summary[CRASHCONTEXT_LOCATION_FIELD]['line']
+                frame_id = "{0}({1}:{2})".format(
+                    method_name,
+                    os.path.basename(src_path),
+                    line)
+                summaries_map_by_frame_id[frame_id] = method_summary
+
+        def expand(summary, k):
+            if k == 0:
+                # Make sure leaf nodes have an empty 'callees' field.
+                leaf_nodes = summary[CRASHCONTEXT_CALLEES_FIELD]
+                for leaf_node in leaf_nodes:
+                    if CRASHCONTEXT_CALLEES_FIELD not in leaf_node:
+                        leaf_node[CRASHCONTEXT_CALLEES_FIELD] = []
+                return summary
+            else:
+                NotImplementedError()  # TODO
+
+        json_frames = []
+        for frame in stacktrace:
+            frame_id = frame.strip()
+            if not frame_id:
+                continue
+            assert frame_id.startswith('at ')
+            frame_id = frame_id[3:]
+            assert frame_id in summaries_map_by_frame_id
+            summary = summaries_map_by_frame_id[frame_id]
+            json_frames.append(expand(summary, k - 1))
+        out_json = {}
+        out_json['stack'] = json_frames
+        out_file = os.path.join(crashcontext_dir, 'crashcontext.json')
+        utils.dump_json_to_path(out_json, out_file)
+
     def read_proc_stats(self):
         proc_stats_path = os.path.join(
             self.args.infer_out,
@@ -513,6 +573,8 @@ class AnalyzerWrapper(object):
         if self.args.analyzer not in [config.ANALYZER_COMPILE,
                                       config.ANALYZER_CAPTURE]:
             if self.analyze() == os.EX_OK:
+                if self.args.analyzer == config.ANALYZER_CRASHCONTEXT:
+                    self.crashcontext_stitch_summaries()
                 reporting_start_time = time.time()
                 report_status = self.create_report()
                 elapsed = utils.elapsed_time(reporting_start_time)

infer/src/backend/infer.ml

@@ -79,6 +79,8 @@ let () =
          ["--use-flavors"]) @
       (match Config.infer_cache with None -> [] | Some s ->
           ["--infer_cache"; s]) @
+      (match Config.stacktrace with None -> [] | Some s ->
+          ["--stacktrace"; s]) @
       "--multicore" :: (string_of_int Config.jobs) ::
       (if not Config.reactive_mode then [] else
          ["--reactive"]) @

infer/src/checkers/BoundedCallTree.ml

@@ -30,14 +30,16 @@ module TransferFunctions (CFG : ProcCfg.S) = struct
   let json_of_summary caller astate loc loc_type =
     let procs = Domain.elements astate in
     let json = `Assoc [
-        ("caller", `String (Procname.to_string caller));
+        ("method", `String (Procname.to_unique_id caller));
         ("location", `Assoc [
             ("type", `String loc_type);
             ("file", `String (DB.source_file_to_string loc.Location.file));
             ("line", `Int loc.Location.line);
           ]);
         ("callees", `List (IList.map
-                             (fun pn -> `String (Procname.to_string pn))
+                             (fun pn -> `Assoc [
+                                  ("method", `String (Procname.to_unique_id pn))
+                                ])
                              procs))
       ] in
     json

infer/tests/codetoanalyze/java/crashcontext/BUCK

@@ -0,0 +1,73 @@
+sources = glob(['**/*.java','**/*.json'])
+
+dependencies = [
+    '//dependencies/java/android/support/v4:android-support-v4',
+    '//infer/annotations:annotations',
+    '//infer/lib/java/android:android',
+]
+
+java_library(
+  name = 'checkers',
+  srcs = sources,
+  deps = dependencies,
+  visibility = [
+      'PUBLIC'
+  ]
+)
+
+out = 'out'
+inferconfig_file = '$(location //infer/tests/codetoanalyze/java:inferconfig)'
+copy_inferconfig = ' '.join(['cp', inferconfig_file, '$SRCDIR'])
+clean_cmd = ' '.join(['rm', '-rf', out])
+classpath = ':'.join([('$(classpath ' + path + ')') for path in dependencies])
+
+def mk_infer_cmd(tag, srcs, stacktrace):
+    infer_cmd_main = ' '.join([
+        'infer',
+        '--no-progress-bar',
+        '--absolute-paths',
+        '-o', out,
+        '-a', 'crashcontext',
+        '--stacktrace', stacktrace,
+        '--',
+        'javac',
+        '-cp', classpath,
+        srcs])
+    out_rename = ' '.join(['cp',
+        out + '/crashcontext/crashcontext.json',
+        '$OUT' + "." + tag])
+    return ' && '.join([infer_cmd_main, out_rename])
+
+infer_cmds = [
+    mk_infer_cmd(
+        "MinimalCrashExample",
+        "MinimalCrashExample.java",
+        "MinimalCrashExample.stacktrace.json"
+    ),
+    mk_infer_cmd(
+        "MultiStackFrameCrashExample",
+        "MultiStackFrameCrashExample.java",
+        "MultiStackFrameCrashExample.stacktrace.json"
+    ),
+    mk_infer_cmd(
+        "BranchingCallsExample",
+        "BranchingCallsExample.java",
+        "BranchingCallsExample.stacktrace.json"
+    )
+]
+
+# Copy the last crashcontext.json because buck expects it as the output file.
+# This will only contain the results for the last run infer_cmd above.
+copy_cmd = ' '.join(['cp', out + '/crashcontext/crashcontext.json', '$OUT'])
+command = ' && '.join([clean_cmd, copy_inferconfig, ' && '.join(infer_cmds), copy_cmd])
+
+genrule(
+    name = 'analyze',
+    srcs = sources,
+    out = 'crashcontext.json',
+    cmd = command,
+    deps = dependencies + [':checkers'],
+    visibility = [
+        'PUBLIC',
+    ]
+)

infer/tests/codetoanalyze/java/crashcontext/MinimalCrashExample.stacktrace.json

@@ -1 +1 @@
-{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at endtoend.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["endtoend.java.crashcontext.MinimalCrashExample.main"]}
+{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at codetoanalyze.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["codetoanalyze.java.crashcontext.MinimalCrashExample.main"]}

infer/tests/endtoend/BUCK

@@ -8,6 +8,7 @@ tests_dependencies = [
     '//dependencies/java/opencsv:opencsv',
     '//infer/tests/utils:utils',
     '//infer/tests/codetoanalyze/java/checkers:checkers',
+    '//infer/tests/codetoanalyze/java/crashcontext:crashcontext',
     '//infer/tests/codetoanalyze/java/eradicate:eradicate',
     '//infer/tests/codetoanalyze/java/infer:infer',
     '//infer/tests/codetoanalyze/java/tracing:tracing',
@@ -66,6 +67,7 @@ java_test(
         '//infer/tests/endtoend/java/infer:infer',
         '//infer/tests/endtoend/java/eradicate:eradicate',
         '//infer/tests/endtoend/java/checkers:checkers',
+        '//infer/tests/endtoend/java/crashcontext:crashcontext',
         '//infer/tests/endtoend/java/harness:harness',
         '//infer/tests/endtoend/java/tracing:tracing',
         '//infer/tests/endtoend/java/comparison:comparison',

infer/tests/endtoend/java/crashcontext/BUCK

@@ -0,0 +1,16 @@
+java_test(
+    name='crashcontext',
+    srcs=glob(['*.java']),
+    deps=[
+        '//dependencies/java/guava:guava',
+        '//dependencies/java/junit:hamcrest',
+        '//dependencies/java/junit:junit',
+        '//infer/tests/utils:utils',
+    ],
+    resources=[
+        '//infer/tests/codetoanalyze/java/crashcontext:analyze',
+    ],
+    visibility=[
+        'PUBLIC',
+    ],
+)

infer/tests/endtoend/java/crashcontext/BranchingCallsTest.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2016 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package endtoend.java.crashcontext;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import utils.CrashContextResults;
+
+public class BranchingCallsTest {
+
+  public static final String TAG = "BranchingCallsExample";
+
+  public static final String MAIN_METHOD =
+    "codetoanalyze.java.crashcontext.BranchingCallsExample.main(java.lang.String[]):void";
+
+  public static final String FOO_METHOD =
+    "codetoanalyze.java.crashcontext.BranchingCallsExample.foo():void";
+
+  public static final String PRE_BAR_METHOD =
+    "codetoanalyze.java.crashcontext.BranchingCallsExample.pre_bar():void";
+
+  public static final String BAR_METHOD =
+    "codetoanalyze.java.crashcontext.BranchingCallsExample.bar():void";
+
+  public static final String POST_BAR_METHOD =
+    "codetoanalyze.java.crashcontext.BranchingCallsExample.post_bar():void";
+
+  public static final String TO_STRING_METHOD =
+    "java.lang.String.toString():java.lang.String";
+
+  private static CrashContextResults crashcontext;
+
+  @BeforeClass
+  public static void loadResults() throws IOException {
+    crashcontext =
+      CrashContextResults.loadJSONResults(TAG);
+  }
+
+  @Test
+  public void shapeOfTheStack() {
+      assertThat("The stack trace should contain " + BAR_METHOD,
+                 crashcontext.hasStackFrame(BAR_METHOD, 0));
+      assertThat("The stack trace should contain " + FOO_METHOD,
+                 crashcontext.hasStackFrame(FOO_METHOD, 1));
+      assertThat("The stack trace should contain " + MAIN_METHOD,
+                 crashcontext.hasStackFrame(MAIN_METHOD, 2));
+  }
+
+  @Test
+  public void toStringMethodIsFound() {
+    assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
+               crashcontext.hasMethod(TO_STRING_METHOD));
+    assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
+               "context tree from " + BAR_METHOD,
+               crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD));
+  }
+
+  @Test
+  public void preBarMethodIsFound() {
+    assertThat("Method " + PRE_BAR_METHOD + " should be part of the context",
+               crashcontext.hasMethod(PRE_BAR_METHOD));
+    assertThat("Method " + PRE_BAR_METHOD + " should be reachable in the " +
+               "context tree from " + FOO_METHOD,
+               crashcontext.hasPath(FOO_METHOD, PRE_BAR_METHOD));
+  }
+
+  @Test
+  public void postBarMethodIsOmitted() {
+    assertThat("Method " + POST_BAR_METHOD + " shouldn't be part of the context",
+               crashcontext.hasNotMethod(POST_BAR_METHOD));
+  }
+
+}

infer/tests/endtoend/java/crashcontext/MinimalCrashTest.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package endtoend.java.crashcontext;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import utils.CrashContextResults;
+
+public class MinimalCrashTest {
+
+  public static final String TAG = "MinimalCrashExample";
+
+  public static final String MAIN_METHOD =
+    "codetoanalyze.java.crashcontext.MinimalCrashExample.main(java.lang.String[]):void";
+
+  public static final String TO_STRING_METHOD =
+    "java.lang.String.toString():java.lang.String";
+
+  private static CrashContextResults crashcontext;
+
+  @BeforeClass
+  public static void loadResults() throws IOException {
+    crashcontext =
+      CrashContextResults.loadJSONResults(TAG);
+  }
+
+  @Test
+  public void shapeOfTheStack() {
+      assertThat("The stack trace should contain " + MAIN_METHOD,
+                 crashcontext.hasStackFrame(MAIN_METHOD, 0));
+  }
+
+  @Test
+  public void toStringMethodIsDetected() {
+    assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
+               crashcontext.hasMethod(TO_STRING_METHOD));
+    assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
+               "context tree from " + MAIN_METHOD,
+               crashcontext.hasPath(MAIN_METHOD, TO_STRING_METHOD));
+  }
+
+}

infer/tests/endtoend/java/crashcontext/MultiStackFrameCrashTest.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2016 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package endtoend.java.crashcontext;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import utils.CrashContextResults;
+
+public class MultiStackFrameCrashTest {
+
+  public static final String TAG = "MultiStackFrameCrashExample";
+
+  public static final String MAIN_METHOD =
+    "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.main(java.lang.String[]):void";
+
+  public static final String FOO_METHOD =
+    "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.foo():void";
+
+  public static final String BAR_METHOD =
+    "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.bar():void";
+
+  public static final String TO_STRING_METHOD =
+    "java.lang.String.toString():java.lang.String";
+
+  private static CrashContextResults crashcontext;
+
+  @BeforeClass
+  public static void loadResults() throws IOException {
+    crashcontext =
+      CrashContextResults.loadJSONResults(TAG);
+  }
+
+  @Test
+  public void shapeOfTheStack() {
+      assertThat("The stack trace should contain " + BAR_METHOD,
+                 crashcontext.hasStackFrame(BAR_METHOD, 0));
+      assertThat("The stack trace should contain " + FOO_METHOD,
+                 crashcontext.hasStackFrame(FOO_METHOD, 1));
+      assertThat("The stack trace should contain " + MAIN_METHOD,
+                 crashcontext.hasStackFrame(MAIN_METHOD, 2));
+  }
+
+  @Test
+  public void toStringMethodIsFound() {
+    assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
+               crashcontext.hasMethod(TO_STRING_METHOD));
+    assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
+               "context tree from " + BAR_METHOD,
+               crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD));
+  }
+
+}

infer/tests/utils/CrashContextResults.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2016 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+package utils;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import com.google.common.io.ByteStreams;
+
+public class CrashContextResults {
+
+  private static final String CRASHCONTEXT_JSON =
+    "/buck-out/gen/infer/tests/codetoanalyze/java/crashcontext/analyze/crashcontext.json";
+
+  private JsonNode json;
+  private String filename;
+
+  private CrashContextResults(String tag) throws IOException {
+    String path =
+      System.getProperty("user.dir") + CRASHCONTEXT_JSON + "." + tag;
+    byte[] jsonData = ByteStreams.toByteArray(new FileInputStream(path));
+    ObjectMapper objectMapper = new ObjectMapper();
+    json = objectMapper.readTree(jsonData);
+  }
+
+  public boolean hasStackFrame(String methodSignature, int pos) {
+    return methodSignature.equals(
+      json.path("stack").get(pos).path("method").asText());
+  }
+
+  public boolean hasStackFrame(String methodSignature) {
+    for (JsonNode frame : json.path("stack")) {
+      if(methodSignature.equals(frame.path("method").asText())) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private List<JsonNode> findNodesForMethod(JsonNode node,
+                                            String methodSignature,
+                                            List<JsonNode> accumulator) {
+    if(methodSignature.equals(node.path("method").asText())) {
+      accumulator.add(node);
+    }
+    for(JsonNode callee : node.path("callees")) {
+      findNodesForMethod(callee, methodSignature, accumulator);
+    }
+    return accumulator;
+  }
+
+  private List<JsonNode> findNodesForMethod(String methodSignature) {
+    List<JsonNode> accumulator = new ArrayList<JsonNode>();
+    for (JsonNode frame : json.path("stack")) {
+      findNodesForMethod(frame, methodSignature, accumulator);
+    }
+    return accumulator;
+  }
+
+  public boolean hasMethod(String methodSignature) {
+    return !findNodesForMethod(methodSignature).isEmpty();
+  }
+
+  public boolean hasNotMethod(String methodSignature) {
+    return findNodesForMethod(methodSignature).isEmpty();
+  }
+
+  public boolean hasPath(String methodFrom, String methodTo) {
+    for(JsonNode from : findNodesForMethod(methodFrom)) {
+      if(!findNodesForMethod(from, methodTo, new ArrayList()).isEmpty()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public static CrashContextResults loadJSONResults(String tag) throws IOException {
+    return new CrashContextResults(tag);
+  }
+
+}