Crashbot results stitching and end-to-end testing.

Reviewed By: sblackshear

Differential Revision: D3619339

fbshipit-source-id: 46f3cc1
master
Lázaro Clapp Jiménez Labora 8 years ago committed by Facebook Github Bot 7
parent 33b417c280
commit 9a79e74380

@ -30,6 +30,10 @@ csv.field_size_limit(sys.maxsize)
INFER_ANALYZE_BINARY = 'InferAnalyze'
CRASHCONTEXT_METHOD_FIELD = 'method'
CRASHCONTEXT_LOCATION_FIELD = 'location'
CRASHCONTEXT_CALLEES_FIELD = 'callees'
def get_infer_version():
try:
@ -106,6 +110,13 @@ base_group.add_argument('-nf', '--no-filtering', action='store_true',
help='''Also show the results from the experimental
checks. Warning: some checks may contain many false
alarms''')
base_group.add_argument('-st', '--stacktrace',
dest='stacktrace',
default='',
help='''File containing a JSON encoded stacktrace. For
use with e.g. -a crashcontext. See
tests/codetoanalyze/java/crashcontext/*.json for
examples of the expected format.''')
base_group.add_argument('--fail-on-bug', action='store_true',
help='''Exit with error code %d if Infer found
something to report'''
@ -481,6 +492,55 @@ class AnalyzerWrapper(object):
return exit_status
def crashcontext_stitch_summaries(self):
"""Take crashcontext per-method summaries and join them together to
produce the final crashcontext.json output file."""
crashcontext_dir = os.path.join(self.args.infer_out, 'crashcontext')
summaries_map_by_frame_id = {}
st_json = utils.load_json_from_path(self.args.stacktrace)
stacktrace = st_json['stack_trace']
k = 1 # Where k is the number of levels of inlined calls.
for f in os.listdir(crashcontext_dir):
if f.endswith('.json'):
path = os.path.join(crashcontext_dir, f)
method_summary = utils.load_json_from_path(path)
method_signature = method_summary[CRASHCONTEXT_METHOD_FIELD]
method_name = method_signature.split('(')[0]
src_path = method_summary[CRASHCONTEXT_LOCATION_FIELD]['file']
line = method_summary[CRASHCONTEXT_LOCATION_FIELD]['line']
frame_id = "{0}({1}:{2})".format(
method_name,
os.path.basename(src_path),
line)
summaries_map_by_frame_id[frame_id] = method_summary
def expand(summary, k):
if k == 0:
# Make sure leaf nodes have an empty 'callees' field.
leaf_nodes = summary[CRASHCONTEXT_CALLEES_FIELD]
for leaf_node in leaf_nodes:
if CRASHCONTEXT_CALLEES_FIELD not in leaf_node:
leaf_node[CRASHCONTEXT_CALLEES_FIELD] = []
return summary
else:
NotImplementedError() # TODO
json_frames = []
for frame in stacktrace:
frame_id = frame.strip()
if not frame_id:
continue
assert frame_id.startswith('at ')
frame_id = frame_id[3:]
assert frame_id in summaries_map_by_frame_id
summary = summaries_map_by_frame_id[frame_id]
json_frames.append(expand(summary, k - 1))
out_json = {}
out_json['stack'] = json_frames
out_file = os.path.join(crashcontext_dir, 'crashcontext.json')
utils.dump_json_to_path(out_json, out_file)
def read_proc_stats(self):
proc_stats_path = os.path.join(
self.args.infer_out,
@ -513,6 +573,8 @@ class AnalyzerWrapper(object):
if self.args.analyzer not in [config.ANALYZER_COMPILE,
config.ANALYZER_CAPTURE]:
if self.analyze() == os.EX_OK:
if self.args.analyzer == config.ANALYZER_CRASHCONTEXT:
self.crashcontext_stitch_summaries()
reporting_start_time = time.time()
report_status = self.create_report()
elapsed = utils.elapsed_time(reporting_start_time)

@ -79,6 +79,8 @@ let () =
["--use-flavors"]) @
(match Config.infer_cache with None -> [] | Some s ->
["--infer_cache"; s]) @
(match Config.stacktrace with None -> [] | Some s ->
["--stacktrace"; s]) @
"--multicore" :: (string_of_int Config.jobs) ::
(if not Config.reactive_mode then [] else
["--reactive"]) @

@ -30,14 +30,16 @@ module TransferFunctions (CFG : ProcCfg.S) = struct
let json_of_summary caller astate loc loc_type =
let procs = Domain.elements astate in
let json = `Assoc [
("caller", `String (Procname.to_string caller));
("method", `String (Procname.to_unique_id caller));
("location", `Assoc [
("type", `String loc_type);
("file", `String (DB.source_file_to_string loc.Location.file));
("line", `Int loc.Location.line);
]);
("callees", `List (IList.map
(fun pn -> `String (Procname.to_string pn))
(fun pn -> `Assoc [
("method", `String (Procname.to_unique_id pn))
])
procs))
] in
json

@ -0,0 +1,73 @@
sources = glob(['**/*.java','**/*.json'])
dependencies = [
'//dependencies/java/android/support/v4:android-support-v4',
'//infer/annotations:annotations',
'//infer/lib/java/android:android',
]
java_library(
name = 'checkers',
srcs = sources,
deps = dependencies,
visibility = [
'PUBLIC'
]
)
out = 'out'
inferconfig_file = '$(location //infer/tests/codetoanalyze/java:inferconfig)'
copy_inferconfig = ' '.join(['cp', inferconfig_file, '$SRCDIR'])
clean_cmd = ' '.join(['rm', '-rf', out])
classpath = ':'.join([('$(classpath ' + path + ')') for path in dependencies])
def mk_infer_cmd(tag, srcs, stacktrace):
infer_cmd_main = ' '.join([
'infer',
'--no-progress-bar',
'--absolute-paths',
'-o', out,
'-a', 'crashcontext',
'--stacktrace', stacktrace,
'--',
'javac',
'-cp', classpath,
srcs])
out_rename = ' '.join(['cp',
out + '/crashcontext/crashcontext.json',
'$OUT' + "." + tag])
return ' && '.join([infer_cmd_main, out_rename])
infer_cmds = [
mk_infer_cmd(
"MinimalCrashExample",
"MinimalCrashExample.java",
"MinimalCrashExample.stacktrace.json"
),
mk_infer_cmd(
"MultiStackFrameCrashExample",
"MultiStackFrameCrashExample.java",
"MultiStackFrameCrashExample.stacktrace.json"
),
mk_infer_cmd(
"BranchingCallsExample",
"BranchingCallsExample.java",
"BranchingCallsExample.stacktrace.json"
)
]
# Copy the last crashcontext.json because buck expects it as the output file.
# This will only contain the results for the last run infer_cmd above.
copy_cmd = ' '.join(['cp', out + '/crashcontext/crashcontext.json', '$OUT'])
command = ' && '.join([clean_cmd, copy_inferconfig, ' && '.join(infer_cmds), copy_cmd])
genrule(
name = 'analyze',
srcs = sources,
out = 'crashcontext.json',
cmd = command,
deps = dependencies + [':checkers'],
visibility = [
'PUBLIC',
]
)

@ -1 +1 @@
{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at endtoend.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["endtoend.java.crashcontext.MinimalCrashExample.main"]}
{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at codetoanalyze.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["codetoanalyze.java.crashcontext.MinimalCrashExample.main"]}

@ -8,6 +8,7 @@ tests_dependencies = [
'//dependencies/java/opencsv:opencsv',
'//infer/tests/utils:utils',
'//infer/tests/codetoanalyze/java/checkers:checkers',
'//infer/tests/codetoanalyze/java/crashcontext:crashcontext',
'//infer/tests/codetoanalyze/java/eradicate:eradicate',
'//infer/tests/codetoanalyze/java/infer:infer',
'//infer/tests/codetoanalyze/java/tracing:tracing',
@ -66,6 +67,7 @@ java_test(
'//infer/tests/endtoend/java/infer:infer',
'//infer/tests/endtoend/java/eradicate:eradicate',
'//infer/tests/endtoend/java/checkers:checkers',
'//infer/tests/endtoend/java/crashcontext:crashcontext',
'//infer/tests/endtoend/java/harness:harness',
'//infer/tests/endtoend/java/tracing:tracing',
'//infer/tests/endtoend/java/comparison:comparison',

@ -0,0 +1,16 @@
java_test(
name='crashcontext',
srcs=glob(['*.java']),
deps=[
'//dependencies/java/guava:guava',
'//dependencies/java/junit:hamcrest',
'//dependencies/java/junit:junit',
'//infer/tests/utils:utils',
],
resources=[
'//infer/tests/codetoanalyze/java/crashcontext:analyze',
],
visibility=[
'PUBLIC',
],
)

@ -0,0 +1,85 @@
/*
* Copyright (c) 2016 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
package endtoend.java.crashcontext;
import static org.hamcrest.MatcherAssert.assertThat;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import utils.CrashContextResults;
public class BranchingCallsTest {
public static final String TAG = "BranchingCallsExample";
public static final String MAIN_METHOD =
"codetoanalyze.java.crashcontext.BranchingCallsExample.main(java.lang.String[]):void";
public static final String FOO_METHOD =
"codetoanalyze.java.crashcontext.BranchingCallsExample.foo():void";
public static final String PRE_BAR_METHOD =
"codetoanalyze.java.crashcontext.BranchingCallsExample.pre_bar():void";
public static final String BAR_METHOD =
"codetoanalyze.java.crashcontext.BranchingCallsExample.bar():void";
public static final String POST_BAR_METHOD =
"codetoanalyze.java.crashcontext.BranchingCallsExample.post_bar():void";
public static final String TO_STRING_METHOD =
"java.lang.String.toString():java.lang.String";
private static CrashContextResults crashcontext;
@BeforeClass
public static void loadResults() throws IOException {
crashcontext =
CrashContextResults.loadJSONResults(TAG);
}
@Test
public void shapeOfTheStack() {
assertThat("The stack trace should contain " + BAR_METHOD,
crashcontext.hasStackFrame(BAR_METHOD, 0));
assertThat("The stack trace should contain " + FOO_METHOD,
crashcontext.hasStackFrame(FOO_METHOD, 1));
assertThat("The stack trace should contain " + MAIN_METHOD,
crashcontext.hasStackFrame(MAIN_METHOD, 2));
}
@Test
public void toStringMethodIsFound() {
assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
crashcontext.hasMethod(TO_STRING_METHOD));
assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
"context tree from " + BAR_METHOD,
crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD));
}
@Test
public void preBarMethodIsFound() {
assertThat("Method " + PRE_BAR_METHOD + " should be part of the context",
crashcontext.hasMethod(PRE_BAR_METHOD));
assertThat("Method " + PRE_BAR_METHOD + " should be reachable in the " +
"context tree from " + FOO_METHOD,
crashcontext.hasPath(FOO_METHOD, PRE_BAR_METHOD));
}
@Test
public void postBarMethodIsOmitted() {
assertThat("Method " + POST_BAR_METHOD + " shouldn't be part of the context",
crashcontext.hasNotMethod(POST_BAR_METHOD));
}
}

@ -0,0 +1,54 @@
/*
* Copyright (c) 2016 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
package endtoend.java.crashcontext;
import static org.hamcrest.MatcherAssert.assertThat;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import utils.CrashContextResults;
public class MinimalCrashTest {
public static final String TAG = "MinimalCrashExample";
public static final String MAIN_METHOD =
"codetoanalyze.java.crashcontext.MinimalCrashExample.main(java.lang.String[]):void";
public static final String TO_STRING_METHOD =
"java.lang.String.toString():java.lang.String";
private static CrashContextResults crashcontext;
@BeforeClass
public static void loadResults() throws IOException {
crashcontext =
CrashContextResults.loadJSONResults(TAG);
}
@Test
public void shapeOfTheStack() {
assertThat("The stack trace should contain " + MAIN_METHOD,
crashcontext.hasStackFrame(MAIN_METHOD, 0));
}
@Test
public void toStringMethodIsDetected() {
assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
crashcontext.hasMethod(TO_STRING_METHOD));
assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
"context tree from " + MAIN_METHOD,
crashcontext.hasPath(MAIN_METHOD, TO_STRING_METHOD));
}
}

@ -0,0 +1,64 @@
/*
* Copyright (c) 2016 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
package endtoend.java.crashcontext;
import static org.hamcrest.MatcherAssert.assertThat;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import utils.CrashContextResults;
public class MultiStackFrameCrashTest {
public static final String TAG = "MultiStackFrameCrashExample";
public static final String MAIN_METHOD =
"codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.main(java.lang.String[]):void";
public static final String FOO_METHOD =
"codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.foo():void";
public static final String BAR_METHOD =
"codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.bar():void";
public static final String TO_STRING_METHOD =
"java.lang.String.toString():java.lang.String";
private static CrashContextResults crashcontext;
@BeforeClass
public static void loadResults() throws IOException {
crashcontext =
CrashContextResults.loadJSONResults(TAG);
}
@Test
public void shapeOfTheStack() {
assertThat("The stack trace should contain " + BAR_METHOD,
crashcontext.hasStackFrame(BAR_METHOD, 0));
assertThat("The stack trace should contain " + FOO_METHOD,
crashcontext.hasStackFrame(FOO_METHOD, 1));
assertThat("The stack trace should contain " + MAIN_METHOD,
crashcontext.hasStackFrame(MAIN_METHOD, 2));
}
@Test
public void toStringMethodIsFound() {
assertThat("Method " + TO_STRING_METHOD + " should be part of the context",
crashcontext.hasMethod(TO_STRING_METHOD));
assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " +
"context tree from " + BAR_METHOD,
crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD));
}
}

@ -0,0 +1,93 @@
/*
* Copyright (c) 2016 - present Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
package utils;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.ByteStreams;
public class CrashContextResults {
private static final String CRASHCONTEXT_JSON =
"/buck-out/gen/infer/tests/codetoanalyze/java/crashcontext/analyze/crashcontext.json";
private JsonNode json;
private String filename;
private CrashContextResults(String tag) throws IOException {
String path =
System.getProperty("user.dir") + CRASHCONTEXT_JSON + "." + tag;
byte[] jsonData = ByteStreams.toByteArray(new FileInputStream(path));
ObjectMapper objectMapper = new ObjectMapper();
json = objectMapper.readTree(jsonData);
}
public boolean hasStackFrame(String methodSignature, int pos) {
return methodSignature.equals(
json.path("stack").get(pos).path("method").asText());
}
public boolean hasStackFrame(String methodSignature) {
for (JsonNode frame : json.path("stack")) {
if(methodSignature.equals(frame.path("method").asText())) {
return true;
}
}
return false;
}
private List<JsonNode> findNodesForMethod(JsonNode node,
String methodSignature,
List<JsonNode> accumulator) {
if(methodSignature.equals(node.path("method").asText())) {
accumulator.add(node);
}
for(JsonNode callee : node.path("callees")) {
findNodesForMethod(callee, methodSignature, accumulator);
}
return accumulator;
}
private List<JsonNode> findNodesForMethod(String methodSignature) {
List<JsonNode> accumulator = new ArrayList<JsonNode>();
for (JsonNode frame : json.path("stack")) {
findNodesForMethod(frame, methodSignature, accumulator);
}
return accumulator;
}
public boolean hasMethod(String methodSignature) {
return !findNodesForMethod(methodSignature).isEmpty();
}
public boolean hasNotMethod(String methodSignature) {
return findNodesForMethod(methodSignature).isEmpty();
}
public boolean hasPath(String methodFrom, String methodTo) {
for(JsonNode from : findNodesForMethod(methodFrom)) {
if(!findNodesForMethod(from, methodTo, new ArrayList()).isEmpty()) {
return true;
}
}
return false;
}
public static CrashContextResults loadJSONResults(String tag) throws IOException {
return new CrashContextResults(tag);
}
}
Loading…
Cancel
Save