From 9a79e743802e5179f87fb4314fdd6b3d9fd91889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1zaro=20Clapp=20Jim=C3=A9nez=20Labora?= Date: Tue, 26 Jul 2016 17:53:18 -0700 Subject: [PATCH] Crashbot results stitching and end-to-end testing. Reviewed By: sblackshear Differential Revision: D3619339 fbshipit-source-id: 46f3cc1 --- infer/lib/python/inferlib/analyze.py | 62 +++++++++++++ infer/src/backend/infer.ml | 4 +- infer/src/checkers/BoundedCallTree.ml | 6 +- .../codetoanalyze/java/crashcontext/BUCK | 73 +++++++++++++++ .../MinimalCrashExample.stacktrace.json | 2 +- infer/tests/endtoend/BUCK | 2 + infer/tests/endtoend/java/crashcontext/BUCK | 16 ++++ .../java/crashcontext/BranchingCallsTest.java | 85 +++++++++++++++++ .../java/crashcontext/MinimalCrashTest.java | 54 +++++++++++ .../MultiStackFrameCrashTest.java | 64 +++++++++++++ infer/tests/utils/CrashContextResults.java | 93 +++++++++++++++++++ 11 files changed, 457 insertions(+), 4 deletions(-) create mode 100644 infer/tests/codetoanalyze/java/crashcontext/BUCK create mode 100644 infer/tests/endtoend/java/crashcontext/BUCK create mode 100644 infer/tests/endtoend/java/crashcontext/BranchingCallsTest.java create mode 100644 infer/tests/endtoend/java/crashcontext/MinimalCrashTest.java create mode 100644 infer/tests/endtoend/java/crashcontext/MultiStackFrameCrashTest.java create mode 100644 infer/tests/utils/CrashContextResults.java diff --git a/infer/lib/python/inferlib/analyze.py b/infer/lib/python/inferlib/analyze.py index 5f8bd10ab..e90b6921b 100644 --- a/infer/lib/python/inferlib/analyze.py +++ b/infer/lib/python/inferlib/analyze.py @@ -30,6 +30,10 @@ csv.field_size_limit(sys.maxsize) INFER_ANALYZE_BINARY = 'InferAnalyze' +CRASHCONTEXT_METHOD_FIELD = 'method' +CRASHCONTEXT_LOCATION_FIELD = 'location' +CRASHCONTEXT_CALLEES_FIELD = 'callees' + def get_infer_version(): try: @@ -106,6 +110,13 @@ base_group.add_argument('-nf', '--no-filtering', action='store_true', help='''Also show the results from the experimental checks. Warning: some checks may contain many false alarms''') +base_group.add_argument('-st', '--stacktrace', + dest='stacktrace', + default='', + help='''File containing a JSON encoded stacktrace. For + use with e.g. -a crashcontext. See + tests/codetoanalyze/java/crashcontext/*.json for + examples of the expected format.''') base_group.add_argument('--fail-on-bug', action='store_true', help='''Exit with error code %d if Infer found something to report''' @@ -481,6 +492,55 @@ class AnalyzerWrapper(object): return exit_status + def crashcontext_stitch_summaries(self): + """Take crashcontext per-method summaries and join them together to + produce the final crashcontext.json output file.""" + + crashcontext_dir = os.path.join(self.args.infer_out, 'crashcontext') + summaries_map_by_frame_id = {} + st_json = utils.load_json_from_path(self.args.stacktrace) + stacktrace = st_json['stack_trace'] + k = 1 # Where k is the number of levels of inlined calls. + for f in os.listdir(crashcontext_dir): + if f.endswith('.json'): + path = os.path.join(crashcontext_dir, f) + method_summary = utils.load_json_from_path(path) + method_signature = method_summary[CRASHCONTEXT_METHOD_FIELD] + method_name = method_signature.split('(')[0] + src_path = method_summary[CRASHCONTEXT_LOCATION_FIELD]['file'] + line = method_summary[CRASHCONTEXT_LOCATION_FIELD]['line'] + frame_id = "{0}({1}:{2})".format( + method_name, + os.path.basename(src_path), + line) + summaries_map_by_frame_id[frame_id] = method_summary + + def expand(summary, k): + if k == 0: + # Make sure leaf nodes have an empty 'callees' field. + leaf_nodes = summary[CRASHCONTEXT_CALLEES_FIELD] + for leaf_node in leaf_nodes: + if CRASHCONTEXT_CALLEES_FIELD not in leaf_node: + leaf_node[CRASHCONTEXT_CALLEES_FIELD] = [] + return summary + else: + NotImplementedError() # TODO + + json_frames = [] + for frame in stacktrace: + frame_id = frame.strip() + if not frame_id: + continue + assert frame_id.startswith('at ') + frame_id = frame_id[3:] + assert frame_id in summaries_map_by_frame_id + summary = summaries_map_by_frame_id[frame_id] + json_frames.append(expand(summary, k - 1)) + out_json = {} + out_json['stack'] = json_frames + out_file = os.path.join(crashcontext_dir, 'crashcontext.json') + utils.dump_json_to_path(out_json, out_file) + def read_proc_stats(self): proc_stats_path = os.path.join( self.args.infer_out, @@ -513,6 +573,8 @@ class AnalyzerWrapper(object): if self.args.analyzer not in [config.ANALYZER_COMPILE, config.ANALYZER_CAPTURE]: if self.analyze() == os.EX_OK: + if self.args.analyzer == config.ANALYZER_CRASHCONTEXT: + self.crashcontext_stitch_summaries() reporting_start_time = time.time() report_status = self.create_report() elapsed = utils.elapsed_time(reporting_start_time) diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index 6b1a8fec0..7dec84600 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -78,7 +78,9 @@ let () = (if not Config.flavors || not buck then [] else ["--use-flavors"]) @ (match Config.infer_cache with None -> [] | Some s -> - ["--infer_cache"; s]) @ + ["--infer_cache"; s]) @ + (match Config.stacktrace with None -> [] | Some s -> + ["--stacktrace"; s]) @ "--multicore" :: (string_of_int Config.jobs) :: (if not Config.reactive_mode then [] else ["--reactive"]) @ diff --git a/infer/src/checkers/BoundedCallTree.ml b/infer/src/checkers/BoundedCallTree.ml index 9e24c656f..ef431305e 100644 --- a/infer/src/checkers/BoundedCallTree.ml +++ b/infer/src/checkers/BoundedCallTree.ml @@ -30,14 +30,16 @@ module TransferFunctions (CFG : ProcCfg.S) = struct let json_of_summary caller astate loc loc_type = let procs = Domain.elements astate in let json = `Assoc [ - ("caller", `String (Procname.to_string caller)); + ("method", `String (Procname.to_unique_id caller)); ("location", `Assoc [ ("type", `String loc_type); ("file", `String (DB.source_file_to_string loc.Location.file)); ("line", `Int loc.Location.line); ]); ("callees", `List (IList.map - (fun pn -> `String (Procname.to_string pn)) + (fun pn -> `Assoc [ + ("method", `String (Procname.to_unique_id pn)) + ]) procs)) ] in json diff --git a/infer/tests/codetoanalyze/java/crashcontext/BUCK b/infer/tests/codetoanalyze/java/crashcontext/BUCK new file mode 100644 index 000000000..ba9f4fae4 --- /dev/null +++ b/infer/tests/codetoanalyze/java/crashcontext/BUCK @@ -0,0 +1,73 @@ +sources = glob(['**/*.java','**/*.json']) + +dependencies = [ + '//dependencies/java/android/support/v4:android-support-v4', + '//infer/annotations:annotations', + '//infer/lib/java/android:android', +] + +java_library( + name = 'checkers', + srcs = sources, + deps = dependencies, + visibility = [ + 'PUBLIC' + ] +) + +out = 'out' +inferconfig_file = '$(location //infer/tests/codetoanalyze/java:inferconfig)' +copy_inferconfig = ' '.join(['cp', inferconfig_file, '$SRCDIR']) +clean_cmd = ' '.join(['rm', '-rf', out]) +classpath = ':'.join([('$(classpath ' + path + ')') for path in dependencies]) + +def mk_infer_cmd(tag, srcs, stacktrace): + infer_cmd_main = ' '.join([ + 'infer', + '--no-progress-bar', + '--absolute-paths', + '-o', out, + '-a', 'crashcontext', + '--stacktrace', stacktrace, + '--', + 'javac', + '-cp', classpath, + srcs]) + out_rename = ' '.join(['cp', + out + '/crashcontext/crashcontext.json', + '$OUT' + "." + tag]) + return ' && '.join([infer_cmd_main, out_rename]) + +infer_cmds = [ + mk_infer_cmd( + "MinimalCrashExample", + "MinimalCrashExample.java", + "MinimalCrashExample.stacktrace.json" + ), + mk_infer_cmd( + "MultiStackFrameCrashExample", + "MultiStackFrameCrashExample.java", + "MultiStackFrameCrashExample.stacktrace.json" + ), + mk_infer_cmd( + "BranchingCallsExample", + "BranchingCallsExample.java", + "BranchingCallsExample.stacktrace.json" + ) +] + +# Copy the last crashcontext.json because buck expects it as the output file. +# This will only contain the results for the last run infer_cmd above. +copy_cmd = ' '.join(['cp', out + '/crashcontext/crashcontext.json', '$OUT']) +command = ' && '.join([clean_cmd, copy_inferconfig, ' && '.join(infer_cmds), copy_cmd]) + +genrule( + name = 'analyze', + srcs = sources, + out = 'crashcontext.json', + cmd = command, + deps = dependencies + [':checkers'], + visibility = [ + 'PUBLIC', + ] +) diff --git a/infer/tests/codetoanalyze/java/crashcontext/MinimalCrashExample.stacktrace.json b/infer/tests/codetoanalyze/java/crashcontext/MinimalCrashExample.stacktrace.json index 56301aa24..fdac7aa5d 100644 --- a/infer/tests/codetoanalyze/java/crashcontext/MinimalCrashExample.stacktrace.json +++ b/infer/tests/codetoanalyze/java/crashcontext/MinimalCrashExample.stacktrace.json @@ -1 +1 @@ -{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at endtoend.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["endtoend.java.crashcontext.MinimalCrashExample.main"]} +{"exception_type": "java.lang.NullPointerException", "stack_trace": ["at codetoanalyze.java.crashcontext.MinimalCrashExample.main(MinimalCrashExample.java:16)",""], "exception_message": "", "normvector_stack": ["codetoanalyze.java.crashcontext.MinimalCrashExample.main"]} diff --git a/infer/tests/endtoend/BUCK b/infer/tests/endtoend/BUCK index 3b21756c0..d0f15c5cf 100644 --- a/infer/tests/endtoend/BUCK +++ b/infer/tests/endtoend/BUCK @@ -8,6 +8,7 @@ tests_dependencies = [ '//dependencies/java/opencsv:opencsv', '//infer/tests/utils:utils', '//infer/tests/codetoanalyze/java/checkers:checkers', + '//infer/tests/codetoanalyze/java/crashcontext:crashcontext', '//infer/tests/codetoanalyze/java/eradicate:eradicate', '//infer/tests/codetoanalyze/java/infer:infer', '//infer/tests/codetoanalyze/java/tracing:tracing', @@ -66,6 +67,7 @@ java_test( '//infer/tests/endtoend/java/infer:infer', '//infer/tests/endtoend/java/eradicate:eradicate', '//infer/tests/endtoend/java/checkers:checkers', + '//infer/tests/endtoend/java/crashcontext:crashcontext', '//infer/tests/endtoend/java/harness:harness', '//infer/tests/endtoend/java/tracing:tracing', '//infer/tests/endtoend/java/comparison:comparison', diff --git a/infer/tests/endtoend/java/crashcontext/BUCK b/infer/tests/endtoend/java/crashcontext/BUCK new file mode 100644 index 000000000..4c42188f0 --- /dev/null +++ b/infer/tests/endtoend/java/crashcontext/BUCK @@ -0,0 +1,16 @@ +java_test( + name='crashcontext', + srcs=glob(['*.java']), + deps=[ + '//dependencies/java/guava:guava', + '//dependencies/java/junit:hamcrest', + '//dependencies/java/junit:junit', + '//infer/tests/utils:utils', + ], + resources=[ + '//infer/tests/codetoanalyze/java/crashcontext:analyze', + ], + visibility=[ + 'PUBLIC', + ], +) diff --git a/infer/tests/endtoend/java/crashcontext/BranchingCallsTest.java b/infer/tests/endtoend/java/crashcontext/BranchingCallsTest.java new file mode 100644 index 000000000..310b41300 --- /dev/null +++ b/infer/tests/endtoend/java/crashcontext/BranchingCallsTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +package endtoend.java.crashcontext; + +import static org.hamcrest.MatcherAssert.assertThat; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import utils.CrashContextResults; + +public class BranchingCallsTest { + + public static final String TAG = "BranchingCallsExample"; + + public static final String MAIN_METHOD = + "codetoanalyze.java.crashcontext.BranchingCallsExample.main(java.lang.String[]):void"; + + public static final String FOO_METHOD = + "codetoanalyze.java.crashcontext.BranchingCallsExample.foo():void"; + + public static final String PRE_BAR_METHOD = + "codetoanalyze.java.crashcontext.BranchingCallsExample.pre_bar():void"; + + public static final String BAR_METHOD = + "codetoanalyze.java.crashcontext.BranchingCallsExample.bar():void"; + + public static final String POST_BAR_METHOD = + "codetoanalyze.java.crashcontext.BranchingCallsExample.post_bar():void"; + + public static final String TO_STRING_METHOD = + "java.lang.String.toString():java.lang.String"; + + private static CrashContextResults crashcontext; + + @BeforeClass + public static void loadResults() throws IOException { + crashcontext = + CrashContextResults.loadJSONResults(TAG); + } + + @Test + public void shapeOfTheStack() { + assertThat("The stack trace should contain " + BAR_METHOD, + crashcontext.hasStackFrame(BAR_METHOD, 0)); + assertThat("The stack trace should contain " + FOO_METHOD, + crashcontext.hasStackFrame(FOO_METHOD, 1)); + assertThat("The stack trace should contain " + MAIN_METHOD, + crashcontext.hasStackFrame(MAIN_METHOD, 2)); + } + + @Test + public void toStringMethodIsFound() { + assertThat("Method " + TO_STRING_METHOD + " should be part of the context", + crashcontext.hasMethod(TO_STRING_METHOD)); + assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " + + "context tree from " + BAR_METHOD, + crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD)); + } + + @Test + public void preBarMethodIsFound() { + assertThat("Method " + PRE_BAR_METHOD + " should be part of the context", + crashcontext.hasMethod(PRE_BAR_METHOD)); + assertThat("Method " + PRE_BAR_METHOD + " should be reachable in the " + + "context tree from " + FOO_METHOD, + crashcontext.hasPath(FOO_METHOD, PRE_BAR_METHOD)); + } + + @Test + public void postBarMethodIsOmitted() { + assertThat("Method " + POST_BAR_METHOD + " shouldn't be part of the context", + crashcontext.hasNotMethod(POST_BAR_METHOD)); + } + +} diff --git a/infer/tests/endtoend/java/crashcontext/MinimalCrashTest.java b/infer/tests/endtoend/java/crashcontext/MinimalCrashTest.java new file mode 100644 index 000000000..5ce1d1943 --- /dev/null +++ b/infer/tests/endtoend/java/crashcontext/MinimalCrashTest.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +package endtoend.java.crashcontext; + +import static org.hamcrest.MatcherAssert.assertThat; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import utils.CrashContextResults; + +public class MinimalCrashTest { + + public static final String TAG = "MinimalCrashExample"; + + public static final String MAIN_METHOD = + "codetoanalyze.java.crashcontext.MinimalCrashExample.main(java.lang.String[]):void"; + + public static final String TO_STRING_METHOD = + "java.lang.String.toString():java.lang.String"; + + private static CrashContextResults crashcontext; + + @BeforeClass + public static void loadResults() throws IOException { + crashcontext = + CrashContextResults.loadJSONResults(TAG); + } + + @Test + public void shapeOfTheStack() { + assertThat("The stack trace should contain " + MAIN_METHOD, + crashcontext.hasStackFrame(MAIN_METHOD, 0)); + } + + @Test + public void toStringMethodIsDetected() { + assertThat("Method " + TO_STRING_METHOD + " should be part of the context", + crashcontext.hasMethod(TO_STRING_METHOD)); + assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " + + "context tree from " + MAIN_METHOD, + crashcontext.hasPath(MAIN_METHOD, TO_STRING_METHOD)); + } + +} diff --git a/infer/tests/endtoend/java/crashcontext/MultiStackFrameCrashTest.java b/infer/tests/endtoend/java/crashcontext/MultiStackFrameCrashTest.java new file mode 100644 index 000000000..0711842bd --- /dev/null +++ b/infer/tests/endtoend/java/crashcontext/MultiStackFrameCrashTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +package endtoend.java.crashcontext; + +import static org.hamcrest.MatcherAssert.assertThat; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +import utils.CrashContextResults; + +public class MultiStackFrameCrashTest { + + public static final String TAG = "MultiStackFrameCrashExample"; + + public static final String MAIN_METHOD = + "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.main(java.lang.String[]):void"; + + public static final String FOO_METHOD = + "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.foo():void"; + + public static final String BAR_METHOD = + "codetoanalyze.java.crashcontext.MultiStackFrameCrashExample.bar():void"; + + public static final String TO_STRING_METHOD = + "java.lang.String.toString():java.lang.String"; + + private static CrashContextResults crashcontext; + + @BeforeClass + public static void loadResults() throws IOException { + crashcontext = + CrashContextResults.loadJSONResults(TAG); + } + + @Test + public void shapeOfTheStack() { + assertThat("The stack trace should contain " + BAR_METHOD, + crashcontext.hasStackFrame(BAR_METHOD, 0)); + assertThat("The stack trace should contain " + FOO_METHOD, + crashcontext.hasStackFrame(FOO_METHOD, 1)); + assertThat("The stack trace should contain " + MAIN_METHOD, + crashcontext.hasStackFrame(MAIN_METHOD, 2)); + } + + @Test + public void toStringMethodIsFound() { + assertThat("Method " + TO_STRING_METHOD + " should be part of the context", + crashcontext.hasMethod(TO_STRING_METHOD)); + assertThat("Method " + TO_STRING_METHOD + " should be reachable in the " + + "context tree from " + BAR_METHOD, + crashcontext.hasPath(BAR_METHOD, TO_STRING_METHOD)); + } + +} diff --git a/infer/tests/utils/CrashContextResults.java b/infer/tests/utils/CrashContextResults.java new file mode 100644 index 000000000..2480f6e8b --- /dev/null +++ b/infer/tests/utils/CrashContextResults.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2016 - present Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +package utils; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.google.common.io.ByteStreams; + +public class CrashContextResults { + + private static final String CRASHCONTEXT_JSON = + "/buck-out/gen/infer/tests/codetoanalyze/java/crashcontext/analyze/crashcontext.json"; + + private JsonNode json; + private String filename; + + private CrashContextResults(String tag) throws IOException { + String path = + System.getProperty("user.dir") + CRASHCONTEXT_JSON + "." + tag; + byte[] jsonData = ByteStreams.toByteArray(new FileInputStream(path)); + ObjectMapper objectMapper = new ObjectMapper(); + json = objectMapper.readTree(jsonData); + } + + public boolean hasStackFrame(String methodSignature, int pos) { + return methodSignature.equals( + json.path("stack").get(pos).path("method").asText()); + } + + public boolean hasStackFrame(String methodSignature) { + for (JsonNode frame : json.path("stack")) { + if(methodSignature.equals(frame.path("method").asText())) { + return true; + } + } + return false; + } + + private List findNodesForMethod(JsonNode node, + String methodSignature, + List accumulator) { + if(methodSignature.equals(node.path("method").asText())) { + accumulator.add(node); + } + for(JsonNode callee : node.path("callees")) { + findNodesForMethod(callee, methodSignature, accumulator); + } + return accumulator; + } + + private List findNodesForMethod(String methodSignature) { + List accumulator = new ArrayList(); + for (JsonNode frame : json.path("stack")) { + findNodesForMethod(frame, methodSignature, accumulator); + } + return accumulator; + } + + public boolean hasMethod(String methodSignature) { + return !findNodesForMethod(methodSignature).isEmpty(); + } + + public boolean hasNotMethod(String methodSignature) { + return findNodesForMethod(methodSignature).isEmpty(); + } + + public boolean hasPath(String methodFrom, String methodTo) { + for(JsonNode from : findNodesForMethod(methodFrom)) { + if(!findNodesForMethod(from, methodTo, new ArrayList()).isEmpty()) { + return true; + } + } + return false; + } + + public static CrashContextResults loadJSONResults(String tag) throws IOException { + return new CrashContextResults(tag); + } + +}