From 88b615d8ab6e03009482e57d594e97b8c40f8fdc Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Thu, 12 Nov 2015 05:21:57 -0800 Subject: [PATCH] move code about infer issues formats to issues.py Reviewed By: jeremydubreil Differential Revision: D2642057 fb-gh-sync-id: d22c62f --- infer/lib/python/BuckAnalyze | 4 +- infer/lib/python/inferlib/analyze.py | 182 +-------------------- infer/lib/python/inferlib/capture/buck.py | 2 +- infer/lib/python/inferlib/issues.py | 188 ++++++++++++++++++++++ 4 files changed, 194 insertions(+), 182 deletions(-) diff --git a/infer/lib/python/BuckAnalyze b/infer/lib/python/BuckAnalyze index a46689266..9acd2d0ed 100755 --- a/infer/lib/python/BuckAnalyze +++ b/infer/lib/python/BuckAnalyze @@ -30,7 +30,7 @@ import time import traceback import zipfile -from inferlib import analyze, utils +from inferlib import analyze, issues, utils ANALYSIS_SUMMARY_OUTPUT = 'analysis_summary.txt' @@ -448,7 +448,7 @@ def collect_results(args, start_time): report.flush() print('\n') - analyze.print_errors(json_report, bugs_out) + issues.print_errors(json_report, bugs_out) stats['int']['total_time'] = int(round(utils.elapsed_time(start_time))) diff --git a/infer/lib/python/inferlib/analyze.py b/infer/lib/python/inferlib/analyze.py index dd2b5b3ca..4dfdb1153 100644 --- a/infer/lib/python/inferlib/analyze.py +++ b/infer/lib/python/inferlib/analyze.py @@ -21,9 +21,7 @@ import os import shutil import subprocess import sys -import tempfile import time -import xml.etree.ElementTree as ET from . import config, issues, jwlib, utils @@ -227,180 +225,6 @@ def help_exit(message): exit(1) -def compare_csv_rows(row_1, row_2): - filename_1 = row_1[utils.CSV_INDEX_FILENAME] - filename_2 = row_2[utils.CSV_INDEX_FILENAME] - if filename_1 < filename_2: - return -1 - elif filename_1 > filename_2: - return 1 - else: - line_1 = int(row_1[utils.CSV_INDEX_LINE]) - line_2 = int(row_2[utils.CSV_INDEX_LINE]) - return line_1 - line_2 - - -def compare_json_rows(row_1, row_2): - filename_1 = row_1[utils.JSON_INDEX_FILENAME] - filename_2 = row_2[utils.JSON_INDEX_FILENAME] - if filename_1 < filename_2: - return -1 - elif filename_1 > filename_2: - return 1 - else: - line_1 = int(row_1[utils.JSON_INDEX_LINE]) - line_2 = int(row_2[utils.JSON_INDEX_LINE]) - return line_1 - line_2 - - -def should_report(analyzer, error_kind, error_type, error_bucket): - analyzers_whitelist = [ - config.ANALYZER_ERADICATE, - config.ANALYZER_CHECKERS, - config.ANALYZER_TRACING, - ] - error_kinds = [issues.ISSUE_KIND_ERROR, issues.ISSUE_KIND_WARNING] - null_style_buckets = ['B1', 'B2'] - - if analyzer in analyzers_whitelist: - return True - - if error_kind not in error_kinds: - return False - - if not error_type: - return False - - if error_type in issues.NULL_STYLE_ISSUE_TYPES: - return error_bucket in null_style_buckets - - return error_type in issues.ISSUE_TYPES - - -def should_report_csv(analyzer, row): - error_kind = row[utils.CSV_INDEX_KIND] - error_type = row[utils.CSV_INDEX_TYPE] - error_bucket = '' # can be updated later once we extract it from qualifier - - try: - qualifier_xml = ET.fromstring(row[utils.CSV_INDEX_QUALIFIER_TAGS]) - if qualifier_xml.tag == utils.QUALIFIER_TAGS: - bucket = qualifier_xml.find(utils.BUCKET_TAGS) - if bucket is not None: - error_bucket = bucket.text - except ET.ParseError: - pass # this will skip any invalid xmls - - return should_report(analyzer, error_kind, error_type, error_bucket) - - -def should_report_json(analyzer, row): - error_kind = row[utils.JSON_INDEX_KIND] - error_type = row[utils.JSON_INDEX_TYPE] - error_bucket = '' # can be updated later once we extract it from qualifier - - for qual_tag in row[utils.QUALIFIER_TAGS]: - if qual_tag['tag'] == utils.BUCKET_TAGS: - error_bucket = qual_tag['value'] - break - - return should_report(analyzer, error_kind, error_type, error_bucket) - - -def clean_json(args, json_report): - collected_rows = [] - with codecs.open(json_report, 'r', encoding=utils.LOCALE) as file_in: - rows = json.load(file_in) - for row in rows: - filename = row[utils.JSON_INDEX_FILENAME] - if os.path.isfile(filename): - if args.no_filtering or should_report_json(args.analyzer, row): - collected_rows.append(row) - collected_rows = sorted( - collected_rows, - cmp=compare_json_rows) - temporary_file = tempfile.mktemp() - with codecs.open(temporary_file, 'w', encoding=utils.LOCALE) as file_out: - json.dump(collected_rows, file_out) - file_out.flush() - shutil.move(temporary_file, json_report) - - -def clean_csv(args, csv_report): - collected_rows = [] - with open(csv_report, 'r') as file_in: - reader = utils.locale_csv_reader(file_in) - rows = [row for row in reader] - if len(rows) <= 1: - return rows - else: - for row in rows[1:]: - filename = row[utils.CSV_INDEX_FILENAME] - if os.path.isfile(filename): - if args.no_filtering \ - or should_report_csv(args.analyzer, row): - collected_rows.append(row) - collected_rows = sorted( - collected_rows, - cmp=compare_csv_rows) - collected_rows = [rows[0]] + collected_rows - temporary_file = tempfile.mktemp() - with open(temporary_file, 'w') as file_out: - writer = csv.writer(file_out) - writer.writerows(collected_rows) - file_out.flush() - shutil.move(temporary_file, csv_report) - - -def print_and_write(file_out, message): - print(message) - file_out.write(message + '\n') - - -def print_errors(json_report, bugs_out): - with codecs.open(json_report, 'r', encoding=utils.LOCALE) as file_in: - errors = json.load(file_in) - - errors = filter(lambda row: row[utils.JSON_INDEX_KIND] in - [issues.ISSUE_KIND_ERROR, issues.ISSUE_KIND_WARNING], - errors) - - with codecs.open(bugs_out, 'w', encoding=utils.LOCALE) as file_out: - text_errors_list = [] - for row in errors: - filename = row[utils.JSON_INDEX_FILENAME] - if os.path.isfile(filename): - kind = row[utils.JSON_INDEX_KIND] - line = row[utils.JSON_INDEX_LINE] - error_type = row[utils.JSON_INDEX_TYPE] - msg = row[utils.JSON_INDEX_QUALIFIER] - indenter = utils.Indenter() - indenter.indent_push() - indenter.add( - utils.build_source_context(filename, - utils.TERMINAL_FORMATTER, - int(line))) - source_context = unicode(indenter) - text_errors_list.append( - u'{0}:{1}: {2}: {3}\n {4}\n{5}'.format( - filename, - line, - kind.lower(), - error_type, - msg, - source_context, - ) - ) - n_issues = len(text_errors_list) - if n_issues == 0: - print_and_write(file_out, 'No issues found') - else: - msg = '\nFound %s\n' % utils.get_plural('issue', n_issues) - print_and_write(file_out, msg) - text_errors = '\n\n'.join(text_errors_list) - print_and_write(file_out, text_errors) - - def run_command(cmd, debug_mode, javac_arguments, step, analyzer): if debug_mode: print('\n{0}\n'.format(' '.join(cmd))) @@ -668,8 +492,8 @@ class Infer: logging.error('Error with InferPrint with the command: ' + infer_print_cmd) else: - clean_csv(self.args, csv_report) - clean_json(self.args, json_report) + issues.clean_csv(self.args, csv_report) + issues.clean_json(self.args, json_report) self.update_stats_with_warnings(csv_report) return exit_status @@ -725,7 +549,7 @@ class Infer: utils.JSON_REPORT_FILENAME) bugs_out = os.path.join(self.args.infer_out, utils.BUGS_FILENAME) - print_errors(json_report, bugs_out) + issues.print_errors(json_report, bugs_out) def print_analysis_stats(self): procs_total = self.stats['int']['procedures'] diff --git a/infer/lib/python/inferlib/capture/buck.py b/infer/lib/python/inferlib/capture/buck.py index 08be1803e..096daf14d 100644 --- a/infer/lib/python/inferlib/capture/buck.py +++ b/infer/lib/python/inferlib/capture/buck.py @@ -139,7 +139,7 @@ class BuckAnalyzer: utils.merge_json_reports( result_files, merged_results_path) - # TODO: adapt analyze.print_errors to support json and print on screen + # TODO: adapt issues.print_errors to support json and print on screen print('Results saved in {results_path}'.format( results_path=merged_results_path)) return os.EX_OK diff --git a/infer/lib/python/inferlib/issues.py b/infer/lib/python/inferlib/issues.py index 7cbc76176..204ddac68 100644 --- a/infer/lib/python/inferlib/issues.py +++ b/infer/lib/python/inferlib/issues.py @@ -10,6 +10,20 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals +import codecs +import csv +import json +import os +import shutil +import sys +import tempfile +import xml.etree.ElementTree as ET + +from . import config, utils + + +# Increase the limit of the CSV parser to sys.maxlimit +csv.field_size_limit(sys.maxsize) ISSUE_KIND_ERROR = 'ERROR' ISSUE_KIND_WARNING = 'WARNING' @@ -38,3 +52,177 @@ NULL_STYLE_ISSUE_TYPES = [ 'PARAMETER_NOT_NULL_CHECKED', 'PREMATURE_NIL_TERMINATION_ARGUMENT', ] + + +def clean_csv(args, csv_report): + collected_rows = [] + with open(csv_report, 'r') as file_in: + reader = csv.reader(file_in) + rows = [row for row in reader] + if len(rows) <= 1: + return rows + else: + for row in rows[1:]: + filename = row[utils.CSV_INDEX_FILENAME] + if os.path.isfile(filename): + if args.no_filtering \ + or _should_report_csv(args.analyzer, row): + collected_rows.append(row) + collected_rows = sorted( + collected_rows, + cmp=_compare_csv_rows) + collected_rows = [rows[0]] + collected_rows + temporary_file = tempfile.mktemp() + with open(temporary_file, 'w') as file_out: + writer = csv.writer(file_out) + writer.writerows(collected_rows) + file_out.flush() + shutil.move(temporary_file, csv_report) + + +def clean_json(args, json_report): + collected_rows = [] + with open(json_report, 'r') as file_in: + rows = json.load(file_in) + for row in rows: + filename = row[utils.JSON_INDEX_FILENAME] + if os.path.isfile(filename): + if args.no_filtering \ + or _should_report_json(args.analyzer, row): + collected_rows.append(row) + collected_rows = sorted( + collected_rows, + cmp=_compare_json_rows) + temporary_file = tempfile.mktemp() + with open(temporary_file, 'w') as file_out: + json.dump(collected_rows, file_out) + file_out.flush() + shutil.move(temporary_file, json_report) + + +def print_errors(json_report, bugs_out): + with codecs.open(json_report, 'r', encoding=utils.LOCALE) as file_in: + errors = json.load(file_in) + + errors = filter(lambda row: row[utils.JSON_INDEX_KIND] in + [ISSUE_KIND_ERROR, ISSUE_KIND_WARNING], + errors) + + with codecs.open(bugs_out, 'w', encoding=utils.LOCALE) as file_out: + text_errors_list = [] + for row in errors: + filename = row[utils.JSON_INDEX_FILENAME] + if os.path.isfile(filename): + kind = row[utils.JSON_INDEX_KIND] + line = row[utils.JSON_INDEX_LINE] + error_type = row[utils.JSON_INDEX_TYPE] + msg = row[utils.JSON_INDEX_QUALIFIER] + indenter = utils.Indenter() + indenter.indent_push() + indenter.add( + utils.build_source_context(filename, + utils.TERMINAL_FORMATTER, + int(line))) + source_context = unicode(indenter) + text_errors_list.append( + u'{0}:{1}: {2}: {3}\n {4}\n{5}'.format( + filename, + line, + kind.lower(), + error_type, + msg, + source_context, + ) + ) + n_issues = len(text_errors_list) + if n_issues == 0: + _print_and_write(file_out, 'No issues found') + else: + msg = '\nFound %s\n' % utils.get_plural('issue', n_issues) + _print_and_write(file_out, msg) + text_errors = '\n\n'.join(text_errors_list) + _print_and_write(file_out, text_errors) + + +def _compare_issues(filename_1, line_1, filename_2, line_2): + if filename_1 < filename_2: + return -1 + elif filename_1 > filename_2: + return 1 + else: + return line_1 - line_2 + + +def _compare_csv_rows(row_1, row_2): + filename_1 = row_1[utils.CSV_INDEX_FILENAME] + filename_2 = row_2[utils.CSV_INDEX_FILENAME] + line_1 = int(row_1[utils.CSV_INDEX_LINE]) + line_2 = int(row_2[utils.CSV_INDEX_LINE]) + return _compare_issues(filename_1, line_1, filename_2, line_2) + + +def _compare_json_rows(row_1, row_2): + filename_1 = row_1[utils.JSON_INDEX_FILENAME] + filename_2 = row_2[utils.JSON_INDEX_FILENAME] + line_1 = row_1[utils.JSON_INDEX_LINE] + line_2 = row_2[utils.JSON_INDEX_LINE] + return _compare_issues(filename_1, line_1, filename_2, line_2) + + +def _should_report(analyzer, error_kind, error_type, error_bucket): + analyzers_whitelist = [ + config.ANALYZER_ERADICATE, + config.ANALYZER_CHECKERS, + config.ANALYZER_TRACING, + ] + error_kinds = [ISSUE_KIND_ERROR, ISSUE_KIND_WARNING] + null_style_buckets = ['B1', 'B2'] + + if analyzer in analyzers_whitelist: + return True + + if error_kind not in error_kinds: + return False + + if not error_type: + return False + + if error_type in NULL_STYLE_ISSUE_TYPES: + return error_bucket in null_style_buckets + + return error_type in ISSUE_TYPES + + +def _should_report_csv(analyzer, row): + error_kind = row[utils.CSV_INDEX_KIND] + error_type = row[utils.CSV_INDEX_TYPE] + error_bucket = '' # can be updated later once we extract it from qualifier + + try: + qualifier_xml = ET.fromstring(row[utils.CSV_INDEX_QUALIFIER_TAGS]) + if qualifier_xml.tag == utils.QUALIFIER_TAGS: + bucket = qualifier_xml.find(utils.BUCKET_TAGS) + if bucket is not None: + error_bucket = bucket.text + except ET.ParseError: + pass # this will skip any invalid xmls + + return _should_report(analyzer, error_kind, error_type, error_bucket) + + +def _should_report_json(analyzer, row): + error_kind = row[utils.JSON_INDEX_KIND] + error_type = row[utils.JSON_INDEX_TYPE] + error_bucket = '' # can be updated later once we extract it from qualifier + + for qual_tag in row[utils.QUALIFIER_TAGS]: + if qual_tag['tag'] == utils.BUCKET_TAGS: + error_bucket = qual_tag['value'] + break + + return _should_report(analyzer, error_kind, error_type, error_bucket) + + +def _print_and_write(file_out, message): + print(message) + file_out.write(message + '\n')