From 8d6153d949baf5d42e33f40cfd6cf97b1ebf19a1 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Thu, 26 Mar 2020 05:14:14 -0700 Subject: [PATCH] [python] create console output and report.txt from OCaml Summary: Migrating some more Python code. Still TODO: - xml report (or kill?) - inferTraceBugs Reviewed By: skcho Differential Revision: D20623613 fbshipit-source-id: 701a624a3 --- infer/lib/python/inferlib/issues.py | 14 +-- infer/man/man1/infer-full.txt | 7 ++ infer/man/man1/infer-report.txt | 4 + infer/man/man1/infer.txt | 4 + infer/src/base/Config.ml | 9 ++ infer/src/base/Config.mli | 2 + infer/src/integration/Driver.ml | 24 +++-- infer/src/integration/TextReport.ml | 152 +++++++++++++++++++++++++++ infer/src/integration/TextReport.mli | 12 +++ 9 files changed, 207 insertions(+), 21 deletions(-) create mode 100644 infer/src/integration/TextReport.ml create mode 100644 infer/src/integration/TextReport.mli diff --git a/infer/lib/python/inferlib/issues.py b/infer/lib/python/inferlib/issues.py index 4cb54e8db..3b37e6246 100644 --- a/infer/lib/python/inferlib/issues.py +++ b/infer/lib/python/inferlib/issues.py @@ -194,19 +194,9 @@ def _is_user_visible(report): def print_and_save_errors(infer_out, project_root, json_report, bugs_out, pmd_xml, console_out): - errors = utils.load_json_from_path(json_report) - errors = [e for e in errors if _is_user_visible(e)] - if console_out: - utils.stderr('') - _text_of_report_list(project_root, errors, bugs_out, console_out=True, - limit=10) - plain_out = _text_of_report_list(project_root, errors, bugs_out, - formatter=colorize.PLAIN_FORMATTER) - with codecs.open(bugs_out, 'w', - encoding=config.CODESET, errors='replace') as file_out: - file_out.write(plain_out) - if pmd_xml: + errors = utils.load_json_from_path(json_report) + errors = [e for e in errors if _is_user_visible(e)] xml_out = os.path.join(infer_out, config.PMD_XML_FILENAME) with codecs.open(xml_out, 'w', encoding=config.CODESET, diff --git a/infer/man/man1/infer-full.txt b/infer/man/man1/infer-full.txt index 69bef64ef..d8c44220e 100644 --- a/infer/man/man1/infer-full.txt +++ b/infer/man/man1/infer-full.txt @@ -932,6 +932,10 @@ OPTIONS specified OCaml regex, even if they match the whitelist specified by --report-whitelist-path-regex See also infer-report(1) and infer-run(1). + --report-console-limit int + Maximum number of issues to display on standard output. Unset with + --report-console-limit-reset to show all. See also infer-report(1). + --report-current path report of the latest revision See also infer-reportdiff(1). @@ -1631,6 +1635,9 @@ INTERNAL OPTIONS --report-blacklist-path-regex-reset Set --report-blacklist-path-regex to the empty list. + --report-console-limit-reset + Cancel the effect of --report-console-limit. + --report-current-reset Cancel the effect of --report-current. diff --git a/infer/man/man1/infer-report.txt b/infer/man/man1/infer-report.txt index e2df15731..3487c9d08 100644 --- a/infer/man/man1/infer-report.txt +++ b/infer/man/man1/infer-report.txt @@ -304,6 +304,10 @@ OPTIONS specified OCaml regex, even if they match the whitelist specified by --report-whitelist-path-regex + --report-console-limit int + Maximum number of issues to display on standard output. Unset with + --report-console-limit-reset to show all. + --report-formatter { none | phabricator } Which formatter to use when emitting the report diff --git a/infer/man/man1/infer.txt b/infer/man/man1/infer.txt index 708146252..29f8d4e3f 100644 --- a/infer/man/man1/infer.txt +++ b/infer/man/man1/infer.txt @@ -932,6 +932,10 @@ OPTIONS specified OCaml regex, even if they match the whitelist specified by --report-whitelist-path-regex See also infer-report(1) and infer-run(1). + --report-console-limit int + Maximum number of issues to display on standard output. Unset with + --report-console-limit-reset to show all. See also infer-report(1). + --report-current path report of the latest revision See also infer-reportdiff(1). diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index 6ca43a6f7..dfa9377ec 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -1938,6 +1938,13 @@ and ( report_blacklist_files_containing ~meta:"error_name" ) +and report_console_limit = + CLOpt.mk_int_opt ~long:"report-console-limit" ~default:5 + ~in_help:InferCommand.[(Report, manual_generic)] + "Maximum number of issues to display on standard output. Unset with \ + $(b,--report-console-limit-reset) to show all." + + and report_current = CLOpt.mk_path_opt ~long:"report-current" ~in_help:InferCommand.[(ReportDiff, manual_generic)] @@ -2941,6 +2948,8 @@ and report = !report and report_blacklist_files_containing = !report_blacklist_files_containing +and report_console_limit = !report_console_limit + and report_current = !report_current and report_custom_error = !report_custom_error diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index f82bd5ca0..97686c344 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -531,6 +531,8 @@ val reanalyze : bool val report_blacklist_files_containing : string list +val report_console_limit : int option + val report_current : string option val report_formatter : [`No_formatter | `Phabricator_formatter] diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index ad594f175..2d69512d7 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -278,19 +278,23 @@ let execute_analyze ~changed_files = let report ?(suppress_console = false) () = let issues_json = Config.(results_dir ^/ report_json) in JsonReports.write_reports ~issues_json ~costs_json:Config.(results_dir ^/ costs_report_json) ; - if Config.(test_determinator && process_clang_ast) then - TestDeterminator.merge_test_determinator_results () ; (* Post-process the report according to the user config. By default, calls report.py to create a human-readable report. Do not bother calling the report hook when called from within Buck. *) - match (Config.buck_cache_mode, Config.report_hook) with - | true, _ | false, None -> - () - | false, Some prog -> - (* Create a dummy bugs.txt file for backwards compatibility. TODO: Stop doing that one day. *) - Utils.with_file_out (Config.results_dir ^/ "bugs.txt") ~f:(fun outc -> - Out_channel.output_string outc "The contents of this file have moved to report.txt.\n" ) ; + if not Config.buck_cache_mode then ( + (* Create a dummy bugs.txt file for backwards compatibility. TODO: Stop doing that one day. *) + Utils.with_file_out (Config.results_dir ^/ "bugs.txt") ~f:(fun outc -> + Out_channel.output_string outc "The contents of this file have moved to report.txt.\n" ) ; + TextReport.create_from_json + ~quiet:(Config.quiet || suppress_console) + ~console_limit:Config.report_console_limit + ~report_txt:Config.(results_dir ^/ report_txt) + ~report_json:issues_json ) ; + if Config.(test_determinator && process_clang_ast) then + TestDeterminator.merge_test_determinator_results () ; + match Config.report_hook with + | Some prog when (not Config.buck_cache_mode) && Config.pmd_xml -> let if_true key opt args = if not opt then args else key :: args in let args = if_true "--pmd-xml" Config.pmd_xml @@ -309,6 +313,8 @@ let report ?(suppress_console = false) () = L.external_error "** Error running the reporting script:@\n** %s %s@\n** See error above@." prog (String.concat ~sep:" " args) + | _ -> + () (* shadowed for tracing *) diff --git a/infer/src/integration/TextReport.ml b/infer/src/integration/TextReport.ml new file mode 100644 index 000000000..05ea8aee4 --- /dev/null +++ b/infer/src/integration/TextReport.ml @@ -0,0 +1,152 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) +open! IStd +module F = Format + +[@@@warning "+9"] + +(* how many lines of context around each report *) +let source_context = 2 + +module IssueHash = Caml.Hashtbl.Make (String) + +let pp_n_spaces n fmt = + for _ = 1 to n do + F.pp_print_char fmt ' ' + done + + +module ReportSummary = struct + type t = {mutable n_issues: int; issue_type_counts: int IssueHash.t} + + let mk_empty () = {n_issues= 0; issue_type_counts= IssueHash.create 64} + + let pp fmt {n_issues= _; issue_type_counts} = + let max_issue_length, issue_counts = + IssueHash.to_seq issue_type_counts + |> Seq.fold_left + (fun (max_issue_length, issue_counts) ((issue_type, _) as issue_with_count) -> + let l = String.length issue_type in + (Int.max max_issue_length l, issue_with_count :: issue_counts) ) + (0, []) + in + List.sort issue_counts ~compare:(fun (issue_type1, count1) (issue_type2, count2) -> + (* reverse lexicographic order on (count * issue_type) *) + if Int.equal count2 count1 then String.compare issue_type2 issue_type1 + else Int.compare count2 count1 ) + |> List.iter ~f:(fun (bug_type, count) -> + pp_n_spaces (max_issue_length - String.length bug_type) fmt ; + F.fprintf fmt " %s: %d@\n" bug_type count ) + + + let add_issue summary (jsonbug : Jsonbug_t.jsonbug) = + let bug_count = + IssueHash.find_opt summary.issue_type_counts jsonbug.bug_type |> Option.value ~default:0 + in + IssueHash.replace summary.issue_type_counts jsonbug.bug_type (bug_count + 1) ; + summary.n_issues <- summary.n_issues + 1 ; + (* chain for convenience/pretending it's a functional data structure *) + summary +end + +let pp_jsonbug fmt {Jsonbug_t.file; severity; line; bug_type; qualifier; _} = + F.fprintf fmt "%s:%d: %s: %s@\n %s" file line (String.lowercase severity) bug_type qualifier + + +let pp_source_context fmt {Jsonbug_t.file= source_name; lnum= report_line; cnum= report_col; enum= _} + = + let source_name = + if Filename.is_absolute source_name then source_name else Config.project_root ^/ source_name + in + match Sys.is_file source_name with + | `No | `Unknown -> + () + | `Yes -> + let start_line = max 1 (report_line - source_context) in + (* could go beyond last line *) + let end_line = report_line + source_context in + let n_length = String.length (string_of_int end_line) in + Utils.with_file_in source_name ~f:(fun in_chan -> + Container.fold_until in_chan + ~fold:(In_channel.fold_lines ~fix_win_eol:false) + ~finish:(fun _final_line_number -> ()) + ~init:1 + ~f:(fun line_number line -> + if start_line <= line_number && line_number <= end_line then ( + (* we are inside the context to print *) + F.fprintf fmt " %*d. " n_length line_number ; + if report_col < 0 then + (* no column number, print caret next to the line of the report *) + if Int.equal line_number report_line then F.pp_print_string fmt "> " + else F.pp_print_string fmt " " ; + F.pp_print_string fmt line ; + F.pp_print_newline fmt () ; + if Int.equal line_number report_line && report_col >= 0 then ( + pp_n_spaces (2 + n_length + 1 + report_col) fmt ; + F.pp_print_char fmt '^' ; + F.pp_print_newline fmt () ) ) ; + if line_number < end_line then Continue (line_number + 1) else Stop () ) ) + + +let create_from_json ~quiet ~console_limit ~report_txt ~report_json = + (* TOOD: possible optimisation: stream reading report.json to process each issue one by one *) + let report = Atdgen_runtime.Util.Json.from_file Jsonbug_j.read_report report_json in + let one_issue_to_report_txt fmt (jsonbug : Jsonbug_t.jsonbug) = + F.fprintf fmt "%a@\n%a@\n" pp_jsonbug jsonbug pp_source_context + {Jsonbug_t.file= jsonbug.file; lnum= jsonbug.line; cnum= jsonbug.column; enum= -1} + in + let one_issue_to_console ~console_limit i (jsonbug : Jsonbug_t.jsonbug) = + match console_limit with + | Some limit when i >= limit -> + () + | _ -> + let style = + match jsonbug.severity with + | "ERROR" -> + ANSITerminal.[Foreground Red] + | "WARNING" -> + ANSITerminal.[Foreground Yellow] + | _ -> + [] + in + F.printf "%!" ; + ANSITerminal.print_string style (F.asprintf "%a" pp_jsonbug jsonbug) ; + F.printf "%!" ; + F.printf "@\n%a@\n" pp_source_context + {Jsonbug_t.file= jsonbug.file; lnum= jsonbug.line; cnum= jsonbug.column; enum= -1} + in + Utils.with_file_out report_txt ~f:(fun report_txt_out -> + let report_txt_fmt = F.formatter_of_out_channel report_txt_out in + if not quiet then F.printf "@\n@[" ; + let summary = + List.foldi report ~init:(ReportSummary.mk_empty ()) ~f:(fun i summary jsonbug -> + let summary' = ReportSummary.add_issue summary jsonbug in + one_issue_to_report_txt report_txt_fmt jsonbug ; + if not quiet then one_issue_to_console ~console_limit i jsonbug ; + summary' ) + in + let n_issues = summary.n_issues in + if Int.equal n_issues 0 then ( + if not quiet then ( + F.printf "%!" ; + ANSITerminal.(printf [Background Magenta; Bold; Foreground White]) " No issues found " ; + F.printf "@\n%!" ) ; + F.pp_print_string report_txt_fmt "@\nNo issues found@\n" ) + else + let s_of_issues = if n_issues > 1 then "s" else "" in + if not quiet then ( + F.printf "@\n%!" ; + ANSITerminal.(printf [Bold]) "Found %d issue%s" n_issues s_of_issues ; + ( match console_limit with + | Some limit when n_issues >= limit -> + F.printf " (console output truncated to %d, see '%s' for the full list)" limit + report_txt + | _ -> + () ) ; + F.printf "@\n%a@]%!" ReportSummary.pp summary ) ; + F.fprintf report_txt_fmt "Found %d issue%s@\n%a%!" n_issues s_of_issues ReportSummary.pp + summary ) diff --git a/infer/src/integration/TextReport.mli b/infer/src/integration/TextReport.mli new file mode 100644 index 000000000..b1b0b3347 --- /dev/null +++ b/infer/src/integration/TextReport.mli @@ -0,0 +1,12 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) +open! IStd + +val create_from_json : + quiet:bool -> console_limit:int option -> report_txt:string -> report_json:string -> unit +(** Read [report_json] and produce a textual output in [report_txt]. If [not quiet] then display at + most [console_limit] issues on stdout. If [console_limit] is [None] then display all the issues. *)