[python] create console output and report.txt from OCaml

Summary:
Migrating some more Python code.

Still TODO:
- xml report (or kill?)
- inferTraceBugs

Reviewed By: skcho

Differential Revision: D20623613

fbshipit-source-id: 701a624a3

infer/lib/python/inferlib/issues.py

@@ -194,19 +194,9 @@ def _is_user_visible(report):
 
 def print_and_save_errors(infer_out, project_root, json_report, bugs_out,
                           pmd_xml, console_out):
-    errors = utils.load_json_from_path(json_report)
-    errors = [e for e in errors if _is_user_visible(e)]
-    if console_out:
-        utils.stderr('')
-        _text_of_report_list(project_root, errors, bugs_out, console_out=True,
-                             limit=10)
-    plain_out = _text_of_report_list(project_root, errors, bugs_out,
-                                     formatter=colorize.PLAIN_FORMATTER)
-    with codecs.open(bugs_out, 'w',
-                     encoding=config.CODESET, errors='replace') as file_out:
-        file_out.write(plain_out)
-
     if pmd_xml:
+        errors = utils.load_json_from_path(json_report)
+        errors = [e for e in errors if _is_user_visible(e)]
         xml_out = os.path.join(infer_out, config.PMD_XML_FILENAME)
         with codecs.open(xml_out, 'w',
                          encoding=config.CODESET,

infer/man/man1/infer-full.txt

@@ -932,6 +932,10 @@ OPTIONS
            specified OCaml regex, even if they match the whitelist specified
            by --report-whitelist-path-regex           See also infer-report(1) and infer-run(1).
 
+       --report-console-limit int
+           Maximum number of issues to display on standard output. Unset with
+           --report-console-limit-reset to show all.           See also infer-report(1).
+
        --report-current path
            report of the latest revision           See also infer-reportdiff(1).
 
@@ -1631,6 +1635,9 @@ INTERNAL OPTIONS
        --report-blacklist-path-regex-reset
            Set --report-blacklist-path-regex to the empty list.
 
+       --report-console-limit-reset
+           Cancel the effect of --report-console-limit.
+
        --report-current-reset
            Cancel the effect of --report-current.
 

infer/man/man1/infer-report.txt

@@ -304,6 +304,10 @@ OPTIONS
            specified OCaml regex, even if they match the whitelist specified
            by --report-whitelist-path-regex
 
+       --report-console-limit int
+           Maximum number of issues to display on standard output. Unset with
+           --report-console-limit-reset to show all.
+
        --report-formatter { none | phabricator }
            Which formatter to use when emitting the report
 

infer/man/man1/infer.txt

@@ -932,6 +932,10 @@ OPTIONS
            specified OCaml regex, even if they match the whitelist specified
            by --report-whitelist-path-regex           See also infer-report(1) and infer-run(1).
 
+       --report-console-limit int
+           Maximum number of issues to display on standard output. Unset with
+           --report-console-limit-reset to show all.           See also infer-report(1).
+
        --report-current path
            report of the latest revision           See also infer-reportdiff(1).
 

infer/src/base/Config.ml

@@ -1938,6 +1938,13 @@ and ( report_blacklist_files_containing
       ~meta:"error_name" )
 
 
+and report_console_limit =
+  CLOpt.mk_int_opt ~long:"report-console-limit" ~default:5
+    ~in_help:InferCommand.[(Report, manual_generic)]
+    "Maximum number of issues to display on standard output. Unset with \
+     $(b,--report-console-limit-reset) to show all."
+
+
 and report_current =
   CLOpt.mk_path_opt ~long:"report-current"
     ~in_help:InferCommand.[(ReportDiff, manual_generic)]
@@ -2941,6 +2948,8 @@ and report = !report
 
 and report_blacklist_files_containing = !report_blacklist_files_containing
 
+and report_console_limit = !report_console_limit
+
 and report_current = !report_current
 
 and report_custom_error = !report_custom_error

infer/src/base/Config.mli

@@ -531,6 +531,8 @@ val reanalyze : bool
 
 val report_blacklist_files_containing : string list
 
+val report_console_limit : int option
+
 val report_current : string option
 
 val report_formatter : [`No_formatter | `Phabricator_formatter]

infer/src/integration/Driver.ml

@@ -278,19 +278,23 @@ let execute_analyze ~changed_files =
 let report ?(suppress_console = false) () =
   let issues_json = Config.(results_dir ^/ report_json) in
   JsonReports.write_reports ~issues_json ~costs_json:Config.(results_dir ^/ costs_report_json) ;
-  if Config.(test_determinator && process_clang_ast) then
-    TestDeterminator.merge_test_determinator_results () ;
   (* Post-process the report according to the user config. By default, calls report.py to create a
      human-readable report.
 
      Do not bother calling the report hook when called from within Buck. *)
-  match (Config.buck_cache_mode, Config.report_hook) with
+  if not Config.buck_cache_mode then (
-  | true, _ | false, None ->
+    (* Create a dummy bugs.txt file for backwards compatibility. TODO: Stop doing that one day. *)
-      ()
+    Utils.with_file_out (Config.results_dir ^/ "bugs.txt") ~f:(fun outc ->
-  | false, Some prog ->
+        Out_channel.output_string outc "The contents of this file have moved to report.txt.\n" ) ;
-      (* Create a dummy bugs.txt file for backwards compatibility. TODO: Stop doing that one day. *)
+    TextReport.create_from_json
-      Utils.with_file_out (Config.results_dir ^/ "bugs.txt") ~f:(fun outc ->
+      ~quiet:(Config.quiet || suppress_console)
-          Out_channel.output_string outc "The contents of this file have moved to report.txt.\n" ) ;
+      ~console_limit:Config.report_console_limit
+      ~report_txt:Config.(results_dir ^/ report_txt)
+      ~report_json:issues_json ) ;
+  if Config.(test_determinator && process_clang_ast) then
+    TestDeterminator.merge_test_determinator_results () ;
+  match Config.report_hook with
+  | Some prog when (not Config.buck_cache_mode) && Config.pmd_xml ->
       let if_true key opt args = if not opt then args else key :: args in
       let args =
         if_true "--pmd-xml" Config.pmd_xml
@@ -309,6 +313,8 @@ let report ?(suppress_console = false) () =
         L.external_error
           "** Error running the reporting script:@\n**   %s %s@\n** See error above@." prog
           (String.concat ~sep:" " args)
+  | _ ->
+      ()
 
 
 (* shadowed for tracing *)

infer/src/integration/TextReport.ml

@@ -0,0 +1,152 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+open! IStd
+module F = Format
+
+[@@@warning "+9"]
+
+(* how many lines of context around each report *)
+let source_context = 2
+
+module IssueHash = Caml.Hashtbl.Make (String)
+
+let pp_n_spaces n fmt =
+  for _ = 1 to n do
+    F.pp_print_char fmt ' '
+  done
+
+
+module ReportSummary = struct
+  type t = {mutable n_issues: int; issue_type_counts: int IssueHash.t}
+
+  let mk_empty () = {n_issues= 0; issue_type_counts= IssueHash.create 64}
+
+  let pp fmt {n_issues= _; issue_type_counts} =
+    let max_issue_length, issue_counts =
+      IssueHash.to_seq issue_type_counts
+      |> Seq.fold_left
+           (fun (max_issue_length, issue_counts) ((issue_type, _) as issue_with_count) ->
+             let l = String.length issue_type in
+             (Int.max max_issue_length l, issue_with_count :: issue_counts) )
+           (0, [])
+    in
+    List.sort issue_counts ~compare:(fun (issue_type1, count1) (issue_type2, count2) ->
+        (* reverse lexicographic order on (count * issue_type) *)
+        if Int.equal count2 count1 then String.compare issue_type2 issue_type1
+        else Int.compare count2 count1 )
+    |> List.iter ~f:(fun (bug_type, count) ->
+           pp_n_spaces (max_issue_length - String.length bug_type) fmt ;
+           F.fprintf fmt "  %s: %d@\n" bug_type count )
+
+
+  let add_issue summary (jsonbug : Jsonbug_t.jsonbug) =
+    let bug_count =
+      IssueHash.find_opt summary.issue_type_counts jsonbug.bug_type |> Option.value ~default:0
+    in
+    IssueHash.replace summary.issue_type_counts jsonbug.bug_type (bug_count + 1) ;
+    summary.n_issues <- summary.n_issues + 1 ;
+    (* chain for convenience/pretending it's a functional data structure *)
+    summary
+end
+
+let pp_jsonbug fmt {Jsonbug_t.file; severity; line; bug_type; qualifier; _} =
+  F.fprintf fmt "%s:%d: %s: %s@\n  %s" file line (String.lowercase severity) bug_type qualifier
+
+
+let pp_source_context fmt {Jsonbug_t.file= source_name; lnum= report_line; cnum= report_col; enum= _}
+    =
+  let source_name =
+    if Filename.is_absolute source_name then source_name else Config.project_root ^/ source_name
+  in
+  match Sys.is_file source_name with
+  | `No | `Unknown ->
+      ()
+  | `Yes ->
+      let start_line = max 1 (report_line - source_context) in
+      (* could go beyond last line *)
+      let end_line = report_line + source_context in
+      let n_length = String.length (string_of_int end_line) in
+      Utils.with_file_in source_name ~f:(fun in_chan ->
+          Container.fold_until in_chan
+            ~fold:(In_channel.fold_lines ~fix_win_eol:false)
+            ~finish:(fun _final_line_number -> ())
+            ~init:1
+            ~f:(fun line_number line ->
+              if start_line <= line_number && line_number <= end_line then (
+                (* we are inside the context to print *)
+                F.fprintf fmt "  %*d. " n_length line_number ;
+                if report_col < 0 then
+                  (* no column number, print caret next to the line of the report *)
+                  if Int.equal line_number report_line then F.pp_print_string fmt "> "
+                  else F.pp_print_string fmt "  " ;
+                F.pp_print_string fmt line ;
+                F.pp_print_newline fmt () ;
+                if Int.equal line_number report_line && report_col >= 0 then (
+                  pp_n_spaces (2 + n_length + 1 + report_col) fmt ;
+                  F.pp_print_char fmt '^' ;
+                  F.pp_print_newline fmt () ) ) ;
+              if line_number < end_line then Continue (line_number + 1) else Stop () ) )
+
+
+let create_from_json ~quiet ~console_limit ~report_txt ~report_json =
+  (* TOOD: possible optimisation: stream reading report.json to process each issue one by one *)
+  let report = Atdgen_runtime.Util.Json.from_file Jsonbug_j.read_report report_json in
+  let one_issue_to_report_txt fmt (jsonbug : Jsonbug_t.jsonbug) =
+    F.fprintf fmt "%a@\n%a@\n" pp_jsonbug jsonbug pp_source_context
+      {Jsonbug_t.file= jsonbug.file; lnum= jsonbug.line; cnum= jsonbug.column; enum= -1}
+  in
+  let one_issue_to_console ~console_limit i (jsonbug : Jsonbug_t.jsonbug) =
+    match console_limit with
+    | Some limit when i >= limit ->
+        ()
+    | _ ->
+        let style =
+          match jsonbug.severity with
+          | "ERROR" ->
+              ANSITerminal.[Foreground Red]
+          | "WARNING" ->
+              ANSITerminal.[Foreground Yellow]
+          | _ ->
+              []
+        in
+        F.printf "%!" ;
+        ANSITerminal.print_string style (F.asprintf "%a" pp_jsonbug jsonbug) ;
+        F.printf "%!" ;
+        F.printf "@\n%a@\n" pp_source_context
+          {Jsonbug_t.file= jsonbug.file; lnum= jsonbug.line; cnum= jsonbug.column; enum= -1}
+  in
+  Utils.with_file_out report_txt ~f:(fun report_txt_out ->
+      let report_txt_fmt = F.formatter_of_out_channel report_txt_out in
+      if not quiet then F.printf "@\n@[" ;
+      let summary =
+        List.foldi report ~init:(ReportSummary.mk_empty ()) ~f:(fun i summary jsonbug ->
+            let summary' = ReportSummary.add_issue summary jsonbug in
+            one_issue_to_report_txt report_txt_fmt jsonbug ;
+            if not quiet then one_issue_to_console ~console_limit i jsonbug ;
+            summary' )
+      in
+      let n_issues = summary.n_issues in
+      if Int.equal n_issues 0 then (
+        if not quiet then (
+          F.printf "%!" ;
+          ANSITerminal.(printf [Background Magenta; Bold; Foreground White]) "  No issues found  " ;
+          F.printf "@\n%!" ) ;
+        F.pp_print_string report_txt_fmt "@\nNo issues found@\n" )
+      else
+        let s_of_issues = if n_issues > 1 then "s" else "" in
+        if not quiet then (
+          F.printf "@\n%!" ;
+          ANSITerminal.(printf [Bold]) "Found %d issue%s" n_issues s_of_issues ;
+          ( match console_limit with
+          | Some limit when n_issues >= limit ->
+              F.printf " (console output truncated to %d, see '%s' for the full list)" limit
+                report_txt
+          | _ ->
+              () ) ;
+          F.printf "@\n%a@]%!" ReportSummary.pp summary ) ;
+        F.fprintf report_txt_fmt "Found %d issue%s@\n%a%!" n_issues s_of_issues ReportSummary.pp
+          summary )

infer/src/integration/TextReport.mli

@@ -0,0 +1,12 @@
+(*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+open! IStd
+
+val create_from_json :
+  quiet:bool -> console_limit:int option -> report_txt:string -> report_json:string -> unit
+(** Read [report_json] and produce a textual output in [report_txt]. If [not quiet] then display at
+    most [console_limit] issues on stdout. If [console_limit] is [None] then display all the issues. *)