[memtrace] allow memory tracing of analysis workers

Summary: As per summary. Note that biabduction will make the results imprecise due to async exceptions from the timeout signal handler, so we warn when both are enabled (https://github.com/janestreet/memtrace/issues/2).

Reviewed By: jvillard

Differential Revision: D25219737

fbshipit-source-id: bdef228fc

infer/man/man1/infer-analyze.txt

@@ -219,6 +219,14 @@ OPTIONS
        --max-jobs int
            Maximum number of analysis jobs running simultaneously
 
+       --memtrace-analysis-profiling
+           Activates: Generate OCaml analysis allocation traces in
+           `infer-out/memtrace`. (Conversely:
+           --no-memtrace-analysis-profiling)
+
+       --memtrace-sampling-rate float
+           Sampling rate for Memtrace allocation profiling. Default is 1e-6.
+
        --print-active-checkers
            Activates: Print the active checkers before starting the analysis
            (Conversely: --no-print-active-checkers)

infer/man/man1/infer-full.txt

@@ -822,6 +822,15 @@ OPTIONS
            maximum nesting level are skipped. If omitted, all levels are
            shown.           See also infer-explore(1).
 
+       --memtrace-analysis-profiling
+           Activates: Generate OCaml analysis allocation traces in
+           `infer-out/memtrace`. (Conversely:
+           --no-memtrace-analysis-profiling)           See also infer-analyze(1).
+
+       --memtrace-sampling-rate float
+           Sampling rate for Memtrace allocation profiling. Default is 1e-6.       
+    See also infer-analyze(1).
+
        --merge
            Activates: Merge the captured results directories specified in the
            dependency file. (Conversely: --no-merge)           See also infer-analyze(1).
@@ -1622,6 +1631,9 @@ INTERNAL OPTIONS
        --max-nesting-reset
            Cancel the effect of --max-nesting.
 
+       --memtrace-sampling-rate-reset
+           Cancel the effect of --memtrace-sampling-rate.
+
        --method-decls-info method_decls_info.json
            Specifies the file containing the method declarations info (eg.
            start line, end line, class, method name, etc.) when Infer is run

infer/man/man1/infer.txt

@@ -822,6 +822,15 @@ OPTIONS
            maximum nesting level are skipped. If omitted, all levels are
            shown.           See also infer-explore(1).
 
+       --memtrace-analysis-profiling
+           Activates: Generate OCaml analysis allocation traces in
+           `infer-out/memtrace`. (Conversely:
+           --no-memtrace-analysis-profiling)           See also infer-analyze(1).
+
+       --memtrace-sampling-rate float
+           Sampling rate for Memtrace allocation profiling. Default is 1e-6.       
+    See also infer-analyze(1).
+
        --merge
            Activates: Merge the captured results directories specified in the
            dependency file. (Conversely: --no-merge)           See also infer-analyze(1).

infer/src/backend/InferAnalyze.ml

@@ -165,12 +165,26 @@ let analyze source_files_to_analyze =
     in
     (* Prepare tasks one file at a time while executing in parallel *)
     RestartScheduler.setup () ;
+    let allocation_traces_dir = ResultsDir.get_path AllocationTraces in
+    if Config.memtrace_analysis then (
+      Utils.create_dir allocation_traces_dir ;
+      if Config.is_checker_enabled Biabduction then
+        L.user_warning
+          "Memtrace and biabduction are incompatible \
+           (https://github.com/janestreet/memtrace/issues/2)@\n" ) ;
     let runner =
       (* use a ref to pass data from prologue to epilogue without too much machinery *)
       let gc_stats_pre_fork = ref None in
       let child_prologue () =
         BackendStats.reset () ;
-        gc_stats_pre_fork := Some (GCStats.get ~since:ProgramStart)
+        gc_stats_pre_fork := Some (GCStats.get ~since:ProgramStart) ;
+        if Config.memtrace_analysis then
+          let filename =
+            allocation_traces_dir ^/ F.asprintf "memtrace.%a" Pid.pp (Unix.getpid ())
+          in
+          Memtrace.start_tracing ~context:None ~sampling_rate:Config.memtrace_sampling_rate
+            ~filename
+          |> ignore
       in
       let child_epilogue () =
         let gc_stats_in_fork =

infer/src/backend/dune

@@ -11,8 +11,8 @@
     ATDGenerated -open IBase -open IR -open Absint -open Biabduction -open BO
     -open Nullsafe -open Pulselib -open Checkers -open Costlib -open Quandary
     -open TOPLlib -open Concurrency -open Labs))
- (libraries core IStdlib ATDGenerated IBase IR Absint Biabduction Nullsafe BO
-   Checkers Costlib Quandary TOPLlib Concurrency Labs)
+ (libraries core memtrace IStdlib ATDGenerated IBase IR Absint Biabduction
+   Nullsafe BO Checkers Costlib Quandary TOPLlib Concurrency Labs)
  (preprocess
   (pps ppx_compare ppx_fields_conv ppx_yojson_conv)))
 

infer/src/base/Config.ml

@@ -1599,10 +1599,16 @@ and max_nesting =
      skipped. If omitted, all levels are shown."
 
 
-and method_decls_info =
-  CLOpt.mk_path_opt ~long:"method-decls-info" ~meta:"method_decls_info.json"
-    "Specifies the file containing the method declarations info (eg. start line, end line, class, \
-     method name, etc.) when Infer is run Test Determinator mode with $(b,--test-determinator)."
+and memtrace_analysis =
+  CLOpt.mk_bool ~long:"memtrace-analysis-profiling"
+    ~in_help:InferCommand.[(Analyze, manual_generic)]
+    "Generate OCaml analysis allocation traces in `infer-out/memtrace`."
+
+
+and memtrace_sampling_rate =
+  CLOpt.mk_float_opt ~long:"memtrace-sampling-rate" ~default:1e-6
+    ~in_help:InferCommand.[(Analyze, manual_generic)]
+    "Sampling rate for Memtrace allocation profiling. Default is 1e-6."
 
 
 and merge =
@@ -1611,6 +1617,12 @@ and merge =
     "Merge the captured results directories specified in the dependency file."
 
 
+and method_decls_info =
+  CLOpt.mk_path_opt ~long:"method-decls-info" ~meta:"method_decls_info.json"
+    "Specifies the file containing the method declarations info (eg. start line, end line, class, \
+     method name, etc.) when Infer is run Test Determinator mode with $(b,--test-determinator)."
+
+
 and ml_buckets =
   CLOpt.mk_symbol_seq ~deprecated:["ml_buckets"; "-ml_buckets"] ~long:"ml-buckets"
     ~default:[`MLeak_cf]
@@ -3004,10 +3016,14 @@ and load_average =
 
 and max_nesting = !max_nesting
 
-and method_decls_info = !method_decls_info
+and memtrace_analysis = !memtrace_analysis
+
+and memtrace_sampling_rate = Option.value_exn !memtrace_sampling_rate
 
 and merge = !merge
 
+and method_decls_info = !method_decls_info
+
 and ml_buckets = !ml_buckets
 
 and modified_lines = !modified_lines

infer/src/base/Config.mli

@@ -382,6 +382,10 @@ val liveness_ignored_constant : string list
 
 val max_nesting : int option
 
+val memtrace_analysis : bool
+
+val memtrace_sampling_rate : float
+
 val merge : bool
 
 val method_decls_info : string option

infer/src/base/ResultsDirEntryName.ml

@@ -10,6 +10,7 @@ open! IStd
 let buck_infer_deps_file_name = "infer-deps.txt"
 
 type id =
+  | AllocationTraces
   | CaptureDB
   | CaptureDependencies
   | ChangedFunctions
@@ -52,6 +53,11 @@ type t =
             e.g., a distributed Buck cache. *) }
 
 let of_id = function
+  | AllocationTraces ->
+      { rel_path= "memtrace"
+      ; kind= Directory
+      ; before_incremental_analysis= Delete
+      ; before_caching_capture= Delete }
   | CaptureDependencies ->
       { rel_path= buck_infer_deps_file_name
       ; kind= File

infer/src/base/ResultsDirEntryName.mli

@@ -10,6 +10,7 @@ open! IStd
     directory you probably want to use {!ResultsDir.Entry} instead of this module. *)
 
 type id =
+  | AllocationTraces  (** directory for storing allocation traces *)
   | CaptureDB  (** the capture database *)
   | CaptureDependencies  (** list of infer-out/ directories that contain capture artefacts *)
   | ChangedFunctions  (** results of the clang test determinator *)

infer/src/base/dune

@@ -9,8 +9,8 @@
  (flags
   (:standard -open Core -open IStdlib -open IStd -open ATDGenerated -open
     OpenSource))
- (libraries cmdliner core mtime.clock.os parmap re sqlite3 zip ATDGenerated
-   IStdlib OpenSource)
+ (libraries cmdliner core memtrace mtime.clock.os parmap re sqlite3 zip
+   ATDGenerated IStdlib OpenSource)
  (preprocess
   (pps ppx_blob ppx_compare ppx_enumerate))
  (preprocessor_deps

infer/src/deadcode/dune.in

@@ -9,7 +9,7 @@
  (flags
   (:standard -w +60))
  (libraries javalib ANSITerminal async atdgen base base64 cmdliner core iter
-   mtime.clock.os ocamlgraph oUnit parmap re sawja sqlite3 str unix xmlm
+   memtrace mtime.clock.os ocamlgraph oUnit parmap re sawja sqlite3 str unix xmlm
    yojson zarith zip CStubs)
  (modules All_infer_in_one_file)
  (preprocess

opam

@@ -32,6 +32,7 @@ depends: [
   "dune" {build & >="2.0"}
   "iter"
   "javalib" {>="3.2.1"}
+  "memtrace" {>="0.1.2"}
   "mlgmpidl" {>="1.2.12"}
   "mtime"
   "ocaml" {>="4.09.0"}

opam.locked

@@ -64,6 +64,7 @@ depends: [
   "jane-street-headers" {= "v0.14.0"}
   "javalib" {= "3.2.1"}
   "jst-config" {= "v0.14.0"}
+  "memtrace" {= "0.1.2"}
   "menhir" {= "20200624"}
   "menhirLib" {= "20200624"}
   "menhirSdk" {= "20200624"}
@@ -72,6 +73,7 @@ depends: [
   "num" {= "1.3"}
   "ocaml" {= "4.11.1"}
   "ocaml-compiler-libs" {= "v0.12.1"}
+  "ocaml-config" {= "1"}
   "ocaml-migrate-parsetree" {= "1.7.3"}
   "ocaml-variants" {= "4.11.1+flambda"}
   "ocamlbuild" {= "0.14.0"}