[refactor] Call StatsAggregator from Infer instead of from python

Reviewed By: martinoluca

Differential Revision: D4124689

fbshipit-source-id: fdc8f67

.gitignore

@@ -71,7 +71,6 @@ buck-out/
 /infer/bin/InferClang++
 /infer/bin/InferJava
 /infer/bin/InferPrint
-/infer/bin/InferStatsAggregator
 /infer/bin/InferBuckCompilationDatabase
 /infer/bin/InferUnit
 /infer/bin/Typeprop

Makefile

@@ -295,7 +295,6 @@ endif
 	$(INSTALL_PROGRAM) -C $(INFER_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	$(INSTALL_PROGRAM) -C $(INFERANALYZE_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	$(INSTALL_PROGRAM) -C $(INFERPRINT_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
-	$(INSTALL_PROGRAM) -C $(INFERSTATS_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/
 	(cd $(DESTDIR)$(bindir)/ && \
 	 $(REMOVE) infer && \
 	 $(LN_S) $(libdir)/infer/infer/bin/infer infer)

Makefile.config.in

@@ -95,7 +95,6 @@ CAPTURE_LIB_DIR = $(PYTHON_LIB_DIR)/capture
 INFERANALYZE_BIN = $(BIN_DIR)/InferAnalyze
 INFERCLANG_BIN = $(BIN_DIR)/InferClang
 INFERJAVA_BIN = $(BIN_DIR)/InferJava
-INFERSTATS_BIN = $(BIN_DIR)/InferStatsAggregator
 INFERPRINT_BIN = $(BIN_DIR)/InferPrint
 INFERUNIT_BIN = $(BIN_DIR)/InferUnit
 INFER_BUCK_COMPILATION_DATABASE_BIN = $(BIN_DIR)/InferBuckCompilationDatabase

infer/lib/python/infer.py

@@ -25,7 +25,6 @@ from inferlib import analyze, config, utils
 from inferlib.capture import make
 
 CAPTURE_PACKAGE = 'capture'
-TOP_LEVEL_ENVVAR = 'TOP_LEVEL_INFER_INSTANCE_ALREADY_RUNNING'
 
 # token that identifies the end of the options for infer and the beginning
 # of the compilation command
@@ -126,12 +125,6 @@ def create_argparser(parents=[]):
 
 
 def main():
-    toplevel_envvar_value = os.environ.get(TOP_LEVEL_ENVVAR, None)
-    is_toplevel_instance = False
-    if toplevel_envvar_value is None:
-        os.environ[TOP_LEVEL_ENVVAR] = '1'
-        is_toplevel_instance = True
-
     to_parse, cmd = split_args_to_parse()
     # get the module name (if any), then load it
     capture_module_name = os.path.basename(cmd[0]) if len(cmd) > 0 else None
@@ -215,17 +208,5 @@ def main():
         analysis.analyze_and_report()
         analysis.save_stats()
 
-    if is_toplevel_instance is True:
-        buck_out_for_stats_aggregator = None
-        if (mod_name == 'buck' and
-            os.path.isfile(
-                os.path.join(args.infer_out,
-                             config.INFER_BUCK_DEPS_FILENAME))):
-            buck_out_for_stats_aggregator = 'buck-out'
-        logging.info('Aggregating stats')
-        output = utils.run_infer_stats_aggregator(
-            args.infer_out, buck_out_for_stats_aggregator)
-        logging.info(output)
-
 if __name__ == '__main__':
     main()

infer/lib/python/inferlib/utils.py

@@ -128,23 +128,6 @@ def infer_key(analyzer):
     return '/'.join([analyzer, infer_version()])
 
 
-def run_infer_stats_aggregator(infer_out, buck_out=None):
-    buck_out_args = []
-    if buck_out is not None:
-        buck_out_args = ['--buck-out', buck_out]
-    try:
-        cmd = get_cmd_in_bin_dir('InferStatsAggregator')
-        return subprocess.check_output(
-            [cmd, '--results-dir', infer_out] + buck_out_args,
-            stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as exc:
-        output = 'Error while aggregating stats ({ec}): {output}'\
-            .format(ec=exc.returncode, output=exc.output)
-        stderr('Info: aggregated performance data are not available in '
-               'this run')
-        return output
-
-
 def vcs_branch(dir='.'):
     cwd = os.getcwd()
     devnull = open(os.devnull, 'w')

infer/src/Makefile

@@ -130,9 +130,6 @@ SCRIPT_SOURCES = scripts
 CHECKCOPYRIGHT_BIN = $(SCRIPT_DIR)/checkCopyright
 CHECKCOPYRIGHT_MAIN = $(SCRIPT_SOURCES)/checkCopyright
 
-STATSAGGREGATOR_BIN = $(BIN_DIR)/InferStatsAggregator
-STATSAGGREGATOR_MAIN = backend/StatsAggregator
-
 #### End of declarations ####
 
 ifeq ($(IS_FACEBOOK_TREE),yes)
@@ -160,8 +157,7 @@ INFER_BASE_TARGETS = \
   $(INFERANALYZE_MAIN).native \
   $(INFERPRINT_MAIN).native \
   $(INFERUNIT_MAIN).native \
-  $(CHECKCOPYRIGHT_MAIN).native \
+  $(CHECKCOPYRIGHT_MAIN).native
-  $(STATSAGGREGATOR_MAIN).native
 
 INFER_ALL_TARGETS = $(INFER_BASE_TARGETS) \
   $(INFERJAVA_MAIN).native \
@@ -194,7 +190,6 @@ infer: init $(STACKTREE_ATDGEN_STUBS) $(INFERPRINT_ATDGEN_STUBS)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERANALYZE_MAIN).native $(INFERANALYZE_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERPRINT_MAIN).native $(INFERPRINT_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(CHECKCOPYRIGHT_MAIN).native $(CHECKCOPYRIGHT_BIN)
-	$(COPY) $(INFER_BUILD_DIR)/$(STATSAGGREGATOR_MAIN).native $(STATSAGGREGATOR_BIN)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERUNIT_MAIN).native $(INFERUNIT_BIN)
 ifeq ($(BUILD_JAVA_ANALYZERS),yes)
 	$(COPY) $(INFER_BUILD_DIR)/$(INFERJAVA_MAIN).native $(INFERJAVA_BIN)
@@ -383,7 +378,7 @@ endif
 	$(REMOVE) base/Version.ml.tmp.*
 	$(REMOVE) backend/jsonbug_{j,t}.ml{,i}
 	$(REMOVE) checkers/stacktree_{j,t}.ml{,i}
-	$(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(STATSAGGREGATOR_BIN)
+	$(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN)
 	$(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN)
 	$(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(INFER_BUCK_COMPILATION_DATABASE_BIN)
 	$(REMOVE) $(CLANG_ATDGEN_STUBS)

infer/src/backend/StatsAggregator.re

@@ -165,7 +165,7 @@ let aggregate_stats_by_target tp => {
   {frontend_json_data, backend_json_data, reporting_json_data}
 };
 
-let () = {
+let generate_files () => {
   let infer_out = Config.results_dir;
   let stats_files = collect_all_stats_files ();
   let origin =

infer/src/backend/StatsAggregator.rei

@@ -7,3 +7,8 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 open! Utils;
+
+
+/** Generate files containing statistics aggregated from individual statistics files under
+    Config.results_dir */
+let generate_files: unit => unit;

infer/src/backend/infer.ml

@@ -169,6 +169,7 @@ let analyze = function
 
 let epilogue build_mode =
   if Config.is_originator then (
+    StatsAggregator.generate_files () ;
     if Config.analyzer = Some Config.Crashcontext then
       Crashcontext.crashcontext_epilogue ~in_buck_mode:(build_mode = Buck);
     if Config.fail_on_bug then

infer/src/base/CommandLineOption.ml

@@ -17,8 +17,7 @@ module YBU = Yojson.Basic.Util
 
 (** Each command line option may appear in the --help list of any executable, these tags are used to
     specify which executables for which an option will be documented. *)
-type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator
+type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel
-         | Toplevel
 
 
 (** Association list of executable (base)names to their [exe]s. *)
@@ -28,7 +27,6 @@ let exes = [
   ("InferClang", Clang);
   ("InferJava", Java);
   ("InferPrint", Print);
-  ("InferStatsAggregator", StatsAggregator);
   ("infer", Toplevel);
   ("interactive", Interactive);
 ]
@@ -576,7 +574,7 @@ let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file current_exe e
   let exe_name = Sys.executable_name in
   let should_parse_cl_args = match current_exe with
     | Clang | Interactive -> false
-    | Analyze | BuckCompilationDatabase | Java | Print | StatsAggregator | Toplevel -> true in
+    | Analyze | BuckCompilationDatabase | Java | Print | Toplevel -> true in
   let env_cl_args =
     if should_parse_cl_args then prepend_to_argv env_args
     else env_args in

infer/src/base/CommandLineOption.mli

@@ -11,8 +11,7 @@
 
 open! Utils
 
-type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator
+type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel
-         | Toplevel
 
 (** Association list of executable (base)names to their [exe]s. *)
 val exes : (string * exe) list

infer/src/base/Config.ml

@@ -417,6 +417,9 @@ let resolve path =
   else
     path
 
+
+(** Command Line options *)
+
 (* Declare the phase 1 options *)
 
 let inferconfig_home =
@@ -605,7 +608,7 @@ and buck_build_args =
 
 and buck_out =
   CLOpt.mk_string_opt ~long:"buck-out"
-    ~exes:CLOpt.[StatsAggregator] ~meta:"dir" "Specify the root directory of buck-out"
+    ~exes:CLOpt.[Toplevel] ~meta:"dir" "Specify the root directory of buck-out"
 
 and bugs_csv =
   CLOpt.mk_string_opt ~deprecated:["bugs"] ~long:"issues-csv"
@@ -1096,7 +1099,7 @@ and report_custom_error =
 and results_dir =
   CLOpt.mk_path ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o"
     ~default:(init_work_dir // "infer-out")
-    ~exes:CLOpt.[Analyze;Clang;Java;Print;StatsAggregator]
+    ~exes:CLOpt.[Toplevel;Analyze;Clang;Java;Print]
     ~meta:"dir" "Write results and internal files in the specified directory"
 
 and save_results =
@@ -1242,7 +1245,7 @@ and verbose_out =
 
 and version =
   CLOpt.mk_bool ~deprecated:["version"] ~long:"version"
-    ~exes:CLOpt.[Analyze;Clang;Java;Print] "Print version information and exit"
+    ~exes:CLOpt.[Toplevel;Analyze;Clang;Java;Print] "Print version information and exit"
 
 and version_json =
   CLOpt.mk_bool ~deprecated:["version_json"] ~long:"version-json"
@@ -1309,9 +1312,6 @@ let exe_usage (exe : CLOpt.exe) =
        To process all the .specs in the current directory, pass . as only parameter \
        To process all the .specs in the results directory, use option --results-dir \
        Each spec is printed to standard output unless option -q is used."
-  | StatsAggregator ->
-      "Usage: InferStatsAggregator --results-dir <dir> --buck-out <dir>\n\
-       Aggregates all the perf stats generated by Buck on each target"
   | Toplevel ->
       version_string
 

infer/src/base/Logging.ml

@@ -24,7 +24,6 @@ let log_dir_of_current_exe (current_exe : CommandLineOption.exe) =
   | Interactive -> "interactive"
   | Java -> "java"
   | Print -> "print"
-  | StatsAggregator -> "stats_agregator"
   | Toplevel -> "top_level"