From 3ae2b41241abf8d28fb3e38186cc04f00c8ec26b Mon Sep 17 00:00:00 2001 From: Josh Berdine Date: Fri, 4 Nov 2016 03:55:32 -0700 Subject: [PATCH] [refactor] Call StatsAggregator from Infer instead of from python Reviewed By: martinoluca Differential Revision: D4124689 fbshipit-source-id: fdc8f67 --- .gitignore | 1 - Makefile | 1 - Makefile.config.in | 1 - infer/lib/python/infer.py | 19 ------------------- infer/lib/python/inferlib/utils.py | 17 ----------------- infer/src/Makefile | 9 ++------- infer/src/backend/StatsAggregator.re | 2 +- infer/src/backend/StatsAggregator.rei | 5 +++++ infer/src/backend/infer.ml | 1 + infer/src/base/CommandLineOption.ml | 6 ++---- infer/src/base/CommandLineOption.mli | 3 +-- infer/src/base/Config.ml | 12 ++++++------ infer/src/base/Logging.ml | 1 - 13 files changed, 18 insertions(+), 60 deletions(-) diff --git a/.gitignore b/.gitignore index e58e89e0b..6be662e86 100644 --- a/.gitignore +++ b/.gitignore @@ -71,7 +71,6 @@ buck-out/ /infer/bin/InferClang++ /infer/bin/InferJava /infer/bin/InferPrint -/infer/bin/InferStatsAggregator /infer/bin/InferBuckCompilationDatabase /infer/bin/InferUnit /infer/bin/Typeprop diff --git a/Makefile b/Makefile index c7b3b07fd..338e35010 100644 --- a/Makefile +++ b/Makefile @@ -295,7 +295,6 @@ endif $(INSTALL_PROGRAM) -C $(INFER_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ $(INSTALL_PROGRAM) -C $(INFERANALYZE_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ $(INSTALL_PROGRAM) -C $(INFERPRINT_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ - $(INSTALL_PROGRAM) -C $(INFERSTATS_BIN) $(DESTDIR)$(libdir)/infer/infer/bin/ (cd $(DESTDIR)$(bindir)/ && \ $(REMOVE) infer && \ $(LN_S) $(libdir)/infer/infer/bin/infer infer) diff --git a/Makefile.config.in b/Makefile.config.in index 4974a98ee..7000fda78 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -95,7 +95,6 @@ CAPTURE_LIB_DIR = $(PYTHON_LIB_DIR)/capture INFERANALYZE_BIN = $(BIN_DIR)/InferAnalyze INFERCLANG_BIN = $(BIN_DIR)/InferClang INFERJAVA_BIN = $(BIN_DIR)/InferJava -INFERSTATS_BIN = $(BIN_DIR)/InferStatsAggregator INFERPRINT_BIN = $(BIN_DIR)/InferPrint INFERUNIT_BIN = $(BIN_DIR)/InferUnit INFER_BUCK_COMPILATION_DATABASE_BIN = $(BIN_DIR)/InferBuckCompilationDatabase diff --git a/infer/lib/python/infer.py b/infer/lib/python/infer.py index f46f7ab5a..d84e614c9 100755 --- a/infer/lib/python/infer.py +++ b/infer/lib/python/infer.py @@ -25,7 +25,6 @@ from inferlib import analyze, config, utils from inferlib.capture import make CAPTURE_PACKAGE = 'capture' -TOP_LEVEL_ENVVAR = 'TOP_LEVEL_INFER_INSTANCE_ALREADY_RUNNING' # token that identifies the end of the options for infer and the beginning # of the compilation command @@ -126,12 +125,6 @@ def create_argparser(parents=[]): def main(): - toplevel_envvar_value = os.environ.get(TOP_LEVEL_ENVVAR, None) - is_toplevel_instance = False - if toplevel_envvar_value is None: - os.environ[TOP_LEVEL_ENVVAR] = '1' - is_toplevel_instance = True - to_parse, cmd = split_args_to_parse() # get the module name (if any), then load it capture_module_name = os.path.basename(cmd[0]) if len(cmd) > 0 else None @@ -215,17 +208,5 @@ def main(): analysis.analyze_and_report() analysis.save_stats() - if is_toplevel_instance is True: - buck_out_for_stats_aggregator = None - if (mod_name == 'buck' and - os.path.isfile( - os.path.join(args.infer_out, - config.INFER_BUCK_DEPS_FILENAME))): - buck_out_for_stats_aggregator = 'buck-out' - logging.info('Aggregating stats') - output = utils.run_infer_stats_aggregator( - args.infer_out, buck_out_for_stats_aggregator) - logging.info(output) - if __name__ == '__main__': main() diff --git a/infer/lib/python/inferlib/utils.py b/infer/lib/python/inferlib/utils.py index ccb372c86..fc09714c5 100644 --- a/infer/lib/python/inferlib/utils.py +++ b/infer/lib/python/inferlib/utils.py @@ -128,23 +128,6 @@ def infer_key(analyzer): return '/'.join([analyzer, infer_version()]) -def run_infer_stats_aggregator(infer_out, buck_out=None): - buck_out_args = [] - if buck_out is not None: - buck_out_args = ['--buck-out', buck_out] - try: - cmd = get_cmd_in_bin_dir('InferStatsAggregator') - return subprocess.check_output( - [cmd, '--results-dir', infer_out] + buck_out_args, - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as exc: - output = 'Error while aggregating stats ({ec}): {output}'\ - .format(ec=exc.returncode, output=exc.output) - stderr('Info: aggregated performance data are not available in ' - 'this run') - return output - - def vcs_branch(dir='.'): cwd = os.getcwd() devnull = open(os.devnull, 'w') diff --git a/infer/src/Makefile b/infer/src/Makefile index 698dcbae1..cd4286ed8 100644 --- a/infer/src/Makefile +++ b/infer/src/Makefile @@ -130,9 +130,6 @@ SCRIPT_SOURCES = scripts CHECKCOPYRIGHT_BIN = $(SCRIPT_DIR)/checkCopyright CHECKCOPYRIGHT_MAIN = $(SCRIPT_SOURCES)/checkCopyright -STATSAGGREGATOR_BIN = $(BIN_DIR)/InferStatsAggregator -STATSAGGREGATOR_MAIN = backend/StatsAggregator - #### End of declarations #### ifeq ($(IS_FACEBOOK_TREE),yes) @@ -160,8 +157,7 @@ INFER_BASE_TARGETS = \ $(INFERANALYZE_MAIN).native \ $(INFERPRINT_MAIN).native \ $(INFERUNIT_MAIN).native \ - $(CHECKCOPYRIGHT_MAIN).native \ - $(STATSAGGREGATOR_MAIN).native + $(CHECKCOPYRIGHT_MAIN).native INFER_ALL_TARGETS = $(INFER_BASE_TARGETS) \ $(INFERJAVA_MAIN).native \ @@ -194,7 +190,6 @@ infer: init $(STACKTREE_ATDGEN_STUBS) $(INFERPRINT_ATDGEN_STUBS) $(COPY) $(INFER_BUILD_DIR)/$(INFERANALYZE_MAIN).native $(INFERANALYZE_BIN) $(COPY) $(INFER_BUILD_DIR)/$(INFERPRINT_MAIN).native $(INFERPRINT_BIN) $(COPY) $(INFER_BUILD_DIR)/$(CHECKCOPYRIGHT_MAIN).native $(CHECKCOPYRIGHT_BIN) - $(COPY) $(INFER_BUILD_DIR)/$(STATSAGGREGATOR_MAIN).native $(STATSAGGREGATOR_BIN) $(COPY) $(INFER_BUILD_DIR)/$(INFERUNIT_MAIN).native $(INFERUNIT_BIN) ifeq ($(BUILD_JAVA_ANALYZERS),yes) $(COPY) $(INFER_BUILD_DIR)/$(INFERJAVA_MAIN).native $(INFERJAVA_BIN) @@ -383,7 +378,7 @@ endif $(REMOVE) base/Version.ml.tmp.* $(REMOVE) backend/jsonbug_{j,t}.ml{,i} $(REMOVE) checkers/stacktree_{j,t}.ml{,i} - $(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(STATSAGGREGATOR_BIN) + $(REMOVE) $(INFER_BIN) $(INFERANALYZE_BIN) $(INFERPRINT_BIN) $(REMOVE) $(INFERJAVA_BIN) $(INFERCLANG_BIN) $(REMOVE) $(INFERUNIT_BIN) $(CHECKCOPYRIGHT_BIN) $(INFER_BUCK_COMPILATION_DATABASE_BIN) $(REMOVE) $(CLANG_ATDGEN_STUBS) diff --git a/infer/src/backend/StatsAggregator.re b/infer/src/backend/StatsAggregator.re index b1a3b228a..c94c50170 100644 --- a/infer/src/backend/StatsAggregator.re +++ b/infer/src/backend/StatsAggregator.re @@ -165,7 +165,7 @@ let aggregate_stats_by_target tp => { {frontend_json_data, backend_json_data, reporting_json_data} }; -let () = { +let generate_files () => { let infer_out = Config.results_dir; let stats_files = collect_all_stats_files (); let origin = diff --git a/infer/src/backend/StatsAggregator.rei b/infer/src/backend/StatsAggregator.rei index e92b0aede..ac4642b90 100644 --- a/infer/src/backend/StatsAggregator.rei +++ b/infer/src/backend/StatsAggregator.rei @@ -7,3 +7,8 @@ * of patent rights can be found in the PATENTS file in the same directory. */ open! Utils; + + +/** Generate files containing statistics aggregated from individual statistics files under + Config.results_dir */ +let generate_files: unit => unit; diff --git a/infer/src/backend/infer.ml b/infer/src/backend/infer.ml index 5a94e9634..96386ee5e 100644 --- a/infer/src/backend/infer.ml +++ b/infer/src/backend/infer.ml @@ -169,6 +169,7 @@ let analyze = function let epilogue build_mode = if Config.is_originator then ( + StatsAggregator.generate_files () ; if Config.analyzer = Some Config.Crashcontext then Crashcontext.crashcontext_epilogue ~in_buck_mode:(build_mode = Buck); if Config.fail_on_bug then diff --git a/infer/src/base/CommandLineOption.ml b/infer/src/base/CommandLineOption.ml index 6813e477e..61635fcbe 100644 --- a/infer/src/base/CommandLineOption.ml +++ b/infer/src/base/CommandLineOption.ml @@ -17,8 +17,7 @@ module YBU = Yojson.Basic.Util (** Each command line option may appear in the --help list of any executable, these tags are used to specify which executables for which an option will be documented. *) -type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator - | Toplevel +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel (** Association list of executable (base)names to their [exe]s. *) @@ -28,7 +27,6 @@ let exes = [ ("InferClang", Clang); ("InferJava", Java); ("InferPrint", Print); - ("InferStatsAggregator", StatsAggregator); ("infer", Toplevel); ("interactive", Interactive); ] @@ -576,7 +574,7 @@ let parse ?(incomplete=false) ?(accept_unknown=false) ?config_file current_exe e let exe_name = Sys.executable_name in let should_parse_cl_args = match current_exe with | Clang | Interactive -> false - | Analyze | BuckCompilationDatabase | Java | Print | StatsAggregator | Toplevel -> true in + | Analyze | BuckCompilationDatabase | Java | Print | Toplevel -> true in let env_cl_args = if should_parse_cl_args then prepend_to_argv env_args else env_args in diff --git a/infer/src/base/CommandLineOption.mli b/infer/src/base/CommandLineOption.mli index 64d8c9dde..ac07dfcec 100644 --- a/infer/src/base/CommandLineOption.mli +++ b/infer/src/base/CommandLineOption.mli @@ -11,8 +11,7 @@ open! Utils -type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | StatsAggregator - | Toplevel +type exe = Analyze | BuckCompilationDatabase | Clang | Interactive | Java | Print | Toplevel (** Association list of executable (base)names to their [exe]s. *) val exes : (string * exe) list diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index a9a4f5bc9..e38b58a5f 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -417,6 +417,9 @@ let resolve path = else path + +(** Command Line options *) + (* Declare the phase 1 options *) let inferconfig_home = @@ -605,7 +608,7 @@ and buck_build_args = and buck_out = CLOpt.mk_string_opt ~long:"buck-out" - ~exes:CLOpt.[StatsAggregator] ~meta:"dir" "Specify the root directory of buck-out" + ~exes:CLOpt.[Toplevel] ~meta:"dir" "Specify the root directory of buck-out" and bugs_csv = CLOpt.mk_string_opt ~deprecated:["bugs"] ~long:"issues-csv" @@ -1096,7 +1099,7 @@ and report_custom_error = and results_dir = CLOpt.mk_path ~deprecated:["results_dir"; "-out"] ~long:"results-dir" ~short:"o" ~default:(init_work_dir // "infer-out") - ~exes:CLOpt.[Analyze;Clang;Java;Print;StatsAggregator] + ~exes:CLOpt.[Toplevel;Analyze;Clang;Java;Print] ~meta:"dir" "Write results and internal files in the specified directory" and save_results = @@ -1242,7 +1245,7 @@ and verbose_out = and version = CLOpt.mk_bool ~deprecated:["version"] ~long:"version" - ~exes:CLOpt.[Analyze;Clang;Java;Print] "Print version information and exit" + ~exes:CLOpt.[Toplevel;Analyze;Clang;Java;Print] "Print version information and exit" and version_json = CLOpt.mk_bool ~deprecated:["version_json"] ~long:"version-json" @@ -1309,9 +1312,6 @@ let exe_usage (exe : CLOpt.exe) = To process all the .specs in the current directory, pass . as only parameter \ To process all the .specs in the results directory, use option --results-dir \ Each spec is printed to standard output unless option -q is used." - | StatsAggregator -> - "Usage: InferStatsAggregator --results-dir --buck-out \n\ - Aggregates all the perf stats generated by Buck on each target" | Toplevel -> version_string diff --git a/infer/src/base/Logging.ml b/infer/src/base/Logging.ml index 337bbc404..740ecbb37 100644 --- a/infer/src/base/Logging.ml +++ b/infer/src/base/Logging.ml @@ -24,7 +24,6 @@ let log_dir_of_current_exe (current_exe : CommandLineOption.exe) = | Interactive -> "interactive" | Java -> "java" | Print -> "print" - | StatsAggregator -> "stats_agregator" | Toplevel -> "top_level"