From 4fabf0358343936370fe31c90e24061949e4e441 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Fri, 1 Jun 2018 08:47:51 -0700 Subject: [PATCH] [integrations] support forcing integration in python too Summary: Forcing integration with `--force-integration` would only work for some integrations that stayed in OCaml-land. This propagetes the forcing to infer.py. Also remove some unused options on the Python side, and add more debug information. Fixes #927 Reviewed By: mbouaziz Differential Revision: D8235504 fbshipit-source-id: 1d98543 --- infer/lib/python/infer.py | 23 ++++++++++++++++-- infer/lib/python/inferlib/analyze.py | 14 ++++------- infer/lib/python/inferlib/bucklib.py | 5 ---- infer/lib/python/inferlib/capture/ant.py | 11 ++++----- infer/lib/python/inferlib/capture/gradle.py | 12 +++++----- infer/lib/python/inferlib/capture/util.py | 26 +++++++++++---------- infer/src/integration/Driver.ml | 22 ++++++++--------- infer/src/istd/Pp.ml | 7 +++--- 8 files changed, 65 insertions(+), 55 deletions(-) diff --git a/infer/lib/python/infer.py b/infer/lib/python/infer.py index 5e6f229bd..f6bb638b3 100755 --- a/infer/lib/python/infer.py +++ b/infer/lib/python/infer.py @@ -80,7 +80,7 @@ def split_args_to_parse(): class FailSilentlyArgumentParser(argparse.ArgumentParser): '''We want to leave the handling of printing usage messages to the OCaml code. To do so, swallow error messages from ArgumentParser - and exit with a special error code (101) that infer.ml looks for. + and exit with a special error code that infer.ml looks for. ''' def error(self, message): @@ -115,10 +115,29 @@ def create_argparser(parents=[]): return parser +class IgnoreFailuresArgumentParser(argparse.ArgumentParser): + def error(self, message): + pass + + def print_help(self, file=None): + pass + def main(): to_parse, cmd = split_args_to_parse() + # first pass to see if a capture module is forced + initial_argparser = IgnoreFailuresArgumentParser( + parents=[analyze.infer_parser], + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + initial_args = initial_argparser.parse_args(to_parse) + # get the module name (if any), then load it - capture_module_name = os.path.basename(cmd[0]) if len(cmd) > 0 else None + capture_module_name = None + if initial_args.force_integration is not None: + capture_module_name = initial_args.force_integration + elif len(cmd) > 0: + capture_module_name = os.path.basename(cmd[0]) mod_name = get_module_name(capture_module_name) imported_module = None if mod_name: diff --git a/infer/lib/python/inferlib/analyze.py b/infer/lib/python/inferlib/analyze.py index f0e5c3453..74497f18e 100644 --- a/infer/lib/python/inferlib/analyze.py +++ b/infer/lib/python/inferlib/analyze.py @@ -26,22 +26,16 @@ base_group.add_argument('--continue', dest="continue_capture", action='store_true', help='''Continue the capture, do not delete previous results''') -base_group.add_argument('-r', '--reactive', action='store_true', - help='''Analyze in reactive propagation mode - starting from changed files.''') -base_group.add_argument('--debug-exceptions', action='store_true', - help='''Generate lightweight debugging information: - just print the internal exceptions during analysis''') base_group.add_argument('-g', '--debug', action='store_true', help='Generate all debugging information') -base_group.add_argument('-a', '--analyzer', - help='Select the analyzer within: {0}'.format( - ', '.join(config.ANALYZERS)), - default=config.ANALYZER_INFER) base_group.add_argument('-nf', '--no-filtering', action='store_true', help='''Also show the results from the experimental checks. Warning: some checks may contain many false alarms''') +base_group.add_argument('--force-integration', metavar='', + type=utils.decode, + help='Force the integration to be used regardless of \ + the build command that is passed') base_group.add_argument('--pmd-xml', action='store_true', help='''Output issues in (PMD) XML format.''') diff --git a/infer/lib/python/inferlib/bucklib.py b/infer/lib/python/inferlib/bucklib.py index 224ffddec..9f58f6ba6 100644 --- a/infer/lib/python/inferlib/bucklib.py +++ b/infer/lib/python/inferlib/bucklib.py @@ -240,11 +240,6 @@ class Wrapper: def __init__(self, infer_args, buck_cmd): self.timer = utils.Timer(logging.info) - # The reactive mode is not yet supported - if infer_args.reactive: - sys.stderr.write( - 'Reactive is not supported for Java Buck project. Exiting.\n') - sys.exit(1) self.infer_args = infer_args self.timer.start('Computing library targets') base_cmd, buck_args = parse_buck_command(buck_cmd) diff --git a/infer/lib/python/inferlib/capture/ant.py b/infer/lib/python/inferlib/capture/ant.py index 54b239617..5096422fe 100644 --- a/infer/lib/python/inferlib/capture/ant.py +++ b/infer/lib/python/inferlib/capture/ant.py @@ -32,9 +32,9 @@ class AntCapture: def __init__(self, args, cmd): self.args = args util.log_java_version() - logging.info(util.run_cmd_ignore_fail(['ant', '-version'])) + logging.info(util.run_cmd_ignore_fail([cmd[0], '-version'])) # TODO: make the extraction of targets smarter - self.build_cmd = ['ant', '-verbose'] + cmd[1:] + self.build_cmd = [cmd[0], '-verbose'] + cmd[1:] def is_interesting(self, content): return self.is_quoted(content) or content.endswith('.java') @@ -56,7 +56,7 @@ class AntCapture: calls = [] javac_arguments = [] collect = False - for line in verbose_output: + for line in verbose_output.split('\n'): if javac_pattern in line: if argument_start_pattern in line: collect = True @@ -77,9 +77,8 @@ class AntCapture: return calls def capture(self): - (code, verbose_out) = util.get_build_output(self.build_cmd) + (code, (verbose_out, _)) = util.get_build_output(self.build_cmd) if code != os.EX_OK: return code - clean_cmd = '\'{}\' clean'.format(self.build_cmd[0]) cmds = self.get_infer_commands(verbose_out) - return util.run_compilation_commands(cmds, clean_cmd) + return util.run_compilation_commands(cmds) diff --git a/infer/lib/python/inferlib/capture/gradle.py b/infer/lib/python/inferlib/capture/gradle.py index 2b7101913..95a958bdc 100644 --- a/infer/lib/python/inferlib/capture/gradle.py +++ b/infer/lib/python/inferlib/capture/gradle.py @@ -175,7 +175,7 @@ class GradleCapture: argument_start_pattern = ' Compiler arguments: ' calls = [] seen_build_cmds = set([]) - for line in verbose_output: + for line in verbose_output.split('\n'): if argument_start_pattern in line: content = line.partition(argument_start_pattern)[2].strip() # if we're building both the debug and release configuration @@ -209,9 +209,9 @@ class GradleCapture: def capture(self): print('Running and capturing gradle compilation...') - (code, verbose_out) = util.get_build_output(self.build_cmd) - if code != os.EX_OK: - return code + (build_code, (verbose_out, _)) = util.get_build_output(self.build_cmd) cmds = self.get_infer_commands(verbose_out) - clean_cmd = '%s clean' % self.build_cmd[0] - return util.run_compilation_commands(cmds, clean_cmd) + capture_code = util.run_compilation_commands(cmds) + if build_code != os.EX_OK: + return build_code + return capture_code diff --git a/infer/lib/python/inferlib/capture/util.py b/infer/lib/python/inferlib/capture/util.py index f440982f0..22b10b445 100644 --- a/infer/lib/python/inferlib/capture/util.py +++ b/infer/lib/python/inferlib/capture/util.py @@ -21,25 +21,27 @@ def get_build_output(build_cmd): from inferlib import utils # TODO make it return generator to be able to handle large builds proc = subprocess.Popen(build_cmd, stdout=subprocess.PIPE) - (verbose_out_chars, _) = proc.communicate() - if proc.returncode != 0: + (out_chars, err_chars) = proc.communicate() + out = utils.decode(out_chars) if out_chars is not None else '' + err = utils.decode(err_chars) if err_chars is not None else '' + if proc.returncode != os.EX_OK: utils.stderr( 'ERROR: couldn\'t run compilation command `{}`'.format(build_cmd)) - return (proc.returncode, None) - out = utils.decode(verbose_out_chars).split('\n') - return (os.EX_OK, out) + logging.error( + 'ERROR: couldn\'t run compilation command `{}`:\n\ + *** stdout:\n{}\n*** stderr:\n{}\n' + .format(build_cmd, out, err)) + return (proc.returncode, (out, err)) -def run_compilation_commands(cmds, clean_cmd): - """runs compilation commands, and suggests a project cleaning command - in case there is nothing to compile. +def run_compilation_commands(cmds): + """runs all the commands passed as argument """ - from inferlib import utils # TODO call it in parallel if cmds is None or len(cmds) == 0: - utils.stderr('Nothing to compile. Try running `{}` first.' - .format(clean_cmd)) - return os.EX_NOINPUT + # nothing to capture, the OCaml side will detect that and + # display the appropriate warning + return os.EX_OK for cmd in cmds: if cmd.start() != os.EX_OK: return os.EX_SOFTWARE diff --git a/infer/src/integration/Driver.ml b/infer/src/integration/Driver.ml index b9589cf90..a896c0f5c 100644 --- a/infer/src/integration/Driver.ml +++ b/infer/src/integration/Driver.ml @@ -150,8 +150,7 @@ let command_error_handling ~always_die ~prog ~args = function L.external_error else L.die InternalError in - log "Error running '%s' %a:@\n %s" prog Pp.cli_args args - (Unix.Exit_or_signal.to_string_hum status) + log "%a:@\n %s" Pp.cli_args (prog :: args) (Unix.Exit_or_signal.to_string_hum status) let run_command ~prog ~args ?(cleanup= command_error_handling ~always_die:false ~prog ~args) () = @@ -218,16 +217,17 @@ let capture ~changed_files mode = let infer_py = Config.lib_dir ^/ "python" ^/ "infer.py" in let args = List.rev_append Config.anon_args - ( [ "--analyzer" - ; List.Assoc.find_exn ~equal:Config.equal_analyzer - (List.map ~f:(fun (n, a) -> (a, n)) Config.string_to_analyzer) - Config.analyzer ] - @ ( match Config.blacklist with + ( ( match Config.blacklist with | Some s when in_buck_mode -> ["--blacklist-regex"; s] | _ -> [] ) @ (if not Config.continue_capture then [] else ["--continue"]) + @ ( match Config.force_integration with + | None -> + [] + | Some tool -> + ["--force-integration"; Config.string_of_build_system tool] ) @ ( match Config.java_jar_compiler with | None -> [] @@ -239,7 +239,6 @@ let capture ~changed_files mode = | _ -> [] ) @ (if not Config.debug_mode then [] else ["--debug"]) - @ (if not Config.debug_exceptions then [] else ["--debug-exceptions"]) @ (if Config.filtering then [] else ["--no-filtering"]) @ (if not Config.flavors || not in_buck_mode then [] else ["--use-flavors"]) @ "-j" @@ -248,7 +247,6 @@ let capture ~changed_files mode = @ (if not Config.pmd_xml then [] else ["--pmd-xml"]) @ ["--project-root"; Config.project_root] @ (if not Config.quiet then [] else ["--quiet"]) - @ (if not Config.reactive_mode then [] else ["--reactive"]) @ "--out" :: Config.results_dir :: @@ -464,9 +462,9 @@ let mode_of_build_command build_cmd = | prog :: args -> let build_system = match Config.force_integration with - | Some build_system -> + | Some build_system when CLOpt.is_originator -> build_system - | None -> + | _ -> Config.build_system_of_exe_name (Filename.basename prog) in assert_supported_build_system build_system ; @@ -492,6 +490,8 @@ let mode_of_build_command build_cmd = Python args | BXcode when Config.xcpretty -> XcodeXcpretty (prog, args) + | BBuck when not Config.flavors && Config.reactive_mode -> + L.die UserError "The Buck Java integration does not support --reactive@." | (BAnt | BBuck | BGradle | BNdk | BXcode) as build_system -> PythonCapture (build_system, build_cmd) diff --git a/infer/src/istd/Pp.ml b/infer/src/istd/Pp.ml index 77967c93f..bf1c198ba 100644 --- a/infer/src/istd/Pp.ml +++ b/infer/src/istd/Pp.ml @@ -139,7 +139,7 @@ let cli_args fmt args = let pp_args fmt args = F.fprintf fmt "@[ " ; seq ~sep:"" ~print_env:text_break F.pp_print_string fmt args ; - F.fprintf fmt "@]@\n" + F.fprintf fmt "@]" in let rec pp_argfile_args in_argfiles fmt args = let at_least_one = ref false in @@ -156,8 +156,9 @@ let cli_args fmt args = let in_argfiles' = String.Set.add in_argfiles fname in match In_channel.read_lines fname with | args -> - F.fprintf fmt "++Contents of %s:@\n" (Escape.escape_in_single_quotes fname) ; - pp_args fmt args ; + F.fprintf fmt "++Contents of %s:@\n%a@\n" + (Escape.escape_in_single_quotes fname) + pp_args args ; pp_argfile_args in_argfiles' fmt args ; () | exception exn ->