[integrations] support forcing integration in python too

Summary:
Forcing integration with `--force-integration` would only work for some
integrations that stayed in OCaml-land. This propagetes the forcing to
infer.py.

Also remove some unused options on the Python side, and add more debug
information.

Fixes #927

Reviewed By: mbouaziz

Differential Revision: D8235504

fbshipit-source-id: 1d98543

infer/lib/python/infer.py

@@ -80,7 +80,7 @@ def split_args_to_parse():
 class FailSilentlyArgumentParser(argparse.ArgumentParser):
     '''We want to leave the handling of printing usage messages to the
     OCaml code. To do so, swallow error messages from ArgumentParser
-    and exit with a special error code (101) that infer.ml looks for.
+    and exit with a special error code that infer.ml looks for.
     '''
 
     def error(self, message):
@@ -115,10 +115,29 @@ def create_argparser(parents=[]):
     return parser
 
 
+class IgnoreFailuresArgumentParser(argparse.ArgumentParser):
+    def error(self, message):
+        pass
+
+    def print_help(self, file=None):
+        pass
+
 def main():
     to_parse, cmd = split_args_to_parse()
+    # first pass to see if a capture module is forced
+    initial_argparser = IgnoreFailuresArgumentParser(
+        parents=[analyze.infer_parser],
+        add_help=False,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    initial_args = initial_argparser.parse_args(to_parse)
+
     # get the module name (if any), then load it
-    capture_module_name = os.path.basename(cmd[0]) if len(cmd) > 0 else None
+    capture_module_name = None
+    if initial_args.force_integration is not None:
+        capture_module_name = initial_args.force_integration
+    elif len(cmd) > 0:
+        capture_module_name = os.path.basename(cmd[0])
     mod_name = get_module_name(capture_module_name)
     imported_module = None
     if mod_name:

infer/lib/python/inferlib/analyze.py

@@ -26,22 +26,16 @@ base_group.add_argument('--continue', dest="continue_capture",
                         action='store_true',
                         help='''Continue the capture, do not delete previous
                         results''')
-base_group.add_argument('-r', '--reactive', action='store_true',
-                        help='''Analyze in reactive propagation mode
-                        starting from changed files.''')
-base_group.add_argument('--debug-exceptions', action='store_true',
-                        help='''Generate lightweight debugging information:
-                        just print the internal exceptions during analysis''')
 base_group.add_argument('-g', '--debug', action='store_true',
                         help='Generate all debugging information')
-base_group.add_argument('-a', '--analyzer',
-                        help='Select the analyzer within: {0}'.format(
-                            ', '.join(config.ANALYZERS)),
-                        default=config.ANALYZER_INFER)
 base_group.add_argument('-nf', '--no-filtering', action='store_true',
                         help='''Also show the results from the experimental
                         checks. Warning: some checks may contain many false
                         alarms''')
+base_group.add_argument('--force-integration', metavar='<build system>',
+                        type=utils.decode,
+                        help='Force the integration to be used regardless of \
+                        the build command that is passed')
 base_group.add_argument('--pmd-xml',
                         action='store_true',
                         help='''Output issues in (PMD) XML format.''')

infer/lib/python/inferlib/bucklib.py

@@ -240,11 +240,6 @@ class Wrapper:
 
     def __init__(self, infer_args, buck_cmd):
         self.timer = utils.Timer(logging.info)
-        # The reactive mode is not yet supported
-        if infer_args.reactive:
-            sys.stderr.write(
-                'Reactive is not supported for Java Buck project. Exiting.\n')
-            sys.exit(1)
         self.infer_args = infer_args
         self.timer.start('Computing library targets')
         base_cmd, buck_args = parse_buck_command(buck_cmd)

infer/lib/python/inferlib/capture/ant.py

@@ -32,9 +32,9 @@ class AntCapture:
     def __init__(self, args, cmd):
         self.args = args
         util.log_java_version()
-        logging.info(util.run_cmd_ignore_fail(['ant', '-version']))
+        logging.info(util.run_cmd_ignore_fail([cmd[0], '-version']))
         # TODO: make the extraction of targets smarter
-        self.build_cmd = ['ant', '-verbose'] + cmd[1:]
+        self.build_cmd = [cmd[0], '-verbose'] + cmd[1:]
 
     def is_interesting(self, content):
         return self.is_quoted(content) or content.endswith('.java')
@@ -56,7 +56,7 @@ class AntCapture:
         calls = []
         javac_arguments = []
         collect = False
-        for line in verbose_output:
+        for line in verbose_output.split('\n'):
             if javac_pattern in line:
                 if argument_start_pattern in line:
                     collect = True
@@ -77,9 +77,8 @@ class AntCapture:
         return calls
 
     def capture(self):
-        (code, verbose_out) = util.get_build_output(self.build_cmd)
+        (code, (verbose_out, _)) = util.get_build_output(self.build_cmd)
         if code != os.EX_OK:
             return code
-        clean_cmd = '\'{}\' clean'.format(self.build_cmd[0])
         cmds = self.get_infer_commands(verbose_out)
-        return util.run_compilation_commands(cmds, clean_cmd)
+        return util.run_compilation_commands(cmds)

infer/lib/python/inferlib/capture/gradle.py

@@ -175,7 +175,7 @@ class GradleCapture:
         argument_start_pattern = ' Compiler arguments: '
         calls = []
         seen_build_cmds = set([])
-        for line in verbose_output:
+        for line in verbose_output.split('\n'):
             if argument_start_pattern in line:
                 content = line.partition(argument_start_pattern)[2].strip()
                 # if we're building both the debug and release configuration
@@ -209,9 +209,9 @@ class GradleCapture:
 
     def capture(self):
         print('Running and capturing gradle compilation...')
-        (code, verbose_out) = util.get_build_output(self.build_cmd)
+        (build_code, (verbose_out, _)) = util.get_build_output(self.build_cmd)
-        if code != os.EX_OK:
-            return code
         cmds = self.get_infer_commands(verbose_out)
-        clean_cmd = '%s clean' % self.build_cmd[0]
+        capture_code = util.run_compilation_commands(cmds)
-        return util.run_compilation_commands(cmds, clean_cmd)
+        if build_code != os.EX_OK:
+            return build_code
+        return capture_code

infer/lib/python/inferlib/capture/util.py

@@ -21,25 +21,27 @@ def get_build_output(build_cmd):
     from inferlib import utils
     #  TODO make it return generator to be able to handle large builds
     proc = subprocess.Popen(build_cmd, stdout=subprocess.PIPE)
-    (verbose_out_chars, _) = proc.communicate()
+    (out_chars, err_chars) = proc.communicate()
-    if proc.returncode != 0:
+    out = utils.decode(out_chars) if out_chars is not None else ''
+    err = utils.decode(err_chars) if err_chars is not None else ''
+    if proc.returncode != os.EX_OK:
         utils.stderr(
             'ERROR: couldn\'t run compilation command `{}`'.format(build_cmd))
-        return (proc.returncode, None)
+        logging.error(
-    out = utils.decode(verbose_out_chars).split('\n')
+            'ERROR: couldn\'t run compilation command `{}`:\n\
-    return (os.EX_OK, out)
+            *** stdout:\n{}\n*** stderr:\n{}\n'
+            .format(build_cmd, out, err))
+    return (proc.returncode, (out, err))
 
 
-def run_compilation_commands(cmds, clean_cmd):
+def run_compilation_commands(cmds):
-    """runs compilation commands, and suggests a project cleaning command
+    """runs all the commands passed as argument
-    in case there is nothing to compile.
     """
-    from inferlib import utils
     #  TODO call it in parallel
     if cmds is None or len(cmds) == 0:
-        utils.stderr('Nothing to compile. Try running `{}` first.'
+        # nothing to capture, the OCaml side will detect that and
-                     .format(clean_cmd))
+        # display the appropriate warning
-        return os.EX_NOINPUT
+        return os.EX_OK
     for cmd in cmds:
         if cmd.start() != os.EX_OK:
             return os.EX_SOFTWARE

infer/src/integration/Driver.ml

@@ -150,8 +150,7 @@ let command_error_handling ~always_die ~prog ~args = function
           L.external_error
         else L.die InternalError
       in
-      log "Error running '%s' %a:@\n  %s" prog Pp.cli_args args
+      log "%a:@\n  %s" Pp.cli_args (prog :: args) (Unix.Exit_or_signal.to_string_hum status)
-        (Unix.Exit_or_signal.to_string_hum status)
 
 
 let run_command ~prog ~args ?(cleanup= command_error_handling ~always_die:false ~prog ~args) () =
@@ -218,16 +217,17 @@ let capture ~changed_files mode =
       let infer_py = Config.lib_dir ^/ "python" ^/ "infer.py" in
       let args =
         List.rev_append Config.anon_args
-          ( [ "--analyzer"
+          ( ( match Config.blacklist with
-            ; List.Assoc.find_exn ~equal:Config.equal_analyzer
-                (List.map ~f:(fun (n, a) -> (a, n)) Config.string_to_analyzer)
-                Config.analyzer ]
-          @ ( match Config.blacklist with
             | Some s when in_buck_mode ->
                 ["--blacklist-regex"; s]
             | _ ->
                 [] )
           @ (if not Config.continue_capture then [] else ["--continue"])
+          @ ( match Config.force_integration with
+            | None ->
+                []
+            | Some tool ->
+                ["--force-integration"; Config.string_of_build_system tool] )
           @ ( match Config.java_jar_compiler with
             | None ->
                 []
@@ -239,7 +239,6 @@ let capture ~changed_files mode =
             | _ ->
                 [] )
           @ (if not Config.debug_mode then [] else ["--debug"])
-          @ (if not Config.debug_exceptions then [] else ["--debug-exceptions"])
           @ (if Config.filtering then [] else ["--no-filtering"])
           @ (if not Config.flavors || not in_buck_mode then [] else ["--use-flavors"])
           @ "-j"
@@ -248,7 +247,6 @@ let capture ~changed_files mode =
           @ (if not Config.pmd_xml then [] else ["--pmd-xml"])
           @ ["--project-root"; Config.project_root]
           @ (if not Config.quiet then [] else ["--quiet"])
-          @ (if not Config.reactive_mode then [] else ["--reactive"])
           @ "--out"
             :: Config.results_dir
             ::
@@ -464,9 +462,9 @@ let mode_of_build_command build_cmd =
   | prog :: args ->
       let build_system =
         match Config.force_integration with
-        | Some build_system ->
+        | Some build_system when CLOpt.is_originator ->
             build_system
-        | None ->
+        | _ ->
             Config.build_system_of_exe_name (Filename.basename prog)
       in
       assert_supported_build_system build_system ;
@@ -492,6 +490,8 @@ let mode_of_build_command build_cmd =
           Python args
       | BXcode when Config.xcpretty ->
           XcodeXcpretty (prog, args)
+      | BBuck when not Config.flavors && Config.reactive_mode ->
+          L.die UserError "The Buck Java integration does not support --reactive@."
       | (BAnt | BBuck | BGradle | BNdk | BXcode) as build_system ->
           PythonCapture (build_system, build_cmd)
 

infer/src/istd/Pp.ml

@@ -139,7 +139,7 @@ let cli_args fmt args =
   let pp_args fmt args =
     F.fprintf fmt "@[<hov2>  " ;
     seq ~sep:"" ~print_env:text_break F.pp_print_string fmt args ;
-    F.fprintf fmt "@]@\n"
+    F.fprintf fmt "@]"
   in
   let rec pp_argfile_args in_argfiles fmt args =
     let at_least_one = ref false in
@@ -156,8 +156,9 @@ let cli_args fmt args =
       let in_argfiles' = String.Set.add in_argfiles fname in
       match In_channel.read_lines fname with
       | args ->
-          F.fprintf fmt "++Contents of %s:@\n" (Escape.escape_in_single_quotes fname) ;
+          F.fprintf fmt "++Contents of %s:@\n%a@\n"
-          pp_args fmt args ;
+            (Escape.escape_in_single_quotes fname)
+            pp_args args ;
           pp_argfile_args in_argfiles' fmt args ;
           ()
       | exception exn ->