From a3452e51198d4bf8a30d84a9b9c394f1cfb68909 Mon Sep 17 00:00:00 2001 From: Alexander Slesarev Date: Wed, 9 Aug 2017 01:03:05 -0700 Subject: [PATCH] [infer][PR] Updated fix for the gradle capture issue regarding projects with space-containing paths Summary: Fix for the #669. Recursion problem has been reproduced (see the new doctest in the latest commit) and eliminated. I guess the new version will work faster too (no more unnecessary slicing). Closes https://github.com/facebook/infer/pull/697 Reviewed By: sblackshear Differential Revision: D5507925 Pulled By: jvillard fbshipit-source-id: 21fa07b --- infer/lib/python/inferlib/capture/gradle.py | 148 ++++++++++++++++++-- infer/lib/python/inferlib/capture/util.py | 4 +- 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/infer/lib/python/inferlib/capture/gradle.py b/infer/lib/python/inferlib/capture/gradle.py index 5aee3cc59..03a301b37 100644 --- a/infer/lib/python/inferlib/capture/gradle.py +++ b/infer/lib/python/inferlib/capture/gradle.py @@ -5,13 +5,12 @@ # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. +import itertools import logging import os import util import tempfile -from inferlib import config, jwlib, utils - MODULE_NAME = __name__ MODULE_DESCRIPTION = '''Run analysis of code built with a command like: gradle [options] [task] @@ -31,9 +30,136 @@ def gen_instance(*args): create_argparser = util.base_argparser(MODULE_DESCRIPTION, MODULE_NAME) +def extract_filepath(parts): + size = len(parts) + pos = size - 1 + while pos >= 0: + path = ' '.join(itertools.islice(parts, pos, None)) + if os.path.isfile(path): + return parts[:pos], path + pos -= 1 + return parts, None + + +def pop(the_list): + if len(the_list) > 0: + return the_list.pop() + return None + + +def extract_argfiles_from_rev(javac_arguments): + """Extract class names and @argfiles from the reversed list.""" + # Reverse the list, so it's in a natural order now + javac_arguments = list(reversed(javac_arguments)) + java_opts = [] + saved = [] + java_arg = pop(javac_arguments) + while java_arg: + if java_arg.startswith('@'): + # Probably got an @argfile + path = ' '.join([java_arg[1:]] + saved) + if os.path.isfile(path): + java_opts.insert(0, '@' + path) + saved = [] + else: + # @ at the middle of the path + saved.insert(0, java_arg) + else: + # Either a class name or a part of the @argfile path + saved.insert(0, java_arg) + java_arg = pop(javac_arguments) + + # Only class names left + java_opts[0:0] = saved + + return java_opts + + +# Please run the doctests using: +# $ python -m doctest -v gradle.py +def extract_all(javac_arguments): + """Extract Java filenames and Javac options from the Javac arguments. + + >>> os.path.isfile = lambda s: s[1:].startswith('path/to/') + >>> extract_all([]) + {'files': [], 'opts': []} + >>> extract_all(['-opt1', 'optval1', '/path/to/1.java']) + {'files': ['/path/to/1.java'], 'opts': ['-opt1', 'optval1']} + >>> extract_all(['-opt1', 'optval1', '/path/to/a', 'b/1.java']) + {'files': ['/path/to/a b/1.java'], 'opts': ['-opt1', 'optval1']} + >>> extract_all(['-opt1', 'opt', 'val1', '/path/to/1.java']) + {'files': ['/path/to/1.java'], 'opts': ['-opt1', 'opt val1']} + >>> extract_all(['-opt1', '/path/to/a', 'b/c', 'd/1.java', '-opt2']) + {'files': ['/path/to/a b/c d/1.java'], 'opts': ['-opt1', '-opt2']} + >>> extract_all(['-opt1', 'optval1', '-path/to/1.java']) + {'files': ['-path/to/1.java'], 'opts': ['-opt1', 'optval1']} + >>> extract_all(['-opt1', 'optval1', '/path/to/', '-1.java']) + {'files': ['/path/to/ -1.java'], 'opts': ['-opt1', 'optval1']} + >>> extract_all(['undef1', 'undef2']) + {'files': [], 'opts': ['undef1', 'undef2']} + >>> extract_all(['-o', '/path/to/1.java', 'cls.class', '@/path/to/1']) + {'files': ['/path/to/1.java'], 'opts': ['-o', 'cls.class', '@/path/to/1']} + >>> extract_all(['-opt1', 'optval1', '/path/to/1.java', 'cls.class']) + {'files': ['/path/to/1.java'], 'opts': ['-opt1', 'optval1', 'cls.class']} + >>> extract_all(['cls.class', '@/path/to/a', 'b.txt']) + {'files': [], 'opts': ['cls.class', '@/path/to/a b.txt']} + >>> extract_all(['cls.class', '@/path/to/a', '@b.txt']) + {'files': [], 'opts': ['cls.class', '@/path/to/a @b.txt']} + >>> v = extract_all(['-opt1', 'optval1'] * 1000 + ['/path/to/1.java']) + >>> len(v['opts']) + 2000 + """ + java_files = [] + java_opts = [] + # Reversed Javac options parameters + rev_opt_params = [] + java_arg = pop(javac_arguments) + while java_arg: + if java_arg.endswith('.java'): + # Probably got a file + remainder, path = extract_filepath(javac_arguments + [java_arg]) + if path is not None: + java_files.append(path) + javac_arguments = remainder + # The file name can't be in the middle of the option + java_opts.extend(extract_argfiles_from_rev(rev_opt_params)) + rev_opt_params = [] + else: + # A use-case here: *.java dir as an option parameter + rev_opt_params.append(java_arg) + elif java_arg.startswith('-'): + # Got a Javac option + option = [java_arg] + if len(rev_opt_params) > 0: + option.append(' '.join(reversed(rev_opt_params))) + rev_opt_params = [] + java_opts[0:0] = option + else: + # Got Javac option parameter + rev_opt_params.append(java_arg) + java_arg = pop(javac_arguments) + + # We may have class names and @argfiles besides java files and options + java_opts.extend(extract_argfiles_from_rev(rev_opt_params)) + + return {'files': java_files, 'opts': java_opts} + + +def normalize(path): + from inferlib import utils + # From Javac docs: If a filename contains embedded spaces, + # put the whole filename in double quotes + quoted_path = path + if ' ' in path: + quoted_path = '"' + path + '"' + return utils.encode(quoted_path) + + class GradleCapture: def __init__(self, args, cmd): + from inferlib import config, utils + self.args = args # TODO: make the extraction of targets smarter self.build_cmd = [cmd[0], '--debug'] + cmd[1:] @@ -46,6 +172,8 @@ class GradleCapture: logging.info('Running with:\n' + utils.decode(version_str)) def get_infer_commands(self, verbose_output): + from inferlib import config, jwlib + argument_start_pattern = ' Compiler arguments: ' calls = [] seen_build_cmds = set([]) @@ -59,14 +187,12 @@ class GradleCapture: if build_agnostic_cmd in seen_build_cmds: continue seen_build_cmds.add(build_agnostic_cmd) - javac_arguments = content.split(' ') - java_files = [] - java_args = [] - for java_arg in javac_arguments: - if java_arg.endswith('.java'): - java_files.append(java_arg) - else: - java_args.append(java_arg) + # Filter out the empty elements + arguments = list(filter(None, content.split(' '))) + extracted = extract_all(arguments) + java_files = extracted['files'] + java_args = extracted['opts'] + with tempfile.NamedTemporaryFile( mode='w', suffix='.txt', @@ -74,7 +200,7 @@ class GradleCapture: dir=os.path.join(self.args.infer_out, config.JAVAC_FILELISTS_FILENAME), delete=False) as sources: - sources.write('\n'.join(map(utils.encode, java_files))) + sources.write('\n'.join(map(normalize, java_files))) sources.flush() java_args.append('@' + sources.name) capture = jwlib.create_infer_command(java_args) diff --git a/infer/lib/python/inferlib/capture/util.py b/infer/lib/python/inferlib/capture/util.py index dd807d2cd..4ffbae911 100644 --- a/infer/lib/python/inferlib/capture/util.py +++ b/infer/lib/python/inferlib/capture/util.py @@ -18,10 +18,9 @@ import logging import subprocess import traceback -from inferlib import utils - def get_build_output(build_cmd): + from inferlib import utils # TODO make it return generator to be able to handle large builds proc = subprocess.Popen(build_cmd, stdout=subprocess.PIPE) (verbose_out_chars, _) = proc.communicate() @@ -37,6 +36,7 @@ def run_compilation_commands(cmds, clean_cmd): """runs compilation commands, and suggests a project cleaning command in case there is nothing to compile. """ + from inferlib import utils # TODO call it in parallel if cmds is None or len(cmds) == 0: utils.stderr('Nothing to compile. Try running `{}` first.'