From d4521fd70d9cb9545943f5ce2d84cd410fe8b0e8 Mon Sep 17 00:00:00 2001 From: Jules Villard Date: Thu, 5 May 2016 06:54:37 -0700 Subject: [PATCH] handle utf8 PWD Summary: - [python] decode strings coming from `os.*` commands - [python] decode strings coming from the command-line - [python] encode a few remaining unicodes into strings - [java] replace lex/yacc parser for javac verbose output by regex-based matching to handle unicode in paths - [make] random fix of `make test` to have `make clean test` work - [integration tests] add e2e build integration tests for utf8 in the PWD Closes #76 Reviewed By: martinoluca Differential Revision: D3240809 fb-gh-sync-id: 8c2e1ed fbshipit-source-id: 8c2e1ed --- Makefile.in | 2 +- infer/lib/python/infer | 34 +++-- infer/lib/python/inferlib/analyze.py | 22 ++-- infer/lib/python/inferlib/bucklib.py | 8 +- infer/lib/python/inferlib/capture/gradle.py | 12 +- infer/lib/python/inferlib/capture/make.py | 9 +- infer/lib/python/inferlib/capture/util.py | 2 +- .../lib/python/inferlib/capture/xcodebuild.py | 6 +- infer/lib/python/inferlib/jwlib.py | 10 +- infer/lib/python/inferlib/utils.py | 30 ++++- infer/src/java/jClasspath.ml | 45 ++++--- infer/src/java/jVerboseLexer.mll | 61 --------- infer/src/java/jVerboseParser.mly | 52 -------- .../build_systems/build_integration_tests.py | 117 +++++++++++++++--- .../codetoanalyze/utf8_in_pwd/Hello.java} | 19 ++- .../codetoanalyze/utf8_in_pwd/cmake | 1 + .../codetoanalyze/utf8_in_pwd/gradle | 1 + .../codetoanalyze/utf8_in_pwd/make | 1 + .../expected_outputs/cmake_report.json | 2 +- .../expected_outputs/javac_report.json | 7 ++ .../utf8_in_pwd_cmake_report.json | 7 ++ .../utf8_in_pwd_gradle_report.json | 17 +++ .../utf8_in_pwd_javac_report.json | 7 ++ .../utf8_in_pwd_make_report.json | 7 ++ 24 files changed, 280 insertions(+), 199 deletions(-) delete mode 100644 infer/src/java/jVerboseLexer.mll delete mode 100644 infer/src/java/jVerboseParser.mly rename infer/{src/java/jVerbose.ml => tests/build_systems/codetoanalyze/utf8_in_pwd/Hello.java} (56%) create mode 120000 infer/tests/build_systems/codetoanalyze/utf8_in_pwd/cmake create mode 120000 infer/tests/build_systems/codetoanalyze/utf8_in_pwd/gradle create mode 120000 infer/tests/build_systems/codetoanalyze/utf8_in_pwd/make create mode 100644 infer/tests/build_systems/expected_outputs/javac_report.json create mode 100644 infer/tests/build_systems/expected_outputs/utf8_in_pwd_cmake_report.json create mode 100644 infer/tests/build_systems/expected_outputs/utf8_in_pwd_gradle_report.json create mode 100644 infer/tests/build_systems/expected_outputs/utf8_in_pwd_javac_report.json create mode 100644 infer/tests/build_systems/expected_outputs/utf8_in_pwd_make_report.json diff --git a/Makefile.in b/Makefile.in index f0c7a0ce4..8241ac805 100644 --- a/Makefile.in +++ b/Makefile.in @@ -147,7 +147,7 @@ endif @rm -fr __test-infer-out__ test: - $(MAKE) -C $(INFER_DIR) $(INFER_BIN) + $(MAKE) -C $(INFER_DIR) $(INFER_BIN_RELPATH) $(MAKE) -C $(SRC_DIR) init $(MAKE) -j$(NCPU) test_build $(MAKE) -C $(SRC_DIR) mod_dep.dot diff --git a/infer/lib/python/infer b/infer/lib/python/infer index 6eb3a7161..9b416feed 100755 --- a/infer/lib/python/infer +++ b/infer/lib/python/infer @@ -69,11 +69,11 @@ def load_module(mod_name): def split_args_to_parse(): + sys_argv = map(utils.decode, sys.argv) dd_index = \ - sys.argv.index(CMD_MARKER) if CMD_MARKER in sys.argv else len(sys.argv) - cmd_raw = sys.argv[dd_index + 1:] - return (map(utils.decode, sys.argv[1:dd_index]), - map(utils.decode, cmd_raw)) + sys_argv.index(CMD_MARKER) if CMD_MARKER in sys_argv else len(sys_argv) + cmd_raw = sys_argv[dd_index + 1:] + return (sys_argv[1:dd_index], cmd_raw) def create_argparser(parents=[]): @@ -131,18 +131,28 @@ def main(): touch_if_present=not args.continue_capture) utils.configure_logging(args) - logging.info('Running command %s', ' '.join(sys.argv)) - logging.info('Path to infer script %s (%s)', __file__, - os.path.realpath(__file__)) - logging.info(analyze.get_infer_version()) - logging.info('Platform: %s', platform.platform()) - logging.info('PATH=%s', os.getenv('PATH')) - logging.info('SHELL=%s', os.getenv('SHELL')) - logging.info('PWD=%s', os.getenv('PWD')) logging.info('output of locale.getdefaultlocale(): %s', str(locale.getdefaultlocale())) logging.info('encoding we chose in the end: %s', config.CODESET) + logging.info('Running command %s', + ' '.join(map(utils.decode, sys.argv))) + logging.info('Path to infer script %s (%s)', utils.decode(__file__), + os.path.realpath(utils.decode(__file__))) + logging.info(analyze.get_infer_version()) + logging.info('Platform: %s', utils.decode(platform.platform())) + + def log_getenv(k): + v = os.getenv(k) + if v is not None: + v = utils.decode(v) + else: + v = '' + logging.info('%s=%s', k, v) + + log_getenv('PATH') + log_getenv('SHELL') + log_getenv('PWD') capture_exitcode = imported_module.gen_instance(args, cmd).capture() if capture_exitcode != os.EX_OK: diff --git a/infer/lib/python/inferlib/analyze.py b/infer/lib/python/inferlib/analyze.py index bd2f177c2..f2a606f73 100644 --- a/infer/lib/python/inferlib/analyze.py +++ b/infer/lib/python/inferlib/analyze.py @@ -56,7 +56,9 @@ class VersionAction(argparse._VersionAction): base_parser = argparse.ArgumentParser(add_help=False) base_group = base_parser.add_argument_group('global arguments') base_group.add_argument('-o', '--out', metavar='', - default=config.DEFAULT_INFER_OUT, dest='infer_out', + default=utils.encode(config.DEFAULT_INFER_OUT), + dest='infer_out', + type=utils.decode, action=utils.AbsolutePathAction, help='Set the Infer results directory') base_group.add_argument('-i', '--incremental', @@ -129,10 +131,11 @@ infer_group.add_argument('--infer_cache', metavar='', help='Select a directory to contain the infer cache') infer_group.add_argument('-pr', '--project_root', - dest='project_root', - default=os.getcwd(), - help='Location of the project root ' - '(default is current directory)') + dest='project_root', + default=os.getcwd(), + type=utils.decode, + help='Location of the project root ' + '(default is current directory)') infer_group.add_argument('--absolute-paths', action='store_true', @@ -280,7 +283,7 @@ class AnalyzerWrapper(object): # to be reported infer_options += ['-allow_specs_cleanup'] - infer_options += ['-inferconfig_home', os.getcwd()] + infer_options += ['-inferconfig_home', utils.decode(os.getcwd())] if self.args.analyzer == config.ANALYZER_ERADICATE: infer_options += ['-eradicate'] @@ -336,7 +339,8 @@ class AnalyzerWrapper(object): exit_status = os.EX_OK if self.javac is not None and self.args.buck: - infer_options += ['-project_root', os.getcwd(), '-java'] + infer_options += ['-project_root', utils.decode(os.getcwd()), + '-java'] if self.javac.args.classpath is not None: for path in self.javac.args.classpath.split(os.pathsep): if os.path.isfile(path): @@ -348,7 +352,9 @@ class AnalyzerWrapper(object): config.ANALYZER_TRACING]: os.environ['INFER_ONDEMAND'] = 'Y' - os.environ['INFER_OPTIONS'] = ' '.join(infer_options) + infer_options = map(utils.decode_or_not, infer_options) + infer_options_str = ' '.join(infer_options) + os.environ['INFER_OPTIONS'] = utils.encode(infer_options_str) javac_original_arguments = \ self.javac.original_arguments if self.javac is not None else [] diff --git a/infer/lib/python/inferlib/bucklib.py b/infer/lib/python/inferlib/bucklib.py index bc43bc46a..3323ee644 100644 --- a/infer/lib/python/inferlib/bucklib.py +++ b/infer/lib/python/inferlib/bucklib.py @@ -35,7 +35,7 @@ from inferlib import analyze, config, issues, utils ANALYSIS_SUMMARY_OUTPUT = 'analysis_summary.txt' -DEFAULT_BUCK_OUT = os.path.join(os.getcwd(), 'buck-out') +DEFAULT_BUCK_OUT = os.path.join(utils.decode(os.getcwd()), 'buck-out') DEFAULT_BUCK_OUT_GEN = os.path.join(DEFAULT_BUCK_OUT, 'gen') INFER_CSV_REPORT = os.path.join(config.BUCK_INFER_OUT, @@ -103,7 +103,7 @@ def prepare_build(args): dir='.') as infer_script: logging.info('Creating %s' % infer_script.name) infer_script.file.write( - INFER_SCRIPT.format(sys.executable, infer).encode()) + utils.encode(INFER_SCRIPT.format(sys.executable, infer))) st = os.stat(infer_script.name) os.chmod(infer_script.name, st.st_mode | stat.S_IEXEC) @@ -148,7 +148,7 @@ def init_stats(args, start_time): 'analyzer': args.analyzer, 'machine': platform.machine(), 'node': platform.node(), - 'project': os.path.basename(os.getcwd()), + 'project': utils.decode(os.path.basename(os.getcwd())), 'revision': utils.vcs_revision(), 'branch': utils.vcs_branch(), 'system': platform.system(), @@ -415,7 +415,7 @@ def cleanup(temp_files): parser = argparse.ArgumentParser() -parser.add_argument('--build-report', metavar='PATH', type=str) +parser.add_argument('--build-report', metavar='PATH', type=utils.decode) parser.add_argument('--deep', action='store_true') parser.add_argument('--keep-going', action='store_true') parser.add_argument('--load-limit', '-L') diff --git a/infer/lib/python/inferlib/capture/gradle.py b/infer/lib/python/inferlib/capture/gradle.py index fa8a34c7f..d3a3e40d8 100644 --- a/infer/lib/python/inferlib/capture/gradle.py +++ b/infer/lib/python/inferlib/capture/gradle.py @@ -10,8 +10,7 @@ import os import util import tempfile -from inferlib import jwlib -from inferlib import config +from inferlib import config, jwlib, utils MODULE_NAME = __name__ MODULE_DESCRIPTION = '''Run analysis of code built with a command like: @@ -43,7 +42,7 @@ class GradleCapture: config.JAVAC_FILELISTS_FILENAME) if not os.path.exists(path): os.mkdir(path) - logging.info('Running with:\n' + version_str) + logging.info('Running with:\n' + utils.decode(version_str)) def get_infer_commands(self, verbose_output): argument_start_pattern = ' Compiler arguments: ' @@ -74,12 +73,11 @@ class GradleCapture: dir=os.path.join(self.args.infer_out, config.JAVAC_FILELISTS_FILENAME), delete=False) as sources: - sources.write('\n'.join(java_files)) + sources.write('\n'.join(map(utils.encode, java_files))) sources.flush() java_args.append('@' + sources.name) - capture = jwlib.create_infer_command(self.args, - java_args) - calls.append(capture) + capture = jwlib.create_infer_command(self.args, java_args) + calls.append(capture) return calls def capture(self): diff --git a/infer/lib/python/inferlib/capture/make.py b/infer/lib/python/inferlib/capture/make.py index 8178a7b53..411281a3a 100644 --- a/infer/lib/python/inferlib/capture/make.py +++ b/infer/lib/python/inferlib/capture/make.py @@ -48,7 +48,7 @@ class MakeCapture: cmd[0] = command_name def get_envvars(self): - env_vars = dict(os.environ) + env_vars = utils.read_env() wrappers_path = config.WRAPPERS_DIRECTORY env_vars['INFER_OLD_PATH'] = env_vars['PATH'] env_vars['PATH'] = '{wrappers}{sep}{path}'.format( @@ -63,9 +63,10 @@ class MakeCapture: def capture(self): try: - env = self.get_envvars() - logging.info('Running command %s with env:\n%s' % (self.cmd, env)) - subprocess.check_call(self.cmd, env=env) + env = utils.encode_env(self.get_envvars()) + cmd = map(utils.encode, self.cmd) + logging.info('Running command %s with env:\n%s' % (cmd, env)) + subprocess.check_call(cmd, env=env) capture_dir = os.path.join(self.args.infer_out, 'captured') if len(os.listdir(capture_dir)) < 1: # Don't return with a failure code unless we're diff --git a/infer/lib/python/inferlib/capture/util.py b/infer/lib/python/inferlib/capture/util.py index e051388fc..e2614edf1 100644 --- a/infer/lib/python/inferlib/capture/util.py +++ b/infer/lib/python/inferlib/capture/util.py @@ -24,7 +24,7 @@ def get_build_output(build_cmd): # TODO make it return generator to be able to handle large builds proc = subprocess.Popen(build_cmd, stdout=subprocess.PIPE) (verbose_out_chars, _) = proc.communicate() - return verbose_out_chars.split('\n') + return utils.decode(verbose_out_chars).split('\n') def run_compilation_commands(cmds, clean_cmd): diff --git a/infer/lib/python/inferlib/capture/xcodebuild.py b/infer/lib/python/inferlib/capture/xcodebuild.py index 4f510f93d..cdeda76cc 100644 --- a/infer/lib/python/inferlib/capture/xcodebuild.py +++ b/infer/lib/python/inferlib/capture/xcodebuild.py @@ -50,7 +50,7 @@ class XcodebuildCapture: self.cmd = cmd def get_envvars(self): - env_vars = dict(os.environ) + env_vars = utils.read_env() env_vars['FCP_APPLE_CLANG'] = self.apple_clang_path @@ -70,7 +70,9 @@ class XcodebuildCapture: self.cmd += ['GCC_PRECOMPILE_PREFIX_HEADER=NO'] try: - subprocess.check_call(self.cmd, env=self.get_envvars()) + env = utils.encode_env(self.get_envvars()) + cmd = map(utils.encode, self.cmd) + subprocess.check_call(cmd, env=env) return os.EX_OK except subprocess.CalledProcessError as exc: if self.args.debug: diff --git a/infer/lib/python/inferlib/jwlib.py b/infer/lib/python/inferlib/jwlib.py index bb9333687..ac92f5bf8 100644 --- a/infer/lib/python/inferlib/jwlib.py +++ b/infer/lib/python/inferlib/jwlib.py @@ -22,16 +22,16 @@ from . import analyze, config, utils # javac options parser = argparse.ArgumentParser() -current_directory = os.getcwd() +current_directory = utils.decode(os.getcwd()) parser.add_argument('-version', action='store_true') parser.add_argument('-deprecation', action='store_true') -parser.add_argument('-cp', '-classpath', type=str, +parser.add_argument('-cp', '-classpath', type=utils.decode, dest='classpath', default=os.getcwd()) -parser.add_argument('-bootclasspath', type=str) +parser.add_argument('-bootclasspath', type=utils.decode) parser.add_argument('-d', dest='classes_out', default=current_directory) -parser.add_argument('-processorpath', type=str, dest='processorpath') -parser.add_argument('-processor', type=str, dest='processor') +parser.add_argument('-processorpath', type=utils.decode, dest='processorpath') +parser.add_argument('-processor', type=utils.decode, dest='processor') def _get_javac_args(args): diff --git a/infer/lib/python/inferlib/utils.py b/infer/lib/python/inferlib/utils.py index ea384a62e..7412aa855 100644 --- a/infer/lib/python/inferlib/utils.py +++ b/infer/lib/python/inferlib/utils.py @@ -295,6 +295,20 @@ def encode(u, errors='replace'): return u.encode(encoding=config.CODESET, errors=errors) +def decode_or_not(s, errors='replace'): + try: + return decode(s, errors) + except UnicodeEncodeError: + return s + + +def encode_or_not(u, errors='replace'): + try: + return encode(u, errors) + except UnicodeDecodeError: + return u + + def stdout(s, errors='replace'): print(encode(s, errors=errors)) @@ -312,7 +326,21 @@ def merge_and_dedup_files_into_path(files_to_merge, dest): fdest.writelines(lines) +def read_env(): + env = dict(os.environ).copy() + for k, v in env.iteritems(): + env[k] = decode(v) + return env + + +def encode_env(env): + new_env = env.copy() + for k, v in new_env.iteritems(): + new_env[k] = encode(v) + return new_env + + class AbsolutePathAction(argparse.Action): """Convert a path from relative to absolute in the arg parser""" def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, os.path.abspath(values)) + setattr(namespace, self.dest, encode(os.path.abspath(values))) diff --git a/infer/src/java/jClasspath.ml b/infer/src/java/jClasspath.ml index e19e1f283..81b337f93 100644 --- a/infer/src/java/jClasspath.ml +++ b/infer/src/java/jClasspath.ml @@ -162,26 +162,39 @@ let add_source_file path map = let load_sources_and_classes () = let file_in = open_in !javac_verbose_out in + let class_filename_re = + Str.regexp + "\\[wrote RegularFileObject\\[\\(.*\\)\\]\\]" in + let source_filename_re = + Str.regexp + "\\[parsing started RegularFileObject\\[\\(.*\\)\\]\\]" in + let classpath_re = + Str.regexp + "\\[search path for class files: \\(.*\\)\\]" in let rec loop paths roots sources classes = try - let lexbuf = Lexing.from_string (input_line file_in) in - match JVerboseParser.line JVerboseLexer.token lexbuf with - | JVerbose.Source path -> - loop paths roots (add_source_file path sources) classes - | JVerbose.Class path -> - let cn, root_info = Javalib.extract_class_name_from_file path in - let root_dir = if root_info = "" then Filename.current_dir_name else root_info in - let updated_roots = - if IList.exists (fun p -> p = root_dir) roots then roots - else root_dir:: roots in - loop paths updated_roots sources (JBasics.ClassSet.add cn classes) - | JVerbose.Classpath parsed_paths -> - loop parsed_paths roots sources classes + let line = input_line file_in in + if Str.string_match class_filename_re line 0 then + let path = Str.matched_group 1 line in + let cn, root_info = Javalib.extract_class_name_from_file path in + let root_dir = if root_info = "" then Filename.current_dir_name else root_info in + let updated_roots = + if IList.exists (fun p -> p = root_dir) roots then roots + else root_dir:: roots in + loop paths updated_roots sources (JBasics.ClassSet.add cn classes) + else if Str.string_match source_filename_re line 0 then + let path = Str.matched_group 1 line in + loop paths roots (add_source_file path sources) classes + else if Str.string_match classpath_re line 0 then + let classpath = Str.matched_group 1 line in + let parsed_paths = Str.split (Str.regexp_string ",") classpath in + loop parsed_paths roots sources classes + else + (* skip this line *) + loop paths roots sources classes with | JBasics.Class_structure_error _ - | Parsing.Parse_error - | Invalid_argument _ - | Failure "lexing: empty token" -> loop paths roots sources classes + | Invalid_argument _ -> loop paths roots sources classes | End_of_file -> close_in file_in; let classpath = IList.fold_left append_path "" (roots @ (add_android_jar paths)) in diff --git a/infer/src/java/jVerboseLexer.mll b/infer/src/java/jVerboseLexer.mll deleted file mode 100644 index 4f1fe16c2..000000000 --- a/infer/src/java/jVerboseLexer.mll +++ /dev/null @@ -1,61 +0,0 @@ -(* - * Copyright (c) 2015 - present Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - *) - -{ - open JVerboseParser -} - -let space = [' ' '\t'] - -let lowerletter = ['a'-'z'] -let upperletter = ['A'-'Z'] -let letter = lowerletter | upperletter - -let underscore = '_' -let minus = '-' -let dot = '.' -let hash = '#' -let dollar = '$' -let equal = '=' -let tilde = '~' - -let digit = ['0'-'9'] -let number = digit* | digit+ '.' digit* | digit* '.' digit+ -let char = letter | digit - -let dir_sep = '/' - -let dot_java = dot "java" -let dot_class = dot "class" -let path = (char | dir_sep | underscore | minus | dot | hash | dollar | equal | tilde)+ -let source_basename = path dot_java -let class_basename = path dot_class -let source_filename = (dir_sep path | path)* source_basename -let class_filename = (dir_sep path | path)* class_basename - -let search_path = "search path for class files" - -rule token = parse -| [' ' '\t'] { token lexbuf } (* skip blanks *) -| ['\n'] { EOL } -| '[' { LEFT_SQUARE_BRACKET } -| ']' { RIGHT_SQUARE_BRACKET } -| ':' { COLON } -| ',' { COMMA } -| "\'\'" { EMPTY } -| "parsing" { PARSING } -| "started" { STARTED } -| "wrote" { WROTE } -| search_path { SEARCH_PATH } -| "RegularFileObject" { REGULARFILEOBJECT } -| "ZipFileIndexFileObject" { ZIPFILEINDEXFILEOBJECT } -| source_filename as p { SOURCE_FILENAME p } -| class_filename as p { CLASS_FILENAME p } -| path as p { PATH p } -| eof { EOF } diff --git a/infer/src/java/jVerboseParser.mly b/infer/src/java/jVerboseParser.mly deleted file mode 100644 index d25e7d9b7..000000000 --- a/infer/src/java/jVerboseParser.mly +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2009 - 2013 Monoidics ltd. - * Copyright (c) 2013 - present Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ - -%{ - -%} - -%token EOL EOF -%token LEFT_SQUARE_BRACKET RIGHT_SQUARE_BRACKET -%token COLON COMMA -%token EMPTY -%token PARSING STARTED WROTE SEARCH_PATH -%token REGULARFILEOBJECT ZIPFILEINDEXFILEOBJECT -%token CLASS_FILENAME -%token SOURCE_FILENAME -%token PATH -%start line -%type line -%% - - -line: - | source_filename { $1 } - | class_filename { $1 } - | classpath { $1 } -; - -source_filename: - LEFT_SQUARE_BRACKET PARSING STARTED REGULARFILEOBJECT LEFT_SQUARE_BRACKET SOURCE_FILENAME RIGHT_SQUARE_BRACKET RIGHT_SQUARE_BRACKET { JVerbose.Source $6 } -; - -class_filename: - LEFT_SQUARE_BRACKET WROTE REGULARFILEOBJECT LEFT_SQUARE_BRACKET CLASS_FILENAME RIGHT_SQUARE_BRACKET RIGHT_SQUARE_BRACKET { JVerbose.Class $5 } -; - -classpath: - LEFT_SQUARE_BRACKET SEARCH_PATH COLON classpath_parts RIGHT_SQUARE_BRACKET { JVerbose.Classpath $4 } -; - -classpath_parts: - | EMPTY { [] } - | PATH { [$1] } - | EMPTY COMMA classpath_parts { $3 } - | PATH COMMA classpath_parts { $1 :: $3 } -; diff --git a/infer/tests/build_systems/build_integration_tests.py b/infer/tests/build_systems/build_integration_tests.py index 5fef4162f..c0cdea941 100755 --- a/infer/tests/build_systems/build_integration_tests.py +++ b/infer/tests/build_systems/build_integration_tests.py @@ -1,4 +1,5 @@ #!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- # Copyright (c) 2015 - present Facebook, Inc. # All rights reserved. # @@ -34,7 +35,7 @@ sys.path.insert(0, os.path.join(SCRIPT_DIR, os.pardir, os.pardir, 'lib', 'python')) -from inferlib import issues, utils +from inferlib import config, issues, utils ROOT_DIR = os.path.join(SCRIPT_DIR, os.pardir, os.pardir, os.pardir) @@ -54,7 +55,17 @@ REPORT_FIELDS = [ CODETOANALYZE_DIR = os.path.join(SCRIPT_DIR, 'codetoanalyze') EXPECTED_OUTPUTS_DIR = os.path.join(SCRIPT_DIR, 'expected_outputs') -ALL_TESTS = ['ant', 'buck', 'gradle', 'make', 'locale', 'waf'] +ALL_TESTS = [ + 'ant', + 'buck', + 'cmake', + 'gradle', + 'javac', + 'locale', + 'make', + 'utf8_in_pwd', + 'waf', +] to_test = ALL_TESTS @@ -188,6 +199,13 @@ def do_test(errors, expected_errors_filename): check_results(errors, patterns) +def make_paths_relative_in_report(root, errors): + for error in errors: + # remove "root/" from each file name + rel_fname = error[issues.JSON_INDEX_FILENAME][len(root) + 1:] + error[issues.JSON_INDEX_FILENAME] = rel_fname + + class BuildIntegrationTest(unittest.TestCase): def test_ant_integration(self): @@ -205,14 +223,38 @@ class BuildIntegrationTest(unittest.TestCase): original = os.path.join(EXPECTED_OUTPUTS_DIR, 'ant_report.json') do_test(errors, original) + def test_javac_integration( + self, + enabled=None, + root=os.path.join(ROOT_DIR, 'examples'), + report_name='javac_report.json'): + if enabled is None: + enabled = 'javac' in to_test + if not enabled: + print('\nSkipping javac integration test') + return - def test_gradle_integration(self): - if 'gradle' not in to_test: + print('\nRunning javac integration test') + errors = run_analysis( + root, + [], + [['javac', 'Hello.java']], + INFER_EXECUTABLE) + original = os.path.join(EXPECTED_OUTPUTS_DIR, report_name) + do_test(errors, original) + + def test_gradle_integration( + self, + enabled=None, + root=os.path.join(ROOT_DIR, 'examples', 'java_hello'), + report_name='gradle_report.json'): + if enabled is None: + enabled = 'gradle' in to_test + if not enabled: print('\nSkipping Gradle integration test') return print('\nRunning Gradle integration test using mock gradle') - root = os.path.join(ROOT_DIR, 'examples', 'java_hello') env = os.environ env['PATH'] = '{}:{}'.format( os.path.join(SCRIPT_DIR, 'mock'), @@ -224,7 +266,7 @@ class BuildIntegrationTest(unittest.TestCase): [['gradle', 'build']], INFER_EXECUTABLE, env=env) - original = os.path.join(EXPECTED_OUTPUTS_DIR, 'gradle_report.json') + original = os.path.join(EXPECTED_OUTPUTS_DIR, report_name) do_test(errors, original) def test_buck_integration(self): @@ -242,19 +284,24 @@ class BuildIntegrationTest(unittest.TestCase): original = os.path.join(EXPECTED_OUTPUTS_DIR, 'buck_report.json') do_test(errors, original) - def test_make_integration(self): - if 'make' not in to_test: + def test_make_integration( + self, + enabled=None, + root=os.path.join(CODETOANALYZE_DIR, 'make'), + report_name='make_report.json'): + if enabled is None: + enabled = 'make' in to_test + if not enabled: print('\nSkipping make integration test') return print('\nRunning make integration test') - root = os.path.join(CODETOANALYZE_DIR, 'make') errors = run_analysis( root, [['make', 'clean']], [['make', 'all']], INFER_EXECUTABLE) - original = os.path.join(EXPECTED_OUTPUTS_DIR, 'make_report.json') + original = os.path.join(EXPECTED_OUTPUTS_DIR, report_name) do_test(errors, original) def test_wonky_locale_integration(self): @@ -293,14 +340,23 @@ class BuildIntegrationTest(unittest.TestCase): original = os.path.join(EXPECTED_OUTPUTS_DIR, 'waf_report.json') do_test(errors, original) - def test_cmake_integration(self): - if not ('cmake' in to_test and + def test_cmake_integration( + self, + enabled=None, + root=os.path.join(CODETOANALYZE_DIR, 'cmake'), + report_name='cmake_report.json'): + if enabled is None: + enabled = 'cmake' in to_test + if not (enabled and is_tool_available(['cmake', '--version'])): print('\nSkipping cmake integration test') return print('\nRunning cmake integration test') - root = os.path.join(CODETOANALYZE_DIR, 'cmake', 'build') + orig_root = root + root = os.path.join(root, 'build') + # remove build/ directory just in case + shutil.rmtree(root, True) errors = run_analysis( root, [], @@ -308,9 +364,42 @@ class BuildIntegrationTest(unittest.TestCase): INFER_EXECUTABLE) # remove build/ directory shutil.rmtree(root) - original = os.path.join(EXPECTED_OUTPUTS_DIR, 'cmake_report.json') + original = os.path.join(EXPECTED_OUTPUTS_DIR, report_name) + # cmake produces absolute paths using the real path + make_paths_relative_in_report(os.path.realpath(orig_root), errors) do_test(errors, original) + def test_utf8_in_pwd_integration(self): + if not 'utf8_in_pwd' in to_test: + print('\nSkipping utf8_in_pwd integration test') + return + print('\nRunning utf8_in_pwd integration test') + + utf8_in_pwd_path = os.path.join(CODETOANALYZE_DIR, 'utf8_ιn_pwd') + + # copy non-unicode dir to one with unicode in it + shutil.rmtree(utf8_in_pwd_path, True) # remove just in case + shutil.copytree(os.path.join(CODETOANALYZE_DIR, 'utf8_in_pwd'), + utf8_in_pwd_path) + + self.test_cmake_integration( + enabled=True, + root=os.path.join(utf8_in_pwd_path, 'cmake'), + report_name='utf8_in_pwd_cmake_report.json') + self.test_gradle_integration( + enabled=True, + root=os.path.join(utf8_in_pwd_path, 'gradle'), + report_name='utf8_in_pwd_gradle_report.json') + self.test_javac_integration( + enabled=True, + root=os.path.join(utf8_in_pwd_path), + report_name='utf8_in_pwd_javac_report.json') + self.test_make_integration( + enabled=True, + root=os.path.join(utf8_in_pwd_path, 'make'), + report_name='utf8_in_pwd_make_report.json') + shutil.rmtree(utf8_in_pwd_path, True) # remove copied dir + if __name__ == '__main__': # hackish capturing of the arguments after '--' diff --git a/infer/src/java/jVerbose.ml b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/Hello.java similarity index 56% rename from infer/src/java/jVerbose.ml rename to infer/tests/build_systems/codetoanalyze/utf8_in_pwd/Hello.java index eb0efff4c..fdb567e99 100644 --- a/infer/src/java/jVerbose.ml +++ b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/Hello.java @@ -1,16 +1,15 @@ -(* - * Copyright (c) 2009 - 2013 Monoidics ltd. - * Copyright (c) 2013 - present Facebook, Inc. +/* + * Copyright (c) 2015 - present Facebook, Inc. * All rights reserved. * * This source code is licensed under the BSD style license found in the * LICENSE file in the root directory of this source tree. An additional grant * of patent rights can be found in the PATENTS file in the same directory. - *) + */ -open! Utils - -type parsed_data = - | Source of string - | Class of string - | Classpath of string list +class Hello { + int test() { + String s = null; + return s.length(); + } +} diff --git a/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/cmake b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/cmake new file mode 120000 index 000000000..8e8a460f7 --- /dev/null +++ b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/cmake @@ -0,0 +1 @@ +../cmake/ \ No newline at end of file diff --git a/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/gradle b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/gradle new file mode 120000 index 000000000..5530a6853 --- /dev/null +++ b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/gradle @@ -0,0 +1 @@ +../../../../../examples/java_hello/ \ No newline at end of file diff --git a/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/make b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/make new file mode 120000 index 000000000..b74302805 --- /dev/null +++ b/infer/tests/build_systems/codetoanalyze/utf8_in_pwd/make @@ -0,0 +1 @@ +../make/ \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/cmake_report.json b/infer/tests/build_systems/expected_outputs/cmake_report.json index b29ee489e..a6ed10eff 100644 --- a/infer/tests/build_systems/expected_outputs/cmake_report.json +++ b/infer/tests/build_systems/expected_outputs/cmake_report.json @@ -1,7 +1,7 @@ [ { "bug_type": "NULL_DEREFERENCE", - "file": "/home/jul/infer/infer/tests/build_systems/codetoanalyze/cmake/hello.c", + "file": "hello.c", "procedure": "test" } ] \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/javac_report.json b/infer/tests/build_systems/expected_outputs/javac_report.json new file mode 100644 index 000000000..32cd14214 --- /dev/null +++ b/infer/tests/build_systems/expected_outputs/javac_report.json @@ -0,0 +1,7 @@ +[ + { + "bug_type": "NULL_DEREFERENCE", + "file": "Hello.java", + "procedure": "int Hello.test()" + } +] \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/utf8_in_pwd_cmake_report.json b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_cmake_report.json new file mode 100644 index 000000000..a6ed10eff --- /dev/null +++ b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_cmake_report.json @@ -0,0 +1,7 @@ +[ + { + "bug_type": "NULL_DEREFERENCE", + "file": "hello.c", + "procedure": "test" + } +] \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/utf8_in_pwd_gradle_report.json b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_gradle_report.json new file mode 100644 index 000000000..b3f642cea --- /dev/null +++ b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_gradle_report.json @@ -0,0 +1,17 @@ +[ + { + "bug_type": "NULL_DEREFERENCE", + "file": "Hello.java", + "procedure": "void Hello.mayCauseNPE()" + }, + { + "bug_type": "RESOURCE_LEAK", + "file": "Hello.java", + "procedure": "void Hello.mayLeakResource()" + }, + { + "bug_type": "RESOURCE_LEAK", + "file": "Hello.java", + "procedure": "void Hello.twoResources()" + } +] \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/utf8_in_pwd_javac_report.json b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_javac_report.json new file mode 100644 index 000000000..32cd14214 --- /dev/null +++ b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_javac_report.json @@ -0,0 +1,7 @@ +[ + { + "bug_type": "NULL_DEREFERENCE", + "file": "Hello.java", + "procedure": "int Hello.test()" + } +] \ No newline at end of file diff --git a/infer/tests/build_systems/expected_outputs/utf8_in_pwd_make_report.json b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_make_report.json new file mode 100644 index 000000000..b43a81460 --- /dev/null +++ b/infer/tests/build_systems/expected_outputs/utf8_in_pwd_make_report.json @@ -0,0 +1,7 @@ +[ + { + "bug_type": "NULL_DEREFERENCE", + "file": "utf8_in_function_names.c", + "procedure": "test_\uc131\uacf5" + } +] \ No newline at end of file