infer_clone/infer/tests/build_systems/build_integration_tests.py

#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
# Copyright (c) 2015 - present Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

# example usage:
# # run all the tests
# ./build_integration_test.py
# # run only the ant and gradle tests
# ./build_integration_test.py -- ant gradle
# # run no test
# ./build_integration_test.py --
# # run only the waf tests and record the output
# INFER_RECORD_INTEGRATION_TESTS=1 ./build_integration_test.py -- waf

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import json
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import unittest

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0,
                os.path.join(SCRIPT_DIR,
                             os.pardir, os.pardir, 'lib', 'python'))

from inferlib import config, issues, utils


ROOT_DIR = os.path.join(SCRIPT_DIR, os.pardir, os.pardir, os.pardir)

INFER_BIN = os.path.join(ROOT_DIR, 'infer', 'bin', 'infer')
INFERPRINT_BIN = os.path.join(ROOT_DIR, 'infer', 'bin', 'InferPrint')

CLANG_BIN = os.path.join(ROOT_DIR, 'facebook-clang-plugins', 'clang',
                         'install', 'bin', 'clang')

REPORT_JSON = 'report.json'

RECORD_ENV = 'INFER_RECORD_INTEGRATION_TESTS'

REPORT_FIELDS = [
    issues.JSON_INDEX_FILENAME,
    issues.JSON_INDEX_PROCEDURE,
    issues.JSON_INDEX_TYPE,
]

CODETOANALYZE_DIR = os.path.join(SCRIPT_DIR, 'codetoanalyze')
EXPECTED_OUTPUTS_DIR = os.path.join(SCRIPT_DIR, 'expected_outputs')

ALL_TESTS = [
    'cmake',
    'componentkit',
    'delete',
    'fail',
    'gradle',
    'javac',
    'locale',
    'make',
    'multiclang',
    'ndk-build',
    'pmd-xml',
    'reactive',
    'unknown_ext',
    'utf8_in_pwd',
    'waf',
]

to_test = ALL_TESTS


def should_record_tests():
    return RECORD_ENV in os.environ and os.environ[RECORD_ENV] == '1'


def quote(s):
    return '\"%s\"' % s


def string_of_error(e):
    line = ''
    if issues.JSON_INDEX_LINE in e:
        line = ' on line %s ' % e[issues.JSON_INDEX_LINE]
    msg = '%s in file %s, procedure %s%s' % (
        e[issues.JSON_INDEX_TYPE],
        quote(e[issues.JSON_INDEX_FILENAME]),
        quote(e[issues.JSON_INDEX_PROCEDURE]),
        line,
    )
    return msg


def save_report(reports, filename):
    # sorting to avoid spurious differences between two lists of reports
    reports.sort()

    def filter_report(report):
        return dict((k, v) for (k, v) in report.items() if k in REPORT_FIELDS)

    def should_report(report):
        return len(report) > 0

    filtered = filter(should_report, map(filter_report, reports))
    utils.dump_json_to_path(filtered, filename,
                            separators=(',', ': '), sort_keys=True)


def run_analysis(clean_cmds, build_cmds, extra_check, should_fail, env=None):
    for clean_cmd in clean_cmds:
        subprocess.check_call(clean_cmd, env=env)

    temp_out_dir = tempfile.mkdtemp(suffix='_out', prefix='infer_')
    for build_cmd in build_cmds:
        extra_args = (build_cmd['infer_args']
                      if 'infer_args' in build_cmd
                      else [])
        infer_cmd = ([INFER_BIN, '-o', temp_out_dir] +
                     extra_args +
                     ['--'] +
                     build_cmd['compile'])
        # Only record the output of the last build command. We record
        # all of them but each command overwrites the output of the
        # previous one.
        with tempfile.TemporaryFile(
                mode='w',
                suffix='.out',
                prefix='analysis_') as analysis_output:
            try:
                subprocess.check_call(infer_cmd,
                                      stdout=analysis_output, env=env)
                if should_fail is not None:
                    # hacky since we should clean up infer-out, etc. as below
                    # if you made the test fails, this is your punishment
                    assert False
            except subprocess.CalledProcessError, exn:
                if exn.returncode != should_fail:
                    raise

        # Set this to True to create an issues.exp file using the
        # results of the test. This is a temporary hack to aid
        # migrating the tests from this file to Makefiles. It can be
        # useful to compare the result of your migrated test with the
        # issues.exp that this gives you.
        if False:
            inferprint_cmd = (
                [INFERPRINT_BIN, '-q', '--issues-tests', 'issues.exp',
                 '--from-json-report',
                 os.path.join(temp_out_dir, 'report.json')] +
                extra_args)
            subprocess.check_call(inferprint_cmd, env=env)

    json_path = os.path.join(temp_out_dir, REPORT_JSON)
    found_errors = utils.load_json_from_path(json_path)
    extra_check(temp_out_dir)
    shutil.rmtree(temp_out_dir)
    os.chdir(SCRIPT_DIR)

    return found_errors


def match_pattern(f, p):
    for key in p.keys():
        if f[key] != p[key]:
            return False
    return True


def is_expected(e, patterns):
    for p in patterns:
        if match_pattern(e, p):
            return True
    return False


def is_missing(p, errors):
    for e in errors:
        if match_pattern(e, p):
            return False
    return True


def unexpected_errors(errors, patterns):
    return [e for e in errors if not is_expected(e, patterns)]


def missing_errors(errors, patterns):
    return [p for p in patterns if is_missing(p, errors)]


def check_results(errors, patterns):
    unexpected = unexpected_errors(errors, patterns)
    if unexpected != []:
        utils.stderr('\nInfer found the following unexpected errors:')
        for e in unexpected:
            utils.stderr('\t{}\n'.format(string_of_error(e)))
    missing = missing_errors(errors, patterns)
    if missing != []:
        utils.stderr('\nInfer did not find the following errors:')
        for p in missing:
            utils.stderr('\t{}\n'.format(string_of_error(p)))
    assert unexpected == []
    assert missing == []


def is_tool_available(cmd):
    try:
        with open(os.devnull, 'w') as devnull:
            subprocess.call(cmd, stdout=devnull)
    except OSError as e:
        if e.errno == os.errno.ENOENT:
            return False
        else:
            raise
    return True


def do_test(errors, expected_errors_filename):
    if should_record_tests():
        save_report(errors, expected_errors_filename)
        return
    else:
        patterns = utils.load_json_from_path(expected_errors_filename)
        check_results(errors, patterns)


def make_paths_relative_in_report(root, errors):
    for error in errors:
        # remove "root/" from each file name
        rel_fname = error[issues.JSON_INDEX_FILENAME][len(root) + 1:]
        error[issues.JSON_INDEX_FILENAME] = rel_fname
    return errors


def test(name,
         readable_name,
         root,
         compile_commands,
         clean_commands=[],
         env=None,
         available=lambda: True,
         enabled=None,
         report_fname=None,
         extra_check=lambda x: None,
         should_fail=None,
         preprocess=lambda: None,
         postprocess=lambda errors: errors):
    """Run a test.

    Arguments:
    - [name] is used to test if the test is enabled by default (but
      see [enabled])
    - [root] the directory from which to run the test
    - [compile_commands] the commands to be captured by Infer
    - [clean_commands] commands to setup the build directory prior to
      running Infer
    - [env] the environment in which to run all the commands
    - [available] a test to determine whether the test can be run
    - [enabled] whether the test should attempt to run. By default it
      is enabled if [[name] in [to_test]]
    - [report_fname] where to find the expected Infer results
    - [extra_check] some function that will be given the temporary
      results directory as argument
    - [should_fail] if not None then running infer is expected to fail
      with [should_fail] error code
    - [preprocess] a function to run before the clean and compile
      commands. If the function returns something non-None, use that as
      the compile commands.
    - [postprocess] a function that takes in an Infer report and can
      modify them. It must return an Infer report.

    Returns [True] if the test ran, [False] otherwise.

    """
    # python can't into using values of arguments in the default
    # values of other arguments
    if enabled is None:
        enabled = name in to_test
    if report_fname is None:
        report_fname = '%s_report.json' % name

    if not (enabled and available()):
        print('Skipping %s integration test' % readable_name)
        return False

    print('\nRunning %s integration test' % readable_name)

    if not os.path.exists(root):
        os.makedirs(root)
    os.chdir(root)

    pre = preprocess()
    if pre is not None:
        compile_commands = pre

    # rerun this in case preprocess() deleted the current directory
    if not os.path.exists(root):
        os.makedirs(root)
    os.chdir(root)

    errors = run_analysis(
        clean_commands,
        compile_commands,
        extra_check=extra_check,
        should_fail=should_fail,
        env=env)
    original = os.path.join(EXPECTED_OUTPUTS_DIR, report_fname)
    do_test(postprocess(errors), original)
    return True

class BuildIntegrationTest(unittest.TestCase):

    def test_javac_integration(
            self,
            enabled=None,
            root=os.path.join(ROOT_DIR, 'examples'),
            report_fname='javac_report.json'):
        test('javac', 'javac',
             root,
             [{'compile': ['javac', 'Hello.java']}],
             enabled=enabled,
             report_fname=report_fname)

    def test_gradle_integration(
            self,
            enabled=None,
            root=os.path.join(ROOT_DIR, 'examples', 'java_hello'),
            report_fname='gradle_report.json'):
        env = os.environ.copy()
        env['PATH'] = '{}:{}'.format(
            os.path.join(SCRIPT_DIR, 'mock'),
            os.getenv('PATH'),
        )
        test('gradle', 'Gradle',
             root,
             [{'compile': ['gradle', 'build']}],
             enabled=enabled,
             report_fname=report_fname,
             env=env)

    def test_make_integration(
            self,
            enabled=None,
            root=os.path.join(CODETOANALYZE_DIR, 'make'),
            report_fname='make_report.json'):
        test('make', 'make',
             root,
             [{'compile': ['make', 'all']}],
             clean_commands=[['make', 'clean']],
             enabled=enabled,
             report_fname=report_fname)

    def test_ndkbuild_integration(self):
        root = os.path.join(CODETOANALYZE_DIR, 'ndk-build', 'hello_app')
        gen_lib_dir = os.path.join(root, 'libs')
        gen_obj_dir = os.path.join(root, 'obj')
        env = os.environ.copy()
        ndk_dir = os.getenv('ANDROID_NDK',
                            os.path.join(os.path.sep,
                                         'opt',
                                         'android_ndk',
                                         'r10e'))
        env['PATH'] = '{}:{}'.format(os.getenv('PATH'), ndk_dir)
        if test('ndk-build', 'ndk-build',
                root,
                [{'compile': ['ndk-build', '-B',
                              'NDK_LIBS_OUT=./libs', 'NDK_OUT=./obj']}],
                clean_commands=[['ndk-build', 'clean']],
                available=lambda: is_tool_available([
                    os.path.join(ndk_dir, 'ndk-build'), '-v']),
                env=env):
            # remove libs/ and obj/ directories
            shutil.rmtree(gen_lib_dir)
            shutil.rmtree(gen_obj_dir)

    def test_wonky_locale_integration(self):
        env = os.environ.copy()
        env['LC_ALL'] = 'C'
        test('locale', 'wonky locale',
             os.path.join(CODETOANALYZE_DIR, 'make'),
             [{'compile': ['clang', '-c', 'utf8_in_function_names.c']},
              {'compile': ['clang', '-c', 'utf8_in_function_names.c']}],
             env=env)

    def test_waf_integration(self):
        test('waf', 'waf',
             os.path.join(CODETOANALYZE_DIR, 'make'),
             [{'compile': ['./waf', 'build']}],
             clean_commands=[['make', 'clean']])

    def test_cmake_integration(
            self,
            enabled=None,
            root=os.path.join(CODETOANALYZE_DIR, 'cmake'),
            report_fname='cmake_report.json'):
        build_root = os.path.join(root, 'build')
        if test('cmake', 'CMake',
                build_root,
                [{'compile': ['cmake', '..']},
                 {'compile': ['make', 'clean', 'all']}],
                available=lambda: is_tool_available(['cmake', '--version']),
                enabled=enabled,
                # remove build/ directory just in case
                preprocess=lambda: shutil.rmtree(build_root, True),
                # cmake produces absolute paths using the real path
                postprocess=(lambda errors:
                             make_paths_relative_in_report(
                                 os.path.realpath(root), errors))):
            # remove build/ directory
            shutil.rmtree(build_root)

    def test_utf8_in_pwd_integration(self):
        if not 'utf8_in_pwd' in to_test:
            print('\nSkipping utf8_in_pwd integration test')
            return
        print('\nRunning utf8_in_pwd integration test')

        utf8_in_pwd_path = os.path.join(CODETOANALYZE_DIR, u'utf8_\u03B9n_pwd')

        # copy non-unicode dir to one with unicode in it
        shutil.rmtree(utf8_in_pwd_path, True) # remove just in case
        shutil.copytree(os.path.join(CODETOANALYZE_DIR, 'utf8_in_pwd'),
                        utf8_in_pwd_path)

        self.test_cmake_integration(
            enabled=True,
            root=os.path.join(utf8_in_pwd_path, 'cmake'),
            report_fname='utf8_in_pwd_cmake_report.json')
        self.test_gradle_integration(
            enabled=True,
            root=os.path.join(utf8_in_pwd_path, 'gradle'),
            report_fname='utf8_in_pwd_gradle_report.json')
        self.test_javac_integration(
            enabled=True,
            root=os.path.join(utf8_in_pwd_path),
            report_fname='utf8_in_pwd_javac_report.json')
        self.test_make_integration(
            enabled=True,
            root=os.path.join(utf8_in_pwd_path, 'make'),
            report_fname='utf8_in_pwd_make_report.json')
        shutil.rmtree(utf8_in_pwd_path, True) # remove copied dir

    def test_unknown_extension(self):
        test('unknown_ext', 'unknown extension',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-x', 'c', '-c', 'hello.unknown_ext']}])

    def test_clang_multiple_source_files(self):
        test('multiclang', 'clang multiple source files',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-c', 'hello.c', 'hello2.c']}])

    def test_reactive_multiple_capture(self):
        reactive_args = ['-a', 'capture', '--reactive', '--continue']
        test('reactive', 'reactive with multiple capture',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-c', 'hello.c'],
               'infer_args': reactive_args},
              {'compile': ['clang', '-c', 'hello2.c'],
               'infer_args': reactive_args},
              {'compile': ['analyze']}])

    def test_clang_component_kit_imports(self):
        test('componentkit', 'component quality analyzer  skips imports',
             os.path.join(CODETOANALYZE_DIR, 'componentkit'),
             [{'compile': ['clang', '-x', 'objective-c++', '-std=c++11', '-c',
                           '-fblocks', 'TestIgnoreImports.mm'],
               'infer_args': ['--cxx', '--no-filtering', '-a', 'linters']}])

    def test_fail_on_issue(self):
        test('fail', '--fail-on-issue flag',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-c', 'hello.c'],
               'infer_args': ['--fail-on-issue']}],
             should_fail=2)

    def test_pmd_xml_output(self):
        def pmd_check(infer_out):
            assert os.path.exists(os.path.join(infer_out, 'report.xml'))
        try:
            from lxml import etree
            has_lxml = True
        except ImportError:
            has_lxml = False

        test('pmd-xml', 'PMD XML output',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-c', 'hello.c'],
               'infer_args': ['--pmd-xml']}],
             extra_check=pmd_check,
             available=lambda: has_lxml)

    def test_infer_deletes_infer_out(self):
        # Test that two consecutive analyses do not pollute each other:
        # the expected results of running infer on hello.c then on
        # hello2.c is that only the bug in hello2.c is reported.
        test('delete', 'infer deletes infer-out',
             CODETOANALYZE_DIR,
             [{'compile': ['clang', '-c', 'hello.c']},
              {'compile': ['clang', '-c', 'hello2.c']}])


if __name__ == '__main__':
    # hackish capturing of the arguments after '--'
    try:
        i = sys.argv.index('--')
        to_test = sys.argv[i + 1:]
        sys.argv = sys.argv[:i]
    except ValueError:
        pass

    unittest.main()  # run all the tests