You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

649 lines
21 KiB

import glob
import os
import shutil
import subprocess
import sys
import tempfile
import warnings
from sysconfig import get_config_var, get_config_vars, get_path
from .runners import (
CCompilerRunner,
CppCompilerRunner,
FortranCompilerRunner
)
from .util import (
get_abspath, make_dirs, copy, Glob, ArbitraryDepthGlob,
glob_at_depth, import_module_from_file, pyx_is_cplus,
sha256_of_string, sha256_of_file, CompileError
)
if os.name == 'posix':
objext = '.o'
elif os.name == 'nt':
objext = '.obj'
else:
warnings.warn("Unknown os.name: {}".format(os.name))
objext = '.o'
def compile_sources(files, Runner=None, destdir=None, cwd=None, keep_dir_struct=False,
per_file_kwargs=None, **kwargs):
""" Compile source code files to object files.
Parameters
==========
files : iterable of str
Paths to source files, if ``cwd`` is given, the paths are taken as relative.
Runner: CompilerRunner subclass (optional)
Could be e.g. ``FortranCompilerRunner``. Will be inferred from filename
extensions if missing.
destdir: str
Output directory, if cwd is given, the path is taken as relative.
cwd: str
Working directory. Specify to have compiler run in other directory.
also used as root of relative paths.
keep_dir_struct: bool
Reproduce directory structure in `destdir`. default: ``False``
per_file_kwargs: dict
Dict mapping instances in ``files`` to keyword arguments.
\\*\\*kwargs: dict
Default keyword arguments to pass to ``Runner``.
"""
_per_file_kwargs = {}
if per_file_kwargs is not None:
for k, v in per_file_kwargs.items():
if isinstance(k, Glob):
for path in glob.glob(k.pathname):
_per_file_kwargs[path] = v
elif isinstance(k, ArbitraryDepthGlob):
for path in glob_at_depth(k.filename, cwd):
_per_file_kwargs[path] = v
else:
_per_file_kwargs[k] = v
# Set up destination directory
destdir = destdir or '.'
if not os.path.isdir(destdir):
if os.path.exists(destdir):
raise OSError("{} is not a directory".format(destdir))
else:
make_dirs(destdir)
if cwd is None:
cwd = '.'
for f in files:
copy(f, destdir, only_update=True, dest_is_dir=True)
# Compile files and return list of paths to the objects
dstpaths = []
for f in files:
if keep_dir_struct:
name, ext = os.path.splitext(f)
else:
name, ext = os.path.splitext(os.path.basename(f))
file_kwargs = kwargs.copy()
file_kwargs.update(_per_file_kwargs.get(f, {}))
dstpaths.append(src2obj(f, Runner, cwd=cwd, **file_kwargs))
return dstpaths
def get_mixed_fort_c_linker(vendor=None, cplus=False, cwd=None):
vendor = vendor or os.environ.get('SYMPY_COMPILER_VENDOR', 'gnu')
if vendor.lower() == 'intel':
if cplus:
return (FortranCompilerRunner,
{'flags': ['-nofor_main', '-cxxlib']}, vendor)
else:
return (FortranCompilerRunner,
{'flags': ['-nofor_main']}, vendor)
elif vendor.lower() == 'gnu' or 'llvm':
if cplus:
return (CppCompilerRunner,
{'lib_options': ['fortran']}, vendor)
else:
return (FortranCompilerRunner,
{}, vendor)
else:
raise ValueError("No vendor found.")
def link(obj_files, out_file=None, shared=False, Runner=None,
cwd=None, cplus=False, fort=False, **kwargs):
""" Link object files.
Parameters
==========
obj_files: iterable of str
Paths to object files.
out_file: str (optional)
Path to executable/shared library, if ``None`` it will be
deduced from the last item in obj_files.
shared: bool
Generate a shared library?
Runner: CompilerRunner subclass (optional)
If not given the ``cplus`` and ``fort`` flags will be inspected
(fallback is the C compiler).
cwd: str
Path to the root of relative paths and working directory for compiler.
cplus: bool
C++ objects? default: ``False``.
fort: bool
Fortran objects? default: ``False``.
\\*\\*kwargs: dict
Keyword arguments passed to ``Runner``.
Returns
=======
The absolute path to the generated shared object / executable.
"""
if out_file is None:
out_file, ext = os.path.splitext(os.path.basename(obj_files[-1]))
if shared:
out_file += get_config_var('EXT_SUFFIX')
if not Runner:
if fort:
Runner, extra_kwargs, vendor = \
get_mixed_fort_c_linker(
vendor=kwargs.get('vendor', None),
cplus=cplus,
cwd=cwd,
)
for k, v in extra_kwargs.items():
if k in kwargs:
kwargs[k].expand(v)
else:
kwargs[k] = v
else:
if cplus:
Runner = CppCompilerRunner
else:
Runner = CCompilerRunner
flags = kwargs.pop('flags', [])
if shared:
if '-shared' not in flags:
flags.append('-shared')
run_linker = kwargs.pop('run_linker', True)
if not run_linker:
raise ValueError("run_linker was set to False (nonsensical).")
out_file = get_abspath(out_file, cwd=cwd)
runner = Runner(obj_files, out_file, flags, cwd=cwd, **kwargs)
runner.run()
return out_file
def link_py_so(obj_files, so_file=None, cwd=None, libraries=None,
cplus=False, fort=False, **kwargs):
""" Link Python extension module (shared object) for importing
Parameters
==========
obj_files: iterable of str
Paths to object files to be linked.
so_file: str
Name (path) of shared object file to create. If not specified it will
have the basname of the last object file in `obj_files` but with the
extension '.so' (Unix).
cwd: path string
Root of relative paths and working directory of linker.
libraries: iterable of strings
Libraries to link against, e.g. ['m'].
cplus: bool
Any C++ objects? default: ``False``.
fort: bool
Any Fortran objects? default: ``False``.
kwargs**: dict
Keyword arguments passed to ``link(...)``.
Returns
=======
Absolute path to the generate shared object.
"""
libraries = libraries or []
include_dirs = kwargs.pop('include_dirs', [])
library_dirs = kwargs.pop('library_dirs', [])
# Add Python include and library directories
# PY_LDFLAGS does not available on all python implementations
# e.g. when with pypy, so it's LDFLAGS we need to use
if sys.platform == "win32":
warnings.warn("Windows not yet supported.")
elif sys.platform == 'darwin':
cfgDict = get_config_vars()
kwargs['linkline'] = kwargs.get('linkline', []) + [cfgDict['LDFLAGS']]
library_dirs += [cfgDict['LIBDIR']]
# In macOS, linker needs to compile frameworks
# e.g. "-framework CoreFoundation"
is_framework = False
for opt in cfgDict['LIBS'].split():
if is_framework:
kwargs['linkline'] = kwargs.get('linkline', []) + ['-framework', opt]
is_framework = False
elif opt.startswith('-l'):
libraries.append(opt[2:])
elif opt.startswith('-framework'):
is_framework = True
# The python library is not included in LIBS
libfile = cfgDict['LIBRARY']
libname = ".".join(libfile.split('.')[:-1])[3:]
libraries.append(libname)
elif sys.platform[:3] == 'aix':
# Don't use the default code below
pass
else:
if get_config_var('Py_ENABLE_SHARED'):
cfgDict = get_config_vars()
kwargs['linkline'] = kwargs.get('linkline', []) + [cfgDict['LDFLAGS']]
library_dirs += [cfgDict['LIBDIR']]
for opt in cfgDict['BLDLIBRARY'].split():
if opt.startswith('-l'):
libraries += [opt[2:]]
else:
pass
flags = kwargs.pop('flags', [])
needed_flags = ('-pthread',)
for flag in needed_flags:
if flag not in flags:
flags.append(flag)
return link(obj_files, shared=True, flags=flags, cwd=cwd,
cplus=cplus, fort=fort, include_dirs=include_dirs,
libraries=libraries, library_dirs=library_dirs, **kwargs)
def simple_cythonize(src, destdir=None, cwd=None, **cy_kwargs):
""" Generates a C file from a Cython source file.
Parameters
==========
src: str
Path to Cython source.
destdir: str (optional)
Path to output directory (default: '.').
cwd: path string (optional)
Root of relative paths (default: '.').
**cy_kwargs:
Second argument passed to cy_compile. Generates a .cpp file if ``cplus=True`` in ``cy_kwargs``,
else a .c file.
"""
from Cython.Compiler.Main import (
default_options, CompilationOptions
)
from Cython.Compiler.Main import compile as cy_compile
assert src.lower().endswith('.pyx') or src.lower().endswith('.py')
cwd = cwd or '.'
destdir = destdir or '.'
ext = '.cpp' if cy_kwargs.get('cplus', False) else '.c'
c_name = os.path.splitext(os.path.basename(src))[0] + ext
dstfile = os.path.join(destdir, c_name)
if cwd:
ori_dir = os.getcwd()
else:
ori_dir = '.'
os.chdir(cwd)
try:
cy_options = CompilationOptions(default_options)
cy_options.__dict__.update(cy_kwargs)
# Set language_level if not set by cy_kwargs
# as not setting it is deprecated
if 'language_level' not in cy_kwargs:
cy_options.__dict__['language_level'] = 3
cy_result = cy_compile([src], cy_options)
if cy_result.num_errors > 0:
raise ValueError("Cython compilation failed.")
# Move generated C file to destination
# In macOS, the generated C file is in the same directory as the source
# but the /var is a symlink to /private/var, so we need to use realpath
if os.path.realpath(os.path.dirname(src)) != os.path.realpath(destdir):
if os.path.exists(dstfile):
os.unlink(dstfile)
shutil.move(os.path.join(os.path.dirname(src), c_name), destdir)
finally:
os.chdir(ori_dir)
return dstfile
extension_mapping = {
'.c': (CCompilerRunner, None),
'.cpp': (CppCompilerRunner, None),
'.cxx': (CppCompilerRunner, None),
'.f': (FortranCompilerRunner, None),
'.for': (FortranCompilerRunner, None),
'.ftn': (FortranCompilerRunner, None),
'.f90': (FortranCompilerRunner, None), # ifort only knows about .f90
'.f95': (FortranCompilerRunner, 'f95'),
'.f03': (FortranCompilerRunner, 'f2003'),
'.f08': (FortranCompilerRunner, 'f2008'),
}
def src2obj(srcpath, Runner=None, objpath=None, cwd=None, inc_py=False, **kwargs):
""" Compiles a source code file to an object file.
Files ending with '.pyx' assumed to be cython files and
are dispatched to pyx2obj.
Parameters
==========
srcpath: str
Path to source file.
Runner: CompilerRunner subclass (optional)
If ``None``: deduced from extension of srcpath.
objpath : str (optional)
Path to generated object. If ``None``: deduced from ``srcpath``.
cwd: str (optional)
Working directory and root of relative paths. If ``None``: current dir.
inc_py: bool
Add Python include path to kwarg "include_dirs". Default: False
\\*\\*kwargs: dict
keyword arguments passed to Runner or pyx2obj
"""
name, ext = os.path.splitext(os.path.basename(srcpath))
if objpath is None:
if os.path.isabs(srcpath):
objpath = '.'
else:
objpath = os.path.dirname(srcpath)
objpath = objpath or '.' # avoid objpath == ''
if os.path.isdir(objpath):
objpath = os.path.join(objpath, name + objext)
include_dirs = kwargs.pop('include_dirs', [])
if inc_py:
py_inc_dir = get_path('include')
if py_inc_dir not in include_dirs:
include_dirs.append(py_inc_dir)
if ext.lower() == '.pyx':
return pyx2obj(srcpath, objpath=objpath, include_dirs=include_dirs, cwd=cwd,
**kwargs)
if Runner is None:
Runner, std = extension_mapping[ext.lower()]
if 'std' not in kwargs:
kwargs['std'] = std
flags = kwargs.pop('flags', [])
needed_flags = ('-fPIC',)
for flag in needed_flags:
if flag not in flags:
flags.append(flag)
# src2obj implies not running the linker...
run_linker = kwargs.pop('run_linker', False)
if run_linker:
raise CompileError("src2obj called with run_linker=True")
runner = Runner([srcpath], objpath, include_dirs=include_dirs,
run_linker=run_linker, cwd=cwd, flags=flags, **kwargs)
runner.run()
return objpath
def pyx2obj(pyxpath, objpath=None, destdir=None, cwd=None,
include_dirs=None, cy_kwargs=None, cplus=None, **kwargs):
"""
Convenience function
If cwd is specified, pyxpath and dst are taken to be relative
If only_update is set to `True` the modification time is checked
and compilation is only run if the source is newer than the
destination
Parameters
==========
pyxpath: str
Path to Cython source file.
objpath: str (optional)
Path to object file to generate.
destdir: str (optional)
Directory to put generated C file. When ``None``: directory of ``objpath``.
cwd: str (optional)
Working directory and root of relative paths.
include_dirs: iterable of path strings (optional)
Passed onto src2obj and via cy_kwargs['include_path']
to simple_cythonize.
cy_kwargs: dict (optional)
Keyword arguments passed onto `simple_cythonize`
cplus: bool (optional)
Indicate whether C++ is used. default: auto-detect using ``.util.pyx_is_cplus``.
compile_kwargs: dict
keyword arguments passed onto src2obj
Returns
=======
Absolute path of generated object file.
"""
assert pyxpath.endswith('.pyx')
cwd = cwd or '.'
objpath = objpath or '.'
destdir = destdir or os.path.dirname(objpath)
abs_objpath = get_abspath(objpath, cwd=cwd)
if os.path.isdir(abs_objpath):
pyx_fname = os.path.basename(pyxpath)
name, ext = os.path.splitext(pyx_fname)
objpath = os.path.join(objpath, name + objext)
cy_kwargs = cy_kwargs or {}
cy_kwargs['output_dir'] = cwd
if cplus is None:
cplus = pyx_is_cplus(pyxpath)
cy_kwargs['cplus'] = cplus
interm_c_file = simple_cythonize(pyxpath, destdir=destdir, cwd=cwd, **cy_kwargs)
include_dirs = include_dirs or []
flags = kwargs.pop('flags', [])
needed_flags = ('-fwrapv', '-pthread', '-fPIC')
for flag in needed_flags:
if flag not in flags:
flags.append(flag)
options = kwargs.pop('options', [])
if kwargs.pop('strict_aliasing', False):
raise CompileError("Cython requires strict aliasing to be disabled.")
# Let's be explicit about standard
if cplus:
std = kwargs.pop('std', 'c++98')
else:
std = kwargs.pop('std', 'c99')
return src2obj(interm_c_file, objpath=objpath, cwd=cwd,
include_dirs=include_dirs, flags=flags, std=std,
options=options, inc_py=True, strict_aliasing=False,
**kwargs)
def _any_X(srcs, cls):
for src in srcs:
name, ext = os.path.splitext(src)
key = ext.lower()
if key in extension_mapping:
if extension_mapping[key][0] == cls:
return True
return False
def any_fortran_src(srcs):
return _any_X(srcs, FortranCompilerRunner)
def any_cplus_src(srcs):
return _any_X(srcs, CppCompilerRunner)
def compile_link_import_py_ext(sources, extname=None, build_dir='.', compile_kwargs=None,
link_kwargs=None):
""" Compiles sources to a shared object (Python extension) and imports it
Sources in ``sources`` which is imported. If shared object is newer than the sources, they
are not recompiled but instead it is imported.
Parameters
==========
sources : string
List of paths to sources.
extname : string
Name of extension (default: ``None``).
If ``None``: taken from the last file in ``sources`` without extension.
build_dir: str
Path to directory in which objects files etc. are generated.
compile_kwargs: dict
keyword arguments passed to ``compile_sources``
link_kwargs: dict
keyword arguments passed to ``link_py_so``
Returns
=======
The imported module from of the Python extension.
"""
if extname is None:
extname = os.path.splitext(os.path.basename(sources[-1]))[0]
compile_kwargs = compile_kwargs or {}
link_kwargs = link_kwargs or {}
try:
mod = import_module_from_file(os.path.join(build_dir, extname), sources)
except ImportError:
objs = compile_sources(list(map(get_abspath, sources)), destdir=build_dir,
cwd=build_dir, **compile_kwargs)
so = link_py_so(objs, cwd=build_dir, fort=any_fortran_src(sources),
cplus=any_cplus_src(sources), **link_kwargs)
mod = import_module_from_file(so)
return mod
def _write_sources_to_build_dir(sources, build_dir):
build_dir = build_dir or tempfile.mkdtemp()
if not os.path.isdir(build_dir):
raise OSError("Non-existent directory: ", build_dir)
source_files = []
for name, src in sources:
dest = os.path.join(build_dir, name)
differs = True
sha256_in_mem = sha256_of_string(src.encode('utf-8')).hexdigest()
if os.path.exists(dest):
if os.path.exists(dest + '.sha256'):
with open(dest + '.sha256') as fh:
sha256_on_disk = fh.read()
else:
sha256_on_disk = sha256_of_file(dest).hexdigest()
differs = sha256_on_disk != sha256_in_mem
if differs:
with open(dest, 'wt') as fh:
fh.write(src)
with open(dest + '.sha256', 'wt') as fh:
fh.write(sha256_in_mem)
source_files.append(dest)
return source_files, build_dir
def compile_link_import_strings(sources, build_dir=None, **kwargs):
""" Compiles, links and imports extension module from source.
Parameters
==========
sources : iterable of name/source pair tuples
build_dir : string (default: None)
Path. ``None`` implies use a temporary directory.
**kwargs:
Keyword arguments passed onto `compile_link_import_py_ext`.
Returns
=======
mod : module
The compiled and imported extension module.
info : dict
Containing ``build_dir`` as 'build_dir'.
"""
source_files, build_dir = _write_sources_to_build_dir(sources, build_dir)
mod = compile_link_import_py_ext(source_files, build_dir=build_dir, **kwargs)
info = {"build_dir": build_dir}
return mod, info
def compile_run_strings(sources, build_dir=None, clean=False, compile_kwargs=None, link_kwargs=None):
""" Compiles, links and runs a program built from sources.
Parameters
==========
sources : iterable of name/source pair tuples
build_dir : string (default: None)
Path. ``None`` implies use a temporary directory.
clean : bool
Whether to remove build_dir after use. This will only have an
effect if ``build_dir`` is ``None`` (which creates a temporary directory).
Passing ``clean == True`` and ``build_dir != None`` raises a ``ValueError``.
This will also set ``build_dir`` in returned info dictionary to ``None``.
compile_kwargs: dict
Keyword arguments passed onto ``compile_sources``
link_kwargs: dict
Keyword arguments passed onto ``link``
Returns
=======
(stdout, stderr): pair of strings
info: dict
Containing exit status as 'exit_status' and ``build_dir`` as 'build_dir'
"""
if clean and build_dir is not None:
raise ValueError("Automatic removal of build_dir is only available for temporary directory.")
try:
source_files, build_dir = _write_sources_to_build_dir(sources, build_dir)
objs = compile_sources(list(map(get_abspath, source_files)), destdir=build_dir,
cwd=build_dir, **(compile_kwargs or {}))
prog = link(objs, cwd=build_dir,
fort=any_fortran_src(source_files),
cplus=any_cplus_src(source_files), **(link_kwargs or {}))
p = subprocess.Popen([prog], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
exit_status = p.wait()
stdout, stderr = [txt.decode('utf-8') for txt in p.communicate()]
finally:
if clean and os.path.isdir(build_dir):
shutil.rmtree(build_dir)
build_dir = None
info = {"exit_status": exit_status, "build_dir": build_dir}
return (stdout, stderr), info