From 6bb89b717a5547a7bc8dac02af5e23ceda4efa43 Mon Sep 17 00:00:00 2001 From: dhirschf Date: Tue, 13 Jun 2017 17:02:49 +1000 Subject: [PATCH] Workaround for Windows Containers On Windows `os.stat` treats host mapped volumes as broken symlinks --- notebook/services/contents/filemanager.py | 12 +++-- notebook/services/kernels/kernelmanager.py | 4 +- notebook/utils.py | 52 +++++++++++++--------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/notebook/services/contents/filemanager.py b/notebook/services/contents/filemanager.py index a05253ac3..ee6036e4a 100644 --- a/notebook/services/contents/filemanager.py +++ b/notebook/services/contents/filemanager.py @@ -9,6 +9,7 @@ import io import os import shutil import stat +import sys import warnings import mimetypes import nbformat @@ -18,6 +19,7 @@ from tornado import web from .filecheckpoints import FileCheckpoints from .fileio import FileManagerMixin from .manager import ContentsManager +from ...utils import f_stat, WINDOWS_CONTAINER, exists from ipython_genutils.importstring import import_item from traitlets import Any, Unicode, Bool, TraitError, observe, default, validate @@ -220,12 +222,12 @@ class FileContentsManager(FileManagerMixin, ContentsManager): """ path = path.strip('/') os_path = self._get_os_path(path=path) - return os.path.exists(os_path) + return exists(os_path) def _base_model(self, path): """Build the common base of a contents model""" os_path = self._get_os_path(path) - info = os.stat(os_path) + info = f_stat(os_path) last_modified = tz.utcfromtimestamp(info.st_mtime) created = tz.utcfromtimestamp(info.st_ctime) # Create the base model. @@ -275,7 +277,7 @@ class FileContentsManager(FileManagerMixin, ContentsManager): continue try: - st = os.stat(os_path) + st = f_stat(os_path) except OSError as e: # skip over broken symlinks in listing if e.errno == errno.ENOENT: @@ -284,7 +286,9 @@ class FileContentsManager(FileManagerMixin, ContentsManager): self.log.warning("Error stat-ing %s: %s", os_path, e) continue - if not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode): + if (not WINDOWS_CONTAINER + and not stat.S_ISREG(st.st_mode) + and not stat.S_ISDIR(st.st_mode)): self.log.debug("%s not a regular file", os_path) continue diff --git a/notebook/services/kernels/kernelmanager.py b/notebook/services/kernels/kernelmanager.py index 569c60426..2453d6f05 100644 --- a/notebook/services/kernels/kernelmanager.py +++ b/notebook/services/kernels/kernelmanager.py @@ -17,7 +17,7 @@ from jupyter_client.session import Session from jupyter_client.multikernelmanager import MultiKernelManager from traitlets import Bool, Dict, List, Unicode, TraitError, Integer, default, validate -from notebook.utils import to_os_path +from notebook.utils import to_os_path, exists from notebook._tz import utcnow, isoformat from ipython_genutils.py3compat import getcwd @@ -55,7 +55,7 @@ class MappingKernelManager(MultiKernelManager): if not os.path.isabs(value): # If we receive a non-absolute path, make it absolute. value = os.path.abspath(value) - if not os.path.exists(value) or not os.path.isdir(value): + if not exists(value) or not os.path.isdir(value): raise TraitError("kernel root dir %r is not a directory" % value) return value diff --git a/notebook/utils.py b/notebook/utils.py index 2ad3710a9..27c54b87d 100644 --- a/notebook/utils.py +++ b/notebook/utils.py @@ -24,6 +24,24 @@ from ipython_genutils import py3compat # It is used by BSD to indicate hidden files. UF_HIDDEN = getattr(stat, 'UF_HIDDEN', 32768) +# Replace `os.stat` which can't stat a host mapped volume from inside a Windows Container. +WINDOWS_CONTAINER = sys.platform == 'win32' and os.environ.get('USERNAME') == 'ContainerAdministrator' +if WINDOWS_CONTAINER: + f_stat = os.lstat +else: + f_stat = os.stat + + +def exists(path): + """Replacement for `os.path.exists` which works for host mapped volumes + on Windows containers + """ + try: + f_stat(path) + except OSError: + return False + return True + def url_path_join(*pieces): """Join components of url into a relative url @@ -58,10 +76,10 @@ def url2path(url): pieces = [ unquote(p) for p in url.split('/') ] path = os.path.join(*pieces) return path - + def url_escape(path): """Escape special characters in a URL path - + Turns '/foo bar/' into '/foo%20bar/' """ parts = py3compat.unicode_to_str(path, encoding='utf8').split('/') @@ -69,7 +87,7 @@ def url_escape(path): def url_unescape(path): """Unescape special characters in a URL path - + Turns '/foo%20bar/' into '/foo bar/' """ return u'/'.join([ @@ -77,7 +95,6 @@ def url_unescape(path): for p in py3compat.unicode_to_str(path, encoding='utf8').split('/') ]) -_win32_FILE_ATTRIBUTE_HIDDEN = 0x02 def is_file_hidden_win(abs_path, stat_res=None): """Is a file hidden? @@ -98,22 +115,15 @@ def is_file_hidden_win(abs_path, stat_res=None): if os.path.basename(abs_path).startswith('.'): return True - # check that dirs can be listed - if os.path.isdir(abs_path): - # can't trust os.access on Windows because it seems to always return True - try: - os.stat(abs_path) - except OSError: - # stat may fail on Windows junctions or non-user-readable dirs - return True - + win32_FILE_ATTRIBUTE_HIDDEN = 0x02 try: attrs = ctypes.windll.kernel32.GetFileAttributesW( - py3compat.cast_unicode(abs_path)) + py3compat.cast_unicode(abs_path) + ) except AttributeError: pass else: - if attrs > 0 and attrs & _win32_FILE_ATTRIBUTE_HIDDEN: + if attrs > 0 and attrs & win32_FILE_ATTRIBUTE_HIDDEN: return True return False @@ -164,12 +174,12 @@ else: def is_hidden(abs_path, abs_root=''): """Is a file hidden or contained in a hidden directory? - + This will start with the rightmost path element and work backwards to the given root to see if a path is hidden or in a hidden directory. Hidden is - determined by either name starting with '.' or the UF_HIDDEN flag as + determined by either name starting with '.' or the UF_HIDDEN flag as reported by stat. - + Parameters ---------- abs_path : unicode @@ -191,12 +201,12 @@ def is_hidden(abs_path, abs_root=''): # is_file_hidden() already checked the file, so start from its parent dir path = os.path.dirname(abs_path) while path and path.startswith(abs_root) and path != abs_root: - if not os.path.exists(path): + if not exists(path): path = os.path.dirname(path) continue try: # may fail on Windows junctions - st = os.stat(path) + st = f_stat(path) except OSError: return True if getattr(st, 'st_flags', 0) & UF_HIDDEN: @@ -244,7 +254,7 @@ def to_os_path(path, root=''): def to_api_path(os_path, root=''): """Convert a filesystem path to an API path - + If given, root will be removed from the path. root must be a filesystem path already. """