Merge pull request #1854 from takluyver/dir-list-performance

Improve performance of directory listing

notebook/services/contents/filemanager.py

@@ -4,9 +4,11 @@
 # Distributed under the terms of the Modified BSD License.
 
 
+import errno
 import io
 import os
 import shutil
+import stat
 import warnings
 import mimetypes
 import nbformat
@@ -22,7 +24,7 @@ from traitlets import Any, Unicode, Bool, TraitError, observe, default, validate
 from ipython_genutils.py3compat import getcwd, string_types
 from . import tz
 from notebook.utils import (
-    is_hidden,
+    is_hidden, is_file_hidden,
     to_api_path,
 )
 
@@ -269,14 +271,22 @@ class FileContentsManager(FileManagerMixin, ContentsManager):
                     self.log.warning(
                         "failed to decode filename '%s': %s", name, e)
                     continue
-                # skip over broken symlinks in listing
-                if not os.path.exists(os_path):
-                    self.log.warning("%s doesn't exist", os_path)
+
+                try:
+                    st = os.stat(os_path)
+                except OSError as e:
+                    # skip over broken symlinks in listing
+                    if e.errno == errno.ENOENT:
+                        self.log.warning("%s doesn't exist", os_path)
+                    else:
+                        self.log.warning("Error stat-ing %s: %s", (os_path, e))
                     continue
-                elif not os.path.isfile(os_path) and not os.path.isdir(os_path):
+
+                if not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode):
                     self.log.debug("%s not a regular file", os_path)
                     continue
-                if self.should_list(name) and not is_hidden(os_path, self.root_dir):
+
+                if self.should_list(name) and not is_file_hidden(os_path, stat_res=st):
                     contents.append(self.get(
                         path='%s/%s' % (path, name),
                         content=False)

notebook/tests/test_utils.py

@@ -9,7 +9,7 @@ import os
 import nose.tools as nt
 
 from traitlets.tests.utils import check_help_all_output
-from notebook.utils import url_escape, url_unescape, is_hidden
+from notebook.utils import url_escape, url_unescape, is_hidden, is_file_hidden
 from ipython_genutils.py3compat import cast_unicode
 from ipython_genutils.tempdir import TemporaryDirectory
 from ipython_genutils.testing.decorators import skip_if_not_win32
@@ -60,14 +60,23 @@ def test_is_hidden():
         subdir1 = os.path.join(root, 'subdir')
         os.makedirs(subdir1)
         nt.assert_equal(is_hidden(subdir1, root), False)
+        nt.assert_equal(is_file_hidden(subdir1), False)
         subdir2 = os.path.join(root, '.subdir2')
         os.makedirs(subdir2)
         nt.assert_equal(is_hidden(subdir2, root), True)
+        nt.assert_equal(is_file_hidden(subdir2), True)
         subdir34 = os.path.join(root, 'subdir3', '.subdir4')
         os.makedirs(subdir34)
         nt.assert_equal(is_hidden(subdir34, root), True)
         nt.assert_equal(is_hidden(subdir34), True)
 
+        subdir56 = os.path.join(root, '.subdir5', 'subdir6')
+        os.makedirs(subdir56)
+        nt.assert_equal(is_hidden(subdir56, root), True)
+        nt.assert_equal(is_hidden(subdir56), True)
+        nt.assert_equal(is_file_hidden(subdir56), False)
+        nt.assert_equal(is_file_hidden(subdir56, os.stat(subdir56)), False)
+
 @skip_if_not_win32
 def test_is_hidden_win32():
     with TemporaryDirectory() as root:
@@ -78,4 +87,5 @@ def test_is_hidden_win32():
         r = ctypes.windll.kernel32.SetFileAttributesW(subdir1, 0x02)
         print(r)
         assert is_hidden(subdir1, root)
+        assert is_file_hidden(subdir1)
 

notebook/utils.py

@@ -79,6 +79,89 @@ def url_unescape(path):
 
 _win32_FILE_ATTRIBUTE_HIDDEN = 0x02
 
+def is_file_hidden_win(abs_path, stat_res=None):
+    """Is a file hidden?
+
+    This only checks the file itself; it should be called in combination with
+    checking the directory containing the file.
+
+    Use is_hidden() instead to check the file and its parent directories.
+
+    Parameters
+    ----------
+    abs_path : unicode
+        The absolute path to check.
+    stat_res : os.stat_result, optional
+        Ignored on Windows, exists for compatibility with POSIX version of the
+        function.
+    """
+    if os.path.basename(abs_path).startswith('.'):
+        return True
+
+    # check that dirs can be listed
+    if os.path.isdir(abs_path):
+        # can't trust os.access on Windows because it seems to always return True
+        try:
+            os.stat(abs_path)
+        except OSError:
+            # stat may fail on Windows junctions or non-user-readable dirs
+            return True
+
+    try:
+        attrs = ctypes.windll.kernel32.GetFileAttributesW(
+            py3compat.cast_unicode(abs_path))
+    except AttributeError:
+        pass
+    else:
+        if attrs > 0 and attrs & _win32_FILE_ATTRIBUTE_HIDDEN:
+            return True
+
+    return False
+
+def is_file_hidden_posix(abs_path, stat_res=None):
+    """Is a file hidden?
+
+    This only checks the file itself; it should be called in combination with
+    checking the directory containing the file.
+
+    Use is_hidden() instead to check the file and its parent directories.
+
+    Parameters
+    ----------
+    abs_path : unicode
+        The absolute path to check.
+    stat_res : os.stat_result, optional
+        The result of calling stat() on abs_path. If not passed, this function
+        will call stat() internally.
+    """
+    if os.path.basename(abs_path).startswith('.'):
+        return True
+
+    if stat_res is None:
+        try:
+            stat_res = os.stat(abs_path)
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                return False
+            raise
+
+    # check that dirs can be listed
+    if stat.S_ISDIR(stat_res.st_mode):
+        # use x-access, not actual listing, in case of slow/large listings
+        if not os.access(abs_path, os.X_OK | os.R_OK):
+            return True
+
+    # check UF_HIDDEN
+    if getattr(stat_res, 'st_flags', 0) & UF_HIDDEN:
+        return True
+
+    return False
+
+if sys.platform == 'win32':
+    is_file_hidden = is_file_hidden_win
+else:
+    is_file_hidden = is_file_hidden_posix
+
 def is_hidden(abs_path, abs_root=''):
     """Is a file hidden or contained in a hidden directory?
     
@@ -95,28 +178,18 @@ def is_hidden(abs_path, abs_root=''):
         The absolute path of the root directory in which hidden directories
         should be checked for.
     """
+    if is_file_hidden(abs_path):
+        return True
+
     if not abs_root:
         abs_root = abs_path.split(os.sep, 1)[0] + os.sep
     inside_root = abs_path[len(abs_root):]
     if any(part.startswith('.') for part in inside_root.split(os.sep)):
         return True
-    
-    # check that dirs can be listed
-    if os.path.isdir(abs_path):
-        if sys.platform == 'win32':
-            # can't trust os.access on Windows because it seems to always return True
-            try:
-                os.stat(abs_path)
-            except OSError:
-                # stat may fail on Windows junctions or non-user-readable dirs
-                return True
-        else:
-            # use x-access, not actual listing, in case of slow/large listings
-            if not os.access(abs_path, os.X_OK | os.R_OK):
-                return True
-    
-    # check UF_HIDDEN on any location up to root
-    path = abs_path
+
+    # check UF_HIDDEN on any location up to root.
+    # is_file_hidden() already checked the file, so start from its parent dir
+    path = os.path.dirname(abs_path)
     while path and path.startswith(abs_root) and path != abs_root:
         if not os.path.exists(path):
             path = os.path.dirname(path)
@@ -129,15 +202,6 @@ def is_hidden(abs_path, abs_root=''):
         if getattr(st, 'st_flags', 0) & UF_HIDDEN:
             return True
         path = os.path.dirname(path)
-    
-    if sys.platform == 'win32':
-        try:
-            attrs = ctypes.windll.kernel32.GetFileAttributesW(py3compat.cast_unicode(abs_path))
-        except AttributeError:
-            pass
-        else:
-            if attrs > 0 and attrs & _win32_FILE_ATTRIBUTE_HIDDEN:
-                return True
 
     return False