Merge pull request #4438 from takluyver/1codebase

Single codebase Python 3 support (again)

IPython/html/base/handlers.py

@@ -34,6 +34,7 @@ except ImportError:
 
 from IPython.config import Application
 from IPython.utils.path import filefind
+from IPython.utils.py3compat import string_types
 
 # UF_HIDDEN is a stat flag not defined in the stat module.
 # It is used by BSD to indicate hidden files.
@@ -307,7 +308,7 @@ class FileFindHandler(web.StaticFileHandler):
     _static_paths = {}
     
     def initialize(self, path, default_filename=None):
-        if isinstance(path, basestring):
+        if isinstance(path, string_types):
             path = [path]
         
         self.root = tuple(

IPython/html/base/zmqhandlers.py

@@ -16,7 +16,10 @@ Authors:
 # Imports
 #-----------------------------------------------------------------------------
 
-import Cookie
+try:
+    from http.cookies import SimpleCookie  # Py 3
+except ImportError:
+    from Cookie import SimpleCookie  # Py 2
 import logging
 from tornado import web
 from tornado import websocket
@@ -102,7 +105,7 @@ class AuthenticatedZMQStreamHandler(ZMQStreamHandler, IPythonHandler):
             logging.error("First ws message didn't have the form 'identity:[cookie]' - %r", msg)
         
         try:
-            self.request._cookies = Cookie.SimpleCookie(msg)
+            self.request._cookies = SimpleCookie(msg)
         except:
             self.log.warn("couldn't parse cookie string: %s",msg, exc_info=True)
 

IPython/html/notebook/handlers.py

@@ -16,16 +16,12 @@ Authors:
 # Imports
 #-----------------------------------------------------------------------------
 
-import os
-import json
-
 from tornado import web
 HTTPError = web.HTTPError
 
 from ..base.handlers import IPythonHandler
 from ..services.notebooks.handlers import _notebook_path_regex, _path_regex
-from ..utils import url_path_join, url_escape, url_unescape
-from urllib import quote
+from ..utils import url_path_join, url_escape
 
 #-----------------------------------------------------------------------------
 # Handlers

IPython/html/notebookapp.py

@@ -5,6 +5,7 @@ Authors:
 
 * Brian Granger
 """
+from __future__ import print_function
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
 #
@@ -628,7 +629,7 @@ class NotebookApp(BaseIPythonApplication):
         time.sleep(0.1)
         info = self.log.info
         info('interrupted')
-        print self.notebook_info()
+        print(self.notebook_info())
         sys.stdout.write("Shutdown this notebook server (y/[n])? ")
         sys.stdout.flush()
         r,w,x = select.select([sys.stdin], [], [], 5)
@@ -639,8 +640,8 @@ class NotebookApp(BaseIPythonApplication):
                 ioloop.IOLoop.instance().stop()
                 return
         else:
-            print "No answer for 5s:",
-        print "resuming operation..."
+            print("No answer for 5s:", end=' ')
+        print("resuming operation...")
         # no answer, or answer is no:
         # set it back to original SIGINT handler
         # use IOLoop.add_callback because signal.signal must be called
@@ -652,7 +653,7 @@ class NotebookApp(BaseIPythonApplication):
         ioloop.IOLoop.instance().stop()
 
     def _signal_info(self, sig, frame):
-        print self.notebook_info()
+        print(self.notebook_info())
     
     def init_components(self):
         """Check the components submodule, and warn if it's unclean"""

IPython/html/services/notebooks/tests/test_nbmanager.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 """Tests for the notebook manager."""
+from __future__ import print_function
 
 import os
 
@@ -66,7 +67,7 @@ class TestNotebookManager(TestCase):
         try:
             os.makedirs(os_path)
         except OSError:
-            print "Directory already exists."
+            print("Directory already exists.")
 
     def test_create_notebook_model(self):
         with TemporaryDirectory() as td:

IPython/html/services/sessions/sessionmanager.py

@@ -22,6 +22,7 @@ import sqlite3
 from tornado import web
 
 from IPython.config.configurable import LoggingConfigurable
+from IPython.utils.py3compat import unicode_type
 
 #-----------------------------------------------------------------------------
 # Classes
@@ -66,7 +67,7 @@ class SessionManager(LoggingConfigurable):
 
     def new_session_id(self):
         "Create a uuid for a new session"
-        return unicode(uuid.uuid4())
+        return unicode_type(uuid.uuid4())
 
     def create_session(self, name=None, path=None, kernel_id=None, ws_url=None):
         """Creates a session and returns its model"""
@@ -132,7 +133,7 @@ class SessionManager(LoggingConfigurable):
 
         query = "SELECT * FROM session WHERE %s" % (' AND '.join(conditions))
 
-        self.cursor.execute(query, kwargs.values())
+        self.cursor.execute(query, list(kwargs.values()))
         model = self.cursor.fetchone()
         if model is None:
             q = []
@@ -169,7 +170,7 @@ class SessionManager(LoggingConfigurable):
                 raise TypeError("No such column: %r" % column)
             sets.append("%s=?" % column)
         query = "UPDATE session SET %s WHERE session_id=?" % (', '.join(sets))
-        self.cursor.execute(query, kwargs.values() + [session_id])
+        self.cursor.execute(query, list(kwargs.values()) + [session_id])
 
     @staticmethod
     def row_factory(cursor, row):

IPython/html/services/sessions/tests/test_sessions_api.py

@@ -1,5 +1,6 @@
 """Test the sessions web service API."""
 
+import errno
 import io
 import os
 import json
@@ -51,7 +52,12 @@ class SessionAPITest(NotebookTestBase):
     """Test the sessions web service API"""
     def setUp(self):
         nbdir = self.notebook_dir.name
-        os.mkdir(pjoin(nbdir, 'foo'))
+        try:
+            os.mkdir(pjoin(nbdir, 'foo'))
+        except OSError as e:
+            # Deleting the folder in an earlier test may have failed
+            if e.errno != errno.EEXIST:
+                raise
 
         with io.open(pjoin(nbdir, 'foo', 'nb1.ipynb'), 'w') as f:
             nb = new_notebook(name='nb1')
@@ -62,7 +68,8 @@ class SessionAPITest(NotebookTestBase):
     def tearDown(self):
         for session in self.sess_api.list().json():
             self.sess_api.delete(session['id'])
-        shutil.rmtree(pjoin(self.notebook_dir.name, 'foo'))
+        shutil.rmtree(pjoin(self.notebook_dir.name, 'foo'),
+                      ignore_errors=True)
 
     def test_create(self):
         sessions = self.sess_api.list().json()

IPython/html/utils.py

@@ -13,7 +13,10 @@ Authors:
 #-----------------------------------------------------------------------------
 
 import os
-from urllib import quote, unquote
+try:
+    from urllib.parse import quote, unquote
+except ImportError:
+    from urllib import quote, unquote
 
 from IPython.utils import py3compat
 

IPython/testing/iptest.py

@@ -382,9 +382,21 @@ class StreamCapturer(Thread):
                 continue
 
             ready = select(streams, [], [], 0.5)[0]
+            dead = []
             with self.buffer_lock:
                 for fd in ready:
-                    self.buffer.write(os.read(fd, 1024))
+                    try:
+                        self.buffer.write(os.read(fd, 1024))
+                    except OSError as e:
+                        import errno
+                        if e.errno == errno.EBADF:
+                            dead.append(fd)
+                        else:
+                            raise
+
+            with self.streams_lock:
+                for fd in dead:
+                    self.streams.remove(fd)
     
     def add_stream(self, fd):
         with self.streams_lock:

IPython/testing/iptestcontroller.py

@@ -224,7 +224,8 @@ def prepare_controllers(options):
     testgroups = options.testgroups
 
     if testgroups:
-        py_testgroups = [g for g in testgroups if g in py_test_group_names]
+        py_testgroups = [g for g in testgroups if (g in py_test_group_names) \
+                                                or g.startswith('IPython')]
         js_testgroups = [g for g in testgroups if g in js_test_group_names]
     else:
         py_testgroups = py_test_group_names
@@ -236,7 +237,7 @@ def prepare_controllers(options):
     c_py = [PyTestController(name) for name in py_testgroups]
 
     configure_py_controllers(c_py, xunit=options.xunit,
-            coverage=options.coverage)
+            coverage=options.coverage, extra_args=options.extra_args)
 
     controllers = c_py + c_js
     to_run = [c for c in controllers if c.will_run]

IPython/utils/tests/test_io.py

@@ -15,14 +15,18 @@ from __future__ import print_function
 
 import sys
 
-from StringIO import StringIO
 from subprocess import Popen, PIPE
 import unittest
 
 import nose.tools as nt
 
 from IPython.utils.io import Tee, capture_output
-from IPython.utils.py3compat import doctest_refactor_print
+from IPython.utils.py3compat import doctest_refactor_print, PY3
+
+if PY3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Tests

setup.py

@@ -51,10 +51,6 @@ if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
-# On Python 3, we need distribute (new setuptools) to do the 2to3 conversion
-if PY3:
-    import setuptools
-
 # Our own imports
 from setupbase import target_update
 
@@ -63,6 +59,7 @@ from setupbase import (
     find_packages,
     find_package_data,
     find_scripts,
+    build_scripts_rename,
     find_data_files,
     check_for_dependencies,
     git_prebuild,
@@ -266,7 +263,7 @@ if 'setuptools' in sys.modules:
     setup_args['cmdclass']['develop'] = require_submodules(develop)
     
     setuptools_extra_args['zip_safe'] = False
-    setuptools_extra_args['entry_points'] = find_scripts(True)
+    setuptools_extra_args['entry_points'] = find_scripts(True, suffix = '3' if PY3 else '')
     setup_args['extras_require'] = dict(
         parallel = 'pyzmq>=2.1.11',
         qtconsole = ['pyzmq>=2.1.11', 'pygments'],
@@ -314,29 +311,15 @@ if 'setuptools' in sys.modules:
                                  {"install_script":
                                   "ipython_win_post_install.py"}}
 
-    if PY3:
-        setuptools_extra_args['use_2to3'] = True
-        # we try to make a 2.6, 2.7, and 3.1 to 3.3 python compatible code
-        # so we explicitly disable some 2to3 fixes to be sure we aren't forgetting
-        # anything.
-        setuptools_extra_args['use_2to3_exclude_fixers'] = [
-                'lib2to3.fixes.fix_apply',
-                'lib2to3.fixes.fix_except',
-                'lib2to3.fixes.fix_has_key',
-                'lib2to3.fixes.fix_next',
-                'lib2to3.fixes.fix_repr',
-                'lib2to3.fixes.fix_tuple_params',
-                ]
-        from setuptools.command.build_py import build_py
-        setup_args['cmdclass'] = {'build_py': git_prebuild('IPython', build_cmd=build_py)}
-        setuptools_extra_args['entry_points'] = find_scripts(True, suffix='3')
-        setuptools._dont_write_bytecode = True
 else:
     # If we are running without setuptools, call this function which will
     # check for dependencies an inform the user what is needed.  This is
     # just to make life easy for users.
     check_for_dependencies()
     setup_args['scripts'] = find_scripts(False)
+    if PY3:
+        # Rename scripts with '3' suffix
+        setup_args['cmdclass']['build_scripts'] = build_scripts_rename
 
 #---------------------------------------------------------------------------
 # Do the actual setup now

setupbase.py

@@ -28,6 +28,7 @@ try:
 except:
     from ConfigParser import ConfigParser
 from distutils.command.build_py import build_py
+from distutils.command.build_scripts import build_scripts
 from distutils.cmd import Command
 from glob import glob
 from subprocess import call
@@ -347,6 +348,21 @@ def find_scripts(entry_points=False, suffix=''):
         ]
     return scripts
 
+class build_scripts_rename(build_scripts):
+    """Use this on Python 3 to rename scripts to ipython3 etc."""
+    _suffix = '3'
+    
+    def copy_scripts(self):
+        outfiles, updated_files = super(build_scripts_rename, self).copy_scripts()
+        new_outfiles = [p + self._suffix for p in outfiles]
+        updated_files = [p + self._suffix for p in updated_files]
+        for old, new in zip(outfiles, new_outfiles):
+            if os.path.exists(new):
+                os.unlink(new) 
+            self.move_file(old, new)
+        return new_outfiles, updated_files
+            
+
 #---------------------------------------------------------------------------
 # Verify all dependencies
 #---------------------------------------------------------------------------