Massive package and module reorganization of all of IPython.

* New top-level sub-packages have been created: :mod:`IPython.core`, 
  :mod:`IPython.lib`, :mod:`IPython.utils`, :mod:`IPython.deathrow`,
  :mod:`IPython.quarantine`.  All existing top-level modules have been
  moved to appropriate sub-packages.  All internal import statements
  have been updated and tests have been added.  The build system (setup.py
  and friends) have been updated.
* Compatability modules have been created for :mod:`IPython.Shell`, 
  :mod:`IPython.ipapi` and :mod:`IPython.iplib` that display warnings
  and then load the actual implementation from :mod:`IPython.core`.
* :mod:`Extensions` has been moved to :mod:`extensions`.

IPython/testing/iptest.py

@@ -31,7 +31,7 @@ import warnings
 import nose.plugins.builtin
 from nose.core import TestProgram
 
-from IPython.platutils import find_cmd
+from IPython.utils.platutils import find_cmd
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 
 pjoin = path.join
@@ -58,75 +58,85 @@ have_foolscap = test_for('foolscap')
 have_objc = test_for('objc')
 have_pexpect = test_for('pexpect')
 
-# For the IPythonDoctest plugin, we need to exclude certain patterns that cause
-# testing problems.  We should strive to minimize the number of skipped
-# modules, since this means untested code.  As the testing machinery
-# solidifies, this list should eventually become empty.
-EXCLUDE = [pjoin('IPython', 'external'),
-           pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
-           pjoin('IPython_doctest_plugin'),
-           pjoin('IPython', 'Gnuplot'),
-           pjoin('IPython', 'Extensions', 'ipy_'),
-           pjoin('IPython', 'Extensions', 'PhysicalQInput'),
-           pjoin('IPython', 'Extensions', 'PhysicalQInteractive'),
-           pjoin('IPython', 'Extensions', 'InterpreterPasteInput'),
-           pjoin('IPython', 'Extensions', 'scitedirector'),
-           pjoin('IPython', 'Extensions', 'numeric_formats'),
-           pjoin('IPython', 'testing', 'attic'),
-           pjoin('IPython', 'testing', 'tutils'),
-           pjoin('IPython', 'testing', 'tools'),
-           pjoin('IPython', 'testing', 'mkdoctests'),
-           ]
-
-if not have_wx:
-    EXCLUDE.append(pjoin('IPython', 'Extensions', 'igrid'))
-    EXCLUDE.append(pjoin('IPython', 'gui'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
-
-if not have_wx_aui:
-    EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
-
-if not have_objc:
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
-
-if not have_curses:
-    EXCLUDE.append(pjoin('IPython', 'Extensions', 'ibrowse'))
-
-if not sys.platform == 'win32':
-    EXCLUDE.append(pjoin('IPython', 'platutils_win32'))
-
-# These have to be skipped on win32 because the use echo, rm, cd, etc.
-# See ticket https://bugs.launchpad.net/bugs/366982
-if sys.platform == 'win32':
-    EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
-    EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
-
-if not os.name == 'posix':
-    EXCLUDE.append(pjoin('IPython', 'platutils_posix'))
-
-if not have_pexpect:
-    EXCLUDE.append(pjoin('IPython', 'irunner'))
-
-if not have_twisted:
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_linefrontend'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 'test_frontendbase'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
-                         'test_prefilterfrontend'))
-    EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
-                         'test_asyncfrontendbase'))
-    EXCLUDE.append(pjoin('IPython', 'kernel', 'error'))
-    EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
-    EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
-    EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 'test_decorators_trial'))
-
-
-# This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
-if sys.platform == 'win32':
-    EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+
+def make_exclude():
+
+    # For the IPythonDoctest plugin, we need to exclude certain patterns that cause
+    # testing problems.  We should strive to minimize the number of skipped
+    # modules, since this means untested code.  As the testing machinery
+    # solidifies, this list should eventually become empty.
+    EXCLUDE = [pjoin('IPython', 'external'),
+               pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
+               pjoin('IPython_doctest_plugin'),
+               pjoin('IPython', 'extensions', 'ipy_'),
+               pjoin('IPython', 'extensions', 'PhysicalQInput'),
+               pjoin('IPython', 'extensions', 'PhysicalQInteractive'),
+               pjoin('IPython', 'extensions', 'InterpreterPasteInput'),
+               pjoin('IPython', 'extensions', 'scitedirector'),
+               pjoin('IPython', 'extensions', 'numeric_formats'),
+               pjoin('IPython', 'testing', 'attic'),
+               pjoin('IPython', 'testing', 'tools'),
+               pjoin('IPython', 'testing', 'mkdoctests')
+               ]
+
+    if not have_wx:
+        EXCLUDE.append(pjoin('IPython', 'extensions', 'igrid'))
+        EXCLUDE.append(pjoin('IPython', 'gui'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'wx'))
+
+    if not have_wx_aui:
+        EXCLUDE.append(pjoin('IPython', 'gui', 'wx', 'wxIPython'))
+
+    if not have_objc:
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'cocoa'))
+
+    if not have_curses:
+        EXCLUDE.append(pjoin('IPython', 'extensions', 'ibrowse'))
+
+    if not sys.platform == 'win32':
+        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_win32'))
+
+    # These have to be skipped on win32 because the use echo, rm, cd, etc.
+    # See ticket https://bugs.launchpad.net/bugs/366982
+    if sys.platform == 'win32':
+        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'test_exampleip'))
+        EXCLUDE.append(pjoin('IPython', 'testing', 'plugin', 'dtexample'))
+
+    if not os.name == 'posix':
+        EXCLUDE.append(pjoin('IPython', 'utils', 'platutils_posix'))
+
+    if not have_pexpect:
+        EXCLUDE.append(pjoin('IPython', 'scripts', 'irunner'))
+
+    # This is scary.  We still have things in frontend and testing that
+    # are being tested by nose that use twisted.  We need to rethink
+    # how we are isolating dependencies in testing.
+    if not (have_twisted and have_zi and have_foolscap):
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'asyncfrontendbase'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'prefilterfrontend'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'frontendbase'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'linefrontendbase'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+                             'test_linefrontend'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests', 
+                             'test_frontendbase'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+                             'test_prefilterfrontend'))
+        EXCLUDE.append(pjoin('IPython', 'frontend', 'tests',
+                             'test_asyncfrontendbase')),
+        EXCLUDE.append(pjoin('IPython', 'testing', 'parametric'))
+        EXCLUDE.append(pjoin('IPython', 'testing', 'util'))
+        EXCLUDE.append(pjoin('IPython', 'testing', 'tests', 
+                             'test_decorators_trial'))
+
+    # Skip shell always because of a bug in FakeModule.
+    EXCLUDE.append(pjoin('IPython', 'core', 'shell'))
+
+    # This is needed for the reg-exp to match on win32 in the ipdoctest plugin.
+    if sys.platform == 'win32':
+        EXCLUDE = [s.replace('\\','\\\\') for s in EXCLUDE]
+
+    return EXCLUDE
 
 
 #-----------------------------------------------------------------------------
@@ -178,13 +188,12 @@ def run_iptest():
 
     # Construct list of plugins, omitting the existing doctest plugin, which
     # ours replaces (and extends).
+    EXCLUDE = make_exclude()
     plugins = [IPythonDoctest(EXCLUDE)]
     for p in nose.plugins.builtin.plugins:
         plug = p()
         if plug.name == 'doctest':
             continue
-
-        #print '*** adding plugin:',plug.name  # dbg
         plugins.append(plug)
 
     TestProgram(argv=argv,plugins=plugins)
@@ -234,58 +243,28 @@ class IPTester(object):
 
 
 def make_runners():
-    """Define the modules and packages that need to be tested.
+    """Define the top-level packages that need to be tested.
     """
-    
-    # This omits additional top-level modules that should not be doctested.
-    # XXX: Shell.py is also ommited because of a bug in the skip_doctest
-    # decorator.  See ticket https://bugs.launchpad.net/bugs/366209
-    top_mod = \
-      ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
-       'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
-       'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
-       'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
-       'iplib.py', 'ipmaker.py', 'ipstruct.py', 'Itpl.py',
-       'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
-       'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
-       'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
-       'shellglobals.py', 'strdispatch.py', 
-       'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
-       # See note above for why this is skipped
-       # 'Shell.py',
-       'winconsole.py']
-
-    if have_twisted:
-        top_mod.append('twshell.py')
-
-    if have_pexpect:
-        top_mod.append('irunner.py')
 
-    if sys.platform == 'win32':
-        top_mod.append('platutils_win32.py')
-    elif os.name == 'posix':
-        top_mod.append('platutils_posix.py')
-    else:
-        top_mod.append('platutils_dummy.py')
-
-    # These are tested by nose, so skip IPython.kernel
-    top_pack = ['config','Extensions','frontend',
-                'testing','tests','tools','UserConfig']
+    nose_packages = ['config', 'core', 'extensions',
+                     'frontend', 'lib', 'quarantine',
+                     'scripts', 'testing', 'utils']
+    trial_packages = ['kernel']
 
     if have_wx:
-        top_pack.append('gui')
+        nose_packages.append('gui')
 
-    modules  = ['IPython.%s' % m[:-3] for m in top_mod ]
-    packages = ['IPython.%s' % m for m in top_pack ]
+    nose_packages = ['IPython.%s' % m for m in nose_packages ]
+    trial_packages = ['IPython.%s' % m for m in trial_packages ]
 
     # Make runners
-    runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))
+    runners = dict()
     
-    # Test IPython.kernel using trial if twisted is installed
+    nose_runners = dict(zip(nose_packages, [IPTester(params=v) for v in nose_packages]))
     if have_zi and have_twisted and have_foolscap:
-        runners['trial'] = IPTester('trial',['IPython'])
-
-    runners['modules'] = IPTester(params=modules)
+        trial_runners = dict(zip(trial_packages, [IPTester('trial',params=v) for v in trial_packages]))
+    runners.update(nose_runners)
+    runners.update(trial_runners)
 
     return runners
 
@@ -298,7 +277,9 @@ def run_iptestall():
     and packages of IPython to be tested each in their own subprocess using
     nose or twisted.trial appropriately.
     """
+
     runners = make_runners()
+
     # Run all test runners, tracking execution time
     failed = {}
     t_start = time.time()

setup.py

@@ -29,8 +29,7 @@ if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
-# Local imports
-from IPython.genutils import target_update
+from IPython.utils.genutils import target_update
 
 from setupbase import (
     setup_args, 
@@ -42,6 +41,7 @@ from setupbase import (
 )
 
 isfile = os.path.isfile
+pjoin = os.path.join
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
@@ -123,7 +123,7 @@ if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
 
         # First, compute all the dependencies that can force us to rebuild the
         # docs.  Start with the main release file that contains metadata
-        docdeps = ['IPython/Release.py']
+        docdeps = ['IPython/core/release.py']
         # Inculde all the reST sources
         pjoin = os.path.join
         for dirpath,dirnames,filenames in os.walk('docs/source'):
@@ -167,13 +167,14 @@ if 'setuptools' in sys.modules:
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': [
-            'ipython = IPython.ipapi:launch_new_instance',
-            'pycolor = IPython.PyColorize:main',
+            'ipython = IPython.core.ipapi:launch_new_instance',
+            'pycolor = IPython.utils.PyColorize:main',
             'ipcontroller = IPython.kernel.scripts.ipcontroller:main',
             'ipengine = IPython.kernel.scripts.ipengine:main',
             'ipcluster = IPython.kernel.scripts.ipcluster:main',
             'ipythonx = IPython.frontend.wx.ipythonx:main',
             'iptest = IPython.testing.iptest:main',
+            'irunner = IPython.lib.irunner:main'
         ]
     }
     setup_args['extras_require'] = dict(
@@ -190,9 +191,9 @@ if 'setuptools' in sys.modules:
     scripts = []
 else:
     # package_data of setuptools was introduced to distutils in 2.4
-    cfgfiles = filter(isfile, glob('IPython/UserConfig/*'))
+    cfgfiles = filter(isfile, glob(pjoin('IPython','config','userconfig')))
     if sys.version_info < (2,4):
-        data_files.append(('lib', 'IPython/UserConfig', cfgfiles))
+        data_files.append(('lib', pjoin('IPython','config','userconfig'), cfgfiles))
     # If we are running without setuptools, call this function which will
     # check for dependencies an inform the user what is needed.  This is
     # just to make life easy for users.

setupbase.py

@@ -57,8 +57,8 @@ def file_doesnt_endwith(test,endings):
 # Basic project information
 #---------------------------------------------------------------------------
 
-# Release.py contains version, authors, license, url, keywords, etc.
-execfile(pjoin('IPython','Release.py'))
+# release.py contains version, authors, license, url, keywords, etc.
+execfile(pjoin('IPython','core','release.py'))
 
 # Create a dict with the basic information
 # This dict is eventually passed to setup after additional keys are added.
@@ -104,21 +104,30 @@ def find_packages():
     """
     packages = ['IPython']
     add_package(packages, 'config', tests=True)
-    add_package(packages , 'Extensions')
+    add_package(packages, 'config.userconfig')
+    add_package(packages, 'core', tests=True)
+    add_package(packages, 'deathrow', tests=True)
+    add_package(packages , 'extensions')
     add_package(packages, 'external')
-    add_package(packages, 'gui')
-    add_package(packages, 'gui.wx')
     add_package(packages, 'frontend', tests=True)
+    # Don't include the cocoa frontend for now as it is not stable
+    if sys.platform == 'darwin' and False:
+        add_package(packages, 'frontend.cocoa', tests=True, others=['plugin'])
+        add_package(packages, 'frontend.cocoa.examples')
+        add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox')
+        add_package(packages, 'frontend.cocoa.examples.IPython1Sandbox.English.lproj')
     add_package(packages, 'frontend.process')
     add_package(packages, 'frontend.wx')
-    add_package(packages, 'frontend.cocoa', tests=True)
+    add_package(packages, 'gui')
+    add_package(packages, 'gui.wx')
     add_package(packages, 'kernel', config=True, tests=True, scripts=True)
     add_package(packages, 'kernel.core', config=True, tests=True)
+    add_package(packages, 'lib', tests=True)
+    add_package(packages, 'quarantine', tests=True)
+    add_package(packages, 'scripts')
     add_package(packages, 'testing', tests=True)
-    add_package(packages, 'tests')
     add_package(packages, 'testing.plugin', tests=False)
-    add_package(packages, 'tools', tests=True)
-    add_package(packages, 'UserConfig')
+    add_package(packages, 'utils', tests=True)
     return packages
 
 #---------------------------------------------------------------------------
@@ -132,8 +141,7 @@ def find_package_data():
     # This is not enough for these things to appear in an sdist.
     # We need to muck with the MANIFEST to get this to work
     package_data = {
-        'IPython.UserConfig' : ['*'],
-        'IPython.tools.tests' : ['*.txt'],
+        'IPython.config.userconfig' : ['*'],
         'IPython.testing' : ['*.txt']
     }
     return package_data
@@ -187,17 +195,24 @@ def find_data_files():
     Most of these are docs.
     """
     
-    docdirbase  = 'share/doc/ipython'
-    manpagebase = 'share/man/man1'
-
+    docdirbase  = pjoin('share', 'doc', 'ipython')
+    manpagebase = pjoin('share', 'man', 'man1')
+    
     # Simple file lists can be made by hand
-    manpages  = filter(isfile, glob('docs/man/*.1.gz'))
-    igridhelpfiles = filter(isfile, glob('IPython/Extensions/igrid_help.*'))
+    manpages  = filter(isfile, glob(pjoin('docs','man','*.1.gz')))
+    igridhelpfiles = filter(isfile, glob(pjoin('IPython','extensions','igrid_help.*')))
 
     # For nested structures, use the utility above
-    example_files = make_dir_struct('data','docs/examples',
-                                    pjoin(docdirbase,'examples'))
-    manual_files = make_dir_struct('data','docs/dist',pjoin(docdirbase,'manual'))
+    example_files = make_dir_struct(
+        'data',
+        pjoin('docs','examples'),
+        pjoin(docdirbase,'examples')
+    )
+    manual_files = make_dir_struct(
+        'data',
+        pjoin('docs','dist'),
+        pjoin(docdirbase,'manual')
+    )
 
     # And assemble the entire output list
     data_files = [ ('data',manpagebase, manpages),
@@ -245,16 +260,18 @@ def find_scripts():
     """
     Find IPython's scripts.
     """
-    scripts = ['IPython/kernel/scripts/ipengine',
-               'IPython/kernel/scripts/ipcontroller',
-               'IPython/kernel/scripts/ipcluster',
-               'scripts/ipython',
-               'scripts/ipythonx',
-               'scripts/ipython-wx',
-               'scripts/pycolor',
-               'scripts/irunner',
-               'scripts/iptest',
-               ]
+    kernel_scripts = pjoin('IPython','kernel','scripts')
+    main_scripts = pjoin('IPython','scripts')
+    scripts = [pjoin(kernel_scripts, 'ipengine'),
+               pjoin(kernel_scripts, 'ipcontroller'),
+               pjoin(kernel_scripts, 'ipcluster'),
+               pjoin(main_scripts, 'ipython'),
+               pjoin(main_scripts, 'ipythonx'),
+               pjoin(main_scripts, 'ipython-wx'),
+               pjoin(main_scripts, 'pycolor'),
+               pjoin(main_scripts, 'irunner'),
+               pjoin(main_scripts, 'iptest')
+              ]
     
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
@@ -264,7 +281,7 @@ def find_scripts():
         if len(sys.argv) > 2 and ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print >> sys.stderr,"ERROR: bdist_wininst must be run alone. Exiting."
             sys.exit(1)
-        scripts.append('scripts/ipython_win_post_install.py')
+        scripts.append(pjoin('scripts','ipython_win_post_install.py'))
     
     return scripts