ensure submodules exist prior to doing anything

git submodule update fires before distutils gets fired up
because distutils cannot be trusted.

setup.py

@@ -67,7 +67,11 @@ from setupbase import (
     find_scripts,
     find_data_files,
     check_for_dependencies,
-    record_commit_info,
+    git_prebuild,
+    check_for_submodules,
+    update_submodules,
+    require_submodules,
+    UpdateSubmodules,
 )
 from setupext import setupext
 
@@ -105,6 +109,25 @@ if os_name == 'windows' and 'sdist' in sys.argv:
     print('The sdist command is not available under Windows.  Exiting.')
     sys.exit(1)
 
+#-------------------------------------------------------------------------------
+# Make sure we aren't trying to run without submodules
+#-------------------------------------------------------------------------------
+
+def ensure_submodules_exist():
+    """Check out git submodules before distutils can do anything
+    
+    Because distutils cannot be trusted to update the tree
+    after everything has been set in motion.
+    """
+    # don't do anything if nothing is actually supposed to happen
+    for do_nothing in ('-h', '--help', '--help-commands', 'clean'):
+        if do_nothing in sys.argv:
+            return
+    if not check_for_submodules():
+        update_submodules()
+
+ensure_submodules_exist()
+
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
@@ -193,9 +216,10 @@ class UploadWindowsInstallers(upload):
             self.upload_file('bdist_wininst', 'any', dist_file)
 
 setup_args['cmdclass'] = {
-    'build_py': record_commit_info('IPython'),
-    'sdist' : record_commit_info('IPython', sdist),
+    'build_py': git_prebuild('IPython'),
+    'sdist' : git_prebuild('IPython', sdist),
     'upload_wininst' : UploadWindowsInstallers,
+    'submodule' : UpdateSubmodules,
 }
 
 #---------------------------------------------------------------------------
@@ -223,6 +247,10 @@ if len(needs_setuptools.intersection(sys.argv)) > 0:
 setuptools_extra_args = {}
 
 if 'setuptools' in sys.modules:
+    # setup.py develop should check for submodules
+    from setuptools.command.develop import develop
+    setup_args['cmdclass']['develop'] = require_submodules(develop)
+    
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = find_scripts(True)
     setup_args['extras_require'] = dict(
@@ -277,7 +305,7 @@ if 'setuptools' in sys.modules:
                 'lib2to3.fixes.fix_tuple_params',
                 ]
         from setuptools.command.build_py import build_py
-        setup_args['cmdclass'] = {'build_py': record_commit_info('IPython', build_cmd=build_py)}
+        setup_args['cmdclass'] = {'build_py': git_prebuild('IPython', build_cmd=build_py)}
         setuptools_extra_args['entry_points'] = find_scripts(True, suffix='3')
         setuptools._dont_write_bytecode = True
 else:

setupbase.py

@@ -28,6 +28,7 @@ try:
 except:
     from ConfigParser import ConfigParser
 from distutils.command.build_py import build_py
+from distutils.cmd import Command
 from glob import glob
 
 from setupext import install_data_ext
@@ -368,17 +369,79 @@ def check_for_dependencies():
     check_for_pyzmq()
     check_for_readline()
 
-def record_commit_info(pkg_dir, build_cmd=build_py):
-    """ Return extended build or sdist command class for recording commit
+#---------------------------------------------------------------------------
+# VCS related
+#---------------------------------------------------------------------------
+
+def check_for_submodules():
+    """return False if there are any submodules that need to be checked out,
+    True otherwise.
+    
+    This doesn't check if they are up to date, only existence.
+    """
+    here = os.path.dirname(__file__)
+    submodules = [
+        os.path.join(here, 'IPython', 'frontend', 'html', 'notebook', 'static', 'external')
+    ]
+    for submodule in submodules:
+        if not os.path.exists(submodule):
+            return False
+    return True
+
+def update_submodules():
+    """update git submodules"""
+    import subprocess
+    print("updating git submodules")
+    subprocess.check_call('git submodule init'.split())
+    subprocess.check_call('git submodule update --recursive'.split())
+
+class UpdateSubmodules(Command):
+    """Update git submodules
+    
+    IPython's external javascript dependencies live in a separate repo.
+    """
+    description = "Update git submodules"
+    user_options = []
+    
+    def initialize_options(self):
+        pass
+    
+    def finalize_options(self):
+        pass
+    
+    def run(self):
+        failure = False
+        try:
+            self.spawn('git submodule init'.split())
+            self.spawn('git submodule update --recursive'.split())
+        except Exception as e:
+            failure = e
+            print(e)
+        
+        if not check_for_submodules():
+            print("submodules could not be checked out")
+            sys.exit(1)
+        
+        # re-scan package data after update
+        self.distribution.package_data = find_package_data()
+
+def git_prebuild(pkg_dir, build_cmd=build_py):
+    """Return extended build or sdist command class for recording commit
     
     records git commit in IPython.utils._sysinfo.commit
     
     for use in IPython.utils.sysinfo.sys_info() calls after installation.
+    
+    Also ensures that submodules exist prior to running
     """
     
     class MyBuildPy(build_cmd):
         ''' Subclass to write commit data into installation tree '''
         def run(self):
+            if not check_for_submodules():
+                print("submodules missing! Run `setup.py submodule` and try again")
+                sys.exit(1)
+            
             build_cmd.run(self)
             # this one will only fire for build commands
             if hasattr(self, 'build_lib'):
@@ -416,3 +479,14 @@ def record_commit_info(pkg_dir, build_cmd=build_py):
                     'commit = "%s"\n' % repo_commit,
                 ])
     return MyBuildPy
+
+
+def require_submodules(command):
+    """decorator for instructing a command to check for submodules before running"""
+    class DecoratedCommand(command):
+        def run(self):
+            if not check_for_submodules():
+                print("submodules missing! Run `setup.py submodule` and try again")
+                sys.exit(1)
+            command.run(self)
+    return DecoratedCommand