restore r.js-based build system from 4.x

bower.json

@@ -6,6 +6,7 @@
     "bootstrap": "components/bootstrap#~3.3",
     "bootstrap-tour": "0.9.0",
     "codemirror": "components/codemirror#~5.16",
+    "es6-promise": "~1.0",
     "font-awesome": "components/font-awesome#~4.2.0",
     "google-caja": "5669",
     "jquery": "components/jquery#~2.0",
@@ -13,8 +14,10 @@
     "jquery-ui": "components/jqueryui#~1.10",
     "marked": "~0.3",
     "MathJax": "components/MathJax#~2.6",
+    "moment": "~2.8.4",
     "requirejs": "~2.1",
     "text-encoding": "~0.1",
-    "underscore": "components/underscore#~1.8.3"
+    "underscore": "components/underscore#~1.8.3",
+    "xterm.js": "sourcelair/xterm.js#~2.1.0"
   }
 }

package.json

@@ -9,38 +9,12 @@
     "url": "https://github.com/jupyter/notebook.git"
   },
   "scripts": {
-    "lint": "eslint --quiet notebook/",
-    "bower": "bower install --allow-root --config.interactive=false",
-    "build:watch": "concurrent \"npm run build:css:watch\" \"npm run build:js:watch\"",
-    "build": "npm run build:js && npm run build:css",
-    "build:css": "concurrent \"npm run build:css:ipython\" \"npm run build:css:style\"",
-    "build:css:ipython": "lessc --include-path=notebook/static notebook/static/style/ipython.less notebook/static/style/ipython.min.css",
-    "build:css:style": "lessc --include-path=notebook/static notebook/static/style/style.less notebook/static/style/style.min.css",
-    "build:css:watch": "./scripts/less-watch ./notebook/static",
-    "build:js": "webpack",
-    "build:js:watch": "npm run build:js -- --watch"
+    "bower": "bower install",
+    "build": "python setup.py js css"
   },
   "devDependencies": {
-    "babel-cli": "^6.7.5",
-    "babel-core": "^6.7.4",
-    "babel-loader": "^6.2.4",
-    "babel-preset-es2015": "^6.6.0",
     "bower": "*",
-    "concurrently": "^1.0.0",
-    "css-loader": "^0.23.1",
-    "eslint": "^2.8.0",
-    "json-loader": "^0.5.4",
     "less": "~2",
-    "requirejs": "^2.1.17",
-    "style-loader": "^0.13.1",
-    "underscore": "^1.8.3",
-    "webpack": "^1.12.13"
-  },
-  "dependencies": {
-    "es6-promise": "^4.0.5",
-    "moment": "^2.8.4",
-    "preact": "^4.5.1",
-    "preact-compat": "^1.7.0",
-    "xterm": "^2.1.0"
+    "requirejs": "^2.1.17"
   }
 }

setup.py

@@ -55,7 +55,7 @@ from setupbase import (
     check_package_data_first,
     CompileCSS,
     CompileJS,
-    JavascriptDependencies,
+    Bower,
     JavascriptVersion,
     css_js_prerelease,
 )
@@ -119,7 +119,7 @@ setup_args['cmdclass'] = {
     'sdist' : css_js_prerelease(sdist, strict=True),
     'css' : CompileCSS,
     'js' : CompileJS,
-    'jsdeps' : JavascriptDependencies,
+    'jsdeps' : Bower,
     'jsversion' : JavascriptVersion,
 }
 

setupbase.py

@@ -19,10 +19,10 @@ import sys
 import pipes
 from distutils import log
 from distutils.cmd import Command
-from distutils.version import LooseVersion
 from fnmatch import fnmatch
 from glob import glob
-from subprocess import check_call, check_output
+from multiprocessing.pool import ThreadPool
+from subprocess import check_call
 
 if sys.platform == 'win32':
     from subprocess import list2cmdline
@@ -119,14 +119,8 @@ def find_package_data():
     # for verification purposes, explicitly add main.min.js
     # so that installation will fail if they are missing
     for app in ['auth', 'edit', 'notebook', 'terminal', 'tree']:
-        static_data.extend([
-            pjoin('static', app, 'js', 'built', '*main.min.js'),
-        ])
-    static_data.extend([
-        pjoin('static', 'built', '*index.js'),
-        pjoin('static', 'services', 'built', '*contents.js'),
-    ])
-
+        static_data.append(pjoin('static', app, 'js', 'main.min.js'))
+    
     components = pjoin("static", "components")
     # select the components we actually need to install
     # (there are lots of resources we bundle for sdist-reasons that we don't actually use)
@@ -136,6 +130,7 @@ def find_package_data():
         pjoin(components, "bootstrap-tour", "build", "css", "bootstrap-tour.min.css"),
         pjoin(components, "bootstrap-tour", "build", "js", "bootstrap-tour.min.js"),
         pjoin(components, "font-awesome", "css", "*.css"),
+        pjoin(components, "es6-promise", "*.js"),
         pjoin(components, "font-awesome", "fonts", "*.*"),
         pjoin(components, "google-caja", "html-css-sanitizer-minified.js"),
         pjoin(components, "jquery", "jquery.min.js"),
@@ -147,6 +142,10 @@ def find_package_data():
         pjoin(components, "marked", "lib", "marked.js"),
         pjoin(components, "requirejs", "require.js"),
         pjoin(components, "underscore", "underscore-min.js"),
+        pjoin(components, "moment", "moment.js"),
+        pjoin(components, "moment", "min", "moment.min.js"),
+        pjoin(components, "xterm.js", "dist", "xterm.js"),
+        pjoin(components, "xterm.js", "dist", "xterm.css"),
         pjoin(components, "text-encoding", "lib", "encoding.js"),
     ])
 
@@ -322,49 +321,61 @@ def run(cmd, *args, **kwargs):
     return check_call(cmd, *args, **kwargs)
 
 
-def npm_install(cwd):
-    """Run npm install in a directory and dedupe if necessary"""
-
-    try:
-        run(['npm', 'install', '--progress=false'], cwd=cwd)
-    except OSError as e:
-        print("Failed to run `npm install`: %s" % e, file=sys.stderr)
-        print("npm is required to build a development version of the notebook.", file=sys.stderr)
-        raise
-
-    shell = (sys.platform == 'win32')
-    version = check_output(['npm', '--version'], shell=shell).decode('utf-8')
-    if LooseVersion(version) < LooseVersion('3.0'):
-        try:
-            run(['npm', 'dedupe'], cwd=cwd)
-        except Exception as e:
-            print("Failed to run `npm dedupe`: %s" % e, file=sys.stderr)
-            print("Please install npm v3+ to build a development version of the notebook.")
-            raise
-
-
-class JavascriptDependencies(Command):
-    description = "Fetch Javascript dependencies with npm and bower"
+class Bower(Command):
+    description = "fetch static client-side components with bower"
+    
+    user_options = [
+        ('force', 'f', "force fetching of bower dependencies"),
+    ]
     
     def initialize_options(self):
-        pass
+        self.force = False
     
     def finalize_options(self):
-        pass
+        self.force = bool(self.force)
     
     bower_dir = pjoin(static, 'components')
     node_modules = pjoin(repo_root, 'node_modules')
     
-    def run(self):
-        npm_install(repo_root)
+    def should_run(self):
+        if self.force:
+            return True
+        if not os.path.exists(self.bower_dir):
+            return True
+        return mtime(self.bower_dir) < mtime(pjoin(repo_root, 'bower.json'))
 
+    def should_run_npm(self):
+        if not which('npm'):
+            print("npm unavailable", file=sys.stderr)
+            return False
+        if not os.path.exists(self.node_modules):
+            return True
+        return mtime(self.node_modules) < mtime(pjoin(repo_root, 'package.json'))
+    
+    def run(self):
+        if not self.should_run():
+            print("bower dependencies up to date")
+            return
+        
+        if self.should_run_npm():
+            print("installing build dependencies with npm")
+            run(['npm', 'install'], cwd=repo_root)
+            os.utime(self.node_modules, None)
+        
+        env = os.environ.copy()
+        env['PATH'] = npm_path
+        
         try:
-            run(['npm', 'run', 'bower'], cwd=repo_root)
-        except Exception as e:
-            print("Failed to run `npm run bower`: %s" % e, file=sys.stderr)
+            run(
+                ['bower', 'install', '--allow-root', '--config.interactive=false'],
+                cwd=repo_root,
+                env=env
+            )
+        except OSError as e:
+            print("Failed to run bower: %s" % e, file=sys.stderr)
             print("You can install js dependencies with `npm install`", file=sys.stderr)
             raise
-
+        os.utime(self.bower_dir, None)
         # update package data in case this created new files
         update_package_data(self.distribution)
 
@@ -385,17 +396,29 @@ class CompileCSS(Command):
     def finalize_options(self):
         pass
 
+    sources = []
     targets = []
     for name in ('ipython', 'style'):
+        sources.append(pjoin(static, 'style', '%s.less' % name))
         targets.append(pjoin(static, 'style', '%s.min.css' % name))
 
     def run(self):
-        try:
-            run(['npm', 'run', 'build:css'])
-        except OSError as e:
-            print("Failed to run npm run build:css : %s" % e, file=sys.stderr)
-            print("You can install js dependencies with `npm install`", file=sys.stderr)
-            raise
+        self.run_command('jsdeps')
+        env = os.environ.copy()
+        env['PATH'] = npm_path
+        
+        for src, dst in zip(self.sources, self.targets):
+            try:
+                run(['lessc',
+                    '--source-map',
+                    '--include-path=%s' % pipes.quote(static),
+                    src,
+                    dst,
+                ], cwd=repo_root, env=env)
+            except OSError as e:
+                print("Failed to build css: %s" % e, file=sys.stderr)
+                print("You can install js dependencies with `npm install`", file=sys.stderr)
+                raise
         # update package data in case this created new files
         update_package_data(self.distribution)
 
@@ -416,39 +439,55 @@ class CompileJS(Command):
     def finalize_options(self):
         self.force = bool(self.force)
 
-    target = pjoin(static, 'built', 'index.js')
-    targets = [target]
+    apps = ['notebook', 'tree', 'edit', 'terminal', 'auth']
+    targets = [ pjoin(static, app, 'js', 'main.min.js') for app in apps ]
     
-    def sources(self):
+    def sources(self, name):
         """Generator yielding .js sources that an application depends on"""
-        yield pjoin(repo_root, 'package.json')
-        yield pjoin(repo_root, 'webpack.config.js')
-        for parent, dirs, files in os.walk(static):
-            if os.path.basename(parent) in {'MathJax', 'built'}:
+        yield pjoin(static, name, 'js', 'main.js')
+
+        for sec in [name, 'base', 'auth']:
+            for f in glob(pjoin(static, sec, 'js', '*.js')):
+                if not f.endswith('.min.js'):
+                    yield f
+        yield pjoin(static, 'services', 'config.js')
+        if name == 'notebook':
+            for f in glob(pjoin(static, 'services', '*', '*.js')):
+                yield f
+        for parent, dirs, files in os.walk(pjoin(static, 'components')):
+            if os.path.basename(parent) == 'MathJax':
                 # don't look in MathJax, since it takes forever to walk it
-                # also don't look at build targets as sources
                 dirs[:] = []
                 continue
             for f in files:
-                if not f.endswith('.js'):
-                    continue
                 yield pjoin(parent, f)
     
-    def should_run(self):
-        if self.force or not os.path.exists(self.target):
-            print("Missing %s" % self.target)
+    def should_run(self, name, target):
+        if self.force or not os.path.exists(target):
             return True
-        target_mtime = mtime(self.target)
-        for source in self.sources():
+        target_mtime = mtime(target)
+        for source in self.sources(name):
             if mtime(source) > target_mtime:
-                print('%s > %s' % (source, self.target))
+                print(source, target)
                 return True
         return False
+        
+    def build_main(self, name):
+        """Build main.min.js"""
+        target = pjoin(static, name, 'js', 'main.min.js')
+        
+        if not self.should_run(name, target):
+            log.info("%s up to date" % target)
+            return
+        log.info("Rebuilding %s" % target)
+        run(['node', 'tools/build-main.js', name])
 
     def run(self):
         self.run_command('jsdeps')
-        if self.should_run():
-            run(['npm', 'run', 'build:js'])
+        env = os.environ.copy()
+        env['PATH'] = npm_path
+        pool = ThreadPool()
+        pool.map(self.build_main, self.apps)
         # update package data in case this created new files
         update_package_data(self.distribution)
 
@@ -466,27 +505,17 @@ class JavascriptVersion(Command):
     
     def run(self):
         nsfile = pjoin(repo_root, "notebook", "static", "base", "js", "namespace.js")
-        lines = []
-        found = False
         with open(nsfile) as f:
-            for line in f.readlines():
-                if line.strip().startswith("Jupyter.version"):
-                    found = True
-                    new_line = '    Jupyter.version = "{0}";\n'.format(version)
-                    if new_line == line:
-                        # no change, don't rewrite file
-                        return
-                    lines.append(new_line)
-                else:
-                    lines.append(line)
-
-        if not found:
-            raise RuntimeError("Didn't find Jupyter.version line in %s" % nsfile)
-        
-        print("Writing version=%s to %s" % (version, nsfile))
+            lines = f.readlines()
         with open(nsfile, 'w') as f:
+            found = False
             for line in lines:
+                if line.strip().startswith("Jupyter.version"):
+                    line = '    Jupyter.version = "{0}";\n'.format(version)
+                    found = True
                 f.write(line)
+            if not found:
+                raise RuntimeError("Didn't find Jupyter.version line in %s" % nsfile)
 
 
 def css_js_prerelease(command, strict=False):
@@ -497,8 +526,7 @@ def css_js_prerelease(command, strict=False):
             jsdeps = self.distribution.get_command_obj('jsdeps')
             js = self.distribution.get_command_obj('js')
             css = self.distribution.get_command_obj('css')
-
-            js.force = strict
+            jsdeps.force = js.force = strict
 
             targets = [ jsdeps.bower_dir ]
             targets.extend(js.targets)

tools/build-main.js

@@ -0,0 +1,68 @@
+// build main.min.js
+// spawned by gulp to allow parallelism
+
+var rjs = require('requirejs').optimize;
+
+var name = process.argv[2];
+
+var rjs_config = {
+  name: name + '/js/main',
+  out: './notebook/static/' + name + '/js/main.min.js',
+  baseUrl: 'notebook/static',
+  preserveLicenseComments: false, // license comments conflict with sourcemap generation
+  generateSourceMaps: true,
+  optimize: "none",
+  paths: {
+    underscore : 'components/underscore/underscore-min',
+    backbone : 'components/backbone/backbone-min',
+    jquery: 'components/jquery/jquery.min',
+    bootstrap: 'components/bootstrap/js/bootstrap.min',
+    bootstraptour: 'components/bootstrap-tour/build/js/bootstrap-tour.min',
+    "jquery-ui": 'components/jquery-ui/ui/minified/jquery-ui.min',
+    moment: 'components/moment/moment',
+    codemirror: 'components/codemirror',
+    xterm: 'components/xterm.js/dist/xterm',
+    typeahead: 'components/jquery-typeahead/dist/jquery.typeahead',
+    contents: 'empty:',
+    custom: 'empty:',
+  },
+  map: { // for backward compatibility
+    "*": {
+        "jqueryui": "jquery-ui",
+    }
+  },
+  shim: {
+    typeahead: {
+            deps: ["jquery"],
+            exports: "typeahead"
+          },
+    underscore: {
+      exports: '_'
+    },
+    backbone: {
+      deps: ["underscore", "jquery"],
+      exports: "Backbone"
+    },
+    bootstrap: {
+      deps: ["jquery"],
+      exports: "bootstrap"
+    },
+    bootstraptour: {
+      deps: ["bootstrap"],
+      exports: "Tour"
+    },
+    "jquery-ui": {
+      deps: ["jquery"],
+      exports: "$"
+    }
+  },
+
+  exclude: [
+    "custom/custom",
+  ]
+};
+
+rjs(rjs_config, console.log, function (err) {
+  console.log("Failed to build", name, err);
+  process.exit(1);
+});

webpack.config.js

@@ -1,93 +0,0 @@
-// Copyright (c) Jupyter Development Team.
-// Distributed under the terms of the Modified BSD License.
-
-// See https://github.com/webpack/css-loader/issues/144
-var webpack = require('webpack');
-var _ = require('underscore');
-var path = require('path');
-var sourcemaps = 'inline-source-map';
-
-if(process.argv.indexOf('-w') !== -1 || process.argv.indexOf('-w') !== -1  ){
-  console.log('watch mode detected, will switch to cheap sourcemaps');
-  sourcemaps = 'eval-source-map';
-
-}
-var commonConfig = {
-    resolve: {
-        root: [
-            '.', /* allows npm packages to be loaded */
-            './notebook/static'
-        ].map(function(p) {return path.resolve(p);}),
-        modulesDirectories: [
-            "components", /* bower */
-            "node_modules" /* npm */
-        ]
-    },
-    bail: true,
-    module: {
-        loaders: [
-            { test: /\.js$/, exclude: /node_modules|\/notebook\/static\/component/, loader: "babel-loader"},
-            { test: /\.css$/, loader: "style-loader!css-loader" },
-            { test: /\.json$/, loader: "json-loader" },
-            // jquery-ui loads some images
-            { test: /\.(jpg|png|gif)$/, loader: "file" },
-            // required to load font-awesome
-            { test: /\.woff2(\?v=\d+\.\d+\.\d+)?$/, loader: "url?limit=10000&minetype=application/font-woff" },
-            { test: /\.woff(\?v=\d+\.\d+\.\d+)?$/, loader: "url?limit=10000&minetype=application/font-woff" },
-            { test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, loader: "url?limit=10000&minetype=application/octet-stream" },
-            { test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, loader: "file" },
-            { test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, loader: "url?limit=10000&minetype=image/svg+xml" }
-        ]
-    },
-    externals: {
-      jquery: '$',
-      bootstrap: '$',
-      bootstraptour: 'Tour',
-      'jquery-ui': '$',
-      typeahead: '$.typeahead',
-      'codemirror': 'CodeMirror',
-      'codemirror/lib/codemirror': 'CodeMirror',
-      'codemirror/mode/meta': 'CodeMirror',
-      // Account for relative paths from other CodeMirror files
-      '../../lib/codemirror': 'CodeMirror',
-      '../lib/codemirror': 'CodeMirror'
-    },
-};
-
-function buildConfig(appName) {
-    if (typeof appName !== 'string') return appName;
-    return _.extend({}, commonConfig, {
-        entry: ['es6-promise/auto','./notebook/static/' + appName + '/js/main.js'],
-        output: {
-            filename: 'main.min.js',
-            path: path.join(__dirname, 'notebook', 'static', appName, 'js', 'built')
-        },
-        devtool: sourcemaps,
-    });
-}
-
-module.exports = [
-    'auth',
-    'edit',
-    'terminal',
-    'tree',
-    'notebook',
-    _.extend({}, commonConfig, {
-        entry: ['es6-promise/auto', './notebook/static/services/contents.js'],
-        output: {
-            filename: 'contents.js',
-            path: path.join(__dirname, 'notebook', 'static', 'services', 'built'),
-            libraryTarget: 'amd'
-        },
-        devtool: sourcemaps,
-    }),
-    _.extend({}, commonConfig, {
-        entry: ['es6-promise/auto', './notebook/static/index.js'],
-        output: {
-            filename: 'index.js',
-            path: path.join(__dirname, 'notebook', 'static', 'built'),
-            libraryTarget: 'amd'
-        },
-        devtool: sourcemaps,
-    }),
-].map(buildConfig);