From 8fb670904b473bf14abf2784282cf9f953af8c3d Mon Sep 17 00:00:00 2001 From: Peter Parente Date: Tue, 28 Jun 2016 16:46:40 -0400 Subject: [PATCH 01/15] First pass migrating from jupyter_cms * Change handler for consistency with nbconvert * Change bundler function API to take ContentManager models * Change CLI to work as jupyter bundler * Change UI to fit into existing template / JS structure (c) Copyright IBM Corp. 2016 --- notebook/bundler/__init__.py | 0 notebook/bundler/handlers.py | 242 +++++++++++++++++++++++ notebook/bundler/tests/__init__.py | 0 notebook/bundlerextensions.py | 259 +++++++++++++++++++++++++ notebook/notebookapp.py | 1 + notebook/static/notebook/js/main.js | 3 +- notebook/static/notebook/js/menubar.js | 70 ++++++- notebook/templates/notebook.html | 5 +- scripts/jupyter-bundler | 8 + setup.py | 1 + 10 files changed, 578 insertions(+), 11 deletions(-) create mode 100644 notebook/bundler/__init__.py create mode 100644 notebook/bundler/handlers.py create mode 100644 notebook/bundler/tests/__init__.py create mode 100644 notebook/bundlerextensions.py create mode 100644 scripts/jupyter-bundler diff --git a/notebook/bundler/__init__.py b/notebook/bundler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/notebook/bundler/handlers.py b/notebook/bundler/handlers.py new file mode 100644 index 000000000..c8bdd0fc6 --- /dev/null +++ b/notebook/bundler/handlers.py @@ -0,0 +1,242 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shutil +import errno +import nbformat +import fnmatch +import glob +from notebook.utils import url_path_join, url2path +from notebook.base.handlers import IPythonHandler, path_regex +from notebook.services.config import ConfigManager +from ipython_genutils.importstring import import_item +from tornado import web, gen + +class BundlerTools(object): + '''Set of common tools to aid bundler implementations.''' + def get_file_references(self, abs_nb_path, version): + ''' + Gets a list of files referenced either in Markdown fenced code blocks + or in HTML comments from the notebook. Expands patterns expressed in + gitignore syntax (https://git-scm.com/docs/gitignore). Returns the + fully expanded list of filenames relative to the notebook dirname. + + NOTE: Temporarily changes the current working directory when called. + + :param abs_nb_path: Absolute path of the notebook on disk + :param version: Version of the notebook document format to use + :returns: List of filename strings relative to the notebook path + ''' + ref_patterns = self.get_reference_patterns(abs_nb_path, version) + expanded = self.expand_references(os.path.dirname(abs_nb_path), ref_patterns) + return expanded + + def get_reference_patterns(self, abs_nb_path, version): + ''' + Gets a list of reference patterns either in Markdown fenced code blocks + or in HTML comments from the notebook. + + :param abs_nb_path: Absolute path of the notebook on disk + :param version: Version of the notebook document format to use + :returns: List of pattern strings from the notebook + ''' + notebook = nbformat.read(abs_nb_path, version) + referenced_list = [] + for cell in notebook.cells: + references = self.get_cell_reference_patterns(cell) + if references: + referenced_list = referenced_list + references + return referenced_list + + def get_cell_reference_patterns(self, cell): + ''' + Retrieves the list of references from a single notebook cell. Looks for + fenced code blocks or HTML comments in Markdown cells, e.g., + + ``` + some.csv + foo/ + !foo/bar + ``` + + or + + + + :param cell: Notebook cell object + :returns: List of strings + ''' + referenced = [] + # invisible after execution: unrendered HTML comment + if cell.get('cell_type').startswith('markdown') and cell.get('source').startswith(''): + break + # Trying to go out of the current directory leads to + # trouble when deploying + if line.find('../') < 0 and not line.startswith('#'): + referenced.append(line) + # visible after execution: rendered as a code element within a pre element + elif cell.get('cell_type').startswith('markdown') and cell.get('source').find('```') >= 0: + source = cell.get('source') + offset = source.find('```') + lines = source[offset + len('```'):].splitlines() + for line in lines: + if line.startswith('```'): + break + # Trying to go out of the current directory leads to + # trouble when deploying + if line.find('../') < 0 and not line.startswith('#'): + referenced.append(line) + + # Clean out blank references + return [ref for ref in referenced if ref.strip()] + + def expand_references(self, root_path, references): + ''' + Expands a set of reference patterns by evaluating them against the + given root directory. Expansions are performed against patterns + expressed in the same manner as in gitignore + (https://git-scm.com/docs/gitignore). + + :param root_path: Assumed root directory for the patterns + :param references: List of reference patterns + :returns: List of filename strings relative to the root path + ''' + globbed = [] + negations = [] + must_walk = [] + for pattern in references: + if pattern and pattern.find('/') < 0: + # simple shell glob + cwd = os.getcwd() + os.chdir(root_path) + if pattern.startswith('!'): + negations = negations + glob.glob(pattern[1:]) + else: + globbed = globbed + glob.glob(pattern) + os.chdir(cwd) + elif pattern: + must_walk.append(pattern) + + for pattern in must_walk: + pattern_is_negation = pattern.startswith('!') + if pattern_is_negation: + testpattern = pattern[1:] + else: + testpattern = pattern + for root, _, filenames in os.walk(root_path): + for filename in filenames: + joined = os.path.join(root[len(root_path) + 1:], filename) + if testpattern.endswith('/'): + if joined.startswith(testpattern): + if pattern_is_negation: + negations.append(joined) + else: + globbed.append(joined) + elif testpattern.find('**') >= 0: + # path wildcard + ends = testpattern.split('**') + if len(ends) == 2: + if joined.startswith(ends[0]) and joined.endswith(ends[1]): + if pattern_is_negation: + negations.append(joined) + else: + globbed.append(joined) + else: + # segments should be respected + if fnmatch.fnmatch(joined, testpattern): + if pattern_is_negation: + negations.append(joined) + else: + globbed.append(joined) + + for negated in negations: + try: + globbed.remove(negated) + except ValueError as err: + pass + return set(globbed) + + def copy_filelist(self, src, dst, src_relative_filenames): + ''' + Copies the given list of files, relative to src, into dst, creating + directories along the way as needed and ignore existence errors. + Skips any files that do not exist. Does not create empty directories + from src in dst. + + :param src: Root of the source directory + :param dst: Root of the destination directory + :param src_relative_filenames: List of filename relative to src + ''' + for filename in src_relative_filenames: + # Only consider the file if it exists in src + if os.path.isfile(os.path.join(src, filename)): + parent_relative = os.path.dirname(filename) + if parent_relative: + # Make sure the parent directory exists + parent_dst = os.path.join(dst, parent_relative) + try: + os.makedirs(parent_dst) + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise exc + shutil.copy2(os.path.join(src, filename), os.path.join(dst, filename)) + +class BundlerHandler(IPythonHandler): + def initialize(self): + # Create common tools for bundler plugin to use + self.tools = BundlerTools() + + def get_bundler(self, bundler_id): + ''' + :param bundler_id: Unique ID within the notebook/jupyter_cms_bundlers + config section. + :returns: Dict of bundler metadata with keys label, group, module_name + :raises KeyError: If the bundler is not registered + ''' + cm = ConfigManager() + return cm.get('notebook').get('bundlers', {})[bundler_id] + + @web.authenticated + @gen.coroutine + def get(self, path): + """Bundle the given notebook. + """ + bundler_id = self.get_query_argument('bundler') + model = self.contents_manager.get(path=url2path(path)) + + try: + bundler = self.get_bundler(bundler_id) + except KeyError: + raise web.HTTPError(404, 'Bundler %s not found' % bundler_id) + + module_name = bundler['module_name'] + try: + # no-op in python3, decode error in python2 + module_name = str(module_name) + except UnicodeEncodeError: + # Encode unicode as utf-8 in python2 else import_item fails + module_name = module_name.encode('utf-8') + + try: + bundler_mod = import_item(module_name) + except ImportError: + raise web.HTTPError(500, 'Could not import bundler %s ' % bundler_id) + + # Let the bundler respond in any way it sees fit and assume it will + # finish the request + yield gen.maybe_future(bundler_mod.bundle(self, model)) + +_bundler_id_regex = r'(?P[A-Za-z0-9_]+)' + +default_handlers = [ + (r"/bundle/(.*)", BundlerHandler) +] diff --git a/notebook/bundler/tests/__init__.py b/notebook/bundler/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/notebook/bundlerextensions.py b/notebook/bundlerextensions.py new file mode 100644 index 000000000..ac97bac32 --- /dev/null +++ b/notebook/bundlerextensions.py @@ -0,0 +1,259 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import sys +import os + +from .nbextensions import (BaseNBExtensionApp, _get_config_dir) +from ._version import __version__ + +from traitlets.config.manager import BaseJSONConfigManager +from traitlets.utils.importstring import import_item +from traitlets import Bool + +BUNDLER_SECTION = "notebook" +BUNDLER_SUBSECTION = "bundlers" + +def _get_bundler_metadata(module): + """Gets the list of bundlers associated with a Python package. + + Returns a tuple of (the module, [{ + 'name': 'unique name of the bundler', + 'label': 'file menu item label for the bundler', + 'module_name': 'dotted package/module name containing the bundler', + 'group': 'download or deploy parent menu item' + }]) + + Parameters + ---------- + + module : str + Importable Python module exposing the + magic-named `_jupyter_bundler_paths` function + """ + m = import_item(module) + if not hasattr(m, '_jupyter_bundler_paths'): + raise KeyError('The Python module {} does not contain a valid bundler'.format(module)) + bundlers = m._jupyter_bundler_paths() + return m, bundlers + +def _set_bundler_state(name, label, module_name, group, state, + user=True, sys_prefix=False, logger=None): + """Set whether a bundler is enabled or disabled. + + Returns True if the final state is the one requested. + + Parameters + ---------- + name : string + Unique name of the bundler + label : string + Human-readable label for the bundler menu item in the notebook UI + module_name : string + Dotted module/package name containing the bundler + group : string + 'download' or 'deploy' indicating the parent menu containing the label + state : bool + The state in which to leave the extension + user : bool [default: True] + Whether to update the user's .jupyter/nbconfig directory + sys_prefix : bool [default: False] + Whether to update the sys.prefix, i.e. environment. Will override + `user`. + logger : Jupyter logger [optional] + Logger instance to use + """ + user = False if sys_prefix else user + config_dir = os.path.join( + _get_config_dir(user=user, sys_prefix=sys_prefix), 'nbconfig') + cm = BaseJSONConfigManager(config_dir=config_dir) + + if logger: + logger.info("{} {} bundler {}...".format( + "Enabling" if state else "Disabling", + name, + module_name + )) + + if state: + cm.update(BUNDLER_SECTION, { + BUNDLER_SUBSECTION: { + name: { + "label": label, + "module_name": module_name, + "group" : group + } + } + }) + else: + cm.update(BUNDLER_SECTION, { + BUNDLER_SUBSECTION: { + name: None + } + }) + + return (cm.get(BUNDLER_SECTION) + .get(BUNDLER_SUBSECTION, {}) + .get(name) is not None) == state + + +def _set_bundler_state_python(state, module, user, sys_prefix, logger=None): + """Enables or disables bundlers defined in a Python package. + + Returns a list of whether the state was achieved for each bundler. + + Parameters + ---------- + state : Bool + Whether the extensions should be enabled + module : str + Importable Python module exposing the + magic-named `_jupyter_bundler_paths` function + user : bool + Whether to enable in the user's nbconfig directory. + sys_prefix : bool + Enable/disable in the sys.prefix, i.e. environment + logger : Jupyter logger [optional] + Logger instance to use + """ + m, bundlers = _get_bundler_metadata(module) + return [_set_bundler_state(name=bundler["name"], + label=bundler["label"], + module_name=bundler["module_name"], + group=bundler["group"], + state=state, + user=user, sys_prefix=sys_prefix, + logger=logger) + for bundler in bundlers] + +def enable_bundler_python(module, user=True, sys_prefix=False, logger=None): + """Enables bundlers defined in a Python package. + + Returns whether each bundle defined in the packaged was enabled or not. + + Parameters + ---------- + module : str + Importable Python module exposing the + magic-named `_jupyter_bundler_paths` function + user : bool [default: True] + Whether to enable in the user's nbconfig directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment. Will override + `user` + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_bundler_state_python(True, module, user, sys_prefix, + logger=logger) + +def disable_bundler_python(module, user=True, sys_prefix=False, logger=None): + """Disables bundlers defined in a Python package. + + Returns whether each bundle defined in the packaged was enabled or not. + + Parameters + ---------- + module : str + Importable Python module exposing the + magic-named `_jupyter_bundler_paths` function + user : bool [default: True] + Whether to enable in the user's nbconfig directory. + sys_prefix : bool [default: False] + Whether to enable in the sys.prefix, i.e. environment. Will override + `user` + logger : Jupyter logger [optional] + Logger instance to use + """ + return _set_bundler_state_python(False, module, user, sys_prefix, + logger=logger) + +class ToggleNBBundlerApp(BaseNBExtensionApp): + """A base class for apps that enable/disable bundlers""" + name = "jupyter bundler enable/disable" + version = __version__ + description = "Enable/disable a bundler in configuration." + + user = Bool(True, config=True, help="Apply the configuration only for the current user (default)") + + _toggle_value = None + + def _config_file_name_default(self): + """The default config file name.""" + return 'jupyter_notebook_config' + + def toggle_bundler_python(self, module): + """Toggle some extensions in an importable Python module. + + Returns a list of booleans indicating whether the state was changed as + requested. + + Parameters + ---------- + module : str + Importable Python module exposing the + magic-named `_jupyter_bundler_paths` function + """ + toggle = (enable_bundler_python if self._toggle_value + else disable_bundler_python) + return toggle(module, + user=self.user, + sys_prefix=self.sys_prefix, + logger=self.log) + + def start(self): + if not self.extra_args: + sys.exit('Please specify an nbextension/package to enable or disable') + elif len(self.extra_args) > 1: + sys.exit('Please specify one nbextension/package at a time') + if self.python: + self.toggle_bundler_python(self.extra_args[0]) + else: + raise NotImplementedError('Cannot install bundlers from non-Python packages') + +class EnableNBBundlerApp(ToggleNBBundlerApp): + """An App that enables bundlers""" + name = "jupyter bundler enable" + description = """ + Enable a bundler in frontend configuration. + + Usage + jupyter bundler enable [--system|--sys-prefix] + """ + _toggle_value = True + +class DisableNBBundlerApp(ToggleNBBundlerApp): + """An App that disables bundlers""" + name = "jupyter bundler disable" + description = """ + Disable a bundler in frontend configuration. + + Usage + jupyter bundler disable [--system|--sys-prefix] + """ + _toggle_value = None + +class NBBundlerApp(BaseNBExtensionApp): + """Base jupyter bundler command entry point""" + name = "jupyter bundler" + version = __version__ + description = "Work with Jupyter notebook bundlers" + examples = """ +jupyter bundler enable --py # enable all bundlers in a Python package +jupyter bundler disable --py # disable all bundlers in a Python package +""" + + subcommands = dict( + enable=(EnableNBBundlerApp, "Enable a bundler"), + disable=(DisableNBBundlerApp, "Disable a bundler") + ) + + def start(self): + """Perform the App's functions as configured""" + super(NBBundlerApp, self).start() + + # The above should have called a subcommand and raised NoStart; if we + # get here, it didn't, so we should self.log.info a message. + subcmds = ", ".join(sorted(self.subcommands)) + sys.exit("Please supply at least one subcommand: %s" % subcmds) + +main = NBBundlerApp.launch_instance diff --git a/notebook/notebookapp.py b/notebook/notebookapp.py index a0972475c..a58a84c76 100644 --- a/notebook/notebookapp.py +++ b/notebook/notebookapp.py @@ -260,6 +260,7 @@ class NotebookWebApplication(web.Application): handlers.extend(load_handlers('files.handlers')) handlers.extend(load_handlers('notebook.handlers')) handlers.extend(load_handlers('nbconvert.handlers')) + handlers.extend(load_handlers('bundler.handlers')) handlers.extend(load_handlers('kernelspecs.handlers')) handlers.extend(load_handlers('edit.handlers')) handlers.extend(load_handlers('services.api.handlers')) diff --git a/notebook/static/notebook/js/main.js b/notebook/static/notebook/js/main.js index dc6af6edc..de3def780 100644 --- a/notebook/static/notebook/js/main.js +++ b/notebook/static/notebook/js/main.js @@ -141,7 +141,8 @@ require([ events: events, save_widget: save_widget, quick_help: quick_help, - actions: acts}, + actions: acts, + config: config_section}, common_options)); var notification_area = new notificationarea.NotebookNotificationArea( '#notification_area', { diff --git a/notebook/static/notebook/js/menubar.js b/notebook/static/notebook/js/menubar.js index 04984803a..bccc22b7c 100644 --- a/notebook/static/notebook/js/menubar.js +++ b/notebook/static/notebook/js/menubar.js @@ -29,6 +29,7 @@ define([ * base_url : string * notebook_path : string * notebook_name : string + * config: ConfigSection instance */ options = options || {}; this.base_url = options.base_url || utils.get_body_data("baseUrl"); @@ -40,6 +41,7 @@ define([ this.save_widget = options.save_widget; this.quick_help = options.quick_help; this.actions = options.actions; + this.config = options.config; try { this.tour = new tour.Tour(this.notebook, this.events); @@ -51,6 +53,7 @@ define([ if (this.selector !== undefined) { this.element = $(selector); this.style(); + this.add_bundler_items(); this.bind_events(); } }; @@ -66,6 +69,63 @@ define([ } ); }; + + MenuBar.prototype.add_bundler_items = function() { + var that = this; + this.config.loaded.then(function() { + var bundlers = that.config.data.bundlers; + if(bundlers) { + // TODO: maybe sort by label + for(var bundler_id in bundlers) { + var bundler = bundlers[bundler_id]; + var group = that.element.find('#'+bundler.group+'_menu') + + // Basic validation to ensure valid menu options + if(!group.length) { + console.warn('unknown group', bundler.group, 'for bundler ID', bundler_id, '; skipping'); + continue; + } else if(!bundler.label) { + console.warn('no label for bundler ID', bundler_id, '; skipping'); + continue; + } + + // New menu item in the right submenu to trigger that._bundle + group.parent().removeClass('hidden'); + var $li = $('
  • ') + .appendTo(group); + $('') + .attr('href', '#') + .text(bundler.label) + .appendTo($li) + .on('click', that._bundle.bind(that, bundler_id)) + .appendTo($li); + } + } + }); + }; + + MenuBar.prototype._new_window = function(url) { + var w = window.open('', IPython._target); + if (this.notebook.dirty && this.notebook.writable) { + this.notebook.save_notebook().then(function() { + w.location = url; + }); + } else { + w.location = url; + } + }; + + MenuBar.prototype._bundle = function(bundler_id) { + // Read notebook path and base url here in case they change + var notebook_path = utils.encode_uri_components(this.notebook.notebook_path); + var url = utils.url_path_join( + this.base_url, + 'bundle', + notebook_path + ) + '?bundler=' + utils.encode_uri_components(bundler_id); + + this._new_window(url); + }; MenuBar.prototype._nbconvert = function (format, download) { download = download || false; @@ -77,14 +137,7 @@ define([ notebook_path ) + "?download=" + download.toString(); - var w = window.open('', IPython._target); - if (this.notebook.dirty && this.notebook.writable) { - this.notebook.save_notebook().then(function() { - w.location = url; - }); - } else { - w.location = url; - } + this._new_window(url); }; MenuBar.prototype._size_header = function() { @@ -154,7 +207,6 @@ define([ that._nbconvert('script', true); }); - this.events.on('trust_changed.Notebook', function (event, trusted) { if (trusted) { that.element.find('#trust_notebook') diff --git a/notebook/templates/notebook.html b/notebook/templates/notebook.html index f8e4f3c07..8ac71de66 100644 --- a/notebook/templates/notebook.html +++ b/notebook/templates/notebook.html @@ -103,7 +103,7 @@ data-notebook-path="{{notebook_path | urlencode}}"