From 58c0a97ec38b265e8ff0943eb99e2d2ece762c7c Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 10 Nov 2014 16:28:20 -0800 Subject: [PATCH 1/4] add pre/post-save hooks - `ContentsManager.pre_save_hook` runs on the path and model with content - `FileContentsManager.post_save_hook` runs on the filesystem path and model without content - use pre_save_hook for things like stripping output - use post_save_hook for things like nbconvert --to python --- IPython/html/services/contents/filemanager.py | 42 ++++++++++++++++- IPython/html/services/contents/manager.py | 45 ++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py index 2ca1f41c5..620f45bcd 100644 --- a/IPython/html/services/contents/filemanager.py +++ b/IPython/html/services/contents/filemanager.py @@ -16,8 +16,9 @@ from tornado import web from .manager import ContentsManager from IPython import nbformat from IPython.utils.io import atomic_writing +from IPython.utils.importstring import import_item from IPython.utils.path import ensure_dir_exists -from IPython.utils.traitlets import Unicode, Bool, TraitError +from IPython.utils.traitlets import Any, Unicode, Bool, TraitError from IPython.utils.py3compat import getcwd, str_to_unicode from IPython.utils import tz from IPython.html.utils import is_hidden, to_os_path, to_api_path @@ -71,6 +72,40 @@ class FileContentsManager(ContentsManager): Use `ipython nbconvert --to python [notebook]` instead. """) + post_save_hook = Any(None, config=True, + help="""Python callable or importstring thereof + + to be called on the path of a file just saved. + + This can be used to + This can be used to process the file on disk, + such as converting the notebook to other formats, such as Python or HTML via nbconvert + + It will be called as (all arguments passed by keyword): + + hook(os_path=os_path, model=model, contents_manager=instance) + + path: the filesystem path to the file just written + model: the model representing the file + contents_manager: this ContentsManager instance + """ + ) + def _post_save_hook_changed(self, name, old, new): + if new and isinstance(new, string_types): + self.post_save_hook = import_item(self.post_save_hook) + elif new: + if not callable(new): + raise TraitError("post_save_hook must be callable") + + def run_post_save_hook(self, model, os_path): + """Run the post-save hook if defined, and log errors""" + if self.post_save_hook: + try: + self.log.debug("Running post-save hook on %s", os_path) + self.post_save_hook(os_path=os_path, model=model, contents_manager=self) + except Exception: + self.log.error("Post-save hook failed on %s", os_path, exc_info=True) + def _root_dir_changed(self, name, old, new): """Do a bit of validation of the root_dir.""" if not os.path.isabs(new): @@ -404,6 +439,8 @@ class FileContentsManager(ContentsManager): if 'content' not in model and model['type'] != 'directory': raise web.HTTPError(400, u'No file content provided') + self.run_pre_save_hook(model=model, path=path) + # One checkpoint should always exist if self.file_exists(path) and not self.list_checkpoints(path): self.create_checkpoint(path) @@ -433,6 +470,9 @@ class FileContentsManager(ContentsManager): model = self.get(path, content=False) if validation_message: model['message'] = validation_message + + self.run_post_save_hook(model=model, os_path=os_path) + return model def update(self, model, path): diff --git a/IPython/html/services/contents/manager.py b/IPython/html/services/contents/manager.py index 1c87064ca..055ade53a 100644 --- a/IPython/html/services/contents/manager.py +++ b/IPython/html/services/contents/manager.py @@ -14,7 +14,9 @@ from tornado.web import HTTPError from IPython.config.configurable import LoggingConfigurable from IPython.nbformat import sign, validate, ValidationError from IPython.nbformat.v4 import new_notebook -from IPython.utils.traitlets import Instance, Unicode, List +from IPython.utils.importstring import import_item +from IPython.utils.traitlets import Instance, Unicode, List, Any, TraitError +from IPython.utils.py3compat import string_types copy_pat = re.compile(r'\-Copy\d*\.') @@ -60,6 +62,41 @@ class ContentsManager(LoggingConfigurable): help="The base name used when creating untitled directories." ) + pre_save_hook = Any(None, config=True, + help="""Python callable or importstring thereof + + To be called on a contents model prior to save. + + This can be used to process the structure, + such as removing notebook outputs or other side effects that + should not be saved. + + It will be called as (all arguments passed by keyword): + + hook(path=path, model=model, contents_manager=self) + + model: the model to be saved. Includes file contents. + modifying this dict will affect the file that is stored. + path: the API path of the save destination + contents_manager: this ContentsManager instance + """ + ) + def _pre_save_hook_changed(self, name, old, new): + if new and isinstance(new, string_types): + self.pre_save_hook = import_item(self.pre_save_hook) + elif new: + if not callable(new): + raise TraitError("pre_save_hook must be callable") + + def run_pre_save_hook(self, model, path, **kwargs): + """Run the pre-save hook if defined, and log errors""" + if self.pre_save_hook: + try: + self.log.debug("Running pre-save hook on %s", path) + self.pre_save_hook(model=model, path=path, contents_manager=self, **kwargs) + except Exception: + self.log.error("Pre-save hook failed on %s", path, exc_info=True) + # ContentsManager API part 1: methods that must be # implemented in subclasses. @@ -142,7 +179,11 @@ class ContentsManager(LoggingConfigurable): raise NotImplementedError('must be implemented in a subclass') def save(self, model, path): - """Save the file or directory and return the model with no content.""" + """Save the file or directory and return the model with no content. + + Save implementations should call self.run_pre_save_hook(model=model, path=path) + prior to writing any data. + """ raise NotImplementedError('must be implemented in a subclass') def update(self, model, path): From 1731b9b44982532719e9a722412f37d7df5597c5 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 18 Nov 2014 14:53:33 -0800 Subject: [PATCH 2/4] `--script` triggers post_save hook with nbconvert --- IPython/html/services/contents/filemanager.py | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py index 620f45bcd..45099fb39 100644 --- a/IPython/html/services/contents/filemanager.py +++ b/IPython/html/services/contents/filemanager.py @@ -23,6 +23,27 @@ from IPython.utils.py3compat import getcwd, str_to_unicode from IPython.utils import tz from IPython.html.utils import is_hidden, to_os_path, to_api_path +_script_exporter = None +def _post_save_script(model, os_path, contents_manager, **kwargs): + """convert notebooks to Python script after save with nbconvert + + replaces `ipython notebook --script` + """ + from IPython.nbconvert.exporters.python import PythonExporter + + if model['type'] != 'notebook': + return + global _script_exporter + if _script_exporter is None: + _script_exporter = PythonExporter(parent=contents_manager) + log = contents_manager.log + + base, ext = os.path.splitext(os_path) + py_fname = base + '.py' + log.info("Writing %s", py_fname) + py, resources = _script_exporter.from_filename(os_path) + with io.open(py_fname, 'w', encoding='utf-8') as f: + f.write(py) class FileContentsManager(ContentsManager): @@ -65,13 +86,23 @@ class FileContentsManager(ContentsManager): with atomic_writing(os_path, *args, **kwargs) as f: yield f - save_script = Bool(False, config=True, help='DEPRECATED, IGNORED') + save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook') def _save_script_changed(self): self.log.warn(""" - Automatically saving notebooks as scripts has been removed. - Use `ipython nbconvert --to python [notebook]` instead. + `--script` is deprecated. You can trigger nbconvert via pre- or post-save hooks: + + ContentsManager.pre_save_hook + FileContentsManager.post_save_hook + + A post-save hook has been registered that calls: + + ipython nbconvert --to python [notebook] + + which behaves similar to `--script`. """) + self.post_save_hook = _post_save_script + post_save_hook = Any(None, config=True, help="""Python callable or importstring thereof From f2343e4ec50e90ce62ddecd018b169a299e20ba4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Dec 2014 15:26:13 -0800 Subject: [PATCH 3/4] update `--script` behavior to use `nbconvert --to script` --- IPython/html/services/contents/filemanager.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py index 45099fb39..cdcda190a 100644 --- a/IPython/html/services/contents/filemanager.py +++ b/IPython/html/services/contents/filemanager.py @@ -19,31 +19,34 @@ from IPython.utils.io import atomic_writing from IPython.utils.importstring import import_item from IPython.utils.path import ensure_dir_exists from IPython.utils.traitlets import Any, Unicode, Bool, TraitError -from IPython.utils.py3compat import getcwd, str_to_unicode +from IPython.utils.py3compat import getcwd, str_to_unicode, string_types from IPython.utils import tz from IPython.html.utils import is_hidden, to_os_path, to_api_path _script_exporter = None + def _post_save_script(model, os_path, contents_manager, **kwargs): """convert notebooks to Python script after save with nbconvert replaces `ipython notebook --script` """ - from IPython.nbconvert.exporters.python import PythonExporter + from IPython.nbconvert.exporters.script import ScriptExporter if model['type'] != 'notebook': return + global _script_exporter if _script_exporter is None: - _script_exporter = PythonExporter(parent=contents_manager) + _script_exporter = ScriptExporter(parent=contents_manager) log = contents_manager.log base, ext = os.path.splitext(os_path) py_fname = base + '.py' - log.info("Writing %s", py_fname) - py, resources = _script_exporter.from_filename(os_path) - with io.open(py_fname, 'w', encoding='utf-8') as f: - f.write(py) + script, resources = _script_exporter.from_filename(os_path) + script_fname = base + resources.get('output_extension', '.txt') + log.info("Saving script /%s", to_api_path(script_fname, contents_manager.root_dir)) + with io.open(script_fname, 'w', encoding='utf-8') as f: + f.write(script) class FileContentsManager(ContentsManager): @@ -96,9 +99,9 @@ class FileContentsManager(ContentsManager): A post-save hook has been registered that calls: - ipython nbconvert --to python [notebook] + ipython nbconvert --to script [notebook] - which behaves similar to `--script`. + which behaves similarly to `--script`. """) self.post_save_hook = _post_save_script From ee00dcf083122ea80e6f48793a3c8f1e4f3b880f Mon Sep 17 00:00:00 2001 From: Min RK Date: Sun, 7 Dec 2014 11:20:30 -0800 Subject: [PATCH 4/4] docstring --- IPython/html/services/contents/filemanager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py index cdcda190a..ec186de96 100644 --- a/IPython/html/services/contents/filemanager.py +++ b/IPython/html/services/contents/filemanager.py @@ -111,9 +111,8 @@ class FileContentsManager(ContentsManager): to be called on the path of a file just saved. - This can be used to This can be used to process the file on disk, - such as converting the notebook to other formats, such as Python or HTML via nbconvert + such as converting the notebook to a script or HTML via nbconvert. It will be called as (all arguments passed by keyword):