Merge pull request #6598 from minrk/nbformat-backport

nbformat validation

IPython/html/services/contents/filemanager.py

@@ -176,6 +176,7 @@ class FileContentsManager(ContentsManager):
         model['created'] = created
         model['content'] = None
         model['format'] = None
+        model['message'] = None
         return model
 
     def _dir_model(self, name, path='', content=True):
@@ -258,6 +259,7 @@ class FileContentsManager(ContentsManager):
             self.mark_trusted_cells(nb, name, path)
             model['content'] = nb
             model['format'] = 'json'
+            self.validate_notebook_model(model)
         return model
 
     def get_model(self, name, path='', content=True):
@@ -301,7 +303,7 @@ class FileContentsManager(ContentsManager):
             nb['metadata']['name'] = u''
 
         with atomic_writing(os_path, encoding='utf-8') as f:
-            current.write(nb, f, u'json')
+            current.write(nb, f, version=nb.nbformat)
 
     def _save_file(self, os_path, model, name='', path=''):
         """save a non-notebook file"""
@@ -366,7 +368,14 @@ class FileContentsManager(ContentsManager):
         except Exception as e:
             raise web.HTTPError(400, u'Unexpected error while saving file: %s %s' % (os_path, e))
 
+        validation_message = None
+        if model['type'] == 'notebook':
+            self.validate_notebook_model(model)
+            validation_message = model.get('message', None)
+
         model = self.get_model(new_name, new_path, content=False)
+        if validation_message:
+            model['message'] = validation_message
         return model
 
     def update(self, model, name, path=''):

IPython/html/services/contents/manager.py

@@ -5,6 +5,7 @@
 
 from fnmatch import fnmatch
 import itertools
+import json
 import os
 
 from tornado.web import HTTPError
@@ -216,6 +217,16 @@ class ContentsManager(LoggingConfigurable):
                 break
         return name
 
+    def validate_notebook_model(self, model):
+        """Add failed-validation message to model"""
+        try:
+            current.validate(model['content'])
+        except current.ValidationError as e:
+            model['message'] = 'Notebook Validation failed: {}:\n{}'.format(
+                e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'),
+            )
+        return model
+
     def create_file(self, model=None, path='', ext='.ipynb'):
         """Create a new file or directory and return its model with no content."""
         path = path.strip('/')
@@ -304,6 +315,8 @@ class ContentsManager(LoggingConfigurable):
         path : string
             The notebook's directory (for logging)
         """
+        if nb['nbformat'] != current.nbformat:
+            return
         if self.notary.check_cells(nb):
             self.notary.sign(nb)
         else:

IPython/html/services/contents/tests/test_manager.py

@@ -311,13 +311,13 @@ class TestContentsManager(TestCase):
         cm.mark_trusted_cells(nb, name, path)
         for cell in nb.worksheets[0].cells:
             if cell.cell_type == 'code':
-                assert not cell.trusted
+                assert not cell.metadata.trusted
 
         cm.trust_notebook(name, path)
         nb = cm.get_model(name, path)['content']
         for cell in nb.worksheets[0].cells:
             if cell.cell_type == 'code':
-                assert cell.trusted
+                assert cell.metadata.trusted
 
     def test_check_and_sign(self):
         cm = self.contents_manager

IPython/html/static/notebook/js/codecell.js

@@ -472,7 +472,7 @@ define([
             } else {
                 this.set_input_prompt();
             }
-            this.output_area.trusted = data.trusted || false;
+            this.output_area.trusted = data.metadata.trusted || false;
             this.output_area.fromJSON(data.outputs);
             if (data.collapsed !== undefined) {
                 if (data.collapsed) {
@@ -495,8 +495,8 @@ define([
         var outputs = this.output_area.toJSON();
         data.outputs = outputs;
         data.language = 'python';
-        data.trusted = this.output_area.trusted;
-        data.collapsed = this.collapsed;
+        data.metadata.trusted = this.output_area.trusted;
+        data.collapsed = this.output_area.collapsed;
         return data;
     };
 

IPython/html/static/notebook/js/notebook.js

@@ -1792,6 +1792,7 @@ define([
      * @param {Object} data JSON representation of a notebook
      */
     Notebook.prototype.fromJSON = function (data) {
+
         var content = data.content;
         var ncells = this.ncells();
         var i;
@@ -1941,6 +1942,7 @@ define([
             type : "PUT",
             data : JSON.stringify(model),
             headers : {'Content-Type': 'application/json'},
+            dataType : "json",
             success : $.proxy(this.save_notebook_success, this, start),
             error : $.proxy(this.save_notebook_error, this)
         };
@@ -1970,6 +1972,30 @@ define([
      */
     Notebook.prototype.save_notebook_success = function (start, data, status, xhr) {
         this.set_dirty(false);
+        if (data.message) {
+            // save succeeded, but validation failed.
+            var body = $("<div>");
+            var title = "Notebook validation failed";
+
+            body.append($("<p>").text(
+                "The save operation succeeded," +
+                " but the notebook does not appear to be valid." +
+                " The validation error was:"
+            )).append($("<div>").addClass("validation-error").append(
+                $("<pre>").text(data.message)
+            ));
+            dialog.modal({
+                notebook: this,
+                keyboard_manager: this.keyboard_manager,
+                title: title,
+                body: body,
+                buttons : {
+                    OK : {
+                        "class" : "btn-primary"
+                    }
+                }
+            });
+        }
         this.events.trigger('notebook_saved.Notebook');
         this._update_autosave_interval(start);
         if (this._checkpoint_after_save) {
@@ -2244,7 +2270,57 @@ define([
      * @param {jqXHR} xhr jQuery Ajax object
      */
     Notebook.prototype.load_notebook_success = function (data, status, xhr) {
-        this.fromJSON(data);
+        var failed;
+        try {
+            this.fromJSON(data);
+        } catch (e) {
+            failed = e;
+            console.log("Notebook failed to load from JSON:", e);
+        }
+        if (failed || data.message) {
+            // *either* fromJSON failed or validation failed
+            var body = $("<div>");
+            var title;
+            if (failed) {
+                title = "Notebook failed to load";
+                body.append($("<p>").text(
+                    "The error was: "
+                )).append($("<div>").addClass("js-error").text(
+                    failed.toString()
+                )).append($("<p>").text(
+                    "See the error console for details."
+                ));
+            } else {
+                title = "Notebook validation failed";
+            }
+
+            if (data.message) {
+                var msg;
+                if (failed) {
+                    msg = "The notebook also failed validation:"
+                } else {
+                    msg = "An invalid notebook may not function properly." +
+                    " The validation error was:"
+                }
+                body.append($("<p>").text(
+                    msg
+                )).append($("<div>").addClass("validation-error").append(
+                    $("<pre>").text(data.message)
+                ));
+            }
+
+            dialog.modal({
+                notebook: this,
+                keyboard_manager: this.keyboard_manager,
+                title: title,
+                body: body,
+                buttons : {
+                    OK : {
+                        "class" : "btn-primary"
+                    }
+                }
+            });
+        }
         if (this.ncells() === 0) {
             this.insert_cell_below('code');
             this.edit_mode(0);

IPython/testing/iptest.py

@@ -14,19 +14,11 @@ itself from the command line. There are two ways of running this script:
 
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2009-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
 from __future__ import print_function
 
-# Stdlib
 import glob
 from io import BytesIO
 import os
@@ -35,7 +27,6 @@ import sys
 from threading import Thread, Lock, Event
 import warnings
 
-# Now, proceed to import nose itself
 import nose.plugins.builtin
 from nose.plugins.xunit import Xunit
 from nose import SkipTest
@@ -43,7 +34,6 @@ from nose.core import TestProgram
 from nose.plugins import Plugin
 from nose.util import safe_str
 
-# Our own imports
 from IPython.utils.process import is_cmd_found
 from IPython.utils.importstring import import_item
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
@@ -148,7 +138,6 @@ have['mistune'] = test_for('mistune')
 have['requests'] = test_for('requests')
 have['sphinx'] = test_for('sphinx')
 have['jsonschema'] = test_for('jsonschema')
-have['jsonpointer'] = test_for('jsonpointer')
 have['casperjs'] = is_cmd_found('casperjs')
 have['phantomjs'] = is_cmd_found('phantomjs')
 have['slimerjs'] = is_cmd_found('slimerjs')
@@ -268,7 +257,7 @@ test_sections['qt'].requires('zmq', 'qt', 'pygments')
 
 # html:
 sec = test_sections['html']
-sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema', 'jsonpointer')
+sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema')
 # The notebook 'static' directory contains JS, css and other
 # files for web serving.  Occasionally projects may put a .py
 # file in there (MathJax ships a conf.py), so we might as
@@ -286,7 +275,7 @@ test_sections['config'].exclude('profile')
 
 # nbconvert:
 sec = test_sections['nbconvert']
-sec.requires('pygments', 'jinja2', 'jsonschema', 'jsonpointer', 'mistune')
+sec.requires('pygments', 'jinja2', 'jsonschema', 'mistune')
 # Exclude nbconvert directories containing config files used to test.
 # Executing the config files with iptest would cause an exception.
 sec.exclude('tests.files')
@@ -296,7 +285,7 @@ if not have['tornado']:
     sec.exclude('nbconvert.post_processors.tests.test_serve')
 
 # nbformat:
-test_sections['nbformat'].requires('jsonschema', 'jsonpointer')
+test_sections['nbformat'].requires('jsonschema')
 
 #-----------------------------------------------------------------------------
 # Functions and classes

IPython/testing/iptestcontroller.py

@@ -218,7 +218,7 @@ def all_js_groups():
 class JSController(TestController):
     """Run CasperJS tests """
     requirements =  ['zmq', 'tornado', 'jinja2', 'casperjs', 'sqlite3',
-                     'jsonschema', 'jsonpointer']
+                     'jsonschema']
     display_slimer_output = False
 
     def __init__(self, section, xunit=True, engine='phantomjs'):

setup.py

@@ -275,7 +275,7 @@ extras_require = dict(
     doc = ['Sphinx>=1.1', 'numpydoc'],
     test = ['nose>=0.10.1'],
     terminal = [],
-    nbformat = ['jsonschema>=2.0', 'jsonpointer>=1.3'],
+    nbformat = ['jsonschema>=2.0'],
     notebook = ['tornado>=3.1', 'pyzmq>=2.1.11', 'jinja2', 'pygments', 'mistune>=0.3.1'],
     nbconvert = ['pygments', 'jinja2', 'mistune>=0.3.1']
 )

setupbase.py

@@ -194,7 +194,10 @@ def find_package_data():
                 'preprocessors/tests/files/*.*',
             ],
         'IPython.nbconvert.filters' : ['marked.js'],
-        'IPython.nbformat' : ['tests/*.ipynb','v3/v3.withref.json']
+        'IPython.nbformat' : [
+            'tests/*.ipynb',
+            'v3/nbformat.v3.schema.json',
+            ]
     }
     
     return package_data