Merge pull request #4656 from takluyver/nbconvert-service

Nbconvert HTTP service

IPython/html/base/handlers.py

@@ -354,6 +354,14 @@ class TrailingSlashHandler(web.RequestHandler):
     def get(self):
         self.redirect(self.request.uri.rstrip('/'))
 
+#-----------------------------------------------------------------------------
+# URL pattern fragments for re-use
+#-----------------------------------------------------------------------------
+
+path_regex = r"(?P<path>(?:/.*)*)"
+notebook_name_regex = r"(?P<name>[^/]+\.ipynb)"
+notebook_path_regex = "%s/%s" % (path_regex, notebook_name_regex)
+
 #-----------------------------------------------------------------------------
 # URL to handler mappings
 #-----------------------------------------------------------------------------

IPython/html/nbconvert/handlers.py

@@ -0,0 +1,117 @@
+import io
+import os
+import zipfile
+
+from tornado import web
+
+from ..base.handlers import IPythonHandler, notebook_path_regex
+from IPython.nbformat.current import to_notebook_json
+from IPython.nbconvert.exporters.export import exporter_map
+from IPython.utils import tz
+from IPython.utils.py3compat import cast_bytes
+
+import sys
+
+def find_resource_files(output_files_dir):
+    files = []
+    for dirpath, dirnames, filenames in os.walk(output_files_dir):
+        files.extend([os.path.join(dirpath, f) for f in filenames])
+    return files
+
+def respond_zip(handler, name, output, resources):
+    """Zip up the output and resource files and respond with the zip file.
+
+    Returns True if it has served a zip file, False if there are no resource
+    files, in which case we serve the plain output file.
+    """
+    # Check if we have resource files we need to zip
+    output_files = resources.get('outputs', None)
+    if not output_files:
+        return False
+
+    # Headers
+    zip_filename = os.path.splitext(name)[0] + '.zip'
+    handler.set_header('Content-Disposition',
+                       'attachment; filename="%s"' % zip_filename)
+    handler.set_header('Content-Type', 'application/zip')
+
+    # Prepare the zip file
+    buffer = io.BytesIO()
+    zipf = zipfile.ZipFile(buffer, mode='w', compression=zipfile.ZIP_DEFLATED)
+    output_filename = os.path.splitext(name)[0] + '.' + resources['output_extension']
+    zipf.writestr(output_filename, cast_bytes(output, 'utf-8'))
+    for filename, data in output_files.items():
+        zipf.writestr(os.path.basename(filename), data)
+    zipf.close()
+
+    handler.finish(buffer.getvalue())
+    return True
+
+class NbconvertFileHandler(IPythonHandler):
+
+    SUPPORTED_METHODS = ('GET',)
+    
+    @web.authenticated
+    def get(self, format, path='', name=None):
+        exporter = exporter_map[format](config=self.config)
+        
+        path = path.strip('/')
+        os_path = self.notebook_manager.get_os_path(name, path)
+        if not os.path.isfile(os_path):
+            raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
+
+        info = os.stat(os_path)
+        self.set_header('Last-Modified', tz.utcfromtimestamp(info.st_mtime))
+
+        output, resources = exporter.from_filename(os_path)
+
+        if respond_zip(self, name, output, resources):
+            return
+
+        # Force download if requested
+        if self.get_argument('download', 'false').lower() == 'true':
+            filename = os.path.splitext(name)[0] + '.' + resources['output_extension']
+            self.set_header('Content-Disposition',
+                               'attachment; filename="%s"' % filename)
+
+        # MIME type
+        if exporter.output_mimetype:
+            self.set_header('Content-Type',
+                            '%s; charset=utf-8' % exporter.output_mimetype)
+
+        self.finish(output)
+
+class NbconvertPostHandler(IPythonHandler):
+    SUPPORTED_METHODS = ('POST',)
+
+    @web.authenticated 
+    def post(self, format):
+        exporter = exporter_map[format](config=self.config)
+        
+        model = self.get_json_body()
+        nbnode = to_notebook_json(model['content'])
+
+        output, resources = exporter.from_notebook_node(nbnode)
+
+        if respond_zip(self, nbnode.metadata.name, output, resources):
+            return
+
+        # MIME type
+        if exporter.output_mimetype:
+            self.set_header('Content-Type',
+                            '%s; charset=utf-8' % exporter.output_mimetype)
+
+        self.finish(output)
+
+#-----------------------------------------------------------------------------
+# URL to handler mappings
+#-----------------------------------------------------------------------------
+
+_format_regex = r"(?P<format>\w+)"
+
+
+default_handlers = [
+    (r"/nbconvert/%s%s" % (_format_regex, notebook_path_regex),
+         NbconvertFileHandler),
+    (r"/nbconvert/%s" % _format_regex, NbconvertPostHandler),
+]

IPython/html/nbconvert/tests/test_nbconvert_handlers.py

@@ -0,0 +1,120 @@
+# coding: utf-8
+import base64
+import io
+import json
+import os
+from os.path import join as pjoin
+import shutil
+
+import requests
+
+from IPython.html.utils import url_path_join
+from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error
+from IPython.nbformat.current import (new_notebook, write, new_worksheet,
+                                      new_heading_cell, new_code_cell,
+                                      new_output)
+
+class NbconvertAPI(object):
+    """Wrapper for nbconvert API calls."""
+    def __init__(self, base_url):
+        self.base_url = base_url
+
+    def _req(self, verb, path, body=None, params=None):
+        response = requests.request(verb,
+                url_path_join(self.base_url, 'nbconvert', path),
+                data=body, params=params,
+        )
+        response.raise_for_status()
+        return response
+
+    def from_file(self, format, path, name, download=False):
+        return self._req('GET', url_path_join(format, path, name),
+                         params={'download':download})
+
+    def from_post(self, format, nbmodel):
+        body = json.dumps(nbmodel)
+        return self._req('POST', format, body)
+
+    def list_formats(self):
+        return self._req('GET', '')
+
+png_green_pixel = base64.encodestring(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00'
+b'\x00\x00\x01\x00\x00x00\x01\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\x0cIDAT'
+b'\x08\xd7c\x90\xfb\xcf\x00\x00\x02\\\x01\x1e.~d\x87\x00\x00\x00\x00IEND\xaeB`\x82')
+
+class APITest(NotebookTestBase):
+    def setUp(self):
+        nbdir = self.notebook_dir.name
+        
+        if not os.path.isdir(pjoin(nbdir, 'foo')):
+            os.mkdir(pjoin(nbdir, 'foo'))
+        
+        nb = new_notebook(name='testnb')
+        
+        ws = new_worksheet()
+        nb.worksheets = [ws]
+        ws.cells.append(new_heading_cell(u'Created by test ³'))
+        cc1 = new_code_cell(input=u'print(2*6)')
+        cc1.outputs.append(new_output(output_text=u'12'))
+        cc1.outputs.append(new_output(output_png=png_green_pixel, output_type='pyout'))
+        ws.cells.append(cc1)
+        
+        with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w',
+                     encoding='utf-8') as f:
+            write(nb, f, format='ipynb')
+
+        self.nbconvert_api = NbconvertAPI(self.base_url())
+
+    def tearDown(self):
+        nbdir = self.notebook_dir.name
+
+        for dname in ['foo']:
+            shutil.rmtree(pjoin(nbdir, dname), ignore_errors=True)
+    
+    def test_from_file(self):
+        r = self.nbconvert_api.from_file('html', 'foo', 'testnb.ipynb')
+        self.assertEqual(r.status_code, 200)
+        self.assertIn(u'text/html', r.headers['Content-Type'])
+        self.assertIn(u'Created by test', r.text)
+        self.assertIn(u'print', r.text)
+
+        r = self.nbconvert_api.from_file('python', 'foo', 'testnb.ipynb')
+        self.assertIn(u'text/x-python', r.headers['Content-Type'])
+        self.assertIn(u'print(2*6)', r.text)
+
+    def test_from_file_404(self):
+        with assert_http_error(404):
+            self.nbconvert_api.from_file('html', 'foo', 'thisdoesntexist.ipynb')
+
+    def test_from_file_download(self):
+        r = self.nbconvert_api.from_file('python', 'foo', 'testnb.ipynb', download=True)
+        content_disposition = r.headers['Content-Disposition']
+        self.assertIn('attachment', content_disposition)
+        self.assertIn('testnb.py', content_disposition)
+
+    def test_from_file_zip(self):
+        r = self.nbconvert_api.from_file('latex', 'foo', 'testnb.ipynb', download=True)
+        self.assertIn(u'application/zip', r.headers['Content-Type'])
+        self.assertIn(u'.zip', r.headers['Content-Disposition'])
+
+    def test_from_post(self):
+        nbmodel_url = url_path_join(self.base_url(), 'api/notebooks/foo/testnb.ipynb')
+        nbmodel = requests.get(nbmodel_url).json()
+        
+        r = self.nbconvert_api.from_post(format='html', nbmodel=nbmodel)
+        self.assertEqual(r.status_code, 200)
+        self.assertIn(u'text/html', r.headers['Content-Type'])
+        self.assertIn(u'Created by test', r.text)
+        self.assertIn(u'print', r.text)
+        
+        r = self.nbconvert_api.from_post(format='python', nbmodel=nbmodel)
+        self.assertIn(u'text/x-python', r.headers['Content-Type'])
+        self.assertIn(u'print(2*6)', r.text)
+
+    def test_from_post_zip(self):
+        nbmodel_url = url_path_join(self.base_url(), 'api/notebooks/foo/testnb.ipynb')
+        nbmodel = requests.get(nbmodel_url).json()
+
+        r = self.nbconvert_api.from_post(format='latex', nbmodel=nbmodel)
+        self.assertIn(u'application/zip', r.headers['Content-Type'])
+        self.assertIn(u'.zip', r.headers['Content-Disposition'])

IPython/html/notebook/handlers.py

@@ -20,8 +20,7 @@ import os
 from tornado import web
 HTTPError = web.HTTPError
 
-from ..base.handlers import IPythonHandler
-from ..services.notebooks.handlers import _notebook_path_regex, _path_regex
+from ..base.handlers import IPythonHandler, notebook_path_regex, path_regex
 from ..utils import url_path_join, url_escape
 
 #-----------------------------------------------------------------------------
@@ -85,7 +84,7 @@ class NotebookRedirectHandler(IPythonHandler):
 
 
 default_handlers = [
-    (r"/notebooks%s" % _notebook_path_regex, NotebookHandler),
-    (r"/notebooks%s" % _path_regex, NotebookRedirectHandler),
+    (r"/notebooks%s" % notebook_path_regex, NotebookHandler),
+    (r"/notebooks%s" % path_regex, NotebookRedirectHandler),
 ]
 

IPython/html/notebookapp.py

@@ -192,10 +192,12 @@ class NotebookWebApplication(web.Application):
         handlers.extend(load_handlers('auth.login'))
         handlers.extend(load_handlers('auth.logout'))
         handlers.extend(load_handlers('notebook.handlers'))
+        handlers.extend(load_handlers('nbconvert.handlers'))
         handlers.extend(load_handlers('services.kernels.handlers'))
         handlers.extend(load_handlers('services.notebooks.handlers'))
         handlers.extend(load_handlers('services.clusters.handlers'))
         handlers.extend(load_handlers('services.sessions.handlers'))
+        handlers.extend(load_handlers('services.nbconvert.handlers'))
         handlers.extend([
             (r"/files/(.*)", AuthenticatedFileHandler, {'path' : settings['notebook_manager'].notebook_dir}),
             (r"/nbextensions/(.*)", FileFindHandler, {'path' : settings['nbextensions_path']}),

IPython/html/services/nbconvert/handlers.py

@@ -0,0 +1,23 @@
+import json
+
+from tornado import web
+
+from ...base.handlers import IPythonHandler, json_errors
+from IPython.nbconvert.exporters.export import exporter_map
+
+class NbconvertRootHandler(IPythonHandler):
+    SUPPORTED_METHODS = ('GET',)
+
+    @web.authenticated
+    @json_errors
+    def get(self):
+        res = {}
+        for format, exporter in exporter_map.items():
+            res[format] = info = {}
+            info['output_mimetype'] = exporter.output_mimetype
+
+        self.finish(json.dumps(res))
+
+default_handlers = [
+    (r"/api/nbconvert", NbconvertRootHandler),
+]

IPython/html/services/nbconvert/tests/test_nbconvert_api.py

@@ -0,0 +1,31 @@
+import requests
+
+from IPython.html.utils import url_path_join
+from IPython.html.tests.launchnotebook import NotebookTestBase
+
+class NbconvertAPI(object):
+    """Wrapper for nbconvert API calls."""
+    def __init__(self, base_url):
+        self.base_url = base_url
+
+    def _req(self, verb, path, body=None, params=None):
+        response = requests.request(verb,
+                url_path_join(self.base_url, 'api/nbconvert', path),
+                data=body, params=params,
+        )
+        response.raise_for_status()
+        return response
+
+    def list_formats(self):
+        return self._req('GET', '')
+
+class APITest(NotebookTestBase):
+    def setUp(self):
+        self.nbconvert_api = NbconvertAPI(self.base_url())
+
+    def test_list_formats(self):
+        formats = self.nbconvert_api.list_formats().json()
+        self.assertIsInstance(formats, dict)
+        self.assertIn('python', formats)
+        self.assertIn('html', formats)
+        self.assertEqual(formats['python']['output_mimetype'], 'text/x-python')

IPython/html/services/notebooks/filenbmanager.py

@@ -178,7 +178,7 @@ class FileNotebookManager(NotebookManager):
         return notebooks
 
     def get_notebook_model(self, name, path='', content=True):
-        """ Takes a path and name for a notebook and returns it's model
+        """ Takes a path and name for a notebook and returns its model
         
         Parameters
         ----------

IPython/html/services/notebooks/handlers.py

@@ -23,7 +23,9 @@ from tornado import web
 from IPython.html.utils import url_path_join, url_escape
 from IPython.utils.jsonutil import date_default
 
-from IPython.html.base.handlers import IPythonHandler, json_errors
+from IPython.html.base.handlers import (IPythonHandler, json_errors,
+                                    notebook_path_regex, path_regex,
+                                    notebook_name_regex)
 
 #-----------------------------------------------------------------------------
 # Notebook web service handlers
@@ -264,17 +266,14 @@ class ModifyNotebookCheckpointsHandler(IPythonHandler):
 #-----------------------------------------------------------------------------
 
 
-_path_regex = r"(?P<path>(?:/.*)*)"
 _checkpoint_id_regex = r"(?P<checkpoint_id>[\w-]+)"
-_notebook_name_regex = r"(?P<name>[^/]+\.ipynb)"
-_notebook_path_regex = "%s/%s" % (_path_regex, _notebook_name_regex)
 
 default_handlers = [
-    (r"/api/notebooks%s/checkpoints" % _notebook_path_regex, NotebookCheckpointsHandler),
-    (r"/api/notebooks%s/checkpoints/%s" % (_notebook_path_regex, _checkpoint_id_regex),
+    (r"/api/notebooks%s/checkpoints" % notebook_path_regex, NotebookCheckpointsHandler),
+    (r"/api/notebooks%s/checkpoints/%s" % (notebook_path_regex, _checkpoint_id_regex),
         ModifyNotebookCheckpointsHandler),
-    (r"/api/notebooks%s" % _notebook_path_regex, NotebookHandler),
-    (r"/api/notebooks%s" % _path_regex, NotebookHandler),
+    (r"/api/notebooks%s" % notebook_path_regex, NotebookHandler),
+    (r"/api/notebooks%s" % path_regex, NotebookHandler),
 ]
 
 

IPython/html/static/notebook/js/celltoolbarpresets/rawcell.js

@@ -22,7 +22,7 @@
     ["reST", "text/restructuredtext"],
     ["HTML", "text/html"],
     ["Markdown", "text/markdown"],
-    ["Python", "application/x-python"],
+    ["Python", "text/x-python"],
     ["Custom", "dialog"],
 
     ],
@@ -87,4 +87,4 @@
   CellToolbar.register_preset('Raw Cell Format', raw_cell_preset);
   console.log('Raw Cell Format toolbar preset loaded.');
 
-}(IPython));
+}(IPython));

IPython/html/static/notebook/js/menubar.js

@@ -69,6 +69,22 @@ var IPython = (function (IPython) {
         );
     };
 
+    MenuBar.prototype._nbconvert = function (format, download) {
+        download = download || false;
+        var notebook_name = IPython.notebook.get_notebook_name();
+        if (IPython.notebook.dirty) {
+            IPython.notebook.save_notebook({async : false});
+        }
+        var url = utils.url_path_join(
+            this.baseProjectUrl(),
+            'nbconvert',
+            format,
+            this.notebookPath(),
+            notebook_name + '.ipynb'
+        ) + "?download=" + download.toString();
+
+        window.open(url);
+    }
 
     MenuBar.prototype.bind_events = function () {
         //  File
@@ -102,25 +118,22 @@ var IPython = (function (IPython) {
             window.location.assign(url);
         });
         
-        /* FIXME: download-as-py doesn't work right now
-         * We will need nbconvert hooked up to get this back
-        
+        this.element.find('#print_preview').click(function () {
+            that._nbconvert('html', false);
+        });
+
         this.element.find('#download_py').click(function () {
-            var notebook_name = IPython.notebook.get_notebook_name();
-            if (IPython.notebook.dirty) {
-                IPython.notebook.save_notebook({async : false});
-            }
-            var url = utils.url_path_join(
-                that.baseProjectUrl(),
-                'api/notebooks',
-                that.notebookPath(),
-                notebook_name + '.ipynb?format=py&download=True'
-            );
-            window.location.assign(url);
+            that._nbconvert('python', true);
         });
-        
-        */
-        
+
+        this.element.find('#download_html').click(function () {
+            that._nbconvert('html', true);
+        });
+
+        this.element.find('#download_rst').click(function () {
+            that._nbconvert('rst', true);
+        });
+
         this.element.find('#rename_notebook').click(function () {
             IPython.save_widget.rename_notebook();
         });

IPython/html/templates/notebook.html

@@ -77,10 +77,13 @@ class="notebook_app"
                   </ul>
                 </li>
                 <li class="divider"></li>
+                <li id="print_preview"><a href="#">Print Preview</a></li>
                 <li class="dropdown-submenu"><a href="#">Download as</a>
                     <ul class="dropdown-menu">
                         <li id="download_ipynb"><a href="#">IPython Notebook (.ipynb)</a></li>
-                        <!-- <li id="download_py"><a href="#">Python (.py)</a></li> -->
+                        <li id="download_py"><a href="#">Python (.py)</a></li>
+                        <li id="download_html"><a href="#">HTML (.html)</a></li>
+                        <li id="download_rst"><a href="#">reST (.rst)</a></li>
                     </ul>
                 </li>
                 <li class="divider"></li>

IPython/html/tests/launchnotebook.py

@@ -1,13 +1,14 @@
 """Base class for notebook tests."""
 
-import os
 import sys
 import time
 import requests
 from contextlib import contextmanager
-from subprocess import Popen, PIPE
+from subprocess import Popen, STDOUT
 from unittest import TestCase
 
+import nose
+
 from IPython.utils.tempdir import TemporaryDirectory
 
 class NotebookTestBase(TestCase):
@@ -55,10 +56,9 @@ class NotebookTestBase(TestCase):
             '--ipython-dir=%s' % cls.ipython_dir.name,
             '--notebook-dir=%s' % cls.notebook_dir.name,
         ]
-        devnull = open(os.devnull, 'w')
         cls.notebook = Popen(notebook_args,
-            stdout=devnull,
-            stderr=devnull,
+            stdout=nose.iptest_stdstreams_fileno(),
+            stderr=STDOUT,
         )
         cls.wait_until_alive()
 

IPython/html/tree/handlers.py

@@ -18,9 +18,8 @@ Authors:
 import os
 
 from tornado import web
-from ..base.handlers import IPythonHandler
+from ..base.handlers import IPythonHandler, notebook_path_regex, path_regex
 from ..utils import url_path_join, path2url, url2path, url_escape
-from ..services.notebooks.handlers import _notebook_path_regex, _path_regex
 
 #-----------------------------------------------------------------------------
 # Handlers
@@ -70,8 +69,8 @@ class TreeRedirectHandler(IPythonHandler):
 
 
 default_handlers = [
-    (r"/tree%s" % _notebook_path_regex, TreeHandler),
-    (r"/tree%s" % _path_regex, TreeHandler),
+    (r"/tree%s" % notebook_path_regex, TreeHandler),
+    (r"/tree%s" % path_regex, TreeHandler),
     (r"/tree", TreeHandler),
     (r"/", TreeRedirectHandler),
     ]

IPython/nbconvert/exporters/python.py

@@ -29,6 +29,4 @@ class PythonExporter(TemplateExporter):
         'py', config=True, 
         help="Extension of the file that should be written to disk")
 
-    def _raw_mimetype_default(self):
-        return 'application/x-python'
-
+    output_mimetype = 'text/x-python'