From 02fa348c69bc6c1053d326a8486bba9e99d92525 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 8 May 2014 16:18:23 -0700 Subject: [PATCH 1/6] Create REST API for kernel specs --- IPython/html/base/handlers.py | 4 + IPython/html/notebookapp.py | 24 ++++-- IPython/html/services/kernelspecs/handlers.py | 75 +++++++++++++++++++ 3 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 IPython/html/services/kernelspecs/handlers.py diff --git a/IPython/html/base/handlers.py b/IPython/html/base/handlers.py index 03070c0ac..56772f979 100644 --- a/IPython/html/base/handlers.py +++ b/IPython/html/base/handlers.py @@ -158,6 +158,10 @@ class IPythonHandler(AuthenticatedHandler): def session_manager(self): return self.settings['session_manager'] + @property + def kernel_spec_manager(self): + return self.settings['kernel_spec_manager'] + @property def project_dir(self): return self.notebook_manager.notebook_dir diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index fc08f6b1d..c36d901d5 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -67,12 +67,13 @@ from IPython.core.application import ( ) from IPython.core.profiledir import ProfileDir from IPython.kernel import KernelManager +from IPython.kernel.kernelspec import KernelSpecManager from IPython.kernel.zmq.session import default_secure, Session from IPython.nbformat.sign import NotebookNotary from IPython.utils.importstring import import_item from IPython.utils import submodule from IPython.utils.traitlets import ( - Dict, Unicode, Integer, List, Bool, Bytes, + Dict, Unicode, Integer, List, Bool, Bytes, Instance, DottedObjectName, TraitError, ) from IPython.utils import py3compat @@ -118,19 +119,21 @@ def load_handlers(name): class NotebookWebApplication(web.Application): def __init__(self, ipython_app, kernel_manager, notebook_manager, - cluster_manager, session_manager, log, base_url, - settings_overrides, jinja_env_options): + cluster_manager, session_manager, kernel_spec_manager, log, + base_url, settings_overrides, jinja_env_options): settings = self.init_settings( ipython_app, kernel_manager, notebook_manager, cluster_manager, - session_manager, log, base_url, settings_overrides, jinja_env_options) + session_manager, kernel_spec_manager, log, base_url, + settings_overrides, jinja_env_options) handlers = self.init_handlers(settings) super(NotebookWebApplication, self).__init__(handlers, **settings) def init_settings(self, ipython_app, kernel_manager, notebook_manager, - cluster_manager, session_manager, log, base_url, - settings_overrides, jinja_env_options=None): + cluster_manager, session_manager, kernel_spec_manager, + log, base_url, settings_overrides, + jinja_env_options=None): # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and # base_url will always be unicode, which will in turn # make the patterns unicode, and ultimately result in unicode @@ -162,6 +165,7 @@ class NotebookWebApplication(web.Application): notebook_manager=notebook_manager, cluster_manager=cluster_manager, session_manager=session_manager, + kernel_spec_manager=kernel_spec_manager, # IPython stuff nbextensions_path = ipython_app.nbextensions_path, @@ -188,6 +192,7 @@ class NotebookWebApplication(web.Application): handlers.extend(load_handlers('services.clusters.handlers')) handlers.extend(load_handlers('services.sessions.handlers')) handlers.extend(load_handlers('services.nbconvert.handlers')) + handlers.extend(load_handlers('services.kernelspecs.handlers')) # FIXME: /files/ should be handled by the Contents service when it exists nbm = settings['notebook_manager'] if hasattr(nbm, 'notebook_dir'): @@ -510,6 +515,11 @@ class NotebookApp(BaseIPythonApplication): help='The cluster manager class to use.' ) + kernel_spec_manager = Instance(KernelSpecManager) + + def _kernel_spec_manager_default(self): + return KernelSpecManager(ipython_dir=self.ipython_dir) + trust_xheaders = Bool(False, config=True, help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL") @@ -616,7 +626,7 @@ class NotebookApp(BaseIPythonApplication): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication( self, self.kernel_manager, self.notebook_manager, - self.cluster_manager, self.session_manager, + self.cluster_manager, self.session_manager, self.kernel_spec_manager, self.log, self.base_url, self.webapp_settings, self.jinja_environment_options ) diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py new file mode 100644 index 000000000..1840cf402 --- /dev/null +++ b/IPython/html/services/kernelspecs/handlers.py @@ -0,0 +1,75 @@ +"""Tornado handlers for kernel specifications.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import logging +from tornado import web + +from zmq.utils import jsonapi + +from ...base.handlers import IPythonHandler, json_errors, path_regex + + +class MainKernelSpecHandler(IPythonHandler): + SUPPORTED_METHODS = ('GET',) + + @web.authenticated + @json_errors + def get(self): + ksm = self.kernel_spec_manager + results = [] + for kernel_name in ksm.find_kernel_specs(): + results.append(dict(name=kernel_name, + display_name=ksm.get_kernel_spec(kernel_name).display_name)) + + self.set_header("Content-Type", 'application/json') + self.finish(jsonapi.dumps(results)) + + +class KernelSpecHandler(IPythonHandler): + SUPPORTED_METHODS = ('GET',) + + @web.authenticated + @json_errors + def get(self, kernel_name): + ksm = self.kernel_spec_manager + kernelspec = ksm.get_kernel_spec(kernel_name) + self.set_header("Content-Type", 'application/json') + self.finish(kernelspec.to_json()) + + +class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): + SUPPORTED_METHODS = ('GET', 'HEAD') + + def initialize(self): + web.StaticFileHandler.initialize(self, path='') + + def get(self, kernel_name, path, include_body=True): + ksm = self.kernel_spec_manager + self.root = ksm.get_kernel_spec(kernel_name).resource_dir + self.log.warn("Set root: %s", self.root) + return web.StaticFileHandler.get(self, path, include_body=include_body) + +# @classmethod +# def get_absolute_path(cls, root, path): +# res = web.StaticFileHandler.get_absolute_path(cls, root, path) +# self.log.warn("Full path: %s", res) +# return res + + def head(self, kernel_name, path): + self.get(kernel_name, path, include_body=False) + + +#----------------------------------------------------------------------------- +# URL to handler mappings +#----------------------------------------------------------------------------- + + +_kernel_name_regex = r"(?P\w+)" + +default_handlers = [ + (r"/api/kernelspecs", MainKernelSpecHandler), + (r"/api/kernelspecs/%s" % _kernel_name_regex, KernelSpecHandler), + (r"/api/kernelspecs/%s/(?P.*)" % _kernel_name_regex, KernelSpecResourceHandler), +] From 1dabf3c54773c47c9ed0a597d72c9fbca70236dc Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Thu, 8 May 2014 17:32:47 -0700 Subject: [PATCH 2/6] Test kernel specs REST API And fix kernel not found producing a 404 code. --- IPython/html/services/kernelspecs/handlers.py | 24 ++--- .../kernelspecs/tests/test_kernelspecs_api.py | 94 +++++++++++++++++++ 2 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py index 1840cf402..6e0b0c7a1 100644 --- a/IPython/html/services/kernelspecs/handlers.py +++ b/IPython/html/services/kernelspecs/handlers.py @@ -3,12 +3,11 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. -import logging from tornado import web from zmq.utils import jsonapi -from ...base.handlers import IPythonHandler, json_errors, path_regex +from ...base.handlers import IPythonHandler, json_errors class MainKernelSpecHandler(IPythonHandler): @@ -34,7 +33,10 @@ class KernelSpecHandler(IPythonHandler): @json_errors def get(self, kernel_name): ksm = self.kernel_spec_manager - kernelspec = ksm.get_kernel_spec(kernel_name) + try: + kernelspec = ksm.get_kernel_spec(kernel_name) + except KeyError: + raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) self.set_header("Content-Type", 'application/json') self.finish(kernelspec.to_json()) @@ -47,24 +49,18 @@ class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): def get(self, kernel_name, path, include_body=True): ksm = self.kernel_spec_manager - self.root = ksm.get_kernel_spec(kernel_name).resource_dir - self.log.warn("Set root: %s", self.root) + try: + self.root = ksm.get_kernel_spec(kernel_name).resource_dir + except KeyError: + raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) + self.log.debug("Serving kernel resource from: %s", self.root) return web.StaticFileHandler.get(self, path, include_body=include_body) -# @classmethod -# def get_absolute_path(cls, root, path): -# res = web.StaticFileHandler.get_absolute_path(cls, root, path) -# self.log.warn("Full path: %s", res) -# return res - def head(self, kernel_name, path): self.get(kernel_name, path, include_body=False) -#----------------------------------------------------------------------------- # URL to handler mappings -#----------------------------------------------------------------------------- - _kernel_name_regex = r"(?P\w+)" diff --git a/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py new file mode 100644 index 000000000..0a7086bee --- /dev/null +++ b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py @@ -0,0 +1,94 @@ +# coding: utf-8 +"""Test the kernel specs webservice API.""" + +import errno +import io +import json +import os + +pjoin = os.path.join + +import requests + +from IPython.html.utils import url_path_join +from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error + +# Copied from IPython.kernel.tests.test_kernelspec so updating that doesn't +# break these tests +sample_kernel_json = {'argv':['cat', '{connection_file}'], + 'display_name':'Test kernel', + 'language':'bash', + } + +some_resource = u"The very model of a modern major general" + + +class KernelSpecAPI(object): + """Wrapper for notebook API calls.""" + def __init__(self, base_url): + self.base_url = base_url + + def _req(self, verb, path, body=None): + response = requests.request(verb, + url_path_join(self.base_url, 'api/kernelspecs', path), + data=body, + ) + response.raise_for_status() + return response + + def list(self): + return self._req('GET', '') + + def kernel_spec_info(self, name): + return self._req('GET', name) + + def kernel_resource(self, name, path): + return self._req('GET', url_path_join(name, path)) + +class APITest(NotebookTestBase): + """Test the kernelspec web service API""" + def setUp(self): + ipydir = self.ipython_dir.name + sample_kernel_dir = pjoin(ipydir, 'kernels', 'sample') + try: + os.makedirs(sample_kernel_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + with open(pjoin(sample_kernel_dir, 'kernel.json'), 'w') as f: + json.dump(sample_kernel_json, f) + + with io.open(pjoin(sample_kernel_dir, 'resource.txt'), 'w', + encoding='utf-8') as f: + f.write(some_resource) + + self.ks_api = KernelSpecAPI(self.base_url()) + + def test_list_kernelspecs(self): + specs = self.ks_api.list().json() + assert isinstance(specs, list) + + # 2: the sample kernelspec created in setUp, and the native Python kernel + self.assertEqual(len(specs), 2) + assert any(s == {'name': 'sample', 'display_name': 'Test kernel'} + for s in specs), specs + + def test_get_kernelspec(self): + spec = self.ks_api.kernel_spec_info('Sample').json() # Case insensitive + self.assertEqual(spec['language'], 'bash') + + def test_get_nonexistant_kernelspec(self): + with assert_http_error(404): + self.ks_api.kernel_spec_info('nonexistant') + + def test_get_kernel_resource_file(self): + res = self.ks_api.kernel_resource('sAmple', 'resource.txt') + self.assertEqual(res.text, some_resource) + + def test_get_nonexistant_resource(self): + with assert_http_error(404): + self.ks_api.kernel_resource('nonexistant', 'resource.txt') + + with assert_http_error(404): + self.ks_api.kernel_resource('sample', 'nonexistant.txt') From f6628b7006a1167d670d1dc669831251de2222f7 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Sat, 10 May 2014 12:22:20 -0700 Subject: [PATCH 3/6] Refactor kernelspec resource handler to separate URL prefix --- IPython/html/kernelspecs/handlers.py | 25 +++++++++++++++++++ IPython/html/notebookapp.py | 1 + IPython/html/services/kernelspecs/handlers.py | 24 ++---------------- .../kernelspecs/tests/test_kernelspecs_api.py | 8 +++--- 4 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 IPython/html/kernelspecs/handlers.py diff --git a/IPython/html/kernelspecs/handlers.py b/IPython/html/kernelspecs/handlers.py new file mode 100644 index 000000000..9940440b6 --- /dev/null +++ b/IPython/html/kernelspecs/handlers.py @@ -0,0 +1,25 @@ +from tornado import web +from ..base.handlers import IPythonHandler +from ..services.kernelspecs.handlers import kernel_name_regex + +class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): + SUPPORTED_METHODS = ('GET', 'HEAD') + + def initialize(self): + web.StaticFileHandler.initialize(self, path='') + + def get(self, kernel_name, path, include_body=True): + ksm = self.kernel_spec_manager + try: + self.root = ksm.get_kernel_spec(kernel_name).resource_dir + except KeyError: + raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) + self.log.debug("Serving kernel resource from: %s", self.root) + return web.StaticFileHandler.get(self, path, include_body=include_body) + + def head(self, kernel_name, path): + self.get(kernel_name, path, include_body=False) + +default_handlers = [ + (r"/kernelspecs/%s/(?P.*)" % kernel_name_regex, KernelSpecResourceHandler), +] \ No newline at end of file diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index c36d901d5..9c555d640 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -187,6 +187,7 @@ class NotebookWebApplication(web.Application): handlers.extend(load_handlers('auth.logout')) handlers.extend(load_handlers('notebook.handlers')) handlers.extend(load_handlers('nbconvert.handlers')) + handlers.extend(load_handlers('kernelspecs.handlers')) handlers.extend(load_handlers('services.kernels.handlers')) handlers.extend(load_handlers('services.notebooks.handlers')) handlers.extend(load_handlers('services.clusters.handlers')) diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py index 6e0b0c7a1..3aeb5545f 100644 --- a/IPython/html/services/kernelspecs/handlers.py +++ b/IPython/html/services/kernelspecs/handlers.py @@ -41,31 +41,11 @@ class KernelSpecHandler(IPythonHandler): self.finish(kernelspec.to_json()) -class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): - SUPPORTED_METHODS = ('GET', 'HEAD') - - def initialize(self): - web.StaticFileHandler.initialize(self, path='') - - def get(self, kernel_name, path, include_body=True): - ksm = self.kernel_spec_manager - try: - self.root = ksm.get_kernel_spec(kernel_name).resource_dir - except KeyError: - raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) - self.log.debug("Serving kernel resource from: %s", self.root) - return web.StaticFileHandler.get(self, path, include_body=include_body) - - def head(self, kernel_name, path): - self.get(kernel_name, path, include_body=False) - - # URL to handler mappings -_kernel_name_regex = r"(?P\w+)" +kernel_name_regex = r"(?P\w+)" default_handlers = [ (r"/api/kernelspecs", MainKernelSpecHandler), - (r"/api/kernelspecs/%s" % _kernel_name_regex, KernelSpecHandler), - (r"/api/kernelspecs/%s/(?P.*)" % _kernel_name_regex, KernelSpecResourceHandler), + (r"/api/kernelspecs/%s" % kernel_name_regex, KernelSpecHandler), ] diff --git a/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py index 0a7086bee..90224151f 100644 --- a/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py +++ b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py @@ -30,20 +30,20 @@ class KernelSpecAPI(object): def _req(self, verb, path, body=None): response = requests.request(verb, - url_path_join(self.base_url, 'api/kernelspecs', path), + url_path_join(self.base_url, path), data=body, ) response.raise_for_status() return response def list(self): - return self._req('GET', '') + return self._req('GET', 'api/kernelspecs') def kernel_spec_info(self, name): - return self._req('GET', name) + return self._req('GET', url_path_join('api/kernelspecs', name)) def kernel_resource(self, name, path): - return self._req('GET', url_path_join(name, path)) + return self._req('GET', url_path_join('kernelspecs', name, path)) class APITest(NotebookTestBase): """Test the kernelspec web service API""" From 76fdfe9eb7531a57595d06f07043f85edd1affdc Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 12 May 2014 12:14:37 -0700 Subject: [PATCH 4/6] Put full kernel info in REST API response for all kernels --- IPython/html/services/kernelspecs/handlers.py | 5 +++-- .../services/kernelspecs/tests/test_kernelspecs_api.py | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py index 3aeb5545f..975cd5d9f 100644 --- a/IPython/html/services/kernelspecs/handlers.py +++ b/IPython/html/services/kernelspecs/handlers.py @@ -19,8 +19,9 @@ class MainKernelSpecHandler(IPythonHandler): ksm = self.kernel_spec_manager results = [] for kernel_name in ksm.find_kernel_specs(): - results.append(dict(name=kernel_name, - display_name=ksm.get_kernel_spec(kernel_name).display_name)) + d = ksm.get_kernel_spec(kernel_name).to_dict() + d['name'] = kernel_name + results.append(d) self.set_header("Content-Type", 'application/json') self.finish(jsonapi.dumps(results)) diff --git a/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py index 90224151f..ad2a5fbdb 100644 --- a/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py +++ b/IPython/html/services/kernelspecs/tests/test_kernelspecs_api.py @@ -68,11 +68,14 @@ class APITest(NotebookTestBase): def test_list_kernelspecs(self): specs = self.ks_api.list().json() assert isinstance(specs, list) - + # 2: the sample kernelspec created in setUp, and the native Python kernel self.assertEqual(len(specs), 2) - assert any(s == {'name': 'sample', 'display_name': 'Test kernel'} - for s in specs), specs + + def is_sample_kernelspec(s): + return s['name'] == 'sample' and s['display_name'] == 'Test kernel' + + assert any(is_sample_kernelspec(s) for s in specs), specs def test_get_kernelspec(self): spec = self.ks_api.kernel_spec_info('Sample').json() # Case insensitive From 85ce34bc66e0ec7dafb8ad12c9da30d702dda732 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 16 Jun 2014 15:24:30 -0700 Subject: [PATCH 5/6] Kernel resource handlers now require authenticated user --- IPython/html/kernelspecs/handlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/html/kernelspecs/handlers.py b/IPython/html/kernelspecs/handlers.py index 9940440b6..26eecf1b1 100644 --- a/IPython/html/kernelspecs/handlers.py +++ b/IPython/html/kernelspecs/handlers.py @@ -8,6 +8,7 @@ class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): def initialize(self): web.StaticFileHandler.initialize(self, path='') + @web.authenticated def get(self, kernel_name, path, include_body=True): ksm = self.kernel_spec_manager try: @@ -16,7 +17,8 @@ class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) self.log.debug("Serving kernel resource from: %s", self.root) return web.StaticFileHandler.get(self, path, include_body=include_body) - + + @web.authenticated def head(self, kernel_name, path): self.get(kernel_name, path, include_body=False) From ebf0ea32ef818ef570dd5565a72c65f2005b656b Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 18 Jun 2014 12:51:41 -0700 Subject: [PATCH 6/6] Use stdlib json, not zmq jsonapi --- IPython/html/services/kernelspecs/handlers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py index 975cd5d9f..dbe8382a8 100644 --- a/IPython/html/services/kernelspecs/handlers.py +++ b/IPython/html/services/kernelspecs/handlers.py @@ -2,11 +2,9 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. - +import json from tornado import web -from zmq.utils import jsonapi - from ...base.handlers import IPythonHandler, json_errors @@ -24,7 +22,7 @@ class MainKernelSpecHandler(IPythonHandler): results.append(d) self.set_header("Content-Type", 'application/json') - self.finish(jsonapi.dumps(results)) + self.finish(json.dumps(results)) class KernelSpecHandler(IPythonHandler):