From a76a5466276c557985c4a6a6d84f3f2ad2429eff Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 14 Oct 2014 12:40:21 -0700 Subject: [PATCH 1/6] Add REST API for retrieving, storing and updating config --- IPython/html/base/handlers.py | 4 ++ IPython/html/notebookapp.py | 2 + IPython/html/services/config/__init__.py | 0 IPython/html/services/config/handlers.py | 65 ++++++++++++++++++ .../services/config/tests/test_config_api.py | 67 +++++++++++++++++++ 5 files changed, 138 insertions(+) create mode 100644 IPython/html/services/config/__init__.py create mode 100644 IPython/html/services/config/handlers.py create mode 100644 IPython/html/services/config/tests/test_config_api.py diff --git a/IPython/html/base/handlers.py b/IPython/html/base/handlers.py index a52f1d6f9..7c5d27d55 100644 --- a/IPython/html/base/handlers.py +++ b/IPython/html/base/handlers.py @@ -120,6 +120,10 @@ class IPythonHandler(AuthenticatedHandler): return Application.instance().log else: return app_log + + @property + def profile_dir(self): + return self.settings.get('profile_dir', '') #--------------------------------------------------------------- # URLs diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index bf7f4f8d5..cc3793308 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -174,6 +174,7 @@ class NotebookWebApplication(web.Application): config=ipython_app.config, jinja2_env=env, terminals_available=False, # Set later if terminals are available + profile_dir = ipython_app.profile_dir.location, ) # allow custom overrides for the tornado web app. @@ -191,6 +192,7 @@ class NotebookWebApplication(web.Application): handlers.extend(load_handlers('notebook.handlers')) handlers.extend(load_handlers('nbconvert.handlers')) handlers.extend(load_handlers('kernelspecs.handlers')) + handlers.extend(load_handlers('services.config.handlers')) handlers.extend(load_handlers('services.kernels.handlers')) handlers.extend(load_handlers('services.contents.handlers')) handlers.extend(load_handlers('services.clusters.handlers')) diff --git a/IPython/html/services/config/__init__.py b/IPython/html/services/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py new file mode 100644 index 000000000..9829681ba --- /dev/null +++ b/IPython/html/services/config/handlers.py @@ -0,0 +1,65 @@ +"""Tornado handlers for kernel specifications.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +import io +from tornado import web + +from ...base.handlers import IPythonHandler, json_errors + + +class ConfigHandler(IPythonHandler): + SUPPORTED_METHODS = ('GET', 'PUT', 'PATCH') + + def file_name(self, section_name): + return os.path.join(self.profile_dir, 'nb_%s_config.json' % section_name) + + @web.authenticated + @json_errors + def get(self, section_name): + self.set_header("Content-Type", 'application/json') + filename = self.file_name(section_name) + if os.path.isfile(filename): + with io.open(filename, encoding='utf-8') as f: + self.finish(f.read()) + else: + self.finish("{}") + + @web.authenticated + @json_errors + def put(self, section_name): + filename = self.file_name(section_name) + with open(filename, 'wb') as f: + f.write(self.request.body) + self.set_status(204) + + @web.authenticated + @json_errors + def patch(self, section_name): + filename = self.file_name(section_name) + if os.path.isfile(filename): + with io.open(filename, encoding='utf-8') as f: + section = json.load(f) + else: + section = {} + + for k, v in self.get_json_body().items(): + if v is None: + section.pop(k, None) + else: + section[k] = v + + with io.open(filename, 'w', encoding='utf-8') as f: + json.dump(section, f) + self.set_status(204) + + +# URL to handler mappings + +section_name_regex = r"(?P\w+)" + +default_handlers = [ + (r"/api/config/%s" % section_name_regex, ConfigHandler), +] diff --git a/IPython/html/services/config/tests/test_config_api.py b/IPython/html/services/config/tests/test_config_api.py new file mode 100644 index 000000000..fac4e15ad --- /dev/null +++ b/IPython/html/services/config/tests/test_config_api.py @@ -0,0 +1,67 @@ +# coding: utf-8 +"""Test the kernel specs webservice API.""" + +import json + +import requests + +from IPython.html.utils import url_path_join +from IPython.html.tests.launchnotebook import NotebookTestBase + + +class ConfigAPI(object): + """Wrapper for notebook API calls.""" + def __init__(self, base_url): + self.base_url = base_url + + def _req(self, verb, section, body=None): + response = requests.request(verb, + url_path_join(self.base_url, 'api/config', section), + data=body, + ) + response.raise_for_status() + return response + + def get(self, section): + return self._req('GET', section) + + def set(self, section, values): + return self._req('PUT', section, json.dumps(values)) + + def modify(self, section, values): + return self._req('PATCH', section, json.dumps(values)) + +class APITest(NotebookTestBase): + """Test the kernelspec web service API""" + def setUp(self): + self.config_api = ConfigAPI(self.base_url()) + + def test_create_retrieve_config(self): + sample = {'foo': 'bar', 'baz': 73} + r = self.config_api.set('example', sample) + self.assertEqual(r.status_code, 204) + + r = self.config_api.get('example') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json(), sample) + + def test_modify(self): + sample = {'foo': 'bar', 'baz': 73} + self.config_api.set('example', sample) + + r = self.config_api.modify('example', {'foo': None, # should delete foo + 'baz': 75, + 'wib': [1,2,3], + }) + self.assertEqual(r.status_code, 204) + + r = self.config_api.get('example') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json(), {'baz': 75, 'wib': [1,2,3]}) + + def test_get_unknown(self): + # We should get an empty config dictionary instead of a 404 + r = self.config_api.get('nonexistant') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json(), {}) + From 6786f86c638847582706cebc7f650115c0ea3b3e Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 14 Oct 2014 15:47:09 -0700 Subject: [PATCH 2/6] Fix docstring, validate JSON on PUT --- IPython/html/services/config/handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py index 9829681ba..4f27dd392 100644 --- a/IPython/html/services/config/handlers.py +++ b/IPython/html/services/config/handlers.py @@ -1,4 +1,4 @@ -"""Tornado handlers for kernel specifications.""" +"""Tornado handlers for frontend config storage.""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. @@ -30,6 +30,7 @@ class ConfigHandler(IPythonHandler): @web.authenticated @json_errors def put(self, section_name): + self.get_json_body() # Will raise 400 if content is not valid JSON filename = self.file_name(section_name) with open(filename, 'wb') as f: f.write(self.request.body) From 425d5a1c02b54d2a45ff33b942a5aedc592f7bd1 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 20 Oct 2014 14:14:53 -0700 Subject: [PATCH 3/6] Apply JSON config updates recursively --- IPython/html/services/config/handlers.py | 26 +++++++++++++++---- .../services/config/tests/test_config_api.py | 12 ++++++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py index 4f27dd392..a9247469b 100644 --- a/IPython/html/services/config/handlers.py +++ b/IPython/html/services/config/handlers.py @@ -9,6 +9,25 @@ from tornado import web from ...base.handlers import IPythonHandler, json_errors +def recursive_update(target, new): + """Recursively update one dictionary using another. + + None values will delete their keys. + """ + for k, v in new.items(): + if isinstance(v, dict): + if k not in target: + target[k] = {} + recursive_update(target[k], v) + if not target[k]: + # Prune empty subdicts + del target[k] + + elif v is None: + target.pop(k, None) + + else: + target[k] = v class ConfigHandler(IPythonHandler): SUPPORTED_METHODS = ('GET', 'PUT', 'PATCH') @@ -46,11 +65,8 @@ class ConfigHandler(IPythonHandler): else: section = {} - for k, v in self.get_json_body().items(): - if v is None: - section.pop(k, None) - else: - section[k] = v + update = self.get_json_body() + recursive_update(section, update) with io.open(filename, 'w', encoding='utf-8') as f: json.dump(section, f) diff --git a/IPython/html/services/config/tests/test_config_api.py b/IPython/html/services/config/tests/test_config_api.py index fac4e15ad..aece326b9 100644 --- a/IPython/html/services/config/tests/test_config_api.py +++ b/IPython/html/services/config/tests/test_config_api.py @@ -1,5 +1,5 @@ # coding: utf-8 -"""Test the kernel specs webservice API.""" +"""Test the config webservice API.""" import json @@ -32,7 +32,7 @@ class ConfigAPI(object): return self._req('PATCH', section, json.dumps(values)) class APITest(NotebookTestBase): - """Test the kernelspec web service API""" + """Test the config web service API""" def setUp(self): self.config_api = ConfigAPI(self.base_url()) @@ -46,18 +46,22 @@ class APITest(NotebookTestBase): self.assertEqual(r.json(), sample) def test_modify(self): - sample = {'foo': 'bar', 'baz': 73} + sample = {'foo': 'bar', 'baz': 73, + 'sub': {'a': 6, 'b': 7}, 'sub2': {'c': 8}} self.config_api.set('example', sample) r = self.config_api.modify('example', {'foo': None, # should delete foo 'baz': 75, 'wib': [1,2,3], + 'sub': {'a': 8, 'b': None, 'd': 9}, + 'sub2': {'c': None} # should delete sub2 }) self.assertEqual(r.status_code, 204) r = self.config_api.get('example') self.assertEqual(r.status_code, 200) - self.assertEqual(r.json(), {'baz': 75, 'wib': [1,2,3]}) + self.assertEqual(r.json(), {'baz': 75, 'wib': [1,2,3], + 'sub': {'a': 8, 'd': 9}}) def test_get_unknown(self): # We should get an empty config dictionary instead of a 404 From c26b0f6961aa214c59db9060562fc50683bd3600 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 20 Oct 2014 14:56:10 -0700 Subject: [PATCH 4/6] Fix writing JSON on Python 2 --- IPython/html/services/config/handlers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py index a9247469b..04d7e193e 100644 --- a/IPython/html/services/config/handlers.py +++ b/IPython/html/services/config/handlers.py @@ -7,6 +7,7 @@ import os import io from tornado import web +from IPython.utils.py3compat import PY3 from ...base.handlers import IPythonHandler, json_errors def recursive_update(target, new): @@ -68,8 +69,13 @@ class ConfigHandler(IPythonHandler): update = self.get_json_body() recursive_update(section, update) - with io.open(filename, 'w', encoding='utf-8') as f: + if PY3: + f = io.open(filename, 'w', encoding='utf-8') + else: + f = open(filename, 'wb') + with f: json.dump(section, f) + self.set_status(204) From ae6ae8f7b06f23e7c0da20fe3c8e8e6132e948a5 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Mon, 27 Oct 2014 16:32:55 -0700 Subject: [PATCH 5/6] Return updated config from PATCH requests --- IPython/html/services/config/handlers.py | 2 +- IPython/html/services/config/tests/test_config_api.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py index 04d7e193e..7950aab0a 100644 --- a/IPython/html/services/config/handlers.py +++ b/IPython/html/services/config/handlers.py @@ -76,7 +76,7 @@ class ConfigHandler(IPythonHandler): with f: json.dump(section, f) - self.set_status(204) + self.finish(json.dumps(section)) # URL to handler mappings diff --git a/IPython/html/services/config/tests/test_config_api.py b/IPython/html/services/config/tests/test_config_api.py index aece326b9..463fc4e25 100644 --- a/IPython/html/services/config/tests/test_config_api.py +++ b/IPython/html/services/config/tests/test_config_api.py @@ -56,9 +56,6 @@ class APITest(NotebookTestBase): 'sub': {'a': 8, 'b': None, 'd': 9}, 'sub2': {'c': None} # should delete sub2 }) - self.assertEqual(r.status_code, 204) - - r = self.config_api.get('example') self.assertEqual(r.status_code, 200) self.assertEqual(r.json(), {'baz': 75, 'wib': [1,2,3], 'sub': {'a': 8, 'd': 9}}) From 0f44eced721071f08c5c9dede0984eb2e81f4b40 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 12 Nov 2014 15:53:35 -0800 Subject: [PATCH 6/6] Put frontend config files in profile_foo/nbconfig/ subdir --- IPython/html/services/config/handlers.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/IPython/html/services/config/handlers.py b/IPython/html/services/config/handlers.py index 7950aab0a..411a0abab 100644 --- a/IPython/html/services/config/handlers.py +++ b/IPython/html/services/config/handlers.py @@ -5,6 +5,7 @@ import json import os import io +import errno from tornado import web from IPython.utils.py3compat import PY3 @@ -33,8 +34,19 @@ def recursive_update(target, new): class ConfigHandler(IPythonHandler): SUPPORTED_METHODS = ('GET', 'PUT', 'PATCH') + @property + def config_dir(self): + return os.path.join(self.profile_dir, 'nbconfig') + + def ensure_config_dir_exists(self): + try: + os.mkdir(self.config_dir, 0o755) + except OSError as e: + if e.errno != errno.EEXIST: + raise + def file_name(self, section_name): - return os.path.join(self.profile_dir, 'nb_%s_config.json' % section_name) + return os.path.join(self.config_dir, section_name+'.json') @web.authenticated @json_errors @@ -52,6 +64,7 @@ class ConfigHandler(IPythonHandler): def put(self, section_name): self.get_json_body() # Will raise 400 if content is not valid JSON filename = self.file_name(section_name) + self.ensure_config_dir_exists() with open(filename, 'wb') as f: f.write(self.request.body) self.set_status(204) @@ -69,6 +82,7 @@ class ConfigHandler(IPythonHandler): update = self.get_json_body() recursive_update(section, update) + self.ensure_config_dir_exists() if PY3: f = io.open(filename, 'w', encoding='utf-8') else: