diff --git a/notebook/notebookapp.py b/notebook/notebookapp.py index 498405a6d..4b6a98dd8 100755 --- a/notebook/notebookapp.py +++ b/notebook/notebookapp.py @@ -75,6 +75,7 @@ from .services.kernels.kernelmanager import MappingKernelManager from .services.config import ConfigManager from .services.contents.manager import ContentsManager from .services.contents.filemanager import FileContentsManager +from .services.contents.largefilemanager import LargeFileManager from .services.sessions.sessionmanager import SessionManager from .auth.login import LoginHandler @@ -843,7 +844,7 @@ class NotebookApp(JupyterApp): self.log.info("Using MathJax configuration file: %s", change['new']) contents_manager_class = Type( - default_value=FileContentsManager, + default_value=LargeFileManager, klass=ContentsManager, config=True, help='The notebook manager class to use.' diff --git a/notebook/services/contents/handlers.py b/notebook/services/contents/handlers.py index d4e528abd..640309065 100644 --- a/notebook/services/contents/handlers.py +++ b/notebook/services/contents/handlers.py @@ -176,7 +176,9 @@ class ContentsHandler(APIHandler): @gen.coroutine def _save(self, model, path): """Save an existing file.""" - self.log.info(u"Saving file at %s", path) + chunk = model.get("chunk", None) + if not chunk or chunk == -1: # Avoid tedious log information + self.log.info(u"Saving file at %s", path) model = yield gen.maybe_future(self.contents_manager.save(model, path)) validate_model(model, expect_content=False) self._finish_model(model) diff --git a/notebook/services/contents/largefilemanager.py b/notebook/services/contents/largefilemanager.py new file mode 100644 index 000000000..59328ef7e --- /dev/null +++ b/notebook/services/contents/largefilemanager.py @@ -0,0 +1,70 @@ +from notebook.services.contents.filemanager import FileContentsManager +from contextlib import contextmanager +from tornado import web +import nbformat +import base64 +import os, io + +class LargeFileManager(FileContentsManager): + """Handle large file upload.""" + + def save(self, model, path=''): + """Save the file model and return the model with no content.""" + chunk = model.get('chunk', None) + if chunk is not None: + path = path.strip('/') + + if 'type' not in model: + raise web.HTTPError(400, u'No file type provided') + if model['type'] != 'file': + raise web.HTTPError(400, u'File type "{}" is not supported for large file transfer'.format(model['type'])) + if 'content' not in model and model['type'] != 'directory': + raise web.HTTPError(400, u'No file content provided') + + os_path = self._get_os_path(path) + + try: + if chunk == 1: + self.log.debug("Saving %s", os_path) + self.run_pre_save_hook(model=model, path=path) + super(LargeFileManager, self)._save_file(os_path, model['content'], model.get('format')) + else: + self._save_large_file(os_path, model['content'], model.get('format')) + except web.HTTPError: + raise + except Exception as e: + self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True) + raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e)) + + model = self.get(path, content=False) + + # Last chunk + if chunk == -1: + self.run_post_save_hook(model=model, os_path=os_path) + return model + else: + return super(LargeFileManager, self).save(model, path) + + def _save_large_file(self, os_path, content, format): + """Save content of a generic file.""" + if format not in {'text', 'base64'}: + raise web.HTTPError( + 400, + "Must specify format of file contents as 'text' or 'base64'", + ) + try: + if format == 'text': + bcontent = content.encode('utf8') + else: + b64_bytes = content.encode('ascii') + bcontent = base64.decodestring(b64_bytes) + except Exception as e: + raise web.HTTPError( + 400, u'Encoding error saving %s: %s' % (os_path, e) + ) + + with self.perm_to_403(os_path): + if os.path.islink(os_path): + os_path = os.path.join(os.path.dirname(os_path), os.readlink(os_path)) + with io.open(os_path, 'ab') as f: + f.write(bcontent) diff --git a/notebook/services/contents/tests/test_largefilemanager.py b/notebook/services/contents/tests/test_largefilemanager.py new file mode 100644 index 000000000..13d294b9b --- /dev/null +++ b/notebook/services/contents/tests/test_largefilemanager.py @@ -0,0 +1,113 @@ +from unittest import TestCase +from ipython_genutils.tempdir import TemporaryDirectory +from ..largefilemanager import LargeFileManager +import os +from tornado import web + + +def _make_dir(contents_manager, api_path): + """ + Make a directory. + """ + os_path = contents_manager._get_os_path(api_path) + try: + os.makedirs(os_path) + except OSError: + print("Directory already exists: %r" % os_path) + + +class TestLargeFileManager(TestCase): + + def setUp(self): + self._temp_dir = TemporaryDirectory() + self.td = self._temp_dir.name + self.contents_manager = LargeFileManager(root_dir=self.td) + + def make_dir(self, api_path): + """make a subdirectory at api_path + + override in subclasses if contents are not on the filesystem. + """ + _make_dir(self.contents_manager, api_path) + + def test_save(self): + + cm = self.contents_manager + # Create a notebook + model = cm.new_untitled(type='notebook') + name = model['name'] + path = model['path'] + + # Get the model with 'content' + full_model = cm.get(path) + # Save the notebook + model = cm.save(full_model, path) + assert isinstance(model, dict) + self.assertIn('name', model) + self.assertIn('path', model) + self.assertEqual(model['name'], name) + self.assertEqual(model['path'], path) + + try: + model = {'name': 'test', 'path': 'test', 'chunk': 1} + cm.save(model, model['path']) + except web.HTTPError as e: + self.assertEqual('HTTP 400: Bad Request (No file type provided)', str(e)) + + try: + model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'notebook'} + cm.save(model, model['path']) + except web.HTTPError as e: + self.assertEqual('HTTP 400: Bad Request (File type "notebook" is not supported for large file transfer)', str(e)) + + try: + model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'file'} + cm.save(model, model['path']) + except web.HTTPError as e: + self.assertEqual('HTTP 400: Bad Request (No file content provided)', str(e)) + + try: + model = {'name': 'test', 'path': 'test', 'chunk': 2, 'type': 'file', + 'content': u'test', 'format': 'json'} + cm.save(model, model['path']) + except web.HTTPError as e: + self.assertEqual("HTTP 400: Bad Request (Must specify format of file contents as 'text' or 'base64')", + str(e)) + + # Save model for different chunks + model = {'name': 'test', 'path': 'test', 'type': 'file', + 'content': u'test==', 'format': 'text'} + name = model['name'] + path = model['path'] + cm.save(model, path) + + for chunk in (1, 2, -1): + for fm in ('text', 'base64'): + full_model = cm.get(path) + full_model['chunk'] = chunk + full_model['format'] = fm + model_res = cm.save(full_model, path) + assert isinstance(model_res, dict) + + self.assertIn('name', model_res) + self.assertIn('path', model_res) + self.assertNotIn('chunk', model_res) + self.assertEqual(model_res['name'], name) + self.assertEqual(model_res['path'], path) + + # Test in sub-directory + # Create a directory and notebook in that directory + sub_dir = '/foo/' + self.make_dir('foo') + model = cm.new_untitled(path=sub_dir, type='notebook') + name = model['name'] + path = model['path'] + model = cm.get(path) + + # Change the name in the model for rename + model = cm.save(model, path) + assert isinstance(model, dict) + self.assertIn('name', model) + self.assertIn('path', model) + self.assertEqual(model['name'], 'Untitled.ipynb') + self.assertEqual(model['path'], 'foo/Untitled.ipynb') diff --git a/notebook/static/tree/js/notebooklist.js b/notebook/static/tree/js/notebooklist.js index ff146f3fe..8ac4db1d6 100644 --- a/notebook/static/tree/js/notebooklist.js +++ b/notebook/static/tree/js/notebooklist.js @@ -278,31 +278,38 @@ define([ var name_and_ext = utils.splitext(f.name); var file_ext = name_and_ext[1]; - // skip large files with a warning if (f.size > this._max_upload_size_mb * 1024 * 1024) { dialog.modal({ - title : 'Cannot upload file', - body : "Cannot upload file (>" + this._max_upload_size_mb + " MB) '" + f.name + "'", - buttons : {'OK' : { 'class' : 'btn-primary' }} + title : 'Large file size warning', + body : "The file size is " + Math.round(f.size / (1024 * 1024)) + "MB. Do you still want to upload it?", + buttons : { + Cancel: {}, + Ok: { + class: "btn-primary", + click: function() { + that.add_large_file_upload_button(f); + } + } + } }); - continue; } - - var reader = new FileReader(); - if (file_ext === '.ipynb') { - reader.readAsText(f); - } else { - // read non-notebook files as binary - reader.readAsArrayBuffer(f); + else{ + var reader = new FileReader(); + if (file_ext === '.ipynb') { + reader.readAsText(f); + } else { + // read non-notebook files as binary + reader.readAsArrayBuffer(f); + } + var item = that.new_item(0, true); + item.addClass('new-file'); + that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file'); + // Store the list item in the reader so we can use it later + // to know which item it belongs to. + $(reader).data('item', item); + reader.onload = reader_onload; + reader.onerror = reader_onerror; } - var item = that.new_item(0, true); - item.addClass('new-file'); - that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file'); - // Store the list item in the reader so we can use it later - // to know which item it belongs to. - $(reader).data('item', item); - reader.onload = reader_onload; - reader.onerror = reader_onerror; } // Replace the file input form wth a clone of itself. This is required to // reset the form. Otherwise, if you upload a file, delete it and try to @@ -1085,6 +1092,188 @@ define([ }); }; + // Add a new class for large file upload + NotebookList.prototype.add_large_file_upload_button = function (file) { + var that = this; + var item = that.new_item(0, true); + item.addClass('new-file'); + that.add_name_input(file.name, item, 'file'); + var cancel_button = $('