Merge pull request #2162 from blackrock/master

Fix for uploading large files crashing the browser (issue #96)

notebook/notebookapp.py

@@ -75,6 +75,7 @@ from .services.kernels.kernelmanager import MappingKernelManager
 from .services.config import ConfigManager
 from .services.contents.manager import ContentsManager
 from .services.contents.filemanager import FileContentsManager
+from .services.contents.largefilemanager import LargeFileManager
 from .services.sessions.sessionmanager import SessionManager
 
 from .auth.login import LoginHandler
@@ -843,7 +844,7 @@ class NotebookApp(JupyterApp):
         self.log.info("Using MathJax configuration file: %s", change['new'])
 
     contents_manager_class = Type(
-        default_value=FileContentsManager,
+        default_value=LargeFileManager,
         klass=ContentsManager,
         config=True,
         help='The notebook manager class to use.'

notebook/services/contents/handlers.py

@@ -176,7 +176,9 @@ class ContentsHandler(APIHandler):
     @gen.coroutine
     def _save(self, model, path):
         """Save an existing file."""
-        self.log.info(u"Saving file at %s", path)
+        chunk = model.get("chunk", None) 
+        if not chunk or chunk == -1:  # Avoid tedious log information
+            self.log.info(u"Saving file at %s", path)  
         model = yield gen.maybe_future(self.contents_manager.save(model, path))
         validate_model(model, expect_content=False)
         self._finish_model(model)

notebook/services/contents/largefilemanager.py

@@ -0,0 +1,70 @@
+from notebook.services.contents.filemanager import FileContentsManager
+from contextlib import contextmanager
+from tornado import web
+import nbformat
+import base64
+import os, io
+
+class LargeFileManager(FileContentsManager):
+    """Handle large file upload."""
+
+    def save(self, model, path=''):
+        """Save the file model and return the model with no content."""
+        chunk = model.get('chunk', None)
+        if chunk is not None:
+            path = path.strip('/')
+            
+            if 'type' not in model:
+                raise web.HTTPError(400, u'No file type provided')
+            if model['type'] != 'file':
+                raise web.HTTPError(400, u'File type "{}" is not supported for large file transfer'.format(model['type']))
+            if 'content' not in model and model['type'] != 'directory':
+                raise web.HTTPError(400, u'No file content provided')
+
+            os_path = self._get_os_path(path)
+
+            try:
+                if chunk == 1:
+                    self.log.debug("Saving %s", os_path)
+                    self.run_pre_save_hook(model=model, path=path)
+                    super(LargeFileManager, self)._save_file(os_path, model['content'], model.get('format'))
+                else:
+                    self._save_large_file(os_path, model['content'], model.get('format'))
+            except web.HTTPError:
+                raise
+            except Exception as e:
+                self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True)
+                raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e))
+
+            model = self.get(path, content=False)
+
+            # Last chunk
+            if chunk == -1:
+                self.run_post_save_hook(model=model, os_path=os_path)
+            return model
+        else:
+            return super(LargeFileManager, self).save(model, path)
+
+    def _save_large_file(self, os_path, content, format):
+        """Save content of a generic file."""
+        if format not in {'text', 'base64'}:
+            raise web.HTTPError(
+                400,
+                "Must specify format of file contents as 'text' or 'base64'",
+            )
+        try:
+            if format == 'text':
+                bcontent = content.encode('utf8')
+            else:
+                b64_bytes = content.encode('ascii')
+                bcontent = base64.decodestring(b64_bytes)
+        except Exception as e:
+            raise web.HTTPError(
+                400, u'Encoding error saving %s: %s' % (os_path, e)
+            )
+
+        with self.perm_to_403(os_path):
+            if os.path.islink(os_path):
+                os_path = os.path.join(os.path.dirname(os_path), os.readlink(os_path))
+            with io.open(os_path, 'ab') as f:
+                f.write(bcontent)

notebook/services/contents/tests/test_largefilemanager.py

@@ -0,0 +1,113 @@
+from unittest import TestCase
+from ipython_genutils.tempdir import TemporaryDirectory
+from ..largefilemanager import LargeFileManager
+import os
+from tornado import web
+
+
+def _make_dir(contents_manager, api_path):
+    """
+    Make a directory.
+    """
+    os_path = contents_manager._get_os_path(api_path)
+    try:
+        os.makedirs(os_path)
+    except OSError:
+        print("Directory already exists: %r" % os_path)
+
+
+class TestLargeFileManager(TestCase):
+
+    def setUp(self):
+        self._temp_dir = TemporaryDirectory()
+        self.td = self._temp_dir.name
+        self.contents_manager = LargeFileManager(root_dir=self.td)
+
+    def make_dir(self, api_path):
+        """make a subdirectory at api_path
+
+        override in subclasses if contents are not on the filesystem.
+        """
+        _make_dir(self.contents_manager, api_path)
+
+    def test_save(self):
+
+        cm = self.contents_manager
+        # Create a notebook
+        model = cm.new_untitled(type='notebook')
+        name = model['name']
+        path = model['path']
+
+        # Get the model with 'content'
+        full_model = cm.get(path)
+        # Save the notebook
+        model = cm.save(full_model, path)
+        assert isinstance(model, dict)
+        self.assertIn('name', model)
+        self.assertIn('path', model)
+        self.assertEqual(model['name'], name)
+        self.assertEqual(model['path'], path)
+
+        try:
+            model = {'name': 'test', 'path': 'test', 'chunk': 1}
+            cm.save(model, model['path'])
+        except web.HTTPError as e:
+            self.assertEqual('HTTP 400: Bad Request (No file type provided)', str(e))
+
+        try:
+            model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'notebook'}
+            cm.save(model, model['path'])
+        except web.HTTPError as e:
+            self.assertEqual('HTTP 400: Bad Request (File type "notebook" is not supported for large file transfer)', str(e))
+
+        try:
+            model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'file'}
+            cm.save(model, model['path'])
+        except web.HTTPError as e:
+            self.assertEqual('HTTP 400: Bad Request (No file content provided)', str(e))
+
+        try:
+            model = {'name': 'test', 'path': 'test', 'chunk': 2, 'type': 'file',
+                     'content': u'test', 'format': 'json'}
+            cm.save(model, model['path'])
+        except web.HTTPError as e:
+            self.assertEqual("HTTP 400: Bad Request (Must specify format of file contents as 'text' or 'base64')",
+            				 str(e))
+
+        # Save model for different chunks
+        model = {'name': 'test', 'path': 'test', 'type': 'file',
+                 'content': u'test==', 'format': 'text'}
+        name = model['name']
+        path = model['path']
+        cm.save(model, path)
+
+        for chunk in (1, 2, -1):
+            for fm in ('text', 'base64'):
+                full_model = cm.get(path)
+                full_model['chunk'] = chunk
+                full_model['format'] = fm
+                model_res = cm.save(full_model, path)
+                assert isinstance(model_res, dict)
+
+                self.assertIn('name', model_res)
+                self.assertIn('path', model_res)
+                self.assertNotIn('chunk', model_res)
+                self.assertEqual(model_res['name'], name)
+                self.assertEqual(model_res['path'], path)
+
+        # Test in sub-directory
+        # Create a directory and notebook in that directory
+        sub_dir = '/foo/'
+        self.make_dir('foo')
+        model = cm.new_untitled(path=sub_dir, type='notebook')
+        name = model['name']
+        path = model['path']
+        model = cm.get(path)
+
+        # Change the name in the model for rename
+        model = cm.save(model, path)
+        assert isinstance(model, dict)
+        self.assertIn('name', model)
+        self.assertIn('path', model)
+        self.assertEqual(model['name'], 'Untitled.ipynb')
+        self.assertEqual(model['path'], 'foo/Untitled.ipynb')

notebook/static/tree/js/notebooklist.js

@@ -278,31 +278,38 @@ define([
             var name_and_ext = utils.splitext(f.name);
             var file_ext = name_and_ext[1];
 
-            // skip large files with a warning
             if (f.size > this._max_upload_size_mb * 1024 * 1024) {
                 dialog.modal({
-                    title : 'Cannot upload file',
-                    body : "Cannot upload file (>" + this._max_upload_size_mb + " MB) '" + f.name + "'",
-                    buttons : {'OK' : { 'class' : 'btn-primary' }}
+                    title : 'Large file size warning',
+                    body : "The file size is " + Math.round(f.size / (1024 * 1024)) + "MB. Do you still want to upload it?",
+                    buttons : {
+                        Cancel: {},
+                        Ok: {
+                            class: "btn-primary",
+                            click: function() {
+                                that.add_large_file_upload_button(f);
+                            }
+                        }
+                    }
                 });
-                continue;
             }
-
-            var reader = new FileReader();
-            if (file_ext === '.ipynb') {
-                reader.readAsText(f);
-            } else {
-                // read non-notebook files as binary
-                reader.readAsArrayBuffer(f);
+            else{
+                var reader = new FileReader();
+                if (file_ext === '.ipynb') {
+                    reader.readAsText(f);
+                } else {
+                    // read non-notebook files as binary
+                    reader.readAsArrayBuffer(f);
+                }
+                var item = that.new_item(0, true);
+                item.addClass('new-file');
+                that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file');
+                // Store the list item in the reader so we can use it later
+                // to know which item it belongs to.
+                $(reader).data('item', item);
+                reader.onload = reader_onload;
+                reader.onerror = reader_onerror;
             }
-            var item = that.new_item(0, true);
-            item.addClass('new-file');
-            that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file');
-            // Store the list item in the reader so we can use it later
-            // to know which item it belongs to.
-            $(reader).data('item', item);
-            reader.onload = reader_onload;
-            reader.onerror = reader_onerror;
         }
         // Replace the file input form wth a clone of itself. This is required to
         // reset the form. Otherwise, if you upload a file, delete it and try to
@@ -1085,6 +1092,188 @@ define([
         });
     };
 
+    // Add a new class for large file upload
+    NotebookList.prototype.add_large_file_upload_button = function (file) {
+        var that = this;
+        var item = that.new_item(0, true);
+        item.addClass('new-file');
+        that.add_name_input(file.name, item, 'file');
+        var cancel_button = $('<button/>').text("Cancel")
+            .addClass("btn btn-default btn-xs")
+            .click(function (e) {
+                item.remove();
+                return false;
+            });
+
+        var upload_button = $('<button/>').text("Upload")
+            .addClass('btn btn-primary btn-xs upload_button')
+            .click(function (e) {
+                var filename = item.find('.item_name > input').val();
+                var path = utils.url_path_join(that.notebook_path, filename);
+                var format = 'text';
+                if (filename.length === 0 || filename[0] === '.') {
+                    dialog.modal({
+                        title : 'Invalid file name',
+                        body : "File names must be at least one character and not start with a dot",
+                        buttons : {'OK' : { 'class' : 'btn-primary' }}
+                    });
+                    return false;
+                }
+                
+                var check_exist = function () {
+                    var exists = false;
+                    $.each(that.element.find('.list_item:not(.new-file)'), function(k,v){
+                    if ($(v).data('name') === filename) { exists = true; return false; }
+                    });
+                    return exists
+                }
+                var exists = check_exist();
+                
+                var add_uploading_button = function (f, item) {
+                    // change buttons, add a progress bar
+                    var uploading_button = item.find('.upload_button').text("Uploading");
+                    var progress_bar = $('<span/>')
+                        .addClass('progress-bar')
+                        .css('top', '0')
+                        .css('left', '0')
+                        .css('width', '0')
+                        .css('height', '3px')
+                        .css('border-radius', '0 0 0 0')
+                        .css('display', 'inline-block')
+                        .css('position', 'absolute');
+
+                    var parse_large_file = function (f, item) {
+                        // codes inspired by http://stackoverflow.com/a/28318964
+                        var chunk_size = 1024 * 1024;
+                        var offset = 0;
+                        var chunk = 0;
+                        var chunk_reader = null;
+                        var upload_file = null;
+                        
+                        var large_reader_onload = function (event) {
+                            if (event.target.error == null) {
+                                offset += chunk_size;
+                                if (offset >= f.size) {
+                                    chunk = -1;
+                                } else {
+                                    chunk += 1;
+                                }
+                                // callback for handling reading: reader_onload in add_upload_button
+                                var item = $(event.target).data('item');
+                                that.add_file_data(event.target.result, item);
+                                upload_file(item, chunk);  // Do the upload
+                            } else {
+                                console.log("Read error: " + event.target.error);
+                                return;
+                            }
+                        };
+                        var on_error = function (event) {
+                            var item = $(event.target).data('item');
+                            var name = item.data('name');
+                            item.remove();
+                            var _exists = check_exist();
+                            if (_exists) {
+                                that.contents.delete(path);
+                            }
+                            dialog.modal({
+                                title : 'Failed to read file',
+                                body : "Failed to read file '" + name + "'",
+                                buttons : {'OK' : { 'class' : 'btn-primary' }}
+                            });
+                        }
+
+                        chunk_reader = function (_offset, _f) {
+                            var reader = new FileReader();
+                            var blob = _f.slice(_offset, chunk_size + _offset);
+                            // Load everything as ArrayBuffer
+                            reader.readAsArrayBuffer(blob);
+                            // Store the list item in the reader so we can use it later
+                            // to know which item it belongs to.
+                            $(reader).data('item', item);
+                            reader.onload = large_reader_onload;
+                            reader.onerror = on_error;
+                        };
+
+                        // These codes to upload file in original class
+                        var upload_file = function(item, chunk) {
+                            var filedata = item.data('filedata');
+                            if (filedata instanceof ArrayBuffer) {
+                                // base64-encode binary file data
+                                var bytes = '';
+                                var buf = new Uint8Array(filedata);
+                                var nbytes = buf.byteLength;
+                                for (var i=0; i<nbytes; i++) {
+                                    bytes += String.fromCharCode(buf[i]);
+                                }
+                                filedata = btoa(bytes);
+                                format = 'base64';
+                            }
+                            var model = { name: filename, path: path };
+
+                            var name_and_ext = utils.splitext(filename);
+                            var file_ext = name_and_ext[1];
+                            var content_type;
+                            // Treat everything as generic file
+                            model.type = 'file';
+                            model.format = format;
+                            content_type = 'application/octet-stream';
+
+                            model.chunk = chunk;
+                            model.content = filedata;
+                            
+                            var on_success = function (event) {
+                                if (offset < f.size) {
+                                    // of to the next chunk
+                                    chunk_reader(offset, f);
+                                    // change progress bar and progress button
+                                    var progress = offset / f.size * 100;
+                                    progress = progress > 100 ? 100 : progress;
+                                    uploading_button.text(progress.toFixed(0)+'%');
+                                    progress_bar.css('width', progress+'%')
+                                        .attr('aria-valuenow', progress.toString());
+                                } else {
+                                    item.removeClass('new-file');
+                                    that.add_link(model, item);
+                                    that.session_list.load_sessions();
+                                }
+                            };
+                            that.contents.save(path, model).then(on_success, on_error);
+                        }
+
+                        // now let's start the read with the first block
+                        chunk_reader(offset, f);
+                    };
+                    item.find('.item_buttons')
+                        .append(progress_bar);
+                    parse_large_file(f, item);
+                };
+                if (exists) {
+                    dialog.modal({
+                        title : "Replace file",
+                        body : 'There is already a file named ' + filename + ', do you want to replace it?',
+                        default_button: "Cancel",
+                        buttons : {
+                            Overwrite : {
+                                class: "btn-danger",
+                                click: function () {
+                                    add_uploading_button(file, item);
+                                }
+                            },
+                            Cancel : {
+                                click: function() { item.remove(); }
+                            }
+                        }
+                    });
+                } else {
+                    add_uploading_button(file, item);
+                }
+
+                return false;
+            });
+        item.find(".item_buttons").empty()
+            .append(upload_button)
+            .append(cancel_button);
+    }
 
     NotebookList.prototype.add_upload_button = function (item) {
         var that = this;