diff --git a/notebook/notebookapp.py b/notebook/notebookapp.py
index 498405a6d..4b6a98dd8 100755
--- a/notebook/notebookapp.py
+++ b/notebook/notebookapp.py
@@ -75,6 +75,7 @@ from .services.kernels.kernelmanager import MappingKernelManager
from .services.config import ConfigManager
from .services.contents.manager import ContentsManager
from .services.contents.filemanager import FileContentsManager
+from .services.contents.largefilemanager import LargeFileManager
from .services.sessions.sessionmanager import SessionManager
from .auth.login import LoginHandler
@@ -843,7 +844,7 @@ class NotebookApp(JupyterApp):
self.log.info("Using MathJax configuration file: %s", change['new'])
contents_manager_class = Type(
- default_value=FileContentsManager,
+ default_value=LargeFileManager,
klass=ContentsManager,
config=True,
help='The notebook manager class to use.'
diff --git a/notebook/services/contents/handlers.py b/notebook/services/contents/handlers.py
index d4e528abd..640309065 100644
--- a/notebook/services/contents/handlers.py
+++ b/notebook/services/contents/handlers.py
@@ -176,7 +176,9 @@ class ContentsHandler(APIHandler):
@gen.coroutine
def _save(self, model, path):
"""Save an existing file."""
- self.log.info(u"Saving file at %s", path)
+ chunk = model.get("chunk", None)
+ if not chunk or chunk == -1: # Avoid tedious log information
+ self.log.info(u"Saving file at %s", path)
model = yield gen.maybe_future(self.contents_manager.save(model, path))
validate_model(model, expect_content=False)
self._finish_model(model)
diff --git a/notebook/services/contents/largefilemanager.py b/notebook/services/contents/largefilemanager.py
new file mode 100644
index 000000000..59328ef7e
--- /dev/null
+++ b/notebook/services/contents/largefilemanager.py
@@ -0,0 +1,70 @@
+from notebook.services.contents.filemanager import FileContentsManager
+from contextlib import contextmanager
+from tornado import web
+import nbformat
+import base64
+import os, io
+
+class LargeFileManager(FileContentsManager):
+ """Handle large file upload."""
+
+ def save(self, model, path=''):
+ """Save the file model and return the model with no content."""
+ chunk = model.get('chunk', None)
+ if chunk is not None:
+ path = path.strip('/')
+
+ if 'type' not in model:
+ raise web.HTTPError(400, u'No file type provided')
+ if model['type'] != 'file':
+ raise web.HTTPError(400, u'File type "{}" is not supported for large file transfer'.format(model['type']))
+ if 'content' not in model and model['type'] != 'directory':
+ raise web.HTTPError(400, u'No file content provided')
+
+ os_path = self._get_os_path(path)
+
+ try:
+ if chunk == 1:
+ self.log.debug("Saving %s", os_path)
+ self.run_pre_save_hook(model=model, path=path)
+ super(LargeFileManager, self)._save_file(os_path, model['content'], model.get('format'))
+ else:
+ self._save_large_file(os_path, model['content'], model.get('format'))
+ except web.HTTPError:
+ raise
+ except Exception as e:
+ self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True)
+ raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e))
+
+ model = self.get(path, content=False)
+
+ # Last chunk
+ if chunk == -1:
+ self.run_post_save_hook(model=model, os_path=os_path)
+ return model
+ else:
+ return super(LargeFileManager, self).save(model, path)
+
+ def _save_large_file(self, os_path, content, format):
+ """Save content of a generic file."""
+ if format not in {'text', 'base64'}:
+ raise web.HTTPError(
+ 400,
+ "Must specify format of file contents as 'text' or 'base64'",
+ )
+ try:
+ if format == 'text':
+ bcontent = content.encode('utf8')
+ else:
+ b64_bytes = content.encode('ascii')
+ bcontent = base64.decodestring(b64_bytes)
+ except Exception as e:
+ raise web.HTTPError(
+ 400, u'Encoding error saving %s: %s' % (os_path, e)
+ )
+
+ with self.perm_to_403(os_path):
+ if os.path.islink(os_path):
+ os_path = os.path.join(os.path.dirname(os_path), os.readlink(os_path))
+ with io.open(os_path, 'ab') as f:
+ f.write(bcontent)
diff --git a/notebook/services/contents/tests/test_largefilemanager.py b/notebook/services/contents/tests/test_largefilemanager.py
new file mode 100644
index 000000000..13d294b9b
--- /dev/null
+++ b/notebook/services/contents/tests/test_largefilemanager.py
@@ -0,0 +1,113 @@
+from unittest import TestCase
+from ipython_genutils.tempdir import TemporaryDirectory
+from ..largefilemanager import LargeFileManager
+import os
+from tornado import web
+
+
+def _make_dir(contents_manager, api_path):
+ """
+ Make a directory.
+ """
+ os_path = contents_manager._get_os_path(api_path)
+ try:
+ os.makedirs(os_path)
+ except OSError:
+ print("Directory already exists: %r" % os_path)
+
+
+class TestLargeFileManager(TestCase):
+
+ def setUp(self):
+ self._temp_dir = TemporaryDirectory()
+ self.td = self._temp_dir.name
+ self.contents_manager = LargeFileManager(root_dir=self.td)
+
+ def make_dir(self, api_path):
+ """make a subdirectory at api_path
+
+ override in subclasses if contents are not on the filesystem.
+ """
+ _make_dir(self.contents_manager, api_path)
+
+ def test_save(self):
+
+ cm = self.contents_manager
+ # Create a notebook
+ model = cm.new_untitled(type='notebook')
+ name = model['name']
+ path = model['path']
+
+ # Get the model with 'content'
+ full_model = cm.get(path)
+ # Save the notebook
+ model = cm.save(full_model, path)
+ assert isinstance(model, dict)
+ self.assertIn('name', model)
+ self.assertIn('path', model)
+ self.assertEqual(model['name'], name)
+ self.assertEqual(model['path'], path)
+
+ try:
+ model = {'name': 'test', 'path': 'test', 'chunk': 1}
+ cm.save(model, model['path'])
+ except web.HTTPError as e:
+ self.assertEqual('HTTP 400: Bad Request (No file type provided)', str(e))
+
+ try:
+ model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'notebook'}
+ cm.save(model, model['path'])
+ except web.HTTPError as e:
+ self.assertEqual('HTTP 400: Bad Request (File type "notebook" is not supported for large file transfer)', str(e))
+
+ try:
+ model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'file'}
+ cm.save(model, model['path'])
+ except web.HTTPError as e:
+ self.assertEqual('HTTP 400: Bad Request (No file content provided)', str(e))
+
+ try:
+ model = {'name': 'test', 'path': 'test', 'chunk': 2, 'type': 'file',
+ 'content': u'test', 'format': 'json'}
+ cm.save(model, model['path'])
+ except web.HTTPError as e:
+ self.assertEqual("HTTP 400: Bad Request (Must specify format of file contents as 'text' or 'base64')",
+ str(e))
+
+ # Save model for different chunks
+ model = {'name': 'test', 'path': 'test', 'type': 'file',
+ 'content': u'test==', 'format': 'text'}
+ name = model['name']
+ path = model['path']
+ cm.save(model, path)
+
+ for chunk in (1, 2, -1):
+ for fm in ('text', 'base64'):
+ full_model = cm.get(path)
+ full_model['chunk'] = chunk
+ full_model['format'] = fm
+ model_res = cm.save(full_model, path)
+ assert isinstance(model_res, dict)
+
+ self.assertIn('name', model_res)
+ self.assertIn('path', model_res)
+ self.assertNotIn('chunk', model_res)
+ self.assertEqual(model_res['name'], name)
+ self.assertEqual(model_res['path'], path)
+
+ # Test in sub-directory
+ # Create a directory and notebook in that directory
+ sub_dir = '/foo/'
+ self.make_dir('foo')
+ model = cm.new_untitled(path=sub_dir, type='notebook')
+ name = model['name']
+ path = model['path']
+ model = cm.get(path)
+
+ # Change the name in the model for rename
+ model = cm.save(model, path)
+ assert isinstance(model, dict)
+ self.assertIn('name', model)
+ self.assertIn('path', model)
+ self.assertEqual(model['name'], 'Untitled.ipynb')
+ self.assertEqual(model['path'], 'foo/Untitled.ipynb')
diff --git a/notebook/static/tree/js/notebooklist.js b/notebook/static/tree/js/notebooklist.js
index ff146f3fe..8ac4db1d6 100644
--- a/notebook/static/tree/js/notebooklist.js
+++ b/notebook/static/tree/js/notebooklist.js
@@ -278,31 +278,38 @@ define([
var name_and_ext = utils.splitext(f.name);
var file_ext = name_and_ext[1];
- // skip large files with a warning
if (f.size > this._max_upload_size_mb * 1024 * 1024) {
dialog.modal({
- title : 'Cannot upload file',
- body : "Cannot upload file (>" + this._max_upload_size_mb + " MB) '" + f.name + "'",
- buttons : {'OK' : { 'class' : 'btn-primary' }}
+ title : 'Large file size warning',
+ body : "The file size is " + Math.round(f.size / (1024 * 1024)) + "MB. Do you still want to upload it?",
+ buttons : {
+ Cancel: {},
+ Ok: {
+ class: "btn-primary",
+ click: function() {
+ that.add_large_file_upload_button(f);
+ }
+ }
+ }
});
- continue;
}
-
- var reader = new FileReader();
- if (file_ext === '.ipynb') {
- reader.readAsText(f);
- } else {
- // read non-notebook files as binary
- reader.readAsArrayBuffer(f);
+ else{
+ var reader = new FileReader();
+ if (file_ext === '.ipynb') {
+ reader.readAsText(f);
+ } else {
+ // read non-notebook files as binary
+ reader.readAsArrayBuffer(f);
+ }
+ var item = that.new_item(0, true);
+ item.addClass('new-file');
+ that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file');
+ // Store the list item in the reader so we can use it later
+ // to know which item it belongs to.
+ $(reader).data('item', item);
+ reader.onload = reader_onload;
+ reader.onerror = reader_onerror;
}
- var item = that.new_item(0, true);
- item.addClass('new-file');
- that.add_name_input(f.name, item, file_ext === '.ipynb' ? 'notebook' : 'file');
- // Store the list item in the reader so we can use it later
- // to know which item it belongs to.
- $(reader).data('item', item);
- reader.onload = reader_onload;
- reader.onerror = reader_onerror;
}
// Replace the file input form wth a clone of itself. This is required to
// reset the form. Otherwise, if you upload a file, delete it and try to
@@ -1085,6 +1092,188 @@ define([
});
};
+ // Add a new class for large file upload
+ NotebookList.prototype.add_large_file_upload_button = function (file) {
+ var that = this;
+ var item = that.new_item(0, true);
+ item.addClass('new-file');
+ that.add_name_input(file.name, item, 'file');
+ var cancel_button = $('').text("Cancel")
+ .addClass("btn btn-default btn-xs")
+ .click(function (e) {
+ item.remove();
+ return false;
+ });
+
+ var upload_button = $('').text("Upload")
+ .addClass('btn btn-primary btn-xs upload_button')
+ .click(function (e) {
+ var filename = item.find('.item_name > input').val();
+ var path = utils.url_path_join(that.notebook_path, filename);
+ var format = 'text';
+ if (filename.length === 0 || filename[0] === '.') {
+ dialog.modal({
+ title : 'Invalid file name',
+ body : "File names must be at least one character and not start with a dot",
+ buttons : {'OK' : { 'class' : 'btn-primary' }}
+ });
+ return false;
+ }
+
+ var check_exist = function () {
+ var exists = false;
+ $.each(that.element.find('.list_item:not(.new-file)'), function(k,v){
+ if ($(v).data('name') === filename) { exists = true; return false; }
+ });
+ return exists
+ }
+ var exists = check_exist();
+
+ var add_uploading_button = function (f, item) {
+ // change buttons, add a progress bar
+ var uploading_button = item.find('.upload_button').text("Uploading");
+ var progress_bar = $('')
+ .addClass('progress-bar')
+ .css('top', '0')
+ .css('left', '0')
+ .css('width', '0')
+ .css('height', '3px')
+ .css('border-radius', '0 0 0 0')
+ .css('display', 'inline-block')
+ .css('position', 'absolute');
+
+ var parse_large_file = function (f, item) {
+ // codes inspired by http://stackoverflow.com/a/28318964
+ var chunk_size = 1024 * 1024;
+ var offset = 0;
+ var chunk = 0;
+ var chunk_reader = null;
+ var upload_file = null;
+
+ var large_reader_onload = function (event) {
+ if (event.target.error == null) {
+ offset += chunk_size;
+ if (offset >= f.size) {
+ chunk = -1;
+ } else {
+ chunk += 1;
+ }
+ // callback for handling reading: reader_onload in add_upload_button
+ var item = $(event.target).data('item');
+ that.add_file_data(event.target.result, item);
+ upload_file(item, chunk); // Do the upload
+ } else {
+ console.log("Read error: " + event.target.error);
+ return;
+ }
+ };
+ var on_error = function (event) {
+ var item = $(event.target).data('item');
+ var name = item.data('name');
+ item.remove();
+ var _exists = check_exist();
+ if (_exists) {
+ that.contents.delete(path);
+ }
+ dialog.modal({
+ title : 'Failed to read file',
+ body : "Failed to read file '" + name + "'",
+ buttons : {'OK' : { 'class' : 'btn-primary' }}
+ });
+ }
+
+ chunk_reader = function (_offset, _f) {
+ var reader = new FileReader();
+ var blob = _f.slice(_offset, chunk_size + _offset);
+ // Load everything as ArrayBuffer
+ reader.readAsArrayBuffer(blob);
+ // Store the list item in the reader so we can use it later
+ // to know which item it belongs to.
+ $(reader).data('item', item);
+ reader.onload = large_reader_onload;
+ reader.onerror = on_error;
+ };
+
+ // These codes to upload file in original class
+ var upload_file = function(item, chunk) {
+ var filedata = item.data('filedata');
+ if (filedata instanceof ArrayBuffer) {
+ // base64-encode binary file data
+ var bytes = '';
+ var buf = new Uint8Array(filedata);
+ var nbytes = buf.byteLength;
+ for (var i=0; i 100 ? 100 : progress;
+ uploading_button.text(progress.toFixed(0)+'%');
+ progress_bar.css('width', progress+'%')
+ .attr('aria-valuenow', progress.toString());
+ } else {
+ item.removeClass('new-file');
+ that.add_link(model, item);
+ that.session_list.load_sessions();
+ }
+ };
+ that.contents.save(path, model).then(on_success, on_error);
+ }
+
+ // now let's start the read with the first block
+ chunk_reader(offset, f);
+ };
+ item.find('.item_buttons')
+ .append(progress_bar);
+ parse_large_file(f, item);
+ };
+ if (exists) {
+ dialog.modal({
+ title : "Replace file",
+ body : 'There is already a file named ' + filename + ', do you want to replace it?',
+ default_button: "Cancel",
+ buttons : {
+ Overwrite : {
+ class: "btn-danger",
+ click: function () {
+ add_uploading_button(file, item);
+ }
+ },
+ Cancel : {
+ click: function() { item.remove(); }
+ }
+ }
+ });
+ } else {
+ add_uploading_button(file, item);
+ }
+
+ return false;
+ });
+ item.find(".item_buttons").empty()
+ .append(upload_button)
+ .append(cancel_button);
+ }
NotebookList.prototype.add_upload_button = function (item) {
var that = this;