Merge pull request #6876 from minrk/falloff-reconnect

Add exponential falloff to kernel reconnect

IPython/html/static/notebook/js/menubar.js

@@ -295,6 +295,9 @@ define([
         this.element.find('#restart_kernel').click(function () {
             that.notebook.restart_kernel();
         });
+        this.element.find('#reconnect_kernel').click(function () {
+            that.notebook.kernel.reconnect();
+        });
         // Help
         if (this.tour) {
             this.element.find('#notebook_tour').click(function () {

IPython/html/static/notebook/js/notificationarea.js

@@ -132,6 +132,13 @@ define([
             knw.warning("Connecting to kernel");
         });
 
+        this.events.on('kernel_connection_dead.Kernel', function (evt, info) {
+            knw.danger("Not Connected", undefined, function () {
+                // schedule reconnect a short time in the future, don't reconnect immediately
+                setTimeout($.proxy(info.kernel.reconnect, info.kernel), 500);
+            }, {title: 'click to reconnect'});
+        });
+
         this.events.on('kernel_connected.Kernel', function () {
             knw.info("Connected", 500);
         });

IPython/html/static/services/kernels/kernel.js

@@ -66,6 +66,7 @@ define([
 
         this._autorestart_attempt = 0;
         this._reconnect_attempt = 0;
+        this.reconnect_limit = 7;
     };
 
     /**
@@ -332,8 +333,15 @@ define([
      * @function reconnect
      */
     Kernel.prototype.reconnect = function () {
-        this.events.trigger('kernel_reconnecting.Kernel', {kernel: this});
-        setTimeout($.proxy(this.start_channels, this), 3000);
+        if (this.is_connected()) {
+            return;
+        }
+        this._reconnect_attempt = this._reconnect_attempt + 1;
+        this.events.trigger('kernel_reconnecting.Kernel', {
+            kernel: this,
+            attempt: this._reconnect_attempt,
+        });
+        this.start_channels();
     };
 
     /**
@@ -524,12 +532,27 @@ define([
         this.events.trigger('kernel_disconnected.Kernel', {kernel: this});
         if (error) {
             console.log('WebSocket connection failed: ', ws_url);
-            this._reconnect_attempt = this._reconnect_attempt + 1;
             this.events.trigger('kernel_connection_failed.Kernel', {kernel: this, ws_url: ws_url, attempt: this._reconnect_attempt});
         }
-        this.reconnect();
+        this._schedule_reconnect();
     };
-
+    
+    Kernel.prototype._schedule_reconnect = function () {
+        // function to call when kernel connection is lost
+        // schedules reconnect, or fires 'connection_dead' if reconnect limit is hit
+        if (this._reconnect_attempt < this.reconnect_limit) {
+            var timeout = Math.pow(2, this._reconnect_attempt);
+            console.log("Connection lost, reconnecting in " + timeout + " seconds.");
+            setTimeout($.proxy(this.reconnect, this), 1e3 * timeout);
+        } else {
+            this.events.trigger('kernel_connection_dead.Kernel', {
+                kernel: this,
+                reconnect_attempt: this._reconnect_attempt,
+            });
+            console.log("Failed to reconnect, giving up.");
+        }
+    };
+    
     /**
      * Close the websocket channels. After successful close, the value
      * in `this.channels[channel_name]` will be null.

IPython/html/templates/notebook.html

@@ -230,10 +230,16 @@ class="notebook_app"
                     <ul id="kernel_menu" class="dropdown-menu">
                         <li id="int_kernel"
                             title="Send KeyboardInterrupt (CTRL-C) to the Kernel">
-                            <a href="#">Interrupt</a></li>
+                            <a href="#">Interrupt</a>
+                        </li>
                         <li id="restart_kernel"
                             title="Restart the Kernel">
-                            <a href="#">Restart</a></li>
+                            <a href="#">Restart</a>
+                        </li>
+                        <li id="reconnect_kernel"
+                            title="Reconnect to the Kernel">
+                            <a href="#">Reconnect</a>
+                        </li>
                         <li class="divider"></li>
                         <li id="menu-change-kernel" class="dropdown-submenu">
                             <a href="#">Change kernel</a>

IPython/html/tests/services/kernel.js

@@ -257,7 +257,11 @@ casper.notebook_test(function () {
         'ws_closed_error',
         [
             'kernel_disconnected.Kernel',
-            'kernel_connection_failed.Kernel'
+            'kernel_connection_failed.Kernel',
+            'kernel_reconnecting.Kernel',
+            'kernel_connected.Kernel',
+            'kernel_busy.Kernel',
+            'kernel_idle.Kernel'
         ],
         function () {
             this.thenEvaluate(function () {
@@ -265,6 +269,8 @@ casper.notebook_test(function () {
             });
         }
     );
+    // wait for any last idle/busy messages to be handled
+    this.wait_for_kernel_ready();
 
     // start the kernel back up
     this.thenEvaluate(function () {