Improve IRC connection handling

Godeps/Godeps.json

@@ -53,6 +53,10 @@
 			"ImportPath": "github.com/inconshreveable/mousetrap",
 			"Rev": "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
 		},
+		{
+			"ImportPath": "github.com/jpillora/backoff",
+			"Rev": "2ff7c4694083b5dbd71b21fd7cb7577477a74b31"
+		},
 		{
 			"ImportPath": "github.com/kr/pretty",
 			"Comment": "go.weekly.2011-12-22-27-ge6ac2fc",

Godeps/_workspace/src/github.com/jpillora/backoff/README.md

@@ -0,0 +1,138 @@
+# Backoff
+
+A simple backoff algorithm in Go (Golang)
+
+[![GoDoc](https://godoc.org/github.com/jpillora/backoff?status.svg)](https://godoc.org/github.com/jpillora/backoff)
+
+### Install
+
+```
+$ go get -v github.com/jpillora/backoff
+```
+
+### Usage
+
+Backoff is a `time.Duration` counter. It starts at `Min`. After every call to `Duration()` it is  multiplied by `Factor`. It is capped at `Max`. It returns to `Min` on every call to `Reset()`. `Jitter` adds randomness ([see below](#example-using-jitter)). Used in conjunction with the `time` package.
+
+---
+
+#### Simple example
+
+``` go
+
+b := &backoff.Backoff{
+	//These are the defaults
+	Min:    100 * time.Millisecond,
+	Max:    10 * time.Second,
+	Factor: 2,
+	Jitter: false,
+}
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+
+fmt.Printf("Reset!\n")
+b.Reset()
+
+fmt.Printf("%s\n", b.Duration())
+```
+
+```
+100ms
+200ms
+400ms
+Reset!
+100ms
+```
+
+---
+
+#### Example using `net` package
+
+``` go
+b := &backoff.Backoff{
+    Max:    5 * time.Minute,
+}
+
+for {
+	conn, err := net.Dial("tcp", "example.com:5309")
+	if err != nil {
+		d := b.Duration()
+		fmt.Printf("%s, reconnecting in %s", err, d)
+		time.Sleep(d)
+		continue
+	}
+	//connected
+	b.Reset()
+	conn.Write([]byte("hello world!"))
+	// ... Read ... Write ... etc
+	conn.Close()
+	//disconnected
+}
+
+```
+
+---
+
+#### Example using `Jitter`
+
+Enabling `Jitter` adds some randomization to the backoff durations. [See Amazon's writeup of performance gains using jitter](http://www.awsarchitectureblog.com/2015/03/backoff.html). Seeding is not necessary but doing so gives repeatable results.
+
+```go
+import "math/rand"
+
+b := &backoff.Backoff{
+	Jitter: true,
+}
+
+rand.Seed(42)
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+
+fmt.Printf("Reset!\n")
+b.Reset()
+
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+fmt.Printf("%s\n", b.Duration())
+```
+
+```
+100ms
+106.600049ms
+281.228155ms
+Reset!
+100ms
+104.381845ms
+214.957989ms
+```
+
+#### Credits
+
+Ported from some JavaScript written by [@tj](https://github.com/tj)
+
+#### MIT License
+
+Copyright © 2015 Jaime Pillora <dev@jpillora.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/jpillora/backoff/backoff.go

@@ -0,0 +1,54 @@
+package backoff
+
+import (
+	"math"
+	"math/rand"
+	"time"
+)
+
+//Backoff is a time.Duration counter. It starts at Min.
+//After every call to Duration() it is  multiplied by Factor.
+//It is capped at Max. It returns to Min on every call to Reset().
+//Used in conjunction with the time package.
+type Backoff struct {
+	//Factor is the multiplying factor for each increment step
+	attempts, Factor float64
+	//Jitter eases contention by randomizing backoff steps
+	Jitter bool
+	//Min and Max are the minimum and maximum values of the counter
+	Min, Max time.Duration
+}
+
+//Returns the current value of the counter and then
+//multiplies it Factor
+func (b *Backoff) Duration() time.Duration {
+	//Zero-values are nonsensical, so we use
+	//them to apply defaults
+	if b.Min == 0 {
+		b.Min = 100 * time.Millisecond
+	}
+	if b.Max == 0 {
+		b.Max = 10 * time.Second
+	}
+	if b.Factor == 0 {
+		b.Factor = 2
+	}
+	//calculate this duration
+	dur := float64(b.Min) * math.Pow(b.Factor, b.attempts)
+	if b.Jitter == true {
+		dur = rand.Float64()*(dur-float64(b.Min)) + float64(b.Min)
+	}
+	//cap!
+	if dur > float64(b.Max) {
+		return b.Max
+	}
+	//bump attempts count
+	b.attempts++
+	//return as a time.Duration
+	return time.Duration(dur)
+}
+
+//Resets the current value of the counter back to Min
+func (b *Backoff) Reset() {
+	b.attempts = 0
+}

Godeps/_workspace/src/github.com/jpillora/backoff/backoff_test.go

@@ -0,0 +1,81 @@
+package backoff
+
+import (
+	"testing"
+	"time"
+)
+
+func Test1(t *testing.T) {
+
+	b := &Backoff{
+		Min:    100 * time.Millisecond,
+		Max:    10 * time.Second,
+		Factor: 2,
+	}
+
+	equals(t, b.Duration(), 100*time.Millisecond)
+	equals(t, b.Duration(), 200*time.Millisecond)
+	equals(t, b.Duration(), 400*time.Millisecond)
+	b.Reset()
+	equals(t, b.Duration(), 100*time.Millisecond)
+}
+
+func Test2(t *testing.T) {
+
+	b := &Backoff{
+		Min:    100 * time.Millisecond,
+		Max:    10 * time.Second,
+		Factor: 1.5,
+	}
+
+	equals(t, b.Duration(), 100*time.Millisecond)
+	equals(t, b.Duration(), 150*time.Millisecond)
+	equals(t, b.Duration(), 225*time.Millisecond)
+	b.Reset()
+	equals(t, b.Duration(), 100*time.Millisecond)
+}
+
+func Test3(t *testing.T) {
+
+	b := &Backoff{
+		Min:    100 * time.Nanosecond,
+		Max:    10 * time.Second,
+		Factor: 1.75,
+	}
+
+	equals(t, b.Duration(), 100*time.Nanosecond)
+	equals(t, b.Duration(), 175*time.Nanosecond)
+	equals(t, b.Duration(), 306*time.Nanosecond)
+	b.Reset()
+	equals(t, b.Duration(), 100*time.Nanosecond)
+}
+
+func TestJitter(t *testing.T) {
+	b := &Backoff{
+		Min:    100 * time.Millisecond,
+		Max:    10 * time.Second,
+		Factor: 2,
+		Jitter: true,
+	}
+
+	equals(t, b.Duration(), 100*time.Millisecond)
+	between(t, b.Duration(), 100*time.Millisecond, 200*time.Millisecond)
+	between(t, b.Duration(), 100*time.Millisecond, 400*time.Millisecond)
+	b.Reset()
+	equals(t, b.Duration(), 100*time.Millisecond)
+}
+
+func between(t *testing.T, actual, low, high time.Duration) {
+	if actual < low {
+		t.Fatalf("Got %s, Expecting >= %s", actual, low)
+	}
+	if actual > high {
+		t.Fatalf("Got %s, Expecting <= %s", actual, high)
+	}
+}
+
+func equals(t *testing.T, d1, d2 time.Duration) {
+	if d1 != d2 {
+		t.Fatalf("Got %s, Expecting %s", d1, d2)
+	}
+}

irc/client.go

@@ -31,6 +31,7 @@ type Client struct {
 	quit      chan struct{}
 	reconnect chan struct{}
 	ready     sync.WaitGroup
+	sendRecv  sync.WaitGroup
 	once      resync.Once
 	lock      sync.Mutex
 }
@@ -83,11 +84,6 @@ func (c *Client) Quit() {
 			c.write("QUIT")
 		}
 		close(c.quit)
-		c.lock.Lock()
-		if c.conn != nil {
-			c.conn.Close()
-		}
-		c.lock.Unlock()
 	}()
 }
 

irc/conn.go

@@ -7,6 +7,8 @@ import (
 	"net"
 	"strings"
 	"time"
+
+	"github.com/khlieng/dispatch/Godeps/_workspace/src/github.com/jpillora/backoff"
 )
 
 func (c *Client) Connect(address string) {
@@ -82,11 +84,15 @@ func (c *Client) connect() error {
 }
 
 func (c *Client) tryConnect() {
-	// TODO: backoff
+	b := &backoff.Backoff{
+		Jitter: true,
+	}
+
 	for {
 		select {
 		case <-c.quit:
 			return
+
 		default:
 		}
 
@@ -94,30 +100,36 @@ func (c *Client) tryConnect() {
 		if err == nil {
 			return
 		}
+
+		time.Sleep(b.Duration())
 	}
 }
 
 func (c *Client) run() {
 	c.tryConnect()
+
 	for {
 		select {
 		case <-c.quit:
 			c.close()
-			c.lock.Lock()
-			c.connected = false
-			c.lock.Unlock()
 			return
 
 		case <-c.reconnect:
+			c.sendRecv.Wait()
 			c.reconnect = make(chan struct{})
 			c.once.Reset()
+
 			c.tryConnect()
 		}
 	}
 }
 
 func (c *Client) send() {
+	c.sendRecv.Add(1)
+	defer c.sendRecv.Done()
+
 	c.ready.Wait()
+
 	for {
 		select {
 		case <-c.quit:
@@ -136,18 +148,24 @@ func (c *Client) send() {
 }
 
 func (c *Client) recv() {
-	defer c.conn.Close()
+	c.sendRecv.Add(1)
+	defer c.sendRecv.Done()
+
 	for {
 		line, err := c.reader.ReadString('\n')
 		if err != nil {
-			c.lock.Lock()
-			c.connected = false
-			c.lock.Unlock()
-			c.once.Do(c.ready.Done)
 			select {
 			case <-c.quit:
 				return
+
 			default:
+				c.lock.Lock()
+				c.connected = false
+				c.lock.Unlock()
+
+				c.once.Do(c.ready.Done)
+				c.conn.Close()
+
 				close(c.reconnect)
 				return
 			}
@@ -168,8 +186,14 @@ func (c *Client) recv() {
 
 func (c *Client) close() {
 	if c.Connected() {
+		c.lock.Lock()
+		c.connected = false
+		c.lock.Unlock()
+
 		c.once.Do(c.ready.Done)
+		c.conn.Close()
 	}
+
 	close(c.out)
 	close(c.Messages)
 }