Updated Godeps

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "github.com/docker/libnetwork",
-	"GoVersion": "go1.4.1",
+	"GoVersion": "go1.4.2",
 	"Packages": [
 		"./..."
 	],
@@ -15,16 +15,15 @@
 			"Comment": "v0.6.4-12-g467d9d5",
 			"Rev": "467d9d55c2d2c17248441a8fc661561161f40d5e"
 		},
+		{
+			"ImportPath": "github.com/armon/go-metrics",
+			"Rev": "eb0af217e5e9747e41dd5303755356b62d28e3ec"
+		},
 		{
 			"ImportPath": "github.com/coreos/go-etcd/etcd",
 			"Comment": "v2.0.0-7-g73a8ef7",
 			"Rev": "73a8ef737e8ea002281a28b4cb92a1de121ad4c6"
 		},
-		{
-			"ImportPath": "github.com/deckarep/golang-set",
-			"Comment": "v1-26-gef32fa3",
-			"Rev": "ef32fa3046d9f249d399f98ebaf9be944430fd1d"
-		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/homedir",
 			"Comment": "v1.4.1-4106-g637023a",
@@ -70,26 +69,11 @@
 			"Comment": "v1.4.1-4106-g637023a",
 			"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
 		},
-		{
-			"ImportPath": "github.com/docker/docker/api/client",
-			"Comment": "v1.4.1-4106-g637023a",
-			"Rev": "637023a5f8d8347a0e271c09d5c9bc84fbc97693"
-		},
 		{
 			"ImportPath": "github.com/docker/libcontainer/user",
 			"Comment": "v1.4.0-495-g3e66118",
 			"Rev": "3e661186ba24f259d3860f067df052c7f6904bee"
 		},
-		{
-			"ImportPath": "github.com/docker/swarm/discovery",
-			"Comment": "v0.3.0-rc2",
-			"Rev": "a5b2e57496762cb6971eb65809b2e428cb179719"
-		},
-		{
-			"ImportPath": "github.com/docker/swarm/pkg/store",
-			"Comment": "v0.3.0-rc2",
-			"Rev": "a5b2e57496762cb6971eb65809b2e428cb179719"
-		},
 		{
 			"ImportPath": "github.com/docker/libkv",
 			"Rev": "ab16c3d4a8785a9877c62d0b11ea4441cf09120c"
@@ -112,6 +96,19 @@
 			"Comment": "v0.5.0rc1-66-g954aec6",
 			"Rev": "954aec66231b79c161a4122b023fbcad13047f79"
 		},
+		{
+			"ImportPath": "github.com/hashicorp/go-msgpack/codec",
+			"Rev": "71c2886f5a673a35f909803f38ece5810165097b"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/memberlist",
+			"Rev": "9a1e242e454d2443df330bdd51a436d5a9058fc4"
+		},
+		{
+			"ImportPath": "github.com/hashicorp/serf/serf",
+			"Comment": "v0.6.4",
+			"Rev": "7151adcef72687bf95f451a2e0ba15cb19412bf2"
+		},
 		{
 			"ImportPath": "github.com/samuel/go-zookeeper/zk",
 			"Rev": "d0e0d8e11f318e000a8cc434616d69e329edc374"

Godeps/_workspace/src/github.com/armon/go-metrics/.gitignore

@@ -0,0 +1,22 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe

Godeps/_workspace/src/github.com/armon/go-metrics/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Armon Dadgar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/armon/go-metrics/README.md

@@ -0,0 +1,68 @@
+go-metrics
+==========
+
+This library provides a `metrics` package which can be used to instrument code,
+expose application metrics, and profile runtime performance in a flexible manner.
+
+Sinks
+=====
+
+The `metrics` package makes use of a `MetricSink` interface to support delivery
+to any type of backend. Currently the following sinks are provided:
+
+* StatsiteSink : Sinks to a statsite instance (TCP)
+* StatsdSink: Sinks to a statsd / statsite instance (UDP)
+* InmemSink : Provides in-memory aggregation, can be used to export stats
+* FanoutSink : Sinks to multiple sinks. Enables writing to multiple statsite instances for example.
+* BlackholeSink : Sinks to nowhere
+
+In addition to the sinks, the `InmemSignal` can be used to catch a signal,
+and dump a formatted output of recent metrics. For example, when a process gets
+a SIGUSR1, it can dump to stderr recent performance metrics for debugging.
+
+Examples
+========
+
+Here is an example of using the package:
+
+    func SlowMethod() {
+        // Profiling the runtime of a method
+        defer metrics.MeasureSince([]string{"SlowMethod"}, time.Now())
+    }
+
+    // Configure a statsite sink as the global metrics sink
+    sink, _ := metrics.NewStatsiteSink("statsite:8125")
+    metrics.NewGlobal(metrics.DefaultConfig("service-name"), sink)
+
+    // Emit a Key/Value pair
+    metrics.EmitKey([]string{"questions", "meaning of life"}, 42)
+
+
+Here is an example of setting up an signal handler:
+
+    // Setup the inmem sink and signal handler
+    inm := NewInmemSink(10*time.Second, time.Minute)
+    sig := DefaultInmemSignal(inm)
+    metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm)
+
+    // Run some code
+    inm.SetGauge([]string{"foo"}, 42)
+	inm.EmitKey([]string{"bar"}, 30)
+
+	inm.IncrCounter([]string{"baz"}, 42)
+	inm.IncrCounter([]string{"baz"}, 1)
+	inm.IncrCounter([]string{"baz"}, 80)
+
+	inm.AddSample([]string{"method", "wow"}, 42)
+	inm.AddSample([]string{"method", "wow"}, 100)
+	inm.AddSample([]string{"method", "wow"}, 22)
+
+    ....
+
+When a signal comes in, output like the following will be dumped to stderr:
+
+    [2014-01-28 14:57:33.04 -0800 PST][G] 'foo': 42.000
+    [2014-01-28 14:57:33.04 -0800 PST][P] 'bar': 30.000
+	[2014-01-28 14:57:33.04 -0800 PST][C] 'baz': Count: 3 Min: 1.000 Mean: 41.000 Max: 80.000 Stddev: 39.509
+    [2014-01-28 14:57:33.04 -0800 PST][S] 'method.wow': Count: 3 Min: 22.000 Mean: 54.667 Max: 100.000 Stddev: 40.513
+

Godeps/_workspace/src/github.com/armon/go-metrics/const_unix.go

@@ -0,0 +1,12 @@
+// +build !windows
+
+package metrics
+
+import (
+	"syscall"
+)
+
+const (
+	// DefaultSignal is used with DefaultInmemSignal
+	DefaultSignal = syscall.SIGUSR1
+)

Godeps/_workspace/src/github.com/armon/go-metrics/const_windows.go

@@ -0,0 +1,13 @@
+// +build windows
+
+package metrics
+
+import (
+	"syscall"
+)
+
+const (
+	// DefaultSignal is used with DefaultInmemSignal
+	// Windows has no SIGUSR1, use SIGBREAK
+	DefaultSignal = syscall.Signal(21)
+)

Godeps/_workspace/src/github.com/armon/go-metrics/inmem.go

@@ -0,0 +1,239 @@
+package metrics
+
+import (
+	"fmt"
+	"math"
+	"strings"
+	"sync"
+	"time"
+)
+
+// InmemSink provides a MetricSink that does in-memory aggregation
+// without sending metrics over a network. It can be embedded within
+// an application to provide profiling information.
+type InmemSink struct {
+	// How long is each aggregation interval
+	interval time.Duration
+
+	// Retain controls how many metrics interval we keep
+	retain time.Duration
+
+	// maxIntervals is the maximum length of intervals.
+	// It is retain / interval.
+	maxIntervals int
+
+	// intervals is a slice of the retained intervals
+	intervals    []*IntervalMetrics
+	intervalLock sync.RWMutex
+}
+
+// IntervalMetrics stores the aggregated metrics
+// for a specific interval
+type IntervalMetrics struct {
+	sync.RWMutex
+
+	// The start time of the interval
+	Interval time.Time
+
+	// Gauges maps the key to the last set value
+	Gauges map[string]float32
+
+	// Points maps the string to the list of emitted values
+	// from EmitKey
+	Points map[string][]float32
+
+	// Counters maps the string key to a sum of the counter
+	// values
+	Counters map[string]*AggregateSample
+
+	// Samples maps the key to an AggregateSample,
+	// which has the rolled up view of a sample
+	Samples map[string]*AggregateSample
+}
+
+// NewIntervalMetrics creates a new IntervalMetrics for a given interval
+func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
+	return &IntervalMetrics{
+		Interval: intv,
+		Gauges:   make(map[string]float32),
+		Points:   make(map[string][]float32),
+		Counters: make(map[string]*AggregateSample),
+		Samples:  make(map[string]*AggregateSample),
+	}
+}
+
+// AggregateSample is used to hold aggregate metrics
+// about a sample
+type AggregateSample struct {
+	Count int     // The count of emitted pairs
+	Sum   float64 // The sum of values
+	SumSq float64 // The sum of squared values
+	Min   float64 // Minimum value
+	Max   float64 // Maximum value
+}
+
+// Computes a Stddev of the values
+func (a *AggregateSample) Stddev() float64 {
+	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
+	div := float64(a.Count * (a.Count - 1))
+	if div == 0 {
+		return 0
+	}
+	return math.Sqrt(num / div)
+}
+
+// Computes a mean of the values
+func (a *AggregateSample) Mean() float64 {
+	if a.Count == 0 {
+		return 0
+	}
+	return a.Sum / float64(a.Count)
+}
+
+// Ingest is used to update a sample
+func (a *AggregateSample) Ingest(v float64) {
+	a.Count++
+	a.Sum += v
+	a.SumSq += (v * v)
+	if v < a.Min || a.Count == 1 {
+		a.Min = v
+	}
+	if v > a.Max || a.Count == 1 {
+		a.Max = v
+	}
+}
+
+func (a *AggregateSample) String() string {
+	if a.Count == 0 {
+		return "Count: 0"
+	} else if a.Stddev() == 0 {
+		return fmt.Sprintf("Count: %d Sum: %0.3f", a.Count, a.Sum)
+	} else {
+		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f",
+			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum)
+	}
+}
+
+// NewInmemSink is used to construct a new in-memory sink.
+// Uses an aggregation interval and maximum retention period.
+func NewInmemSink(interval, retain time.Duration) *InmemSink {
+	i := &InmemSink{
+		interval:     interval,
+		retain:       retain,
+		maxIntervals: int(retain / interval),
+	}
+	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
+	return i
+}
+
+func (i *InmemSink) SetGauge(key []string, val float32) {
+	k := i.flattenKey(key)
+	intv := i.getInterval()
+
+	intv.Lock()
+	defer intv.Unlock()
+	intv.Gauges[k] = val
+}
+
+func (i *InmemSink) EmitKey(key []string, val float32) {
+	k := i.flattenKey(key)
+	intv := i.getInterval()
+
+	intv.Lock()
+	defer intv.Unlock()
+	vals := intv.Points[k]
+	intv.Points[k] = append(vals, val)
+}
+
+func (i *InmemSink) IncrCounter(key []string, val float32) {
+	k := i.flattenKey(key)
+	intv := i.getInterval()
+
+	intv.Lock()
+	defer intv.Unlock()
+
+	agg := intv.Counters[k]
+	if agg == nil {
+		agg = &AggregateSample{}
+		intv.Counters[k] = agg
+	}
+	agg.Ingest(float64(val))
+}
+
+func (i *InmemSink) AddSample(key []string, val float32) {
+	k := i.flattenKey(key)
+	intv := i.getInterval()
+
+	intv.Lock()
+	defer intv.Unlock()
+
+	agg := intv.Samples[k]
+	if agg == nil {
+		agg = &AggregateSample{}
+		intv.Samples[k] = agg
+	}
+	agg.Ingest(float64(val))
+}
+
+// Data is used to retrieve all the aggregated metrics
+// Intervals may be in use, and a read lock should be acquired
+func (i *InmemSink) Data() []*IntervalMetrics {
+	// Get the current interval, forces creation
+	i.getInterval()
+
+	i.intervalLock.RLock()
+	defer i.intervalLock.RUnlock()
+
+	intervals := make([]*IntervalMetrics, len(i.intervals))
+	copy(intervals, i.intervals)
+	return intervals
+}
+
+func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
+	i.intervalLock.RLock()
+	defer i.intervalLock.RUnlock()
+
+	n := len(i.intervals)
+	if n > 0 && i.intervals[n-1].Interval == intv {
+		return i.intervals[n-1]
+	}
+	return nil
+}
+
+func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
+	i.intervalLock.Lock()
+	defer i.intervalLock.Unlock()
+
+	// Check for an existing interval
+	n := len(i.intervals)
+	if n > 0 && i.intervals[n-1].Interval == intv {
+		return i.intervals[n-1]
+	}
+
+	// Add the current interval
+	current := NewIntervalMetrics(intv)
+	i.intervals = append(i.intervals, current)
+	n++
+
+	// Truncate the intervals if they are too long
+	if n >= i.maxIntervals {
+		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
+		i.intervals = i.intervals[:i.maxIntervals]
+	}
+	return current
+}
+
+// getInterval returns the current interval to write to
+func (i *InmemSink) getInterval() *IntervalMetrics {
+	intv := time.Now().Truncate(i.interval)
+	if m := i.getExistingInterval(intv); m != nil {
+		return m
+	}
+	return i.createInterval(intv)
+}
+
+// Flattens the key for formatting, removes spaces
+func (i *InmemSink) flattenKey(parts []string) string {
+	joined := strings.Join(parts, ".")
+	return strings.Replace(joined, " ", "_", -1)
+}

Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal.go

@@ -0,0 +1,100 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+)
+
+// InmemSignal is used to listen for a given signal, and when received,
+// to dump the current metrics from the InmemSink to an io.Writer
+type InmemSignal struct {
+	signal syscall.Signal
+	inm    *InmemSink
+	w      io.Writer
+	sigCh  chan os.Signal
+
+	stop     bool
+	stopCh   chan struct{}
+	stopLock sync.Mutex
+}
+
+// NewInmemSignal creates a new InmemSignal which listens for a given signal,
+// and dumps the current metrics out to a writer
+func NewInmemSignal(inmem *InmemSink, sig syscall.Signal, w io.Writer) *InmemSignal {
+	i := &InmemSignal{
+		signal: sig,
+		inm:    inmem,
+		w:      w,
+		sigCh:  make(chan os.Signal, 1),
+		stopCh: make(chan struct{}),
+	}
+	signal.Notify(i.sigCh, sig)
+	go i.run()
+	return i
+}
+
+// DefaultInmemSignal returns a new InmemSignal that responds to SIGUSR1
+// and writes output to stderr. Windows uses SIGBREAK
+func DefaultInmemSignal(inmem *InmemSink) *InmemSignal {
+	return NewInmemSignal(inmem, DefaultSignal, os.Stderr)
+}
+
+// Stop is used to stop the InmemSignal from listening
+func (i *InmemSignal) Stop() {
+	i.stopLock.Lock()
+	defer i.stopLock.Unlock()
+
+	if i.stop {
+		return
+	}
+	i.stop = true
+	close(i.stopCh)
+	signal.Stop(i.sigCh)
+}
+
+// run is a long running routine that handles signals
+func (i *InmemSignal) run() {
+	for {
+		select {
+		case <-i.sigCh:
+			i.dumpStats()
+		case <-i.stopCh:
+			return
+		}
+	}
+}
+
+// dumpStats is used to dump the data to output writer
+func (i *InmemSignal) dumpStats() {
+	buf := bytes.NewBuffer(nil)
+
+	data := i.inm.Data()
+	// Skip the last period which is still being aggregated
+	for i := 0; i < len(data)-1; i++ {
+		intv := data[i]
+		intv.RLock()
+		for name, val := range intv.Gauges {
+			fmt.Fprintf(buf, "[%v][G] '%s': %0.3f\n", intv.Interval, name, val)
+		}
+		for name, vals := range intv.Points {
+			for _, val := range vals {
+				fmt.Fprintf(buf, "[%v][P] '%s': %0.3f\n", intv.Interval, name, val)
+			}
+		}
+		for name, agg := range intv.Counters {
+			fmt.Fprintf(buf, "[%v][C] '%s': %s\n", intv.Interval, name, agg)
+		}
+		for name, agg := range intv.Samples {
+			fmt.Fprintf(buf, "[%v][S] '%s': %s\n", intv.Interval, name, agg)
+		}
+		intv.RUnlock()
+	}
+
+	// Write out the bytes
+	i.w.Write(buf.Bytes())
+}

Godeps/_workspace/src/github.com/armon/go-metrics/inmem_signal_test.go

@@ -0,0 +1,46 @@
+package metrics
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+)
+
+func TestInmemSignal(t *testing.T) {
+	buf := bytes.NewBuffer(nil)
+	inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond)
+	sig := NewInmemSignal(inm, syscall.SIGUSR1, buf)
+	defer sig.Stop()
+
+	inm.SetGauge([]string{"foo"}, 42)
+	inm.EmitKey([]string{"bar"}, 42)
+	inm.IncrCounter([]string{"baz"}, 42)
+	inm.AddSample([]string{"wow"}, 42)
+
+	// Wait for period to end
+	time.Sleep(15 * time.Millisecond)
+
+	// Send signal!
+	syscall.Kill(os.Getpid(), syscall.SIGUSR1)
+
+	// Wait for flush
+	time.Sleep(10 * time.Millisecond)
+
+	// Check the output
+	out := string(buf.Bytes())
+	if !strings.Contains(out, "[G] 'foo': 42") {
+		t.Fatalf("bad: %v", out)
+	}
+	if !strings.Contains(out, "[P] 'bar': 42") {
+		t.Fatalf("bad: %v", out)
+	}
+	if !strings.Contains(out, "[C] 'baz': Count: 1 Sum: 42") {
+		t.Fatalf("bad: %v", out)
+	}
+	if !strings.Contains(out, "[S] 'wow': Count: 1 Sum: 42") {
+		t.Fatalf("bad: %v", out)
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/inmem_test.go

@@ -0,0 +1,95 @@
+package metrics
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+func TestInmemSink(t *testing.T) {
+	inm := NewInmemSink(10*time.Millisecond, 50*time.Millisecond)
+
+	data := inm.Data()
+	if len(data) != 1 {
+		t.Fatalf("bad: %v", data)
+	}
+
+	// Add data points
+	inm.SetGauge([]string{"foo", "bar"}, 42)
+	inm.EmitKey([]string{"foo", "bar"}, 42)
+	inm.IncrCounter([]string{"foo", "bar"}, 20)
+	inm.IncrCounter([]string{"foo", "bar"}, 22)
+	inm.AddSample([]string{"foo", "bar"}, 20)
+	inm.AddSample([]string{"foo", "bar"}, 22)
+
+	data = inm.Data()
+	if len(data) != 1 {
+		t.Fatalf("bad: %v", data)
+	}
+
+	intvM := data[0]
+	intvM.RLock()
+
+	if time.Now().Sub(intvM.Interval) > 10*time.Millisecond {
+		t.Fatalf("interval too old")
+	}
+	if intvM.Gauges["foo.bar"] != 42 {
+		t.Fatalf("bad val: %v", intvM.Gauges)
+	}
+	if intvM.Points["foo.bar"][0] != 42 {
+		t.Fatalf("bad val: %v", intvM.Points)
+	}
+
+	agg := intvM.Counters["foo.bar"]
+	if agg.Count != 2 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.Sum != 42 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.SumSq != 884 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.Min != 20 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.Max != 22 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.Mean() != 21 {
+		t.Fatalf("bad val: %v", agg)
+	}
+	if agg.Stddev() != math.Sqrt(2) {
+		t.Fatalf("bad val: %v", agg)
+	}
+
+	if agg = intvM.Samples["foo.bar"]; agg == nil {
+		t.Fatalf("missing sample")
+	}
+
+	intvM.RUnlock()
+
+	for i := 1; i < 10; i++ {
+		time.Sleep(10 * time.Millisecond)
+		inm.SetGauge([]string{"foo", "bar"}, 42)
+		data = inm.Data()
+		if len(data) != min(i+1, 5) {
+			t.Fatalf("bad: %v", data)
+		}
+	}
+
+	// Should not exceed 5 intervals!
+	time.Sleep(10 * time.Millisecond)
+	inm.SetGauge([]string{"foo", "bar"}, 42)
+	data = inm.Data()
+	if len(data) != 5 {
+		t.Fatalf("bad: %v", data)
+	}
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

Godeps/_workspace/src/github.com/armon/go-metrics/metrics.go

@@ -0,0 +1,115 @@
+package metrics
+
+import (
+	"runtime"
+	"time"
+)
+
+func (m *Metrics) SetGauge(key []string, val float32) {
+	if m.HostName != "" && m.EnableHostname {
+		key = insert(0, m.HostName, key)
+	}
+	if m.EnableTypePrefix {
+		key = insert(0, "gauge", key)
+	}
+	if m.ServiceName != "" {
+		key = insert(0, m.ServiceName, key)
+	}
+	m.sink.SetGauge(key, val)
+}
+
+func (m *Metrics) EmitKey(key []string, val float32) {
+	if m.EnableTypePrefix {
+		key = insert(0, "kv", key)
+	}
+	if m.ServiceName != "" {
+		key = insert(0, m.ServiceName, key)
+	}
+	m.sink.EmitKey(key, val)
+}
+
+func (m *Metrics) IncrCounter(key []string, val float32) {
+	if m.EnableTypePrefix {
+		key = insert(0, "counter", key)
+	}
+	if m.ServiceName != "" {
+		key = insert(0, m.ServiceName, key)
+	}
+	m.sink.IncrCounter(key, val)
+}
+
+func (m *Metrics) AddSample(key []string, val float32) {
+	if m.EnableTypePrefix {
+		key = insert(0, "sample", key)
+	}
+	if m.ServiceName != "" {
+		key = insert(0, m.ServiceName, key)
+	}
+	m.sink.AddSample(key, val)
+}
+
+func (m *Metrics) MeasureSince(key []string, start time.Time) {
+	if m.EnableTypePrefix {
+		key = insert(0, "timer", key)
+	}
+	if m.ServiceName != "" {
+		key = insert(0, m.ServiceName, key)
+	}
+	now := time.Now()
+	elapsed := now.Sub(start)
+	msec := float32(elapsed.Nanoseconds()) / float32(m.TimerGranularity)
+	m.sink.AddSample(key, msec)
+}
+
+// Periodically collects runtime stats to publish
+func (m *Metrics) collectStats() {
+	for {
+		time.Sleep(m.ProfileInterval)
+		m.emitRuntimeStats()
+	}
+}
+
+// Emits various runtime statsitics
+func (m *Metrics) emitRuntimeStats() {
+	// Export number of Goroutines
+	numRoutines := runtime.NumGoroutine()
+	m.SetGauge([]string{"runtime", "num_goroutines"}, float32(numRoutines))
+
+	// Export memory stats
+	var stats runtime.MemStats
+	runtime.ReadMemStats(&stats)
+	m.SetGauge([]string{"runtime", "alloc_bytes"}, float32(stats.Alloc))
+	m.SetGauge([]string{"runtime", "sys_bytes"}, float32(stats.Sys))
+	m.SetGauge([]string{"runtime", "malloc_count"}, float32(stats.Mallocs))
+	m.SetGauge([]string{"runtime", "free_count"}, float32(stats.Frees))
+	m.SetGauge([]string{"runtime", "heap_objects"}, float32(stats.HeapObjects))
+	m.SetGauge([]string{"runtime", "total_gc_pause_ns"}, float32(stats.PauseTotalNs))
+	m.SetGauge([]string{"runtime", "total_gc_runs"}, float32(stats.NumGC))
+
+	// Export info about the last few GC runs
+	num := stats.NumGC
+
+	// Handle wrap around
+	if num < m.lastNumGC {
+		m.lastNumGC = 0
+	}
+
+	// Ensure we don't scan more than 256
+	if num-m.lastNumGC >= 256 {
+		m.lastNumGC = num - 255
+	}
+
+	for i := m.lastNumGC; i < num; i++ {
+		pause := stats.PauseNs[i%256]
+		m.AddSample([]string{"runtime", "gc_pause_ns"}, float32(pause))
+	}
+	m.lastNumGC = num
+}
+
+// Inserts a string value at an index into the slice
+func insert(i int, v string, s []string) []string {
+	s = append(s, "")
+	copy(s[i+1:], s[i:])
+	s[i] = v
+	return s
+}

Godeps/_workspace/src/github.com/armon/go-metrics/metrics_test.go

@@ -0,0 +1,262 @@
+package metrics
+
+import (
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+func mockMetric() (*MockSink, *Metrics) {
+	m := &MockSink{}
+	met := &Metrics{sink: m}
+	return m, met
+}
+
+func TestMetrics_SetGauge(t *testing.T) {
+	m, met := mockMetric()
+	met.SetGauge([]string{"key"}, float32(1))
+	if m.keys[0][0] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.HostName = "test"
+	met.EnableHostname = true
+	met.SetGauge([]string{"key"}, float32(1))
+	if m.keys[0][0] != "test" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.EnableTypePrefix = true
+	met.SetGauge([]string{"key"}, float32(1))
+	if m.keys[0][0] != "gauge" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.ServiceName = "service"
+	met.SetGauge([]string{"key"}, float32(1))
+	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+}
+
+func TestMetrics_EmitKey(t *testing.T) {
+	m, met := mockMetric()
+	met.EmitKey([]string{"key"}, float32(1))
+	if m.keys[0][0] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.EnableTypePrefix = true
+	met.EmitKey([]string{"key"}, float32(1))
+	if m.keys[0][0] != "kv" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.ServiceName = "service"
+	met.EmitKey([]string{"key"}, float32(1))
+	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+}
+
+func TestMetrics_IncrCounter(t *testing.T) {
+	m, met := mockMetric()
+	met.IncrCounter([]string{"key"}, float32(1))
+	if m.keys[0][0] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.EnableTypePrefix = true
+	met.IncrCounter([]string{"key"}, float32(1))
+	if m.keys[0][0] != "counter" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.ServiceName = "service"
+	met.IncrCounter([]string{"key"}, float32(1))
+	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+}
+
+func TestMetrics_AddSample(t *testing.T) {
+	m, met := mockMetric()
+	met.AddSample([]string{"key"}, float32(1))
+	if m.keys[0][0] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.EnableTypePrefix = true
+	met.AddSample([]string{"key"}, float32(1))
+	if m.keys[0][0] != "sample" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.ServiceName = "service"
+	met.AddSample([]string{"key"}, float32(1))
+	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] != 1 {
+		t.Fatalf("")
+	}
+}
+
+func TestMetrics_MeasureSince(t *testing.T) {
+	m, met := mockMetric()
+	met.TimerGranularity = time.Millisecond
+	n := time.Now()
+	met.MeasureSince([]string{"key"}, n)
+	if m.keys[0][0] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] > 0.1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.TimerGranularity = time.Millisecond
+	met.EnableTypePrefix = true
+	met.MeasureSince([]string{"key"}, n)
+	if m.keys[0][0] != "timer" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] > 0.1 {
+		t.Fatalf("")
+	}
+
+	m, met = mockMetric()
+	met.TimerGranularity = time.Millisecond
+	met.ServiceName = "service"
+	met.MeasureSince([]string{"key"}, n)
+	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
+		t.Fatalf("")
+	}
+	if m.vals[0] > 0.1 {
+		t.Fatalf("")
+	}
+}
+
+func TestMetrics_EmitRuntimeStats(t *testing.T) {
+	runtime.GC()
+	m, met := mockMetric()
+	met.emitRuntimeStats()
+
+	if m.keys[0][0] != "runtime" || m.keys[0][1] != "num_goroutines" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[0] <= 1 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[1][0] != "runtime" || m.keys[1][1] != "alloc_bytes" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[1] <= 100000 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[2][0] != "runtime" || m.keys[2][1] != "sys_bytes" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[2] <= 100000 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[3][0] != "runtime" || m.keys[3][1] != "malloc_count" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[3] <= 100 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[4][0] != "runtime" || m.keys[4][1] != "free_count" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[4] <= 100 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[5][0] != "runtime" || m.keys[5][1] != "heap_objects" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[5] <= 200 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[6][0] != "runtime" || m.keys[6][1] != "total_gc_pause_ns" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[6] <= 100000 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[7][0] != "runtime" || m.keys[7][1] != "total_gc_runs" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[7] <= 1 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+
+	if m.keys[8][0] != "runtime" || m.keys[8][1] != "gc_pause_ns" {
+		t.Fatalf("bad key %v", m.keys)
+	}
+	if m.vals[8] <= 1000 {
+		t.Fatalf("bad val: %v", m.vals)
+	}
+}
+
+func TestInsert(t *testing.T) {
+	k := []string{"hi", "bob"}
+	exp := []string{"hi", "there", "bob"}
+	out := insert(1, "there", k)
+	if !reflect.DeepEqual(exp, out) {
+		t.Fatalf("bad insert %v %v", exp, out)
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/sink.go

@@ -0,0 +1,52 @@
+package metrics
+
+// The MetricSink interface is used to transmit metrics information
+// to an external system
+type MetricSink interface {
+	// A Gauge should retain the last value it is set to
+	SetGauge(key []string, val float32)
+
+	// Should emit a Key/Value pair for each call
+	EmitKey(key []string, val float32)
+
+	// Counters should accumulate values
+	IncrCounter(key []string, val float32)
+
+	// Samples are for timing information, where quantiles are used
+	AddSample(key []string, val float32)
+}
+
+// BlackholeSink is used to just blackhole messages
+type BlackholeSink struct{}
+
+func (*BlackholeSink) SetGauge(key []string, val float32)    {}
+func (*BlackholeSink) EmitKey(key []string, val float32)     {}
+func (*BlackholeSink) IncrCounter(key []string, val float32) {}
+func (*BlackholeSink) AddSample(key []string, val float32)   {}
+
+// FanoutSink is used to sink to fanout values to multiple sinks
+type FanoutSink []MetricSink
+
+func (fh FanoutSink) SetGauge(key []string, val float32) {
+	for _, s := range fh {
+		s.SetGauge(key, val)
+	}
+}
+
+func (fh FanoutSink) EmitKey(key []string, val float32) {
+	for _, s := range fh {
+		s.EmitKey(key, val)
+	}
+}
+
+func (fh FanoutSink) IncrCounter(key []string, val float32) {
+	for _, s := range fh {
+		s.IncrCounter(key, val)
+	}
+}
+
+func (fh FanoutSink) AddSample(key []string, val float32) {
+	for _, s := range fh {
+		s.AddSample(key, val)
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/sink_test.go

@@ -0,0 +1,120 @@
+package metrics
+
+import (
+	"reflect"
+	"testing"
+)
+
+type MockSink struct {
+	keys [][]string
+	vals []float32
+}
+
+func (m *MockSink) SetGauge(key []string, val float32) {
+	m.keys = append(m.keys, key)
+	m.vals = append(m.vals, val)
+}
+func (m *MockSink) EmitKey(key []string, val float32) {
+	m.keys = append(m.keys, key)
+	m.vals = append(m.vals, val)
+}
+func (m *MockSink) IncrCounter(key []string, val float32) {
+	m.keys = append(m.keys, key)
+	m.vals = append(m.vals, val)
+}
+func (m *MockSink) AddSample(key []string, val float32) {
+	m.keys = append(m.keys, key)
+	m.vals = append(m.vals, val)
+}
+
+func TestFanoutSink_Gauge(t *testing.T) {
+	m1 := &MockSink{}
+	m2 := &MockSink{}
+	fh := &FanoutSink{m1, m2}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	fh.SetGauge(k, v)
+
+	if !reflect.DeepEqual(m1.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m2.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m1.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+	if !reflect.DeepEqual(m2.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func TestFanoutSink_Key(t *testing.T) {
+	m1 := &MockSink{}
+	m2 := &MockSink{}
+	fh := &FanoutSink{m1, m2}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	fh.EmitKey(k, v)
+
+	if !reflect.DeepEqual(m1.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m2.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m1.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+	if !reflect.DeepEqual(m2.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func TestFanoutSink_Counter(t *testing.T) {
+	m1 := &MockSink{}
+	m2 := &MockSink{}
+	fh := &FanoutSink{m1, m2}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	fh.IncrCounter(k, v)
+
+	if !reflect.DeepEqual(m1.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m2.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m1.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+	if !reflect.DeepEqual(m2.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func TestFanoutSink_Sample(t *testing.T) {
+	m1 := &MockSink{}
+	m2 := &MockSink{}
+	fh := &FanoutSink{m1, m2}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	fh.AddSample(k, v)
+
+	if !reflect.DeepEqual(m1.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m2.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m1.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+	if !reflect.DeepEqual(m2.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/start.go

@@ -0,0 +1,95 @@
+package metrics
+
+import (
+	"os"
+	"time"
+)
+
+// Config is used to configure metrics settings
+type Config struct {
+	ServiceName          string        // Prefixed with keys to seperate services
+	HostName             string        // Hostname to use. If not provided and EnableHostname, it will be os.Hostname
+	EnableHostname       bool          // Enable prefixing gauge values with hostname
+	EnableRuntimeMetrics bool          // Enables profiling of runtime metrics (GC, Goroutines, Memory)
+	EnableTypePrefix     bool          // Prefixes key with a type ("counter", "gauge", "timer")
+	TimerGranularity     time.Duration // Granularity of timers.
+	ProfileInterval      time.Duration // Interval to profile runtime metrics
+}
+
+// Metrics represents an instance of a metrics sink that can
+// be used to emit
+type Metrics struct {
+	Config
+	lastNumGC uint32
+	sink      MetricSink
+}
+
+// Shared global metrics instance
+var globalMetrics *Metrics
+
+func init() {
+	// Initialize to a blackhole sink to avoid errors
+	globalMetrics = &Metrics{sink: &BlackholeSink{}}
+}
+
+// DefaultConfig provides a sane default configuration
+func DefaultConfig(serviceName string) *Config {
+	c := &Config{
+		ServiceName:          serviceName, // Use client provided service
+		HostName:             "",
+		EnableHostname:       true,             // Enable hostname prefix
+		EnableRuntimeMetrics: true,             // Enable runtime profiling
+		EnableTypePrefix:     false,            // Disable type prefix
+		TimerGranularity:     time.Millisecond, // Timers are in milliseconds
+		ProfileInterval:      time.Second,      // Poll runtime every second
+	}
+
+	// Try to get the hostname
+	name, _ := os.Hostname()
+	c.HostName = name
+	return c
+}
+
+// New is used to create a new instance of Metrics
+func New(conf *Config, sink MetricSink) (*Metrics, error) {
+	met := &Metrics{}
+	met.Config = *conf
+	met.sink = sink
+
+	// Start the runtime collector
+	if conf.EnableRuntimeMetrics {
+		go met.collectStats()
+	}
+	return met, nil
+}
+
+// NewGlobal is the same as New, but it assigns the metrics object to be
+// used globally as well as returning it.
+func NewGlobal(conf *Config, sink MetricSink) (*Metrics, error) {
+	metrics, err := New(conf, sink)
+	if err == nil {
+		globalMetrics = metrics
+	}
+	return metrics, err
+}
+
+// Proxy all the methods to the globalMetrics instance
+func SetGauge(key []string, val float32) {
+	globalMetrics.SetGauge(key, val)
+}
+
+func EmitKey(key []string, val float32) {
+	globalMetrics.EmitKey(key, val)
+}
+
+func IncrCounter(key []string, val float32) {
+	globalMetrics.IncrCounter(key, val)
+}
+
+func AddSample(key []string, val float32) {
+	globalMetrics.AddSample(key, val)
+}
+
+func MeasureSince(key []string, start time.Time) {
+	globalMetrics.MeasureSince(key, start)
+}

Godeps/_workspace/src/github.com/armon/go-metrics/start_test.go

@@ -0,0 +1,110 @@
+package metrics
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	conf := DefaultConfig("service")
+	if conf.ServiceName != "service" {
+		t.Fatalf("Bad name")
+	}
+	if conf.HostName == "" {
+		t.Fatalf("missing hostname")
+	}
+	if !conf.EnableHostname || !conf.EnableRuntimeMetrics {
+		t.Fatalf("expect true")
+	}
+	if conf.EnableTypePrefix {
+		t.Fatalf("expect false")
+	}
+	if conf.TimerGranularity != time.Millisecond {
+		t.Fatalf("bad granularity")
+	}
+	if conf.ProfileInterval != time.Second {
+		t.Fatalf("bad interval")
+	}
+}
+
+func Test_GlobalMetrics_SetGauge(t *testing.T) {
+	m := &MockSink{}
+	globalMetrics = &Metrics{sink: m}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	SetGauge(k, v)
+
+	if !reflect.DeepEqual(m.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func Test_GlobalMetrics_EmitKey(t *testing.T) {
+	m := &MockSink{}
+	globalMetrics = &Metrics{sink: m}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	EmitKey(k, v)
+
+	if !reflect.DeepEqual(m.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func Test_GlobalMetrics_IncrCounter(t *testing.T) {
+	m := &MockSink{}
+	globalMetrics = &Metrics{sink: m}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	IncrCounter(k, v)
+
+	if !reflect.DeepEqual(m.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func Test_GlobalMetrics_AddSample(t *testing.T) {
+	m := &MockSink{}
+	globalMetrics = &Metrics{sink: m}
+
+	k := []string{"test"}
+	v := float32(42.0)
+	AddSample(k, v)
+
+	if !reflect.DeepEqual(m.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if !reflect.DeepEqual(m.vals[0], v) {
+		t.Fatalf("val not equal")
+	}
+}
+
+func Test_GlobalMetrics_MeasureSince(t *testing.T) {
+	m := &MockSink{}
+	globalMetrics = &Metrics{sink: m}
+	globalMetrics.TimerGranularity = time.Millisecond
+
+	k := []string{"test"}
+	now := time.Now()
+	MeasureSince(k, now)
+
+	if !reflect.DeepEqual(m.keys[0], k) {
+		t.Fatalf("key not equal")
+	}
+	if m.vals[0] > 0.1 {
+		t.Fatalf("val too large %v", m.vals[0])
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/statsd.go

@@ -0,0 +1,154 @@
+package metrics
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+	"time"
+)
+
+const (
+	// statsdMaxLen is the maximum size of a packet
+	// to send to statsd
+	statsdMaxLen = 1400
+)
+
+// StatsdSink provides a MetricSink that can be used
+// with a statsite or statsd metrics server. It uses
+// only UDP packets, while StatsiteSink uses TCP.
+type StatsdSink struct {
+	addr        string
+	metricQueue chan string
+}
+
+// NewStatsdSink is used to create a new StatsdSink
+func NewStatsdSink(addr string) (*StatsdSink, error) {
+	s := &StatsdSink{
+		addr:        addr,
+		metricQueue: make(chan string, 4096),
+	}
+	go s.flushMetrics()
+	return s, nil
+}
+
+// Close is used to stop flushing to statsd
+func (s *StatsdSink) Shutdown() {
+	close(s.metricQueue)
+}
+
+func (s *StatsdSink) SetGauge(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val))
+}
+
+func (s *StatsdSink) EmitKey(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val))
+}
+
+func (s *StatsdSink) IncrCounter(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val))
+}
+
+func (s *StatsdSink) AddSample(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val))
+}
+
+// Flattens the key for formatting, removes spaces
+func (s *StatsdSink) flattenKey(parts []string) string {
+	joined := strings.Join(parts, ".")
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case ':':
+			fallthrough
+		case ' ':
+			return '_'
+		default:
+			return r
+		}
+	}, joined)
+}
+
+// Does a non-blocking push to the metrics queue
+func (s *StatsdSink) pushMetric(m string) {
+	select {
+	case s.metricQueue <- m:
+	default:
+	}
+}
+
+// Flushes metrics
+func (s *StatsdSink) flushMetrics() {
+	var sock net.Conn
+	var err error
+	var wait <-chan time.Time
+	ticker := time.NewTicker(flushInterval)
+	defer ticker.Stop()
+
+CONNECT:
+	// Create a buffer
+	buf := bytes.NewBuffer(nil)
+
+	// Attempt to connect
+	sock, err = net.Dial("udp", s.addr)
+	if err != nil {
+		log.Printf("[ERR] Error connecting to statsd! Err: %s", err)
+		goto WAIT
+	}
+
+	for {
+		select {
+		case metric, ok := <-s.metricQueue:
+			// Get a metric from the queue
+			if !ok {
+				goto QUIT
+			}
+
+			// Check if this would overflow the packet size
+			if len(metric)+buf.Len() > statsdMaxLen {
+				_, err := sock.Write(buf.Bytes())
+				buf.Reset()
+				if err != nil {
+					log.Printf("[ERR] Error writing to statsd! Err: %s", err)
+					goto WAIT
+				}
+			}
+
+			// Append to the buffer
+			buf.WriteString(metric)
+
+		case <-ticker.C:
+			if buf.Len() == 0 {
+				continue
+			}
+
+			_, err := sock.Write(buf.Bytes())
+			buf.Reset()
+			if err != nil {
+				log.Printf("[ERR] Error flushing to statsd! Err: %s", err)
+				goto WAIT
+			}
+		}
+	}
+
+WAIT:
+	// Wait for a while
+	wait = time.After(time.Duration(5) * time.Second)
+	for {
+		select {
+		// Dequeue the messages to avoid backlog
+		case _, ok := <-s.metricQueue:
+			if !ok {
+				goto QUIT
+			}
+		case <-wait:
+			goto CONNECT
+		}
+	}
+QUIT:
+	s.metricQueue = nil
+}

Godeps/_workspace/src/github.com/armon/go-metrics/statsd_test.go

@@ -0,0 +1,105 @@
+package metrics
+
+import (
+	"bufio"
+	"bytes"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestStatsd_Flatten(t *testing.T) {
+	s := &StatsdSink{}
+	flat := s.flattenKey([]string{"a", "b", "c", "d"})
+	if flat != "a.b.c.d" {
+		t.Fatalf("Bad flat")
+	}
+}
+
+func TestStatsd_PushFullQueue(t *testing.T) {
+	q := make(chan string, 1)
+	q <- "full"
+
+	s := &StatsdSink{metricQueue: q}
+	s.pushMetric("omit")
+
+	out := <-q
+	if out != "full" {
+		t.Fatalf("bad val %v", out)
+	}
+
+	select {
+	case v := <-q:
+		t.Fatalf("bad val %v", v)
+	default:
+	}
+}
+
+func TestStatsd_Conn(t *testing.T) {
+	addr := "127.0.0.1:7524"
+	done := make(chan bool)
+	go func() {
+		list, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 7524})
+		if err != nil {
+			panic(err)
+		}
+		defer list.Close()
+		buf := make([]byte, 1500)
+		n, err := list.Read(buf)
+		if err != nil {
+			panic(err)
+		}
+		buf = buf[:n]
+		reader := bufio.NewReader(bytes.NewReader(buf))
+
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "gauge.val:1.000000|g\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "key.other:2.000000|kv\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "counter.me:3.000000|c\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "sample.slow_thingy:4.000000|ms\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		done <- true
+	}()
+	s, err := NewStatsdSink(addr)
+	if err != nil {
+		t.Fatalf("bad error")
+	}
+
+	s.SetGauge([]string{"gauge", "val"}, float32(1))
+	s.EmitKey([]string{"key", "other"}, float32(2))
+	s.IncrCounter([]string{"counter", "me"}, float32(3))
+	s.AddSample([]string{"sample", "slow thingy"}, float32(4))
+
+	select {
+	case <-done:
+		s.Shutdown()
+	case <-time.After(3 * time.Second):
+		t.Fatalf("timeout")
+	}
+}

Godeps/_workspace/src/github.com/armon/go-metrics/statsite.go

@@ -0,0 +1,142 @@
+package metrics
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+	"time"
+)
+
+const (
+	// We force flush the statsite metrics after this period of
+	// inactivity. Prevents stats from getting stuck in a buffer
+	// forever.
+	flushInterval = 100 * time.Millisecond
+)
+
+// StatsiteSink provides a MetricSink that can be used with a
+// statsite metrics server
+type StatsiteSink struct {
+	addr        string
+	metricQueue chan string
+}
+
+// NewStatsiteSink is used to create a new StatsiteSink
+func NewStatsiteSink(addr string) (*StatsiteSink, error) {
+	s := &StatsiteSink{
+		addr:        addr,
+		metricQueue: make(chan string, 4096),
+	}
+	go s.flushMetrics()
+	return s, nil
+}
+
+// Close is used to stop flushing to statsite
+func (s *StatsiteSink) Shutdown() {
+	close(s.metricQueue)
+}
+
+func (s *StatsiteSink) SetGauge(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val))
+}
+
+func (s *StatsiteSink) EmitKey(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val))
+}
+
+func (s *StatsiteSink) IncrCounter(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val))
+}
+
+func (s *StatsiteSink) AddSample(key []string, val float32) {
+	flatKey := s.flattenKey(key)
+	s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val))
+}
+
+// Flattens the key for formatting, removes spaces
+func (s *StatsiteSink) flattenKey(parts []string) string {
+	joined := strings.Join(parts, ".")
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case ':':
+			fallthrough
+		case ' ':
+			return '_'
+		default:
+			return r
+		}
+	}, joined)
+}
+
+// Does a non-blocking push to the metrics queue
+func (s *StatsiteSink) pushMetric(m string) {
+	select {
+	case s.metricQueue <- m:
+	default:
+	}
+}
+
+// Flushes metrics
+func (s *StatsiteSink) flushMetrics() {
+	var sock net.Conn
+	var err error
+	var wait <-chan time.Time
+	var buffered *bufio.Writer
+	ticker := time.NewTicker(flushInterval)
+	defer ticker.Stop()
+
+CONNECT:
+	// Attempt to connect
+	sock, err = net.Dial("tcp", s.addr)
+	if err != nil {
+		log.Printf("[ERR] Error connecting to statsite! Err: %s", err)
+		goto WAIT
+	}
+
+	// Create a buffered writer
+	buffered = bufio.NewWriter(sock)
+
+	for {
+		select {
+		case metric, ok := <-s.metricQueue:
+			// Get a metric from the queue
+			if !ok {
+				goto QUIT
+			}
+
+			// Try to send to statsite
+			_, err := buffered.Write([]byte(metric))
+			if err != nil {
+				log.Printf("[ERR] Error writing to statsite! Err: %s", err)
+				goto WAIT
+			}
+		case <-ticker.C:
+			if err := buffered.Flush(); err != nil {
+				log.Printf("[ERR] Error flushing to statsite! Err: %s", err)
+				goto WAIT
+			}
+		}
+	}
+
+WAIT:
+	// Wait for a while
+	wait = time.After(time.Duration(5) * time.Second)
+	for {
+		select {
+		// Dequeue the messages to avoid backlog
+		case _, ok := <-s.metricQueue:
+			if !ok {
+				goto QUIT
+			}
+		case <-wait:
+			goto CONNECT
+		}
+	}
+QUIT:
+	s.metricQueue = nil
+}

Godeps/_workspace/src/github.com/armon/go-metrics/statsite_test.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"bufio"
+	"net"
+	"testing"
+	"time"
+)
+
+func acceptConn(addr string) net.Conn {
+	ln, _ := net.Listen("tcp", addr)
+	conn, _ := ln.Accept()
+	return conn
+}
+
+func TestStatsite_Flatten(t *testing.T) {
+	s := &StatsiteSink{}
+	flat := s.flattenKey([]string{"a", "b", "c", "d"})
+	if flat != "a.b.c.d" {
+		t.Fatalf("Bad flat")
+	}
+}
+
+func TestStatsite_PushFullQueue(t *testing.T) {
+	q := make(chan string, 1)
+	q <- "full"
+
+	s := &StatsiteSink{metricQueue: q}
+	s.pushMetric("omit")
+
+	out := <-q
+	if out != "full" {
+		t.Fatalf("bad val %v", out)
+	}
+
+	select {
+	case v := <-q:
+		t.Fatalf("bad val %v", v)
+	default:
+	}
+}
+
+func TestStatsite_Conn(t *testing.T) {
+	addr := "localhost:7523"
+	done := make(chan bool)
+	go func() {
+		conn := acceptConn(addr)
+		reader := bufio.NewReader(conn)
+
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "gauge.val:1.000000|g\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "key.other:2.000000|kv\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "counter.me:3.000000|c\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		line, err = reader.ReadString('\n')
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if line != "sample.slow_thingy:4.000000|ms\n" {
+			t.Fatalf("bad line %s", line)
+		}
+
+		conn.Close()
+		done <- true
+	}()
+	s, err := NewStatsiteSink(addr)
+	if err != nil {
+		t.Fatalf("bad error")
+	}
+
+	s.SetGauge([]string{"gauge", "val"}, float32(1))
+	s.EmitKey([]string{"key", "other"}, float32(2))
+	s.IncrCounter([]string{"counter", "me"}, float32(3))
+	s.AddSample([]string{"sample", "slow thingy"}, float32(4))
+
+	select {
+	case <-done:
+		s.Shutdown()
+	case <-time.After(3 * time.Second):
+		t.Fatalf("timeout")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/0doc.go

@@ -0,0 +1,143 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+/*
+High Performance, Feature-Rich Idiomatic Go encoding library for msgpack and binc .
+
+Supported Serialization formats are:
+
+  - msgpack: [https://github.com/msgpack/msgpack]
+  - binc: [http://github.com/ugorji/binc]
+
+To install:
+
+    go get github.com/ugorji/go/codec
+
+The idiomatic Go support is as seen in other encoding packages in
+the standard library (ie json, xml, gob, etc).
+
+Rich Feature Set includes:
+
+  - Simple but extremely powerful and feature-rich API
+  - Very High Performance.
+    Our extensive benchmarks show us outperforming Gob, Json and Bson by 2-4X.
+    This was achieved by taking extreme care on:
+      - managing allocation
+      - function frame size (important due to Go's use of split stacks),
+      - reflection use (and by-passing reflection for common types)
+      - recursion implications
+      - zero-copy mode (encoding/decoding to byte slice without using temp buffers)
+  - Correct.
+    Care was taken to precisely handle corner cases like:
+      overflows, nil maps and slices, nil value in stream, etc.
+  - Efficient zero-copying into temporary byte buffers
+    when encoding into or decoding from a byte slice.
+  - Standard field renaming via tags
+  - Encoding from any value
+    (struct, slice, map, primitives, pointers, interface{}, etc)
+  - Decoding into pointer to any non-nil typed value
+    (struct, slice, map, int, float32, bool, string, reflect.Value, etc)
+  - Supports extension functions to handle the encode/decode of custom types
+  - Support Go 1.2 encoding.BinaryMarshaler/BinaryUnmarshaler
+  - Schema-less decoding
+    (decode into a pointer to a nil interface{} as opposed to a typed non-nil value).
+    Includes Options to configure what specific map or slice type to use
+    when decoding an encoded list or map into a nil interface{}
+  - Provides a RPC Server and Client Codec for net/rpc communication protocol.
+  - Msgpack Specific:
+      - Provides extension functions to handle spec-defined extensions (binary, timestamp)
+      - Options to resolve ambiguities in handling raw bytes (as string or []byte)
+        during schema-less decoding (decoding into a nil interface{})
+      - RPC Server/Client Codec for msgpack-rpc protocol defined at:
+        https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
+  - Fast Paths for some container types:
+    For some container types, we circumvent reflection and its associated overhead
+    and allocation costs, and encode/decode directly. These types are:
+	    []interface{}
+	    []int
+	    []string
+	    map[interface{}]interface{}
+	    map[int]interface{}
+	    map[string]interface{}
+
+Extension Support
+
+Users can register a function to handle the encoding or decoding of
+their custom types.
+
+There are no restrictions on what the custom type can be. Some examples:
+
+    type BisSet   []int
+    type BitSet64 uint64
+    type UUID     string
+    type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
+    type GifImage struct { ... }
+
+As an illustration, MyStructWithUnexportedFields would normally be
+encoded as an empty map because it has no exported fields, while UUID
+would be encoded as a string. However, with extension support, you can
+encode any of these however you like.
+
+RPC
+
+RPC Client and Server Codecs are implemented, so the codecs can be used
+with the standard net/rpc package.
+
+Usage
+
+Typical usage model:
+
+    // create and configure Handle
+    var (
+      bh codec.BincHandle
+      mh codec.MsgpackHandle
+    )
+
+    mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
+
+    // configure extensions
+    // e.g. for msgpack, define functions and enable Time support for tag 1
+    // mh.AddExt(reflect.TypeOf(time.Time{}), 1, myMsgpackTimeEncodeExtFn, myMsgpackTimeDecodeExtFn)
+
+    // create and use decoder/encoder
+    var (
+      r io.Reader
+      w io.Writer
+      b []byte
+      h = &bh // or mh to use msgpack
+    )
+
+    dec = codec.NewDecoder(r, h)
+    dec = codec.NewDecoderBytes(b, h)
+    err = dec.Decode(&v)
+
+    enc = codec.NewEncoder(w, h)
+    enc = codec.NewEncoderBytes(&b, h)
+    err = enc.Encode(v)
+
+    //RPC Server
+    go func() {
+        for {
+            conn, err := listener.Accept()
+            rpcCodec := codec.GoRpc.ServerCodec(conn, h)
+            //OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
+            rpc.ServeCodec(rpcCodec)
+        }
+    }()
+
+    //RPC Communication (client side)
+    conn, err = net.Dial("tcp", "localhost:5555")
+    rpcCodec := codec.GoRpc.ClientCodec(conn, h)
+    //OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
+    client := rpc.NewClientWithCodec(rpcCodec)
+
+Representative Benchmark Results
+
+Run the benchmark suite using:
+   go test -bi -bench=. -benchmem
+
+To run full benchmark suite (including against vmsgpack and bson),
+see notes in ext_dep_test.go
+
+*/
+package codec

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/README.md

@@ -0,0 +1,174 @@
+# Codec
+
+High Performance and Feature-Rich Idiomatic Go Library providing
+encode/decode support for different serialization formats.
+
+Supported Serialization formats are:
+
+  - msgpack: [https://github.com/msgpack/msgpack]
+  - binc: [http://github.com/ugorji/binc]
+
+To install:
+
+    go get github.com/ugorji/go/codec
+
+Online documentation: [http://godoc.org/github.com/ugorji/go/codec]
+
+The idiomatic Go support is as seen in other encoding packages in
+the standard library (ie json, xml, gob, etc).
+
+Rich Feature Set includes:
+
+  - Simple but extremely powerful and feature-rich API
+  - Very High Performance.   
+    Our extensive benchmarks show us outperforming Gob, Json and Bson by 2-4X.
+    This was achieved by taking extreme care on:
+      - managing allocation
+      - function frame size (important due to Go's use of split stacks),
+      - reflection use (and by-passing reflection for common types)
+      - recursion implications
+      - zero-copy mode (encoding/decoding to byte slice without using temp buffers)
+  - Correct.  
+    Care was taken to precisely handle corner cases like: 
+      overflows, nil maps and slices, nil value in stream, etc.
+  - Efficient zero-copying into temporary byte buffers  
+    when encoding into or decoding from a byte slice.
+  - Standard field renaming via tags
+  - Encoding from any value  
+    (struct, slice, map, primitives, pointers, interface{}, etc)
+  - Decoding into pointer to any non-nil typed value  
+    (struct, slice, map, int, float32, bool, string, reflect.Value, etc)
+  - Supports extension functions to handle the encode/decode of custom types
+  - Support Go 1.2 encoding.BinaryMarshaler/BinaryUnmarshaler
+  - Schema-less decoding  
+    (decode into a pointer to a nil interface{} as opposed to a typed non-nil value).  
+    Includes Options to configure what specific map or slice type to use 
+    when decoding an encoded list or map into a nil interface{}
+  - Provides a RPC Server and Client Codec for net/rpc communication protocol.
+  - Msgpack Specific:
+      - Provides extension functions to handle spec-defined extensions (binary, timestamp)
+      - Options to resolve ambiguities in handling raw bytes (as string or []byte)  
+        during schema-less decoding (decoding into a nil interface{})
+      - RPC Server/Client Codec for msgpack-rpc protocol defined at: 
+        https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
+  - Fast Paths for some container types:  
+    For some container types, we circumvent reflection and its associated overhead
+    and allocation costs, and encode/decode directly. These types are:  
+	    []interface{}
+	    []int
+	    []string
+	    map[interface{}]interface{}
+	    map[int]interface{}
+	    map[string]interface{}
+
+## Extension Support
+
+Users can register a function to handle the encoding or decoding of
+their custom types.
+
+There are no restrictions on what the custom type can be. Some examples:
+
+    type BisSet   []int
+    type BitSet64 uint64
+    type UUID     string
+    type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
+    type GifImage struct { ... }
+
+As an illustration, MyStructWithUnexportedFields would normally be
+encoded as an empty map because it has no exported fields, while UUID
+would be encoded as a string. However, with extension support, you can
+encode any of these however you like.
+
+## RPC
+
+RPC Client and Server Codecs are implemented, so the codecs can be used
+with the standard net/rpc package.
+
+## Usage
+
+Typical usage model:
+
+    // create and configure Handle
+    var (
+      bh codec.BincHandle
+      mh codec.MsgpackHandle
+    )
+
+    mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
+    
+    // configure extensions
+    // e.g. for msgpack, define functions and enable Time support for tag 1
+    // mh.AddExt(reflect.TypeOf(time.Time{}), 1, myMsgpackTimeEncodeExtFn, myMsgpackTimeDecodeExtFn)
+
+    // create and use decoder/encoder
+    var (
+      r io.Reader
+      w io.Writer
+      b []byte
+      h = &bh // or mh to use msgpack
+    )
+    
+    dec = codec.NewDecoder(r, h)
+    dec = codec.NewDecoderBytes(b, h)
+    err = dec.Decode(&v) 
+    
+    enc = codec.NewEncoder(w, h)
+    enc = codec.NewEncoderBytes(&b, h)
+    err = enc.Encode(v)
+    
+    //RPC Server
+    go func() {
+        for {
+            conn, err := listener.Accept()
+            rpcCodec := codec.GoRpc.ServerCodec(conn, h)
+            //OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
+            rpc.ServeCodec(rpcCodec)
+        }
+    }()
+
+    //RPC Communication (client side)
+    conn, err = net.Dial("tcp", "localhost:5555")
+    rpcCodec := codec.GoRpc.ClientCodec(conn, h)
+    //OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
+    client := rpc.NewClientWithCodec(rpcCodec)
+
+## Representative Benchmark Results
+
+A sample run of benchmark using "go test -bi -bench=. -benchmem":
+
+    /proc/cpuinfo: Intel(R) Core(TM) i7-2630QM CPU @ 2.00GHz (HT)
+    
+    ..............................................
+    BENCHMARK INIT: 2013-10-16 11:02:50.345970786 -0400 EDT
+    To run full benchmark comparing encodings (MsgPack, Binc, JSON, GOB, etc), use: "go test -bench=."
+    Benchmark: 
+    	Struct recursive Depth:             1
+    	ApproxDeepSize Of benchmark Struct: 4694 bytes
+    Benchmark One-Pass Run:
+    	 v-msgpack: len: 1600 bytes
+    	      bson: len: 3025 bytes
+    	   msgpack: len: 1560 bytes
+    	      binc: len: 1187 bytes
+    	       gob: len: 1972 bytes
+    	      json: len: 2538 bytes
+    ..............................................
+    PASS
+    Benchmark__Msgpack____Encode	   50000	     54359 ns/op	   14953 B/op	      83 allocs/op
+    Benchmark__Msgpack____Decode	   10000	    106531 ns/op	   14990 B/op	     410 allocs/op
+    Benchmark__Binc_NoSym_Encode	   50000	     53956 ns/op	   14966 B/op	      83 allocs/op
+    Benchmark__Binc_NoSym_Decode	   10000	    103751 ns/op	   14529 B/op	     386 allocs/op
+    Benchmark__Binc_Sym___Encode	   50000	     65961 ns/op	   17130 B/op	      88 allocs/op
+    Benchmark__Binc_Sym___Decode	   10000	    106310 ns/op	   15857 B/op	     287 allocs/op
+    Benchmark__Gob________Encode	   10000	    135944 ns/op	   21189 B/op	     237 allocs/op
+    Benchmark__Gob________Decode	    5000	    405390 ns/op	   83460 B/op	    1841 allocs/op
+    Benchmark__Json_______Encode	   20000	     79412 ns/op	   13874 B/op	     102 allocs/op
+    Benchmark__Json_______Decode	   10000	    247979 ns/op	   14202 B/op	     493 allocs/op
+    Benchmark__Bson_______Encode	   10000	    121762 ns/op	   27814 B/op	     514 allocs/op
+    Benchmark__Bson_______Decode	   10000	    162126 ns/op	   16514 B/op	     789 allocs/op
+    Benchmark__VMsgpack___Encode	   50000	     69155 ns/op	   12370 B/op	     344 allocs/op
+    Benchmark__VMsgpack___Decode	   10000	    151609 ns/op	   20307 B/op	     571 allocs/op
+    ok  	ugorji.net/codec	30.827s
+
+To run full benchmark suite (including against vmsgpack and bson), 
+see notes in ext\_dep\_test.go
+

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/bench_test.go

@@ -0,0 +1,319 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+import (
+	"bytes"
+	"encoding/gob"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// Sample way to run:
+// go test -bi -bv -bd=1 -benchmem -bench=.
+
+var (
+	_       = fmt.Printf
+	benchTs *TestStruc
+
+	approxSize int
+
+	benchDoInitBench     bool
+	benchVerify          bool
+	benchUnscientificRes bool = false
+	//depth of 0 maps to ~400bytes json-encoded string, 1 maps to ~1400 bytes, etc
+	//For depth>1, we likely trigger stack growth for encoders, making benchmarking unreliable.
+	benchDepth     int
+	benchInitDebug bool
+	benchCheckers  []benchChecker
+)
+
+type benchEncFn func(interface{}) ([]byte, error)
+type benchDecFn func([]byte, interface{}) error
+type benchIntfFn func() interface{}
+
+type benchChecker struct {
+	name     string
+	encodefn benchEncFn
+	decodefn benchDecFn
+}
+
+func benchInitFlags() {
+	flag.BoolVar(&benchInitDebug, "bg", false, "Bench Debug")
+	flag.IntVar(&benchDepth, "bd", 1, "Bench Depth: If >1, potential unreliable results due to stack growth")
+	flag.BoolVar(&benchDoInitBench, "bi", false, "Run Bench Init")
+	flag.BoolVar(&benchVerify, "bv", false, "Verify Decoded Value during Benchmark")
+	flag.BoolVar(&benchUnscientificRes, "bu", false, "Show Unscientific Results during Benchmark")
+}
+
+func benchInit() {
+	benchTs = newTestStruc(benchDepth, true)
+	approxSize = approxDataSize(reflect.ValueOf(benchTs))
+	bytesLen := 1024 * 4 * (benchDepth + 1) * (benchDepth + 1)
+	if bytesLen < approxSize {
+		bytesLen = approxSize
+	}
+
+	benchCheckers = append(benchCheckers,
+		benchChecker{"msgpack", fnMsgpackEncodeFn, fnMsgpackDecodeFn},
+		benchChecker{"binc-nosym", fnBincNoSymEncodeFn, fnBincNoSymDecodeFn},
+		benchChecker{"binc-sym", fnBincSymEncodeFn, fnBincSymDecodeFn},
+		benchChecker{"simple", fnSimpleEncodeFn, fnSimpleDecodeFn},
+		benchChecker{"gob", fnGobEncodeFn, fnGobDecodeFn},
+		benchChecker{"json", fnJsonEncodeFn, fnJsonDecodeFn},
+	)
+	if benchDoInitBench {
+		runBenchInit()
+	}
+}
+
+func runBenchInit() {
+	logT(nil, "..............................................")
+	logT(nil, "BENCHMARK INIT: %v", time.Now())
+	logT(nil, "To run full benchmark comparing encodings (MsgPack, Binc, Simple, JSON, GOB, etc), "+
+		"use: \"go test -bench=.\"")
+	logT(nil, "Benchmark: ")
+	logT(nil, "\tStruct recursive Depth:             %d", benchDepth)
+	if approxSize > 0 {
+		logT(nil, "\tApproxDeepSize Of benchmark Struct: %d bytes", approxSize)
+	}
+	if benchUnscientificRes {
+		logT(nil, "Benchmark One-Pass Run (with Unscientific Encode/Decode times): ")
+	} else {
+		logT(nil, "Benchmark One-Pass Run:")
+	}
+	for _, bc := range benchCheckers {
+		doBenchCheck(bc.name, bc.encodefn, bc.decodefn)
+	}
+	logT(nil, "..............................................")
+	if benchInitDebug {
+		logT(nil, "<<<<====>>>> depth: %v, ts: %#v\n", benchDepth, benchTs)
+	}
+}
+
+func fnBenchNewTs() interface{} {
+	return new(TestStruc)
+}
+
+func doBenchCheck(name string, encfn benchEncFn, decfn benchDecFn) {
+	runtime.GC()
+	tnow := time.Now()
+	buf, err := encfn(benchTs)
+	if err != nil {
+		logT(nil, "\t%10s: **** Error encoding benchTs: %v", name, err)
+	}
+	encDur := time.Now().Sub(tnow)
+	encLen := len(buf)
+	runtime.GC()
+	if !benchUnscientificRes {
+		logT(nil, "\t%10s: len: %d bytes\n", name, encLen)
+		return
+	}
+	tnow = time.Now()
+	if err = decfn(buf, new(TestStruc)); err != nil {
+		logT(nil, "\t%10s: **** Error decoding into new TestStruc: %v", name, err)
+	}
+	decDur := time.Now().Sub(tnow)
+	logT(nil, "\t%10s: len: %d bytes, encode: %v, decode: %v\n", name, encLen, encDur, decDur)
+}
+
+func fnBenchmarkEncode(b *testing.B, encName string, ts interface{}, encfn benchEncFn) {
+	runtime.GC()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := encfn(ts)
+		if err != nil {
+			logT(b, "Error encoding benchTs: %s: %v", encName, err)
+			b.FailNow()
+		}
+	}
+}
+
+func fnBenchmarkDecode(b *testing.B, encName string, ts interface{},
+	encfn benchEncFn, decfn benchDecFn, newfn benchIntfFn,
+) {
+	buf, err := encfn(ts)
+	if err != nil {
+		logT(b, "Error encoding benchTs: %s: %v", encName, err)
+		b.FailNow()
+	}
+	runtime.GC()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		ts = newfn()
+		if err = decfn(buf, ts); err != nil {
+			logT(b, "Error decoding into new TestStruc: %s: %v", encName, err)
+			b.FailNow()
+		}
+		if benchVerify {
+			if vts, vok := ts.(*TestStruc); vok {
+				verifyTsTree(b, vts)
+			}
+		}
+	}
+}
+
+func verifyTsTree(b *testing.B, ts *TestStruc) {
+	var ts0, ts1m, ts2m, ts1s, ts2s *TestStruc
+	ts0 = ts
+
+	if benchDepth > 0 {
+		ts1m, ts1s = verifyCheckAndGet(b, ts0)
+	}
+
+	if benchDepth > 1 {
+		ts2m, ts2s = verifyCheckAndGet(b, ts1m)
+	}
+	for _, tsx := range []*TestStruc{ts0, ts1m, ts2m, ts1s, ts2s} {
+		if tsx != nil {
+			verifyOneOne(b, tsx)
+		}
+	}
+}
+
+func verifyCheckAndGet(b *testing.B, ts0 *TestStruc) (ts1m *TestStruc, ts1s *TestStruc) {
+	// if len(ts1m.Ms) <= 2 {
+	// 	logT(b, "Error: ts1m.Ms len should be > 2. Got: %v", len(ts1m.Ms))
+	// 	b.FailNow()
+	// }
+	if len(ts0.Its) == 0 {
+		logT(b, "Error: ts0.Islice len should be > 0. Got: %v", len(ts0.Its))
+		b.FailNow()
+	}
+	ts1m = ts0.Mtsptr["0"]
+	ts1s = ts0.Its[0]
+	if ts1m == nil || ts1s == nil {
+		logT(b, "Error: At benchDepth 1, No *TestStruc found")
+		b.FailNow()
+	}
+	return
+}
+
+func verifyOneOne(b *testing.B, ts *TestStruc) {
+	if ts.I64slice[2] != int64(3) {
+		logT(b, "Error: Decode failed by checking values")
+		b.FailNow()
+	}
+}
+
+func fnMsgpackEncodeFn(ts interface{}) (bs []byte, err error) {
+	err = NewEncoderBytes(&bs, testMsgpackH).Encode(ts)
+	return
+}
+
+func fnMsgpackDecodeFn(buf []byte, ts interface{}) error {
+	return NewDecoderBytes(buf, testMsgpackH).Decode(ts)
+}
+
+func fnBincEncodeFn(ts interface{}, sym AsSymbolFlag) (bs []byte, err error) {
+	tSym := testBincH.AsSymbols
+	testBincH.AsSymbols = sym
+	err = NewEncoderBytes(&bs, testBincH).Encode(ts)
+	testBincH.AsSymbols = tSym
+	return
+}
+
+func fnBincDecodeFn(buf []byte, ts interface{}, sym AsSymbolFlag) (err error) {
+	tSym := testBincH.AsSymbols
+	testBincH.AsSymbols = sym
+	err = NewDecoderBytes(buf, testBincH).Decode(ts)
+	testBincH.AsSymbols = tSym
+	return
+}
+
+func fnBincNoSymEncodeFn(ts interface{}) (bs []byte, err error) {
+	return fnBincEncodeFn(ts, AsSymbolNone)
+}
+
+func fnBincNoSymDecodeFn(buf []byte, ts interface{}) error {
+	return fnBincDecodeFn(buf, ts, AsSymbolNone)
+}
+
+func fnBincSymEncodeFn(ts interface{}) (bs []byte, err error) {
+	return fnBincEncodeFn(ts, AsSymbolAll)
+}
+
+func fnBincSymDecodeFn(buf []byte, ts interface{}) error {
+	return fnBincDecodeFn(buf, ts, AsSymbolAll)
+}
+
+func fnSimpleEncodeFn(ts interface{}) (bs []byte, err error) {
+	err = NewEncoderBytes(&bs, testSimpleH).Encode(ts)
+	return
+}
+
+func fnSimpleDecodeFn(buf []byte, ts interface{}) error {
+	return NewDecoderBytes(buf, testSimpleH).Decode(ts)
+}
+
+func fnGobEncodeFn(ts interface{}) ([]byte, error) {
+	bbuf := new(bytes.Buffer)
+	err := gob.NewEncoder(bbuf).Encode(ts)
+	return bbuf.Bytes(), err
+}
+
+func fnGobDecodeFn(buf []byte, ts interface{}) error {
+	return gob.NewDecoder(bytes.NewBuffer(buf)).Decode(ts)
+}
+
+func fnJsonEncodeFn(ts interface{}) ([]byte, error) {
+	return json.Marshal(ts)
+}
+
+func fnJsonDecodeFn(buf []byte, ts interface{}) error {
+	return json.Unmarshal(buf, ts)
+}
+
+func Benchmark__Msgpack____Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "msgpack", benchTs, fnMsgpackEncodeFn)
+}
+
+func Benchmark__Msgpack____Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "msgpack", benchTs, fnMsgpackEncodeFn, fnMsgpackDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Binc_NoSym_Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "binc", benchTs, fnBincNoSymEncodeFn)
+}
+
+func Benchmark__Binc_NoSym_Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "binc", benchTs, fnBincNoSymEncodeFn, fnBincNoSymDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Binc_Sym___Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "binc", benchTs, fnBincSymEncodeFn)
+}
+
+func Benchmark__Binc_Sym___Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "binc", benchTs, fnBincSymEncodeFn, fnBincSymDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Simple____Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "simple", benchTs, fnSimpleEncodeFn)
+}
+
+func Benchmark__Simple____Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "simple", benchTs, fnSimpleEncodeFn, fnSimpleDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Gob________Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "gob", benchTs, fnGobEncodeFn)
+}
+
+func Benchmark__Gob________Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "gob", benchTs, fnGobEncodeFn, fnGobDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Json_______Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "json", benchTs, fnJsonEncodeFn)
+}
+
+func Benchmark__Json_______Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "json", benchTs, fnJsonEncodeFn, fnJsonDecodeFn, fnBenchNewTs)
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/binc.go

@@ -0,0 +1,786 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+import (
+	"math"
+	// "reflect"
+	// "sync/atomic"
+	"time"
+	//"fmt"
+)
+
+const bincDoPrune = true // No longer needed. Needed before as C lib did not support pruning.
+
+//var _ = fmt.Printf
+
+// vd as low 4 bits (there are 16 slots)
+const (
+	bincVdSpecial byte = iota
+	bincVdPosInt
+	bincVdNegInt
+	bincVdFloat
+
+	bincVdString
+	bincVdByteArray
+	bincVdArray
+	bincVdMap
+
+	bincVdTimestamp
+	bincVdSmallInt
+	bincVdUnicodeOther
+	bincVdSymbol
+
+	bincVdDecimal
+	_               // open slot
+	_               // open slot
+	bincVdCustomExt = 0x0f
+)
+
+const (
+	bincSpNil byte = iota
+	bincSpFalse
+	bincSpTrue
+	bincSpNan
+	bincSpPosInf
+	bincSpNegInf
+	bincSpZeroFloat
+	bincSpZero
+	bincSpNegOne
+)
+
+const (
+	bincFlBin16 byte = iota
+	bincFlBin32
+	_ // bincFlBin32e
+	bincFlBin64
+	_ // bincFlBin64e
+	// others not currently supported
+)
+
+type bincEncDriver struct {
+	w encWriter
+	m map[string]uint16 // symbols
+	s uint32            // symbols sequencer
+	b [8]byte
+}
+
+func (e *bincEncDriver) isBuiltinType(rt uintptr) bool {
+	return rt == timeTypId
+}
+
+func (e *bincEncDriver) encodeBuiltin(rt uintptr, v interface{}) {
+	switch rt {
+	case timeTypId:
+		bs := encodeTime(v.(time.Time))
+		e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
+		e.w.writeb(bs)
+	}
+}
+
+func (e *bincEncDriver) encodeNil() {
+	e.w.writen1(bincVdSpecial<<4 | bincSpNil)
+}
+
+func (e *bincEncDriver) encodeBool(b bool) {
+	if b {
+		e.w.writen1(bincVdSpecial<<4 | bincSpTrue)
+	} else {
+		e.w.writen1(bincVdSpecial<<4 | bincSpFalse)
+	}
+}
+
+func (e *bincEncDriver) encodeFloat32(f float32) {
+	if f == 0 {
+		e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat)
+		return
+	}
+	e.w.writen1(bincVdFloat<<4 | bincFlBin32)
+	e.w.writeUint32(math.Float32bits(f))
+}
+
+func (e *bincEncDriver) encodeFloat64(f float64) {
+	if f == 0 {
+		e.w.writen1(bincVdSpecial<<4 | bincSpZeroFloat)
+		return
+	}
+	bigen.PutUint64(e.b[:], math.Float64bits(f))
+	if bincDoPrune {
+		i := 7
+		for ; i >= 0 && (e.b[i] == 0); i-- {
+		}
+		i++
+		if i <= 6 {
+			e.w.writen1(bincVdFloat<<4 | 0x8 | bincFlBin64)
+			e.w.writen1(byte(i))
+			e.w.writeb(e.b[:i])
+			return
+		}
+	}
+	e.w.writen1(bincVdFloat<<4 | bincFlBin64)
+	e.w.writeb(e.b[:])
+}
+
+func (e *bincEncDriver) encIntegerPrune(bd byte, pos bool, v uint64, lim uint8) {
+	if lim == 4 {
+		bigen.PutUint32(e.b[:lim], uint32(v))
+	} else {
+		bigen.PutUint64(e.b[:lim], v)
+	}
+	if bincDoPrune {
+		i := pruneSignExt(e.b[:lim], pos)
+		e.w.writen1(bd | lim - 1 - byte(i))
+		e.w.writeb(e.b[i:lim])
+	} else {
+		e.w.writen1(bd | lim - 1)
+		e.w.writeb(e.b[:lim])
+	}
+}
+
+func (e *bincEncDriver) encodeInt(v int64) {
+	const nbd byte = bincVdNegInt << 4
+	switch {
+	case v >= 0:
+		e.encUint(bincVdPosInt<<4, true, uint64(v))
+	case v == -1:
+		e.w.writen1(bincVdSpecial<<4 | bincSpNegOne)
+	default:
+		e.encUint(bincVdNegInt<<4, false, uint64(-v))
+	}
+}
+
+func (e *bincEncDriver) encodeUint(v uint64) {
+	e.encUint(bincVdPosInt<<4, true, v)
+}
+
+func (e *bincEncDriver) encUint(bd byte, pos bool, v uint64) {
+	switch {
+	case v == 0:
+		e.w.writen1(bincVdSpecial<<4 | bincSpZero)
+	case pos && v >= 1 && v <= 16:
+		e.w.writen1(bincVdSmallInt<<4 | byte(v-1))
+	case v <= math.MaxUint8:
+		e.w.writen2(bd|0x0, byte(v))
+	case v <= math.MaxUint16:
+		e.w.writen1(bd | 0x01)
+		e.w.writeUint16(uint16(v))
+	case v <= math.MaxUint32:
+		e.encIntegerPrune(bd, pos, v, 4)
+	default:
+		e.encIntegerPrune(bd, pos, v, 8)
+	}
+}
+
+func (e *bincEncDriver) encodeExtPreamble(xtag byte, length int) {
+	e.encLen(bincVdCustomExt<<4, uint64(length))
+	e.w.writen1(xtag)
+}
+
+func (e *bincEncDriver) encodeArrayPreamble(length int) {
+	e.encLen(bincVdArray<<4, uint64(length))
+}
+
+func (e *bincEncDriver) encodeMapPreamble(length int) {
+	e.encLen(bincVdMap<<4, uint64(length))
+}
+
+func (e *bincEncDriver) encodeString(c charEncoding, v string) {
+	l := uint64(len(v))
+	e.encBytesLen(c, l)
+	if l > 0 {
+		e.w.writestr(v)
+	}
+}
+
+func (e *bincEncDriver) encodeSymbol(v string) {
+	// if WriteSymbolsNoRefs {
+	// 	e.encodeString(c_UTF8, v)
+	// 	return
+	// }
+
+	//symbols only offer benefit when string length > 1.
+	//This is because strings with length 1 take only 2 bytes to store
+	//(bd with embedded length, and single byte for string val).
+
+	l := len(v)
+	switch l {
+	case 0:
+		e.encBytesLen(c_UTF8, 0)
+		return
+	case 1:
+		e.encBytesLen(c_UTF8, 1)
+		e.w.writen1(v[0])
+		return
+	}
+	if e.m == nil {
+		e.m = make(map[string]uint16, 16)
+	}
+	ui, ok := e.m[v]
+	if ok {
+		if ui <= math.MaxUint8 {
+			e.w.writen2(bincVdSymbol<<4, byte(ui))
+		} else {
+			e.w.writen1(bincVdSymbol<<4 | 0x8)
+			e.w.writeUint16(ui)
+		}
+	} else {
+		e.s++
+		ui = uint16(e.s)
+		//ui = uint16(atomic.AddUint32(&e.s, 1))
+		e.m[v] = ui
+		var lenprec uint8
+		switch {
+		case l <= math.MaxUint8:
+			// lenprec = 0
+		case l <= math.MaxUint16:
+			lenprec = 1
+		case int64(l) <= math.MaxUint32:
+			lenprec = 2
+		default:
+			lenprec = 3
+		}
+		if ui <= math.MaxUint8 {
+			e.w.writen2(bincVdSymbol<<4|0x0|0x4|lenprec, byte(ui))
+		} else {
+			e.w.writen1(bincVdSymbol<<4 | 0x8 | 0x4 | lenprec)
+			e.w.writeUint16(ui)
+		}
+		switch lenprec {
+		case 0:
+			e.w.writen1(byte(l))
+		case 1:
+			e.w.writeUint16(uint16(l))
+		case 2:
+			e.w.writeUint32(uint32(l))
+		default:
+			e.w.writeUint64(uint64(l))
+		}
+		e.w.writestr(v)
+	}
+}
+
+func (e *bincEncDriver) encodeStringBytes(c charEncoding, v []byte) {
+	l := uint64(len(v))
+	e.encBytesLen(c, l)
+	if l > 0 {
+		e.w.writeb(v)
+	}
+}
+
+func (e *bincEncDriver) encBytesLen(c charEncoding, length uint64) {
+	//TODO: support bincUnicodeOther (for now, just use string or bytearray)
+	if c == c_RAW {
+		e.encLen(bincVdByteArray<<4, length)
+	} else {
+		e.encLen(bincVdString<<4, length)
+	}
+}
+
+func (e *bincEncDriver) encLen(bd byte, l uint64) {
+	if l < 12 {
+		e.w.writen1(bd | uint8(l+4))
+	} else {
+		e.encLenNumber(bd, l)
+	}
+}
+
+func (e *bincEncDriver) encLenNumber(bd byte, v uint64) {
+	switch {
+	case v <= math.MaxUint8:
+		e.w.writen2(bd, byte(v))
+	case v <= math.MaxUint16:
+		e.w.writen1(bd | 0x01)
+		e.w.writeUint16(uint16(v))
+	case v <= math.MaxUint32:
+		e.w.writen1(bd | 0x02)
+		e.w.writeUint32(uint32(v))
+	default:
+		e.w.writen1(bd | 0x03)
+		e.w.writeUint64(uint64(v))
+	}
+}
+
+//------------------------------------
+
+type bincDecDriver struct {
+	r      decReader
+	bdRead bool
+	bdType valueType
+	bd     byte
+	vd     byte
+	vs     byte
+	b      [8]byte
+	m      map[uint32]string // symbols (use uint32 as key, as map optimizes for it)
+}
+
+func (d *bincDecDriver) initReadNext() {
+	if d.bdRead {
+		return
+	}
+	d.bd = d.r.readn1()
+	d.vd = d.bd >> 4
+	d.vs = d.bd & 0x0f
+	d.bdRead = true
+	d.bdType = valueTypeUnset
+}
+
+func (d *bincDecDriver) currentEncodedType() valueType {
+	if d.bdType == valueTypeUnset {
+		switch d.vd {
+		case bincVdSpecial:
+			switch d.vs {
+			case bincSpNil:
+				d.bdType = valueTypeNil
+			case bincSpFalse, bincSpTrue:
+				d.bdType = valueTypeBool
+			case bincSpNan, bincSpNegInf, bincSpPosInf, bincSpZeroFloat:
+				d.bdType = valueTypeFloat
+			case bincSpZero:
+				d.bdType = valueTypeUint
+			case bincSpNegOne:
+				d.bdType = valueTypeInt
+			default:
+				decErr("currentEncodedType: Unrecognized special value 0x%x", d.vs)
+			}
+		case bincVdSmallInt:
+			d.bdType = valueTypeUint
+		case bincVdPosInt:
+			d.bdType = valueTypeUint
+		case bincVdNegInt:
+			d.bdType = valueTypeInt
+		case bincVdFloat:
+			d.bdType = valueTypeFloat
+		case bincVdString:
+			d.bdType = valueTypeString
+		case bincVdSymbol:
+			d.bdType = valueTypeSymbol
+		case bincVdByteArray:
+			d.bdType = valueTypeBytes
+		case bincVdTimestamp:
+			d.bdType = valueTypeTimestamp
+		case bincVdCustomExt:
+			d.bdType = valueTypeExt
+		case bincVdArray:
+			d.bdType = valueTypeArray
+		case bincVdMap:
+			d.bdType = valueTypeMap
+		default:
+			decErr("currentEncodedType: Unrecognized d.vd: 0x%x", d.vd)
+		}
+	}
+	return d.bdType
+}
+
+func (d *bincDecDriver) tryDecodeAsNil() bool {
+	if d.bd == bincVdSpecial<<4|bincSpNil {
+		d.bdRead = false
+		return true
+	}
+	return false
+}
+
+func (d *bincDecDriver) isBuiltinType(rt uintptr) bool {
+	return rt == timeTypId
+}
+
+func (d *bincDecDriver) decodeBuiltin(rt uintptr, v interface{}) {
+	switch rt {
+	case timeTypId:
+		if d.vd != bincVdTimestamp {
+			decErr("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd)
+		}
+		tt, err := decodeTime(d.r.readn(int(d.vs)))
+		if err != nil {
+			panic(err)
+		}
+		var vt *time.Time = v.(*time.Time)
+		*vt = tt
+		d.bdRead = false
+	}
+}
+
+func (d *bincDecDriver) decFloatPre(vs, defaultLen byte) {
+	if vs&0x8 == 0 {
+		d.r.readb(d.b[0:defaultLen])
+	} else {
+		l := d.r.readn1()
+		if l > 8 {
+			decErr("At most 8 bytes used to represent float. Received: %v bytes", l)
+		}
+		for i := l; i < 8; i++ {
+			d.b[i] = 0
+		}
+		d.r.readb(d.b[0:l])
+	}
+}
+
+func (d *bincDecDriver) decFloat() (f float64) {
+	//if true { f = math.Float64frombits(d.r.readUint64()); break; }
+	switch vs := d.vs; vs & 0x7 {
+	case bincFlBin32:
+		d.decFloatPre(vs, 4)
+		f = float64(math.Float32frombits(bigen.Uint32(d.b[0:4])))
+	case bincFlBin64:
+		d.decFloatPre(vs, 8)
+		f = math.Float64frombits(bigen.Uint64(d.b[0:8]))
+	default:
+		decErr("only float32 and float64 are supported. d.vd: 0x%x, d.vs: 0x%x", d.vd, d.vs)
+	}
+	return
+}
+
+func (d *bincDecDriver) decUint() (v uint64) {
+	// need to inline the code (interface conversion and type assertion expensive)
+	switch d.vs {
+	case 0:
+		v = uint64(d.r.readn1())
+	case 1:
+		d.r.readb(d.b[6:])
+		v = uint64(bigen.Uint16(d.b[6:]))
+	case 2:
+		d.b[4] = 0
+		d.r.readb(d.b[5:])
+		v = uint64(bigen.Uint32(d.b[4:]))
+	case 3:
+		d.r.readb(d.b[4:])
+		v = uint64(bigen.Uint32(d.b[4:]))
+	case 4, 5, 6:
+		lim := int(7 - d.vs)
+		d.r.readb(d.b[lim:])
+		for i := 0; i < lim; i++ {
+			d.b[i] = 0
+		}
+		v = uint64(bigen.Uint64(d.b[:]))
+	case 7:
+		d.r.readb(d.b[:])
+		v = uint64(bigen.Uint64(d.b[:]))
+	default:
+		decErr("unsigned integers with greater than 64 bits of precision not supported")
+	}
+	return
+}
+
+func (d *bincDecDriver) decIntAny() (ui uint64, i int64, neg bool) {
+	switch d.vd {
+	case bincVdPosInt:
+		ui = d.decUint()
+		i = int64(ui)
+	case bincVdNegInt:
+		ui = d.decUint()
+		i = -(int64(ui))
+		neg = true
+	case bincVdSmallInt:
+		i = int64(d.vs) + 1
+		ui = uint64(d.vs) + 1
+	case bincVdSpecial:
+		switch d.vs {
+		case bincSpZero:
+			//i = 0
+		case bincSpNegOne:
+			neg = true
+			ui = 1
+			i = -1
+		default:
+			decErr("numeric decode fails for special value: d.vs: 0x%x", d.vs)
+		}
+	default:
+		decErr("number can only be decoded from uint or int values. d.bd: 0x%x, d.vd: 0x%x", d.bd, d.vd)
+	}
+	return
+}
+
+func (d *bincDecDriver) decodeInt(bitsize uint8) (i int64) {
+	_, i, _ = d.decIntAny()
+	checkOverflow(0, i, bitsize)
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decodeUint(bitsize uint8) (ui uint64) {
+	ui, i, neg := d.decIntAny()
+	if neg {
+		decErr("Assigning negative signed value: %v, to unsigned type", i)
+	}
+	checkOverflow(ui, 0, bitsize)
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
+	switch d.vd {
+	case bincVdSpecial:
+		d.bdRead = false
+		switch d.vs {
+		case bincSpNan:
+			return math.NaN()
+		case bincSpPosInf:
+			return math.Inf(1)
+		case bincSpZeroFloat, bincSpZero:
+			return
+		case bincSpNegInf:
+			return math.Inf(-1)
+		default:
+			decErr("Invalid d.vs decoding float where d.vd=bincVdSpecial: %v", d.vs)
+		}
+	case bincVdFloat:
+		f = d.decFloat()
+	default:
+		_, i, _ := d.decIntAny()
+		f = float64(i)
+	}
+	checkOverflowFloat32(f, chkOverflow32)
+	d.bdRead = false
+	return
+}
+
+// bool can be decoded from bool only (single byte).
+func (d *bincDecDriver) decodeBool() (b bool) {
+	switch d.bd {
+	case (bincVdSpecial | bincSpFalse):
+		// b = false
+	case (bincVdSpecial | bincSpTrue):
+		b = true
+	default:
+		decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) readMapLen() (length int) {
+	if d.vd != bincVdMap {
+		decErr("Invalid d.vd for map. Expecting 0x%x. Got: 0x%x", bincVdMap, d.vd)
+	}
+	length = d.decLen()
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) readArrayLen() (length int) {
+	if d.vd != bincVdArray {
+		decErr("Invalid d.vd for array. Expecting 0x%x. Got: 0x%x", bincVdArray, d.vd)
+	}
+	length = d.decLen()
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decLen() int {
+	if d.vs <= 3 {
+		return int(d.decUint())
+	}
+	return int(d.vs - 4)
+}
+
+func (d *bincDecDriver) decodeString() (s string) {
+	switch d.vd {
+	case bincVdString, bincVdByteArray:
+		if length := d.decLen(); length > 0 {
+			s = string(d.r.readn(length))
+		}
+	case bincVdSymbol:
+		//from vs: extract numSymbolBytes, containsStringVal, strLenPrecision,
+		//extract symbol
+		//if containsStringVal, read it and put in map
+		//else look in map for string value
+		var symbol uint32
+		vs := d.vs
+		//fmt.Printf(">>>> d.vs: 0b%b, & 0x8: %v, & 0x4: %v\n", d.vs, vs & 0x8, vs & 0x4)
+		if vs&0x8 == 0 {
+			symbol = uint32(d.r.readn1())
+		} else {
+			symbol = uint32(d.r.readUint16())
+		}
+		if d.m == nil {
+			d.m = make(map[uint32]string, 16)
+		}
+
+		if vs&0x4 == 0 {
+			s = d.m[symbol]
+		} else {
+			var slen int
+			switch vs & 0x3 {
+			case 0:
+				slen = int(d.r.readn1())
+			case 1:
+				slen = int(d.r.readUint16())
+			case 2:
+				slen = int(d.r.readUint32())
+			case 3:
+				slen = int(d.r.readUint64())
+			}
+			s = string(d.r.readn(slen))
+			d.m[symbol] = s
+		}
+	default:
+		decErr("Invalid d.vd for string. Expecting string:0x%x, bytearray:0x%x or symbol: 0x%x. Got: 0x%x",
+			bincVdString, bincVdByteArray, bincVdSymbol, d.vd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
+	var clen int
+	switch d.vd {
+	case bincVdString, bincVdByteArray:
+		clen = d.decLen()
+	default:
+		decErr("Invalid d.vd for bytes. Expecting string:0x%x or bytearray:0x%x. Got: 0x%x",
+			bincVdString, bincVdByteArray, d.vd)
+	}
+	if clen > 0 {
+		// if no contents in stream, don't update the passed byteslice
+		if len(bs) != clen {
+			if len(bs) > clen {
+				bs = bs[:clen]
+			} else {
+				bs = make([]byte, clen)
+			}
+			bsOut = bs
+			changed = true
+		}
+		d.r.readb(bs)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
+	switch d.vd {
+	case bincVdCustomExt:
+		l := d.decLen()
+		xtag = d.r.readn1()
+		if verifyTag && xtag != tag {
+			decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
+		}
+		xbs = d.r.readn(l)
+	case bincVdByteArray:
+		xbs, _ = d.decodeBytes(nil)
+	default:
+		decErr("Invalid d.vd for extensions (Expecting extensions or byte array). Got: 0x%x", d.vd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *bincDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+	d.initReadNext()
+
+	switch d.vd {
+	case bincVdSpecial:
+		switch d.vs {
+		case bincSpNil:
+			vt = valueTypeNil
+		case bincSpFalse:
+			vt = valueTypeBool
+			v = false
+		case bincSpTrue:
+			vt = valueTypeBool
+			v = true
+		case bincSpNan:
+			vt = valueTypeFloat
+			v = math.NaN()
+		case bincSpPosInf:
+			vt = valueTypeFloat
+			v = math.Inf(1)
+		case bincSpNegInf:
+			vt = valueTypeFloat
+			v = math.Inf(-1)
+		case bincSpZeroFloat:
+			vt = valueTypeFloat
+			v = float64(0)
+		case bincSpZero:
+			vt = valueTypeUint
+			v = int64(0) // int8(0)
+		case bincSpNegOne:
+			vt = valueTypeInt
+			v = int64(-1) // int8(-1)
+		default:
+			decErr("decodeNaked: Unrecognized special value 0x%x", d.vs)
+		}
+	case bincVdSmallInt:
+		vt = valueTypeUint
+		v = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1
+	case bincVdPosInt:
+		vt = valueTypeUint
+		v = d.decUint()
+	case bincVdNegInt:
+		vt = valueTypeInt
+		v = -(int64(d.decUint()))
+	case bincVdFloat:
+		vt = valueTypeFloat
+		v = d.decFloat()
+	case bincVdSymbol:
+		vt = valueTypeSymbol
+		v = d.decodeString()
+	case bincVdString:
+		vt = valueTypeString
+		v = d.decodeString()
+	case bincVdByteArray:
+		vt = valueTypeBytes
+		v, _ = d.decodeBytes(nil)
+	case bincVdTimestamp:
+		vt = valueTypeTimestamp
+		tt, err := decodeTime(d.r.readn(int(d.vs)))
+		if err != nil {
+			panic(err)
+		}
+		v = tt
+	case bincVdCustomExt:
+		vt = valueTypeExt
+		l := d.decLen()
+		var re RawExt
+		re.Tag = d.r.readn1()
+		re.Data = d.r.readn(l)
+		v = &re
+		vt = valueTypeExt
+	case bincVdArray:
+		vt = valueTypeArray
+		decodeFurther = true
+	case bincVdMap:
+		vt = valueTypeMap
+		decodeFurther = true
+	default:
+		decErr("decodeNaked: Unrecognized d.vd: 0x%x", d.vd)
+	}
+
+	if !decodeFurther {
+		d.bdRead = false
+	}
+	return
+}
+
+//------------------------------------
+
+//BincHandle is a Handle for the Binc Schema-Free Encoding Format
+//defined at https://github.com/ugorji/binc .
+//
+//BincHandle currently supports all Binc features with the following EXCEPTIONS:
+//  - only integers up to 64 bits of precision are supported.
+//    big integers are unsupported.
+//  - Only IEEE 754 binary32 and binary64 floats are supported (ie Go float32 and float64 types).
+//    extended precision and decimal IEEE 754 floats are unsupported.
+//  - Only UTF-8 strings supported.
+//    Unicode_Other Binc types (UTF16, UTF32) are currently unsupported.
+//Note that these EXCEPTIONS are temporary and full support is possible and may happen soon.
+type BincHandle struct {
+	BasicHandle
+}
+
+func (h *BincHandle) newEncDriver(w encWriter) encDriver {
+	return &bincEncDriver{w: w}
+}
+
+func (h *BincHandle) newDecDriver(r decReader) decDriver {
+	return &bincDecDriver{r: r}
+}
+
+func (_ *BincHandle) writeExt() bool {
+	return true
+}
+
+func (h *BincHandle) getBasicHandle() *BasicHandle {
+	return &h.BasicHandle
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/ext_dep_test.go

@@ -0,0 +1,75 @@
+// //+build ignore
+
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+// This file includes benchmarks which have dependencies on 3rdparty
+// packages (bson and vmihailenco/msgpack) which must be installed locally.
+//
+// To run the benchmarks including these 3rdparty packages, first
+//   - Uncomment first line in this file (put // // in front of it)
+//   - Get those packages:
+//       go get github.com/vmihailenco/msgpack
+//       go get labix.org/v2/mgo/bson
+//   - Run:
+//       go test -bi -bench=.
+
+import (
+	"testing"
+
+	vmsgpack "github.com/vmihailenco/msgpack"
+	"labix.org/v2/mgo/bson"
+)
+
+func init() {
+	benchCheckers = append(benchCheckers,
+		benchChecker{"v-msgpack", fnVMsgpackEncodeFn, fnVMsgpackDecodeFn},
+		benchChecker{"bson", fnBsonEncodeFn, fnBsonDecodeFn},
+	)
+}
+
+func fnVMsgpackEncodeFn(ts interface{}) ([]byte, error) {
+	return vmsgpack.Marshal(ts)
+}
+
+func fnVMsgpackDecodeFn(buf []byte, ts interface{}) error {
+	return vmsgpack.Unmarshal(buf, ts)
+}
+
+func fnBsonEncodeFn(ts interface{}) ([]byte, error) {
+	return bson.Marshal(ts)
+}
+
+func fnBsonDecodeFn(buf []byte, ts interface{}) error {
+	return bson.Unmarshal(buf, ts)
+}
+
+func Benchmark__Bson_______Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "bson", benchTs, fnBsonEncodeFn)
+}
+
+func Benchmark__Bson_______Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "bson", benchTs, fnBsonEncodeFn, fnBsonDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__VMsgpack___Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "v-msgpack", benchTs, fnVMsgpackEncodeFn)
+}
+
+func Benchmark__VMsgpack___Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "v-msgpack", benchTs, fnVMsgpackEncodeFn, fnVMsgpackDecodeFn, fnBenchNewTs)
+}
+
+func TestMsgpackPythonGenStreams(t *testing.T) {
+	doTestMsgpackPythonGenStreams(t)
+}
+
+func TestMsgpackRpcSpecGoClientToPythonSvc(t *testing.T) {
+	doTestMsgpackRpcSpecGoClientToPythonSvc(t)
+}
+
+func TestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) {
+	doTestMsgpackRpcSpecPythonClientToGoSvc(t)
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/helper.go

@@ -0,0 +1,589 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+// Contains code shared by both encode and decode.
+
+import (
+	"encoding/binary"
+	"fmt"
+	"math"
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+const (
+	structTagName = "codec"
+
+	// Support
+	//    encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error)
+	//    encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error
+	// This constant flag will enable or disable it.
+	supportBinaryMarshal = true
+
+	// Each Encoder or Decoder uses a cache of functions based on conditionals,
+	// so that the conditionals are not run every time.
+	//
+	// Either a map or a slice is used to keep track of the functions.
+	// The map is more natural, but has a higher cost than a slice/array.
+	// This flag (useMapForCodecCache) controls which is used.
+	useMapForCodecCache = false
+
+	// For some common container types, we can short-circuit an elaborate
+	// reflection dance and call encode/decode directly.
+	// The currently supported types are:
+	//    - slices of strings, or id's (int64,uint64) or interfaces.
+	//    - maps of str->str, str->intf, id(int64,uint64)->intf, intf->intf
+	shortCircuitReflectToFastPath = true
+
+	// for debugging, set this to false, to catch panic traces.
+	// Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic.
+	recoverPanicToErr = true
+)
+
+type charEncoding uint8
+
+const (
+	c_RAW charEncoding = iota
+	c_UTF8
+	c_UTF16LE
+	c_UTF16BE
+	c_UTF32LE
+	c_UTF32BE
+)
+
+// valueType is the stream type
+type valueType uint8
+
+const (
+	valueTypeUnset valueType = iota
+	valueTypeNil
+	valueTypeInt
+	valueTypeUint
+	valueTypeFloat
+	valueTypeBool
+	valueTypeString
+	valueTypeSymbol
+	valueTypeBytes
+	valueTypeMap
+	valueTypeArray
+	valueTypeTimestamp
+	valueTypeExt
+
+	valueTypeInvalid = 0xff
+)
+
+var (
+	bigen               = binary.BigEndian
+	structInfoFieldName = "_struct"
+
+	cachedTypeInfo      = make(map[uintptr]*typeInfo, 4)
+	cachedTypeInfoMutex sync.RWMutex
+
+	intfSliceTyp = reflect.TypeOf([]interface{}(nil))
+	intfTyp      = intfSliceTyp.Elem()
+
+	strSliceTyp     = reflect.TypeOf([]string(nil))
+	boolSliceTyp    = reflect.TypeOf([]bool(nil))
+	uintSliceTyp    = reflect.TypeOf([]uint(nil))
+	uint8SliceTyp   = reflect.TypeOf([]uint8(nil))
+	uint16SliceTyp  = reflect.TypeOf([]uint16(nil))
+	uint32SliceTyp  = reflect.TypeOf([]uint32(nil))
+	uint64SliceTyp  = reflect.TypeOf([]uint64(nil))
+	intSliceTyp     = reflect.TypeOf([]int(nil))
+	int8SliceTyp    = reflect.TypeOf([]int8(nil))
+	int16SliceTyp   = reflect.TypeOf([]int16(nil))
+	int32SliceTyp   = reflect.TypeOf([]int32(nil))
+	int64SliceTyp   = reflect.TypeOf([]int64(nil))
+	float32SliceTyp = reflect.TypeOf([]float32(nil))
+	float64SliceTyp = reflect.TypeOf([]float64(nil))
+
+	mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
+	mapStrIntfTyp  = reflect.TypeOf(map[string]interface{}(nil))
+	mapStrStrTyp   = reflect.TypeOf(map[string]string(nil))
+
+	mapIntIntfTyp    = reflect.TypeOf(map[int]interface{}(nil))
+	mapInt64IntfTyp  = reflect.TypeOf(map[int64]interface{}(nil))
+	mapUintIntfTyp   = reflect.TypeOf(map[uint]interface{}(nil))
+	mapUint64IntfTyp = reflect.TypeOf(map[uint64]interface{}(nil))
+
+	stringTyp = reflect.TypeOf("")
+	timeTyp   = reflect.TypeOf(time.Time{})
+	rawExtTyp = reflect.TypeOf(RawExt{})
+
+	mapBySliceTyp        = reflect.TypeOf((*MapBySlice)(nil)).Elem()
+	binaryMarshalerTyp   = reflect.TypeOf((*binaryMarshaler)(nil)).Elem()
+	binaryUnmarshalerTyp = reflect.TypeOf((*binaryUnmarshaler)(nil)).Elem()
+
+	rawExtTypId = reflect.ValueOf(rawExtTyp).Pointer()
+	intfTypId   = reflect.ValueOf(intfTyp).Pointer()
+	timeTypId   = reflect.ValueOf(timeTyp).Pointer()
+
+	intfSliceTypId = reflect.ValueOf(intfSliceTyp).Pointer()
+	strSliceTypId  = reflect.ValueOf(strSliceTyp).Pointer()
+
+	boolSliceTypId    = reflect.ValueOf(boolSliceTyp).Pointer()
+	uintSliceTypId    = reflect.ValueOf(uintSliceTyp).Pointer()
+	uint8SliceTypId   = reflect.ValueOf(uint8SliceTyp).Pointer()
+	uint16SliceTypId  = reflect.ValueOf(uint16SliceTyp).Pointer()
+	uint32SliceTypId  = reflect.ValueOf(uint32SliceTyp).Pointer()
+	uint64SliceTypId  = reflect.ValueOf(uint64SliceTyp).Pointer()
+	intSliceTypId     = reflect.ValueOf(intSliceTyp).Pointer()
+	int8SliceTypId    = reflect.ValueOf(int8SliceTyp).Pointer()
+	int16SliceTypId   = reflect.ValueOf(int16SliceTyp).Pointer()
+	int32SliceTypId   = reflect.ValueOf(int32SliceTyp).Pointer()
+	int64SliceTypId   = reflect.ValueOf(int64SliceTyp).Pointer()
+	float32SliceTypId = reflect.ValueOf(float32SliceTyp).Pointer()
+	float64SliceTypId = reflect.ValueOf(float64SliceTyp).Pointer()
+
+	mapStrStrTypId     = reflect.ValueOf(mapStrStrTyp).Pointer()
+	mapIntfIntfTypId   = reflect.ValueOf(mapIntfIntfTyp).Pointer()
+	mapStrIntfTypId    = reflect.ValueOf(mapStrIntfTyp).Pointer()
+	mapIntIntfTypId    = reflect.ValueOf(mapIntIntfTyp).Pointer()
+	mapInt64IntfTypId  = reflect.ValueOf(mapInt64IntfTyp).Pointer()
+	mapUintIntfTypId   = reflect.ValueOf(mapUintIntfTyp).Pointer()
+	mapUint64IntfTypId = reflect.ValueOf(mapUint64IntfTyp).Pointer()
+	// Id = reflect.ValueOf().Pointer()
+	// mapBySliceTypId  = reflect.ValueOf(mapBySliceTyp).Pointer()
+
+	binaryMarshalerTypId   = reflect.ValueOf(binaryMarshalerTyp).Pointer()
+	binaryUnmarshalerTypId = reflect.ValueOf(binaryUnmarshalerTyp).Pointer()
+
+	intBitsize  uint8 = uint8(reflect.TypeOf(int(0)).Bits())
+	uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())
+
+	bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
+	bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+)
+
+type binaryUnmarshaler interface {
+	UnmarshalBinary(data []byte) error
+}
+
+type binaryMarshaler interface {
+	MarshalBinary() (data []byte, err error)
+}
+
+// MapBySlice represents a slice which should be encoded as a map in the stream.
+// The slice contains a sequence of key-value pairs.
+type MapBySlice interface {
+	MapBySlice()
+}
+
+// WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED.
+//
+// BasicHandle encapsulates the common options and extension functions.
+type BasicHandle struct {
+	extHandle
+	EncodeOptions
+	DecodeOptions
+}
+
+// Handle is the interface for a specific encoding format.
+//
+// Typically, a Handle is pre-configured before first time use,
+// and not modified while in use. Such a pre-configured Handle
+// is safe for concurrent access.
+type Handle interface {
+	writeExt() bool
+	getBasicHandle() *BasicHandle
+	newEncDriver(w encWriter) encDriver
+	newDecDriver(r decReader) decDriver
+}
+
+// RawExt represents raw unprocessed extension data.
+type RawExt struct {
+	Tag  byte
+	Data []byte
+}
+
+type extTypeTagFn struct {
+	rtid  uintptr
+	rt    reflect.Type
+	tag   byte
+	encFn func(reflect.Value) ([]byte, error)
+	decFn func(reflect.Value, []byte) error
+}
+
+type extHandle []*extTypeTagFn
+
+// AddExt registers an encode and decode function for a reflect.Type.
+// Note that the type must be a named type, and specifically not
+// a pointer or Interface. An error is returned if that is not honored.
+//
+// To Deregister an ext, call AddExt with 0 tag, nil encfn and nil decfn.
+func (o *extHandle) AddExt(
+	rt reflect.Type,
+	tag byte,
+	encfn func(reflect.Value) ([]byte, error),
+	decfn func(reflect.Value, []byte) error,
+) (err error) {
+	// o is a pointer, because we may need to initialize it
+	if rt.PkgPath() == "" || rt.Kind() == reflect.Interface {
+		err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T",
+			reflect.Zero(rt).Interface())
+		return
+	}
+
+	// o cannot be nil, since it is always embedded in a Handle.
+	// if nil, let it panic.
+	// if o == nil {
+	// 	err = errors.New("codec.Handle.AddExt: extHandle cannot be a nil pointer.")
+	// 	return
+	// }
+
+	rtid := reflect.ValueOf(rt).Pointer()
+	for _, v := range *o {
+		if v.rtid == rtid {
+			v.tag, v.encFn, v.decFn = tag, encfn, decfn
+			return
+		}
+	}
+
+	*o = append(*o, &extTypeTagFn{rtid, rt, tag, encfn, decfn})
+	return
+}
+
+func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
+	for _, v := range o {
+		if v.rtid == rtid {
+			return v
+		}
+	}
+	return nil
+}
+
+func (o extHandle) getExtForTag(tag byte) *extTypeTagFn {
+	for _, v := range o {
+		if v.tag == tag {
+			return v
+		}
+	}
+	return nil
+}
+
+func (o extHandle) getDecodeExtForTag(tag byte) (
+	rv reflect.Value, fn func(reflect.Value, []byte) error) {
+	if x := o.getExtForTag(tag); x != nil {
+		// ext is only registered for base
+		rv = reflect.New(x.rt).Elem()
+		fn = x.decFn
+	}
+	return
+}
+
+func (o extHandle) getDecodeExt(rtid uintptr) (tag byte, fn func(reflect.Value, []byte) error) {
+	if x := o.getExt(rtid); x != nil {
+		tag = x.tag
+		fn = x.decFn
+	}
+	return
+}
+
+func (o extHandle) getEncodeExt(rtid uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) {
+	if x := o.getExt(rtid); x != nil {
+		tag = x.tag
+		fn = x.encFn
+	}
+	return
+}
+
+type structFieldInfo struct {
+	encName string // encode name
+
+	// only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set.
+
+	is        []int // (recursive/embedded) field index in struct
+	i         int16 // field index in struct
+	omitEmpty bool
+	toArray   bool // if field is _struct, is the toArray set?
+
+	// tag       string   // tag
+	// name      string   // field name
+	// encNameBs []byte   // encoded name as byte stream
+	// ikind     int      // kind of the field as an int i.e. int(reflect.Kind)
+}
+
+func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
+	if fname == "" {
+		panic("parseStructFieldInfo: No Field Name")
+	}
+	si := structFieldInfo{
+		// name: fname,
+		encName: fname,
+		// tag: stag,
+	}
+
+	if stag != "" {
+		for i, s := range strings.Split(stag, ",") {
+			if i == 0 {
+				if s != "" {
+					si.encName = s
+				}
+			} else {
+				switch s {
+				case "omitempty":
+					si.omitEmpty = true
+				case "toarray":
+					si.toArray = true
+				}
+			}
+		}
+	}
+	// si.encNameBs = []byte(si.encName)
+	return &si
+}
+
+type sfiSortedByEncName []*structFieldInfo
+
+func (p sfiSortedByEncName) Len() int {
+	return len(p)
+}
+
+func (p sfiSortedByEncName) Less(i, j int) bool {
+	return p[i].encName < p[j].encName
+}
+
+func (p sfiSortedByEncName) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+// typeInfo keeps information about each type referenced in the encode/decode sequence.
+//
+// During an encode/decode sequence, we work as below:
+//   - If base is a built in type, en/decode base value
+//   - If base is registered as an extension, en/decode base value
+//   - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method
+//   - Else decode appropriately based on the reflect.Kind
+type typeInfo struct {
+	sfi  []*structFieldInfo // sorted. Used when enc/dec struct to map.
+	sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array.
+
+	rt   reflect.Type
+	rtid uintptr
+
+	// baseId gives pointer to the base reflect.Type, after deferencing
+	// the pointers. E.g. base type of ***time.Time is time.Time.
+	base      reflect.Type
+	baseId    uintptr
+	baseIndir int8 // number of indirections to get to base
+
+	mbs bool // base type (T or *T) is a MapBySlice
+
+	m        bool // base type (T or *T) is a binaryMarshaler
+	unm      bool // base type (T or *T) is a binaryUnmarshaler
+	mIndir   int8 // number of indirections to get to binaryMarshaler type
+	unmIndir int8 // number of indirections to get to binaryUnmarshaler type
+	toArray  bool // whether this (struct) type should be encoded as an array
+}
+
+func (ti *typeInfo) indexForEncName(name string) int {
+	//tisfi := ti.sfi
+	const binarySearchThreshold = 16
+	if sfilen := len(ti.sfi); sfilen < binarySearchThreshold {
+		// linear search. faster than binary search in my testing up to 16-field structs.
+		for i, si := range ti.sfi {
+			if si.encName == name {
+				return i
+			}
+		}
+	} else {
+		// binary search. adapted from sort/search.go.
+		h, i, j := 0, 0, sfilen
+		for i < j {
+			h = i + (j-i)/2
+			if ti.sfi[h].encName < name {
+				i = h + 1
+			} else {
+				j = h
+			}
+		}
+		if i < sfilen && ti.sfi[i].encName == name {
+			return i
+		}
+	}
+	return -1
+}
+
+func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
+	var ok bool
+	cachedTypeInfoMutex.RLock()
+	pti, ok = cachedTypeInfo[rtid]
+	cachedTypeInfoMutex.RUnlock()
+	if ok {
+		return
+	}
+
+	cachedTypeInfoMutex.Lock()
+	defer cachedTypeInfoMutex.Unlock()
+	if pti, ok = cachedTypeInfo[rtid]; ok {
+		return
+	}
+
+	ti := typeInfo{rt: rt, rtid: rtid}
+	pti = &ti
+
+	var indir int8
+	if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
+		ti.m, ti.mIndir = true, indir
+	}
+	if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok {
+		ti.unm, ti.unmIndir = true, indir
+	}
+	if ok, _ = implementsIntf(rt, mapBySliceTyp); ok {
+		ti.mbs = true
+	}
+
+	pt := rt
+	var ptIndir int8
+	// for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { }
+	for pt.Kind() == reflect.Ptr {
+		pt = pt.Elem()
+		ptIndir++
+	}
+	if ptIndir == 0 {
+		ti.base = rt
+		ti.baseId = rtid
+	} else {
+		ti.base = pt
+		ti.baseId = reflect.ValueOf(pt).Pointer()
+		ti.baseIndir = ptIndir
+	}
+
+	if rt.Kind() == reflect.Struct {
+		var siInfo *structFieldInfo
+		if f, ok := rt.FieldByName(structInfoFieldName); ok {
+			siInfo = parseStructFieldInfo(structInfoFieldName, f.Tag.Get(structTagName))
+			ti.toArray = siInfo.toArray
+		}
+		sfip := make([]*structFieldInfo, 0, rt.NumField())
+		rgetTypeInfo(rt, nil, make(map[string]bool), &sfip, siInfo)
+
+		// // try to put all si close together
+		// const tryToPutAllStructFieldInfoTogether = true
+		// if tryToPutAllStructFieldInfoTogether {
+		// 	sfip2 := make([]structFieldInfo, len(sfip))
+		// 	for i, si := range sfip {
+		// 		sfip2[i] = *si
+		// 	}
+		// 	for i := range sfip {
+		// 		sfip[i] = &sfip2[i]
+		// 	}
+		// }
+
+		ti.sfip = make([]*structFieldInfo, len(sfip))
+		ti.sfi = make([]*structFieldInfo, len(sfip))
+		copy(ti.sfip, sfip)
+		sort.Sort(sfiSortedByEncName(sfip))
+		copy(ti.sfi, sfip)
+	}
+	// sfi = sfip
+	cachedTypeInfo[rtid] = pti
+	return
+}
+
+func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
+	sfi *[]*structFieldInfo, siInfo *structFieldInfo,
+) {
+	// for rt.Kind() == reflect.Ptr {
+	// 	// indexstack = append(indexstack, 0)
+	// 	rt = rt.Elem()
+	// }
+	for j := 0; j < rt.NumField(); j++ {
+		f := rt.Field(j)
+		stag := f.Tag.Get(structTagName)
+		if stag == "-" {
+			continue
+		}
+		if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
+			continue
+		}
+		// if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it.
+		if f.Anonymous && stag == "" {
+			ft := f.Type
+			for ft.Kind() == reflect.Ptr {
+				ft = ft.Elem()
+			}
+			if ft.Kind() == reflect.Struct {
+				indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+				rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo)
+				continue
+			}
+		}
+		// do not let fields with same name in embedded structs override field at higher level.
+		// this must be done after anonymous check, to allow anonymous field
+		// still include their child fields
+		if _, ok := fnameToHastag[f.Name]; ok {
+			continue
+		}
+		si := parseStructFieldInfo(f.Name, stag)
+		// si.ikind = int(f.Type.Kind())
+		if len(indexstack) == 0 {
+			si.i = int16(j)
+		} else {
+			si.i = -1
+			si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+		}
+
+		if siInfo != nil {
+			if siInfo.omitEmpty {
+				si.omitEmpty = true
+			}
+		}
+		*sfi = append(*sfi, si)
+		fnameToHastag[f.Name] = stag != ""
+	}
+}
+
+func panicToErr(err *error) {
+	if recoverPanicToErr {
+		if x := recover(); x != nil {
+			//debug.PrintStack()
+			panicValToErr(x, err)
+		}
+	}
+}
+
+func doPanic(tag string, format string, params ...interface{}) {
+	params2 := make([]interface{}, len(params)+1)
+	params2[0] = tag
+	copy(params2[1:], params)
+	panic(fmt.Errorf("%s: "+format, params2...))
+}
+
+func checkOverflowFloat32(f float64, doCheck bool) {
+	if !doCheck {
+		return
+	}
+	// check overflow (logic adapted from std pkg reflect/value.go OverflowFloat()
+	f2 := f
+	if f2 < 0 {
+		f2 = -f
+	}
+	if math.MaxFloat32 < f2 && f2 <= math.MaxFloat64 {
+		decErr("Overflow float32 value: %v", f2)
+	}
+}
+
+func checkOverflow(ui uint64, i int64, bitsize uint8) {
+	// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
+	if bitsize == 0 {
+		return
+	}
+	if i != 0 {
+		if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
+			decErr("Overflow int value: %v", i)
+		}
+	}
+	if ui != 0 {
+		if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
+			decErr("Overflow uint value: %v", ui)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/helper_internal.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+// All non-std package dependencies live in this file,
+// so porting to different environment is easy (just update functions).
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"reflect"
+)
+
+var (
+	raisePanicAfterRecover = false
+	debugging              = true
+)
+
+func panicValToErr(panicVal interface{}, err *error) {
+	switch xerr := panicVal.(type) {
+	case error:
+		*err = xerr
+	case string:
+		*err = errors.New(xerr)
+	default:
+		*err = fmt.Errorf("%v", panicVal)
+	}
+	if raisePanicAfterRecover {
+		panic(panicVal)
+	}
+	return
+}
+
+func isEmptyValueDeref(v reflect.Value, deref bool) bool {
+	switch v.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		return v.Len() == 0
+	case reflect.Bool:
+		return !v.Bool()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return v.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return v.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return v.Float() == 0
+	case reflect.Interface, reflect.Ptr:
+		if deref {
+			if v.IsNil() {
+				return true
+			}
+			return isEmptyValueDeref(v.Elem(), deref)
+		} else {
+			return v.IsNil()
+		}
+	case reflect.Struct:
+		// return true if all fields are empty. else return false.
+
+		// we cannot use equality check, because some fields may be maps/slices/etc
+		// and consequently the structs are not comparable.
+		// return v.Interface() == reflect.Zero(v.Type()).Interface()
+		for i, n := 0, v.NumField(); i < n; i++ {
+			if !isEmptyValueDeref(v.Field(i), deref) {
+				return false
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func isEmptyValue(v reflect.Value) bool {
+	return isEmptyValueDeref(v, true)
+}
+
+func debugf(format string, args ...interface{}) {
+	if debugging {
+		if len(format) == 0 || format[len(format)-1] != '\n' {
+			format = format + "\n"
+		}
+		fmt.Printf(format, args...)
+	}
+}
+
+func pruneSignExt(v []byte, pos bool) (n int) {
+	if len(v) < 2 {
+	} else if pos && v[0] == 0 {
+		for ; v[n] == 0 && n+1 < len(v) && (v[n+1]&(1<<7) == 0); n++ {
+		}
+	} else if !pos && v[0] == 0xff {
+		for ; v[n] == 0xff && n+1 < len(v) && (v[n+1]&(1<<7) != 0); n++ {
+		}
+	}
+	return
+}
+
+func implementsIntf(typ, iTyp reflect.Type) (success bool, indir int8) {
+	if typ == nil {
+		return
+	}
+	rt := typ
+	// The type might be a pointer and we need to keep
+	// dereferencing to the base type until we find an implementation.
+	for {
+		if rt.Implements(iTyp) {
+			return true, indir
+		}
+		if p := rt; p.Kind() == reflect.Ptr {
+			indir++
+			if indir >= math.MaxInt8 { // insane number of indirections
+				return false, 0
+			}
+			rt = p.Elem()
+			continue
+		}
+		break
+	}
+	// No luck yet, but if this is a base type (non-pointer), the pointer might satisfy.
+	if typ.Kind() != reflect.Ptr {
+		// Not a pointer, but does the pointer work?
+		if reflect.PtrTo(typ).Implements(iTyp) {
+			return true, -1
+		}
+	}
+	return false, 0
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/msgpack.go

@@ -0,0 +1,816 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+/*
+MSGPACK
+
+Msgpack-c implementation powers the c, c++, python, ruby, etc libraries.
+We need to maintain compatibility with it and how it encodes integer values
+without caring about the type.
+
+For compatibility with behaviour of msgpack-c reference implementation:
+  - Go intX (>0) and uintX
+       IS ENCODED AS
+    msgpack +ve fixnum, unsigned
+  - Go intX (<0)
+       IS ENCODED AS
+    msgpack -ve fixnum, signed
+
+*/
+package codec
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"net/rpc"
+)
+
+const (
+	mpPosFixNumMin byte = 0x00
+	mpPosFixNumMax      = 0x7f
+	mpFixMapMin         = 0x80
+	mpFixMapMax         = 0x8f
+	mpFixArrayMin       = 0x90
+	mpFixArrayMax       = 0x9f
+	mpFixStrMin         = 0xa0
+	mpFixStrMax         = 0xbf
+	mpNil               = 0xc0
+	_                   = 0xc1
+	mpFalse             = 0xc2
+	mpTrue              = 0xc3
+	mpFloat             = 0xca
+	mpDouble            = 0xcb
+	mpUint8             = 0xcc
+	mpUint16            = 0xcd
+	mpUint32            = 0xce
+	mpUint64            = 0xcf
+	mpInt8              = 0xd0
+	mpInt16             = 0xd1
+	mpInt32             = 0xd2
+	mpInt64             = 0xd3
+
+	// extensions below
+	mpBin8     = 0xc4
+	mpBin16    = 0xc5
+	mpBin32    = 0xc6
+	mpExt8     = 0xc7
+	mpExt16    = 0xc8
+	mpExt32    = 0xc9
+	mpFixExt1  = 0xd4
+	mpFixExt2  = 0xd5
+	mpFixExt4  = 0xd6
+	mpFixExt8  = 0xd7
+	mpFixExt16 = 0xd8
+
+	mpStr8  = 0xd9 // new
+	mpStr16 = 0xda
+	mpStr32 = 0xdb
+
+	mpArray16 = 0xdc
+	mpArray32 = 0xdd
+
+	mpMap16 = 0xde
+	mpMap32 = 0xdf
+
+	mpNegFixNumMin = 0xe0
+	mpNegFixNumMax = 0xff
+)
+
+// MsgpackSpecRpcMultiArgs is a special type which signifies to the MsgpackSpecRpcCodec
+// that the backend RPC service takes multiple arguments, which have been arranged
+// in sequence in the slice.
+//
+// The Codec then passes it AS-IS to the rpc service (without wrapping it in an
+// array of 1 element).
+type MsgpackSpecRpcMultiArgs []interface{}
+
+// A MsgpackContainer type specifies the different types of msgpackContainers.
+type msgpackContainerType struct {
+	fixCutoff                   int
+	bFixMin, b8, b16, b32       byte
+	hasFixMin, has8, has8Always bool
+}
+
+var (
+	msgpackContainerStr  = msgpackContainerType{32, mpFixStrMin, mpStr8, mpStr16, mpStr32, true, true, false}
+	msgpackContainerBin  = msgpackContainerType{0, 0, mpBin8, mpBin16, mpBin32, false, true, true}
+	msgpackContainerList = msgpackContainerType{16, mpFixArrayMin, 0, mpArray16, mpArray32, true, false, false}
+	msgpackContainerMap  = msgpackContainerType{16, mpFixMapMin, 0, mpMap16, mpMap32, true, false, false}
+)
+
+//---------------------------------------------
+
+type msgpackEncDriver struct {
+	w encWriter
+	h *MsgpackHandle
+}
+
+func (e *msgpackEncDriver) isBuiltinType(rt uintptr) bool {
+	//no builtin types. All encodings are based on kinds. Types supported as extensions.
+	return false
+}
+
+func (e *msgpackEncDriver) encodeBuiltin(rt uintptr, v interface{}) {}
+
+func (e *msgpackEncDriver) encodeNil() {
+	e.w.writen1(mpNil)
+}
+
+func (e *msgpackEncDriver) encodeInt(i int64) {
+
+	switch {
+	case i >= 0:
+		e.encodeUint(uint64(i))
+	case i >= -32:
+		e.w.writen1(byte(i))
+	case i >= math.MinInt8:
+		e.w.writen2(mpInt8, byte(i))
+	case i >= math.MinInt16:
+		e.w.writen1(mpInt16)
+		e.w.writeUint16(uint16(i))
+	case i >= math.MinInt32:
+		e.w.writen1(mpInt32)
+		e.w.writeUint32(uint32(i))
+	default:
+		e.w.writen1(mpInt64)
+		e.w.writeUint64(uint64(i))
+	}
+}
+
+func (e *msgpackEncDriver) encodeUint(i uint64) {
+	switch {
+	case i <= math.MaxInt8:
+		e.w.writen1(byte(i))
+	case i <= math.MaxUint8:
+		e.w.writen2(mpUint8, byte(i))
+	case i <= math.MaxUint16:
+		e.w.writen1(mpUint16)
+		e.w.writeUint16(uint16(i))
+	case i <= math.MaxUint32:
+		e.w.writen1(mpUint32)
+		e.w.writeUint32(uint32(i))
+	default:
+		e.w.writen1(mpUint64)
+		e.w.writeUint64(uint64(i))
+	}
+}
+
+func (e *msgpackEncDriver) encodeBool(b bool) {
+	if b {
+		e.w.writen1(mpTrue)
+	} else {
+		e.w.writen1(mpFalse)
+	}
+}
+
+func (e *msgpackEncDriver) encodeFloat32(f float32) {
+	e.w.writen1(mpFloat)
+	e.w.writeUint32(math.Float32bits(f))
+}
+
+func (e *msgpackEncDriver) encodeFloat64(f float64) {
+	e.w.writen1(mpDouble)
+	e.w.writeUint64(math.Float64bits(f))
+}
+
+func (e *msgpackEncDriver) encodeExtPreamble(xtag byte, l int) {
+	switch {
+	case l == 1:
+		e.w.writen2(mpFixExt1, xtag)
+	case l == 2:
+		e.w.writen2(mpFixExt2, xtag)
+	case l == 4:
+		e.w.writen2(mpFixExt4, xtag)
+	case l == 8:
+		e.w.writen2(mpFixExt8, xtag)
+	case l == 16:
+		e.w.writen2(mpFixExt16, xtag)
+	case l < 256:
+		e.w.writen2(mpExt8, byte(l))
+		e.w.writen1(xtag)
+	case l < 65536:
+		e.w.writen1(mpExt16)
+		e.w.writeUint16(uint16(l))
+		e.w.writen1(xtag)
+	default:
+		e.w.writen1(mpExt32)
+		e.w.writeUint32(uint32(l))
+		e.w.writen1(xtag)
+	}
+}
+
+func (e *msgpackEncDriver) encodeArrayPreamble(length int) {
+	e.writeContainerLen(msgpackContainerList, length)
+}
+
+func (e *msgpackEncDriver) encodeMapPreamble(length int) {
+	e.writeContainerLen(msgpackContainerMap, length)
+}
+
+func (e *msgpackEncDriver) encodeString(c charEncoding, s string) {
+	if c == c_RAW && e.h.WriteExt {
+		e.writeContainerLen(msgpackContainerBin, len(s))
+	} else {
+		e.writeContainerLen(msgpackContainerStr, len(s))
+	}
+	if len(s) > 0 {
+		e.w.writestr(s)
+	}
+}
+
+func (e *msgpackEncDriver) encodeSymbol(v string) {
+	e.encodeString(c_UTF8, v)
+}
+
+func (e *msgpackEncDriver) encodeStringBytes(c charEncoding, bs []byte) {
+	if c == c_RAW && e.h.WriteExt {
+		e.writeContainerLen(msgpackContainerBin, len(bs))
+	} else {
+		e.writeContainerLen(msgpackContainerStr, len(bs))
+	}
+	if len(bs) > 0 {
+		e.w.writeb(bs)
+	}
+}
+
+func (e *msgpackEncDriver) writeContainerLen(ct msgpackContainerType, l int) {
+	switch {
+	case ct.hasFixMin && l < ct.fixCutoff:
+		e.w.writen1(ct.bFixMin | byte(l))
+	case ct.has8 && l < 256 && (ct.has8Always || e.h.WriteExt):
+		e.w.writen2(ct.b8, uint8(l))
+	case l < 65536:
+		e.w.writen1(ct.b16)
+		e.w.writeUint16(uint16(l))
+	default:
+		e.w.writen1(ct.b32)
+		e.w.writeUint32(uint32(l))
+	}
+}
+
+//---------------------------------------------
+
+type msgpackDecDriver struct {
+	r      decReader
+	h      *MsgpackHandle
+	bd     byte
+	bdRead bool
+	bdType valueType
+}
+
+func (d *msgpackDecDriver) isBuiltinType(rt uintptr) bool {
+	//no builtin types. All encodings are based on kinds. Types supported as extensions.
+	return false
+}
+
+func (d *msgpackDecDriver) decodeBuiltin(rt uintptr, v interface{}) {}
+
+// Note: This returns either a primitive (int, bool, etc) for non-containers,
+// or a containerType, or a specific type denoting nil or extension.
+// It is called when a nil interface{} is passed, leaving it up to the DecDriver
+// to introspect the stream and decide how best to decode.
+// It deciphers the value by looking at the stream first.
+func (d *msgpackDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+	d.initReadNext()
+	bd := d.bd
+
+	switch bd {
+	case mpNil:
+		vt = valueTypeNil
+		d.bdRead = false
+	case mpFalse:
+		vt = valueTypeBool
+		v = false
+	case mpTrue:
+		vt = valueTypeBool
+		v = true
+
+	case mpFloat:
+		vt = valueTypeFloat
+		v = float64(math.Float32frombits(d.r.readUint32()))
+	case mpDouble:
+		vt = valueTypeFloat
+		v = math.Float64frombits(d.r.readUint64())
+
+	case mpUint8:
+		vt = valueTypeUint
+		v = uint64(d.r.readn1())
+	case mpUint16:
+		vt = valueTypeUint
+		v = uint64(d.r.readUint16())
+	case mpUint32:
+		vt = valueTypeUint
+		v = uint64(d.r.readUint32())
+	case mpUint64:
+		vt = valueTypeUint
+		v = uint64(d.r.readUint64())
+
+	case mpInt8:
+		vt = valueTypeInt
+		v = int64(int8(d.r.readn1()))
+	case mpInt16:
+		vt = valueTypeInt
+		v = int64(int16(d.r.readUint16()))
+	case mpInt32:
+		vt = valueTypeInt
+		v = int64(int32(d.r.readUint32()))
+	case mpInt64:
+		vt = valueTypeInt
+		v = int64(int64(d.r.readUint64()))
+
+	default:
+		switch {
+		case bd >= mpPosFixNumMin && bd <= mpPosFixNumMax:
+			// positive fixnum (always signed)
+			vt = valueTypeInt
+			v = int64(int8(bd))
+		case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
+			// negative fixnum
+			vt = valueTypeInt
+			v = int64(int8(bd))
+		case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
+			if d.h.RawToString {
+				var rvm string
+				vt = valueTypeString
+				v = &rvm
+			} else {
+				var rvm = []byte{}
+				vt = valueTypeBytes
+				v = &rvm
+			}
+			decodeFurther = true
+		case bd == mpBin8, bd == mpBin16, bd == mpBin32:
+			var rvm = []byte{}
+			vt = valueTypeBytes
+			v = &rvm
+			decodeFurther = true
+		case bd == mpArray16, bd == mpArray32, bd >= mpFixArrayMin && bd <= mpFixArrayMax:
+			vt = valueTypeArray
+			decodeFurther = true
+		case bd == mpMap16, bd == mpMap32, bd >= mpFixMapMin && bd <= mpFixMapMax:
+			vt = valueTypeMap
+			decodeFurther = true
+		case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
+			clen := d.readExtLen()
+			var re RawExt
+			re.Tag = d.r.readn1()
+			re.Data = d.r.readn(clen)
+			v = &re
+			vt = valueTypeExt
+		default:
+			decErr("Nil-Deciphered DecodeValue: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
+		}
+	}
+	if !decodeFurther {
+		d.bdRead = false
+	}
+	return
+}
+
+// int can be decoded from msgpack type: intXXX or uintXXX
+func (d *msgpackDecDriver) decodeInt(bitsize uint8) (i int64) {
+	switch d.bd {
+	case mpUint8:
+		i = int64(uint64(d.r.readn1()))
+	case mpUint16:
+		i = int64(uint64(d.r.readUint16()))
+	case mpUint32:
+		i = int64(uint64(d.r.readUint32()))
+	case mpUint64:
+		i = int64(d.r.readUint64())
+	case mpInt8:
+		i = int64(int8(d.r.readn1()))
+	case mpInt16:
+		i = int64(int16(d.r.readUint16()))
+	case mpInt32:
+		i = int64(int32(d.r.readUint32()))
+	case mpInt64:
+		i = int64(d.r.readUint64())
+	default:
+		switch {
+		case d.bd >= mpPosFixNumMin && d.bd <= mpPosFixNumMax:
+			i = int64(int8(d.bd))
+		case d.bd >= mpNegFixNumMin && d.bd <= mpNegFixNumMax:
+			i = int64(int8(d.bd))
+		default:
+			decErr("Unhandled single-byte unsigned integer value: %s: %x", msgBadDesc, d.bd)
+		}
+	}
+	// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
+	if bitsize > 0 {
+		if trunc := (i << (64 - bitsize)) >> (64 - bitsize); i != trunc {
+			decErr("Overflow int value: %v", i)
+		}
+	}
+	d.bdRead = false
+	return
+}
+
+// uint can be decoded from msgpack type: intXXX or uintXXX
+func (d *msgpackDecDriver) decodeUint(bitsize uint8) (ui uint64) {
+	switch d.bd {
+	case mpUint8:
+		ui = uint64(d.r.readn1())
+	case mpUint16:
+		ui = uint64(d.r.readUint16())
+	case mpUint32:
+		ui = uint64(d.r.readUint32())
+	case mpUint64:
+		ui = d.r.readUint64()
+	case mpInt8:
+		if i := int64(int8(d.r.readn1())); i >= 0 {
+			ui = uint64(i)
+		} else {
+			decErr("Assigning negative signed value: %v, to unsigned type", i)
+		}
+	case mpInt16:
+		if i := int64(int16(d.r.readUint16())); i >= 0 {
+			ui = uint64(i)
+		} else {
+			decErr("Assigning negative signed value: %v, to unsigned type", i)
+		}
+	case mpInt32:
+		if i := int64(int32(d.r.readUint32())); i >= 0 {
+			ui = uint64(i)
+		} else {
+			decErr("Assigning negative signed value: %v, to unsigned type", i)
+		}
+	case mpInt64:
+		if i := int64(d.r.readUint64()); i >= 0 {
+			ui = uint64(i)
+		} else {
+			decErr("Assigning negative signed value: %v, to unsigned type", i)
+		}
+	default:
+		switch {
+		case d.bd >= mpPosFixNumMin && d.bd <= mpPosFixNumMax:
+			ui = uint64(d.bd)
+		case d.bd >= mpNegFixNumMin && d.bd <= mpNegFixNumMax:
+			decErr("Assigning negative signed value: %v, to unsigned type", int(d.bd))
+		default:
+			decErr("Unhandled single-byte unsigned integer value: %s: %x", msgBadDesc, d.bd)
+		}
+	}
+	// check overflow (logic adapted from std pkg reflect/value.go OverflowUint()
+	if bitsize > 0 {
+		if trunc := (ui << (64 - bitsize)) >> (64 - bitsize); ui != trunc {
+			decErr("Overflow uint value: %v", ui)
+		}
+	}
+	d.bdRead = false
+	return
+}
+
+// float can either be decoded from msgpack type: float, double or intX
+func (d *msgpackDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
+	switch d.bd {
+	case mpFloat:
+		f = float64(math.Float32frombits(d.r.readUint32()))
+	case mpDouble:
+		f = math.Float64frombits(d.r.readUint64())
+	default:
+		f = float64(d.decodeInt(0))
+	}
+	checkOverflowFloat32(f, chkOverflow32)
+	d.bdRead = false
+	return
+}
+
+// bool can be decoded from bool, fixnum 0 or 1.
+func (d *msgpackDecDriver) decodeBool() (b bool) {
+	switch d.bd {
+	case mpFalse, 0:
+		// b = false
+	case mpTrue, 1:
+		b = true
+	default:
+		decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *msgpackDecDriver) decodeString() (s string) {
+	clen := d.readContainerLen(msgpackContainerStr)
+	if clen > 0 {
+		s = string(d.r.readn(clen))
+	}
+	d.bdRead = false
+	return
+}
+
+// Callers must check if changed=true (to decide whether to replace the one they have)
+func (d *msgpackDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
+	// bytes can be decoded from msgpackContainerStr or msgpackContainerBin
+	var clen int
+	switch d.bd {
+	case mpBin8, mpBin16, mpBin32:
+		clen = d.readContainerLen(msgpackContainerBin)
+	default:
+		clen = d.readContainerLen(msgpackContainerStr)
+	}
+	// if clen < 0 {
+	// 	changed = true
+	// 	panic("length cannot be zero. this cannot be nil.")
+	// }
+	if clen > 0 {
+		// if no contents in stream, don't update the passed byteslice
+		if len(bs) != clen {
+			// Return changed=true if length of passed slice diff from length of bytes in stream
+			if len(bs) > clen {
+				bs = bs[:clen]
+			} else {
+				bs = make([]byte, clen)
+			}
+			bsOut = bs
+			changed = true
+		}
+		d.r.readb(bs)
+	}
+	d.bdRead = false
+	return
+}
+
+// Every top-level decode funcs (i.e. decodeValue, decode) must call this first.
+func (d *msgpackDecDriver) initReadNext() {
+	if d.bdRead {
+		return
+	}
+	d.bd = d.r.readn1()
+	d.bdRead = true
+	d.bdType = valueTypeUnset
+}
+
+func (d *msgpackDecDriver) currentEncodedType() valueType {
+	if d.bdType == valueTypeUnset {
+		bd := d.bd
+		switch bd {
+		case mpNil:
+			d.bdType = valueTypeNil
+		case mpFalse, mpTrue:
+			d.bdType = valueTypeBool
+		case mpFloat, mpDouble:
+			d.bdType = valueTypeFloat
+		case mpUint8, mpUint16, mpUint32, mpUint64:
+			d.bdType = valueTypeUint
+		case mpInt8, mpInt16, mpInt32, mpInt64:
+			d.bdType = valueTypeInt
+		default:
+			switch {
+			case bd >= mpPosFixNumMin && bd <= mpPosFixNumMax:
+				d.bdType = valueTypeInt
+			case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
+				d.bdType = valueTypeInt
+			case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
+				if d.h.RawToString {
+					d.bdType = valueTypeString
+				} else {
+					d.bdType = valueTypeBytes
+				}
+			case bd == mpBin8, bd == mpBin16, bd == mpBin32:
+				d.bdType = valueTypeBytes
+			case bd == mpArray16, bd == mpArray32, bd >= mpFixArrayMin && bd <= mpFixArrayMax:
+				d.bdType = valueTypeArray
+			case bd == mpMap16, bd == mpMap32, bd >= mpFixMapMin && bd <= mpFixMapMax:
+				d.bdType = valueTypeMap
+			case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
+				d.bdType = valueTypeExt
+			default:
+				decErr("currentEncodedType: Undeciphered descriptor: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
+			}
+		}
+	}
+	return d.bdType
+}
+
+func (d *msgpackDecDriver) tryDecodeAsNil() bool {
+	if d.bd == mpNil {
+		d.bdRead = false
+		return true
+	}
+	return false
+}
+
+func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int) {
+	bd := d.bd
+	switch {
+	case bd == mpNil:
+		clen = -1 // to represent nil
+	case bd == ct.b8:
+		clen = int(d.r.readn1())
+	case bd == ct.b16:
+		clen = int(d.r.readUint16())
+	case bd == ct.b32:
+		clen = int(d.r.readUint32())
+	case (ct.bFixMin & bd) == ct.bFixMin:
+		clen = int(ct.bFixMin ^ bd)
+	default:
+		decErr("readContainerLen: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *msgpackDecDriver) readMapLen() int {
+	return d.readContainerLen(msgpackContainerMap)
+}
+
+func (d *msgpackDecDriver) readArrayLen() int {
+	return d.readContainerLen(msgpackContainerList)
+}
+
+func (d *msgpackDecDriver) readExtLen() (clen int) {
+	switch d.bd {
+	case mpNil:
+		clen = -1 // to represent nil
+	case mpFixExt1:
+		clen = 1
+	case mpFixExt2:
+		clen = 2
+	case mpFixExt4:
+		clen = 4
+	case mpFixExt8:
+		clen = 8
+	case mpFixExt16:
+		clen = 16
+	case mpExt8:
+		clen = int(d.r.readn1())
+	case mpExt16:
+		clen = int(d.r.readUint16())
+	case mpExt32:
+		clen = int(d.r.readUint32())
+	default:
+		decErr("decoding ext bytes: found unexpected byte: %x", d.bd)
+	}
+	return
+}
+
+func (d *msgpackDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
+	xbd := d.bd
+	switch {
+	case xbd == mpBin8, xbd == mpBin16, xbd == mpBin32:
+		xbs, _ = d.decodeBytes(nil)
+	case xbd == mpStr8, xbd == mpStr16, xbd == mpStr32,
+		xbd >= mpFixStrMin && xbd <= mpFixStrMax:
+		xbs = []byte(d.decodeString())
+	default:
+		clen := d.readExtLen()
+		xtag = d.r.readn1()
+		if verifyTag && xtag != tag {
+			decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
+		}
+		xbs = d.r.readn(clen)
+	}
+	d.bdRead = false
+	return
+}
+
+//--------------------------------------------------
+
+//MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
+type MsgpackHandle struct {
+	BasicHandle
+
+	// RawToString controls how raw bytes are decoded into a nil interface{}.
+	RawToString bool
+	// WriteExt flag supports encoding configured extensions with extension tags.
+	// It also controls whether other elements of the new spec are encoded (ie Str8).
+	//
+	// With WriteExt=false, configured extensions are serialized as raw bytes
+	// and Str8 is not encoded.
+	//
+	// A stream can still be decoded into a typed value, provided an appropriate value
+	// is provided, but the type cannot be inferred from the stream. If no appropriate
+	// type is provided (e.g. decoding into a nil interface{}), you get back
+	// a []byte or string based on the setting of RawToString.
+	WriteExt bool
+}
+
+func (h *MsgpackHandle) newEncDriver(w encWriter) encDriver {
+	return &msgpackEncDriver{w: w, h: h}
+}
+
+func (h *MsgpackHandle) newDecDriver(r decReader) decDriver {
+	return &msgpackDecDriver{r: r, h: h}
+}
+
+func (h *MsgpackHandle) writeExt() bool {
+	return h.WriteExt
+}
+
+func (h *MsgpackHandle) getBasicHandle() *BasicHandle {
+	return &h.BasicHandle
+}
+
+//--------------------------------------------------
+
+type msgpackSpecRpcCodec struct {
+	rpcCodec
+}
+
+// /////////////// Spec RPC Codec ///////////////////
+func (c *msgpackSpecRpcCodec) WriteRequest(r *rpc.Request, body interface{}) error {
+	// WriteRequest can write to both a Go service, and other services that do
+	// not abide by the 1 argument rule of a Go service.
+	// We discriminate based on if the body is a MsgpackSpecRpcMultiArgs
+	var bodyArr []interface{}
+	if m, ok := body.(MsgpackSpecRpcMultiArgs); ok {
+		bodyArr = ([]interface{})(m)
+	} else {
+		bodyArr = []interface{}{body}
+	}
+	r2 := []interface{}{0, uint32(r.Seq), r.ServiceMethod, bodyArr}
+	return c.write(r2, nil, false, true)
+}
+
+func (c *msgpackSpecRpcCodec) WriteResponse(r *rpc.Response, body interface{}) error {
+	var moe interface{}
+	if r.Error != "" {
+		moe = r.Error
+	}
+	if moe != nil && body != nil {
+		body = nil
+	}
+	r2 := []interface{}{1, uint32(r.Seq), moe, body}
+	return c.write(r2, nil, false, true)
+}
+
+func (c *msgpackSpecRpcCodec) ReadResponseHeader(r *rpc.Response) error {
+	return c.parseCustomHeader(1, &r.Seq, &r.Error)
+}
+
+func (c *msgpackSpecRpcCodec) ReadRequestHeader(r *rpc.Request) error {
+	return c.parseCustomHeader(0, &r.Seq, &r.ServiceMethod)
+}
+
+func (c *msgpackSpecRpcCodec) ReadRequestBody(body interface{}) error {
+	if body == nil { // read and discard
+		return c.read(nil)
+	}
+	bodyArr := []interface{}{body}
+	return c.read(&bodyArr)
+}
+
+func (c *msgpackSpecRpcCodec) parseCustomHeader(expectTypeByte byte, msgid *uint64, methodOrError *string) (err error) {
+
+	if c.cls {
+		return io.EOF
+	}
+
+	// We read the response header by hand
+	// so that the body can be decoded on its own from the stream at a later time.
+
+	const fia byte = 0x94 //four item array descriptor value
+	// Not sure why the panic of EOF is swallowed above.
+	// if bs1 := c.dec.r.readn1(); bs1 != fia {
+	// 	err = fmt.Errorf("Unexpected value for array descriptor: Expecting %v. Received %v", fia, bs1)
+	// 	return
+	// }
+	var b byte
+	b, err = c.br.ReadByte()
+	if err != nil {
+		return
+	}
+	if b != fia {
+		err = fmt.Errorf("Unexpected value for array descriptor: Expecting %v. Received %v", fia, b)
+		return
+	}
+
+	if err = c.read(&b); err != nil {
+		return
+	}
+	if b != expectTypeByte {
+		err = fmt.Errorf("Unexpected byte descriptor in header. Expecting %v. Received %v", expectTypeByte, b)
+		return
+	}
+	if err = c.read(msgid); err != nil {
+		return
+	}
+	if err = c.read(methodOrError); err != nil {
+		return
+	}
+	return
+}
+
+//--------------------------------------------------
+
+// msgpackSpecRpc is the implementation of Rpc that uses custom communication protocol
+// as defined in the msgpack spec at https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
+type msgpackSpecRpc struct{}
+
+// MsgpackSpecRpc implements Rpc using the communication protocol defined in
+// the msgpack spec at https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md .
+// Its methods (ServerCodec and ClientCodec) return values that implement RpcCodecBuffered.
+var MsgpackSpecRpc msgpackSpecRpc
+
+func (x msgpackSpecRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec {
+	return &msgpackSpecRpcCodec{newRPCCodec(conn, h)}
+}
+
+func (x msgpackSpecRpc) ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec {
+	return &msgpackSpecRpcCodec{newRPCCodec(conn, h)}
+}
+
+var _ decDriver = (*msgpackDecDriver)(nil)
+var _ encDriver = (*msgpackEncDriver)(nil)

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/msgpack_test.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+# This will create golden files in a directory passed to it.
+# A Test calls this internally to create the golden files
+# So it can process them (so we don't have to checkin the files).
+
+import msgpack, msgpackrpc, sys, os, threading
+
+def get_test_data_list():
+    # get list with all primitive types, and a combo type
+    l0 = [ 
+        -8,
+         -1616,
+         -32323232,
+         -6464646464646464,
+         192,
+         1616,
+         32323232,
+         6464646464646464,
+         192,
+         -3232.0,
+         -6464646464.0,
+         3232.0,
+         6464646464.0,
+         False,
+         True,
+         None,
+         "someday",
+         "",
+         "bytestring",
+         1328176922000002000,
+         -2206187877999998000,
+         0,
+         -6795364578871345152
+         ]
+    l1 = [
+        { "true": True,
+          "false": False },
+        { "true": "True",
+          "false": False,
+          "uint16(1616)": 1616 },
+        { "list": [1616, 32323232, True, -3232.0, {"TRUE":True, "FALSE":False}, [True, False] ],
+          "int32":32323232, "bool": True, 
+          "LONG STRING": "123456789012345678901234567890123456789012345678901234567890",
+          "SHORT STRING": "1234567890" },	
+        { True: "true", 8: False, "false": 0 }
+        ]
+    
+    l = []
+    l.extend(l0)
+    l.append(l0)
+    l.extend(l1)
+    return l
+
+def build_test_data(destdir):
+    l = get_test_data_list()
+    for i in range(len(l)):
+        packer = msgpack.Packer()
+        serialized = packer.pack(l[i])
+        f = open(os.path.join(destdir, str(i) + '.golden'), 'wb')
+        f.write(serialized)
+        f.close()
+
+def doRpcServer(port, stopTimeSec):
+    class EchoHandler(object):
+        def Echo123(self, msg1, msg2, msg3):
+            return ("1:%s 2:%s 3:%s" % (msg1, msg2, msg3))
+        def EchoStruct(self, msg):
+            return ("%s" % msg)
+    
+    addr = msgpackrpc.Address('localhost', port)
+    server = msgpackrpc.Server(EchoHandler())
+    server.listen(addr)
+    # run thread to stop it after stopTimeSec seconds if > 0
+    if stopTimeSec > 0:
+        def myStopRpcServer():
+            server.stop()
+        t = threading.Timer(stopTimeSec, myStopRpcServer)
+        t.start()
+    server.start()
+
+def doRpcClientToPythonSvc(port):
+    address = msgpackrpc.Address('localhost', port)
+    client = msgpackrpc.Client(address, unpack_encoding='utf-8')
+    print client.call("Echo123", "A1", "B2", "C3")
+    print client.call("EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})
+   
+def doRpcClientToGoSvc(port):
+    # print ">>>> port: ", port, " <<<<<"
+    address = msgpackrpc.Address('localhost', port)
+    client = msgpackrpc.Client(address, unpack_encoding='utf-8')
+    print client.call("TestRpcInt.Echo123", ["A1", "B2", "C3"])
+    print client.call("TestRpcInt.EchoStruct", {"A" :"Aa", "B":"Bb", "C":"Cc"})
+
+def doMain(args):
+    if len(args) == 2 and args[0] == "testdata":
+        build_test_data(args[1])
+    elif len(args) == 3 and args[0] == "rpc-server":
+        doRpcServer(int(args[1]), int(args[2]))
+    elif len(args) == 2 and args[0] == "rpc-client-python-service":
+        doRpcClientToPythonSvc(int(args[1]))
+    elif len(args) == 2 and args[0] == "rpc-client-go-service":
+        doRpcClientToGoSvc(int(args[1]))
+    else:
+        print("Usage: msgpack_test.py " + 
+              "[testdata|rpc-server|rpc-client-python-service|rpc-client-go-service] ...")
+    
+if __name__ == "__main__":
+    doMain(sys.argv[1:])
+

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/rpc.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+import (
+	"bufio"
+	"io"
+	"net/rpc"
+	"sync"
+)
+
+// Rpc provides a rpc Server or Client Codec for rpc communication.
+type Rpc interface {
+	ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec
+	ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec
+}
+
+// RpcCodecBuffered allows access to the underlying bufio.Reader/Writer
+// used by the rpc connection. It accomodates use-cases where the connection
+// should be used by rpc and non-rpc functions, e.g. streaming a file after
+// sending an rpc response.
+type RpcCodecBuffered interface {
+	BufferedReader() *bufio.Reader
+	BufferedWriter() *bufio.Writer
+}
+
+// -------------------------------------
+
+// rpcCodec defines the struct members and common methods.
+type rpcCodec struct {
+	rwc io.ReadWriteCloser
+	dec *Decoder
+	enc *Encoder
+	bw  *bufio.Writer
+	br  *bufio.Reader
+	mu  sync.Mutex
+	cls bool
+}
+
+func newRPCCodec(conn io.ReadWriteCloser, h Handle) rpcCodec {
+	bw := bufio.NewWriter(conn)
+	br := bufio.NewReader(conn)
+	return rpcCodec{
+		rwc: conn,
+		bw:  bw,
+		br:  br,
+		enc: NewEncoder(bw, h),
+		dec: NewDecoder(br, h),
+	}
+}
+
+func (c *rpcCodec) BufferedReader() *bufio.Reader {
+	return c.br
+}
+
+func (c *rpcCodec) BufferedWriter() *bufio.Writer {
+	return c.bw
+}
+
+func (c *rpcCodec) write(obj1, obj2 interface{}, writeObj2, doFlush bool) (err error) {
+	if c.cls {
+		return io.EOF
+	}
+	if err = c.enc.Encode(obj1); err != nil {
+		return
+	}
+	if writeObj2 {
+		if err = c.enc.Encode(obj2); err != nil {
+			return
+		}
+	}
+	if doFlush && c.bw != nil {
+		return c.bw.Flush()
+	}
+	return
+}
+
+func (c *rpcCodec) read(obj interface{}) (err error) {
+	if c.cls {
+		return io.EOF
+	}
+	//If nil is passed in, we should still attempt to read content to nowhere.
+	if obj == nil {
+		var obj2 interface{}
+		return c.dec.Decode(&obj2)
+	}
+	return c.dec.Decode(obj)
+}
+
+func (c *rpcCodec) Close() error {
+	if c.cls {
+		return io.EOF
+	}
+	c.cls = true
+	return c.rwc.Close()
+}
+
+func (c *rpcCodec) ReadResponseBody(body interface{}) error {
+	return c.read(body)
+}
+
+// -------------------------------------
+
+type goRpcCodec struct {
+	rpcCodec
+}
+
+func (c *goRpcCodec) WriteRequest(r *rpc.Request, body interface{}) error {
+	// Must protect for concurrent access as per API
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.write(r, body, true, true)
+}
+
+func (c *goRpcCodec) WriteResponse(r *rpc.Response, body interface{}) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.write(r, body, true, true)
+}
+
+func (c *goRpcCodec) ReadResponseHeader(r *rpc.Response) error {
+	return c.read(r)
+}
+
+func (c *goRpcCodec) ReadRequestHeader(r *rpc.Request) error {
+	return c.read(r)
+}
+
+func (c *goRpcCodec) ReadRequestBody(body interface{}) error {
+	return c.read(body)
+}
+
+// -------------------------------------
+
+// goRpc is the implementation of Rpc that uses the communication protocol
+// as defined in net/rpc package.
+type goRpc struct{}
+
+// GoRpc implements Rpc using the communication protocol defined in net/rpc package.
+// Its methods (ServerCodec and ClientCodec) return values that implement RpcCodecBuffered.
+var GoRpc goRpc
+
+func (x goRpc) ServerCodec(conn io.ReadWriteCloser, h Handle) rpc.ServerCodec {
+	return &goRpcCodec{newRPCCodec(conn, h)}
+}
+
+func (x goRpc) ClientCodec(conn io.ReadWriteCloser, h Handle) rpc.ClientCodec {
+	return &goRpcCodec{newRPCCodec(conn, h)}
+}
+
+var _ RpcCodecBuffered = (*rpcCodec)(nil) // ensure *rpcCodec implements RpcCodecBuffered

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/simple.go

@@ -0,0 +1,461 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+import "math"
+
+const (
+	_               uint8 = iota
+	simpleVdNil           = 1
+	simpleVdFalse         = 2
+	simpleVdTrue          = 3
+	simpleVdFloat32       = 4
+	simpleVdFloat64       = 5
+
+	// each lasts for 4 (ie n, n+1, n+2, n+3)
+	simpleVdPosInt = 8
+	simpleVdNegInt = 12
+
+	// containers: each lasts for 4 (ie n, n+1, n+2, ... n+7)
+	simpleVdString    = 216
+	simpleVdByteArray = 224
+	simpleVdArray     = 232
+	simpleVdMap       = 240
+	simpleVdExt       = 248
+)
+
+type simpleEncDriver struct {
+	h *SimpleHandle
+	w encWriter
+	//b [8]byte
+}
+
+func (e *simpleEncDriver) isBuiltinType(rt uintptr) bool {
+	return false
+}
+
+func (e *simpleEncDriver) encodeBuiltin(rt uintptr, v interface{}) {
+}
+
+func (e *simpleEncDriver) encodeNil() {
+	e.w.writen1(simpleVdNil)
+}
+
+func (e *simpleEncDriver) encodeBool(b bool) {
+	if b {
+		e.w.writen1(simpleVdTrue)
+	} else {
+		e.w.writen1(simpleVdFalse)
+	}
+}
+
+func (e *simpleEncDriver) encodeFloat32(f float32) {
+	e.w.writen1(simpleVdFloat32)
+	e.w.writeUint32(math.Float32bits(f))
+}
+
+func (e *simpleEncDriver) encodeFloat64(f float64) {
+	e.w.writen1(simpleVdFloat64)
+	e.w.writeUint64(math.Float64bits(f))
+}
+
+func (e *simpleEncDriver) encodeInt(v int64) {
+	if v < 0 {
+		e.encUint(uint64(-v), simpleVdNegInt)
+	} else {
+		e.encUint(uint64(v), simpleVdPosInt)
+	}
+}
+
+func (e *simpleEncDriver) encodeUint(v uint64) {
+	e.encUint(v, simpleVdPosInt)
+}
+
+func (e *simpleEncDriver) encUint(v uint64, bd uint8) {
+	switch {
+	case v <= math.MaxUint8:
+		e.w.writen2(bd, uint8(v))
+	case v <= math.MaxUint16:
+		e.w.writen1(bd + 1)
+		e.w.writeUint16(uint16(v))
+	case v <= math.MaxUint32:
+		e.w.writen1(bd + 2)
+		e.w.writeUint32(uint32(v))
+	case v <= math.MaxUint64:
+		e.w.writen1(bd + 3)
+		e.w.writeUint64(v)
+	}
+}
+
+func (e *simpleEncDriver) encLen(bd byte, length int) {
+	switch {
+	case length == 0:
+		e.w.writen1(bd)
+	case length <= math.MaxUint8:
+		e.w.writen1(bd + 1)
+		e.w.writen1(uint8(length))
+	case length <= math.MaxUint16:
+		e.w.writen1(bd + 2)
+		e.w.writeUint16(uint16(length))
+	case int64(length) <= math.MaxUint32:
+		e.w.writen1(bd + 3)
+		e.w.writeUint32(uint32(length))
+	default:
+		e.w.writen1(bd + 4)
+		e.w.writeUint64(uint64(length))
+	}
+}
+
+func (e *simpleEncDriver) encodeExtPreamble(xtag byte, length int) {
+	e.encLen(simpleVdExt, length)
+	e.w.writen1(xtag)
+}
+
+func (e *simpleEncDriver) encodeArrayPreamble(length int) {
+	e.encLen(simpleVdArray, length)
+}
+
+func (e *simpleEncDriver) encodeMapPreamble(length int) {
+	e.encLen(simpleVdMap, length)
+}
+
+func (e *simpleEncDriver) encodeString(c charEncoding, v string) {
+	e.encLen(simpleVdString, len(v))
+	e.w.writestr(v)
+}
+
+func (e *simpleEncDriver) encodeSymbol(v string) {
+	e.encodeString(c_UTF8, v)
+}
+
+func (e *simpleEncDriver) encodeStringBytes(c charEncoding, v []byte) {
+	e.encLen(simpleVdByteArray, len(v))
+	e.w.writeb(v)
+}
+
+//------------------------------------
+
+type simpleDecDriver struct {
+	h      *SimpleHandle
+	r      decReader
+	bdRead bool
+	bdType valueType
+	bd     byte
+	//b      [8]byte
+}
+
+func (d *simpleDecDriver) initReadNext() {
+	if d.bdRead {
+		return
+	}
+	d.bd = d.r.readn1()
+	d.bdRead = true
+	d.bdType = valueTypeUnset
+}
+
+func (d *simpleDecDriver) currentEncodedType() valueType {
+	if d.bdType == valueTypeUnset {
+		switch d.bd {
+		case simpleVdNil:
+			d.bdType = valueTypeNil
+		case simpleVdTrue, simpleVdFalse:
+			d.bdType = valueTypeBool
+		case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3:
+			d.bdType = valueTypeUint
+		case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3:
+			d.bdType = valueTypeInt
+		case simpleVdFloat32, simpleVdFloat64:
+			d.bdType = valueTypeFloat
+		case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4:
+			d.bdType = valueTypeString
+		case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
+			d.bdType = valueTypeBytes
+		case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
+			d.bdType = valueTypeExt
+		case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4:
+			d.bdType = valueTypeArray
+		case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4:
+			d.bdType = valueTypeMap
+		default:
+			decErr("currentEncodedType: Unrecognized d.vd: 0x%x", d.bd)
+		}
+	}
+	return d.bdType
+}
+
+func (d *simpleDecDriver) tryDecodeAsNil() bool {
+	if d.bd == simpleVdNil {
+		d.bdRead = false
+		return true
+	}
+	return false
+}
+
+func (d *simpleDecDriver) isBuiltinType(rt uintptr) bool {
+	return false
+}
+
+func (d *simpleDecDriver) decodeBuiltin(rt uintptr, v interface{}) {
+}
+
+func (d *simpleDecDriver) decIntAny() (ui uint64, i int64, neg bool) {
+	switch d.bd {
+	case simpleVdPosInt:
+		ui = uint64(d.r.readn1())
+		i = int64(ui)
+	case simpleVdPosInt + 1:
+		ui = uint64(d.r.readUint16())
+		i = int64(ui)
+	case simpleVdPosInt + 2:
+		ui = uint64(d.r.readUint32())
+		i = int64(ui)
+	case simpleVdPosInt + 3:
+		ui = uint64(d.r.readUint64())
+		i = int64(ui)
+	case simpleVdNegInt:
+		ui = uint64(d.r.readn1())
+		i = -(int64(ui))
+		neg = true
+	case simpleVdNegInt + 1:
+		ui = uint64(d.r.readUint16())
+		i = -(int64(ui))
+		neg = true
+	case simpleVdNegInt + 2:
+		ui = uint64(d.r.readUint32())
+		i = -(int64(ui))
+		neg = true
+	case simpleVdNegInt + 3:
+		ui = uint64(d.r.readUint64())
+		i = -(int64(ui))
+		neg = true
+	default:
+		decErr("decIntAny: Integer only valid from pos/neg integer1..8. Invalid descriptor: %v", d.bd)
+	}
+	// don't do this check, because callers may only want the unsigned value.
+	// if ui > math.MaxInt64 {
+	// 	decErr("decIntAny: Integer out of range for signed int64: %v", ui)
+	// }
+	return
+}
+
+func (d *simpleDecDriver) decodeInt(bitsize uint8) (i int64) {
+	_, i, _ = d.decIntAny()
+	checkOverflow(0, i, bitsize)
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) decodeUint(bitsize uint8) (ui uint64) {
+	ui, i, neg := d.decIntAny()
+	if neg {
+		decErr("Assigning negative signed value: %v, to unsigned type", i)
+	}
+	checkOverflow(ui, 0, bitsize)
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) decodeFloat(chkOverflow32 bool) (f float64) {
+	switch d.bd {
+	case simpleVdFloat32:
+		f = float64(math.Float32frombits(d.r.readUint32()))
+	case simpleVdFloat64:
+		f = math.Float64frombits(d.r.readUint64())
+	default:
+		if d.bd >= simpleVdPosInt && d.bd <= simpleVdNegInt+3 {
+			_, i, _ := d.decIntAny()
+			f = float64(i)
+		} else {
+			decErr("Float only valid from float32/64: Invalid descriptor: %v", d.bd)
+		}
+	}
+	checkOverflowFloat32(f, chkOverflow32)
+	d.bdRead = false
+	return
+}
+
+// bool can be decoded from bool only (single byte).
+func (d *simpleDecDriver) decodeBool() (b bool) {
+	switch d.bd {
+	case simpleVdTrue:
+		b = true
+	case simpleVdFalse:
+	default:
+		decErr("Invalid single-byte value for bool: %s: %x", msgBadDesc, d.bd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) readMapLen() (length int) {
+	d.bdRead = false
+	return d.decLen()
+}
+
+func (d *simpleDecDriver) readArrayLen() (length int) {
+	d.bdRead = false
+	return d.decLen()
+}
+
+func (d *simpleDecDriver) decLen() int {
+	switch d.bd % 8 {
+	case 0:
+		return 0
+	case 1:
+		return int(d.r.readn1())
+	case 2:
+		return int(d.r.readUint16())
+	case 3:
+		ui := uint64(d.r.readUint32())
+		checkOverflow(ui, 0, intBitsize)
+		return int(ui)
+	case 4:
+		ui := d.r.readUint64()
+		checkOverflow(ui, 0, intBitsize)
+		return int(ui)
+	}
+	decErr("decLen: Cannot read length: bd%8 must be in range 0..4. Got: %d", d.bd%8)
+	return -1
+}
+
+func (d *simpleDecDriver) decodeString() (s string) {
+	s = string(d.r.readn(d.decLen()))
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
+	if clen := d.decLen(); clen > 0 {
+		// if no contents in stream, don't update the passed byteslice
+		if len(bs) != clen {
+			if len(bs) > clen {
+				bs = bs[:clen]
+			} else {
+				bs = make([]byte, clen)
+			}
+			bsOut = bs
+			changed = true
+		}
+		d.r.readb(bs)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
+	switch d.bd {
+	case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
+		l := d.decLen()
+		xtag = d.r.readn1()
+		if verifyTag && xtag != tag {
+			decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
+		}
+		xbs = d.r.readn(l)
+	case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
+		xbs, _ = d.decodeBytes(nil)
+	default:
+		decErr("Invalid d.vd for extensions (Expecting extensions or byte array). Got: 0x%x", d.bd)
+	}
+	d.bdRead = false
+	return
+}
+
+func (d *simpleDecDriver) decodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+	d.initReadNext()
+
+	switch d.bd {
+	case simpleVdNil:
+		vt = valueTypeNil
+	case simpleVdFalse:
+		vt = valueTypeBool
+		v = false
+	case simpleVdTrue:
+		vt = valueTypeBool
+		v = true
+	case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3:
+		vt = valueTypeUint
+		ui, _, _ := d.decIntAny()
+		v = ui
+	case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3:
+		vt = valueTypeInt
+		_, i, _ := d.decIntAny()
+		v = i
+	case simpleVdFloat32:
+		vt = valueTypeFloat
+		v = d.decodeFloat(true)
+	case simpleVdFloat64:
+		vt = valueTypeFloat
+		v = d.decodeFloat(false)
+	case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4:
+		vt = valueTypeString
+		v = d.decodeString()
+	case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
+		vt = valueTypeBytes
+		v, _ = d.decodeBytes(nil)
+	case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
+		vt = valueTypeExt
+		l := d.decLen()
+		var re RawExt
+		re.Tag = d.r.readn1()
+		re.Data = d.r.readn(l)
+		v = &re
+		vt = valueTypeExt
+	case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4:
+		vt = valueTypeArray
+		decodeFurther = true
+	case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4:
+		vt = valueTypeMap
+		decodeFurther = true
+	default:
+		decErr("decodeNaked: Unrecognized d.vd: 0x%x", d.bd)
+	}
+
+	if !decodeFurther {
+		d.bdRead = false
+	}
+	return
+}
+
+//------------------------------------
+
+// SimpleHandle is a Handle for a very simple encoding format.
+//
+// simple is a simplistic codec similar to binc, but not as compact.
+//   - Encoding of a value is always preceeded by the descriptor byte (bd)
+//   - True, false, nil are encoded fully in 1 byte (the descriptor)
+//   - Integers (intXXX, uintXXX) are encoded in 1, 2, 4 or 8 bytes (plus a descriptor byte).
+//     There are positive (uintXXX and intXXX >= 0) and negative (intXXX < 0) integers.
+//   - Floats are encoded in 4 or 8 bytes (plus a descriptor byte)
+//   - Lenght of containers (strings, bytes, array, map, extensions)
+//     are encoded in 0, 1, 2, 4 or 8 bytes.
+//     Zero-length containers have no length encoded.
+//     For others, the number of bytes is given by pow(2, bd%3)
+//   - maps are encoded as [bd] [length] [[key][value]]...
+//   - arrays are encoded as [bd] [length] [value]...
+//   - extensions are encoded as [bd] [length] [tag] [byte]...
+//   - strings/bytearrays are encoded as [bd] [length] [byte]...
+//
+// The full spec will be published soon.
+type SimpleHandle struct {
+	BasicHandle
+}
+
+func (h *SimpleHandle) newEncDriver(w encWriter) encDriver {
+	return &simpleEncDriver{w: w, h: h}
+}
+
+func (h *SimpleHandle) newDecDriver(r decReader) decDriver {
+	return &simpleDecDriver{r: r, h: h}
+}
+
+func (_ *SimpleHandle) writeExt() bool {
+	return true
+}
+
+func (h *SimpleHandle) getBasicHandle() *BasicHandle {
+	return &h.BasicHandle
+}
+
+var _ decDriver = (*simpleDecDriver)(nil)
+var _ encDriver = (*simpleEncDriver)(nil)

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/time.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+import (
+	"time"
+)
+
+var (
+	timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
+)
+
+// EncodeTime encodes a time.Time as a []byte, including
+// information on the instant in time and UTC offset.
+//
+// Format Description
+//
+//   A timestamp is composed of 3 components:
+//
+//   - secs: signed integer representing seconds since unix epoch
+//   - nsces: unsigned integer representing fractional seconds as a
+//     nanosecond offset within secs, in the range 0 <= nsecs < 1e9
+//   - tz: signed integer representing timezone offset in minutes east of UTC,
+//     and a dst (daylight savings time) flag
+//
+//   When encoding a timestamp, the first byte is the descriptor, which
+//   defines which components are encoded and how many bytes are used to
+//   encode secs and nsecs components. *If secs/nsecs is 0 or tz is UTC, it
+//   is not encoded in the byte array explicitly*.
+//
+//       Descriptor 8 bits are of the form `A B C DDD EE`:
+//           A:   Is secs component encoded? 1 = true
+//           B:   Is nsecs component encoded? 1 = true
+//           C:   Is tz component encoded? 1 = true
+//           DDD: Number of extra bytes for secs (range 0-7).
+//                If A = 1, secs encoded in DDD+1 bytes.
+//                    If A = 0, secs is not encoded, and is assumed to be 0.
+//                    If A = 1, then we need at least 1 byte to encode secs.
+//                    DDD says the number of extra bytes beyond that 1.
+//                    E.g. if DDD=0, then secs is represented in 1 byte.
+//                         if DDD=2, then secs is represented in 3 bytes.
+//           EE:  Number of extra bytes for nsecs (range 0-3).
+//                If B = 1, nsecs encoded in EE+1 bytes (similar to secs/DDD above)
+//
+//   Following the descriptor bytes, subsequent bytes are:
+//
+//       secs component encoded in `DDD + 1` bytes (if A == 1)
+//       nsecs component encoded in `EE + 1` bytes (if B == 1)
+//       tz component encoded in 2 bytes (if C == 1)
+//
+//   secs and nsecs components are integers encoded in a BigEndian
+//   2-complement encoding format.
+//
+//   tz component is encoded as 2 bytes (16 bits). Most significant bit 15 to
+//   Least significant bit 0 are described below:
+//
+//       Timezone offset has a range of -12:00 to +14:00 (ie -720 to +840 minutes).
+//       Bit 15 = have\_dst: set to 1 if we set the dst flag.
+//       Bit 14 = dst\_on: set to 1 if dst is in effect at the time, or 0 if not.
+//       Bits 13..0 = timezone offset in minutes. It is a signed integer in Big Endian format.
+//
+func encodeTime(t time.Time) []byte {
+	//t := rv.Interface().(time.Time)
+	tsecs, tnsecs := t.Unix(), t.Nanosecond()
+	var (
+		bd   byte
+		btmp [8]byte
+		bs   [16]byte
+		i    int = 1
+	)
+	l := t.Location()
+	if l == time.UTC {
+		l = nil
+	}
+	if tsecs != 0 {
+		bd = bd | 0x80
+		bigen.PutUint64(btmp[:], uint64(tsecs))
+		f := pruneSignExt(btmp[:], tsecs >= 0)
+		bd = bd | (byte(7-f) << 2)
+		copy(bs[i:], btmp[f:])
+		i = i + (8 - f)
+	}
+	if tnsecs != 0 {
+		bd = bd | 0x40
+		bigen.PutUint32(btmp[:4], uint32(tnsecs))
+		f := pruneSignExt(btmp[:4], true)
+		bd = bd | byte(3-f)
+		copy(bs[i:], btmp[f:4])
+		i = i + (4 - f)
+	}
+	if l != nil {
+		bd = bd | 0x20
+		// Note that Go Libs do not give access to dst flag.
+		_, zoneOffset := t.Zone()
+		//zoneName, zoneOffset := t.Zone()
+		zoneOffset /= 60
+		z := uint16(zoneOffset)
+		bigen.PutUint16(btmp[:2], z)
+		// clear dst flags
+		bs[i] = btmp[0] & 0x3f
+		bs[i+1] = btmp[1]
+		i = i + 2
+	}
+	bs[0] = bd
+	return bs[0:i]
+}
+
+// DecodeTime decodes a []byte into a time.Time.
+func decodeTime(bs []byte) (tt time.Time, err error) {
+	bd := bs[0]
+	var (
+		tsec  int64
+		tnsec uint32
+		tz    uint16
+		i     byte = 1
+		i2    byte
+		n     byte
+	)
+	if bd&(1<<7) != 0 {
+		var btmp [8]byte
+		n = ((bd >> 2) & 0x7) + 1
+		i2 = i + n
+		copy(btmp[8-n:], bs[i:i2])
+		//if first bit of bs[i] is set, then fill btmp[0..8-n] with 0xff (ie sign extend it)
+		if bs[i]&(1<<7) != 0 {
+			copy(btmp[0:8-n], bsAll0xff)
+			//for j,k := byte(0), 8-n; j < k; j++ {	btmp[j] = 0xff }
+		}
+		i = i2
+		tsec = int64(bigen.Uint64(btmp[:]))
+	}
+	if bd&(1<<6) != 0 {
+		var btmp [4]byte
+		n = (bd & 0x3) + 1
+		i2 = i + n
+		copy(btmp[4-n:], bs[i:i2])
+		i = i2
+		tnsec = bigen.Uint32(btmp[:])
+	}
+	if bd&(1<<5) == 0 {
+		tt = time.Unix(tsec, int64(tnsec)).UTC()
+		return
+	}
+	// In stdlib time.Parse, when a date is parsed without a zone name, it uses "" as zone name.
+	// However, we need name here, so it can be shown when time is printed.
+	// Zone name is in form: UTC-08:00.
+	// Note that Go Libs do not give access to dst flag, so we ignore dst bits
+
+	i2 = i + 2
+	tz = bigen.Uint16(bs[i:i2])
+	i = i2
+	// sign extend sign bit into top 2 MSB (which were dst bits):
+	if tz&(1<<13) == 0 { // positive
+		tz = tz & 0x3fff //clear 2 MSBs: dst bits
+	} else { // negative
+		tz = tz | 0xc000 //set 2 MSBs: dst bits
+		//tzname[3] = '-' (TODO: verify. this works here)
+	}
+	tzint := int16(tz)
+	if tzint == 0 {
+		tt = time.Unix(tsec, int64(tnsec)).UTC()
+	} else {
+		// For Go Time, do not use a descriptive timezone.
+		// It's unnecessary, and makes it harder to do a reflect.DeepEqual.
+		// The Offset already tells what the offset should be, if not on UTC and unknown zone name.
+		// var zoneName = timeLocUTCName(tzint)
+		tt = time.Unix(tsec, int64(tnsec)).In(time.FixedZone("", int(tzint)*60))
+	}
+	return
+}
+
+func timeLocUTCName(tzint int16) string {
+	if tzint == 0 {
+		return "UTC"
+	}
+	var tzname = []byte("UTC+00:00")
+	//tzname := fmt.Sprintf("UTC%s%02d:%02d", tzsign, tz/60, tz%60) //perf issue using Sprintf. inline below.
+	//tzhr, tzmin := tz/60, tz%60 //faster if u convert to int first
+	var tzhr, tzmin int16
+	if tzint < 0 {
+		tzname[3] = '-' // (TODO: verify. this works here)
+		tzhr, tzmin = -tzint/60, (-tzint)%60
+	} else {
+		tzhr, tzmin = tzint/60, tzint%60
+	}
+	tzname[4] = timeDigits[tzhr/10]
+	tzname[5] = timeDigits[tzhr%10]
+	tzname[7] = timeDigits[tzmin/10]
+	tzname[8] = timeDigits[tzmin%10]
+	return string(tzname)
+	//return time.FixedZone(string(tzname), int(tzint)*60)
+}

Godeps/_workspace/src/github.com/hashicorp/go-msgpack/codec/z_helper_test.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2012, 2013 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a BSD-style license found in the LICENSE file.
+
+package codec
+
+// All non-std package dependencies related to testing live in this file,
+// so porting to different environment is easy (just update functions).
+//
+// Also, this file is called z_helper_test, to give a "hint" to compiler
+// that its init() function should be called last. (not guaranteed by spec)
+
+import (
+	"errors"
+	"reflect"
+	"flag"
+	"testing"
+)
+
+var (
+	testLogToT    = true
+	failNowOnFail = true
+)
+
+func init() {
+	testInitFlags()
+	benchInitFlags()
+	flag.Parse()
+	testInit()
+	benchInit()
+}
+
+func checkErrT(t *testing.T, err error) {
+	if err != nil {
+		logT(t, err.Error())
+		failT(t)
+	}
+}
+
+func checkEqualT(t *testing.T, v1 interface{}, v2 interface{}, desc string) (err error) {
+	if err = deepEqual(v1, v2); err != nil {
+		logT(t, "Not Equal: %s: %v. v1: %v, v2: %v", desc, err, v1, v2)
+		failT(t)
+	}
+	return
+}
+
+func logT(x interface{}, format string, args ...interface{}) {
+	if t, ok := x.(*testing.T); ok && t != nil && testLogToT {
+		t.Logf(format, args...)
+	} else if b, ok := x.(*testing.B); ok && b != nil && testLogToT {
+		b.Logf(format, args...)
+	} else {
+		debugf(format, args...)
+	}
+}
+
+func failT(t *testing.T) {
+	if failNowOnFail {
+		t.FailNow()
+	} else {
+		t.Fail()
+	}
+}
+
+func deepEqual(v1, v2 interface{}) (err error) {
+	if !reflect.DeepEqual(v1, v2) {
+		err = errors.New("Not Match")
+	}
+	return
+}
+
+func approxDataSize(rv reflect.Value) (sum int) {
+	switch rk := rv.Kind(); rk {
+	case reflect.Invalid:
+	case reflect.Ptr, reflect.Interface:
+		sum += int(rv.Type().Size())
+		sum += approxDataSize(rv.Elem())
+	case reflect.Slice:
+		sum += int(rv.Type().Size())
+		for j := 0; j < rv.Len(); j++ {
+			sum += approxDataSize(rv.Index(j))
+		}
+	case reflect.String:
+		sum += int(rv.Type().Size())
+		sum += rv.Len()
+	case reflect.Map:
+		sum += int(rv.Type().Size())
+		for _, mk := range rv.MapKeys() {
+			sum += approxDataSize(mk)
+			sum += approxDataSize(rv.MapIndex(mk))
+		}
+	case reflect.Struct:
+		//struct size already includes the full data size.
+		//sum += int(rv.Type().Size())
+		for j := 0; j < rv.NumField(); j++ {
+			sum += approxDataSize(rv.Field(j))
+		}
+	default:
+		//pure value types
+		sum += int(rv.Type().Size())
+	}
+	return
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/.gitignore

@@ -0,0 +1,25 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+.vagrant/
+

Godeps/_workspace/src/github.com/hashicorp/memberlist/LICENSE

@@ -0,0 +1,354 @@
+Mozilla Public License, version 2.0
+
+1. Definitions
+
+1.1. “Contributor”
+
+     means each individual or legal entity that creates, contributes to the
+     creation of, or owns Covered Software.
+
+1.2. “Contributor Version”
+
+     means the combination of the Contributions of others (if any) used by a
+     Contributor and that particular Contributor’s Contribution.
+
+1.3. “Contribution”
+
+     means Covered Software of a particular Contributor.
+
+1.4. “Covered Software”
+
+     means Source Code Form to which the initial Contributor has attached the
+     notice in Exhibit A, the Executable Form of such Source Code Form, and
+     Modifications of such Source Code Form, in each case including portions
+     thereof.
+
+1.5. “Incompatible With Secondary Licenses”
+     means
+
+     a. that the initial Contributor has attached the notice described in
+        Exhibit B to the Covered Software; or
+
+     b. that the Covered Software was made available under the terms of version
+        1.1 or earlier of the License, but not also under the terms of a
+        Secondary License.
+
+1.6. “Executable Form”
+
+     means any form of the work other than Source Code Form.
+
+1.7. “Larger Work”
+
+     means a work that combines Covered Software with other material, in a separate
+     file or files, that is not Covered Software.
+
+1.8. “License”
+
+     means this document.
+
+1.9. “Licensable”
+
+     means having the right to grant, to the maximum extent possible, whether at the
+     time of the initial grant or subsequently, any and all of the rights conveyed by
+     this License.
+
+1.10. “Modifications”
+
+     means any of the following:
+
+     a. any file in Source Code Form that results from an addition to, deletion
+        from, or modification of the contents of Covered Software; or
+
+     b. any new file in Source Code Form that contains any Covered Software.
+
+1.11. “Patent Claims” of a Contributor
+
+      means any patent claim(s), including without limitation, method, process,
+      and apparatus claims, in any patent Licensable by such Contributor that
+      would be infringed, but for the grant of the License, by the making,
+      using, selling, offering for sale, having made, import, or transfer of
+      either its Contributions or its Contributor Version.
+
+1.12. “Secondary License”
+
+      means either the GNU General Public License, Version 2.0, the GNU Lesser
+      General Public License, Version 2.1, the GNU Affero General Public
+      License, Version 3.0, or any later versions of those licenses.
+
+1.13. “Source Code Form”
+
+      means the form of the work preferred for making modifications.
+
+1.14. “You” (or “Your”)
+
+      means an individual or a legal entity exercising rights under this
+      License. For legal entities, “You” includes any entity that controls, is
+      controlled by, or is under common control with You. For purposes of this
+      definition, “control” means (a) the power, direct or indirect, to cause
+      the direction or management of such entity, whether by contract or
+      otherwise, or (b) ownership of more than fifty percent (50%) of the
+      outstanding shares or beneficial ownership of such entity.
+
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+     Each Contributor hereby grants You a world-wide, royalty-free,
+     non-exclusive license:
+
+     a. under intellectual property rights (other than patent or trademark)
+        Licensable by such Contributor to use, reproduce, make available,
+        modify, display, perform, distribute, and otherwise exploit its
+        Contributions, either on an unmodified basis, with Modifications, or as
+        part of a Larger Work; and
+
+     b. under Patent Claims of such Contributor to make, use, sell, offer for
+        sale, have made, import, and otherwise transfer either its Contributions
+        or its Contributor Version.
+
+2.2. Effective Date
+
+     The licenses granted in Section 2.1 with respect to any Contribution become
+     effective for each Contribution on the date the Contributor first distributes
+     such Contribution.
+
+2.3. Limitations on Grant Scope
+
+     The licenses granted in this Section 2 are the only rights granted under this
+     License. No additional rights or licenses will be implied from the distribution
+     or licensing of Covered Software under this License. Notwithstanding Section
+     2.1(b) above, no patent license is granted by a Contributor:
+
+     a. for any code that a Contributor has removed from Covered Software; or
+
+     b. for infringements caused by: (i) Your and any other third party’s
+        modifications of Covered Software, or (ii) the combination of its
+        Contributions with other software (except as part of its Contributor
+        Version); or
+
+     c. under Patent Claims infringed by Covered Software in the absence of its
+        Contributions.
+
+     This License does not grant any rights in the trademarks, service marks, or
+     logos of any Contributor (except as may be necessary to comply with the
+     notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+     No Contributor makes additional grants as a result of Your choice to
+     distribute the Covered Software under a subsequent version of this License
+     (see Section 10.2) or under the terms of a Secondary License (if permitted
+     under the terms of Section 3.3).
+
+2.5. Representation
+
+     Each Contributor represents that the Contributor believes its Contributions
+     are its original creation(s) or it has sufficient rights to grant the
+     rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+     This License is not intended to limit any rights You have under applicable
+     copyright doctrines of fair use, fair dealing, or other equivalents.
+
+2.7. Conditions
+
+     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+     Section 2.1.
+
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+     All distribution of Covered Software in Source Code Form, including any
+     Modifications that You create or to which You contribute, must be under the
+     terms of this License. You must inform recipients that the Source Code Form
+     of the Covered Software is governed by the terms of this License, and how
+     they can obtain a copy of this License. You may not attempt to alter or
+     restrict the recipients’ rights in the Source Code Form.
+
+3.2. Distribution of Executable Form
+
+     If You distribute Covered Software in Executable Form then:
+
+     a. such Covered Software must also be made available in Source Code Form,
+        as described in Section 3.1, and You must inform recipients of the
+        Executable Form how they can obtain a copy of such Source Code Form by
+        reasonable means in a timely manner, at a charge no more than the cost
+        of distribution to the recipient; and
+
+     b. You may distribute such Executable Form under the terms of this License,
+        or sublicense it under different terms, provided that the license for
+        the Executable Form does not attempt to limit or alter the recipients’
+        rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+     You may create and distribute a Larger Work under terms of Your choice,
+     provided that You also comply with the requirements of this License for the
+     Covered Software. If the Larger Work is a combination of Covered Software
+     with a work governed by one or more Secondary Licenses, and the Covered
+     Software is not Incompatible With Secondary Licenses, this License permits
+     You to additionally distribute such Covered Software under the terms of
+     such Secondary License(s), so that the recipient of the Larger Work may, at
+     their option, further distribute the Covered Software under the terms of
+     either this License or such Secondary License(s).
+
+3.4. Notices
+
+     You may not remove or alter the substance of any license notices (including
+     copyright notices, patent notices, disclaimers of warranty, or limitations
+     of liability) contained within the Source Code Form of the Covered
+     Software, except that You may alter any license notices to the extent
+     required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+     You may choose to offer, and to charge a fee for, warranty, support,
+     indemnity or liability obligations to one or more recipients of Covered
+     Software. However, You may do so only on Your own behalf, and not on behalf
+     of any Contributor. You must make it absolutely clear that any such
+     warranty, support, indemnity, or liability obligation is offered by You
+     alone, and You hereby agree to indemnify every Contributor for any
+     liability incurred by such Contributor as a result of warranty, support,
+     indemnity or liability terms You offer. You may include additional
+     disclaimers of warranty and limitations of liability specific to any
+     jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+   If it is impossible for You to comply with any of the terms of this License
+   with respect to some or all of the Covered Software due to statute, judicial
+   order, or regulation then You must: (a) comply with the terms of this License
+   to the maximum extent possible; and (b) describe the limitations and the code
+   they affect. Such description must be placed in a text file included with all
+   distributions of the Covered Software under this License. Except to the
+   extent prohibited by statute or regulation, such description must be
+   sufficiently detailed for a recipient of ordinary skill to be able to
+   understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically if You
+     fail to comply with any of its terms. However, if You become compliant,
+     then the rights granted under this License from a particular Contributor
+     are reinstated (a) provisionally, unless and until such Contributor
+     explicitly and finally terminates Your grants, and (b) on an ongoing basis,
+     if such Contributor fails to notify You of the non-compliance by some
+     reasonable means prior to 60 days after You have come back into compliance.
+     Moreover, Your grants from a particular Contributor are reinstated on an
+     ongoing basis if such Contributor notifies You of the non-compliance by
+     some reasonable means, this is the first time You have received notice of
+     non-compliance with this License from such Contributor, and You become
+     compliant prior to 30 days after Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+     infringement claim (excluding declaratory judgment actions, counter-claims,
+     and cross-claims) alleging that a Contributor Version directly or
+     indirectly infringes any patent, then the rights granted to You by any and
+     all Contributors for the Covered Software under Section 2.1 of this License
+     shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
+     license agreements (excluding distributors and resellers) which have been
+     validly granted by You or Your distributors under this License prior to
+     termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+   Covered Software is provided under this License on an “as is” basis, without
+   warranty of any kind, either expressed, implied, or statutory, including,
+   without limitation, warranties that the Covered Software is free of defects,
+   merchantable, fit for a particular purpose or non-infringing. The entire
+   risk as to the quality and performance of the Covered Software is with You.
+   Should any Covered Software prove defective in any respect, You (not any
+   Contributor) assume the cost of any necessary servicing, repair, or
+   correction. This disclaimer of warranty constitutes an essential part of this
+   License. No use of  any Covered Software is authorized under this License
+   except under this disclaimer.
+
+7. Limitation of Liability
+
+   Under no circumstances and under no legal theory, whether tort (including
+   negligence), contract, or otherwise, shall any Contributor, or anyone who
+   distributes Covered Software as permitted above, be liable to You for any
+   direct, indirect, special, incidental, or consequential damages of any
+   character including, without limitation, damages for lost profits, loss of
+   goodwill, work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses, even if such party shall have been
+   informed of the possibility of such damages. This limitation of liability
+   shall not apply to liability for death or personal injury resulting from such
+   party’s negligence to the extent applicable law prohibits such limitation.
+   Some jurisdictions do not allow the exclusion or limitation of incidental or
+   consequential damages, so this exclusion and limitation may not apply to You.
+
+8. Litigation
+
+   Any litigation relating to this License may be brought only in the courts of
+   a jurisdiction where the defendant maintains its principal place of business
+   and such litigation shall be governed by laws of that jurisdiction, without
+   reference to its conflict-of-law provisions. Nothing in this Section shall
+   prevent a party’s ability to bring cross-claims or counter-claims.
+
+9. Miscellaneous
+
+   This License represents the complete agreement concerning the subject matter
+   hereof. If any provision of this License is held to be unenforceable, such
+   provision shall be reformed only to the extent necessary to make it
+   enforceable. Any law or regulation which provides that the language of a
+   contract shall be construed against the drafter shall not be used to construe
+   this License against a Contributor.
+
+
+10. Versions of the License
+
+10.1. New Versions
+
+      Mozilla Foundation is the license steward. Except as provided in Section
+      10.3, no one other than the license steward has the right to modify or
+      publish new versions of this License. Each version will be given a
+      distinguishing version number.
+
+10.2. Effect of New Versions
+
+      You may distribute the Covered Software under the terms of the version of
+      the License under which You originally received the Covered Software, or
+      under the terms of any subsequent version published by the license
+      steward.
+
+10.3. Modified Versions
+
+      If you create software not governed by this License, and you want to
+      create a new license for such software, you may create and use a modified
+      version of this License if you rename the license and remove any
+      references to the name of the license steward (except to note that such
+      modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses
+      If You choose to distribute Source Code Form that is Incompatible With
+      Secondary Licenses under the terms of this version of the License, the
+      notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+
+      This Source Code Form is subject to the
+      terms of the Mozilla Public License, v.
+      2.0. If a copy of the MPL was not
+      distributed with this file, You can
+      obtain one at
+      http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file, then
+You may include the notice in a location (such as a LICENSE file in a relevant
+directory) where a recipient would be likely to look for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - “Incompatible With Secondary Licenses” Notice
+
+      This Source Code Form is “Incompatible
+      With Secondary Licenses”, as defined by
+      the Mozilla Public License, v. 2.0.
+

Godeps/_workspace/src/github.com/hashicorp/memberlist/Makefile

@@ -0,0 +1,14 @@
+test: subnet
+	go test ./...
+
+integ: subnet
+	INTEG_TESTS=yes go test ./...
+
+subnet:
+	./test/setup_subnet.sh
+
+cov:
+	gocov test github.com/hashicorp/memberlist | gocov-html > /tmp/coverage.html
+	open /tmp/coverage.html
+
+.PNONY: test cov integ

Godeps/_workspace/src/github.com/hashicorp/memberlist/README.md

@@ -0,0 +1,137 @@
+# memberlist
+
+memberlist is a [Go](http://www.golang.org) library that manages cluster
+membership and member failure detection using a gossip based protocol.
+
+The use cases for such a library are far-reaching: all distributed systems
+require membership, and memberlist is a re-usable solution to managing
+cluster membership and node failure detection.
+
+memberlist is eventually consistent but converges quickly on average.
+The speed at which it converges can be heavily tuned via various knobs
+on the protocol. Node failures are detected and network partitions are partially
+tolerated by attempting to communicate to potentially dead nodes through
+multiple routes.
+
+## Building
+
+If you wish to build memberlist you'll need Go version 1.2+ installed.
+
+Please check your installation with:
+
+```
+go version
+```
+
+## Usage
+
+Memberlist is surprisingly simple to use. An example is shown below:
+
+```go
+/* Create the initial memberlist from a safe configuration.
+   Please reference the godoc for other default config types.
+   http://godoc.org/github.com/hashicorp/memberlist#Config
+*/
+list, err := memberlist.Create(memberlist.DefaultLocalConfig())
+if err != nil {
+	panic("Failed to create memberlist: " + err.Error())
+}
+
+// Join an existing cluster by specifying at least one known member.
+n, err := list.Join([]string{"1.2.3.4"})
+if err != nil {
+	panic("Failed to join cluster: " + err.Error())
+}
+
+// Ask for members of the cluster
+for _, member := range list.Members() {
+	fmt.Printf("Member: %s %s\n", member.Name, member.Addr)
+}
+
+// Continue doing whatever you need, memberlist will maintain membership
+// information in the background. Delegates can be used for receiving
+// events when members join or leave.
+```
+
+The most difficult part of memberlist is configuring it since it has many
+available knobs in order to tune state propagation delay and convergence times.
+Memberlist provides a default configuration that offers a good starting point,
+but errs on the side of caution, choosing values that are optimized for
+higher convergence at the cost of higher bandwidth usage.
+
+For complete documentation, see the associated [Godoc](http://godoc.org/github.com/hashicorp/memberlist).
+
+## Protocol
+
+memberlist is based on ["SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol"](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf),
+with a few minor adaptations, mostly to increase propogation speed and
+convergence rate.
+
+A high level overview of the memberlist protocol (based on SWIM) is
+described below, but for details please read the full
+[SWIM paper](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf)
+followed by the memberlist source. We welcome any questions related
+to the protocol on our issue tracker.
+
+### Protocol Description
+
+memberlist begins by joining an existing cluster or starting a new
+cluster. If starting a new cluster, additional nodes are expected to join
+it. New nodes in an existing cluster must be given the address of at
+least one existing member in order to join the cluster. The new member
+does a full state sync with the existing member over TCP and begins gossiping its
+existence to the cluster.
+
+Gossip is done over UDP to a with a configurable but fixed fanout and interval.
+This ensures that network usage is constant with regards to number of nodes, as opposed to
+exponential growth that can occur with traditional heartbeat mechanisms.
+Complete state exchanges with a random node are done periodically over
+TCP, but much less often than gossip messages. This increases the likelihood
+that the membership list converges properly since the full state is exchanged
+and merged. The interval between full state exchanges is configurable or can
+be disabled entirely.
+
+Failure detection is done by periodic random probing using a configurable interval.
+If the node fails to ack within a reasonable time (typically some multiple
+of RTT), then an indirect probe is attempted. An indirect probe asks a
+configurable number of random nodes to probe the same node, in case there
+are network issues causing our own node to fail the probe. If both our
+probe and the indirect probes fail within a reasonable time, then the
+node is marked "suspicious" and this knowledge is gossiped to the cluster.
+A suspicious node is still considered a member of cluster. If the suspect member
+of the cluster does not disputes the suspicion within a configurable period of
+time, the node is finally considered dead, and this state is then gossiped
+to the cluster.
+
+This is a brief and incomplete description of the protocol. For a better idea,
+please read the
+[SWIM paper](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf)
+in its entirety, along with the memberlist source code.
+
+### Changes from SWIM
+
+As mentioned earlier, the memberlist protocol is based on SWIM but includes
+minor changes, mostly to increase propogation speed and convergence rates.
+
+The changes from SWIM are noted here:
+
+* memberlist does a full state sync over TCP periodically. SWIM only propagates
+  changes over gossip. While both eventually reach convergence, the full state
+  sync increases the likelihood that nodes are fully converged more quickly,
+  at the expense of more bandwidth usage. This feature can be totally disabled
+  if you wish.
+
+* memberlist has a dedicated gossip layer separate from the failure detection
+  protocol. SWIM only piggybacks gossip messages on top of probe/ack messages.
+  memberlist also piggybacks gossip messages on top of probe/ack messages, but
+  also will periodically send out dedicated gossip messages on their own. This
+  feature lets you have a higher gossip rate (for example once per 200ms)
+  and a slower failure detection rate (such as once per second), resulting
+  in overall faster convergence rates and data propogation speeds. This feature
+  can be totally disabed as well, if you wish.
+
+* memberlist stores around the state of dead nodes for a set amount of time,
+  so that when full syncs are requested, the requester also receives information
+  about dead nodes. Because SWIM doesn't do full syncs, SWIM deletes dead node
+  state immediately upon learning that the node is dead. This change again helps
+  the cluster converge more quickly.

Godeps/_workspace/src/github.com/hashicorp/memberlist/broadcast.go

@@ -0,0 +1,100 @@
+package memberlist
+
+/*
+The broadcast mechanism works by maintaining a sorted list of messages to be
+sent out. When a message is to be broadcast, the retransmit count
+is set to zero and appended to the queue. The retransmit count serves
+as the "priority", ensuring that newer messages get sent first. Once
+a message hits the retransmit limit, it is removed from the queue.
+
+Additionally, older entries can be invalidated by new messages that
+are contradictory. For example, if we send "{suspect M1 inc: 1},
+then a following {alive M1 inc: 2} will invalidate that message
+*/
+
+type memberlistBroadcast struct {
+	node   string
+	msg    []byte
+	notify chan struct{}
+}
+
+func (b *memberlistBroadcast) Invalidates(other Broadcast) bool {
+	// Check if that broadcast is a memberlist type
+	mb, ok := other.(*memberlistBroadcast)
+	if !ok {
+		return false
+	}
+
+	// Invalidates any message about the same node
+	return b.node == mb.node
+}
+
+func (b *memberlistBroadcast) Message() []byte {
+	return b.msg
+}
+
+func (b *memberlistBroadcast) Finished() {
+	select {
+	case b.notify <- struct{}{}:
+	default:
+	}
+}
+
+// encodeAndBroadcast encodes a message and enqueues it for broadcast. Fails
+// silently if there is an encoding error.
+func (m *Memberlist) encodeAndBroadcast(node string, msgType messageType, msg interface{}) {
+	m.encodeBroadcastNotify(node, msgType, msg, nil)
+}
+
+// encodeBroadcastNotify encodes a message and enqueues it for broadcast
+// and notifies the given channel when transmission is finished. Fails
+// silently if there is an encoding error.
+func (m *Memberlist) encodeBroadcastNotify(node string, msgType messageType, msg interface{}, notify chan struct{}) {
+	buf, err := encode(msgType, msg)
+	if err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to encode message for broadcast: %s", err)
+	} else {
+		m.queueBroadcast(node, buf.Bytes(), notify)
+	}
+}
+
+// queueBroadcast is used to start dissemination of a message. It will be
+// sent up to a configured number of times. The message could potentially
+// be invalidated by a future message about the same node
+func (m *Memberlist) queueBroadcast(node string, msg []byte, notify chan struct{}) {
+	b := &memberlistBroadcast{node, msg, notify}
+	m.broadcasts.QueueBroadcast(b)
+}
+
+// getBroadcasts is used to return a slice of broadcasts to send up to
+// a maximum byte size, while imposing a per-broadcast overhead. This is used
+// to fill a UDP packet with piggybacked data
+func (m *Memberlist) getBroadcasts(overhead, limit int) [][]byte {
+	// Get memberlist messages first
+	toSend := m.broadcasts.GetBroadcasts(overhead, limit)
+
+	// Check if the user has anything to broadcast
+	d := m.config.Delegate
+	if d != nil {
+		// Determine the bytes used already
+		bytesUsed := 0
+		for _, msg := range toSend {
+			bytesUsed += len(msg) + overhead
+		}
+
+		// Check space remaining for user messages
+		avail := limit - bytesUsed
+		if avail > overhead+userMsgOverhead {
+			userMsgs := d.GetBroadcasts(overhead+userMsgOverhead, avail)
+
+			// Frame each user message
+			for _, msg := range userMsgs {
+				buf := make([]byte, 1, len(msg)+1)
+				buf[0] = byte(userMsg)
+				buf = append(buf, msg...)
+				toSend = append(toSend, buf)
+			}
+		}
+	}
+	return toSend
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/broadcast_test.go

@@ -0,0 +1,27 @@
+package memberlist
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestMemberlistBroadcast_Invalidates(t *testing.T) {
+	m1 := &memberlistBroadcast{"test", nil, nil}
+	m2 := &memberlistBroadcast{"foo", nil, nil}
+
+	if m1.Invalidates(m2) || m2.Invalidates(m1) {
+		t.Fatalf("unexpected invalidation")
+	}
+
+	if !m1.Invalidates(m1) {
+		t.Fatalf("expected invalidation")
+	}
+}
+
+func TestMemberlistBroadcast_Message(t *testing.T) {
+	m1 := &memberlistBroadcast{"test", []byte("test"), nil}
+	msg := m1.Message()
+	if !reflect.DeepEqual(msg, []byte("test")) {
+		t.Fatalf("messages do not match")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/config.go

@@ -0,0 +1,209 @@
+package memberlist
+
+import (
+	"io"
+	"os"
+	"time"
+)
+
+type Config struct {
+	// The name of this node. This must be unique in the cluster.
+	Name string
+
+	// Configuration related to what address to bind to and ports to
+	// listen on. The port is used for both UDP and TCP gossip.
+	// It is assumed other nodes are running on this port, but they
+	// do not need to.
+	BindAddr string
+	BindPort int
+
+	// Configuration related to what address to advertise to other
+	// cluster members. Used for nat traversal.
+	AdvertiseAddr string
+	AdvertisePort int
+
+	// ProtocolVersion is the configured protocol version that we
+	// will _speak_. This must be between ProtocolVersionMin and
+	// ProtocolVersionMax.
+	ProtocolVersion uint8
+
+	// TCPTimeout is the timeout for establishing a TCP connection with
+	// a remote node for a full state sync.
+	TCPTimeout time.Duration
+
+	// IndirectChecks is the number of nodes that will be asked to perform
+	// an indirect probe of a node in the case a direct probe fails. Memberlist
+	// waits for an ack from any single indirect node, so increasing this
+	// number will increase the likelihood that an indirect probe will succeed
+	// at the expense of bandwidth.
+	IndirectChecks int
+
+	// RetransmitMult is the multiplier for the number of retransmissions
+	// that are attempted for messages broadcasted over gossip. The actual
+	// count of retransmissions is calculated using the formula:
+	//
+	//   Retransmits = RetransmitMult * log(N+1)
+	//
+	// This allows the retransmits to scale properly with cluster size. The
+	// higher the multiplier, the more likely a failed broadcast is to converge
+	// at the expense of increased bandwidth.
+	RetransmitMult int
+
+	// SuspicionMult is the multiplier for determining the time an
+	// inaccessible node is considered suspect before declaring it dead.
+	// The actual timeout is calculated using the formula:
+	//
+	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
+	//
+	// This allows the timeout to scale properly with expected propagation
+	// delay with a larger cluster size. The higher the multiplier, the longer
+	// an inaccessible node is considered part of the cluster before declaring
+	// it dead, giving that suspect node more time to refute if it is indeed
+	// still alive.
+	SuspicionMult int
+
+	// PushPullInterval is the interval between complete state syncs.
+	// Complete state syncs are done with a single node over TCP and are
+	// quite expensive relative to standard gossiped messages. Setting this
+	// to zero will disable state push/pull syncs completely.
+	//
+	// Setting this interval lower (more frequent) will increase convergence
+	// speeds across larger clusters at the expense of increased bandwidth
+	// usage.
+	PushPullInterval time.Duration
+
+	// ProbeInterval and ProbeTimeout are used to configure probing
+	// behavior for memberlist.
+	//
+	// ProbeInterval is the interval between random node probes. Setting
+	// this lower (more frequent) will cause the memberlist cluster to detect
+	// failed nodes more quickly at the expense of increased bandwidth usage.
+	//
+	// ProbeTimeout is the timeout to wait for an ack from a probed node
+	// before assuming it is unhealthy. This should be set to 99-percentile
+	// of RTT (round-trip time) on your network.
+	ProbeInterval time.Duration
+	ProbeTimeout  time.Duration
+
+	// GossipInterval and GossipNodes are used to configure the gossip
+	// behavior of memberlist.
+	//
+	// GossipInterval is the interval between sending messages that need
+	// to be gossiped that haven't been able to piggyback on probing messages.
+	// If this is set to zero, non-piggyback gossip is disabled. By lowering
+	// this value (more frequent) gossip messages are propagated across
+	// the cluster more quickly at the expense of increased bandwidth.
+	//
+	// GossipNodes is the number of random nodes to send gossip messages to
+	// per GossipInterval. Increasing this number causes the gossip messages
+	// to propagate across the cluster more quickly at the expense of
+	// increased bandwidth.
+	GossipInterval time.Duration
+	GossipNodes    int
+
+	// EnableCompression is used to control message compression. This can
+	// be used to reduce bandwidth usage at the cost of slightly more CPU
+	// utilization. This is only available starting at protocol version 1.
+	EnableCompression bool
+
+	// SecretKey is used to initialize the primary encryption key in a keyring.
+	// The primary encryption key is the only key used to encrypt messages and
+	// the first key used while attempting to decrypt messages. Providing a
+	// value for this primary key will enable message-level encryption and
+	// verification, and automatically install the key onto the keyring.
+	SecretKey []byte
+
+	// The keyring holds all of the encryption keys used internally. It is
+	// automatically initialized using the SecretKey and SecretKeys values.
+	Keyring *Keyring
+
+	// Delegate and Events are delegates for receiving and providing
+	// data to memberlist via callback mechanisms. For Delegate, see
+	// the Delegate interface. For Events, see the EventDelegate interface.
+	//
+	// The DelegateProtocolMin/Max are used to guarantee protocol-compatibility
+	// for any custom messages that the delegate might do (broadcasts,
+	// local/remote state, etc.). If you don't set these, then the protocol
+	// versions will just be zero, and version compliance won't be done.
+	Delegate                Delegate
+	DelegateProtocolVersion uint8
+	DelegateProtocolMin     uint8
+	DelegateProtocolMax     uint8
+	Events                  EventDelegate
+	Conflict                ConflictDelegate
+	Merge                   MergeDelegate
+
+	// LogOutput is the writer where logs should be sent. If this is not
+	// set, logging will go to stderr by default.
+	LogOutput io.Writer
+}
+
+// DefaultLANConfig returns a sane set of configurations for Memberlist.
+// It uses the hostname as the node name, and otherwise sets very conservative
+// values that are sane for most LAN environments. The default configuration
+// errs on the side on the side of caution, choosing values that are optimized
+// for higher convergence at the cost of higher bandwidth usage. Regardless,
+// these values are a good starting point when getting started with memberlist.
+func DefaultLANConfig() *Config {
+	hostname, _ := os.Hostname()
+	return &Config{
+		Name:             hostname,
+		BindAddr:         "0.0.0.0",
+		BindPort:         7946,
+		AdvertiseAddr:    "",
+		AdvertisePort:    7946,
+		ProtocolVersion:  ProtocolVersionMax,
+		TCPTimeout:       10 * time.Second,       // Timeout after 10 seconds
+		IndirectChecks:   3,                      // Use 3 nodes for the indirect ping
+		RetransmitMult:   4,                      // Retransmit a message 4 * log(N+1) nodes
+		SuspicionMult:    5,                      // Suspect a node for 5 * log(N+1) * Interval
+		PushPullInterval: 30 * time.Second,       // Low frequency
+		ProbeTimeout:     500 * time.Millisecond, // Reasonable RTT time for LAN
+		ProbeInterval:    1 * time.Second,        // Failure check every second
+
+		GossipNodes:    3,                      // Gossip to 3 nodes
+		GossipInterval: 200 * time.Millisecond, // Gossip more rapidly
+
+		EnableCompression: true, // Enable compression by default
+
+		SecretKey: nil,
+
+		Keyring: nil,
+	}
+}
+
+// DefaultWANConfig works like DefaultConfig, however it returns a configuration
+// that is optimized for most WAN environments. The default configuration is
+// still very conservative and errs on the side of caution.
+func DefaultWANConfig() *Config {
+	conf := DefaultLANConfig()
+	conf.TCPTimeout = 30 * time.Second
+	conf.SuspicionMult = 6
+	conf.PushPullInterval = 60 * time.Second
+	conf.ProbeTimeout = 3 * time.Second
+	conf.ProbeInterval = 5 * time.Second
+	conf.GossipNodes = 4 // Gossip less frequently, but to an additional node
+	conf.GossipInterval = 500 * time.Millisecond
+	return conf
+}
+
+// DefaultLocalConfig works like DefaultConfig, however it returns a configuration
+// that is optimized for a local loopback environments. The default configuration is
+// still very conservative and errs on the side of caution.
+func DefaultLocalConfig() *Config {
+	conf := DefaultLANConfig()
+	conf.TCPTimeout = time.Second
+	conf.IndirectChecks = 1
+	conf.RetransmitMult = 2
+	conf.SuspicionMult = 3
+	conf.PushPullInterval = 15 * time.Second
+	conf.ProbeTimeout = 200 * time.Millisecond
+	conf.ProbeInterval = time.Second
+	conf.GossipInterval = 100 * time.Millisecond
+	return conf
+}
+
+// Returns whether or not encryption is enabled
+func (c *Config) EncryptionEnabled() bool {
+	return c.Keyring != nil && len(c.Keyring.GetKeys()) > 0
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/conflict_delegate.go

@@ -0,0 +1,10 @@
+package memberlist
+
+// ConflictDelegate is a used to inform a client that
+// a node has attempted to join which would result in a
+// name conflict. This happens if two clients are configured
+// with the same name but different addresses.
+type ConflictDelegate interface {
+	// NotifyConflict is invoked when a name conflict is detected
+	NotifyConflict(existing, other *Node)
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/delegate.go

@@ -0,0 +1,36 @@
+package memberlist
+
+// Delegate is the interface that clients must implement if they want to hook
+// into the gossip layer of Memberlist. All the methods must be thread-safe,
+// as they can and generally will be called concurrently.
+type Delegate interface {
+	// NodeMeta is used to retrieve meta-data about the current node
+	// when broadcasting an alive message. It's length is limited to
+	// the given byte size. This metadata is available in the Node structure.
+	NodeMeta(limit int) []byte
+
+	// NotifyMsg is called when a user-data message is received.
+	// Care should be taken that this method does not block, since doing
+	// so would block the entire UDP packet receive loop. Additionally, the byte
+	// slice may be modified after the call returns, so it should be copied if needed.
+	NotifyMsg([]byte)
+
+	// GetBroadcasts is called when user data messages can be broadcast.
+	// It can return a list of buffers to send. Each buffer should assume an
+	// overhead as provided with a limit on the total byte size allowed.
+	// The total byte size of the resulting data to send must not exceed
+	// the limit.
+	GetBroadcasts(overhead, limit int) [][]byte
+
+	// LocalState is used for a TCP Push/Pull. This is sent to
+	// the remote side in addition to the membership information. Any
+	// data can be sent here. See MergeRemoteState as well. The `join`
+	// boolean indicates this is for a join instead of a push/pull.
+	LocalState(join bool) []byte
+
+	// MergeRemoteState is invoked after a TCP Push/Pull. This is the
+	// state received from the remote side and is the result of the
+	// remote side's LocalState call. The 'join'
+	// boolean indicates this is for a join instead of a push/pull.
+	MergeRemoteState(buf []byte, join bool)
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/event_delegate.go

@@ -0,0 +1,61 @@
+package memberlist
+
+// EventDelegate is a simpler delegate that is used only to receive
+// notifications about members joining and leaving. The methods in this
+// delegate may be called by multiple goroutines, but never concurrently.
+// This allows you to reason about ordering.
+type EventDelegate interface {
+	// NotifyJoin is invoked when a node is detected to have joined.
+	// The Node argument must not be modified.
+	NotifyJoin(*Node)
+
+	// NotifyLeave is invoked when a node is detected to have left.
+	// The Node argument must not be modified.
+	NotifyLeave(*Node)
+
+	// NotifyUpdate is invoked when a node is detected to have
+	// updated, usually involving the meta data. The Node argument
+	// must not be modified.
+	NotifyUpdate(*Node)
+}
+
+// ChannelEventDelegate is used to enable an application to receive
+// events about joins and leaves over a channel instead of a direct
+// function call.
+//
+// Care must be taken that events are processed in a timely manner from
+// the channel, since this delegate will block until an event can be sent.
+type ChannelEventDelegate struct {
+	Ch chan<- NodeEvent
+}
+
+// NodeEventType are the types of events that can be sent from the
+// ChannelEventDelegate.
+type NodeEventType int
+
+const (
+	NodeJoin NodeEventType = iota
+	NodeLeave
+	NodeUpdate
+)
+
+// NodeEvent is a single event related to node activity in the memberlist.
+// The Node member of this struct must not be directly modified. It is passed
+// as a pointer to avoid unnecessary copies. If you wish to modify the node,
+// make a copy first.
+type NodeEvent struct {
+	Event NodeEventType
+	Node  *Node
+}
+
+func (c *ChannelEventDelegate) NotifyJoin(n *Node) {
+	c.Ch <- NodeEvent{NodeJoin, n}
+}
+
+func (c *ChannelEventDelegate) NotifyLeave(n *Node) {
+	c.Ch <- NodeEvent{NodeLeave, n}
+}
+
+func (c *ChannelEventDelegate) NotifyUpdate(n *Node) {
+	c.Ch <- NodeEvent{NodeUpdate, n}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/integ_test.go

@@ -0,0 +1,89 @@
+package memberlist
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+// CheckInteg will skip a test if integration testing is not enabled.
+func CheckInteg(t *testing.T) {
+	if !IsInteg() {
+		t.SkipNow()
+	}
+}
+
+// IsInteg returns a boolean telling you if we're in integ testing mode.
+func IsInteg() bool {
+	return os.Getenv("INTEG_TESTS") != ""
+}
+
+// Tests the memberlist by creating a cluster of 100 nodes
+// and checking that we get strong convergence of changes.
+func TestMemberlist_Integ(t *testing.T) {
+	CheckInteg(t)
+
+	num := 16
+	var members []*Memberlist
+
+	secret := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	eventCh := make(chan NodeEvent, num)
+
+	addr := "127.0.0.1"
+	for i := 0; i < num; i++ {
+		c := DefaultLANConfig()
+		c.Name = fmt.Sprintf("%s:%d", addr, 12345+i)
+		c.BindAddr = addr
+		c.BindPort = 12345 + i
+		c.ProbeInterval = 20 * time.Millisecond
+		c.ProbeTimeout = 100 * time.Millisecond
+		c.GossipInterval = 20 * time.Millisecond
+		c.PushPullInterval = 200 * time.Millisecond
+		c.SecretKey = secret
+
+		if i == 0 {
+			c.Events = &ChannelEventDelegate{eventCh}
+		}
+
+		m, err := Create(c)
+		if err != nil {
+			t.Fatalf("unexpected err: %s", err)
+		}
+		members = append(members, m)
+		defer m.Shutdown()
+
+		if i > 0 {
+			last := members[i-1]
+			num, err := m.Join([]string{last.config.Name})
+			if num == 0 || err != nil {
+				t.Fatalf("unexpected err: %s", err)
+			}
+		}
+	}
+
+	// Wait and print debug info
+	breakTimer := time.After(250 * time.Millisecond)
+WAIT:
+	for {
+		select {
+		case e := <-eventCh:
+			if e.Event == NodeJoin {
+				log.Printf("[DEBUG] Node join: %v (%d)", *e.Node, members[0].NumMembers())
+			} else {
+				log.Printf("[DEBUG] Node leave: %v (%d)", *e.Node, members[0].NumMembers())
+			}
+		case <-breakTimer:
+			break WAIT
+		}
+	}
+
+	for idx, m := range members {
+		got := m.NumMembers()
+		if got != num {
+			t.Errorf("bad num members at idx %d. Expected %d. Got %d.",
+				idx, num, got)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/keyring.go

@@ -0,0 +1,144 @@
+package memberlist
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+)
+
+type Keyring struct {
+	// Keys stores the key data used during encryption and decryption. It is
+	// ordered in such a way where the first key (index 0) is the primary key,
+	// which is used for encrypting messages, and is the first key tried during
+	// message decryption.
+	keys [][]byte
+
+	// The keyring lock is used while performing IO operations on the keyring.
+	l sync.Mutex
+}
+
+// Init allocates substructures
+func (k *Keyring) init() {
+	k.keys = make([][]byte, 0)
+}
+
+// NewKeyring constructs a new container for a set of encryption keys. The
+// keyring contains all key data used internally by memberlist.
+//
+// While creating a new keyring, you must do one of:
+//   - Omit keys and primary key, effectively disabling encryption
+//   - Pass a set of keys plus the primary key
+//   - Pass only a primary key
+//
+// If only a primary key is passed, then it will be automatically added to the
+// keyring. If creating a keyring with multiple keys, one key must be designated
+// primary by passing it as the primaryKey. If the primaryKey does not exist in
+// the list of secondary keys, it will be automatically added at position 0.
+func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) {
+	keyring := &Keyring{}
+	keyring.init()
+
+	if len(keys) > 0 || len(primaryKey) > 0 {
+		if len(primaryKey) == 0 {
+			return nil, fmt.Errorf("Empty primary key not allowed")
+		}
+		if err := keyring.AddKey(primaryKey); err != nil {
+			return nil, err
+		}
+		for _, key := range keys {
+			if err := keyring.AddKey(key); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return keyring, nil
+}
+
+// AddKey will install a new key on the ring. Adding a key to the ring will make
+// it available for use in decryption. If the key already exists on the ring,
+// this function will just return noop.
+func (k *Keyring) AddKey(key []byte) error {
+	// Encorce 16-byte key size
+	if len(key) != 16 {
+		return fmt.Errorf("key size must be 16 bytes")
+	}
+
+	// No-op if key is already installed
+	for _, installedKey := range k.keys {
+		if bytes.Equal(installedKey, key) {
+			return nil
+		}
+	}
+
+	keys := append(k.keys, key)
+	primaryKey := k.GetPrimaryKey()
+	if primaryKey == nil {
+		primaryKey = key
+	}
+	k.installKeys(keys, primaryKey)
+	return nil
+}
+
+// UseKey changes the key used to encrypt messages. This is the only key used to
+// encrypt messages, so peers should know this key before this method is called.
+func (k *Keyring) UseKey(key []byte) error {
+	for _, installedKey := range k.keys {
+		if bytes.Equal(key, installedKey) {
+			k.installKeys(k.keys, key)
+			return nil
+		}
+	}
+	return fmt.Errorf("Requested key is not in the keyring")
+}
+
+// RemoveKey drops a key from the keyring. This will return an error if the key
+// requested for removal is currently at position 0 (primary key).
+func (k *Keyring) RemoveKey(key []byte) error {
+	if bytes.Equal(key, k.keys[0]) {
+		return fmt.Errorf("Removing the primary key is not allowed")
+	}
+	for i, installedKey := range k.keys {
+		if bytes.Equal(key, installedKey) {
+			keys := append(k.keys[:i], k.keys[i+1:]...)
+			k.installKeys(keys, k.keys[0])
+		}
+	}
+	return nil
+}
+
+// installKeys will take out a lock on the keyring, and replace the keys with a
+// new set of keys. The key indicated by primaryKey will be installed as the new
+// primary key.
+func (k *Keyring) installKeys(keys [][]byte, primaryKey []byte) {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	newKeys := [][]byte{primaryKey}
+	for _, key := range keys {
+		if !bytes.Equal(key, primaryKey) {
+			newKeys = append(newKeys, key)
+		}
+	}
+	k.keys = newKeys
+}
+
+// GetKeys returns the current set of keys on the ring.
+func (k *Keyring) GetKeys() [][]byte {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	return k.keys
+}
+
+// GetPrimaryKey returns the key on the ring at position 0. This is the key used
+// for encrypting messages, and is the first key tried for decrypting messages.
+func (k *Keyring) GetPrimaryKey() (key []byte) {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	if len(k.keys) > 0 {
+		key = k.keys[0]
+	}
+	return
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/keyring_test.go

@@ -0,0 +1,154 @@
+package memberlist
+
+import (
+	"bytes"
+	"testing"
+)
+
+var TestKeys [][]byte = [][]byte{
+	[]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+	[]byte{15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0},
+	[]byte{8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7},
+}
+
+func TestKeyring_EmptyRing(t *testing.T) {
+	// Keyrings can be created with no encryption keys (disabled encryption)
+	keyring, err := NewKeyring(nil, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	keys := keyring.GetKeys()
+	if len(keys) != 0 {
+		t.Fatalf("Expected 0 keys but have %d", len(keys))
+	}
+}
+
+func TestKeyring_PrimaryOnly(t *testing.T) {
+	// Keyrings can be created using only a primary key
+	keyring, err := NewKeyring(nil, TestKeys[0])
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	keys := keyring.GetKeys()
+	if len(keys) != 1 {
+		t.Fatalf("Expected 1 key but have %d", len(keys))
+	}
+}
+
+func TestKeyring_GetPrimaryKey(t *testing.T) {
+	keyring, err := NewKeyring(TestKeys, TestKeys[1])
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// GetPrimaryKey returns correct key
+	primaryKey := keyring.GetPrimaryKey()
+	if !bytes.Equal(primaryKey, TestKeys[1]) {
+		t.Fatalf("Unexpected primary key: %v", primaryKey)
+	}
+}
+
+func TestKeyring_AddRemoveUse(t *testing.T) {
+	keyring, err := NewKeyring(nil, TestKeys[1])
+	if err != nil {
+		t.Fatalf("err :%s", err)
+	}
+
+	// Use non-existent key throws error
+	if err := keyring.UseKey(TestKeys[2]); err == nil {
+		t.Fatalf("Expected key not installed error")
+	}
+
+	// Add key to ring
+	if err := keyring.AddKey(TestKeys[2]); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	keys := keyring.GetKeys()
+	if !bytes.Equal(keys[0], TestKeys[1]) {
+		t.Fatalf("Unexpected primary key change")
+	}
+
+	if len(keys) != 2 {
+		t.Fatalf("Expected 2 keys but have %d", len(keys))
+	}
+
+	// Use key that exists should succeed
+	if err := keyring.UseKey(TestKeys[2]); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	primaryKey := keyring.GetPrimaryKey()
+	if !bytes.Equal(primaryKey, TestKeys[2]) {
+		t.Fatalf("Unexpected primary key: %v", primaryKey)
+	}
+
+	// Removing primary key should fail
+	if err := keyring.RemoveKey(TestKeys[2]); err == nil {
+		t.Fatalf("Expected primary key removal error")
+	}
+
+	// Removing non-primary key should succeed
+	if err := keyring.RemoveKey(TestKeys[1]); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	keys = keyring.GetKeys()
+	if len(keys) != 1 {
+		t.Fatalf("Expected 1 key but have %d", len(keys))
+	}
+}
+
+func TestKeyRing_MultiKeyEncryptDecrypt(t *testing.T) {
+	plaintext := []byte("this is a plain text message")
+	extra := []byte("random data")
+
+	keyring, err := NewKeyring(TestKeys, TestKeys[0])
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// First encrypt using the primary key and make sure we can decrypt
+	var buf bytes.Buffer
+	err = encryptPayload(1, TestKeys[0], plaintext, extra, &buf)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	msg, err := decryptPayload(keyring.GetKeys(), buf.Bytes(), extra)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !bytes.Equal(msg, plaintext) {
+		t.Fatalf("bad: %v", msg)
+	}
+
+	// Now encrypt with a secondary key and try decrypting again.
+	buf.Reset()
+	err = encryptPayload(1, TestKeys[2], plaintext, extra, &buf)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !bytes.Equal(msg, plaintext) {
+		t.Fatalf("bad: %v", msg)
+	}
+
+	// Remove a key from the ring, and then try decrypting again
+	if err := keyring.RemoveKey(TestKeys[2]); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	msg, err = decryptPayload(keyring.GetKeys(), buf.Bytes(), extra)
+	if err == nil {
+		t.Fatalf("Expected no keys to decrypt message")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/memberlist.go

@@ -0,0 +1,525 @@
+/*
+memberlist is a library that manages cluster
+membership and member failure detection using a gossip based protocol.
+
+The use cases for such a library are far-reaching: all distributed systems
+require membership, and memberlist is a re-usable solution to managing
+cluster membership and node failure detection.
+
+memberlist is eventually consistent but converges quickly on average.
+The speed at which it converges can be heavily tuned via various knobs
+on the protocol. Node failures are detected and network partitions are partially
+tolerated by attempting to communicate to potentially dead nodes through
+multiple routes.
+*/
+package memberlist
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+type Memberlist struct {
+	config         *Config
+	shutdown       bool
+	shutdownCh     chan struct{}
+	leave          bool
+	leaveBroadcast chan struct{}
+
+	udpListener *net.UDPConn
+	tcpListener *net.TCPListener
+	handoff     chan msgHandoff
+
+	sequenceNum uint32 // Local sequence number
+	incarnation uint32 // Local incarnation number
+
+	nodeLock sync.RWMutex
+	nodes    []*nodeState          // Known nodes
+	nodeMap  map[string]*nodeState // Maps Addr.String() -> NodeState
+
+	tickerLock sync.Mutex
+	tickers    []*time.Ticker
+	stopTick   chan struct{}
+	probeIndex int
+
+	ackLock     sync.Mutex
+	ackHandlers map[uint32]*ackHandler
+
+	broadcasts *TransmitLimitedQueue
+
+	startStopLock sync.Mutex
+
+	logger *log.Logger
+}
+
+// newMemberlist creates the network listeners.
+// Does not schedule execution of background maintenence.
+func newMemberlist(conf *Config) (*Memberlist, error) {
+	if conf.ProtocolVersion < ProtocolVersionMin {
+		return nil, fmt.Errorf("Protocol version '%d' too low. Must be in range: [%d, %d]",
+			conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
+	} else if conf.ProtocolVersion > ProtocolVersionMax {
+		return nil, fmt.Errorf("Protocol version '%d' too high. Must be in range: [%d, %d]",
+			conf.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
+	}
+
+	if len(conf.SecretKey) > 0 {
+		if conf.Keyring == nil {
+			keyring, err := NewKeyring(nil, conf.SecretKey)
+			if err != nil {
+				return nil, err
+			}
+			conf.Keyring = keyring
+		} else {
+			if err := conf.Keyring.AddKey(conf.SecretKey); err != nil {
+				return nil, err
+			}
+			if err := conf.Keyring.UseKey(conf.SecretKey); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	tcpAddr := &net.TCPAddr{IP: net.ParseIP(conf.BindAddr), Port: conf.BindPort}
+	tcpLn, err := net.ListenTCP("tcp", tcpAddr)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
+	}
+
+	udpAddr := &net.UDPAddr{IP: net.ParseIP(conf.BindAddr), Port: conf.BindPort}
+	udpLn, err := net.ListenUDP("udp", udpAddr)
+	if err != nil {
+		tcpLn.Close()
+		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
+	}
+
+	// Set the UDP receive window size
+	setUDPRecvBuf(udpLn)
+
+	if conf.LogOutput == nil {
+		conf.LogOutput = os.Stderr
+	}
+	logger := log.New(conf.LogOutput, "", log.LstdFlags)
+
+	m := &Memberlist{
+		config:         conf,
+		shutdownCh:     make(chan struct{}),
+		leaveBroadcast: make(chan struct{}, 1),
+		udpListener:    udpLn,
+		tcpListener:    tcpLn,
+		handoff:        make(chan msgHandoff, 1024),
+		nodeMap:        make(map[string]*nodeState),
+		ackHandlers:    make(map[uint32]*ackHandler),
+		broadcasts:     &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
+		logger:         logger,
+	}
+	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
+	go m.tcpListen()
+	go m.udpListen()
+	go m.udpHandler()
+	return m, nil
+}
+
+// Create will create a new Memberlist using the given configuration.
+// This will not connect to any other node (see Join) yet, but will start
+// all the listeners to allow other nodes to join this memberlist.
+// After creating a Memberlist, the configuration given should not be
+// modified by the user anymore.
+func Create(conf *Config) (*Memberlist, error) {
+	m, err := newMemberlist(conf)
+	if err != nil {
+		return nil, err
+	}
+	if err := m.setAlive(); err != nil {
+		m.Shutdown()
+		return nil, err
+	}
+	m.schedule()
+	return m, nil
+}
+
+// Join is used to take an existing Memberlist and attempt to join a cluster
+// by contacting all the given hosts and performing a state sync. Initially,
+// the Memberlist only contains our own state, so doing this will cause
+// remote nodes to become aware of the existence of this node, effectively
+// joining the cluster.
+//
+// This returns the number of hosts successfully contacted and an error if
+// none could be reached. If an error is returned, the node did not successfully
+// join the cluster.
+func (m *Memberlist) Join(existing []string) (int, error) {
+	// Attempt to join any of them
+	numSuccess := 0
+	var retErr error
+	for _, exist := range existing {
+		addrs, port, err := m.resolveAddr(exist)
+		if err != nil {
+			m.logger.Printf("[WARN] memberlist: Failed to resolve %s: %v", exist, err)
+			retErr = err
+			continue
+		}
+
+		for _, addr := range addrs {
+			if err := m.pushPullNode(addr, port, true); err != nil {
+				retErr = err
+				continue
+			}
+			numSuccess++
+		}
+
+	}
+
+	if numSuccess > 0 {
+		retErr = nil
+	}
+
+	return numSuccess, retErr
+}
+
+// resolveAddr is used to resolve the address into an address,
+// port, and error. If no port is given, use the default
+func (m *Memberlist) resolveAddr(hostStr string) ([][]byte, uint16, error) {
+	ips := make([][]byte, 0)
+	port := uint16(0)
+	host, sport, err := net.SplitHostPort(hostStr)
+	if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+		// error, port missing - we can solve this
+		port = uint16(m.config.BindPort)
+		host = hostStr
+	} else if err != nil {
+		// error, but not missing port
+		return ips, port, err
+	} else if lport, err := strconv.ParseUint(sport, 10, 16); err != nil {
+		// error, when parsing port
+		return ips, port, err
+	} else {
+		// no error
+		port = uint16(lport)
+	}
+
+	// Get the addresses that hostPort might resolve to
+	// ResolveTcpAddr requres ipv6 brackets to separate
+	// port numbers whereas ParseIP doesn't, but luckily
+	// SplitHostPort takes care of the brackets
+	if ip := net.ParseIP(host); ip == nil {
+		if pre, err := net.LookupIP(host); err == nil {
+			for _, ip := range pre {
+				ips = append(ips, ip)
+			}
+		} else {
+			return ips, port, err
+		}
+	} else {
+		ips = append(ips, ip)
+	}
+
+	return ips, port, nil
+}
+
+// setAlive is used to mark this node as being alive. This is the same
+// as if we received an alive notification our own network channel for
+// ourself.
+func (m *Memberlist) setAlive() error {
+
+	var advertiseAddr []byte
+	var advertisePort int
+	if m.config.AdvertiseAddr != "" {
+		// If AdvertiseAddr is not empty, then advertise
+		// the given address and port.
+		ip := net.ParseIP(m.config.AdvertiseAddr)
+		if ip == nil {
+			return fmt.Errorf("Failed to parse advertise address!")
+		}
+
+		// Ensure IPv4 conversion if necessary
+		if ip4 := ip.To4(); ip4 != nil {
+			ip = ip4
+		}
+
+		advertiseAddr = ip
+		advertisePort = m.config.AdvertisePort
+	} else {
+		if m.config.BindAddr == "0.0.0.0" {
+			// Otherwise, if we're not bound to a specific IP,
+			//let's list the interfaces on this machine and use
+			// the first private IP we find.
+			addresses, err := net.InterfaceAddrs()
+			if err != nil {
+				return fmt.Errorf("Failed to get interface addresses! Err: %v", err)
+			}
+
+			// Find private IPv4 address
+			for _, rawAddr := range addresses {
+				var ip net.IP
+				switch addr := rawAddr.(type) {
+				case *net.IPAddr:
+					ip = addr.IP
+				case *net.IPNet:
+					ip = addr.IP
+				default:
+					continue
+				}
+
+				if ip.To4() == nil {
+					continue
+				}
+				if !isPrivateIP(ip.String()) {
+					continue
+				}
+
+				advertiseAddr = ip
+				break
+			}
+
+			// Failed to find private IP, error
+			if advertiseAddr == nil {
+				return fmt.Errorf("No private IP address found, and explicit IP not provided")
+			}
+
+		} else {
+			// Use the IP that we're bound to.
+			addr := m.tcpListener.Addr().(*net.TCPAddr)
+			advertiseAddr = addr.IP
+		}
+		advertisePort = m.config.BindPort
+	}
+
+	// Check if this is a public address without encryption
+	addrStr := net.IP(advertiseAddr).String()
+	if !isPrivateIP(addrStr) && !isLoopbackIP(addrStr) && !m.config.EncryptionEnabled() {
+		m.logger.Printf("[WARN] memberlist: Binding to public address without encryption!")
+	}
+
+	// Get the node meta data
+	var meta []byte
+	if m.config.Delegate != nil {
+		meta = m.config.Delegate.NodeMeta(MetaMaxSize)
+		if len(meta) > MetaMaxSize {
+			panic("Node meta data provided is longer than the limit")
+		}
+	}
+
+	a := alive{
+		Incarnation: m.nextIncarnation(),
+		Node:        m.config.Name,
+		Addr:        advertiseAddr,
+		Port:        uint16(advertisePort),
+		Meta:        meta,
+		Vsn: []uint8{
+			ProtocolVersionMin, ProtocolVersionMax, m.config.ProtocolVersion,
+			m.config.DelegateProtocolMin, m.config.DelegateProtocolMax,
+			m.config.DelegateProtocolVersion,
+		},
+	}
+	m.aliveNode(&a, nil, true)
+
+	return nil
+}
+
+// LocalNode is used to return the local Node
+func (m *Memberlist) LocalNode() *Node {
+	m.nodeLock.RLock()
+	defer m.nodeLock.RUnlock()
+	state := m.nodeMap[m.config.Name]
+	return &state.Node
+}
+
+// UpdateNode is used to trigger re-advertising the local node. This is
+// primarily used with a Delegate to support dynamic updates to the local
+// meta data.  This will block until the update message is successfully
+// broadcasted to a member of the cluster, if any exist or until a specified
+// timeout is reached.
+func (m *Memberlist) UpdateNode(timeout time.Duration) error {
+	// Get the node meta data
+	var meta []byte
+	if m.config.Delegate != nil {
+		meta = m.config.Delegate.NodeMeta(MetaMaxSize)
+		if len(meta) > MetaMaxSize {
+			panic("Node meta data provided is longer than the limit")
+		}
+	}
+
+	// Get the existing node
+	m.nodeLock.RLock()
+	state := m.nodeMap[m.config.Name]
+	m.nodeLock.RUnlock()
+
+	// Format a new alive message
+	a := alive{
+		Incarnation: m.nextIncarnation(),
+		Node:        m.config.Name,
+		Addr:        state.Addr,
+		Port:        state.Port,
+		Meta:        meta,
+		Vsn: []uint8{
+			ProtocolVersionMin, ProtocolVersionMax, m.config.ProtocolVersion,
+			m.config.DelegateProtocolMin, m.config.DelegateProtocolMax,
+			m.config.DelegateProtocolVersion,
+		},
+	}
+	notifyCh := make(chan struct{})
+	m.aliveNode(&a, notifyCh, true)
+
+	// Wait for the broadcast or a timeout
+	if m.anyAlive() {
+		var timeoutCh <-chan time.Time
+		if timeout > 0 {
+			timeoutCh = time.After(timeout)
+		}
+		select {
+		case <-notifyCh:
+		case <-timeoutCh:
+			return fmt.Errorf("timeout waiting for update broadcast")
+		}
+	}
+	return nil
+}
+
+// SendTo is used to directly send a message to another node, without
+// the use of the gossip mechanism. This will encode the message as a
+// user-data message, which a delegate will receive through NotifyMsg
+// The actual data is transmitted over UDP, which means this is a
+// best-effort transmission mechanism, and the maximum size of the
+// message is the size of a single UDP datagram, after compression
+func (m *Memberlist) SendTo(to net.Addr, msg []byte) error {
+	// Encode as a user message
+	buf := make([]byte, 1, len(msg)+1)
+	buf[0] = byte(userMsg)
+	buf = append(buf, msg...)
+
+	// Send the message
+	return m.rawSendMsg(to, buf)
+}
+
+// Members returns a list of all known live nodes. The node structures
+// returned must not be modified. If you wish to modify a Node, make a
+// copy first.
+func (m *Memberlist) Members() []*Node {
+	m.nodeLock.RLock()
+	defer m.nodeLock.RUnlock()
+
+	nodes := make([]*Node, 0, len(m.nodes))
+	for _, n := range m.nodes {
+		if n.State != stateDead {
+			nodes = append(nodes, &n.Node)
+		}
+	}
+
+	return nodes
+}
+
+// NumMembers returns the number of alive nodes currently known. Between
+// the time of calling this and calling Members, the number of alive nodes
+// may have changed, so this shouldn't be used to determine how many
+// members will be returned by Members.
+func (m *Memberlist) NumMembers() (alive int) {
+	m.nodeLock.RLock()
+	defer m.nodeLock.RUnlock()
+
+	for _, n := range m.nodes {
+		if n.State != stateDead {
+			alive++
+		}
+	}
+
+	return
+}
+
+// Leave will broadcast a leave message but will not shutdown the background
+// listeners, meaning the node will continue participating in gossip and state
+// updates.
+//
+// This will block until the leave message is successfully broadcasted to
+// a member of the cluster, if any exist or until a specified timeout
+// is reached.
+//
+// This method is safe to call multiple times, but must not be called
+// after the cluster is already shut down.
+func (m *Memberlist) Leave(timeout time.Duration) error {
+	m.startStopLock.Lock()
+	defer m.startStopLock.Unlock()
+
+	if m.shutdown {
+		panic("leave after shutdown")
+	}
+
+	if !m.leave {
+		m.leave = true
+
+		state, ok := m.nodeMap[m.config.Name]
+		if !ok {
+			m.logger.Printf("[WARN] memberlist: Leave but we're not in the node map.")
+			return nil
+		}
+
+		d := dead{
+			Incarnation: state.Incarnation,
+			Node:        state.Name,
+		}
+		m.deadNode(&d)
+
+		// Block until the broadcast goes out
+		if m.anyAlive() {
+			var timeoutCh <-chan time.Time
+			if timeout > 0 {
+				timeoutCh = time.After(timeout)
+			}
+			select {
+			case <-m.leaveBroadcast:
+			case <-timeoutCh:
+				return fmt.Errorf("timeout waiting for leave broadcast")
+			}
+		}
+	}
+
+	return nil
+}
+
+// Check for any other alive node.
+func (m *Memberlist) anyAlive() bool {
+	m.nodeLock.RLock()
+	defer m.nodeLock.RUnlock()
+	for _, n := range m.nodes {
+		if n.State != stateDead && n.Name != m.config.Name {
+			return true
+		}
+	}
+	return false
+}
+
+// ProtocolVersion returns the protocol version currently in use by
+// this memberlist.
+func (m *Memberlist) ProtocolVersion() uint8 {
+	// NOTE: This method exists so that in the future we can control
+	// any locking if necessary, if we change the protocol version at
+	// runtime, etc.
+	return m.config.ProtocolVersion
+}
+
+// Shutdown will stop any background maintanence of network activity
+// for this memberlist, causing it to appear "dead". A leave message
+// will not be broadcasted prior, so the cluster being left will have
+// to detect this node's shutdown using probing. If you wish to more
+// gracefully exit the cluster, call Leave prior to shutting down.
+//
+// This method is safe to call multiple times.
+func (m *Memberlist) Shutdown() error {
+	m.startStopLock.Lock()
+	defer m.startStopLock.Unlock()
+
+	if m.shutdown {
+		return nil
+	}
+
+	m.shutdown = true
+	close(m.shutdownCh)
+	m.deschedule()
+	m.udpListener.Close()
+	m.tcpListener.Close()
+	return nil
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/merge_delegate.go

@@ -0,0 +1,13 @@
+package memberlist
+
+// MergeDelegate is used to involve a client in
+// a potential cluster merge operation. Namely, when
+// a node does a TCP push/pull (as part of a join),
+// the delegate is involved and allowed to cancel the join
+// based on custom logic. The merge delegate is NOT invoked
+// as part of the push-pull anti-entropy.
+type MergeDelegate interface {
+	// NotifyMerge is invoked when a merge could take place.
+	// Provides a list of the nodes known by the peer.
+	NotifyMerge(peers []*Node) (cancel bool)
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/net.go

@@ -0,0 +1,852 @@
+package memberlist
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/armon/go-metrics"
+	"github.com/hashicorp/go-msgpack/codec"
+)
+
+// This is the minimum and maximum protocol version that we can
+// _understand_. We're allowed to speak at any version within this
+// range. This range is inclusive.
+const (
+	ProtocolVersionMin uint8 = 1
+	ProtocolVersionMax       = 2
+)
+
+// messageType is an integer ID of a type of message that can be received
+// on network channels from other members.
+type messageType uint8
+
+// The list of available message types.
+const (
+	pingMsg messageType = iota
+	indirectPingMsg
+	ackRespMsg
+	suspectMsg
+	aliveMsg
+	deadMsg
+	pushPullMsg
+	compoundMsg
+	userMsg // User mesg, not handled by us
+	compressMsg
+	encryptMsg
+)
+
+// compressionType is used to specify the compression algorithm
+type compressionType uint8
+
+const (
+	lzwAlgo compressionType = iota
+)
+
+const (
+	MetaMaxSize            = 512 // Maximum size for node meta data
+	compoundHeaderOverhead = 2   // Assumed header overhead
+	compoundOverhead       = 2   // Assumed overhead per entry in compoundHeader
+	udpBufSize             = 65536
+	udpRecvBuf             = 2 * 1024 * 1024
+	udpSendBuf             = 1400
+	userMsgOverhead        = 1
+	blockingWarning        = 10 * time.Millisecond // Warn if a UDP packet takes this long to process
+	maxPushStateBytes      = 10 * 1024 * 1024
+)
+
+// ping request sent directly to node
+type ping struct {
+	SeqNo uint32
+
+	// Node is sent so the target can verify they are
+	// the intended recipient. This is to protect again an agent
+	// restart with a new name.
+	Node string
+}
+
+// indirect ping sent to an indirect ndoe
+type indirectPingReq struct {
+	SeqNo  uint32
+	Target []byte
+	Port   uint16
+	Node   string
+}
+
+// ack response is sent for a ping
+type ackResp struct {
+	SeqNo uint32
+}
+
+// suspect is broadcast when we suspect a node is dead
+type suspect struct {
+	Incarnation uint32
+	Node        string
+	From        string // Include who is suspecting
+}
+
+// alive is broadcast when we know a node is alive.
+// Overloaded for nodes joining
+type alive struct {
+	Incarnation uint32
+	Node        string
+	Addr        []byte
+	Port        uint16
+	Meta        []byte
+
+	// The versions of the protocol/delegate that are being spoken, order:
+	// pmin, pmax, pcur, dmin, dmax, dcur
+	Vsn []uint8
+}
+
+// dead is broadcast when we confirm a node is dead
+// Overloaded for nodes leaving
+type dead struct {
+	Incarnation uint32
+	Node        string
+	From        string // Include who is suspecting
+}
+
+// pushPullHeader is used to inform the
+// otherside how many states we are transfering
+type pushPullHeader struct {
+	Nodes        int
+	UserStateLen int  // Encodes the byte lengh of user state
+	Join         bool // Is this a join request or a anti-entropy run
+}
+
+// pushNodeState is used for pushPullReq when we are
+// transfering out node states
+type pushNodeState struct {
+	Name        string
+	Addr        []byte
+	Port        uint16
+	Meta        []byte
+	Incarnation uint32
+	State       nodeStateType
+	Vsn         []uint8 // Protocol versions
+}
+
+// compress is used to wrap an underlying payload
+// using a specified compression algorithm
+type compress struct {
+	Algo compressionType
+	Buf  []byte
+}
+
+// msgHandoff is used to transfer a message between goroutines
+type msgHandoff struct {
+	msgType messageType
+	buf     []byte
+	from    net.Addr
+}
+
+// encryptionVersion returns the encryption version to use
+func (m *Memberlist) encryptionVersion() encryptionVersion {
+	switch m.ProtocolVersion() {
+	case 1:
+		return 0
+	default:
+		return 1
+	}
+}
+
+// setUDPRecvBuf is used to resize the UDP receive window. The function
+// attempts to set the read buffer to `udpRecvBuf` but backs off until
+// the read buffer can be set.
+func setUDPRecvBuf(c *net.UDPConn) {
+	size := udpRecvBuf
+	for {
+		if err := c.SetReadBuffer(size); err == nil {
+			break
+		}
+		size = size / 2
+	}
+}
+
+// tcpListen listens for and handles incoming connections
+func (m *Memberlist) tcpListen() {
+	for {
+		conn, err := m.tcpListener.AcceptTCP()
+		if err != nil {
+			if m.shutdown {
+				break
+			}
+			m.logger.Printf("[ERR] memberlist: Error accepting TCP connection: %s", err)
+			continue
+		}
+		go m.handleConn(conn)
+	}
+}
+
+// handleConn handles a single incoming TCP connection
+func (m *Memberlist) handleConn(conn *net.TCPConn) {
+	m.logger.Printf("[DEBUG] memberlist: Responding to push/pull sync with: %s", conn.RemoteAddr())
+	defer conn.Close()
+	metrics.IncrCounter([]string{"memberlist", "tcp", "accept"}, 1)
+
+	join, remoteNodes, userState, err := m.readRemoteState(conn)
+	if err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to receive remote state: %s", err)
+		return
+	}
+
+	if err := m.sendLocalState(conn, join); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to push local state: %s", err)
+	}
+
+	if err := m.verifyProtocol(remoteNodes); err != nil {
+		m.logger.Printf("[ERR] memberlist: Push/pull verification failed: %s", err)
+		return
+	}
+
+	// Invoke the merge delegate if any
+	if join && m.config.Merge != nil {
+		nodes := make([]*Node, len(remoteNodes))
+		for idx, n := range remoteNodes {
+			nodes[idx] = &Node{
+				Name: n.Name,
+				Addr: n.Addr,
+				Port: n.Port,
+				Meta: n.Meta,
+				PMin: n.Vsn[0],
+				PMax: n.Vsn[1],
+				PCur: n.Vsn[2],
+				DMin: n.Vsn[3],
+				DMax: n.Vsn[4],
+				DCur: n.Vsn[5],
+			}
+		}
+		if m.config.Merge.NotifyMerge(nodes) {
+			m.logger.Printf("[WARN] memberlist: Cluster merge canceled")
+			return
+		}
+	}
+
+	// Merge the membership state
+	m.mergeState(remoteNodes)
+
+	// Invoke the delegate for user state
+	if m.config.Delegate != nil {
+		m.config.Delegate.MergeRemoteState(userState, join)
+	}
+}
+
+// udpListen listens for and handles incoming UDP packets
+func (m *Memberlist) udpListen() {
+	var n int
+	var addr net.Addr
+	var err error
+	var lastPacket time.Time
+	for {
+		// Do a check for potentially blocking operations
+		if !lastPacket.IsZero() && time.Now().Sub(lastPacket) > blockingWarning {
+			diff := time.Now().Sub(lastPacket)
+			m.logger.Printf(
+				"[DEBUG] memberlist: Potential blocking operation. Last command took %v",
+				diff)
+		}
+
+		// Create a new buffer
+		// TODO: Use Sync.Pool eventually
+		buf := make([]byte, udpBufSize)
+
+		// Read a packet
+		n, addr, err = m.udpListener.ReadFrom(buf)
+		if err != nil {
+			if m.shutdown {
+				break
+			}
+			m.logger.Printf("[ERR] memberlist: Error reading UDP packet: %s", err)
+			continue
+		}
+
+		// Check the length
+		if n < 1 {
+			m.logger.Printf("[ERR] memberlist: UDP packet too short (%d bytes). From: %s",
+				len(buf), addr)
+			continue
+		}
+
+		// Capture the current time
+		lastPacket = time.Now()
+
+		// Ingest this packet
+		metrics.IncrCounter([]string{"memberlist", "udp", "received"}, float32(n))
+		m.ingestPacket(buf[:n], addr)
+	}
+}
+
+func (m *Memberlist) ingestPacket(buf []byte, from net.Addr) {
+	// Check if encryption is enabled
+	if m.config.EncryptionEnabled() {
+		// Decrypt the payload
+		plain, err := decryptPayload(m.config.Keyring.GetKeys(), buf, nil)
+		if err != nil {
+			m.logger.Printf("[ERR] memberlist: Decrypt packet failed: %v", err)
+			return
+		}
+
+		// Continue processing the plaintext buffer
+		buf = plain
+	}
+
+	// Handle the command
+	m.handleCommand(buf, from)
+}
+
+func (m *Memberlist) handleCommand(buf []byte, from net.Addr) {
+	// Decode the message type
+	msgType := messageType(buf[0])
+	buf = buf[1:]
+
+	// Switch on the msgType
+	switch msgType {
+	case compoundMsg:
+		m.handleCompound(buf, from)
+	case compressMsg:
+		m.handleCompressed(buf, from)
+
+	case pingMsg:
+		m.handlePing(buf, from)
+	case indirectPingMsg:
+		m.handleIndirectPing(buf, from)
+	case ackRespMsg:
+		m.handleAck(buf, from)
+
+	case suspectMsg:
+		fallthrough
+	case aliveMsg:
+		fallthrough
+	case deadMsg:
+		fallthrough
+	case userMsg:
+		select {
+		case m.handoff <- msgHandoff{msgType, buf, from}:
+		default:
+			m.logger.Printf("[WARN] memberlist: UDP handler queue full, dropping message (%d)", msgType)
+		}
+
+	default:
+		m.logger.Printf("[ERR] memberlist: UDP msg type (%d) not supported. From: %s", msgType, from)
+	}
+}
+
+// udpHandler processes messages received over UDP, but is decoupled
+// from the listener to avoid blocking the listener which may cause
+// ping/ack messages to be delayed.
+func (m *Memberlist) udpHandler() {
+	for {
+		select {
+		case msg := <-m.handoff:
+			msgType := msg.msgType
+			buf := msg.buf
+			from := msg.from
+
+			switch msgType {
+			case suspectMsg:
+				m.handleSuspect(buf, from)
+			case aliveMsg:
+				m.handleAlive(buf, from)
+			case deadMsg:
+				m.handleDead(buf, from)
+			case userMsg:
+				m.handleUser(buf, from)
+			default:
+				m.logger.Printf("[ERR] memberlist: UDP msg type (%d) not supported. From: %s (handler)", msgType, from)
+			}
+
+		case <-m.shutdownCh:
+			return
+		}
+	}
+}
+
+func (m *Memberlist) handleCompound(buf []byte, from net.Addr) {
+	// Decode the parts
+	trunc, parts, err := decodeCompoundMessage(buf)
+	if err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode compound request: %s", err)
+		return
+	}
+
+	// Log any truncation
+	if trunc > 0 {
+		m.logger.Printf("[WARN] memberlist: Compound request had %d truncated messages", trunc)
+	}
+
+	// Handle each message
+	for _, part := range parts {
+		m.handleCommand(part, from)
+	}
+}
+
+func (m *Memberlist) handlePing(buf []byte, from net.Addr) {
+	var p ping
+	if err := decode(buf, &p); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode ping request: %s", err)
+		return
+	}
+	// If node is provided, verify that it is for us
+	if p.Node != "" && p.Node != m.config.Name {
+		m.logger.Printf("[WARN] memberlist: Got ping for unexpected node '%s'", p.Node)
+		return
+	}
+	ack := ackResp{p.SeqNo}
+	if err := m.encodeAndSendMsg(from, ackRespMsg, &ack); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to send ack: %s", err)
+	}
+}
+
+func (m *Memberlist) handleIndirectPing(buf []byte, from net.Addr) {
+	var ind indirectPingReq
+	if err := decode(buf, &ind); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode indirect ping request: %s", err)
+		return
+	}
+
+	// For proto versions < 2, there is no port provided. Mask old
+	// behavior by using the configured port
+	if m.ProtocolVersion() < 2 || ind.Port == 0 {
+		ind.Port = uint16(m.config.BindPort)
+	}
+
+	// Send a ping to the correct host
+	localSeqNo := m.nextSeqNo()
+	ping := ping{SeqNo: localSeqNo, Node: ind.Node}
+	destAddr := &net.UDPAddr{IP: ind.Target, Port: int(ind.Port)}
+
+	// Setup a response handler to relay the ack
+	respHandler := func() {
+		ack := ackResp{ind.SeqNo}
+		if err := m.encodeAndSendMsg(from, ackRespMsg, &ack); err != nil {
+			m.logger.Printf("[ERR] memberlist: Failed to forward ack: %s", err)
+		}
+	}
+	m.setAckHandler(localSeqNo, respHandler, m.config.ProbeTimeout)
+
+	// Send the ping
+	if err := m.encodeAndSendMsg(destAddr, pingMsg, &ping); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to send ping: %s", err)
+	}
+}
+
+func (m *Memberlist) handleAck(buf []byte, from net.Addr) {
+	var ack ackResp
+	if err := decode(buf, &ack); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode ack response: %s", err)
+		return
+	}
+	m.invokeAckHandler(ack.SeqNo)
+}
+
+func (m *Memberlist) handleSuspect(buf []byte, from net.Addr) {
+	var sus suspect
+	if err := decode(buf, &sus); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode suspect message: %s", err)
+		return
+	}
+	m.suspectNode(&sus)
+}
+
+func (m *Memberlist) handleAlive(buf []byte, from net.Addr) {
+	var live alive
+	if err := decode(buf, &live); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode alive message: %s", err)
+		return
+	}
+
+	// For proto versions < 2, there is no port provided. Mask old
+	// behavior by using the configured port
+	if m.ProtocolVersion() < 2 || live.Port == 0 {
+		live.Port = uint16(m.config.BindPort)
+	}
+
+	m.aliveNode(&live, nil, false)
+}
+
+func (m *Memberlist) handleDead(buf []byte, from net.Addr) {
+	var d dead
+	if err := decode(buf, &d); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decode dead message: %s", err)
+		return
+	}
+	m.deadNode(&d)
+}
+
+// handleUser is used to notify channels of incoming user data
+func (m *Memberlist) handleUser(buf []byte, from net.Addr) {
+	d := m.config.Delegate
+	if d != nil {
+		d.NotifyMsg(buf)
+	}
+}
+
+// handleCompressed is used to unpack a compressed message
+func (m *Memberlist) handleCompressed(buf []byte, from net.Addr) {
+	// Try to decode the payload
+	payload, err := decompressPayload(buf)
+	if err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to decompress payload: %v", err)
+		return
+	}
+
+	// Recursively handle the payload
+	m.handleCommand(payload, from)
+}
+
+// encodeAndSendMsg is used to combine the encoding and sending steps
+func (m *Memberlist) encodeAndSendMsg(to net.Addr, msgType messageType, msg interface{}) error {
+	out, err := encode(msgType, msg)
+	if err != nil {
+		return err
+	}
+	if err := m.sendMsg(to, out.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// sendMsg is used to send a UDP message to another host. It will opportunistically
+// create a compoundMsg and piggy back other broadcasts
+func (m *Memberlist) sendMsg(to net.Addr, msg []byte) error {
+	// Check if we can piggy back any messages
+	bytesAvail := udpSendBuf - len(msg) - compoundHeaderOverhead
+	if m.config.EncryptionEnabled() {
+		bytesAvail -= encryptOverhead(m.encryptionVersion())
+	}
+	extra := m.getBroadcasts(compoundOverhead, bytesAvail)
+
+	// Fast path if nothing to piggypack
+	if len(extra) == 0 {
+		return m.rawSendMsg(to, msg)
+	}
+
+	// Join all the messages
+	msgs := make([][]byte, 0, 1+len(extra))
+	msgs = append(msgs, msg)
+	msgs = append(msgs, extra...)
+
+	// Create a compound message
+	compound := makeCompoundMessage(msgs)
+
+	// Send the message
+	return m.rawSendMsg(to, compound.Bytes())
+}
+
+// rawSendMsg is used to send a UDP message to another host without modification
+func (m *Memberlist) rawSendMsg(to net.Addr, msg []byte) error {
+	// Check if we have compression enabled
+	if m.config.EnableCompression {
+		buf, err := compressPayload(msg)
+		if err != nil {
+			m.logger.Printf("[WARN] memberlist: Failed to compress payload: %v", err)
+		} else {
+			// Only use compression if it reduced the size
+			if buf.Len() < len(msg) {
+				msg = buf.Bytes()
+			}
+		}
+	}
+
+	// Check if we have encryption enabled
+	if m.config.EncryptionEnabled() {
+		// Encrypt the payload
+		var buf bytes.Buffer
+		primaryKey := m.config.Keyring.GetPrimaryKey()
+		err := encryptPayload(m.encryptionVersion(), primaryKey, msg, nil, &buf)
+		if err != nil {
+			m.logger.Printf("[ERR] memberlist: Encryption of message failed: %v", err)
+			return err
+		}
+		msg = buf.Bytes()
+	}
+
+	metrics.IncrCounter([]string{"memberlist", "udp", "sent"}, float32(len(msg)))
+	_, err := m.udpListener.WriteTo(msg, to)
+	return err
+}
+
+// sendState is used to initiate a push/pull over TCP with a remote node
+func (m *Memberlist) sendAndReceiveState(addr []byte, port uint16, join bool) ([]pushNodeState, []byte, error) {
+	// Attempt to connect
+	dialer := net.Dialer{Timeout: m.config.TCPTimeout}
+	dest := net.TCPAddr{IP: addr, Port: int(port)}
+	conn, err := dialer.Dial("tcp", dest.String())
+	if err != nil {
+		return nil, nil, err
+	}
+	defer conn.Close()
+	m.logger.Printf("[DEBUG] memberlist: Initiating push/pull sync with: %s", conn.RemoteAddr())
+	metrics.IncrCounter([]string{"memberlist", "tcp", "connect"}, 1)
+
+	// Send our state
+	if err := m.sendLocalState(conn, join); err != nil {
+		return nil, nil, err
+	}
+
+	// Read remote state
+	_, remote, userState, err := m.readRemoteState(conn)
+	if err != nil {
+		err := fmt.Errorf("Reading remote state failed: %v", err)
+		return nil, nil, err
+	}
+
+	// Return the remote state
+	return remote, userState, nil
+}
+
+// sendLocalState is invoked to send our local state over a tcp connection
+func (m *Memberlist) sendLocalState(conn net.Conn, join bool) error {
+	// Setup a deadline
+	conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
+
+	// Prepare the local node state
+	m.nodeLock.RLock()
+	localNodes := make([]pushNodeState, len(m.nodes))
+	for idx, n := range m.nodes {
+		localNodes[idx].Name = n.Name
+		localNodes[idx].Addr = n.Addr
+		localNodes[idx].Port = n.Port
+		localNodes[idx].Incarnation = n.Incarnation
+		localNodes[idx].State = n.State
+		localNodes[idx].Meta = n.Meta
+		localNodes[idx].Vsn = []uint8{
+			n.PMin, n.PMax, n.PCur,
+			n.DMin, n.DMax, n.DCur,
+		}
+	}
+	m.nodeLock.RUnlock()
+
+	// Get the delegate state
+	var userData []byte
+	if m.config.Delegate != nil {
+		userData = m.config.Delegate.LocalState(join)
+	}
+
+	// Create a bytes buffer writer
+	bufConn := bytes.NewBuffer(nil)
+
+	// Send our node state
+	header := pushPullHeader{Nodes: len(localNodes), UserStateLen: len(userData), Join: join}
+	hd := codec.MsgpackHandle{}
+	enc := codec.NewEncoder(bufConn, &hd)
+
+	// Begin state push
+	if _, err := bufConn.Write([]byte{byte(pushPullMsg)}); err != nil {
+		return err
+	}
+
+	if err := enc.Encode(&header); err != nil {
+		return err
+	}
+	for i := 0; i < header.Nodes; i++ {
+		if err := enc.Encode(&localNodes[i]); err != nil {
+			return err
+		}
+	}
+
+	// Write the user state as well
+	if userData != nil {
+		if _, err := bufConn.Write(userData); err != nil {
+			return err
+		}
+	}
+
+	// Get the send buffer
+	sendBuf := bufConn.Bytes()
+
+	// Check if compresion is enabled
+	if m.config.EnableCompression {
+		compBuf, err := compressPayload(bufConn.Bytes())
+		if err != nil {
+			m.logger.Printf("[ERROR] memberlist: Failed to compress local state: %v", err)
+		} else {
+			sendBuf = compBuf.Bytes()
+		}
+	}
+
+	// Check if encryption is enabled
+	if m.config.EncryptionEnabled() {
+		crypt, err := m.encryptLocalState(sendBuf)
+		if err != nil {
+			m.logger.Printf("[ERROR] memberlist: Failed to encrypt local state: %v", err)
+			return err
+		}
+		sendBuf = crypt
+	}
+
+	// Write out the entire send buffer
+	metrics.IncrCounter([]string{"memberlist", "tcp", "sent"}, float32(len(sendBuf)))
+	if _, err := conn.Write(sendBuf); err != nil {
+		return err
+	}
+	return nil
+}
+
+// encryptLocalState is used to help encrypt local state before sending
+func (m *Memberlist) encryptLocalState(sendBuf []byte) ([]byte, error) {
+	var buf bytes.Buffer
+
+	// Write the encryptMsg byte
+	buf.WriteByte(byte(encryptMsg))
+
+	// Write the size of the message
+	sizeBuf := make([]byte, 4)
+	encVsn := m.encryptionVersion()
+	encLen := encryptedLength(encVsn, len(sendBuf))
+	binary.BigEndian.PutUint32(sizeBuf, uint32(encLen))
+	buf.Write(sizeBuf)
+
+	// Write the encrypted cipher text to the buffer
+	key := m.config.Keyring.GetPrimaryKey()
+	err := encryptPayload(encVsn, key, sendBuf, buf.Bytes()[:5], &buf)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// decryptRemoteState is used to help decrypt the remote state
+func (m *Memberlist) decryptRemoteState(bufConn io.Reader) ([]byte, error) {
+	// Read in enough to determine message length
+	cipherText := bytes.NewBuffer(nil)
+	cipherText.WriteByte(byte(encryptMsg))
+	_, err := io.CopyN(cipherText, bufConn, 4)
+	if err != nil {
+		return nil, err
+	}
+
+	// Ensure we aren't asked to download too much. This is to guard against
+	// an attack vector where a huge amount of state is sent
+	moreBytes := binary.BigEndian.Uint32(cipherText.Bytes()[1:5])
+	if moreBytes > maxPushStateBytes {
+		return nil, fmt.Errorf("Remote node state is larger than limit (%d)", moreBytes)
+	}
+
+	// Read in the rest of the payload
+	_, err = io.CopyN(cipherText, bufConn, int64(moreBytes))
+	if err != nil {
+		return nil, err
+	}
+
+	// Decrypt the cipherText
+	dataBytes := cipherText.Bytes()[:5]
+	cipherBytes := cipherText.Bytes()[5:]
+
+	// Decrypt the payload
+	keys := m.config.Keyring.GetKeys()
+	return decryptPayload(keys, cipherBytes, dataBytes)
+}
+
+// recvRemoteState is used to read the remote state from a connection
+func (m *Memberlist) readRemoteState(conn net.Conn) (bool, []pushNodeState, []byte, error) {
+	// Setup a deadline
+	conn.SetDeadline(time.Now().Add(m.config.TCPTimeout))
+
+	// Created a buffered reader
+	var bufConn io.Reader = bufio.NewReader(conn)
+
+	// Read the message type
+	buf := [1]byte{0}
+	if _, err := bufConn.Read(buf[:]); err != nil {
+		return false, nil, nil, err
+	}
+	msgType := messageType(buf[0])
+
+	// Check if the message is encrypted
+	if msgType == encryptMsg {
+		if !m.config.EncryptionEnabled() {
+			return false, nil, nil,
+				fmt.Errorf("Remote state is encrypted and encryption is not configured")
+		}
+
+		plain, err := m.decryptRemoteState(bufConn)
+		if err != nil {
+			return false, nil, nil, err
+		}
+
+		// Reset message type and bufConn
+		msgType = messageType(plain[0])
+		bufConn = bytes.NewReader(plain[1:])
+	} else if m.config.EncryptionEnabled() {
+		return false, nil, nil,
+			fmt.Errorf("Encryption is configured but remote state is not encrypted")
+	}
+
+	// Get the msgPack decoders
+	hd := codec.MsgpackHandle{}
+	dec := codec.NewDecoder(bufConn, &hd)
+
+	// Check if we have a compressed message
+	if msgType == compressMsg {
+		var c compress
+		if err := dec.Decode(&c); err != nil {
+			return false, nil, nil, err
+		}
+		decomp, err := decompressBuffer(&c)
+		if err != nil {
+			return false, nil, nil, err
+		}
+
+		// Reset the message type
+		msgType = messageType(decomp[0])
+
+		// Create a new bufConn
+		bufConn = bytes.NewReader(decomp[1:])
+
+		// Create a new decoder
+		dec = codec.NewDecoder(bufConn, &hd)
+	}
+
+	// Quit if not push/pull
+	if msgType != pushPullMsg {
+		err := fmt.Errorf("received invalid msgType (%d)", msgType)
+		return false, nil, nil, err
+	}
+
+	// Read the push/pull header
+	var header pushPullHeader
+	if err := dec.Decode(&header); err != nil {
+		return false, nil, nil, err
+	}
+
+	// Allocate space for the transfer
+	remoteNodes := make([]pushNodeState, header.Nodes)
+
+	// Try to decode all the states
+	for i := 0; i < header.Nodes; i++ {
+		if err := dec.Decode(&remoteNodes[i]); err != nil {
+			return false, remoteNodes, nil, err
+		}
+	}
+
+	// Read the remote user state into a buffer
+	var userBuf []byte
+	if header.UserStateLen > 0 {
+		userBuf = make([]byte, header.UserStateLen)
+		bytes, err := io.ReadAtLeast(bufConn, userBuf, header.UserStateLen)
+		if err == nil && bytes != header.UserStateLen {
+			err = fmt.Errorf(
+				"Failed to read full user state (%d / %d)",
+				bytes, header.UserStateLen)
+		}
+		if err != nil {
+			return false, remoteNodes, nil, err
+		}
+	}
+
+	// For proto versions < 2, there is no port provided. Mask old
+	// behavior by using the configured port
+	for idx := range remoteNodes {
+		if m.ProtocolVersion() < 2 || remoteNodes[idx].Port == 0 {
+			remoteNodes[idx].Port = uint16(m.config.BindPort)
+		}
+	}
+
+	return header.Join, remoteNodes, userBuf, nil
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/net_test.go

@@ -0,0 +1,476 @@
+package memberlist
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"github.com/hashicorp/go-msgpack/codec"
+	"io"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestHandleCompoundPing(t *testing.T) {
+	m := GetMemberlist(t)
+	m.config.EnableCompression = false
+	defer m.Shutdown()
+
+	var udp *net.UDPConn
+	for port := 60000; port < 61000; port++ {
+		udpAddr := fmt.Sprintf("127.0.0.1:%d", port)
+		udpLn, err := net.ListenPacket("udp", udpAddr)
+		if err == nil {
+			udp = udpLn.(*net.UDPConn)
+			break
+		}
+	}
+
+	if udp == nil {
+		t.Fatalf("no udp listener")
+	}
+
+	// Encode a ping
+	ping := ping{SeqNo: 42}
+	buf, err := encode(pingMsg, ping)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Make a compound message
+	compound := makeCompoundMessage([][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()})
+
+	// Send compound version
+	addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort}
+	udp.WriteTo(compound.Bytes(), addr)
+
+	// Wait for responses
+	go func() {
+		time.Sleep(time.Second)
+		panic("timeout")
+	}()
+
+	for i := 0; i < 3; i++ {
+		in := make([]byte, 1500)
+		n, _, err := udp.ReadFrom(in)
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		in = in[0:n]
+
+		msgType := messageType(in[0])
+		if msgType != ackRespMsg {
+			t.Fatalf("bad response %v", in)
+		}
+
+		var ack ackResp
+		if err := decode(in[1:], &ack); err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+
+		if ack.SeqNo != 42 {
+			t.Fatalf("bad sequence no")
+		}
+	}
+}
+
+func TestHandlePing(t *testing.T) {
+	m := GetMemberlist(t)
+	m.config.EnableCompression = false
+	defer m.Shutdown()
+
+	var udp *net.UDPConn
+	for port := 60000; port < 61000; port++ {
+		udpAddr := fmt.Sprintf("127.0.0.1:%d", port)
+		udpLn, err := net.ListenPacket("udp", udpAddr)
+		if err == nil {
+			udp = udpLn.(*net.UDPConn)
+			break
+		}
+	}
+
+	if udp == nil {
+		t.Fatalf("no udp listener")
+	}
+
+	// Encode a ping
+	ping := ping{SeqNo: 42}
+	buf, err := encode(pingMsg, ping)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Send
+	addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort}
+	udp.WriteTo(buf.Bytes(), addr)
+
+	// Wait for response
+	go func() {
+		time.Sleep(time.Second)
+		panic("timeout")
+	}()
+
+	in := make([]byte, 1500)
+	n, _, err := udp.ReadFrom(in)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+	in = in[0:n]
+
+	msgType := messageType(in[0])
+	if msgType != ackRespMsg {
+		t.Fatalf("bad response %v", in)
+	}
+
+	var ack ackResp
+	if err := decode(in[1:], &ack); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	if ack.SeqNo != 42 {
+		t.Fatalf("bad sequence no")
+	}
+}
+
+func TestHandlePing_WrongNode(t *testing.T) {
+	m := GetMemberlist(t)
+	m.config.EnableCompression = false
+	defer m.Shutdown()
+
+	var udp *net.UDPConn
+	for port := 60000; port < 61000; port++ {
+		udpAddr := fmt.Sprintf("127.0.0.1:%d", port)
+		udpLn, err := net.ListenPacket("udp", udpAddr)
+		if err == nil {
+			udp = udpLn.(*net.UDPConn)
+			break
+		}
+	}
+
+	if udp == nil {
+		t.Fatalf("no udp listener")
+	}
+
+	// Encode a ping, wrong node!
+	ping := ping{SeqNo: 42, Node: m.config.Name + "-bad"}
+	buf, err := encode(pingMsg, ping)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Send
+	addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort}
+	udp.WriteTo(buf.Bytes(), addr)
+
+	// Wait for response
+	udp.SetDeadline(time.Now().Add(50 * time.Millisecond))
+	in := make([]byte, 1500)
+	_, _, err = udp.ReadFrom(in)
+
+	// Should get an i/o timeout
+	if err == nil {
+		t.Fatalf("expected err %s", err)
+	}
+}
+
+func TestHandleIndirectPing(t *testing.T) {
+	m := GetMemberlist(t)
+	m.config.EnableCompression = false
+	defer m.Shutdown()
+
+	var udp *net.UDPConn
+	for port := 60000; port < 61000; port++ {
+		udpAddr := fmt.Sprintf("127.0.0.1:%d", port)
+		udpLn, err := net.ListenPacket("udp", udpAddr)
+		if err == nil {
+			udp = udpLn.(*net.UDPConn)
+			break
+		}
+	}
+
+	if udp == nil {
+		t.Fatalf("no udp listener")
+	}
+
+	// Encode an indirect ping
+	ind := indirectPingReq{
+		SeqNo:  100,
+		Target: net.ParseIP(m.config.BindAddr),
+		Port:   uint16(m.config.BindPort),
+	}
+	buf, err := encode(indirectPingMsg, &ind)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Send
+	addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort}
+	udp.WriteTo(buf.Bytes(), addr)
+
+	// Wait for response
+	go func() {
+		time.Sleep(time.Second)
+		panic("timeout")
+	}()
+
+	in := make([]byte, 1500)
+	n, _, err := udp.ReadFrom(in)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+	in = in[0:n]
+
+	msgType := messageType(in[0])
+	if msgType != ackRespMsg {
+		t.Fatalf("bad response %v", in)
+	}
+
+	var ack ackResp
+	if err := decode(in[1:], &ack); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	if ack.SeqNo != 100 {
+		t.Fatalf("bad sequence no")
+	}
+}
+
+func TestTCPPushPull(t *testing.T) {
+	m := GetMemberlist(t)
+	defer m.Shutdown()
+	m.nodes = append(m.nodes, &nodeState{
+		Node: Node{
+			Name: "Test 0",
+			Addr: net.ParseIP(m.config.BindAddr),
+			Port: uint16(m.config.BindPort),
+		},
+		Incarnation: 0,
+		State:       stateSuspect,
+		StateChange: time.Now().Add(-1 * time.Second),
+	})
+
+	addr := fmt.Sprintf("%s:%d", m.config.BindAddr, m.config.BindPort)
+	conn, err := net.Dial("tcp", addr)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+	defer conn.Close()
+
+	localNodes := make([]pushNodeState, 3)
+	localNodes[0].Name = "Test 0"
+	localNodes[0].Addr = net.ParseIP(m.config.BindAddr)
+	localNodes[0].Port = uint16(m.config.BindPort)
+	localNodes[0].Incarnation = 1
+	localNodes[0].State = stateAlive
+	localNodes[1].Name = "Test 1"
+	localNodes[1].Addr = net.ParseIP(m.config.BindAddr)
+	localNodes[1].Port = uint16(m.config.BindPort)
+	localNodes[1].Incarnation = 1
+	localNodes[1].State = stateAlive
+	localNodes[2].Name = "Test 2"
+	localNodes[2].Addr = net.ParseIP(m.config.BindAddr)
+	localNodes[2].Port = uint16(m.config.BindPort)
+	localNodes[2].Incarnation = 1
+	localNodes[2].State = stateAlive
+
+	// Send our node state
+	header := pushPullHeader{Nodes: 3}
+	hd := codec.MsgpackHandle{}
+	enc := codec.NewEncoder(conn, &hd)
+
+	// Send the push/pull indicator
+	conn.Write([]byte{byte(pushPullMsg)})
+
+	if err := enc.Encode(&header); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+	for i := 0; i < header.Nodes; i++ {
+		if err := enc.Encode(&localNodes[i]); err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+	}
+
+	// Read the message type
+	var msgType messageType
+	if err := binary.Read(conn, binary.BigEndian, &msgType); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	var bufConn io.Reader = conn
+	msghd := codec.MsgpackHandle{}
+	dec := codec.NewDecoder(bufConn, &msghd)
+
+	// Check if we have a compressed message
+	if msgType == compressMsg {
+		var c compress
+		if err := dec.Decode(&c); err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		decomp, err := decompressBuffer(&c)
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+
+		// Reset the message type
+		msgType = messageType(decomp[0])
+
+		// Create a new bufConn
+		bufConn = bytes.NewReader(decomp[1:])
+
+		// Create a new decoder
+		dec = codec.NewDecoder(bufConn, &hd)
+	}
+
+	// Quit if not push/pull
+	if msgType != pushPullMsg {
+		t.Fatalf("bad message type")
+	}
+
+	if err := dec.Decode(&header); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Allocate space for the transfer
+	remoteNodes := make([]pushNodeState, header.Nodes)
+
+	// Try to decode all the states
+	for i := 0; i < header.Nodes; i++ {
+		if err := dec.Decode(&remoteNodes[i]); err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+	}
+
+	if len(remoteNodes) != 1 {
+		t.Fatalf("bad response")
+	}
+
+	n := &remoteNodes[0]
+	if n.Name != "Test 0" {
+		t.Fatalf("bad name")
+	}
+	if bytes.Compare(n.Addr, net.ParseIP(m.config.BindAddr)) != 0 {
+		t.Fatal("bad addr")
+	}
+	if n.Incarnation != 0 {
+		t.Fatal("bad incarnation")
+	}
+	if n.State != stateSuspect {
+		t.Fatal("bad state")
+	}
+}
+
+func TestSendMsg_Piggyback(t *testing.T) {
+	m := GetMemberlist(t)
+	defer m.Shutdown()
+
+	// Add a message to be broadcast
+	a := alive{
+		Incarnation: 10,
+		Node:        "rand",
+		Addr:        []byte{127, 0, 0, 255},
+		Meta:        nil,
+	}
+	m.encodeAndBroadcast("rand", aliveMsg, &a)
+
+	var udp *net.UDPConn
+	for port := 60000; port < 61000; port++ {
+		udpAddr := fmt.Sprintf("127.0.0.1:%d", port)
+		udpLn, err := net.ListenPacket("udp", udpAddr)
+		if err == nil {
+			udp = udpLn.(*net.UDPConn)
+			break
+		}
+	}
+
+	// Encode a ping
+	ping := ping{SeqNo: 42}
+	buf, err := encode(pingMsg, ping)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	// Send
+	addr := &net.UDPAddr{IP: net.ParseIP(m.config.BindAddr), Port: m.config.BindPort}
+	udp.WriteTo(buf.Bytes(), addr)
+
+	// Wait for response
+	go func() {
+		time.Sleep(time.Second)
+		panic("timeout")
+	}()
+
+	in := make([]byte, 1500)
+	n, _, err := udp.ReadFrom(in)
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+	in = in[0:n]
+
+	msgType := messageType(in[0])
+	if msgType != compoundMsg {
+		t.Fatalf("bad response %v", in)
+	}
+
+	// get the parts
+	trunc, parts, err := decodeCompoundMessage(in[1:])
+	if trunc != 0 {
+		t.Fatalf("unexpected truncation")
+	}
+	if len(parts) != 2 {
+		t.Fatalf("unexpected parts %v", parts)
+	}
+	if err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	var ack ackResp
+	if err := decode(parts[0][1:], &ack); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	if ack.SeqNo != 42 {
+		t.Fatalf("bad sequence no")
+	}
+
+	var aliveout alive
+	if err := decode(parts[1][1:], &aliveout); err != nil {
+		t.Fatalf("unexpected err %s", err)
+	}
+
+	if aliveout.Node != "rand" || aliveout.Incarnation != 10 {
+		t.Fatalf("bad mesg")
+	}
+}
+
+func TestEncryptDecryptState(t *testing.T) {
+	state := []byte("this is our internal state...")
+	config := &Config{
+		SecretKey:       []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+		ProtocolVersion: ProtocolVersionMax,
+	}
+
+	m, err := Create(config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer m.Shutdown()
+
+	crypt, err := m.encryptLocalState(state)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Create reader, seek past the type byte
+	buf := bytes.NewReader(crypt)
+	buf.Seek(1, 0)
+
+	plain, err := m.decryptRemoteState(buf)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if !reflect.DeepEqual(state, plain) {
+		t.Fatalf("Decrypt failed: %v", plain)
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/queue.go

@@ -0,0 +1,167 @@
+package memberlist
+
+import (
+	"sort"
+	"sync"
+)
+
+// TransmitLimitedQueue is used to queue messages to broadcast to
+// the cluster (via gossip) but limits the number of transmits per
+// message. It also prioritizes messages with lower transmit counts
+// (hence newer messages).
+type TransmitLimitedQueue struct {
+	// NumNodes returns the number of nodes in the cluster. This is
+	// used to determine the retransmit count, which is calculated
+	// based on the log of this.
+	NumNodes func() int
+
+	// RetransmitMult is the multiplier used to determine the maximum
+	// number of retransmissions attempted.
+	RetransmitMult int
+
+	sync.Mutex
+	bcQueue limitedBroadcasts
+}
+
+type limitedBroadcast struct {
+	transmits int // Number of transmissions attempted.
+	b         Broadcast
+}
+type limitedBroadcasts []*limitedBroadcast
+
+// Broadcast is something that can be broadcasted via gossip to
+// the memberlist cluster.
+type Broadcast interface {
+	// Invalidates checks if enqueuing the current broadcast
+	// invalidates a previous broadcast
+	Invalidates(b Broadcast) bool
+
+	// Returns a byte form of the message
+	Message() []byte
+
+	// Finished is invoked when the message will no longer
+	// be broadcast, either due to invalidation or to the
+	// transmit limit being reached
+	Finished()
+}
+
+// QueueBroadcast is used to enqueue a broadcast
+func (q *TransmitLimitedQueue) QueueBroadcast(b Broadcast) {
+	q.Lock()
+	defer q.Unlock()
+
+	// Check if this message invalidates another
+	n := len(q.bcQueue)
+	for i := 0; i < n; i++ {
+		if b.Invalidates(q.bcQueue[i].b) {
+			q.bcQueue[i].b.Finished()
+			copy(q.bcQueue[i:], q.bcQueue[i+1:])
+			q.bcQueue[n-1] = nil
+			q.bcQueue = q.bcQueue[:n-1]
+			n--
+		}
+	}
+
+	// Append to the queue
+	q.bcQueue = append(q.bcQueue, &limitedBroadcast{0, b})
+}
+
+// GetBroadcasts is used to get a number of broadcasts, up to a byte limit
+// and applying a per-message overhead as provided.
+func (q *TransmitLimitedQueue) GetBroadcasts(overhead, limit int) [][]byte {
+	q.Lock()
+	defer q.Unlock()
+
+	// Fast path the default case
+	if len(q.bcQueue) == 0 {
+		return nil
+	}
+
+	transmitLimit := retransmitLimit(q.RetransmitMult, q.NumNodes())
+	bytesUsed := 0
+	var toSend [][]byte
+
+	for i := len(q.bcQueue) - 1; i >= 0; i-- {
+		// Check if this is within our limits
+		b := q.bcQueue[i]
+		msg := b.b.Message()
+		if bytesUsed+overhead+len(msg) > limit {
+			continue
+		}
+
+		// Add to slice to send
+		bytesUsed += overhead + len(msg)
+		toSend = append(toSend, msg)
+
+		// Check if we should stop transmission
+		b.transmits++
+		if b.transmits >= transmitLimit {
+			b.b.Finished()
+			n := len(q.bcQueue)
+			q.bcQueue[i], q.bcQueue[n-1] = q.bcQueue[n-1], nil
+			q.bcQueue = q.bcQueue[:n-1]
+		}
+	}
+
+	// If we are sending anything, we need to re-sort to deal
+	// with adjusted transmit counts
+	if len(toSend) > 0 {
+		q.bcQueue.Sort()
+	}
+	return toSend
+}
+
+// NumQueued returns the number of queued messages
+func (q *TransmitLimitedQueue) NumQueued() int {
+	q.Lock()
+	defer q.Unlock()
+	return len(q.bcQueue)
+}
+
+// Reset clears all the queued messages
+func (q *TransmitLimitedQueue) Reset() {
+	q.Lock()
+	defer q.Unlock()
+	for _, b := range q.bcQueue {
+		b.b.Finished()
+	}
+	q.bcQueue = nil
+}
+
+// Prune will retain the maxRetain latest messages, and the rest
+// will be discarded. This can be used to prevent unbounded queue sizes
+func (q *TransmitLimitedQueue) Prune(maxRetain int) {
+	q.Lock()
+	defer q.Unlock()
+
+	// Do nothing if queue size is less than the limit
+	n := len(q.bcQueue)
+	if n < maxRetain {
+		return
+	}
+
+	// Invalidate the messages we will be removing
+	for i := 0; i < n-maxRetain; i++ {
+		q.bcQueue[i].b.Finished()
+	}
+
+	// Move the messages, and retain only the last maxRetain
+	copy(q.bcQueue[0:], q.bcQueue[n-maxRetain:])
+	q.bcQueue = q.bcQueue[:maxRetain]
+}
+
+func (b limitedBroadcasts) Len() int {
+	return len(b)
+}
+
+func (b limitedBroadcasts) Less(i, j int) bool {
+	return b[i].transmits < b[j].transmits
+}
+
+func (b limitedBroadcasts) Swap(i, j int) {
+	b[i], b[j] = b[j], b[i]
+}
+
+func (b limitedBroadcasts) Sort() {
+	sort.Sort(sort.Reverse(b))
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/queue_test.go

@@ -0,0 +1,172 @@
+package memberlist
+
+import (
+	"testing"
+)
+
+func TestTransmitLimited_Queue(t *testing.T) {
+	q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 1 }}
+	q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil})
+	q.QueueBroadcast(&memberlistBroadcast{"foo", nil, nil})
+	q.QueueBroadcast(&memberlistBroadcast{"bar", nil, nil})
+
+	if len(q.bcQueue) != 3 {
+		t.Fatalf("bad len")
+	}
+	if q.bcQueue[0].b.(*memberlistBroadcast).node != "test" {
+		t.Fatalf("missing test")
+	}
+	if q.bcQueue[1].b.(*memberlistBroadcast).node != "foo" {
+		t.Fatalf("missing foo")
+	}
+	if q.bcQueue[2].b.(*memberlistBroadcast).node != "bar" {
+		t.Fatalf("missing bar")
+	}
+
+	// Should invalidate previous message
+	q.QueueBroadcast(&memberlistBroadcast{"test", nil, nil})
+
+	if len(q.bcQueue) != 3 {
+		t.Fatalf("bad len")
+	}
+	if q.bcQueue[0].b.(*memberlistBroadcast).node != "foo" {
+		t.Fatalf("missing foo")
+	}
+	if q.bcQueue[1].b.(*memberlistBroadcast).node != "bar" {
+		t.Fatalf("missing bar")
+	}
+	if q.bcQueue[2].b.(*memberlistBroadcast).node != "test" {
+		t.Fatalf("missing test")
+	}
+}
+
+func TestTransmitLimited_GetBroadcasts(t *testing.T) {
+	q := &TransmitLimitedQueue{RetransmitMult: 3, NumNodes: func() int { return 10 }}
+
+	// 18 bytes per message
+	q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil})
+
+	// 2 byte overhead per message, should get all 4 messages
+	all := q.GetBroadcasts(2, 80)
+	if len(all) != 4 {
+		t.Fatalf("missing messages: %v", all)
+	}
+
+	// 3 byte overhead, should only get 3 messages back
+	partial := q.GetBroadcasts(3, 80)
+	if len(partial) != 3 {
+		t.Fatalf("missing messages: %v", partial)
+	}
+}
+
+func TestTransmitLimited_GetBroadcasts_Limit(t *testing.T) {
+	q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }}
+
+	// 18 bytes per message
+	q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil})
+
+	// 3 byte overhead, should only get 3 messages back
+	partial1 := q.GetBroadcasts(3, 80)
+	if len(partial1) != 3 {
+		t.Fatalf("missing messages: %v", partial1)
+	}
+
+	partial2 := q.GetBroadcasts(3, 80)
+	if len(partial2) != 3 {
+		t.Fatalf("missing messages: %v", partial2)
+	}
+
+	// Only two not expired
+	partial3 := q.GetBroadcasts(3, 80)
+	if len(partial3) != 2 {
+		t.Fatalf("missing messages: %v", partial3)
+	}
+
+	// Should get nothing
+	partial5 := q.GetBroadcasts(3, 80)
+	if len(partial5) != 0 {
+		t.Fatalf("missing messages: %v", partial5)
+	}
+}
+
+func TestTransmitLimited_Prune(t *testing.T) {
+	q := &TransmitLimitedQueue{RetransmitMult: 1, NumNodes: func() int { return 10 }}
+
+	ch1 := make(chan struct{}, 1)
+	ch2 := make(chan struct{}, 1)
+
+	// 18 bytes per message
+	q.QueueBroadcast(&memberlistBroadcast{"test", []byte("1. this is a test."), ch1})
+	q.QueueBroadcast(&memberlistBroadcast{"foo", []byte("2. this is a test."), ch2})
+	q.QueueBroadcast(&memberlistBroadcast{"bar", []byte("3. this is a test."), nil})
+	q.QueueBroadcast(&memberlistBroadcast{"baz", []byte("4. this is a test."), nil})
+
+	// Keep only 2
+	q.Prune(2)
+
+	if q.NumQueued() != 2 {
+		t.Fatalf("bad len")
+	}
+
+	// Should notify the first two
+	select {
+	case <-ch1:
+	default:
+		t.Fatalf("expected invalidation")
+	}
+	select {
+	case <-ch2:
+	default:
+		t.Fatalf("expected invalidation")
+	}
+
+	if q.bcQueue[0].b.(*memberlistBroadcast).node != "bar" {
+		t.Fatalf("missing bar")
+	}
+	if q.bcQueue[1].b.(*memberlistBroadcast).node != "baz" {
+		t.Fatalf("missing baz")
+	}
+}
+
+func TestLimitedBroadcastSort(t *testing.T) {
+	bc := limitedBroadcasts([]*limitedBroadcast{
+		&limitedBroadcast{
+			transmits: 0,
+		},
+		&limitedBroadcast{
+			transmits: 10,
+		},
+		&limitedBroadcast{
+			transmits: 3,
+		},
+		&limitedBroadcast{
+			transmits: 4,
+		},
+		&limitedBroadcast{
+			transmits: 7,
+		},
+	})
+	bc.Sort()
+
+	if bc[0].transmits != 10 {
+		t.Fatalf("bad val %v", bc[0])
+	}
+	if bc[1].transmits != 7 {
+		t.Fatalf("bad val %v", bc[7])
+	}
+	if bc[2].transmits != 4 {
+		t.Fatalf("bad val %v", bc[2])
+	}
+	if bc[3].transmits != 3 {
+		t.Fatalf("bad val %v", bc[3])
+	}
+	if bc[4].transmits != 0 {
+		t.Fatalf("bad val %v", bc[4])
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/security.go

@@ -0,0 +1,198 @@
+package memberlist
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"fmt"
+	"io"
+)
+
+/*
+
+Encrypted messages are prefixed with an encryptionVersion byte
+that is used for us to be able to properly encode/decode. We
+currently support the following versions:
+
+ 0 - AES-GCM 128, using PKCS7 padding
+ 1 - AES-GCM 128, no padding. Padding not needed, caused bloat.
+
+*/
+type encryptionVersion uint8
+
+const (
+	minEncryptionVersion encryptionVersion = 0
+	maxEncryptionVersion encryptionVersion = 1
+)
+
+const (
+	versionSize    = 1
+	nonceSize      = 12
+	tagSize        = 16
+	maxPadOverhead = 16
+	blockSize      = aes.BlockSize
+)
+
+// pkcs7encode is used to pad a byte buffer to a specific block size using
+// the PKCS7 algorithm. "Ignores" some bytes to compensate for IV
+func pkcs7encode(buf *bytes.Buffer, ignore, blockSize int) {
+	n := buf.Len() - ignore
+	more := blockSize - (n % blockSize)
+	for i := 0; i < more; i++ {
+		buf.WriteByte(byte(more))
+	}
+}
+
+// pkcs7decode is used to decode a buffer that has been padded
+func pkcs7decode(buf []byte, blockSize int) []byte {
+	if len(buf) == 0 {
+		panic("Cannot decode a PKCS7 buffer of zero length")
+	}
+	n := len(buf)
+	last := buf[n-1]
+	n -= int(last)
+	return buf[:n]
+}
+
+// encryptOverhead returns the maximum possible overhead of encryption by version
+func encryptOverhead(vsn encryptionVersion) int {
+	switch vsn {
+	case 0:
+		return 45 // Version: 1, IV: 12, Padding: 16, Tag: 16
+	case 1:
+		return 29 // Version: 1, IV: 12, Tag: 16
+	default:
+		panic("unsupported version")
+	}
+}
+
+// encryptedLength is used to compute the buffer size needed
+// for a message of given length
+func encryptedLength(vsn encryptionVersion, inp int) int {
+	// If we are on version 1, there is no padding
+	if vsn >= 1 {
+		return versionSize + nonceSize + inp + tagSize
+	}
+
+	// Determine the padding size
+	padding := blockSize - (inp % blockSize)
+
+	// Sum the extra parts to get total size
+	return versionSize + nonceSize + inp + padding + tagSize
+}
+
+// encryptPayload is used to encrypt a message with a given key.
+// We make use of AES-128 in GCM mode. New byte buffer is the version,
+// nonce, ciphertext and tag
+func encryptPayload(vsn encryptionVersion, key []byte, msg []byte, data []byte, dst *bytes.Buffer) error {
+	// Get the AES block cipher
+	aesBlock, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	// Get the GCM cipher mode
+	gcm, err := cipher.NewGCM(aesBlock)
+	if err != nil {
+		return err
+	}
+
+	// Grow the buffer to make room for everything
+	offset := dst.Len()
+	dst.Grow(encryptedLength(vsn, len(msg)))
+
+	// Write the encryption version
+	dst.WriteByte(byte(vsn))
+
+	// Add a random nonce
+	io.CopyN(dst, rand.Reader, nonceSize)
+	afterNonce := dst.Len()
+
+	// Ensure we are correctly padded (only version 0)
+	if vsn == 0 {
+		io.Copy(dst, bytes.NewReader(msg))
+		pkcs7encode(dst, offset+versionSize+nonceSize, aes.BlockSize)
+	}
+
+	// Encrypt message using GCM
+	slice := dst.Bytes()[offset:]
+	nonce := slice[versionSize : versionSize+nonceSize]
+
+	// Message source depends on the encryption version.
+	// Version 0 uses padding, version 1 does not
+	var src []byte
+	if vsn == 0 {
+		src = slice[versionSize+nonceSize:]
+	} else {
+		src = msg
+	}
+	out := gcm.Seal(nil, nonce, src, data)
+
+	// Truncate the plaintext, and write the cipher text
+	dst.Truncate(afterNonce)
+	dst.Write(out)
+	return nil
+}
+
+// decryptMessage performs the actual decryption of ciphertext. This is in its
+// own function to allow it to be called on all keys easily.
+func decryptMessage(key, msg []byte, data []byte) ([]byte, error) {
+	// Get the AES block cipher
+	aesBlock, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the GCM cipher mode
+	gcm, err := cipher.NewGCM(aesBlock)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decrypt the message
+	nonce := msg[versionSize : versionSize+nonceSize]
+	ciphertext := msg[versionSize+nonceSize:]
+	plain, err := gcm.Open(nil, nonce, ciphertext, data)
+	if err != nil {
+		return nil, err
+	}
+
+	// Success!
+	return plain, nil
+}
+
+// decryptPayload is used to decrypt a message with a given key,
+// and verify it's contents. Any padding will be removed, and a
+// slice to the plaintext is returned. Decryption is done IN PLACE!
+func decryptPayload(keys [][]byte, msg []byte, data []byte) ([]byte, error) {
+	// Ensure we have at least one byte
+	if len(msg) == 0 {
+		return nil, fmt.Errorf("Cannot decrypt empty payload")
+	}
+
+	// Verify the version
+	vsn := encryptionVersion(msg[0])
+	if vsn > maxEncryptionVersion {
+		return nil, fmt.Errorf("Unsupported encryption version %d", msg[0])
+	}
+
+	// Ensure the length is sane
+	if len(msg) < encryptedLength(vsn, 0) {
+		return nil, fmt.Errorf("Payload is too small to decrypt: %d", len(msg))
+	}
+
+	for _, key := range keys {
+		plain, err := decryptMessage(key, msg, data)
+		if err == nil {
+			// Remove the PKCS7 padding for vsn 0
+			if vsn == 0 {
+				return pkcs7decode(plain, aes.BlockSize), nil
+			} else {
+				return plain, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("No installed keys could decrypt the message")
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/security_test.go

@@ -0,0 +1,70 @@
+package memberlist
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func TestPKCS7(t *testing.T) {
+	for i := 0; i <= 255; i++ {
+		// Make a buffer of size i
+		buf := []byte{}
+		for j := 0; j < i; j++ {
+			buf = append(buf, byte(i))
+		}
+
+		// Copy to bytes buffer
+		inp := bytes.NewBuffer(nil)
+		inp.Write(buf)
+
+		// Pad this out
+		pkcs7encode(inp, 0, 16)
+
+		// Unpad
+		dec := pkcs7decode(inp.Bytes(), 16)
+
+		// Ensure equivilence
+		if !reflect.DeepEqual(buf, dec) {
+			t.Fatalf("mismatch: %v %v", buf, dec)
+		}
+	}
+
+}
+
+func TestEncryptDecrypt_V0(t *testing.T) {
+	encryptDecryptVersioned(0, t)
+}
+
+func TestEncryptDecrypt_V1(t *testing.T) {
+	encryptDecryptVersioned(1, t)
+}
+
+func encryptDecryptVersioned(vsn encryptionVersion, t *testing.T) {
+	k1 := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}
+	plaintext := []byte("this is a plain text message")
+	extra := []byte("random data")
+
+	var buf bytes.Buffer
+	err := encryptPayload(vsn, k1, plaintext, extra, &buf)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	expLen := encryptedLength(vsn, len(plaintext))
+	if buf.Len() != expLen {
+		t.Fatalf("output length is unexpected %d %d %d", len(plaintext), buf.Len(), expLen)
+	}
+
+	msg, err := decryptPayload([][]byte{k1}, buf.Bytes(), extra)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	cmp := bytes.Compare(msg, plaintext)
+	if cmp != 0 {
+		t.Errorf("len %d %v", len(msg), msg)
+		t.Errorf("len %d %v", len(plaintext), plaintext)
+		t.Fatalf("encrypt/decrypt failed! %d '%s' '%s'", cmp, msg, plaintext)
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/state.go

@@ -0,0 +1,916 @@
+package memberlist
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"github.com/armon/go-metrics"
+)
+
+type nodeStateType int
+
+const (
+	stateAlive nodeStateType = iota
+	stateSuspect
+	stateDead
+)
+
+// Node represents a node in the cluster.
+type Node struct {
+	Name string
+	Addr net.IP
+	Port uint16
+	Meta []byte // Metadata from the delegate for this node.
+	PMin uint8  // Minimum protocol version this understands
+	PMax uint8  // Maximum protocol version this understands
+	PCur uint8  // Current version node is speaking
+	DMin uint8  // Min protocol version for the delegate to understand
+	DMax uint8  // Max protocol version for the delegate to understand
+	DCur uint8  // Current version delegate is speaking
+}
+
+// NodeState is used to manage our state view of another node
+type nodeState struct {
+	Node
+	Incarnation uint32        // Last known incarnation number
+	State       nodeStateType // Current state
+	StateChange time.Time     // Time last state change happened
+}
+
+// ackHandler is used to register handlers for incoming acks
+type ackHandler struct {
+	handler func()
+	timer   *time.Timer
+}
+
+// Schedule is used to ensure the Tick is performed periodically. This
+// function is safe to call multiple times. If the memberlist is already
+// scheduled, then it won't do anything.
+func (m *Memberlist) schedule() {
+	m.tickerLock.Lock()
+	defer m.tickerLock.Unlock()
+
+	// If we already have tickers, then don't do anything, since we're
+	// scheduled
+	if len(m.tickers) > 0 {
+		return
+	}
+
+	// Create the stop tick channel, a blocking channel. We close this
+	// when we should stop the tickers.
+	stopCh := make(chan struct{})
+
+	// Create a new probeTicker
+	if m.config.ProbeInterval > 0 {
+		t := time.NewTicker(m.config.ProbeInterval)
+		go m.triggerFunc(m.config.ProbeInterval, t.C, stopCh, m.probe)
+		m.tickers = append(m.tickers, t)
+	}
+
+	// Create a push pull ticker if needed
+	if m.config.PushPullInterval > 0 {
+		go m.pushPullTrigger(stopCh)
+	}
+
+	// Create a gossip ticker if needed
+	if m.config.GossipInterval > 0 && m.config.GossipNodes > 0 {
+		t := time.NewTicker(m.config.GossipInterval)
+		go m.triggerFunc(m.config.GossipInterval, t.C, stopCh, m.gossip)
+		m.tickers = append(m.tickers, t)
+	}
+
+	// If we made any tickers, then record the stopTick channel for
+	// later.
+	if len(m.tickers) > 0 {
+		m.stopTick = stopCh
+	}
+}
+
+// triggerFunc is used to trigger a function call each time a
+// message is received until a stop tick arrives.
+func (m *Memberlist) triggerFunc(stagger time.Duration, C <-chan time.Time, stop <-chan struct{}, f func()) {
+	// Use a random stagger to avoid syncronizing
+	randStagger := time.Duration(uint64(rand.Int63()) % uint64(stagger))
+	select {
+	case <-time.After(randStagger):
+	case <-stop:
+		return
+	}
+	for {
+		select {
+		case <-C:
+			f()
+		case <-stop:
+			return
+		}
+	}
+}
+
+// pushPullTrigger is used to periodically trigger a push/pull until
+// a stop tick arrives. We don't use triggerFunc since the push/pull
+// timer is dynamically scaled based on cluster size to avoid network
+// saturation
+func (m *Memberlist) pushPullTrigger(stop <-chan struct{}) {
+	interval := m.config.PushPullInterval
+
+	// Use a random stagger to avoid syncronizing
+	randStagger := time.Duration(uint64(rand.Int63()) % uint64(interval))
+	select {
+	case <-time.After(randStagger):
+	case <-stop:
+		return
+	}
+
+	// Tick using a dynamic timer
+	for {
+		m.nodeLock.RLock()
+		tickTime := pushPullScale(interval, len(m.nodes))
+		m.nodeLock.RUnlock()
+		select {
+		case <-time.After(tickTime):
+			m.pushPull()
+		case <-stop:
+			return
+		}
+	}
+}
+
+// Deschedule is used to stop the background maintenence. This is safe
+// to call multiple times.
+func (m *Memberlist) deschedule() {
+	m.tickerLock.Lock()
+	defer m.tickerLock.Unlock()
+
+	// If we have no tickers, then we aren't scheduled.
+	if len(m.tickers) == 0 {
+		return
+	}
+
+	// Close the stop channel so all the ticker listeners stop.
+	close(m.stopTick)
+
+	// Explicitly stop all the tickers themselves so they don't take
+	// up any more resources, and get rid of the list.
+	for _, t := range m.tickers {
+		t.Stop()
+	}
+	m.tickers = nil
+}
+
+// Tick is used to perform a single round of failure detection and gossip
+func (m *Memberlist) probe() {
+	// Track the number of indexes we've considered probing
+	numCheck := 0
+START:
+	m.nodeLock.RLock()
+
+	// Make sure we don't wrap around infinitely
+	if numCheck >= len(m.nodes) {
+		m.nodeLock.RUnlock()
+		return
+	}
+
+	// Handle the wrap around case
+	if m.probeIndex >= len(m.nodes) {
+		m.nodeLock.RUnlock()
+		m.resetNodes()
+		m.probeIndex = 0
+		numCheck++
+		goto START
+	}
+
+	// Determine if we should probe this node
+	skip := false
+	var node nodeState
+
+	node = *m.nodes[m.probeIndex]
+	if node.Name == m.config.Name {
+		skip = true
+	} else if node.State == stateDead {
+		skip = true
+	}
+
+	// Potentially skip
+	m.nodeLock.RUnlock()
+	m.probeIndex++
+	if skip {
+		numCheck++
+		goto START
+	}
+
+	// Probe the specific node
+	m.probeNode(&node)
+}
+
+// probeNode handles a single round of failure checking on a node
+func (m *Memberlist) probeNode(node *nodeState) {
+	defer metrics.MeasureSince([]string{"memberlist", "probeNode"}, time.Now())
+
+	// Send a ping to the node
+	ping := ping{SeqNo: m.nextSeqNo(), Node: node.Name}
+	destAddr := &net.UDPAddr{IP: node.Addr, Port: int(node.Port)}
+
+	// Setup an ack handler
+	ackCh := make(chan bool, m.config.IndirectChecks+1)
+	m.setAckChannel(ping.SeqNo, ackCh, m.config.ProbeInterval)
+
+	// Send the ping message
+	if err := m.encodeAndSendMsg(destAddr, pingMsg, &ping); err != nil {
+		m.logger.Printf("[ERR] memberlist: Failed to send ping: %s", err)
+		return
+	}
+
+	// Wait for response or round-trip-time
+	select {
+	case v := <-ackCh:
+		if v == true {
+			return
+		}
+
+		// As an edge case, if we get a timeout, we need to re-enqueue it
+		// here to break out of the select below
+		if v == false {
+			ackCh <- v
+		}
+	case <-time.After(m.config.ProbeTimeout):
+	}
+
+	// Get some random live nodes
+	m.nodeLock.RLock()
+	excludes := []string{m.config.Name, node.Name}
+	kNodes := kRandomNodes(m.config.IndirectChecks, excludes, m.nodes)
+	m.nodeLock.RUnlock()
+
+	// Attempt an indirect ping
+	ind := indirectPingReq{SeqNo: ping.SeqNo, Target: node.Addr, Port: node.Port, Node: node.Name}
+	for _, peer := range kNodes {
+		destAddr := &net.UDPAddr{IP: peer.Addr, Port: int(peer.Port)}
+		if err := m.encodeAndSendMsg(destAddr, indirectPingMsg, &ind); err != nil {
+			m.logger.Printf("[ERR] memberlist: Failed to send indirect ping: %s", err)
+		}
+	}
+
+	// Wait for the acks or timeout
+	select {
+	case v := <-ackCh:
+		if v == true {
+			return
+		}
+	}
+
+	// No acks received from target, suspect
+	m.logger.Printf("[INFO] memberlist: Suspect %s has failed, no acks received", node.Name)
+	s := suspect{Incarnation: node.Incarnation, Node: node.Name, From: m.config.Name}
+	m.suspectNode(&s)
+}
+
+// resetNodes is used when the tick wraps around. It will reap the
+// dead nodes and shuffle the node list.
+func (m *Memberlist) resetNodes() {
+	m.nodeLock.Lock()
+	defer m.nodeLock.Unlock()
+
+	// Move the dead nodes
+	deadIdx := moveDeadNodes(m.nodes)
+
+	// Deregister the dead nodes
+	for i := deadIdx; i < len(m.nodes); i++ {
+		delete(m.nodeMap, m.nodes[i].Name)
+		m.nodes[i] = nil
+	}
+
+	// Trim the nodes to exclude the dead nodes
+	m.nodes = m.nodes[0:deadIdx]
+
+	// Shuffle live nodes
+	shuffleNodes(m.nodes)
+}
+
+// gossip is invoked every GossipInterval period to broadcast our gossip
+// messages to a few random nodes.
+func (m *Memberlist) gossip() {
+	defer metrics.MeasureSince([]string{"memberlist", "gossip"}, time.Now())
+
+	// Get some random live nodes
+	m.nodeLock.RLock()
+	excludes := []string{m.config.Name}
+	kNodes := kRandomNodes(m.config.GossipNodes, excludes, m.nodes)
+	m.nodeLock.RUnlock()
+
+	// Compute the bytes available
+	bytesAvail := udpSendBuf - compoundHeaderOverhead
+	if m.config.EncryptionEnabled() {
+		bytesAvail -= encryptOverhead(m.encryptionVersion())
+	}
+
+	for _, node := range kNodes {
+		// Get any pending broadcasts
+		msgs := m.getBroadcasts(compoundOverhead, bytesAvail)
+		if len(msgs) == 0 {
+			return
+		}
+
+		// Create a compound message
+		compound := makeCompoundMessage(msgs)
+
+		// Send the compound message
+		destAddr := &net.UDPAddr{IP: node.Addr, Port: int(node.Port)}
+		if err := m.rawSendMsg(destAddr, compound.Bytes()); err != nil {
+			m.logger.Printf("[ERR] memberlist: Failed to send gossip to %s: %s", destAddr, err)
+		}
+	}
+}
+
+// pushPull is invoked periodically to randomly perform a complete state
+// exchange. Used to ensure a high level of convergence, but is also
+// reasonably expensive as the entire state of this node is exchanged
+// with the other node.
+func (m *Memberlist) pushPull() {
+	// Get a random live node
+	m.nodeLock.RLock()
+	excludes := []string{m.config.Name}
+	nodes := kRandomNodes(1, excludes, m.nodes)
+	m.nodeLock.RUnlock()
+
+	// If no nodes, bail
+	if len(nodes) == 0 {
+		return
+	}
+	node := nodes[0]
+
+	// Attempt a push pull
+	if err := m.pushPullNode(node.Addr, node.Port, false); err != nil {
+		m.logger.Printf("[ERR] memberlist: Push/Pull with %s failed: %s", node.Name, err)
+	}
+}
+
+// pushPullNode does a complete state exchange with a specific node.
+func (m *Memberlist) pushPullNode(addr []byte, port uint16, join bool) error {
+	defer metrics.MeasureSince([]string{"memberlist", "pushPullNode"}, time.Now())
+
+	// Attempt to send and receive with the node
+	remote, userState, err := m.sendAndReceiveState(addr, port, join)
+	if err != nil {
+		return err
+	}
+
+	if err := m.verifyProtocol(remote); err != nil {
+		return err
+	}
+
+	// Invoke the merge delegate if any
+	if join && m.config.Merge != nil {
+		nodes := make([]*Node, len(remote))
+		for idx, n := range remote {
+			nodes[idx] = &Node{
+				Name: n.Name,
+				Addr: n.Addr,
+				Port: n.Port,
+				Meta: n.Meta,
+				PMin: n.Vsn[0],
+				PMax: n.Vsn[1],
+				PCur: n.Vsn[2],
+				DMin: n.Vsn[3],
+				DMax: n.Vsn[4],
+				DCur: n.Vsn[5],
+			}
+		}
+		if m.config.Merge.NotifyMerge(nodes) {
+			m.logger.Printf("[WARN] memberlist: Cluster merge canceled")
+			return fmt.Errorf("Merge canceled")
+		}
+	}
+
+	// Merge the state
+	m.mergeState(remote)
+
+	// Invoke the delegate
+	if m.config.Delegate != nil {
+		m.config.Delegate.MergeRemoteState(userState, join)
+	}
+	return nil
+}
+
+// verifyProtocol verifies that all the remote nodes can speak with our
+// nodes and vice versa on both the core protocol as well as the
+// delegate protocol level.
+//
+// The verification works by finding the maximum minimum and
+// minimum maximum understood protocol and delegate versions. In other words,
+// it finds the common denominator of protocol and delegate version ranges
+// for the entire cluster.
+//
+// After this, it goes through the entire cluster (local and remote) and
+// verifies that everyone's speaking protocol versions satisfy this range.
+// If this passes, it means that every node can understand each other.
+func (m *Memberlist) verifyProtocol(remote []pushNodeState) error {
+	m.nodeLock.RLock()
+	defer m.nodeLock.RUnlock()
+
+	// Maximum minimum understood and minimum maximum understood for both
+	// the protocol and delegate versions. We use this to verify everyone
+	// can be understood.
+	var maxpmin, minpmax uint8
+	var maxdmin, mindmax uint8
+	minpmax = math.MaxUint8
+	mindmax = math.MaxUint8
+
+	for _, rn := range remote {
+		// If the node isn't alive, then skip it
+		if rn.State != stateAlive {
+			continue
+		}
+
+		// Skip nodes that don't have versions set, it just means
+		// their version is zero.
+		if len(rn.Vsn) == 0 {
+			continue
+		}
+
+		if rn.Vsn[0] > maxpmin {
+			maxpmin = rn.Vsn[0]
+		}
+
+		if rn.Vsn[1] < minpmax {
+			minpmax = rn.Vsn[1]
+		}
+
+		if rn.Vsn[3] > maxdmin {
+			maxdmin = rn.Vsn[3]
+		}
+
+		if rn.Vsn[4] < mindmax {
+			mindmax = rn.Vsn[4]
+		}
+	}
+
+	for _, n := range m.nodes {
+		// Ignore non-alive nodes
+		if n.State != stateAlive {
+			continue
+		}
+
+		if n.PMin > maxpmin {
+			maxpmin = n.PMin
+		}
+
+		if n.PMax < minpmax {
+			minpmax = n.PMax
+		}
+
+		if n.DMin > maxdmin {
+			maxdmin = n.DMin
+		}
+
+		if n.DMax < mindmax {
+			mindmax = n.DMax
+		}
+	}
+
+	// Now that we definitively know the minimum and maximum understood
+	// version that satisfies the whole cluster, we verify that every
+	// node in the cluster satisifies this.
+	for _, n := range remote {
+		var nPCur, nDCur uint8
+		if len(n.Vsn) > 0 {
+			nPCur = n.Vsn[2]
+			nDCur = n.Vsn[5]
+		}
+
+		if nPCur < maxpmin || nPCur > minpmax {
+			return fmt.Errorf(
+				"Node '%s' protocol version (%d) is incompatible: [%d, %d]",
+				n.Name, nPCur, maxpmin, minpmax)
+		}
+
+		if nDCur < maxdmin || nDCur > mindmax {
+			return fmt.Errorf(
+				"Node '%s' delegate protocol version (%d) is incompatible: [%d, %d]",
+				n.Name, nDCur, maxdmin, mindmax)
+		}
+	}
+
+	for _, n := range m.nodes {
+		nPCur := n.PCur
+		nDCur := n.DCur
+
+		if nPCur < maxpmin || nPCur > minpmax {
+			return fmt.Errorf(
+				"Node '%s' protocol version (%d) is incompatible: [%d, %d]",
+				n.Name, nPCur, maxpmin, minpmax)
+		}
+
+		if nDCur < maxdmin || nDCur > mindmax {
+			return fmt.Errorf(
+				"Node '%s' delegate protocol version (%d) is incompatible: [%d, %d]",
+				n.Name, nDCur, maxdmin, mindmax)
+		}
+	}
+
+	return nil
+}
+
+// nextSeqNo returns a usable sequence number in a thread safe way
+func (m *Memberlist) nextSeqNo() uint32 {
+	return atomic.AddUint32(&m.sequenceNum, 1)
+}
+
+// nextIncarnation returns the next incarnation number in a thread safe way
+func (m *Memberlist) nextIncarnation() uint32 {
+	return atomic.AddUint32(&m.incarnation, 1)
+}
+
+// setAckChannel is used to attach a channel to receive a message when
+// an ack with a given sequence number is received. The channel gets sent
+// false on timeout
+func (m *Memberlist) setAckChannel(seqNo uint32, ch chan bool, timeout time.Duration) {
+	// Create a handler function
+	handler := func() {
+		select {
+		case ch <- true:
+		default:
+		}
+	}
+
+	// Add the handler
+	ah := &ackHandler{handler, nil}
+	m.ackLock.Lock()
+	m.ackHandlers[seqNo] = ah
+	m.ackLock.Unlock()
+
+	// Setup a reaping routing
+	ah.timer = time.AfterFunc(timeout, func() {
+		m.ackLock.Lock()
+		delete(m.ackHandlers, seqNo)
+		m.ackLock.Unlock()
+		select {
+		case ch <- false:
+		default:
+		}
+	})
+}
+
+// setAckHandler is used to attach a handler to be invoked when an
+// ack with a given sequence number is received. If a timeout is reached,
+// the handler is deleted
+func (m *Memberlist) setAckHandler(seqNo uint32, handler func(), timeout time.Duration) {
+	// Add the handler
+	ah := &ackHandler{handler, nil}
+	m.ackLock.Lock()
+	m.ackHandlers[seqNo] = ah
+	m.ackLock.Unlock()
+
+	// Setup a reaping routing
+	ah.timer = time.AfterFunc(timeout, func() {
+		m.ackLock.Lock()
+		delete(m.ackHandlers, seqNo)
+		m.ackLock.Unlock()
+	})
+}
+
+// Invokes an Ack handler if any is associated, and reaps the handler immediately
+func (m *Memberlist) invokeAckHandler(seqNo uint32) {
+	m.ackLock.Lock()
+	ah, ok := m.ackHandlers[seqNo]
+	delete(m.ackHandlers, seqNo)
+	m.ackLock.Unlock()
+	if !ok {
+		return
+	}
+	ah.timer.Stop()
+	ah.handler()
+}
+
+// aliveNode is invoked by the network layer when we get a message about a
+// live node.
+func (m *Memberlist) aliveNode(a *alive, notify chan struct{}, bootstrap bool) {
+	m.nodeLock.Lock()
+	defer m.nodeLock.Unlock()
+	state, ok := m.nodeMap[a.Node]
+
+	// It is possible that during a Leave(), there is already an aliveMsg
+	// in-queue to be processed but blocked by the locks above. If we let
+	// that aliveMsg process, it'll cause us to re-join the cluster. This
+	// ensures that we don't.
+	if m.leave && a.Node == m.config.Name {
+		return
+	}
+
+	// Check if we've never seen this node before, and if not, then
+	// store this node in our node map.
+	if !ok {
+		state = &nodeState{
+			Node: Node{
+				Name: a.Node,
+				Addr: a.Addr,
+				Port: a.Port,
+				Meta: a.Meta,
+			},
+			State: stateDead,
+		}
+
+		// Add to map
+		m.nodeMap[a.Node] = state
+
+		// Get a random offset. This is important to ensure
+		// the failure detection bound is low on average. If all
+		// nodes did an append, failure detection bound would be
+		// very high.
+		n := len(m.nodes)
+		offset := randomOffset(n)
+
+		// Add at the end and swap with the node at the offset
+		m.nodes = append(m.nodes, state)
+		m.nodes[offset], m.nodes[n] = m.nodes[n], m.nodes[offset]
+	}
+
+	// Check if this address is different than the existing node
+	if !bytes.Equal([]byte(state.Addr), a.Addr) || state.Port != a.Port {
+		m.logger.Printf("[ERR] memberlist: Conflicting address for %s. Mine: %v:%d Theirs: %v:%d",
+			state.Name, state.Addr, state.Port, net.IP(a.Addr), a.Port)
+
+		// Inform the conflict delegate if provided
+		if m.config.Conflict != nil {
+			other := Node{
+				Name: a.Node,
+				Addr: a.Addr,
+				Port: a.Port,
+				Meta: a.Meta,
+			}
+			m.config.Conflict.NotifyConflict(&state.Node, &other)
+		}
+		return
+	}
+
+	// Bail if the incarnation number is older, and this is not about us
+	isLocalNode := state.Name == m.config.Name
+	if a.Incarnation <= state.Incarnation && !isLocalNode {
+		return
+	}
+
+	// Bail if strictly less and this is about us
+	if a.Incarnation < state.Incarnation && isLocalNode {
+		return
+	}
+
+	// Update metrics
+	metrics.IncrCounter([]string{"memberlist", "msg", "alive"}, 1)
+
+	// Store the old state and meta data
+	oldState := state.State
+	oldMeta := state.Meta
+
+	// If this is us we need to refute, otherwise re-broadcast
+	if !bootstrap && isLocalNode {
+		// Compute the version vector
+		versions := []uint8{
+			state.PMin, state.PMax, state.PCur,
+			state.DMin, state.DMax, state.DCur,
+		}
+
+		// If the Incarnation is the same, we need special handling, since it
+		// possible for the following situation to happen:
+		// 1) Start with configuration C, join cluster
+		// 2) Hard fail / Kill / Shutdown
+		// 3) Restart with configuration C', join cluster
+		//
+		// In this case, other nodes and the local node see the same incarnation,
+		// but the values may not be the same. For this reason, we always
+		// need to do an equality check for this Incarnation. In most cases,
+		// we just ignore, but we may need to refute.
+		//
+		if a.Incarnation == state.Incarnation &&
+			bytes.Equal(a.Meta, state.Meta) &&
+			bytes.Equal(a.Vsn, versions) {
+			return
+		}
+
+		inc := m.nextIncarnation()
+		for a.Incarnation >= inc {
+			inc = m.nextIncarnation()
+		}
+		state.Incarnation = inc
+
+		a := alive{
+			Incarnation: inc,
+			Node:        state.Name,
+			Addr:        state.Addr,
+			Port:        state.Port,
+			Meta:        state.Meta,
+			Vsn:         versions,
+		}
+		m.encodeBroadcastNotify(a.Node, aliveMsg, a, notify)
+		m.logger.Printf("[WARN] memberlist: Refuting an alive message")
+	} else {
+		m.encodeBroadcastNotify(a.Node, aliveMsg, a, notify)
+
+		// Update protocol versions if it arrived
+		if len(a.Vsn) > 0 {
+			state.PMin = a.Vsn[0]
+			state.PMax = a.Vsn[1]
+			state.PCur = a.Vsn[2]
+			state.DMin = a.Vsn[3]
+			state.DMax = a.Vsn[4]
+			state.DCur = a.Vsn[5]
+		}
+
+		// Update the state and incarnation number
+		state.Incarnation = a.Incarnation
+		state.Meta = a.Meta
+		if state.State != stateAlive {
+			state.State = stateAlive
+			state.StateChange = time.Now()
+		}
+	}
+
+	// Notify the delegate of any relevant updates
+	if m.config.Events != nil {
+		if oldState == stateDead {
+			// if Dead -> Alive, notify of join
+			m.config.Events.NotifyJoin(&state.Node)
+
+		} else if !bytes.Equal(oldMeta, state.Meta) {
+			// if Meta changed, trigger an update notification
+			m.config.Events.NotifyUpdate(&state.Node)
+		}
+	}
+}
+
+// suspectNode is invoked by the network layer when we get a message
+// about a suspect node
+func (m *Memberlist) suspectNode(s *suspect) {
+	m.nodeLock.Lock()
+	defer m.nodeLock.Unlock()
+	state, ok := m.nodeMap[s.Node]
+
+	// If we've never heard about this node before, ignore it
+	if !ok {
+		return
+	}
+
+	// Ignore old incarnation numbers
+	if s.Incarnation < state.Incarnation {
+		return
+	}
+
+	// Ignore non-alive nodes
+	if state.State != stateAlive {
+		return
+	}
+
+	// If this is us we need to refute, otherwise re-broadcast
+	if state.Name == m.config.Name {
+		inc := m.nextIncarnation()
+		for s.Incarnation >= inc {
+			inc = m.nextIncarnation()
+		}
+		state.Incarnation = inc
+
+		a := alive{
+			Incarnation: inc,
+			Node:        state.Name,
+			Addr:        state.Addr,
+			Port:        state.Port,
+			Meta:        state.Meta,
+			Vsn: []uint8{
+				state.PMin, state.PMax, state.PCur,
+				state.DMin, state.DMax, state.DCur,
+			},
+		}
+		m.encodeAndBroadcast(s.Node, aliveMsg, a)
+		m.logger.Printf("[WARN] memberlist: Refuting a suspect message (from: %s)", s.From)
+		return // Do not mark ourself suspect
+	} else {
+		m.encodeAndBroadcast(s.Node, suspectMsg, s)
+	}
+
+	// Update metrics
+	metrics.IncrCounter([]string{"memberlist", "msg", "suspect"}, 1)
+
+	// Update the state
+	state.Incarnation = s.Incarnation
+	state.State = stateSuspect
+	changeTime := time.Now()
+	state.StateChange = changeTime
+
+	// Setup a timeout for this
+	timeout := suspicionTimeout(m.config.SuspicionMult, len(m.nodes), m.config.ProbeInterval)
+	time.AfterFunc(timeout, func() {
+		m.nodeLock.Lock()
+		state, ok := m.nodeMap[s.Node]
+		timeout := ok && state.State == stateSuspect && state.StateChange == changeTime
+		m.nodeLock.Unlock()
+
+		if timeout {
+			m.suspectTimeout(state)
+		}
+	})
+}
+
+// suspectTimeout is invoked when a suspect timeout has occurred
+func (m *Memberlist) suspectTimeout(n *nodeState) {
+	// Construct a dead message
+	m.logger.Printf("[INFO] memberlist: Marking %s as failed, suspect timeout reached", n.Name)
+	d := dead{Incarnation: n.Incarnation, Node: n.Name, From: m.config.Name}
+	m.deadNode(&d)
+}
+
+// deadNode is invoked by the network layer when we get a message
+// about a dead node
+func (m *Memberlist) deadNode(d *dead) {
+	m.nodeLock.Lock()
+	defer m.nodeLock.Unlock()
+	state, ok := m.nodeMap[d.Node]
+
+	// If we've never heard about this node before, ignore it
+	if !ok {
+		return
+	}
+
+	// Ignore old incarnation numbers
+	if d.Incarnation < state.Incarnation {
+		return
+	}
+
+	// Ignore if node is already dead
+	if state.State == stateDead {
+		return
+	}
+
+	// Check if this is us
+	if state.Name == m.config.Name {
+		// If we are not leaving we need to refute
+		if !m.leave {
+			inc := m.nextIncarnation()
+			for d.Incarnation >= inc {
+				inc = m.nextIncarnation()
+			}
+			state.Incarnation = inc
+
+			a := alive{
+				Incarnation: inc,
+				Node:        state.Name,
+				Addr:        state.Addr,
+				Port:        state.Port,
+				Meta:        state.Meta,
+				Vsn: []uint8{
+					state.PMin, state.PMax, state.PCur,
+					state.DMin, state.DMax, state.DCur,
+				},
+			}
+			m.encodeAndBroadcast(d.Node, aliveMsg, a)
+			m.logger.Printf("[WARN] memberlist: Refuting a dead message (from: %s)", d.From)
+			return // Do not mark ourself dead
+		}
+
+		// If we are leaving, we broadcast and wait
+		m.encodeBroadcastNotify(d.Node, deadMsg, d, m.leaveBroadcast)
+	} else {
+		m.encodeAndBroadcast(d.Node, deadMsg, d)
+	}
+
+	// Update metrics
+	metrics.IncrCounter([]string{"memberlist", "msg", "dead"}, 1)
+
+	// Update the state
+	state.Incarnation = d.Incarnation
+	state.State = stateDead
+	state.StateChange = time.Now()
+
+	// Notify of death
+	if m.config.Events != nil {
+		m.config.Events.NotifyLeave(&state.Node)
+	}
+}
+
+// mergeState is invoked by the network layer when we get a Push/Pull
+// state transfer
+func (m *Memberlist) mergeState(remote []pushNodeState) {
+	for _, r := range remote {
+		switch r.State {
+		case stateAlive:
+			a := alive{
+				Incarnation: r.Incarnation,
+				Node:        r.Name,
+				Addr:        r.Addr,
+				Port:        r.Port,
+				Meta:        r.Meta,
+				Vsn:         r.Vsn,
+			}
+			m.aliveNode(&a, nil, false)
+
+		case stateDead:
+			// If the remote node belives a node is dead, we prefer to
+			// suspect that node instead of declaring it dead instantly
+			fallthrough
+		case stateSuspect:
+			s := suspect{Incarnation: r.Incarnation, Node: r.Name, From: m.config.Name}
+			m.suspectNode(&s)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/test/setup_subnet.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# This script makes sure that 127.0.0.x is routable. On Darwin, there
+# is a bug that it isn't routable and this causes errors.
+#
+
+# Check if loopback is setup
+ping -c 1 -W 10 127.0.0.2 > /dev/null 2>&1
+if [ $? -eq 0 ]
+then
+    exit
+fi
+
+# If we're not on OS X, then error
+case $OSTYPE in
+    darwin*)
+        ;;
+    *)
+        echo "Can't setup interfaces on non-Mac. Error!"
+        exit 1
+        ;;
+esac
+
+# Setup loopback
+for ((i=2;i<256;i++))
+do
+    sudo ifconfig lo0 alias 127.0.0.$i up
+done

Godeps/_workspace/src/github.com/hashicorp/memberlist/todo.md

@@ -0,0 +1,6 @@
+# TODO
+* Dynamic RTT discovery
+    * Compute 99th percentile for ping/ack
+    * Better lower bound for ping/ack, faster failure detection
+* Dynamic MTU discovery
+    * Prevent lost updates, increases efficiency

Godeps/_workspace/src/github.com/hashicorp/memberlist/util.go

@@ -0,0 +1,325 @@
+package memberlist
+
+import (
+	"bytes"
+	"compress/lzw"
+	"encoding/binary"
+	"fmt"
+	"github.com/hashicorp/go-msgpack/codec"
+	"io"
+	"math"
+	"math/rand"
+	"net"
+	"time"
+)
+
+// pushPullScale is the minimum number of nodes
+// before we start scaling the push/pull timing. The scale
+// effect is the log2(Nodes) - log2(pushPullScale). This means
+// that the 33rd node will cause us to double the interval,
+// while the 65th will triple it.
+const pushPullScaleThreshold = 32
+
+/*
+ * Contains an entry for each private block:
+ * 10.0.0.0/8
+ * 172.16.0.0/12
+ * 192.168/16
+ */
+var privateBlocks []*net.IPNet
+
+var loopbackBlock *net.IPNet
+
+const (
+	// Constant litWidth 2-8
+	lzwLitWidth = 8
+)
+
+func init() {
+	// Seed the random number generator
+	rand.Seed(time.Now().UnixNano())
+
+	// Add each private block
+	privateBlocks = make([]*net.IPNet, 3)
+	_, block, err := net.ParseCIDR("10.0.0.0/8")
+	if err != nil {
+		panic(fmt.Sprintf("Bad cidr. Got %v", err))
+	}
+	privateBlocks[0] = block
+
+	_, block, err = net.ParseCIDR("172.16.0.0/12")
+	if err != nil {
+		panic(fmt.Sprintf("Bad cidr. Got %v", err))
+	}
+	privateBlocks[1] = block
+
+	_, block, err = net.ParseCIDR("192.168.0.0/16")
+	if err != nil {
+		panic(fmt.Sprintf("Bad cidr. Got %v", err))
+	}
+	privateBlocks[2] = block
+
+	_, block, err = net.ParseCIDR("127.0.0.0/8")
+	if err != nil {
+		panic(fmt.Sprintf("Bad cidr. Got %v", err))
+	}
+	loopbackBlock = block
+}
+
+// Decode reverses the encode operation on a byte slice input
+func decode(buf []byte, out interface{}) error {
+	r := bytes.NewReader(buf)
+	hd := codec.MsgpackHandle{}
+	dec := codec.NewDecoder(r, &hd)
+	return dec.Decode(out)
+}
+
+// Encode writes an encoded object to a new bytes buffer
+func encode(msgType messageType, in interface{}) (*bytes.Buffer, error) {
+	buf := bytes.NewBuffer(nil)
+	buf.WriteByte(uint8(msgType))
+	hd := codec.MsgpackHandle{}
+	enc := codec.NewEncoder(buf, &hd)
+	err := enc.Encode(in)
+	return buf, err
+}
+
+// Returns a random offset between 0 and n
+func randomOffset(n int) int {
+	if n == 0 {
+		return 0
+	}
+	return int(rand.Uint32() % uint32(n))
+}
+
+// suspicionTimeout computes the timeout that should be used when
+// a node is suspected
+func suspicionTimeout(suspicionMult, n int, interval time.Duration) time.Duration {
+	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
+	timeout := time.Duration(suspicionMult) * time.Duration(nodeScale) * interval
+	return timeout
+}
+
+// retransmitLimit computes the limit of retransmissions
+func retransmitLimit(retransmitMult, n int) int {
+	nodeScale := math.Ceil(math.Log10(float64(n + 1)))
+	limit := retransmitMult * int(nodeScale)
+	return limit
+}
+
+// shuffleNodes randomly shuffles the input nodes
+func shuffleNodes(nodes []*nodeState) {
+	for i := range nodes {
+		j := rand.Intn(i + 1)
+		nodes[i], nodes[j] = nodes[j], nodes[i]
+	}
+}
+
+// pushPushScale is used to scale the time interval at which push/pull
+// syncs take place. It is used to prevent network saturation as the
+// cluster size grows
+func pushPullScale(interval time.Duration, n int) time.Duration {
+	// Don't scale until we cross the threshold
+	if n <= pushPullScaleThreshold {
+		return interval
+	}
+
+	multiplier := math.Ceil(math.Log2(float64(n))-math.Log2(pushPullScaleThreshold)) + 1.0
+	return time.Duration(multiplier) * interval
+}
+
+// moveDeadNodes moves all the nodes in the dead state
+// to the end of the slice and returns the index of the first dead node.
+func moveDeadNodes(nodes []*nodeState) int {
+	numDead := 0
+	n := len(nodes)
+	for i := 0; i < n-numDead; i++ {
+		if nodes[i].State != stateDead {
+			continue
+		}
+
+		// Move this node to the end
+		nodes[i], nodes[n-numDead-1] = nodes[n-numDead-1], nodes[i]
+		numDead++
+		i--
+	}
+	return n - numDead
+}
+
+// kRandomNodes is used to select up to k random nodes, excluding a given
+// node and any non-alive nodes. It is possible that less than k nodes are returned.
+func kRandomNodes(k int, excludes []string, nodes []*nodeState) []*nodeState {
+	n := len(nodes)
+	kNodes := make([]*nodeState, 0, k)
+OUTER:
+	// Probe up to 3*n times, with large n this is not necessary
+	// since k << n, but with small n we want search to be
+	// exhaustive
+	for i := 0; i < 3*n && len(kNodes) < k; i++ {
+		// Get random node
+		idx := randomOffset(n)
+		node := nodes[idx]
+
+		// Exclude node if match
+		for _, exclude := range excludes {
+			if node.Name == exclude {
+				continue OUTER
+			}
+		}
+
+		// Exclude if not alive
+		if node.State != stateAlive {
+			continue
+		}
+
+		// Check if we have this node already
+		for j := 0; j < len(kNodes); j++ {
+			if node == kNodes[j] {
+				continue OUTER
+			}
+		}
+
+		// Append the node
+		kNodes = append(kNodes, node)
+	}
+	return kNodes
+}
+
+// makeCompoundMessage takes a list of messages and generates
+// a single compound message containing all of them
+func makeCompoundMessage(msgs [][]byte) *bytes.Buffer {
+	// Create a local buffer
+	buf := bytes.NewBuffer(nil)
+
+	// Write out the type
+	buf.WriteByte(uint8(compoundMsg))
+
+	// Write out the number of message
+	buf.WriteByte(uint8(len(msgs)))
+
+	// Add the message lengths
+	for _, m := range msgs {
+		binary.Write(buf, binary.BigEndian, uint16(len(m)))
+	}
+
+	// Append the messages
+	for _, m := range msgs {
+		buf.Write(m)
+	}
+
+	return buf
+}
+
+// decodeCompoundMessage splits a compound message and returns
+// the slices of individual messages. Also returns the number
+// of truncated messages and any potential error
+func decodeCompoundMessage(buf []byte) (trunc int, parts [][]byte, err error) {
+	if len(buf) < 1 {
+		err = fmt.Errorf("missing compound length byte")
+		return
+	}
+	numParts := uint8(buf[0])
+	buf = buf[1:]
+
+	// Check we have enough bytes
+	if len(buf) < int(numParts*2) {
+		err = fmt.Errorf("truncated len slice")
+		return
+	}
+
+	// Decode the lengths
+	lengths := make([]uint16, numParts)
+	for i := 0; i < int(numParts); i++ {
+		lengths[i] = binary.BigEndian.Uint16(buf[i*2 : i*2+2])
+	}
+	buf = buf[numParts*2:]
+
+	// Split each message
+	for idx, msgLen := range lengths {
+		if len(buf) < int(msgLen) {
+			trunc = int(numParts) - idx
+			return
+		}
+
+		// Extract the slice, seek past on the buffer
+		slice := buf[:msgLen]
+		buf = buf[msgLen:]
+		parts = append(parts, slice)
+	}
+	return
+}
+
+// Returns if the given IP is in a private block
+func isPrivateIP(ip_str string) bool {
+	ip := net.ParseIP(ip_str)
+	for _, priv := range privateBlocks {
+		if priv.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// Returns if the given IP is in a loopback block
+func isLoopbackIP(ip_str string) bool {
+	ip := net.ParseIP(ip_str)
+	return loopbackBlock.Contains(ip)
+}
+
+// compressPayload takes an opaque input buffer, compresses it
+// and wraps it in a compress{} message that is encoded.
+func compressPayload(inp []byte) (*bytes.Buffer, error) {
+	var buf bytes.Buffer
+	compressor := lzw.NewWriter(&buf, lzw.LSB, lzwLitWidth)
+
+	_, err := compressor.Write(inp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Ensure we flush everything out
+	if err := compressor.Close(); err != nil {
+		return nil, err
+	}
+
+	// Create a compressed message
+	c := compress{
+		Algo: lzwAlgo,
+		Buf:  buf.Bytes(),
+	}
+	return encode(compressMsg, &c)
+}
+
+// decompressPayload is used to unpack an encoded compress{}
+// message and return its payload uncompressed
+func decompressPayload(msg []byte) ([]byte, error) {
+	// Decode the message
+	var c compress
+	if err := decode(msg, &c); err != nil {
+		return nil, err
+	}
+	return decompressBuffer(&c)
+}
+
+// decompressBuffer is used to decompress the buffer of
+// a single compress message, handling multiple algorithms
+func decompressBuffer(c *compress) ([]byte, error) {
+	// Verify the algorithm
+	if c.Algo != lzwAlgo {
+		return nil, fmt.Errorf("Cannot decompress unknown algorithm %d", c.Algo)
+	}
+
+	// Create a uncompressor
+	uncomp := lzw.NewReader(bytes.NewReader(c.Buf), lzw.LSB, lzwLitWidth)
+	defer uncomp.Close()
+
+	// Read all the data
+	var b bytes.Buffer
+	_, err := io.Copy(&b, uncomp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return the uncompressed bytes
+	return b.Bytes(), nil
+}

Godeps/_workspace/src/github.com/hashicorp/memberlist/util_test.go

@@ -0,0 +1,293 @@
+package memberlist
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestEncodeDecode(t *testing.T) {
+	msg := &ping{SeqNo: 100}
+	buf, err := encode(pingMsg, msg)
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+	var out ping
+	if err := decode(buf.Bytes()[1:], &out); err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+	if msg.SeqNo != out.SeqNo {
+		t.Fatalf("bad sequence no")
+	}
+}
+
+func TestRandomOffset(t *testing.T) {
+	vals := make(map[int]struct{})
+	for i := 0; i < 100; i++ {
+		offset := randomOffset(2 << 30)
+		if _, ok := vals[offset]; ok {
+			t.Fatalf("got collision")
+		}
+		vals[offset] = struct{}{}
+	}
+}
+
+func TestRandomOffset_Zero(t *testing.T) {
+	offset := randomOffset(0)
+	if offset != 0 {
+		t.Fatalf("bad offset")
+	}
+}
+
+func TestSuspicionTimeout(t *testing.T) {
+	timeout := suspicionTimeout(3, 10, time.Second)
+	if timeout != 6*time.Second {
+		t.Fatalf("bad timeout")
+	}
+}
+
+func TestRetransmitLimit(t *testing.T) {
+	lim := retransmitLimit(3, 0)
+	if lim != 0 {
+		t.Fatalf("bad val %v", lim)
+	}
+	lim = retransmitLimit(3, 1)
+	if lim != 3 {
+		t.Fatalf("bad val %v", lim)
+	}
+	lim = retransmitLimit(3, 99)
+	if lim != 6 {
+		t.Fatalf("bad val %v", lim)
+	}
+}
+
+func TestShuffleNodes(t *testing.T) {
+	orig := []*nodeState{
+		&nodeState{
+			State: stateDead,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateDead,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateDead,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+	}
+	nodes := make([]*nodeState, len(orig))
+	copy(nodes[:], orig[:])
+
+	if !reflect.DeepEqual(nodes, orig) {
+		t.Fatalf("should match")
+	}
+
+	shuffleNodes(nodes)
+
+	if reflect.DeepEqual(nodes, orig) {
+		t.Fatalf("should not match")
+	}
+}
+
+func TestPushPullScale(t *testing.T) {
+	sec := time.Second
+	for i := 0; i <= 32; i++ {
+		if s := pushPullScale(sec, i); s != sec {
+			t.Fatalf("Bad time scale: %v", s)
+		}
+	}
+	for i := 33; i <= 64; i++ {
+		if s := pushPullScale(sec, i); s != 2*sec {
+			t.Fatalf("Bad time scale: %v", s)
+		}
+	}
+	for i := 65; i <= 128; i++ {
+		if s := pushPullScale(sec, i); s != 3*sec {
+			t.Fatalf("Bad time scale: %v", s)
+		}
+	}
+}
+
+func TestMoveDeadNodes(t *testing.T) {
+	nodes := []*nodeState{
+		&nodeState{
+			State: stateDead,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+		&nodeState{
+			State: stateDead,
+		},
+		&nodeState{
+			State: stateAlive,
+		},
+	}
+
+	idx := moveDeadNodes(nodes)
+	if idx != 3 {
+		t.Fatalf("bad index")
+	}
+	for i := 0; i < idx; i++ {
+		if nodes[i].State != stateAlive {
+			t.Fatalf("Bad state %d", i)
+		}
+	}
+	for i := idx; i < len(nodes); i++ {
+		if nodes[i].State != stateDead {
+			t.Fatalf("Bad state %d", i)
+		}
+	}
+}
+
+func TestKRandomNodes(t *testing.T) {
+	nodes := []*nodeState{}
+	for i := 0; i < 90; i++ {
+		// Half the nodes are in a bad state
+		state := stateAlive
+		switch i % 3 {
+		case 0:
+			state = stateAlive
+		case 1:
+			state = stateSuspect
+		case 2:
+			state = stateDead
+		}
+		nodes = append(nodes, &nodeState{
+			Node: Node{
+				Name: fmt.Sprintf("test%d", i),
+			},
+			State: state,
+		})
+	}
+
+	s1 := kRandomNodes(3, []string{"test0"}, nodes)
+	s2 := kRandomNodes(3, []string{"test0"}, nodes)
+	s3 := kRandomNodes(3, []string{"test0"}, nodes)
+
+	if reflect.DeepEqual(s1, s2) {
+		t.Fatalf("unexpected equal")
+	}
+	if reflect.DeepEqual(s1, s3) {
+		t.Fatalf("unexpected equal")
+	}
+	if reflect.DeepEqual(s2, s3) {
+		t.Fatalf("unexpected equal")
+	}
+
+	for _, s := range [][]*nodeState{s1, s2, s3} {
+		if len(s) != 3 {
+			t.Fatalf("bad len")
+		}
+		for _, n := range s {
+			if n.Name == "test0" {
+				t.Fatalf("Bad name")
+			}
+			if n.State != stateAlive {
+				t.Fatalf("Bad state")
+			}
+		}
+	}
+}
+
+func TestMakeCompoundMessage(t *testing.T) {
+	msg := &ping{SeqNo: 100}
+	buf, err := encode(pingMsg, msg)
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+
+	msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()}
+	compound := makeCompoundMessage(msgs)
+
+	if compound.Len() != 3*buf.Len()+3*compoundOverhead+compoundHeaderOverhead {
+		t.Fatalf("bad len")
+	}
+}
+
+func TestDecodeCompoundMessage(t *testing.T) {
+	msg := &ping{SeqNo: 100}
+	buf, err := encode(pingMsg, msg)
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+
+	msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()}
+	compound := makeCompoundMessage(msgs)
+
+	trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:])
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+	if trunc != 0 {
+		t.Fatalf("should not truncate")
+	}
+	if len(parts) != 3 {
+		t.Fatalf("bad parts")
+	}
+	for _, p := range parts {
+		if len(p) != buf.Len() {
+			t.Fatalf("bad part len")
+		}
+	}
+}
+
+func TestDecodeCompoundMessage_Trunc(t *testing.T) {
+	msg := &ping{SeqNo: 100}
+	buf, err := encode(pingMsg, msg)
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+
+	msgs := [][]byte{buf.Bytes(), buf.Bytes(), buf.Bytes()}
+	compound := makeCompoundMessage(msgs)
+
+	trunc, parts, err := decodeCompoundMessage(compound.Bytes()[1:38])
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+	if trunc != 1 {
+		t.Fatalf("truncate: %d", trunc)
+	}
+	if len(parts) != 2 {
+		t.Fatalf("bad parts")
+	}
+	for _, p := range parts {
+		if len(p) != buf.Len() {
+			t.Fatalf("bad part len")
+		}
+	}
+}
+
+func TestCompressDecompressPayload(t *testing.T) {
+	buf, err := compressPayload([]byte("testing"))
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+
+	decomp, err := decompressPayload(buf.Bytes()[1:])
+	if err != nil {
+		t.Fatalf("unexpected err: %s", err)
+	}
+
+	if !reflect.DeepEqual(decomp, []byte("testing")) {
+		t.Fatalf("bad payload: %v", decomp)
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/broadcast.go

@@ -0,0 +1,27 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+)
+
+// broadcast is an implementation of memberlist.Broadcast and is used
+// to manage broadcasts across the memberlist channel that are related
+// only to Serf.
+type broadcast struct {
+	msg    []byte
+	notify chan<- struct{}
+}
+
+func (b *broadcast) Invalidates(other memberlist.Broadcast) bool {
+	return false
+}
+
+func (b *broadcast) Message() []byte {
+	return b.msg
+}
+
+func (b *broadcast) Finished() {
+	if b.notify != nil {
+		close(b.notify)
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/broadcast_test.go

@@ -0,0 +1,32 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+	"testing"
+	"time"
+)
+
+func TestBroadcast_Impl(t *testing.T) {
+	var _ memberlist.Broadcast = &broadcast{}
+}
+
+func TestBroadcastFinished(t *testing.T) {
+	t.Parallel()
+
+	ch := make(chan struct{})
+	b := &broadcast{notify: ch}
+	b.Finished()
+
+	select {
+	case <-ch:
+	case <-time.After(10 * time.Millisecond):
+		t.Fatalf("should have notified")
+	}
+}
+
+func TestBroadcastFinished_nilNotify(t *testing.T) {
+	t.Parallel()
+
+	b := &broadcast{notify: nil}
+	b.Finished()
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce.go

@@ -0,0 +1,80 @@
+package serf
+
+import (
+	"time"
+)
+
+// coalescer is a simple interface that must be implemented to be
+// used inside of a coalesceLoop
+type coalescer interface {
+	// Can the coalescer handle this event, if not it is
+	// directly passed through to the destination channel
+	Handle(Event) bool
+
+	// Invoked to coalesce the given event
+	Coalesce(Event)
+
+	// Invoked to flush the coalesced events
+	Flush(outChan chan<- Event)
+}
+
+// coalescedEventCh returns an event channel where the events are coalesced
+// using the given coalescer.
+func coalescedEventCh(outCh chan<- Event, shutdownCh <-chan struct{},
+	cPeriod time.Duration, qPeriod time.Duration, c coalescer) chan<- Event {
+	inCh := make(chan Event, 1024)
+	go coalesceLoop(inCh, outCh, shutdownCh, cPeriod, qPeriod, c)
+	return inCh
+}
+
+// coalesceLoop is a simple long-running routine that manages the high-level
+// flow of coalescing based on quiescence and a maximum quantum period.
+func coalesceLoop(inCh <-chan Event, outCh chan<- Event, shutdownCh <-chan struct{},
+	coalescePeriod time.Duration, quiescentPeriod time.Duration, c coalescer) {
+	var quiescent <-chan time.Time
+	var quantum <-chan time.Time
+	shutdown := false
+
+INGEST:
+	// Reset the timers
+	quantum = nil
+	quiescent = nil
+
+	for {
+		select {
+		case e := <-inCh:
+			// Ignore any non handled events
+			if !c.Handle(e) {
+				outCh <- e
+				continue
+			}
+
+			// Start a new quantum if we need to
+			// and restart the quiescent timer
+			if quantum == nil {
+				quantum = time.After(coalescePeriod)
+			}
+			quiescent = time.After(quiescentPeriod)
+
+			// Coalesce the event
+			c.Coalesce(e)
+
+		case <-quantum:
+			goto FLUSH
+		case <-quiescent:
+			goto FLUSH
+		case <-shutdownCh:
+			shutdown = true
+			goto FLUSH
+		}
+	}
+
+FLUSH:
+	// Flush the coalesced events
+	c.Flush(outCh)
+
+	// Restart ingestion if we are not done
+	if !shutdown {
+		goto INGEST
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_member.go

@@ -0,0 +1,68 @@
+package serf
+
+type coalesceEvent struct {
+	Type   EventType
+	Member *Member
+}
+
+type memberEventCoalescer struct {
+	lastEvents   map[string]EventType
+	latestEvents map[string]coalesceEvent
+}
+
+func (c *memberEventCoalescer) Handle(e Event) bool {
+	switch e.EventType() {
+	case EventMemberJoin:
+		return true
+	case EventMemberLeave:
+		return true
+	case EventMemberFailed:
+		return true
+	case EventMemberUpdate:
+		return true
+	case EventMemberReap:
+		return true
+	default:
+		return false
+	}
+}
+
+func (c *memberEventCoalescer) Coalesce(raw Event) {
+	e := raw.(MemberEvent)
+	for _, m := range e.Members {
+		c.latestEvents[m.Name] = coalesceEvent{
+			Type:   e.Type,
+			Member: &m,
+		}
+	}
+}
+
+func (c *memberEventCoalescer) Flush(outCh chan<- Event) {
+	// Coalesce the various events we got into a single set of events.
+	events := make(map[EventType]*MemberEvent)
+	for name, cevent := range c.latestEvents {
+		previous, ok := c.lastEvents[name]
+
+		// If we sent the same event before, then ignore
+		// unless it is a MemberUpdate
+		if ok && previous == cevent.Type && cevent.Type != EventMemberUpdate {
+			continue
+		}
+
+		// Update our last event
+		c.lastEvents[name] = cevent.Type
+
+		// Add it to our event
+		newEvent, ok := events[cevent.Type]
+		if !ok {
+			newEvent = &MemberEvent{Type: cevent.Type}
+			events[cevent.Type] = newEvent
+		}
+		newEvent.Members = append(newEvent.Members, *cevent.Member)
+	}
+
+	// Send out those events
+	for _, event := range events {
+		outCh <- *event
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_member_test.go

@@ -0,0 +1,185 @@
+package serf
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+func TestMemberEventCoalesce_Basic(t *testing.T) {
+	outCh := make(chan Event, 64)
+	shutdownCh := make(chan struct{})
+	defer close(shutdownCh)
+
+	c := &memberEventCoalescer{
+		lastEvents:   make(map[string]EventType),
+		latestEvents: make(map[string]coalesceEvent),
+	}
+
+	inCh := coalescedEventCh(outCh, shutdownCh,
+		5*time.Millisecond, 5*time.Millisecond, c)
+
+	send := []Event{
+		MemberEvent{
+			Type:    EventMemberJoin,
+			Members: []Member{Member{Name: "foo"}},
+		},
+		MemberEvent{
+			Type:    EventMemberLeave,
+			Members: []Member{Member{Name: "foo"}},
+		},
+		MemberEvent{
+			Type:    EventMemberLeave,
+			Members: []Member{Member{Name: "bar"}},
+		},
+		MemberEvent{
+			Type:    EventMemberUpdate,
+			Members: []Member{Member{Name: "zip", Tags: map[string]string{"role": "foo"}}},
+		},
+		MemberEvent{
+			Type:    EventMemberUpdate,
+			Members: []Member{Member{Name: "zip", Tags: map[string]string{"role": "bar"}}},
+		},
+		MemberEvent{
+			Type:    EventMemberReap,
+			Members: []Member{Member{Name: "dead"}},
+		},
+	}
+
+	for _, e := range send {
+		inCh <- e
+	}
+
+	events := make(map[EventType]Event)
+	timeout := time.After(10 * time.Millisecond)
+
+MEMBEREVENTFORLOOP:
+	for {
+		select {
+		case e := <-outCh:
+			events[e.EventType()] = e
+		case <-timeout:
+			break MEMBEREVENTFORLOOP
+		}
+	}
+
+	if len(events) != 3 {
+		t.Fatalf("bad: %#v", events)
+	}
+
+	if e, ok := events[EventMemberLeave]; !ok {
+		t.Fatalf("bad: %#v", events)
+	} else {
+		me := e.(MemberEvent)
+
+		if len(me.Members) != 2 {
+			t.Fatalf("bad: %#v", me)
+		}
+
+		expected := []string{"bar", "foo"}
+		names := []string{me.Members[0].Name, me.Members[1].Name}
+		sort.Strings(names)
+
+		if !reflect.DeepEqual(expected, names) {
+			t.Fatalf("bad: %#v", names)
+		}
+	}
+
+	if e, ok := events[EventMemberUpdate]; !ok {
+		t.Fatalf("bad: %#v", events)
+	} else {
+		me := e.(MemberEvent)
+		if len(me.Members) != 1 {
+			t.Fatalf("bad: %#v", me)
+		}
+
+		if me.Members[0].Name != "zip" {
+			t.Fatalf("bad: %#v", me.Members)
+		}
+		if me.Members[0].Tags["role"] != "bar" {
+			t.Fatalf("bad: %#v", me.Members[0].Tags)
+		}
+	}
+
+	if e, ok := events[EventMemberReap]; !ok {
+		t.Fatalf("bad: %#v", events)
+	} else {
+		me := e.(MemberEvent)
+		if len(me.Members) != 1 {
+			t.Fatalf("bad: %#v", me)
+		}
+
+		if me.Members[0].Name != "dead" {
+			t.Fatalf("bad: %#v", me.Members)
+		}
+	}
+}
+
+func TestMemberEventCoalesce_TagUpdate(t *testing.T) {
+	outCh := make(chan Event, 64)
+	shutdownCh := make(chan struct{})
+	defer close(shutdownCh)
+
+	c := &memberEventCoalescer{
+		lastEvents:   make(map[string]EventType),
+		latestEvents: make(map[string]coalesceEvent),
+	}
+
+	inCh := coalescedEventCh(outCh, shutdownCh,
+		5*time.Millisecond, 5*time.Millisecond, c)
+
+	inCh <- MemberEvent{
+		Type:    EventMemberUpdate,
+		Members: []Member{Member{Name: "foo", Tags: map[string]string{"role": "foo"}}},
+	}
+
+	time.Sleep(10 * time.Millisecond)
+
+	select {
+	case e := <-outCh:
+		if e.EventType() != EventMemberUpdate {
+			t.Fatalf("expected update")
+		}
+	default:
+		t.Fatalf("expected update")
+	}
+
+	// Second update should not be suppressed even though
+	// last event was an update
+	inCh <- MemberEvent{
+		Type:    EventMemberUpdate,
+		Members: []Member{Member{Name: "foo", Tags: map[string]string{"role": "bar"}}},
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	select {
+	case e := <-outCh:
+		if e.EventType() != EventMemberUpdate {
+			t.Fatalf("expected update")
+		}
+	default:
+		t.Fatalf("expected update")
+	}
+}
+
+func TestMemberEventCoalesce_passThrough(t *testing.T) {
+	cases := []struct {
+		e      Event
+		handle bool
+	}{
+		{UserEvent{}, false},
+		{MemberEvent{Type: EventMemberJoin}, true},
+		{MemberEvent{Type: EventMemberLeave}, true},
+		{MemberEvent{Type: EventMemberFailed}, true},
+		{MemberEvent{Type: EventMemberUpdate}, true},
+		{MemberEvent{Type: EventMemberReap}, true},
+	}
+
+	for _, tc := range cases {
+		c := &memberEventCoalescer{}
+		if tc.handle != c.Handle(tc.e) {
+			t.Fatalf("bad: %#v", tc.e)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_test.go

@@ -0,0 +1,140 @@
+package serf
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+// Mock EventCounter type
+const EventCounter EventType = 9000
+
+type counterEvent struct {
+	delta int
+}
+
+func (c counterEvent) EventType() EventType {
+	return EventCounter
+}
+func (c counterEvent) String() string {
+	return fmt.Sprintf("CounterEvent %d", c.delta)
+}
+
+// Mock coalescer
+type mockCoalesce struct {
+	value int
+}
+
+func (c *mockCoalesce) Handle(e Event) bool {
+	return e.EventType() == EventCounter
+}
+
+func (c *mockCoalesce) Coalesce(e Event) {
+	c.value += e.(counterEvent).delta
+}
+
+func (c *mockCoalesce) Flush(outChan chan<- Event) {
+	outChan <- counterEvent{c.value}
+	c.value = 0
+}
+
+func testCoalescer(cPeriod, qPeriod time.Duration) (chan<- Event, <-chan Event, chan<- struct{}) {
+	in := make(chan Event, 64)
+	out := make(chan Event)
+	shutdown := make(chan struct{})
+	c := &mockCoalesce{}
+	go coalesceLoop(in, out, shutdown, cPeriod, qPeriod, c)
+	return in, out, shutdown
+}
+
+func TestCoalescer_basic(t *testing.T) {
+	in, out, shutdown := testCoalescer(5*time.Millisecond, time.Second)
+	defer close(shutdown)
+
+	send := []Event{
+		counterEvent{1},
+		counterEvent{39},
+		counterEvent{2},
+	}
+	for _, e := range send {
+		in <- e
+	}
+
+	select {
+	case e := <-out:
+		if e.EventType() != EventCounter {
+			t.Fatalf("expected counter, got: %d", e.EventType())
+		}
+
+		if e.(counterEvent).delta != 42 {
+			t.Fatalf("bad: %#v", e)
+		}
+
+	case <-time.After(50 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+}
+
+func TestCoalescer_quiescent(t *testing.T) {
+	// This tests the quiescence by creating a long coalescence period
+	// with a short quiescent period and waiting only a multiple of the
+	// quiescent period for results.
+	in, out, shutdown := testCoalescer(10*time.Second, 10*time.Millisecond)
+	defer close(shutdown)
+
+	send := []Event{
+		counterEvent{1},
+		counterEvent{39},
+		counterEvent{2},
+	}
+	for _, e := range send {
+		in <- e
+	}
+
+	select {
+	case e := <-out:
+		if e.EventType() != EventCounter {
+			t.Fatalf("expected counter, got: %d", e.EventType())
+		}
+
+		if e.(counterEvent).delta != 42 {
+			t.Fatalf("bad: %#v", e)
+		}
+	case <-time.After(50 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+}
+
+func TestCoalescer_passThrough(t *testing.T) {
+	in, out, shutdown := testCoalescer(time.Second, time.Second)
+	defer close(shutdown)
+
+	send := []Event{
+		UserEvent{
+			Name:    "test",
+			Payload: []byte("foo"),
+		},
+	}
+
+	for _, e := range send {
+		in <- e
+	}
+
+	select {
+	case e := <-out:
+		if e.EventType() != EventUser {
+			t.Fatalf("expected user event, got: %d", e.EventType())
+		}
+
+		if e.(UserEvent).Name != "test" {
+			t.Fatalf("name should be test. %v", e)
+		}
+
+		if !reflect.DeepEqual([]byte("foo"), e.(UserEvent).Payload) {
+			t.Fatalf("bad: %#v", e.(UserEvent).Payload)
+		}
+	case <-time.After(50 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_user.go

@@ -0,0 +1,52 @@
+package serf
+
+type latestUserEvents struct {
+	LTime  LamportTime
+	Events []Event
+}
+
+type userEventCoalescer struct {
+	// Maps an event name into the latest versions
+	events map[string]*latestUserEvents
+}
+
+func (c *userEventCoalescer) Handle(e Event) bool {
+	// Only handle EventUser messages
+	if e.EventType() != EventUser {
+		return false
+	}
+
+	// Check if coalescing is enabled
+	user := e.(UserEvent)
+	return user.Coalesce
+}
+
+func (c *userEventCoalescer) Coalesce(e Event) {
+	user := e.(UserEvent)
+	latest, ok := c.events[user.Name]
+
+	// Create a new entry if there are none, or
+	// if this message has the newest LTime
+	if !ok || latest.LTime < user.LTime {
+		latest = &latestUserEvents{
+			LTime:  user.LTime,
+			Events: []Event{e},
+		}
+		c.events[user.Name] = latest
+		return
+	}
+
+	// If the the same age, save it
+	if latest.LTime == user.LTime {
+		latest.Events = append(latest.Events, e)
+	}
+}
+
+func (c *userEventCoalescer) Flush(outChan chan<- Event) {
+	for _, latest := range c.events {
+		for _, e := range latest.Events {
+			outChan <- e
+		}
+	}
+	c.events = make(map[string]*latestUserEvents)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/coalesce_user_test.go

@@ -0,0 +1,102 @@
+package serf
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestUserEventCoalesce_Basic(t *testing.T) {
+	outCh := make(chan Event, 64)
+	shutdownCh := make(chan struct{})
+	defer close(shutdownCh)
+
+	c := &userEventCoalescer{
+		events: make(map[string]*latestUserEvents),
+	}
+
+	inCh := coalescedEventCh(outCh, shutdownCh,
+		5*time.Millisecond, 5*time.Millisecond, c)
+
+	send := []Event{
+		UserEvent{
+			LTime:    1,
+			Name:     "foo",
+			Coalesce: true,
+		},
+		UserEvent{
+			LTime:    2,
+			Name:     "foo",
+			Coalesce: true,
+		},
+		UserEvent{
+			LTime:    2,
+			Name:     "bar",
+			Payload:  []byte("test1"),
+			Coalesce: true,
+		},
+		UserEvent{
+			LTime:    2,
+			Name:     "bar",
+			Payload:  []byte("test2"),
+			Coalesce: true,
+		},
+	}
+
+	for _, e := range send {
+		inCh <- e
+	}
+
+	var gotFoo, gotBar1, gotBar2 bool
+	timeout := time.After(10 * time.Millisecond)
+USEREVENTFORLOOP:
+	for {
+		select {
+		case e := <-outCh:
+			ue := e.(UserEvent)
+			switch ue.Name {
+			case "foo":
+				if ue.LTime != 2 {
+					t.Fatalf("bad ltime for foo %#v", ue)
+				}
+				gotFoo = true
+			case "bar":
+				if ue.LTime != 2 {
+					t.Fatalf("bad ltime for bar %#v", ue)
+				}
+				if reflect.DeepEqual(ue.Payload, []byte("test1")) {
+					gotBar1 = true
+				}
+				if reflect.DeepEqual(ue.Payload, []byte("test2")) {
+					gotBar2 = true
+				}
+			}
+		case <-timeout:
+			break USEREVENTFORLOOP
+		}
+	}
+
+	if !gotFoo || !gotBar1 || !gotBar2 {
+		t.Fatalf("missing messages %v %v %v", gotFoo, gotBar1, gotBar2)
+	}
+}
+
+func TestUserEventCoalesce_passThrough(t *testing.T) {
+	cases := []struct {
+		e      Event
+		handle bool
+	}{
+		{UserEvent{Coalesce: false}, false},
+		{UserEvent{Coalesce: true}, true},
+		{MemberEvent{Type: EventMemberJoin}, false},
+		{MemberEvent{Type: EventMemberLeave}, false},
+		{MemberEvent{Type: EventMemberFailed}, false},
+	}
+
+	for _, tc := range cases {
+		c := &userEventCoalescer{}
+		if tc.handle != c.Handle(tc.e) {
+			t.Fatalf("bad: %#v", tc.e)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/config.go

@@ -0,0 +1,234 @@
+package serf
+
+import (
+	"io"
+	"os"
+	"time"
+
+	"github.com/hashicorp/memberlist"
+)
+
+// ProtocolVersionMap is the mapping of Serf delegate protocol versions
+// to memberlist protocol versions. We mask the memberlist protocols using
+// our own protocol version.
+var ProtocolVersionMap map[uint8]uint8
+
+func init() {
+	ProtocolVersionMap = map[uint8]uint8{
+		4: 2,
+		3: 2,
+		2: 2,
+	}
+}
+
+// Config is the configuration for creating a Serf instance.
+type Config struct {
+	// The name of this node. This must be unique in the cluster. If this
+	// is not set, Serf will set it to the hostname of the running machine.
+	NodeName string
+
+	// The tags for this role, if any. This is used to provide arbitrary
+	// key/value metadata per-node. For example, a "role" tag may be used to
+	// differentiate "load-balancer" from a "web" role as parts of the same cluster.
+	// Tags are deprecating 'Role', and instead it acts as a special key in this
+	// map.
+	Tags map[string]string
+
+	// EventCh is a channel that receives all the Serf events. The events
+	// are sent on this channel in proper ordering. Care must be taken that
+	// this channel doesn't block, either by processing the events quick
+	// enough or buffering the channel, otherwise it can block state updates
+	// within Serf itself. If no EventCh is specified, no events will be fired,
+	// but point-in-time snapshots of members can still be retrieved by
+	// calling Members on Serf.
+	EventCh chan<- Event
+
+	// ProtocolVersion is the protocol version to speak. This must be between
+	// ProtocolVersionMin and ProtocolVersionMax.
+	ProtocolVersion uint8
+
+	// BroadcastTimeout is the amount of time to wait for a broadcast
+	// message to be sent to the cluster. Broadcast messages are used for
+	// things like leave messages and force remove messages. If this is not
+	// set, a timeout of 5 seconds will be set.
+	BroadcastTimeout time.Duration
+
+	// The settings below relate to Serf's event coalescence feature. Serf
+	// is able to coalesce multiple events into single events in order to
+	// reduce the amount of noise that is sent along the EventCh. For example
+	// if five nodes quickly join, the EventCh will be sent one EventMemberJoin
+	// containing the five nodes rather than five individual EventMemberJoin
+	// events. Coalescence can mitigate potential flapping behavior.
+	//
+	// Coalescence is disabled by default and can be enabled by setting
+	// CoalescePeriod.
+	//
+	// CoalescePeriod specifies the time duration to coalesce events.
+	// For example, if this is set to 5 seconds, then all events received
+	// within 5 seconds that can be coalesced will be.
+	//
+	// QuiescentPeriod specifies the duration of time where if no events
+	// are received, coalescence immediately happens. For example, if
+	// CoalscePeriod is set to 10 seconds but QuiscentPeriod is set to 2
+	// seconds, then the events will be coalesced and dispatched if no
+	// new events are received within 2 seconds of the last event. Otherwise,
+	// every event will always be delayed by at least 10 seconds.
+	CoalescePeriod  time.Duration
+	QuiescentPeriod time.Duration
+
+	// The settings below relate to Serf's user event coalescing feature.
+	// The settings operate like above but only affect user messages and
+	// not the Member* messages that Serf generates.
+	UserCoalescePeriod  time.Duration
+	UserQuiescentPeriod time.Duration
+
+	// The settings below relate to Serf keeping track of recently
+	// failed/left nodes and attempting reconnects.
+	//
+	// ReapInterval is the interval when the reaper runs. If this is not
+	// set (it is zero), it will be set to a reasonable default.
+	//
+	// ReconnectInterval is the interval when we attempt to reconnect
+	// to failed nodes. If this is not set (it is zero), it will be set
+	// to a reasonable default.
+	//
+	// ReconnectTimeout is the amount of time to attempt to reconnect to
+	// a failed node before giving up and considering it completely gone.
+	//
+	// TombstoneTimeout is the amount of time to keep around nodes
+	// that gracefully left as tombstones for syncing state with other
+	// Serf nodes.
+	ReapInterval      time.Duration
+	ReconnectInterval time.Duration
+	ReconnectTimeout  time.Duration
+	TombstoneTimeout  time.Duration
+
+	// QueueDepthWarning is used to generate warning message if the
+	// number of queued messages to broadcast exceeds this number. This
+	// is to provide the user feedback if events are being triggered
+	// faster than they can be disseminated
+	QueueDepthWarning int
+
+	// MaxQueueDepth is used to start dropping messages if the number
+	// of queued messages to broadcast exceeds this number. This is to
+	// prevent an unbounded growth of memory utilization
+	MaxQueueDepth int
+
+	// RecentIntentBuffer is used to set the size of recent join and leave intent
+	// messages that will be buffered. This is used to guard against
+	// the case where Serf broadcasts an intent that arrives before the
+	// Memberlist event. It is important that this not be too small to avoid
+	// continuous rebroadcasting of dead events.
+	RecentIntentBuffer int
+
+	// EventBuffer is used to control how many events are buffered.
+	// This is used to prevent re-delivery of events to a client. The buffer
+	// must be large enough to handle all "recent" events, since Serf will
+	// not deliver messages that are older than the oldest entry in the buffer.
+	// Thus if a client is generating too many events, it's possible that the
+	// buffer gets overrun and messages are not delivered.
+	EventBuffer int
+
+	// QueryBuffer is used to control how many queries are buffered.
+	// This is used to prevent re-delivery of queries to a client. The buffer
+	// must be large enough to handle all "recent" events, since Serf will not
+	// deliver queries older than the oldest entry in the buffer.
+	// Thus if a client is generating too many queries, it's possible that the
+	// buffer gets overrun and messages are not delivered.
+	QueryBuffer int
+
+	// QueryTimeoutMult configures the default timeout multipler for a query to run if no
+	// specific value is provided. Queries are real-time by nature, where the
+	// reply is time sensitive. As a result, results are collected in an async
+	// fashion, however the query must have a bounded duration. We want the timeout
+	// to be long enough that all nodes have time to receive the message, run a handler,
+	// and generate a reply. Once the timeout is exceeded, any further replies are ignored.
+	// The default value is
+	//
+	// Timeout = GossipInterval * QueryTimeoutMult * log(N+1)
+	//
+	QueryTimeoutMult int
+
+	// MemberlistConfig is the memberlist configuration that Serf will
+	// use to do the underlying membership management and gossip. Some
+	// fields in the MemberlistConfig will be overwritten by Serf no
+	// matter what:
+	//
+	//   * Name - This will always be set to the same as the NodeName
+	//     in this configuration.
+	//
+	//   * Events - Serf uses a custom event delegate.
+	//
+	//   * Delegate - Serf uses a custom delegate.
+	//
+	MemberlistConfig *memberlist.Config
+
+	// LogOutput is the location to write logs to. If this is not set,
+	// logs will go to stderr.
+	LogOutput io.Writer
+
+	// SnapshotPath if provided is used to snapshot live nodes as well
+	// as lamport clock values. When Serf is started with a snapshot,
+	// it will attempt to join all the previously known nodes until one
+	// succeeds and will also avoid replaying old user events.
+	SnapshotPath string
+
+	// RejoinAfterLeave controls our interaction with the snapshot file.
+	// When set to false (default), a leave causes a Serf to not rejoin
+	// the cluster until an explicit join is received. If this is set to
+	// true, we ignore the leave, and rejoin the cluster on start.
+	RejoinAfterLeave bool
+
+	// EnableNameConflictResolution controls if Serf will actively attempt
+	// to resolve a name conflict. Since each Serf member must have a unique
+	// name, a cluster can run into issues if multiple nodes claim the same
+	// name. Without automatic resolution, Serf merely logs some warnings, but
+	// otherwise does not take any action. Automatic resolution detects the
+	// conflict and issues a special query which asks the cluster for the
+	// Name -> IP:Port mapping. If there is a simple majority of votes, that
+	// node stays while the other node will leave the cluster and exit.
+	EnableNameConflictResolution bool
+
+	// KeyringFile provides the location of a writable file where Serf can
+	// persist changes to the encryption keyring.
+	KeyringFile string
+
+	// Merge can be optionally provided to intercept a cluster merge
+	// and conditionally abort the merge.
+	Merge MergeDelegate
+}
+
+// Init allocates the subdata structures
+func (c *Config) Init() {
+	if c.Tags == nil {
+		c.Tags = make(map[string]string)
+	}
+}
+
+// DefaultConfig returns a Config struct that contains reasonable defaults
+// for most of the configurations.
+func DefaultConfig() *Config {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+
+	return &Config{
+		NodeName:                     hostname,
+		BroadcastTimeout:             5 * time.Second,
+		EventBuffer:                  512,
+		QueryBuffer:                  512,
+		LogOutput:                    os.Stderr,
+		ProtocolVersion:              ProtocolVersionMax,
+		ReapInterval:                 15 * time.Second,
+		RecentIntentBuffer:           128,
+		ReconnectInterval:            30 * time.Second,
+		ReconnectTimeout:             24 * time.Hour,
+		QueueDepthWarning:            128,
+		MaxQueueDepth:                4096,
+		TombstoneTimeout:             24 * time.Hour,
+		MemberlistConfig:             memberlist.DefaultLANConfig(),
+		QueryTimeoutMult:             16,
+		EnableNameConflictResolution: true,
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/config_test.go

@@ -0,0 +1,12 @@
+package serf
+
+import (
+	"testing"
+)
+
+func TestDefaultConfig(t *testing.T) {
+	c := DefaultConfig()
+	if c.ProtocolVersion != ProtocolVersionMax {
+		t.Fatalf("bad: %#v", c)
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/conflict_delegate.go

@@ -0,0 +1,13 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+)
+
+type conflictDelegate struct {
+	serf *Serf
+}
+
+func (c *conflictDelegate) NotifyConflict(existing, other *memberlist.Node) {
+	c.serf.handleNodeConflict(existing, other)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/delegate.go

@@ -0,0 +1,247 @@
+package serf
+
+import (
+	"fmt"
+	"github.com/armon/go-metrics"
+)
+
+// delegate is the memberlist.Delegate implementation that Serf uses.
+type delegate struct {
+	serf *Serf
+}
+
+func (d *delegate) NodeMeta(limit int) []byte {
+	roleBytes := d.serf.encodeTags(d.serf.config.Tags)
+	if len(roleBytes) > limit {
+		panic(fmt.Errorf("Node tags '%v' exceeds length limit of %d bytes", d.serf.config.Tags, limit))
+	}
+
+	return roleBytes
+}
+
+func (d *delegate) NotifyMsg(buf []byte) {
+	// If we didn't actually receive any data, then ignore it.
+	if len(buf) == 0 {
+		return
+	}
+	metrics.AddSample([]string{"serf", "msgs", "received"}, float32(len(buf)))
+
+	rebroadcast := false
+	rebroadcastQueue := d.serf.broadcasts
+	t := messageType(buf[0])
+	switch t {
+	case messageLeaveType:
+		var leave messageLeave
+		if err := decodeMessage(buf[1:], &leave); err != nil {
+			d.serf.logger.Printf("[ERR] serf: Error decoding leave message: %s", err)
+			break
+		}
+
+		d.serf.logger.Printf("[DEBUG] serf: messageLeaveType: %s", leave.Node)
+		rebroadcast = d.serf.handleNodeLeaveIntent(&leave)
+
+	case messageJoinType:
+		var join messageJoin
+		if err := decodeMessage(buf[1:], &join); err != nil {
+			d.serf.logger.Printf("[ERR] serf: Error decoding join message: %s", err)
+			break
+		}
+
+		d.serf.logger.Printf("[DEBUG] serf: messageJoinType: %s", join.Node)
+		rebroadcast = d.serf.handleNodeJoinIntent(&join)
+
+	case messageUserEventType:
+		var event messageUserEvent
+		if err := decodeMessage(buf[1:], &event); err != nil {
+			d.serf.logger.Printf("[ERR] serf: Error decoding user event message: %s", err)
+			break
+		}
+
+		d.serf.logger.Printf("[DEBUG] serf: messageUserEventType: %s", event.Name)
+		rebroadcast = d.serf.handleUserEvent(&event)
+		rebroadcastQueue = d.serf.eventBroadcasts
+
+	case messageQueryType:
+		var query messageQuery
+		if err := decodeMessage(buf[1:], &query); err != nil {
+			d.serf.logger.Printf("[ERR] serf: Error decoding query message: %s", err)
+			break
+		}
+
+		d.serf.logger.Printf("[DEBUG] serf: messageQueryType: %s", query.Name)
+		rebroadcast = d.serf.handleQuery(&query)
+		rebroadcastQueue = d.serf.queryBroadcasts
+
+	case messageQueryResponseType:
+		var resp messageQueryResponse
+		if err := decodeMessage(buf[1:], &resp); err != nil {
+			d.serf.logger.Printf("[ERR] serf: Error decoding query response message: %s", err)
+			break
+		}
+
+		d.serf.logger.Printf("[DEBUG] serf: messageQueryResponseType: %v", resp.From)
+		d.serf.handleQueryResponse(&resp)
+
+	default:
+		d.serf.logger.Printf("[WARN] serf: Received message of unknown type: %d", t)
+	}
+
+	if rebroadcast {
+		// Copy the buffer since it we cannot rely on the slice not changing
+		newBuf := make([]byte, len(buf))
+		copy(newBuf, buf)
+
+		rebroadcastQueue.QueueBroadcast(&broadcast{
+			msg:    newBuf,
+			notify: nil,
+		})
+	}
+}
+
+func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
+	msgs := d.serf.broadcasts.GetBroadcasts(overhead, limit)
+
+	// Determine the bytes used already
+	bytesUsed := 0
+	for _, msg := range msgs {
+		lm := len(msg)
+		bytesUsed += lm + overhead
+		metrics.AddSample([]string{"serf", "msgs", "sent"}, float32(lm))
+	}
+
+	// Get any additional query broadcasts
+	queryMsgs := d.serf.queryBroadcasts.GetBroadcasts(overhead, limit-bytesUsed)
+	if queryMsgs != nil {
+		for _, m := range queryMsgs {
+			lm := len(m)
+			bytesUsed += lm + overhead
+			metrics.AddSample([]string{"serf", "msgs", "sent"}, float32(lm))
+		}
+		msgs = append(msgs, queryMsgs...)
+	}
+
+	// Get any additional event broadcasts
+	eventMsgs := d.serf.eventBroadcasts.GetBroadcasts(overhead, limit-bytesUsed)
+	if eventMsgs != nil {
+		for _, m := range eventMsgs {
+			lm := len(m)
+			bytesUsed += lm + overhead
+			metrics.AddSample([]string{"serf", "msgs", "sent"}, float32(lm))
+		}
+		msgs = append(msgs, eventMsgs...)
+	}
+
+	return msgs
+}
+
+func (d *delegate) LocalState(join bool) []byte {
+	d.serf.memberLock.RLock()
+	defer d.serf.memberLock.RUnlock()
+	d.serf.eventLock.RLock()
+	defer d.serf.eventLock.RUnlock()
+
+	// Create the message to send
+	pp := messagePushPull{
+		LTime:        d.serf.clock.Time(),
+		StatusLTimes: make(map[string]LamportTime, len(d.serf.members)),
+		LeftMembers:  make([]string, 0, len(d.serf.leftMembers)),
+		EventLTime:   d.serf.eventClock.Time(),
+		Events:       d.serf.eventBuffer,
+		QueryLTime:   d.serf.queryClock.Time(),
+	}
+
+	// Add all the join LTimes
+	for name, member := range d.serf.members {
+		pp.StatusLTimes[name] = member.statusLTime
+	}
+
+	// Add all the left nodes
+	for _, member := range d.serf.leftMembers {
+		pp.LeftMembers = append(pp.LeftMembers, member.Name)
+	}
+
+	// Encode the push pull state
+	buf, err := encodeMessage(messagePushPullType, &pp)
+	if err != nil {
+		d.serf.logger.Printf("[ERR] serf: Failed to encode local state: %v", err)
+		return nil
+	}
+	return buf
+}
+
+func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
+	// Check the message type
+	if messageType(buf[0]) != messagePushPullType {
+		d.serf.logger.Printf("[ERR] serf: Remote state has bad type prefix: %v", buf[0])
+		return
+	}
+
+	// Attempt a decode
+	pp := messagePushPull{}
+	if err := decodeMessage(buf[1:], &pp); err != nil {
+		d.serf.logger.Printf("[ERR] serf: Failed to decode remote state: %v", err)
+		return
+	}
+
+	// Witness the Lamport clocks first.
+	// We subtract 1 since no message with that clock has been sent yet
+	if pp.LTime > 0 {
+		d.serf.clock.Witness(pp.LTime - 1)
+	}
+	if pp.EventLTime > 0 {
+		d.serf.eventClock.Witness(pp.EventLTime - 1)
+	}
+	if pp.QueryLTime > 0 {
+		d.serf.queryClock.Witness(pp.QueryLTime - 1)
+	}
+
+	// Process the left nodes first to avoid the LTimes from being increment
+	// in the wrong order
+	leftMap := make(map[string]struct{}, len(pp.LeftMembers))
+	leave := messageLeave{}
+	for _, name := range pp.LeftMembers {
+		leftMap[name] = struct{}{}
+		leave.LTime = pp.StatusLTimes[name]
+		leave.Node = name
+		d.serf.handleNodeLeaveIntent(&leave)
+	}
+
+	// Update any other LTimes
+	join := messageJoin{}
+	for name, statusLTime := range pp.StatusLTimes {
+		// Skip the left nodes
+		if _, ok := leftMap[name]; ok {
+			continue
+		}
+
+		// Create an artificial join message
+		join.LTime = statusLTime
+		join.Node = name
+		d.serf.handleNodeJoinIntent(&join)
+	}
+
+	// If we are doing a join, and eventJoinIgnore is set
+	// then we set the eventMinTime to the EventLTime. This
+	// prevents any of the incoming events from being processed
+	if isJoin && d.serf.eventJoinIgnore {
+		d.serf.eventLock.Lock()
+		if pp.EventLTime > d.serf.eventMinTime {
+			d.serf.eventMinTime = pp.EventLTime
+		}
+		d.serf.eventLock.Unlock()
+	}
+
+	// Process all the events
+	userEvent := messageUserEvent{}
+	for _, events := range pp.Events {
+		if events == nil {
+			continue
+		}
+		userEvent.LTime = events.LTime
+		for _, e := range events.Events {
+			userEvent.Name = e.Name
+			userEvent.Payload = e.Payload
+			d.serf.handleUserEvent(&userEvent)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/delegate_test.go

@@ -0,0 +1,213 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/testutil"
+	"reflect"
+	"testing"
+)
+
+func TestDelegate_impl(t *testing.T) {
+	var raw interface{}
+	raw = new(delegate)
+	if _, ok := raw.(memberlist.Delegate); !ok {
+		t.Fatal("should be an Delegate")
+	}
+}
+
+func TestDelegate_NodeMeta_Old(t *testing.T) {
+	c := testConfig()
+	c.ProtocolVersion = 2
+	c.Tags["role"] = "test"
+	d := &delegate{&Serf{config: c}}
+	meta := d.NodeMeta(32)
+
+	if !reflect.DeepEqual(meta, []byte("test")) {
+		t.Fatalf("bad meta data: %v", meta)
+	}
+
+	out := d.serf.decodeTags(meta)
+	if out["role"] != "test" {
+		t.Fatalf("bad meta data: %v", meta)
+	}
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("expected panic")
+		}
+	}()
+	d.NodeMeta(1)
+}
+
+func TestDelegate_NodeMeta_New(t *testing.T) {
+	c := testConfig()
+	c.ProtocolVersion = 3
+	c.Tags["role"] = "test"
+	d := &delegate{&Serf{config: c}}
+	meta := d.NodeMeta(32)
+
+	out := d.serf.decodeTags(meta)
+	if out["role"] != "test" {
+		t.Fatalf("bad meta data: %v", meta)
+	}
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("expected panic")
+		}
+	}()
+	d.NodeMeta(1)
+}
+
+// internals
+func TestDelegate_LocalState(t *testing.T) {
+	c1 := testConfig()
+	s1, err := Create(c1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s1.Shutdown()
+
+	c2 := testConfig()
+	s2, err := Create(c2)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s2.Shutdown()
+
+	testutil.Yield()
+
+	_, err = s1.Join([]string{c2.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	err = s1.UserEvent("test", []byte("test"), false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	_, err = s1.Query("foo", nil, nil)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// s2 can leave now
+	s2.Leave()
+
+	// Do a state dump
+	d := c1.MemberlistConfig.Delegate
+	buf := d.LocalState(false)
+
+	// Verify
+	if messageType(buf[0]) != messagePushPullType {
+		t.Fatalf("bad message type")
+	}
+
+	// Attempt a decode
+	pp := messagePushPull{}
+	if err := decodeMessage(buf[1:], &pp); err != nil {
+		t.Fatalf("decode failed %v", err)
+	}
+
+	// Verify lamport
+	if pp.LTime != s1.clock.Time() {
+		t.Fatalf("clock mismatch")
+	}
+
+	// Verify the status
+	if len(pp.StatusLTimes) != 2 {
+		t.Fatalf("missing ltimes")
+	}
+
+	if len(pp.LeftMembers) != 1 {
+		t.Fatalf("missing left members")
+	}
+
+	if pp.EventLTime != s1.eventClock.Time() {
+		t.Fatalf("clock mismatch")
+	}
+
+	if len(pp.Events) != s1.config.EventBuffer {
+		t.Fatalf("should send full event buffer")
+	}
+
+	if pp.QueryLTime != s1.queryClock.Time() {
+		t.Fatalf("clock mismatch")
+	}
+}
+
+// internals
+func TestDelegate_MergeRemoteState(t *testing.T) {
+	c1 := testConfig()
+	s1, err := Create(c1)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s1.Shutdown()
+
+	// Do a state dump
+	d := c1.MemberlistConfig.Delegate
+
+	// Make a fake push pull
+	pp := messagePushPull{
+		LTime: 42,
+		StatusLTimes: map[string]LamportTime{
+			"test": 20,
+			"foo":  15,
+		},
+		LeftMembers: []string{"foo"},
+		EventLTime:  50,
+		Events: []*userEvents{
+			&userEvents{
+				LTime: 45,
+				Events: []userEvent{
+					userEvent{
+						Name:    "test",
+						Payload: nil,
+					},
+				},
+			},
+		},
+		QueryLTime: 100,
+	}
+
+	buf, err := encodeMessage(messagePushPullType, &pp)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Merge in fake state
+	d.MergeRemoteState(buf, false)
+
+	// Verify lamport
+	if s1.clock.Time() != 42 {
+		t.Fatalf("clock mismatch")
+	}
+
+	// Verify pending join for test
+	if s1.recentJoin[0].Node != "test" || s1.recentJoin[0].LTime != 20 {
+		t.Fatalf("bad recent join")
+	}
+
+	// Verify pending leave for foo
+	if s1.recentLeave[0].Node != "foo" || s1.recentLeave[0].LTime != 15 {
+		t.Fatalf("bad recent leave")
+	}
+
+	// Very event time
+	if s1.eventClock.Time() != 50 {
+		t.Fatalf("bad event clock")
+	}
+
+	if s1.eventBuffer[45] == nil {
+		t.Fatalf("missing event buffer for time")
+	}
+	if s1.eventBuffer[45].Events[0].Name != "test" {
+		t.Fatalf("missing event")
+	}
+
+	if s1.queryClock.Time() != 100 {
+		t.Fatalf("bad query clock")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/event.go

@@ -0,0 +1,168 @@
+package serf
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"time"
+)
+
+// EventType are all the types of events that may occur and be sent
+// along the Serf channel.
+type EventType int
+
+const (
+	EventMemberJoin EventType = iota
+	EventMemberLeave
+	EventMemberFailed
+	EventMemberUpdate
+	EventMemberReap
+	EventUser
+	EventQuery
+)
+
+func (t EventType) String() string {
+	switch t {
+	case EventMemberJoin:
+		return "member-join"
+	case EventMemberLeave:
+		return "member-leave"
+	case EventMemberFailed:
+		return "member-failed"
+	case EventMemberUpdate:
+		return "member-update"
+	case EventMemberReap:
+		return "member-reap"
+	case EventUser:
+		return "user"
+	case EventQuery:
+		return "query"
+	default:
+		panic(fmt.Sprintf("unknown event type: %d", t))
+	}
+}
+
+// Event is a generic interface for exposing Serf events
+// Clients will usually need to use a type switches to get
+// to a more useful type
+type Event interface {
+	EventType() EventType
+	String() string
+}
+
+// MemberEvent is the struct used for member related events
+// Because Serf coalesces events, an event may contain multiple members.
+type MemberEvent struct {
+	Type    EventType
+	Members []Member
+}
+
+func (m MemberEvent) EventType() EventType {
+	return m.Type
+}
+
+func (m MemberEvent) String() string {
+	switch m.Type {
+	case EventMemberJoin:
+		return "member-join"
+	case EventMemberLeave:
+		return "member-leave"
+	case EventMemberFailed:
+		return "member-failed"
+	case EventMemberUpdate:
+		return "member-update"
+	case EventMemberReap:
+		return "member-reap"
+	default:
+		panic(fmt.Sprintf("unknown event type: %d", m.Type))
+	}
+}
+
+// UserEvent is the struct used for events that are triggered
+// by the user and are not related to members
+type UserEvent struct {
+	LTime    LamportTime
+	Name     string
+	Payload  []byte
+	Coalesce bool
+}
+
+func (u UserEvent) EventType() EventType {
+	return EventUser
+}
+
+func (u UserEvent) String() string {
+	return fmt.Sprintf("user-event: %s", u.Name)
+}
+
+// Query is the struct used EventQuery type events
+type Query struct {
+	LTime   LamportTime
+	Name    string
+	Payload []byte
+
+	serf     *Serf
+	id       uint32    // ID is not exported, since it may change
+	addr     []byte    // Address to respond to
+	port     uint16    // Port to respond to
+	deadline time.Time // Must respond by this deadline
+	respLock sync.Mutex
+}
+
+func (q *Query) EventType() EventType {
+	return EventQuery
+}
+
+func (q *Query) String() string {
+	return fmt.Sprintf("query: %s", q.Name)
+}
+
+// Deadline returns the time by which a response must be sent
+func (q *Query) Deadline() time.Time {
+	return q.deadline
+}
+
+// Respond is used to send a response to the user query
+func (q *Query) Respond(buf []byte) error {
+	q.respLock.Lock()
+	defer q.respLock.Unlock()
+
+	// Check if we've already responded
+	if q.deadline.IsZero() {
+		return fmt.Errorf("Response already sent")
+	}
+
+	// Ensure we aren't past our response deadline
+	if time.Now().After(q.deadline) {
+		return fmt.Errorf("Response is past the deadline")
+	}
+
+	// Create response
+	resp := messageQueryResponse{
+		LTime:   q.LTime,
+		ID:      q.id,
+		From:    q.serf.config.NodeName,
+		Payload: buf,
+	}
+
+	// Format the response
+	raw, err := encodeMessage(messageQueryResponseType, &resp)
+	if err != nil {
+		return fmt.Errorf("Failed to format response: %v", err)
+	}
+
+	// Check the size limit
+	if len(raw) > QueryResponseSizeLimit {
+		return fmt.Errorf("response exceeds limit of %d bytes", QueryResponseSizeLimit)
+	}
+
+	// Send the response
+	addr := net.UDPAddr{IP: q.addr, Port: int(q.port)}
+	if err := q.serf.memberlist.SendTo(&addr, raw); err != nil {
+		return err
+	}
+
+	// Clera the deadline, response sent
+	q.deadline = time.Time{}
+	return nil
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/event_delegate.go

@@ -0,0 +1,21 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+)
+
+type eventDelegate struct {
+	serf *Serf
+}
+
+func (e *eventDelegate) NotifyJoin(n *memberlist.Node) {
+	e.serf.handleNodeJoin(n)
+}
+
+func (e *eventDelegate) NotifyLeave(n *memberlist.Node) {
+	e.serf.handleNodeLeave(n)
+}
+
+func (e *eventDelegate) NotifyUpdate(n *memberlist.Node) {
+	e.serf.handleNodeUpdate(n)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/event_test.go

@@ -0,0 +1,211 @@
+package serf
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+// testEvents tests that the given node had the given sequence of events
+// on the event channel.
+func testEvents(t *testing.T, ch <-chan Event, node string, expected []EventType) {
+	actual := make([]EventType, 0, len(expected))
+
+TESTEVENTLOOP:
+	for {
+		select {
+		case r := <-ch:
+			e, ok := r.(MemberEvent)
+			if !ok {
+				continue
+			}
+
+			found := false
+			for _, m := range e.Members {
+				if m.Name == node {
+					found = true
+					break
+				}
+			}
+
+			if found {
+				actual = append(actual, e.Type)
+			}
+		case <-time.After(10 * time.Millisecond):
+			break TESTEVENTLOOP
+		}
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("expected events: %v. Got: %v", expected, actual)
+	}
+}
+
+// testUserEvents tests that the given sequence of usr events
+// on the event channel took place.
+func testUserEvents(t *testing.T, ch <-chan Event, expectedName []string, expectedPayload [][]byte) {
+	actualName := make([]string, 0, len(expectedName))
+	actualPayload := make([][]byte, 0, len(expectedPayload))
+
+TESTEVENTLOOP:
+	for {
+		select {
+		case r, ok := <-ch:
+			if !ok {
+				break TESTEVENTLOOP
+			}
+			u, ok := r.(UserEvent)
+			if !ok {
+				continue
+			}
+
+			actualName = append(actualName, u.Name)
+			actualPayload = append(actualPayload, u.Payload)
+		case <-time.After(10 * time.Millisecond):
+			break TESTEVENTLOOP
+		}
+	}
+
+	if !reflect.DeepEqual(actualName, expectedName) {
+		t.Fatalf("expected names: %v. Got: %v", expectedName, actualName)
+	}
+	if !reflect.DeepEqual(actualPayload, expectedPayload) {
+		t.Fatalf("expected payloads: %v. Got: %v", expectedPayload, actualPayload)
+	}
+
+}
+
+// testQueryEvents tests that the given sequence of queries
+// on the event channel took place.
+func testQueryEvents(t *testing.T, ch <-chan Event, expectedName []string, expectedPayload [][]byte) {
+	actualName := make([]string, 0, len(expectedName))
+	actualPayload := make([][]byte, 0, len(expectedPayload))
+
+TESTEVENTLOOP:
+	for {
+		select {
+		case r, ok := <-ch:
+			if !ok {
+				break TESTEVENTLOOP
+			}
+			q, ok := r.(*Query)
+			if !ok {
+				continue
+			}
+
+			actualName = append(actualName, q.Name)
+			actualPayload = append(actualPayload, q.Payload)
+		case <-time.After(10 * time.Millisecond):
+			break TESTEVENTLOOP
+		}
+	}
+
+	if !reflect.DeepEqual(actualName, expectedName) {
+		t.Fatalf("expected names: %v. Got: %v", expectedName, actualName)
+	}
+	if !reflect.DeepEqual(actualPayload, expectedPayload) {
+		t.Fatalf("expected payloads: %v. Got: %v", expectedPayload, actualPayload)
+	}
+
+}
+
+func TestMemberEvent(t *testing.T) {
+	me := MemberEvent{
+		Type:    EventMemberJoin,
+		Members: nil,
+	}
+	if me.EventType() != EventMemberJoin {
+		t.Fatalf("bad event type")
+	}
+	if me.String() != "member-join" {
+		t.Fatalf("bad string val")
+	}
+
+	me.Type = EventMemberLeave
+	if me.EventType() != EventMemberLeave {
+		t.Fatalf("bad event type")
+	}
+	if me.String() != "member-leave" {
+		t.Fatalf("bad string val")
+	}
+
+	me.Type = EventMemberFailed
+	if me.EventType() != EventMemberFailed {
+		t.Fatalf("bad event type")
+	}
+	if me.String() != "member-failed" {
+		t.Fatalf("bad string val")
+	}
+
+	me.Type = EventMemberUpdate
+	if me.EventType() != EventMemberUpdate {
+		t.Fatalf("bad event type")
+	}
+	if me.String() != "member-update" {
+		t.Fatalf("bad string val")
+	}
+
+	me.Type = EventMemberReap
+	if me.EventType() != EventMemberReap {
+		t.Fatalf("bad event type")
+	}
+	if me.String() != "member-reap" {
+		t.Fatalf("bad string val")
+	}
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("expected panic")
+		}
+	}()
+	me.Type = EventUser
+	me.String()
+}
+
+func TestUserEvent(t *testing.T) {
+	ue := UserEvent{
+		Name:    "test",
+		Payload: []byte("foobar"),
+	}
+	if ue.EventType() != EventUser {
+		t.Fatalf("bad event type")
+	}
+	if ue.String() != "user-event: test" {
+		t.Fatalf("bad string val")
+	}
+}
+
+func TestQuery(t *testing.T) {
+	q := Query{
+		LTime:   42,
+		Name:    "update",
+		Payload: []byte("abcd1234"),
+	}
+	if q.EventType() != EventQuery {
+		t.Fatalf("Bad")
+	}
+	if q.String() != "query: update" {
+		t.Fatalf("bad: %v", q.String())
+	}
+}
+
+func TestEventType_String(t *testing.T) {
+	events := []EventType{EventMemberJoin, EventMemberLeave, EventMemberFailed,
+		EventMemberUpdate, EventMemberReap, EventUser, EventQuery}
+	expect := []string{"member-join", "member-leave", "member-failed",
+		"member-update", "member-reap", "user", "query"}
+
+	for idx, event := range events {
+		if event.String() != expect[idx] {
+			t.Fatalf("expect %v got %v", expect[idx], event.String())
+		}
+	}
+
+	other := EventType(100)
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("expected panic")
+		}
+	}()
+	other.String()
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/internal_query.go

@@ -0,0 +1,312 @@
+package serf
+
+import (
+	"encoding/base64"
+	"log"
+	"strings"
+)
+
+const (
+	// This is the prefix we use for queries that are internal to Serf.
+	// They are handled internally, and not forwarded to a client.
+	InternalQueryPrefix = "_serf_"
+
+	// pingQuery is run to check for reachability
+	pingQuery = "ping"
+
+	// conflictQuery is run to resolve a name conflict
+	conflictQuery = "conflict"
+
+	// installKeyQuery is used to install a new key
+	installKeyQuery = "install-key"
+
+	// useKeyQuery is used to change the primary encryption key
+	useKeyQuery = "use-key"
+
+	// removeKeyQuery is used to remove a key from the keyring
+	removeKeyQuery = "remove-key"
+
+	// listKeysQuery is used to list all known keys in the cluster
+	listKeysQuery = "list-keys"
+)
+
+// internalQueryName is used to generate a query name for an internal query
+func internalQueryName(name string) string {
+	return InternalQueryPrefix + name
+}
+
+// serfQueries is used to listen for queries that start with
+// _serf and respond to them as appropriate.
+type serfQueries struct {
+	inCh       chan Event
+	logger     *log.Logger
+	outCh      chan<- Event
+	serf       *Serf
+	shutdownCh <-chan struct{}
+}
+
+// nodeKeyResponse is used to store the result from an individual node while
+// replying to key modification queries
+type nodeKeyResponse struct {
+	// Result indicates true/false if there were errors or not
+	Result bool
+
+	// Message contains error messages or other information
+	Message string
+
+	// Keys is used in listing queries to relay a list of installed keys
+	Keys []string
+}
+
+// newSerfQueries is used to create a new serfQueries. We return an event
+// channel that is ingested and forwarded to an outCh. Any Queries that
+// have the InternalQueryPrefix are handled instead of forwarded.
+func newSerfQueries(serf *Serf, logger *log.Logger, outCh chan<- Event, shutdownCh <-chan struct{}) (chan<- Event, error) {
+	inCh := make(chan Event, 1024)
+	q := &serfQueries{
+		inCh:       inCh,
+		logger:     logger,
+		outCh:      outCh,
+		serf:       serf,
+		shutdownCh: shutdownCh,
+	}
+	go q.stream()
+	return inCh, nil
+}
+
+// stream is a long running routine to ingest the event stream
+func (s *serfQueries) stream() {
+	for {
+		select {
+		case e := <-s.inCh:
+			// Check if this is a query we should process
+			if q, ok := e.(*Query); ok && strings.HasPrefix(q.Name, InternalQueryPrefix) {
+				go s.handleQuery(q)
+
+			} else if s.outCh != nil {
+				s.outCh <- e
+			}
+
+		case <-s.shutdownCh:
+			return
+		}
+	}
+}
+
+// handleQuery is invoked when we get an internal query
+func (s *serfQueries) handleQuery(q *Query) {
+	// Get the queryName after the initial prefix
+	queryName := q.Name[len(InternalQueryPrefix):]
+	switch queryName {
+	case pingQuery:
+		// Nothing to do, we will ack the query
+	case conflictQuery:
+		s.handleConflict(q)
+	case installKeyQuery:
+		s.handleInstallKey(q)
+	case useKeyQuery:
+		s.handleUseKey(q)
+	case removeKeyQuery:
+		s.handleRemoveKey(q)
+	case listKeysQuery:
+		s.handleListKeys(q)
+	default:
+		s.logger.Printf("[WARN] serf: Unhandled internal query '%s'", queryName)
+	}
+}
+
+// handleConflict is invoked when we get a query that is attempting to
+// disambiguate a name conflict. They payload is a node name, and the response
+// should the address we believe that node is at, if any.
+func (s *serfQueries) handleConflict(q *Query) {
+	// The target node name is the payload
+	node := string(q.Payload)
+
+	// Do not respond to the query if it is about us
+	if node == s.serf.config.NodeName {
+		return
+	}
+	s.logger.Printf("[DEBUG] serf: Got conflict resolution query for '%s'", node)
+
+	// Look for the member info
+	var out *Member
+	s.serf.memberLock.Lock()
+	if member, ok := s.serf.members[node]; ok {
+		out = &member.Member
+	}
+	s.serf.memberLock.Unlock()
+
+	// Encode the response
+	buf, err := encodeMessage(messageConflictResponseType, out)
+	if err != nil {
+		s.logger.Printf("[ERR] serf: Failed to encode conflict query response: %v", err)
+		return
+	}
+
+	// Send our answer
+	if err := q.Respond(buf); err != nil {
+		s.logger.Printf("[ERR] serf: Failed to respond to conflict query: %v", err)
+	}
+}
+
+// sendKeyResponse handles responding to key-related queries.
+func (s *serfQueries) sendKeyResponse(q *Query, resp *nodeKeyResponse) {
+	buf, err := encodeMessage(messageKeyResponseType, resp)
+	if err != nil {
+		s.logger.Printf("[ERR] serf: Failed to encode key response: %v", err)
+		return
+	}
+
+	if err := q.Respond(buf); err != nil {
+		s.logger.Printf("[ERR] serf: Failed to respond to key query: %v", err)
+		return
+	}
+}
+
+// handleInstallKey is invoked whenever a new encryption key is received from
+// another member in the cluster, and handles the process of installing it onto
+// the memberlist keyring. This type of query may fail if the provided key does
+// not fit the constraints that memberlist enforces. If the query fails, the
+// response will contain the error message so that it may be relayed.
+func (s *serfQueries) handleInstallKey(q *Query) {
+	response := nodeKeyResponse{Result: false}
+	keyring := s.serf.config.MemberlistConfig.Keyring
+	req := keyRequest{}
+
+	err := decodeMessage(q.Payload[1:], &req)
+	if err != nil {
+		s.logger.Printf("[ERR] serf: Failed to decode key request: %v", err)
+		goto SEND
+	}
+
+	if !s.serf.EncryptionEnabled() {
+		response.Message = "No keyring to modify (encryption not enabled)"
+		s.logger.Printf("[ERR] serf: No keyring to modify (encryption not enabled)")
+		goto SEND
+	}
+
+	s.logger.Printf("[INFO] serf: Received install-key query")
+	if err := keyring.AddKey(req.Key); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to install key: %s", err)
+		goto SEND
+	}
+
+	if err := s.serf.writeKeyringFile(); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to write keyring file: %s", err)
+		goto SEND
+	}
+
+	response.Result = true
+
+SEND:
+	s.sendKeyResponse(q, &response)
+}
+
+// handleUseKey is invoked whenever a query is received to mark a different key
+// in the internal keyring as the primary key. This type of query may fail due
+// to operator error (requested key not in ring), and thus sends error messages
+// back in the response.
+func (s *serfQueries) handleUseKey(q *Query) {
+	response := nodeKeyResponse{Result: false}
+	keyring := s.serf.config.MemberlistConfig.Keyring
+	req := keyRequest{}
+
+	err := decodeMessage(q.Payload[1:], &req)
+	if err != nil {
+		s.logger.Printf("[ERR] serf: Failed to decode key request: %v", err)
+		goto SEND
+	}
+
+	if !s.serf.EncryptionEnabled() {
+		response.Message = "No keyring to modify (encryption not enabled)"
+		s.logger.Printf("[ERR] serf: No keyring to modify (encryption not enabled)")
+		goto SEND
+	}
+
+	s.logger.Printf("[INFO] serf: Received use-key query")
+	if err := keyring.UseKey(req.Key); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to change primary key: %s", err)
+		goto SEND
+	}
+
+	if err := s.serf.writeKeyringFile(); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to write keyring file: %s", err)
+		goto SEND
+	}
+
+	response.Result = true
+
+SEND:
+	s.sendKeyResponse(q, &response)
+}
+
+// handleRemoveKey is invoked when a query is received to remove a particular
+// key from the keyring. This type of query can fail if the key requested for
+// deletion is currently the primary key in the keyring, so therefore it will
+// reply to the query with any relevant errors from the operation.
+func (s *serfQueries) handleRemoveKey(q *Query) {
+	response := nodeKeyResponse{Result: false}
+	keyring := s.serf.config.MemberlistConfig.Keyring
+	req := keyRequest{}
+
+	err := decodeMessage(q.Payload[1:], &req)
+	if err != nil {
+		s.logger.Printf("[ERR] serf: Failed to decode key request: %v", err)
+		goto SEND
+	}
+
+	if !s.serf.EncryptionEnabled() {
+		response.Message = "No keyring to modify (encryption not enabled)"
+		s.logger.Printf("[ERR] serf: No keyring to modify (encryption not enabled)")
+		goto SEND
+	}
+
+	s.logger.Printf("[INFO] serf: Received remove-key query")
+	if err := keyring.RemoveKey(req.Key); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to remove key: %s", err)
+		goto SEND
+	}
+
+	if err := s.serf.writeKeyringFile(); err != nil {
+		response.Message = err.Error()
+		s.logger.Printf("[ERR] serf: Failed to write keyring file: %s", err)
+		goto SEND
+	}
+
+	response.Result = true
+
+SEND:
+	s.sendKeyResponse(q, &response)
+}
+
+// handleListKeys is invoked when a query is received to return a list of all
+// installed keys the Serf instance knows of. For performance, the keys are
+// encoded to base64 on each of the members to remove this burden from the
+// node asking for the results.
+func (s *serfQueries) handleListKeys(q *Query) {
+	response := nodeKeyResponse{Result: false}
+	keyring := s.serf.config.MemberlistConfig.Keyring
+
+	if !s.serf.EncryptionEnabled() {
+		response.Message = "Keyring is empty (encryption not enabled)"
+		s.logger.Printf("[ERR] serf: Keyring is empty (encryption not enabled)")
+		goto SEND
+	}
+
+	s.logger.Printf("[INFO] serf: Received list-keys query")
+	for _, keyBytes := range keyring.GetKeys() {
+		// Encode the keys before sending the response. This should help take
+		// some the burden of doing this off of the asking member.
+		key := base64.StdEncoding.EncodeToString(keyBytes)
+		response.Keys = append(response.Keys, key)
+	}
+	response.Result = true
+
+SEND:
+	s.sendKeyResponse(q, &response)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/internal_query_test.go

@@ -0,0 +1,89 @@
+package serf
+
+import (
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestInternalQueryName(t *testing.T) {
+	name := internalQueryName(conflictQuery)
+	if name != "_serf_conflict" {
+		t.Fatalf("bad: %v", name)
+	}
+}
+
+func TestSerfQueries_Passthrough(t *testing.T) {
+	serf := &Serf{}
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	outCh := make(chan Event, 4)
+	shutdown := make(chan struct{})
+	defer close(shutdown)
+	eventCh, err := newSerfQueries(serf, logger, outCh, shutdown)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Push a user event
+	eventCh <- UserEvent{LTime: 42, Name: "foo"}
+
+	// Push a query
+	eventCh <- &Query{LTime: 42, Name: "foo"}
+
+	// Push a query
+	eventCh <- MemberEvent{Type: EventMemberJoin}
+
+	// Should get passed through
+	for i := 0; i < 3; i++ {
+		select {
+		case <-outCh:
+		case <-time.After(100 * time.Millisecond):
+			t.Fatalf("time out")
+		}
+	}
+}
+
+func TestSerfQueries_Ping(t *testing.T) {
+	serf := &Serf{}
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	outCh := make(chan Event, 4)
+	shutdown := make(chan struct{})
+	defer close(shutdown)
+	eventCh, err := newSerfQueries(serf, logger, outCh, shutdown)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Send a ping
+	eventCh <- &Query{LTime: 42, Name: "_serf_ping"}
+
+	// Should not get passed through
+	select {
+	case <-outCh:
+		t.Fatalf("Should not passthrough query!")
+	case <-time.After(50 * time.Millisecond):
+	}
+}
+
+func TestSerfQueries_Conflict_SameName(t *testing.T) {
+	serf := &Serf{config: &Config{NodeName: "foo"}}
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	outCh := make(chan Event, 4)
+	shutdown := make(chan struct{})
+	defer close(shutdown)
+	eventCh, err := newSerfQueries(serf, logger, outCh, shutdown)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Query for our own name
+	eventCh <- &Query{Name: "_serf_conflict", Payload: []byte("foo")}
+
+	// Should not passthrough OR respond
+	select {
+	case <-outCh:
+		t.Fatalf("Should not passthrough query!")
+	case <-time.After(50 * time.Millisecond):
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/keymanager.go

@@ -0,0 +1,166 @@
+package serf
+
+import (
+	"encoding/base64"
+	"fmt"
+	"sync"
+)
+
+// KeyManager encapsulates all functionality within Serf for handling
+// encryption keyring changes across a cluster.
+type KeyManager struct {
+	serf *Serf
+
+	// Lock to protect read and write operations
+	l sync.RWMutex
+}
+
+// keyRequest is used to contain input parameters which get broadcasted to all
+// nodes as part of a key query operation.
+type keyRequest struct {
+	Key []byte
+}
+
+// KeyResponse is used to relay a query for a list of all keys in use.
+type KeyResponse struct {
+	Messages map[string]string // Map of node name to response message
+	NumNodes int               // Total nodes memberlist knows of
+	NumResp  int               // Total responses received
+	NumErr   int               // Total errors from request
+
+	// Keys is a mapping of the base64-encoded value of the key bytes to the
+	// number of nodes that have the key installed.
+	Keys map[string]int
+}
+
+// streamKeyResp takes care of reading responses from a channel and composing
+// them into a KeyResponse. It will update a KeyResponse *in place* and
+// therefore has nothing to return.
+func (k *KeyManager) streamKeyResp(resp *KeyResponse, ch <-chan NodeResponse) {
+	for r := range ch {
+		var nodeResponse nodeKeyResponse
+
+		resp.NumResp++
+
+		// Decode the response
+		if len(r.Payload) < 1 || messageType(r.Payload[0]) != messageKeyResponseType {
+			resp.Messages[r.From] = fmt.Sprintf(
+				"Invalid key query response type: %v", r.Payload)
+			resp.NumErr++
+			goto NEXT
+		}
+		if err := decodeMessage(r.Payload[1:], &nodeResponse); err != nil {
+			resp.Messages[r.From] = fmt.Sprintf(
+				"Failed to decode key query response: %v", r.Payload)
+			resp.NumErr++
+			goto NEXT
+		}
+
+		if !nodeResponse.Result {
+			resp.Messages[r.From] = nodeResponse.Message
+			resp.NumErr++
+		}
+
+		// Currently only used for key list queries, this adds keys to a counter
+		// and increments them for each node response which contains them.
+		for _, key := range nodeResponse.Keys {
+			if _, ok := resp.Keys[key]; !ok {
+				resp.Keys[key] = 1
+			} else {
+				resp.Keys[key]++
+			}
+		}
+
+	NEXT:
+		// Return early if all nodes have responded. This allows us to avoid
+		// waiting for the full timeout when there is nothing left to do.
+		if resp.NumResp == resp.NumNodes {
+			return
+		}
+	}
+}
+
+// handleKeyRequest performs query broadcasting to all members for any type of
+// key operation and manages gathering responses and packing them up into a
+// KeyResponse for uniform response handling.
+func (k *KeyManager) handleKeyRequest(key, query string) (*KeyResponse, error) {
+	resp := &KeyResponse{
+		Messages: make(map[string]string),
+		Keys:     make(map[string]int),
+	}
+	qName := internalQueryName(query)
+
+	// Decode the new key into raw bytes
+	rawKey, err := base64.StdEncoding.DecodeString(key)
+	if err != nil {
+		return resp, err
+	}
+
+	// Encode the query request
+	req, err := encodeMessage(messageKeyRequestType, keyRequest{Key: rawKey})
+	if err != nil {
+		return resp, err
+	}
+
+	qParam := k.serf.DefaultQueryParams()
+	queryResp, err := k.serf.Query(qName, req, qParam)
+	if err != nil {
+		return resp, err
+	}
+
+	// Handle the response stream and populate the KeyResponse
+	resp.NumNodes = k.serf.memberlist.NumMembers()
+	k.streamKeyResp(resp, queryResp.respCh)
+
+	// Check the response for any reported failure conditions
+	if resp.NumErr != 0 {
+		return resp, fmt.Errorf("%d/%d nodes reported failure", resp.NumErr, resp.NumNodes)
+	}
+	if resp.NumResp != resp.NumNodes {
+		return resp, fmt.Errorf("%d/%d nodes reported success", resp.NumResp, resp.NumNodes)
+	}
+
+	return resp, nil
+}
+
+// InstallKey handles broadcasting a query to all members and gathering
+// responses from each of them, returning a list of messages from each node
+// and any applicable error conditions.
+func (k *KeyManager) InstallKey(key string) (*KeyResponse, error) {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	return k.handleKeyRequest(key, installKeyQuery)
+}
+
+// UseKey handles broadcasting a primary key change to all members in the
+// cluster, and gathering any response messages. If successful, there should
+// be an empty KeyResponse returned.
+func (k *KeyManager) UseKey(key string) (*KeyResponse, error) {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	return k.handleKeyRequest(key, useKeyQuery)
+}
+
+// RemoveKey handles broadcasting a key to the cluster for removal. Each member
+// will receive this event, and if they have the key in their keyring, remove
+// it. If any errors are encountered, RemoveKey will collect and relay them.
+func (k *KeyManager) RemoveKey(key string) (*KeyResponse, error) {
+	k.l.Lock()
+	defer k.l.Unlock()
+
+	return k.handleKeyRequest(key, removeKeyQuery)
+}
+
+// ListKeys is used to collect installed keys from members in a Serf cluster
+// and return an aggregated list of all installed keys. This is useful to
+// operators to ensure that there are no lingering keys installed on any agents.
+// Since having multiple keys installed can cause performance penalties in some
+// cases, it's important to verify this information and remove unneeded keys.
+func (k *KeyManager) ListKeys() (*KeyResponse, error) {
+	k.l.RLock()
+	defer k.l.RUnlock()
+
+	return k.handleKeyRequest("", listKeysQuery)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/keymanager_test.go

@@ -0,0 +1,272 @@
+package serf
+
+import (
+	"bytes"
+	"encoding/base64"
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/testutil"
+	"testing"
+)
+
+func testKeyring() (*memberlist.Keyring, error) {
+	keys := []string{
+		"enjTwAFRe4IE71bOFhirzQ==",
+		"csT9mxI7aTf9ap3HLBbdmA==",
+		"noha2tVc0OyD/2LtCBoAOQ==",
+	}
+
+	keysDecoded := make([][]byte, len(keys))
+	for i, key := range keys {
+		decoded, err := base64.StdEncoding.DecodeString(key)
+		if err != nil {
+			return nil, err
+		}
+		keysDecoded[i] = decoded
+	}
+
+	return memberlist.NewKeyring(keysDecoded, keysDecoded[0])
+}
+
+func testKeyringSerf() (*Serf, error) {
+	config := testConfig()
+
+	keyring, err := testKeyring()
+	if err != nil {
+		return nil, err
+	}
+	config.MemberlistConfig.Keyring = keyring
+
+	s, err := Create(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func keyExistsInRing(kr *memberlist.Keyring, key []byte) bool {
+	for _, installedKey := range kr.GetKeys() {
+		if bytes.Equal(key, installedKey) {
+			return true
+		}
+	}
+	return false
+}
+
+func TestSerf_InstallKey(t *testing.T) {
+	s1, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s1.Shutdown()
+
+	s2, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s2.Shutdown()
+
+	primaryKey := s1.config.MemberlistConfig.Keyring.GetPrimaryKey()
+
+	// Join s1 and s2
+	_, err = s1.Join([]string{s2.config.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	// Begin tests
+	newKey := "l4ZkaypGLT8AsB0LBldthw=="
+	newKeyBytes, err := base64.StdEncoding.DecodeString(newKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	manager := s1.KeyManager()
+
+	// Install a new key onto the existing ring. This is a blocking call, so no
+	// need for a yield.
+	_, err = manager.InstallKey(newKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Key installation did not affect the current primary key
+	if !bytes.Equal(primaryKey, s1.config.MemberlistConfig.Keyring.GetPrimaryKey()) {
+		t.Fatal("Unexpected primary key change on s1")
+	}
+
+	if !bytes.Equal(primaryKey, s2.config.MemberlistConfig.Keyring.GetPrimaryKey()) {
+		t.Fatal("Unexpected primary key change on s2")
+	}
+
+	// New key was successfully broadcasted and installed on all members
+	if !keyExistsInRing(s1.config.MemberlistConfig.Keyring, newKeyBytes) {
+		t.Fatal("Newly-installed key not found in keyring on s1")
+	}
+
+	if !keyExistsInRing(s2.config.MemberlistConfig.Keyring, newKeyBytes) {
+		t.Fatal("Newly-installed key not found in keyring on s2")
+	}
+}
+
+func TestSerf_UseKey(t *testing.T) {
+	s1, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s1.Shutdown()
+
+	s2, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s2.Shutdown()
+
+	// Join s1 and s2
+	_, err = s1.Join([]string{s2.config.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	// Begin tests
+	useKey := "csT9mxI7aTf9ap3HLBbdmA=="
+	useKeyBytes, err := base64.StdEncoding.DecodeString(useKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	manager := s1.KeyManager()
+
+	// Change the primary encryption key
+	_, err = manager.UseKey(useKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// First make sure that the primary key is what we expect it to be
+	if !bytes.Equal(useKeyBytes, s1.config.MemberlistConfig.Keyring.GetPrimaryKey()) {
+		t.Fatal("Unexpected primary key on s1")
+	}
+
+	if !bytes.Equal(useKeyBytes, s2.config.MemberlistConfig.Keyring.GetPrimaryKey()) {
+		t.Fatal("Unexpected primary key on s2")
+	}
+
+	// Make sure an error is thrown if the key doesn't exist
+	_, err = manager.UseKey("aE6AfGEvay+UJbkfxBk4SQ==")
+	if err == nil {
+		t.Fatalf("Expected error changing to non-existent primary key")
+	}
+}
+
+func TestSerf_RemoveKey(t *testing.T) {
+	s1, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s1.Shutdown()
+
+	s2, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s2.Shutdown()
+
+	// Join s1 and s2
+	_, err = s1.Join([]string{s2.config.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	// Begin tests
+	removeKey := "noha2tVc0OyD/2LtCBoAOQ=="
+	removeKeyBytes, err := base64.StdEncoding.DecodeString(removeKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	manager := s1.KeyManager()
+
+	// Remove a key from the ring
+	_, err = manager.RemoveKey(removeKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Key was successfully removed from all members
+	if keyExistsInRing(s1.config.MemberlistConfig.Keyring, removeKeyBytes) {
+		t.Fatal("Key not removed from keyring on s1")
+	}
+
+	if keyExistsInRing(s2.config.MemberlistConfig.Keyring, removeKeyBytes) {
+		t.Fatal("Key not removed from keyring on s2")
+	}
+}
+
+func TestSerf_ListKeys(t *testing.T) {
+	s1, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s1.Shutdown()
+
+	s2, err := testKeyringSerf()
+	if err != nil {
+		t.Fatalf("%s", err)
+	}
+	defer s2.Shutdown()
+
+	manager := s1.KeyManager()
+
+	initialKeyringLen := len(s1.config.MemberlistConfig.Keyring.GetKeys())
+
+	// Extra key on s2 to make sure we see it in the list
+	extraKey := "JHAxGU13qDaXhUW6jIpyog=="
+	extraKeyBytes, err := base64.StdEncoding.DecodeString(extraKey)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	s2.config.MemberlistConfig.Keyring.AddKey(extraKeyBytes)
+
+	// Join s1 and s2
+	_, err = s1.Join([]string{s2.config.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	resp, err := manager.ListKeys()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Found all keys in the list
+	expected := initialKeyringLen + 1
+	if expected != len(resp.Keys) {
+		t.Fatalf("Expected %d keys in result, found %d", expected, len(resp.Keys))
+	}
+
+	found := false
+	for key, _ := range resp.Keys {
+		if key == extraKey {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("Did not find expected key in list: %s", extraKey)
+	}
+
+	// Number of members with extra key installed should be 1
+	for key, num := range resp.Keys {
+		if key == extraKey && num != 1 {
+			t.Fatalf("Expected 1 nodes with key %s but have %d", extraKey, num)
+		}
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/lamport.go

@@ -0,0 +1,45 @@
+package serf
+
+import (
+	"sync/atomic"
+)
+
+// LamportClock is a thread safe implementation of a lamport clock. It
+// uses efficient atomic operations for all of its functions, falling back
+// to a heavy lock only if there are enough CAS failures.
+type LamportClock struct {
+	counter uint64
+}
+
+// LamportTime is the value of a LamportClock.
+type LamportTime uint64
+
+// Time is used to return the current value of the lamport clock
+func (l *LamportClock) Time() LamportTime {
+	return LamportTime(atomic.LoadUint64(&l.counter))
+}
+
+// Increment is used to increment and return the value of the lamport clock
+func (l *LamportClock) Increment() LamportTime {
+	return LamportTime(atomic.AddUint64(&l.counter, 1))
+}
+
+// Witness is called to update our local clock if necessary after
+// witnessing a clock value received from another process
+func (l *LamportClock) Witness(v LamportTime) {
+WITNESS:
+	// If the other value is old, we do not need to do anything
+	cur := atomic.LoadUint64(&l.counter)
+	other := uint64(v)
+	if other < cur {
+		return
+	}
+
+	// Ensure that our local clock is at least one ahead.
+	if !atomic.CompareAndSwapUint64(&l.counter, cur, other+1) {
+		// The CAS failed, so we just retry. Eventually our CAS should
+		// succeed or a future witness will pass us by and our witness
+		// will end.
+		goto WITNESS
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/lamport_test.go

@@ -0,0 +1,39 @@
+package serf
+
+import (
+	"testing"
+)
+
+func TestLamportClock(t *testing.T) {
+	l := &LamportClock{}
+
+	if l.Time() != 0 {
+		t.Fatalf("bad time value")
+	}
+
+	if l.Increment() != 1 {
+		t.Fatalf("bad time value")
+	}
+
+	if l.Time() != 1 {
+		t.Fatalf("bad time value")
+	}
+
+	l.Witness(41)
+
+	if l.Time() != 42 {
+		t.Fatalf("bad time value")
+	}
+
+	l.Witness(41)
+
+	if l.Time() != 42 {
+		t.Fatalf("bad time value")
+	}
+
+	l.Witness(30)
+
+	if l.Time() != 42 {
+		t.Fatalf("bad time value")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/merge_delegate.go

@@ -0,0 +1,35 @@
+package serf
+
+import (
+	"net"
+
+	"github.com/hashicorp/memberlist"
+)
+
+type MergeDelegate interface {
+	NotifyMerge([]*Member) (cancel bool)
+}
+
+type mergeDelegate struct {
+	serf *Serf
+}
+
+func (m *mergeDelegate) NotifyMerge(nodes []*memberlist.Node) (cancel bool) {
+	members := make([]*Member, len(nodes))
+	for idx, n := range nodes {
+		members[idx] = &Member{
+			Name:        n.Name,
+			Addr:        net.IP(n.Addr),
+			Port:        n.Port,
+			Tags:        m.serf.decodeTags(n.Meta),
+			Status:      StatusNone,
+			ProtocolMin: n.PMin,
+			ProtocolMax: n.PMax,
+			ProtocolCur: n.PCur,
+			DelegateMin: n.DMin,
+			DelegateMax: n.DMax,
+			DelegateCur: n.DCur,
+		}
+	}
+	return m.serf.config.Merge.NotifyMerge(members)
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/messages.go

@@ -0,0 +1,147 @@
+package serf
+
+import (
+	"bytes"
+	"github.com/hashicorp/go-msgpack/codec"
+	"time"
+)
+
+// messageType are the types of gossip messages Serf will send along
+// memberlist.
+type messageType uint8
+
+const (
+	messageLeaveType messageType = iota
+	messageJoinType
+	messagePushPullType
+	messageUserEventType
+	messageQueryType
+	messageQueryResponseType
+	messageConflictResponseType
+	messageKeyRequestType
+	messageKeyResponseType
+)
+
+const (
+	// Ack flag is used to force receiver to send an ack back
+	queryFlagAck uint32 = 1 << iota
+
+	// NoBroadcast is used to prevent re-broadcast of a query.
+	// this can be used to selectively send queries to individual members
+	queryFlagNoBroadcast
+)
+
+// filterType is used with a queryFilter to specify the type of
+// filter we are sending
+type filterType uint8
+
+const (
+	filterNodeType filterType = iota
+	filterTagType
+)
+
+// messageJoin is the message broadcasted after we join to
+// associated the node with a lamport clock
+type messageJoin struct {
+	LTime LamportTime
+	Node  string
+}
+
+// messageLeave is the message broadcasted to signal the intentional to
+// leave.
+type messageLeave struct {
+	LTime LamportTime
+	Node  string
+}
+
+// messagePushPullType is used when doing a state exchange. This
+// is a relatively large message, but is sent infrequently
+type messagePushPull struct {
+	LTime        LamportTime            // Current node lamport time
+	StatusLTimes map[string]LamportTime // Maps the node to its status time
+	LeftMembers  []string               // List of left nodes
+	EventLTime   LamportTime            // Lamport time for event clock
+	Events       []*userEvents          // Recent events
+	QueryLTime   LamportTime            // Lamport time for query clock
+}
+
+// messageUserEvent is used for user-generated events
+type messageUserEvent struct {
+	LTime   LamportTime
+	Name    string
+	Payload []byte
+	CC      bool // "Can Coalesce". Zero value is compatible with Serf 0.1
+}
+
+// messageQuery is used for query events
+type messageQuery struct {
+	LTime   LamportTime   // Event lamport time
+	ID      uint32        // Query ID, randomly generated
+	Addr    []byte        // Source address, used for a direct reply
+	Port    uint16        // Source port, used for a direct reply
+	Filters [][]byte      // Potential query filters
+	Flags   uint32        // Used to provide various flags
+	Timeout time.Duration // Maximum time between delivery and response
+	Name    string        // Query name
+	Payload []byte        // Query payload
+}
+
+// Ack checks if the ack flag is set
+func (m *messageQuery) Ack() bool {
+	return (m.Flags & queryFlagAck) != 0
+}
+
+// NoBroadcast checks if the no broadcast flag is set
+func (m *messageQuery) NoBroadcast() bool {
+	return (m.Flags & queryFlagNoBroadcast) != 0
+}
+
+// filterNode is used with the filterNodeType, and is a list
+// of node names
+type filterNode []string
+
+// filterTag is used with the filterTagType and is a regular
+// expression to apply to a tag
+type filterTag struct {
+	Tag  string
+	Expr string
+}
+
+// messageQueryResponse is used to respond to a query
+type messageQueryResponse struct {
+	LTime   LamportTime // Event lamport time
+	ID      uint32      // Query ID
+	From    string      // Node name
+	Flags   uint32      // Used to provide various flags
+	Payload []byte      // Optional response payload
+}
+
+// Ack checks if the ack flag is set
+func (m *messageQueryResponse) Ack() bool {
+	return (m.Flags & queryFlagAck) != 0
+}
+
+func decodeMessage(buf []byte, out interface{}) error {
+	var handle codec.MsgpackHandle
+	return codec.NewDecoder(bytes.NewReader(buf), &handle).Decode(out)
+}
+
+func encodeMessage(t messageType, msg interface{}) ([]byte, error) {
+	buf := bytes.NewBuffer(nil)
+	buf.WriteByte(uint8(t))
+
+	handle := codec.MsgpackHandle{}
+	encoder := codec.NewEncoder(buf, &handle)
+	err := encoder.Encode(msg)
+	return buf.Bytes(), err
+}
+
+func encodeFilter(f filterType, filt interface{}) ([]byte, error) {
+	buf := bytes.NewBuffer(nil)
+	buf.WriteByte(uint8(f))
+
+	handle := codec.MsgpackHandle{}
+	encoder := codec.NewEncoder(buf, &handle)
+	err := encoder.Encode(filt)
+	return buf.Bytes(), err
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/messages_test.go

@@ -0,0 +1,58 @@
+package serf
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestQueryFlags(t *testing.T) {
+	if queryFlagAck != 1 {
+		t.Fatalf("Bad: %v", queryFlagAck)
+	}
+	if queryFlagNoBroadcast != 2 {
+		t.Fatalf("Bad: %v", queryFlagNoBroadcast)
+	}
+}
+
+func TestEncodeMessage(t *testing.T) {
+	in := &messageLeave{Node: "foo"}
+	raw, err := encodeMessage(messageLeaveType, in)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if raw[0] != byte(messageLeaveType) {
+		t.Fatal("should have type header")
+	}
+
+	var out messageLeave
+	if err := decodeMessage(raw[1:], &out); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !reflect.DeepEqual(in, &out) {
+		t.Fatalf("mis-match")
+	}
+}
+
+func TestEncodeFilter(t *testing.T) {
+	nodes := []string{"foo", "bar"}
+
+	raw, err := encodeFilter(filterNodeType, nodes)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if raw[0] != byte(filterNodeType) {
+		t.Fatal("should have type header")
+	}
+
+	var out []string
+	if err := decodeMessage(raw[1:], &out); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !reflect.DeepEqual(nodes, out) {
+		t.Fatalf("mis-match")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/query.go

@@ -0,0 +1,210 @@
+package serf
+
+import (
+	"math"
+	"regexp"
+	"sync"
+	"time"
+)
+
+// QueryParam is provided to Query() to configure the parameters of the
+// query. If not provided, sane defaults will be used.
+type QueryParam struct {
+	// If provided, we restrict the nodes that should respond to those
+	// with names in this list
+	FilterNodes []string
+
+	// FilterTags maps a tag name to a regular expression that is applied
+	// to restrict the nodes that should respond
+	FilterTags map[string]string
+
+	// If true, we are requesting an delivery acknowledgement from
+	// every node that meets the filter requirement. This means nodes
+	// the receive the message but do not pass the filters, will not
+	// send an ack.
+	RequestAck bool
+
+	// The timeout limits how long the query is left open. If not provided,
+	// then a default timeout is used based on the configuration of Serf
+	Timeout time.Duration
+}
+
+// DefaultQueryTimeout returns the default timeout value for a query
+// Computed as GossipInterval * QueryTimeoutMult * log(N+1)
+func (s *Serf) DefaultQueryTimeout() time.Duration {
+	n := s.memberlist.NumMembers()
+	timeout := s.config.MemberlistConfig.GossipInterval
+	timeout *= time.Duration(s.config.QueryTimeoutMult)
+	timeout *= time.Duration(math.Ceil(math.Log10(float64(n + 1))))
+	return timeout
+}
+
+// DefaultQueryParam is used to return the default query parameters
+func (s *Serf) DefaultQueryParams() *QueryParam {
+	return &QueryParam{
+		FilterNodes: nil,
+		FilterTags:  nil,
+		RequestAck:  false,
+		Timeout:     s.DefaultQueryTimeout(),
+	}
+}
+
+// encodeFilters is used to convert the filters into the wire format
+func (q *QueryParam) encodeFilters() ([][]byte, error) {
+	var filters [][]byte
+
+	// Add the node filter
+	if len(q.FilterNodes) > 0 {
+		if buf, err := encodeFilter(filterNodeType, q.FilterNodes); err != nil {
+			return nil, err
+		} else {
+			filters = append(filters, buf)
+		}
+	}
+
+	// Add the tag filters
+	for tag, expr := range q.FilterTags {
+		filt := filterTag{tag, expr}
+		if buf, err := encodeFilter(filterTagType, &filt); err != nil {
+			return nil, err
+		} else {
+			filters = append(filters, buf)
+		}
+	}
+
+	return filters, nil
+}
+
+// QueryResponse is returned for each new Query. It is used to collect
+// Ack's as well as responses and to provide those back to a client.
+type QueryResponse struct {
+	// ackCh is used to send the name of a node for which we've received an ack
+	ackCh chan string
+
+	// deadline is the query end time (start + query timeout)
+	deadline time.Time
+
+	// Query ID
+	id uint32
+
+	// Stores the LTime of the query
+	lTime LamportTime
+
+	// respCh is used to send a response from a node
+	respCh chan NodeResponse
+
+	closed    bool
+	closeLock sync.Mutex
+}
+
+// newQueryResponse is used to construct a new query response
+func newQueryResponse(n int, q *messageQuery) *QueryResponse {
+	resp := &QueryResponse{
+		deadline: time.Now().Add(q.Timeout),
+		id:       q.ID,
+		lTime:    q.LTime,
+		respCh:   make(chan NodeResponse, n),
+	}
+	if q.Ack() {
+		resp.ackCh = make(chan string, n)
+	}
+	return resp
+}
+
+// Close is used to close the query, which will close the underlying
+// channels and prevent further deliveries
+func (r *QueryResponse) Close() {
+	r.closeLock.Lock()
+	defer r.closeLock.Unlock()
+	if r.closed {
+		return
+	}
+	r.closed = true
+	if r.ackCh != nil {
+		close(r.ackCh)
+	}
+	if r.respCh != nil {
+		close(r.respCh)
+	}
+}
+
+// Deadline returns the ending deadline of the query
+func (r *QueryResponse) Deadline() time.Time {
+	return r.deadline
+}
+
+// Finished returns if the query is finished running
+func (r *QueryResponse) Finished() bool {
+	return r.closed || time.Now().After(r.deadline)
+}
+
+// AckCh returns a channel that can be used to listen for acks
+// Channel will be closed when the query is finished. This is nil,
+// if the query did not specify RequestAck.
+func (r *QueryResponse) AckCh() <-chan string {
+	return r.ackCh
+}
+
+// ResponseCh returns a channel that can be used to listen for responses.
+// Channel will be closed when the query is finished.
+func (r *QueryResponse) ResponseCh() <-chan NodeResponse {
+	return r.respCh
+}
+
+// NodeResponse is used to represent a single response from a node
+type NodeResponse struct {
+	From    string
+	Payload []byte
+}
+
+// shouldProcessQuery checks if a query should be proceeded given
+// a set of filers.
+func (s *Serf) shouldProcessQuery(filters [][]byte) bool {
+	for _, filter := range filters {
+		switch filterType(filter[0]) {
+		case filterNodeType:
+			// Decode the filter
+			var nodes filterNode
+			if err := decodeMessage(filter[1:], &nodes); err != nil {
+				s.logger.Printf("[WARN] serf: failed to decode filterNodeType: %v", err)
+				return false
+			}
+
+			// Check if we are being targeted
+			found := false
+			for _, n := range nodes {
+				if n == s.config.NodeName {
+					found = true
+					break
+				}
+			}
+			if !found {
+				return false
+			}
+
+		case filterTagType:
+			// Decode the filter
+			var filt filterTag
+			if err := decodeMessage(filter[1:], &filt); err != nil {
+				s.logger.Printf("[WARN] serf: failed to decode filterTagType: %v", err)
+				return false
+			}
+
+			// Check if we match this regex
+			tags := s.config.Tags
+			matched, err := regexp.MatchString(filt.Expr, tags[filt.Tag])
+			if err != nil {
+				s.logger.Printf("[WARN] serf: failed to compile filter regex (%s): %v", filt.Expr, err)
+				return false
+			}
+			if !matched {
+				return false
+			}
+
+		default:
+			s.logger.Printf("[WARN] serf: query has unrecognized filter type: %d", filter[0])
+			return false
+		}
+	}
+	return true
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/query_test.go

@@ -0,0 +1,138 @@
+package serf
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDefaultQuery(t *testing.T) {
+	s1Config := testConfig()
+	s1, err := Create(s1Config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s1.Shutdown()
+
+	timeout := s1.DefaultQueryTimeout()
+	if timeout != s1Config.MemberlistConfig.GossipInterval*time.Duration(s1Config.QueryTimeoutMult) {
+		t.Fatalf("bad: %v", timeout)
+	}
+
+	params := s1.DefaultQueryParams()
+	if params.FilterNodes != nil {
+		t.Fatalf("bad: %v", *params)
+	}
+	if params.FilterTags != nil {
+		t.Fatalf("bad: %v", *params)
+	}
+	if params.RequestAck {
+		t.Fatalf("bad: %v", *params)
+	}
+	if params.Timeout != timeout {
+		t.Fatalf("bad: %v", *params)
+	}
+}
+
+func TestQueryParams_EncodeFilters(t *testing.T) {
+	q := &QueryParam{
+		FilterNodes: []string{"foo", "bar"},
+		FilterTags: map[string]string{
+			"role":       "^web",
+			"datacenter": "aws$",
+		},
+	}
+
+	filters, err := q.encodeFilters()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if len(filters) != 3 {
+		t.Fatalf("bad: %v", filters)
+	}
+
+	nodeFilt := filters[0]
+	if filterType(nodeFilt[0]) != filterNodeType {
+		t.Fatalf("bad: %v", nodeFilt)
+	}
+
+	tagFilt := filters[1]
+	if filterType(tagFilt[0]) != filterTagType {
+		t.Fatalf("bad: %v", tagFilt)
+	}
+
+	tagFilt = filters[2]
+	if filterType(tagFilt[0]) != filterTagType {
+		t.Fatalf("bad: %v", tagFilt)
+	}
+}
+
+func TestSerf_ShouldProcess(t *testing.T) {
+	s1Config := testConfig()
+	s1Config.NodeName = "zip"
+	s1Config.Tags = map[string]string{
+		"role":       "webserver",
+		"datacenter": "east-aws",
+	}
+	s1, err := Create(s1Config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s1.Shutdown()
+
+	// Try a matching query
+	q := &QueryParam{
+		FilterNodes: []string{"foo", "bar", "zip"},
+		FilterTags: map[string]string{
+			"role":       "^web",
+			"datacenter": "aws$",
+		},
+	}
+	filters, err := q.encodeFilters()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !s1.shouldProcessQuery(filters) {
+		t.Fatalf("expected true")
+	}
+
+	// Omit node
+	q = &QueryParam{
+		FilterNodes: []string{"foo", "bar"},
+	}
+	filters, err = q.encodeFilters()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if s1.shouldProcessQuery(filters) {
+		t.Fatalf("expected false")
+	}
+
+	// Filter on missing tag
+	q = &QueryParam{
+		FilterTags: map[string]string{
+			"other": "cool",
+		},
+	}
+	filters, err = q.encodeFilters()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if s1.shouldProcessQuery(filters) {
+		t.Fatalf("expected false")
+	}
+
+	// Bad tag
+	q = &QueryParam{
+		FilterTags: map[string]string{
+			"role": "db",
+		},
+	}
+	filters, err = q.encodeFilters()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if s1.shouldProcessQuery(filters) {
+		t.Fatalf("expected false")
+	}
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/serf_internals_test.go

@@ -0,0 +1,442 @@
+package serf
+
+import (
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/testutil"
+	"testing"
+)
+
+func TestSerf_joinLeave_ltime(t *testing.T) {
+	s1Config := testConfig()
+	s2Config := testConfig()
+
+	s1, err := Create(s1Config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s1.Shutdown()
+
+	s2, err := Create(s2Config)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s2.Shutdown()
+
+	testutil.Yield()
+
+	_, err = s1.Join([]string{s2Config.MemberlistConfig.BindAddr}, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	if s2.members[s1.config.NodeName].statusLTime != 1 {
+		t.Fatalf("join time is not valid %d",
+			s2.members[s1.config.NodeName].statusLTime)
+	}
+
+	if s2.clock.Time() <= s2.members[s1.config.NodeName].statusLTime {
+		t.Fatalf("join should increment")
+	}
+	oldClock := s2.clock.Time()
+
+	err = s1.Leave()
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	testutil.Yield()
+
+	// s1 clock should exceed s2 due to leave
+	if s2.clock.Time() <= oldClock {
+		t.Fatalf("leave should increment (%d / %d)",
+			s2.clock.Time(), oldClock)
+	}
+}
+
+func TestSerf_join_pendingIntent(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.recentJoin[0] = nodeIntent{5, "test"}
+
+	n := memberlist.Node{Name: "test",
+		Addr: nil,
+		Meta: []byte("test"),
+	}
+
+	s.handleNodeJoin(&n)
+
+	mem := s.members["test"]
+	if mem.statusLTime != 5 {
+		t.Fatalf("bad join time")
+	}
+	if mem.Status != StatusAlive {
+		t.Fatalf("bad status")
+	}
+}
+
+func TestSerf_join_pendingIntents(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.recentJoin[0] = nodeIntent{5, "test"}
+	s.recentLeave[0] = nodeIntent{6, "test"}
+
+	n := memberlist.Node{Name: "test",
+		Addr: nil,
+		Meta: []byte("test"),
+	}
+
+	s.handleNodeJoin(&n)
+
+	mem := s.members["test"]
+	if mem.statusLTime != 6 {
+		t.Fatalf("bad join time")
+	}
+	if mem.Status != StatusLeaving {
+		t.Fatalf("bad status")
+	}
+}
+
+func TestSerf_leaveIntent_bufferEarly(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	// Deliver a leave intent message early
+	j := messageLeave{LTime: 10, Node: "test"}
+	if !s.handleNodeLeaveIntent(&j) {
+		t.Fatalf("should rebroadcast")
+	}
+	if s.handleNodeLeaveIntent(&j) {
+		t.Fatalf("should not rebroadcast")
+	}
+
+	// Check that we buffered
+	if s.recentLeaveIndex != 1 {
+		t.Fatalf("bad index")
+	}
+	if s.recentLeave[0].Node != "test" || s.recentLeave[0].LTime != 10 {
+		t.Fatalf("bad buffer")
+	}
+}
+
+func TestSerf_leaveIntent_oldMessage(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.members["test"] = &memberState{
+		Member: Member{
+			Status: StatusAlive,
+		},
+		statusLTime: 12,
+	}
+
+	j := messageLeave{LTime: 10, Node: "test"}
+	if s.handleNodeLeaveIntent(&j) {
+		t.Fatalf("should not rebroadcast")
+	}
+
+	if s.recentLeaveIndex != 0 {
+		t.Fatalf("bad index")
+	}
+}
+
+func TestSerf_leaveIntent_newer(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.members["test"] = &memberState{
+		Member: Member{
+			Status: StatusAlive,
+		},
+		statusLTime: 12,
+	}
+
+	j := messageLeave{LTime: 14, Node: "test"}
+	if !s.handleNodeLeaveIntent(&j) {
+		t.Fatalf("should rebroadcast")
+	}
+
+	if s.recentLeaveIndex != 0 {
+		t.Fatalf("bad index")
+	}
+
+	if s.members["test"].Status != StatusLeaving {
+		t.Fatalf("should update status")
+	}
+
+	if s.clock.Time() != 15 {
+		t.Fatalf("should update clock")
+	}
+}
+
+func TestSerf_joinIntent_bufferEarly(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	// Deliver a join intent message early
+	j := messageJoin{LTime: 10, Node: "test"}
+	if !s.handleNodeJoinIntent(&j) {
+		t.Fatalf("should rebroadcast")
+	}
+	if s.handleNodeJoinIntent(&j) {
+		t.Fatalf("should not rebroadcast")
+	}
+
+	// Check that we buffered
+	if s.recentJoinIndex != 1 {
+		t.Fatalf("bad index")
+	}
+	if s.recentJoin[0].Node != "test" || s.recentJoin[0].LTime != 10 {
+		t.Fatalf("bad buffer")
+	}
+}
+
+func TestSerf_joinIntent_oldMessage(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.members["test"] = &memberState{
+		statusLTime: 12,
+	}
+
+	j := messageJoin{LTime: 10, Node: "test"}
+	if s.handleNodeJoinIntent(&j) {
+		t.Fatalf("should not rebroadcast")
+	}
+
+	if s.recentJoinIndex != 0 {
+		t.Fatalf("bad index")
+	}
+}
+
+func TestSerf_joinIntent_newer(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.members["test"] = &memberState{
+		statusLTime: 12,
+	}
+
+	// Deliver a join intent message early
+	j := messageJoin{LTime: 14, Node: "test"}
+	if !s.handleNodeJoinIntent(&j) {
+		t.Fatalf("should rebroadcast")
+	}
+
+	if s.recentJoinIndex != 0 {
+		t.Fatalf("bad index")
+	}
+
+	if s.members["test"].statusLTime != 14 {
+		t.Fatalf("should update join time")
+	}
+
+	if s.clock.Time() != 15 {
+		t.Fatalf("should update clock")
+	}
+}
+
+func TestSerf_joinIntent_resetLeaving(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	s.members["test"] = &memberState{
+		Member: Member{
+			Status: StatusLeaving,
+		},
+		statusLTime: 12,
+	}
+
+	j := messageJoin{LTime: 14, Node: "test"}
+	if !s.handleNodeJoinIntent(&j) {
+		t.Fatalf("should rebroadcast")
+	}
+
+	if s.recentJoinIndex != 0 {
+		t.Fatalf("bad index")
+	}
+
+	if s.members["test"].statusLTime != 14 {
+		t.Fatalf("should update join time")
+	}
+	if s.members["test"].Status != StatusAlive {
+		t.Fatalf("should update status")
+	}
+
+	if s.clock.Time() != 15 {
+		t.Fatalf("should update clock")
+	}
+}
+
+func TestSerf_userEvent_oldMessage(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	// increase the ltime artificially
+	s.eventClock.Witness(LamportTime(c.EventBuffer + 1000))
+
+	msg := messageUserEvent{
+		LTime:   1,
+		Name:    "old",
+		Payload: nil,
+	}
+	if s.handleUserEvent(&msg) {
+		t.Fatalf("should not rebroadcast")
+	}
+}
+
+func TestSerf_userEvent_sameClock(t *testing.T) {
+	eventCh := make(chan Event, 4)
+	c := testConfig()
+	c.EventCh = eventCh
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	msg := messageUserEvent{
+		LTime:   1,
+		Name:    "first",
+		Payload: []byte("test"),
+	}
+	if !s.handleUserEvent(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+	msg = messageUserEvent{
+		LTime:   1,
+		Name:    "first",
+		Payload: []byte("newpayload"),
+	}
+	if !s.handleUserEvent(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+	msg = messageUserEvent{
+		LTime:   1,
+		Name:    "second",
+		Payload: []byte("other"),
+	}
+	if !s.handleUserEvent(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+
+	testUserEvents(t, eventCh,
+		[]string{"first", "first", "second"},
+		[][]byte{[]byte("test"), []byte("newpayload"), []byte("other")})
+}
+
+func TestSerf_query_oldMessage(t *testing.T) {
+	c := testConfig()
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	// increase the ltime artificially
+	s.queryClock.Witness(LamportTime(c.QueryBuffer + 1000))
+
+	msg := messageQuery{
+		LTime:   1,
+		Name:    "old",
+		Payload: nil,
+	}
+	if s.handleQuery(&msg) {
+		t.Fatalf("should not rebroadcast")
+	}
+}
+
+func TestSerf_query_sameClock(t *testing.T) {
+	eventCh := make(chan Event, 4)
+	c := testConfig()
+	c.EventCh = eventCh
+	s, err := Create(c)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer s.Shutdown()
+
+	msg := messageQuery{
+		LTime:   1,
+		ID:      1,
+		Name:    "foo",
+		Payload: []byte("test"),
+	}
+	if !s.handleQuery(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+	if s.handleQuery(&msg) {
+		t.Fatalf("should not rebroadcast")
+	}
+	msg = messageQuery{
+		LTime:   1,
+		ID:      2,
+		Name:    "bar",
+		Payload: []byte("newpayload"),
+	}
+	if !s.handleQuery(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+	if s.handleQuery(&msg) {
+		t.Fatalf("should not rebroadcast")
+	}
+	msg = messageQuery{
+		LTime:   1,
+		ID:      3,
+		Name:    "baz",
+		Payload: []byte("other"),
+	}
+	if !s.handleQuery(&msg) {
+		t.Fatalf("should rebroadcast")
+	}
+	if s.handleQuery(&msg) {
+		t.Fatalf("should not rebroadcast")
+	}
+
+	testQueryEvents(t, eventCh,
+		[]string{"foo", "bar", "baz"},
+		[][]byte{[]byte("test"), []byte("newpayload"), []byte("other")})
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/snapshot.go

@@ -0,0 +1,500 @@
+package serf
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/armon/go-metrics"
+)
+
+/*
+Serf supports using a "snapshot" file that contains various
+transactional data that is used to help Serf recover quickly
+and gracefully from a failure. We append member events, as well
+as the latest clock values to the file during normal operation,
+and periodically checkpoint and roll over the file. During a restore,
+we can replay the various member events to recall a list of known
+nodes to re-join, as well as restore our clock values to avoid replaying
+old events.
+*/
+
+const flushInterval = 500 * time.Millisecond
+const clockUpdateInterval = 500 * time.Millisecond
+const tmpExt = ".compact"
+
+// Snapshotter is responsible for ingesting events and persisting
+// them to disk, and providing a recovery mechanism at start time.
+type Snapshotter struct {
+	aliveNodes       map[string]string
+	clock            *LamportClock
+	fh               *os.File
+	buffered         *bufio.Writer
+	inCh             <-chan Event
+	lastFlush        time.Time
+	lastClock        LamportTime
+	lastEventClock   LamportTime
+	lastQueryClock   LamportTime
+	leaveCh          chan struct{}
+	leaving          bool
+	logger           *log.Logger
+	maxSize          int64
+	path             string
+	offset           int64
+	outCh            chan<- Event
+	rejoinAfterLeave bool
+	shutdownCh       <-chan struct{}
+	waitCh           chan struct{}
+}
+
+// PreviousNode is used to represent the previously known alive nodes
+type PreviousNode struct {
+	Name string
+	Addr string
+}
+
+func (p PreviousNode) String() string {
+	return fmt.Sprintf("%s: %s", p.Name, p.Addr)
+}
+
+// NewSnapshotter creates a new Snapshotter that records events up to a
+// max byte size before rotating the file. It can also be used to
+// recover old state. Snapshotter works by reading an event channel it returns,
+// passing through to an output channel, and persisting relevant events to disk.
+// Setting rejoinAfterLeave makes leave not clear the state, and can be used
+// if you intend to rejoin the same cluster after a leave.
+func NewSnapshotter(path string,
+	maxSize int,
+	rejoinAfterLeave bool,
+	logger *log.Logger,
+	clock *LamportClock,
+	outCh chan<- Event,
+	shutdownCh <-chan struct{}) (chan<- Event, *Snapshotter, error) {
+	inCh := make(chan Event, 1024)
+
+	// Try to open the file
+	fh, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0755)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to open snapshot: %v", err)
+	}
+
+	// Determine the offset
+	info, err := fh.Stat()
+	if err != nil {
+		fh.Close()
+		return nil, nil, fmt.Errorf("failed to stat snapshot: %v", err)
+	}
+	offset := info.Size()
+
+	// Create the snapshotter
+	snap := &Snapshotter{
+		aliveNodes:       make(map[string]string),
+		clock:            clock,
+		fh:               fh,
+		buffered:         bufio.NewWriter(fh),
+		inCh:             inCh,
+		lastClock:        0,
+		lastEventClock:   0,
+		lastQueryClock:   0,
+		leaveCh:          make(chan struct{}),
+		logger:           logger,
+		maxSize:          int64(maxSize),
+		path:             path,
+		offset:           offset,
+		outCh:            outCh,
+		rejoinAfterLeave: rejoinAfterLeave,
+		shutdownCh:       shutdownCh,
+		waitCh:           make(chan struct{}),
+	}
+
+	// Recover the last known state
+	if err := snap.replay(); err != nil {
+		fh.Close()
+		return nil, nil, err
+	}
+
+	// Start handling new commands
+	go snap.stream()
+	return inCh, snap, nil
+}
+
+// LastClock returns the last known clock time
+func (s *Snapshotter) LastClock() LamportTime {
+	return s.lastClock
+}
+
+// LastEventClock returns the last known event clock time
+func (s *Snapshotter) LastEventClock() LamportTime {
+	return s.lastEventClock
+}
+
+// LastQueryClock returns the last known query clock time
+func (s *Snapshotter) LastQueryClock() LamportTime {
+	return s.lastQueryClock
+}
+
+// AliveNodes returns the last known alive nodes
+func (s *Snapshotter) AliveNodes() []*PreviousNode {
+	// Copy the previously known
+	previous := make([]*PreviousNode, 0, len(s.aliveNodes))
+	for name, addr := range s.aliveNodes {
+		previous = append(previous, &PreviousNode{name, addr})
+	}
+
+	// Randomize the order, prevents hot shards
+	for i := range previous {
+		j := rand.Intn(i + 1)
+		previous[i], previous[j] = previous[j], previous[i]
+	}
+	return previous
+}
+
+// Wait is used to wait until the snapshotter finishes shut down
+func (s *Snapshotter) Wait() {
+	<-s.waitCh
+}
+
+// Leave is used to remove known nodes to prevent a restart from
+// causing a join. Otherwise nodes will re-join after leaving!
+func (s *Snapshotter) Leave() {
+	select {
+	case s.leaveCh <- struct{}{}:
+	case <-s.shutdownCh:
+	}
+}
+
+// stream is a long running routine that is used to handle events
+func (s *Snapshotter) stream() {
+	for {
+		select {
+		case <-s.leaveCh:
+			s.leaving = true
+
+			// If we plan to re-join, keep our state
+			if !s.rejoinAfterLeave {
+				s.aliveNodes = make(map[string]string)
+			}
+			s.tryAppend("leave\n")
+			if err := s.buffered.Flush(); err != nil {
+				s.logger.Printf("[ERR] serf: failed to flush leave to snapshot: %v", err)
+			}
+			if err := s.fh.Sync(); err != nil {
+				s.logger.Printf("[ERR] serf: failed to sync leave to snapshot: %v", err)
+			}
+
+		case e := <-s.inCh:
+			// Forward the event immediately
+			if s.outCh != nil {
+				s.outCh <- e
+			}
+
+			// Stop recording events after a leave is issued
+			if s.leaving {
+				continue
+			}
+			switch typed := e.(type) {
+			case MemberEvent:
+				s.processMemberEvent(typed)
+			case UserEvent:
+				s.processUserEvent(typed)
+			case *Query:
+				s.processQuery(typed)
+			default:
+				s.logger.Printf("[ERR] serf: Unknown event to snapshot: %#v", e)
+			}
+
+		case <-time.After(clockUpdateInterval):
+			s.updateClock()
+
+		case <-s.shutdownCh:
+			if err := s.buffered.Flush(); err != nil {
+				s.logger.Printf("[ERR] serf: failed to flush snapshot: %v", err)
+			}
+			if err := s.fh.Sync(); err != nil {
+				s.logger.Printf("[ERR] serf: failed to sync snapshot: %v", err)
+			}
+			s.fh.Close()
+			close(s.waitCh)
+			return
+		}
+	}
+}
+
+// processMemberEvent is used to handle a single member event
+func (s *Snapshotter) processMemberEvent(e MemberEvent) {
+	switch e.Type {
+	case EventMemberJoin:
+		for _, mem := range e.Members {
+			addr := net.TCPAddr{IP: mem.Addr, Port: int(mem.Port)}
+			s.aliveNodes[mem.Name] = addr.String()
+			s.tryAppend(fmt.Sprintf("alive: %s %s\n", mem.Name, addr.String()))
+		}
+
+	case EventMemberLeave:
+		fallthrough
+	case EventMemberFailed:
+		for _, mem := range e.Members {
+			delete(s.aliveNodes, mem.Name)
+			s.tryAppend(fmt.Sprintf("not-alive: %s\n", mem.Name))
+		}
+	}
+	s.updateClock()
+}
+
+// updateClock is called periodically to check if we should udpate our
+// clock value. This is done after member events but should also be done
+// periodically due to race conditions with join and leave intents
+func (s *Snapshotter) updateClock() {
+	lastSeen := s.clock.Time() - 1
+	if lastSeen > s.lastClock {
+		s.lastClock = lastSeen
+		s.tryAppend(fmt.Sprintf("clock: %d\n", s.lastClock))
+	}
+}
+
+// processUserEvent is used to handle a single user event
+func (s *Snapshotter) processUserEvent(e UserEvent) {
+	// Ignore old clocks
+	if e.LTime <= s.lastEventClock {
+		return
+	}
+	s.lastEventClock = e.LTime
+	s.tryAppend(fmt.Sprintf("event-clock: %d\n", e.LTime))
+}
+
+// processQuery is used to handle a single query event
+func (s *Snapshotter) processQuery(q *Query) {
+	// Ignore old clocks
+	if q.LTime <= s.lastQueryClock {
+		return
+	}
+	s.lastQueryClock = q.LTime
+	s.tryAppend(fmt.Sprintf("query-clock: %d\n", q.LTime))
+}
+
+// tryAppend will invoke append line but will not return an error
+func (s *Snapshotter) tryAppend(l string) {
+	if err := s.appendLine(l); err != nil {
+		s.logger.Printf("[ERR] serf: Failed to update snapshot: %v", err)
+	}
+}
+
+// appendLine is used to append a line to the existing log
+func (s *Snapshotter) appendLine(l string) error {
+	defer metrics.MeasureSince([]string{"serf", "snapshot", "appendLine"}, time.Now())
+
+	n, err := s.buffered.WriteString(l)
+	if err != nil {
+		return err
+	}
+
+	// Check if we should flush
+	now := time.Now()
+	if now.Sub(s.lastFlush) > flushInterval {
+		s.lastFlush = now
+		if err := s.buffered.Flush(); err != nil {
+			return err
+		}
+	}
+
+	// Check if a compaction is necessary
+	s.offset += int64(n)
+	if s.offset > s.maxSize {
+		return s.compact()
+	}
+	return nil
+}
+
+// Compact is used to compact the snapshot once it is too large
+func (s *Snapshotter) compact() error {
+	defer metrics.MeasureSince([]string{"serf", "snapshot", "compact"}, time.Now())
+
+	// Try to open the file to new fiel
+	newPath := s.path + tmpExt
+	fh, err := os.OpenFile(newPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to open new snapshot: %v", err)
+	}
+
+	// Create a buffered writer
+	buf := bufio.NewWriter(fh)
+
+	// Write out the live nodes
+	var offset int64
+	for name, addr := range s.aliveNodes {
+		line := fmt.Sprintf("alive: %s %s\n", name, addr)
+		n, err := buf.WriteString(line)
+		if err != nil {
+			fh.Close()
+			return err
+		}
+		offset += int64(n)
+	}
+
+	// Write out the clocks
+	line := fmt.Sprintf("clock: %d\n", s.lastClock)
+	n, err := buf.WriteString(line)
+	if err != nil {
+		fh.Close()
+		return err
+	}
+	offset += int64(n)
+
+	line = fmt.Sprintf("event-clock: %d\n", s.lastEventClock)
+	n, err = buf.WriteString(line)
+	if err != nil {
+		fh.Close()
+		return err
+	}
+	offset += int64(n)
+
+	line = fmt.Sprintf("query-clock: %d\n", s.lastQueryClock)
+	n, err = buf.WriteString(line)
+	if err != nil {
+		fh.Close()
+		return err
+	}
+	offset += int64(n)
+
+	// Flush the new snapshot
+	err = buf.Flush()
+	fh.Close()
+	if err != nil {
+		return fmt.Errorf("failed to flush new snapshot: %v", err)
+	}
+
+	// We now need to swap the old snapshot file with the new snapshot.
+	// Turns out, Windows won't let us rename the files if we have
+	// open handles to them or if the destination already exists. This
+	// means we are forced to close the existing handles, delete the
+	// old file, move the new one in place, and then re-open the file
+	// handles.
+
+	// Flush the existing snapshot, ignoring errors since we will
+	// delete it momentarily.
+	s.buffered.Flush()
+	s.buffered = nil
+
+	// Close the file handle to the old snapshot
+	s.fh.Close()
+	s.fh = nil
+
+	// Delete the old file
+	if err := os.Remove(s.path); err != nil {
+		return fmt.Errorf("failed to remove old snapshot: %v", err)
+	}
+
+	// Move the new file into place
+	if err := os.Rename(newPath, s.path); err != nil {
+		return fmt.Errorf("failed to install new snapshot: %v", err)
+	}
+
+	// Open the new snapshot
+	fh, err = os.OpenFile(s.path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to open snapshot: %v", err)
+	}
+	buf = bufio.NewWriter(fh)
+
+	// Rotate our handles
+	s.fh = fh
+	s.buffered = buf
+	s.offset = offset
+	s.lastFlush = time.Now()
+	return nil
+}
+
+// replay is used to seek to reset our internal state by replaying
+// the snapshot file. It is used at initialization time to read old
+// state
+func (s *Snapshotter) replay() error {
+	// Seek to the beginning
+	if _, err := s.fh.Seek(0, os.SEEK_SET); err != nil {
+		return err
+	}
+
+	// Read each line
+	reader := bufio.NewReader(s.fh)
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			break
+		}
+
+		// Skip the newline
+		line = line[:len(line)-1]
+
+		// Switch on the prefix
+		if strings.HasPrefix(line, "alive: ") {
+			info := strings.TrimPrefix(line, "alive: ")
+			addrIdx := strings.LastIndex(info, " ")
+			if addrIdx == -1 {
+				s.logger.Printf("[WARN] serf: Failed to parse address: %v", line)
+				continue
+			}
+			addr := info[addrIdx+1:]
+			name := info[:addrIdx]
+			s.aliveNodes[name] = addr
+
+		} else if strings.HasPrefix(line, "not-alive: ") {
+			name := strings.TrimPrefix(line, "not-alive: ")
+			delete(s.aliveNodes, name)
+
+		} else if strings.HasPrefix(line, "clock: ") {
+			timeStr := strings.TrimPrefix(line, "clock: ")
+			timeInt, err := strconv.ParseUint(timeStr, 10, 64)
+			if err != nil {
+				s.logger.Printf("[WARN] serf: Failed to convert clock time: %v", err)
+				continue
+			}
+			s.lastClock = LamportTime(timeInt)
+
+		} else if strings.HasPrefix(line, "event-clock: ") {
+			timeStr := strings.TrimPrefix(line, "event-clock: ")
+			timeInt, err := strconv.ParseUint(timeStr, 10, 64)
+			if err != nil {
+				s.logger.Printf("[WARN] serf: Failed to convert event clock time: %v", err)
+				continue
+			}
+			s.lastEventClock = LamportTime(timeInt)
+
+		} else if strings.HasPrefix(line, "query-clock: ") {
+			timeStr := strings.TrimPrefix(line, "query-clock: ")
+			timeInt, err := strconv.ParseUint(timeStr, 10, 64)
+			if err != nil {
+				s.logger.Printf("[WARN] serf: Failed to convert query clock time: %v", err)
+				continue
+			}
+			s.lastQueryClock = LamportTime(timeInt)
+
+		} else if line == "leave" {
+			// Ignore a leave if we plan on re-joining
+			if s.rejoinAfterLeave {
+				s.logger.Printf("[INFO] serf: Ignoring previous leave in snapshot")
+				continue
+			}
+			s.aliveNodes = make(map[string]string)
+			s.lastClock = 0
+			s.lastEventClock = 0
+			s.lastQueryClock = 0
+
+		} else if strings.HasPrefix(line, "#") {
+			// Skip comment lines
+
+		} else {
+			s.logger.Printf("[WARN] serf: Unrecognized snapshot line: %v", line)
+		}
+	}
+
+	// Seek to the end
+	if _, err := s.fh.Seek(0, os.SEEK_END); err != nil {
+		return err
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/hashicorp/serf/serf/snapshot_test.go

@@ -0,0 +1,374 @@
+package serf
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestSnapshoter(t *testing.T) {
+	td, err := ioutil.TempDir("", "serf")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	clock := new(LamportClock)
+	outCh := make(chan Event, 64)
+	stopCh := make(chan struct{})
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit, false,
+		logger, clock, outCh, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Write some user events
+	ue := UserEvent{
+		LTime: 42,
+		Name:  "bar",
+	}
+	inCh <- ue
+
+	// Write some queries
+	q := &Query{
+		LTime: 50,
+		Name:  "uptime",
+	}
+	inCh <- q
+
+	// Write some member events
+	clock.Witness(100)
+	meJoin := MemberEvent{
+		Type: EventMemberJoin,
+		Members: []Member{
+			Member{
+				Name: "foo",
+				Addr: []byte{127, 0, 0, 1},
+				Port: 5000,
+			},
+		},
+	}
+	meFail := MemberEvent{
+		Type: EventMemberFailed,
+		Members: []Member{
+			Member{
+				Name: "foo",
+				Addr: []byte{127, 0, 0, 1},
+				Port: 5000,
+			},
+		},
+	}
+	inCh <- meJoin
+	inCh <- meFail
+	inCh <- meJoin
+
+	// Check these get passed through
+	select {
+	case e := <-outCh:
+		if !reflect.DeepEqual(e, ue) {
+			t.Fatalf("expected user event: %#v", e)
+		}
+	case <-time.After(200 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+
+	select {
+	case e := <-outCh:
+		if !reflect.DeepEqual(e, q) {
+			t.Fatalf("expected query event: %#v", e)
+		}
+	case <-time.After(200 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+
+	select {
+	case e := <-outCh:
+		if !reflect.DeepEqual(e, meJoin) {
+			t.Fatalf("expected member event: %#v", e)
+		}
+	case <-time.After(200 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+
+	select {
+	case e := <-outCh:
+		if !reflect.DeepEqual(e, meFail) {
+			t.Fatalf("expected member event: %#v", e)
+		}
+	case <-time.After(200 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+
+	select {
+	case e := <-outCh:
+		if !reflect.DeepEqual(e, meJoin) {
+			t.Fatalf("expected member event: %#v", e)
+		}
+	case <-time.After(200 * time.Millisecond):
+		t.Fatalf("timeout")
+	}
+
+	// Close the snapshoter
+	close(stopCh)
+	snap.Wait()
+
+	// Open the snapshoter
+	stopCh = make(chan struct{})
+	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
+		logger, clock, outCh, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the values
+	if snap.LastClock() != 100 {
+		t.Fatalf("bad clock %d", snap.LastClock())
+	}
+	if snap.LastEventClock() != 42 {
+		t.Fatalf("bad clock %d", snap.LastEventClock())
+	}
+	if snap.LastQueryClock() != 50 {
+		t.Fatalf("bad clock %d", snap.LastQueryClock())
+	}
+
+	prev := snap.AliveNodes()
+	if len(prev) != 1 {
+		t.Fatalf("expected alive: %#v", prev)
+	}
+	if prev[0].Name != "foo" {
+		t.Fatalf("bad name: %#v", prev[0])
+	}
+	if prev[0].Addr != "127.0.0.1:5000" {
+		t.Fatalf("bad addr: %#v", prev[0])
+	}
+}
+
+func TestSnapshoter_forceCompact(t *testing.T) {
+	td, err := ioutil.TempDir("", "serf")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	clock := new(LamportClock)
+	stopCh := make(chan struct{})
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+
+	// Create a very low limit
+	inCh, snap, err := NewSnapshotter(td+"snap", 1024, false,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Write lots of user events
+	for i := 0; i < 1024; i++ {
+		ue := UserEvent{
+			LTime: LamportTime(i),
+		}
+		inCh <- ue
+	}
+
+	// Write lots of queries
+	for i := 0; i < 1024; i++ {
+		q := &Query{
+			LTime: LamportTime(i),
+		}
+		inCh <- q
+	}
+
+	// Wait for drain
+	for len(inCh) > 0 {
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	// Close the snapshoter
+	close(stopCh)
+	snap.Wait()
+
+	// Open the snapshoter
+	stopCh = make(chan struct{})
+	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the values
+	if snap.LastEventClock() != 1023 {
+		t.Fatalf("bad clock %d", snap.LastEventClock())
+	}
+
+	if snap.LastQueryClock() != 1023 {
+		t.Fatalf("bad clock %d", snap.LastQueryClock())
+	}
+
+	close(stopCh)
+	snap.Wait()
+}
+
+func TestSnapshoter_leave(t *testing.T) {
+	td, err := ioutil.TempDir("", "serf")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	clock := new(LamportClock)
+	stopCh := make(chan struct{})
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit, false,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Write a user event
+	ue := UserEvent{
+		LTime: 42,
+		Name:  "bar",
+	}
+	inCh <- ue
+
+	// Write a query
+	q := &Query{
+		LTime: 50,
+		Name:  "uptime",
+	}
+	inCh <- q
+
+	// Write some member events
+	clock.Witness(100)
+	meJoin := MemberEvent{
+		Type: EventMemberJoin,
+		Members: []Member{
+			Member{
+				Name: "foo",
+				Addr: []byte{127, 0, 0, 1},
+				Port: 5000,
+			},
+		},
+	}
+	inCh <- meJoin
+
+	// Wait for drain
+	for len(inCh) > 0 {
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	// Leave the cluster!
+	snap.Leave()
+
+	// Close the snapshoter
+	close(stopCh)
+	snap.Wait()
+
+	// Open the snapshoter
+	stopCh = make(chan struct{})
+	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, false,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the values
+	if snap.LastClock() != 0 {
+		t.Fatalf("bad clock %d", snap.LastClock())
+	}
+	if snap.LastEventClock() != 0 {
+		t.Fatalf("bad clock %d", snap.LastEventClock())
+	}
+	if snap.LastQueryClock() != 0 {
+		t.Fatalf("bad clock %d", snap.LastQueryClock())
+	}
+
+	prev := snap.AliveNodes()
+	if len(prev) != 0 {
+		t.Fatalf("expected none alive: %#v", prev)
+	}
+}
+
+func TestSnapshoter_leave_rejoin(t *testing.T) {
+	td, err := ioutil.TempDir("", "serf")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	defer os.RemoveAll(td)
+
+	clock := new(LamportClock)
+	stopCh := make(chan struct{})
+	logger := log.New(os.Stderr, "", log.LstdFlags)
+	inCh, snap, err := NewSnapshotter(td+"snap", snapshotSizeLimit, true,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Write a user event
+	ue := UserEvent{
+		LTime: 42,
+		Name:  "bar",
+	}
+	inCh <- ue
+
+	// Write a query
+	q := &Query{
+		LTime: 50,
+		Name:  "uptime",
+	}
+	inCh <- q
+
+	// Write some member events
+	clock.Witness(100)
+	meJoin := MemberEvent{
+		Type: EventMemberJoin,
+		Members: []Member{
+			Member{
+				Name: "foo",
+				Addr: []byte{127, 0, 0, 1},
+				Port: 5000,
+			},
+		},
+	}
+	inCh <- meJoin
+
+	// Wait for drain
+	for len(inCh) > 0 {
+		time.Sleep(20 * time.Millisecond)
+	}
+
+	// Leave the cluster!
+	snap.Leave()
+
+	// Close the snapshoter
+	close(stopCh)
+	snap.Wait()
+
+	// Open the snapshoter
+	stopCh = make(chan struct{})
+	_, snap, err = NewSnapshotter(td+"snap", snapshotSizeLimit, true,
+		logger, clock, nil, stopCh)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the values
+	if snap.LastClock() != 100 {
+		t.Fatalf("bad clock %d", snap.LastClock())
+	}
+	if snap.LastEventClock() != 42 {
+		t.Fatalf("bad clock %d", snap.LastEventClock())
+	}
+	if snap.LastQueryClock() != 50 {
+		t.Fatalf("bad clock %d", snap.LastQueryClock())
+	}
+
+	prev := snap.AliveNodes()
+	if len(prev) == 0 {
+		t.Fatalf("expected alive: %#v", prev)
+	}
+}

Godeps/_workspace/src/github.com/stretchr/objx/accessors_test.go

@@ -1,7 +1,7 @@
 package objx
 
 import (
-	"github.com/docker/libnetwork/Godeps/_workspace/src/github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/assert"
 	"testing"
 )
 

Godeps/_workspace/src/github.com/stretchr/objx/conversions_test.go

@@ -1,7 +1,7 @@
 package objx
 
 import (
-	"github.com/docker/libnetwork/Godeps/_workspace/src/github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/assert"
 	"testing"
 )
 

Godeps/_workspace/src/github.com/stretchr/objx/fixture_test.go

@@ -1,7 +1,7 @@
 package objx
 
 import (
-	"github.com/docker/libnetwork/Godeps/_workspace/src/github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/assert"
 	"testing"
 )
 

Godeps/_workspace/src/github.com/stretchr/objx/map_test.go

@@ -1,7 +1,7 @@
 package objx
 
 import (
-	"github.com/docker/libnetwork/Godeps/_workspace/src/github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/assert"
 	"testing"
 )