Clean out remaining cruft from 'tianon-separate-dockerinit'

Conflicts:
	runtime_test.go

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changelog
 
-## 0.6.4 (2013-10-15)
+## 0.6.4 (2013-10-16)
 - Runtime: Add cleanup of container when Start() fails
 - Testing: Catch errClosing error when TCP and UDP proxies are terminated
 - Testing: Add aggregated docker-ci email report

Dockerfile

@@ -24,46 +24,58 @@
 #
 
 docker-version 0.6.1
-from	ubuntu:12.04
-maintainer	Solomon Hykes <solomon@dotcloud.com>
+from    ubuntu:12.10
+maintainer      Solomon Hykes <solomon@dotcloud.com>
 
 # Build dependencies
-run	echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
-run	apt-get update
-run	apt-get install -y -q curl
-run	apt-get install -y -q git
-run	apt-get install -y -q mercurial
-run	apt-get install -y -q build-essential
+run     apt-get update
+run     apt-get install -y -q curl
+run     apt-get install -y -q git
+run     apt-get install -y -q mercurial
+run     apt-get install -y -q build-essential
+run     apt-get install -y -q libsqlite3-dev
 
 # Install Go from source (for eventual cross-compiling)
-env	CGO_ENABLED 0
-run	curl -s https://go.googlecode.com/files/go1.1.2.src.tar.gz | tar -v -C / -xz && mv /go /goroot
-run	cd /goroot/src && ./make.bash
-env GOROOT	/goroot
-env	PATH	$PATH:/goroot/bin
-env	GOPATH	/go:/go/src/github.com/dotcloud/docker/vendor
+run     curl -s https://go.googlecode.com/files/go1.2rc1.src.tar.gz | tar -v -C / -xz && mv /go /goroot
+run     cd /goroot/src && ./make.bash
+env     GOROOT  /goroot
+env     PATH    $PATH:/goroot/bin
+env     GOPATH  /go:/go/src/github.com/dotcloud/docker/vendor
+
+# Create Go cache with tag netgo (for static compilation of Go while preserving CGO support)
+run     go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
+
+# Get lvm2 source for compiling statically
+run     git clone git://git.fedorahosted.org/git/lvm2.git /lvm2
+run     cd /lvm2 && git checkout v2_02_102
+
+# can't use git clone -b because it's not supported by git versions before 1.7.10
+run	cd /lvm2 && ./configure --enable-static_link && make && make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
 
 # Ubuntu stuff
-run	apt-get install -y -q ruby1.9.3 rubygems libffi-dev
-run	gem install --no-rdoc --no-ri fpm
-run	apt-get install -y -q reprepro dpkg-sig
+run     apt-get install -y -q ruby1.9.3 rubygems libffi-dev
+run     gem install --no-rdoc --no-ri fpm
+run     apt-get install -y -q reprepro dpkg-sig
 
 # Install s3cmd 1.0.1 (earlier versions don't support env variables in the config)
-run	apt-get install -y -q python-pip
-run	pip install s3cmd
-run	pip install python-magic
-run	/bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg
+run     apt-get install -y -q python-pip
+run     pip install s3cmd
+run     pip install python-magic
+run     /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg
 
 # Runtime dependencies
-run	apt-get install -y -q iptables
-run	apt-get install -y -q lxc
-run	apt-get install -y -q aufs-tools
+run     apt-get install -y -q iptables
+run     dpkg-divert --local --rename --add /sbin/initctl && \
+        ln -s /bin/true /sbin/initctl && \
+        apt-get install -y -q lxc
 
-volume	/var/lib/docker
-workdir	/go/src/github.com/dotcloud/docker
+volume  /var/lib/docker
+workdir /go/src/github.com/dotcloud/docker
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
 entrypoint ["hack/dind"]
 
 # Upload docker source
-add	.       /go/src/github.com/dotcloud/docker
+add     .		/go/src/github.com/dotcloud/docker
+

VERSION

@@ -1 +1 @@
-0.6.4
+0.6.4-dev

api.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/gograph"
 	"github.com/dotcloud/docker/utils"
 	"github.com/gorilla/mux"
 	"io"
@@ -69,12 +70,12 @@ func httpError(w http.ResponseWriter, err error) {
 		statusCode = http.StatusUnauthorized
 	} else if strings.Contains(err.Error(), "hasn't been activated") {
 		statusCode = http.StatusForbidden
-	}	
-	
+	}
+
 	if err != nil {
 		utils.Errorf("HTTP Error: statusCode=%d %s", statusCode, err.Error())
-		http.Error(w, err.Error(), statusCode)		
-	}	
+		http.Error(w, err.Error(), statusCode)
+	}
 }
 
 func writeJSON(w http.ResponseWriter, code int, v interface{}) error {
@@ -515,16 +516,19 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r
 		return err
 	}
 
-	if !config.NetworkDisabled && len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
+	if !config.NetworkDisabled && len(config.Dns) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
 		out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns))
 		config.Dns = defaultDns
 	}
 
-	id, err := srv.ContainerCreate(config)
+	id, warnings, err := srv.ContainerCreate(config)
 	if err != nil {
 		return err
 	}
 	out.ID = id
+	for _, warning := range warnings {
+		out.Warnings = append(out.Warnings, warning)
+	}
 
 	if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit {
 		log.Println("WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.")
@@ -570,12 +574,17 @@ func deleteContainers(srv *Server, version float64, w http.ResponseWriter, r *ht
 		return fmt.Errorf("Missing parameter")
 	}
 	name := vars["name"]
+
 	removeVolume, err := getBoolParam(r.Form.Get("v"))
 	if err != nil {
 		return err
 	}
+	removeLink, err := getBoolParam(r.Form.Get("link"))
+	if err != nil {
+		return err
+	}
 
-	if err := srv.ContainerDestroy(name, removeVolume); err != nil {
+	if err := srv.ContainerDestroy(name, removeVolume, removeLink); err != nil {
 		return err
 	}
 	w.WriteHeader(http.StatusNoContent)
@@ -623,6 +632,7 @@ func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r
 	}
 	name := vars["name"]
 	if err := srv.ContainerStart(name, hostConfig); err != nil {
+		utils.Debugf("error ContainerStart: %s", err)
 		return err
 	}
 	w.WriteHeader(http.StatusNoContent)
@@ -655,6 +665,7 @@ func postContainersWait(srv *Server, version float64, w http.ResponseWriter, r *
 		return fmt.Errorf("Missing parameter")
 	}
 	name := vars["name"]
+
 	status, err := srv.ContainerWait(name)
 	if err != nil {
 		return err
@@ -975,7 +986,7 @@ func makeHttpHandler(srv *Server, logging bool, localMethod string, localRoute s
 		if err != nil {
 			version = APIVERSION
 		}
-		if srv.enableCors {
+		if srv.runtime.config.EnableCors {
 			writeCorsHeaders(w, r)
 		}
 
@@ -991,6 +1002,73 @@ func makeHttpHandler(srv *Server, logging bool, localMethod string, localRoute s
 	}
 }
 
+func getContainersLinks(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if err := parseForm(r); err != nil {
+		return err
+	}
+
+	runtime := srv.runtime
+	all, err := getBoolParam(r.Form.Get("all"))
+	if err != nil {
+		return err
+	}
+
+	out := []APILink{}
+	err = runtime.containerGraph.Walk("/", func(p string, e *gograph.Entity) error {
+		if container := runtime.Get(e.ID()); container != nil {
+			if !all && strings.Contains(p, container.ID) {
+				return nil
+			}
+			out = append(out, APILink{
+				Path:        p,
+				ContainerID: container.ID,
+				Image:       runtime.repositories.ImageName(container.Image),
+			})
+		}
+		return nil
+	}, -1)
+
+	if err != nil {
+		return err
+	}
+	return writeJSON(w, http.StatusOK, out)
+}
+
+func postContainerLink(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if vars == nil {
+		return fmt.Errorf("Missing parameter")
+	}
+	values := make(map[string]string)
+	if matchesContentType(r.Header.Get("Content-Type"), "application/json") && r.Body != nil {
+		defer r.Body.Close()
+
+		dec := json.NewDecoder(r.Body)
+		if err := dec.Decode(&values); err != nil {
+			return err
+		}
+	} else {
+		return fmt.Errorf("Invalid json body")
+	}
+	currentName := values["currentName"]
+	newName := values["newName"]
+
+	if currentName == "" {
+		return fmt.Errorf("currentName cannot be empty")
+	}
+	if newName == "" {
+		return fmt.Errorf("newName cannot be empty")
+	}
+
+	if err := srv.runtime.RenameLink(currentName, newName); err != nil {
+		if strings.HasSuffix(err.Error(), "name are not unique") {
+			return fmt.Errorf("Conflict, %s already exists", newName)
+		}
+		return err
+	}
+
+	return nil
+}
+
 func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 	r := mux.NewRouter()
 
@@ -1011,6 +1089,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/containers/{name:.*}/json":      getContainersByName,
 			"/containers/{name:.*}/top":       getContainersTop,
 			"/containers/{name:.*}/attach/ws": wsContainersAttach,
+			"/containers/links":               getContainersLinks,
 		},
 		"POST": {
 			"/auth":                         postAuth,
@@ -1029,6 +1108,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/containers/{name:.*}/resize":  postContainersResize,
 			"/containers/{name:.*}/attach":  postContainersAttach,
 			"/containers/{name:.*}/copy":    postContainersCopy,
+			"/containers/link":              postContainerLink,
 		},
 		"DELETE": {
 			"/containers/{name:.*}": deleteContainers,

api_params.go

@@ -1,7 +1,5 @@
 package docker
 
-import "encoding/json"
-
 type APIHistory struct {
 	ID        string   `json:"Id"`
 	Tags      []string `json:",omitempty"`
@@ -19,18 +17,23 @@ type APIImages struct {
 }
 
 type APIInfo struct {
-	Debug              bool
-	Containers         int
-	Images             int
-	NFd                int    `json:",omitempty"`
-	NGoroutines        int    `json:",omitempty"`
-	MemoryLimit        bool   `json:",omitempty"`
-	SwapLimit          bool   `json:",omitempty"`
-	IPv4Forwarding     bool   `json:",omitempty"`
-	LXCVersion         string `json:",omitempty"`
-	NEventsListener    int    `json:",omitempty"`
-	KernelVersion      string `json:",omitempty"`
-	IndexServerAddress string `json:",omitempty"`
+	Debug                  bool
+	Containers             int
+	Images                 int
+	NFd                    int    `json:",omitempty"`
+	NGoroutines            int    `json:",omitempty"`
+	MemoryLimit            bool   `json:",omitempty"`
+	SwapLimit              bool   `json:",omitempty"`
+	IPv4Forwarding         bool   `json:",omitempty"`
+	LXCVersion             string `json:",omitempty"`
+	NEventsListener        int    `json:",omitempty"`
+	KernelVersion          string `json:",omitempty"`
+	IndexServerAddress     string `json:",omitempty"`
+	DevmapperPool          string `json:",omitempty"`
+	DevmapperDataUsed      uint64 `json:",omitempty"`
+	DevmapperDataTotal     uint64 `json:",omitempty"`
+	DevmapperMetadataUsed  uint64 `json:",omitempty"`
+	DevmapperMetadataTotal uint64 `json:",omitempty"`
 }
 
 type APITop struct {
@@ -52,17 +55,18 @@ type APIContainers struct {
 	Ports      []APIPort
 	SizeRw     int64
 	SizeRootFs int64
+	Names      []string
 }
 
 func (self *APIContainers) ToLegacy() APIContainersOld {
 	return APIContainersOld{
-		ID: self.ID,
-		Image: self.Image,
-		Command: self.Command,
-		Created: self.Created,
-		Status: self.Status,
-		Ports: displayablePorts(self.Ports),
-		SizeRw: self.SizeRw,
+		ID:         self.ID,
+		Image:      self.Image,
+		Command:    self.Command,
+		Created:    self.Created,
+		Status:     self.Status,
+		Ports:      displayablePorts(self.Ports),
+		SizeRw:     self.SizeRw,
 		SizeRootFs: self.SizeRootFs,
 	}
 }
@@ -96,14 +100,7 @@ type APIPort struct {
 	PrivatePort int64
 	PublicPort  int64
 	Type        string
-}
-
-func (port *APIPort) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"PrivatePort": port.PrivatePort,
-		"PublicPort":  port.PublicPort,
-		"Type":        port.Type,
-	})
+	IP          string
 }
 
 type APIVersion struct {
@@ -129,3 +126,9 @@ type APICopy struct {
 	Resource string
 	HostPath string
 }
+
+type APILink struct {
+	Path        string
+	ContainerID string
+	Image       string
+}

api_test.go

@@ -112,6 +112,7 @@ func TestGetInfo(t *testing.T) {
 
 func TestGetEvents(t *testing.T) {
 	runtime := mkRuntime(t)
+	defer nuke(runtime)
 	srv := &Server{
 		runtime:   runtime,
 		events:    make([]utils.JSONMessage, 0, 64),
@@ -346,7 +347,7 @@ func TestGetContainersJSON(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"echo", "test"},
 	})
@@ -361,9 +362,11 @@ func TestGetContainersJSON(t *testing.T) {
 	}
 
 	r := httptest.NewRecorder()
-	if err := getContainersJSON(srv, APIVERSION, r, req, nil); err != nil {
-		t.Fatal(err)
-	}
+	setTimeout(t, "getContainerJSON timed out", 5*time.Second, func() {
+		if err := getContainersJSON(srv, APIVERSION, r, req, nil); err != nil {
+			t.Fatal(err)
+		}
+	})
 	containers := []APIContainers{}
 	if err := json.Unmarshal(r.Body.Bytes(), &containers); err != nil {
 		t.Fatal(err)
@@ -383,7 +386,7 @@ func TestGetContainersExport(t *testing.T) {
 	srv := &Server{runtime: runtime}
 
 	// Create a container and remove a file
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"touch", "/test"},
@@ -399,7 +402,7 @@ func TestGetContainersExport(t *testing.T) {
 	}
 
 	r := httptest.NewRecorder()
-	if err = getContainersExport(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil {
+	if err := getContainersExport(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -433,7 +436,7 @@ func TestGetContainersChanges(t *testing.T) {
 	srv := &Server{runtime: runtime}
 
 	// Create a container and remove a file
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"/bin/rm", "/etc/passwd"},
@@ -471,15 +474,12 @@ func TestGetContainersChanges(t *testing.T) {
 
 func TestGetContainersTop(t *testing.T) {
 	t.Skip("Fixme. Skipping test for now. Reported error when testing using dind: 'api_test.go:527: Expected 2 processes, found 0.'")
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/sh", "-c", "cat"},
@@ -561,7 +561,7 @@ func TestGetContainersByName(t *testing.T) {
 	srv := &Server{runtime: runtime}
 
 	// Create a container and remove a file
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"echo", "test"},
@@ -592,7 +592,7 @@ func TestPostCommit(t *testing.T) {
 	srv := &Server{runtime: runtime}
 
 	// Create a container and remove a file
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"touch", "/test"},
@@ -671,13 +671,21 @@ func TestPostContainersCreate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := os.Stat(path.Join(container.rwPath(), "test")); err != nil {
+	if err := container.EnsureMounted(); err != nil {
+		t.Fatalf("Unable to mount container: %s", err)
+	}
+
+	if _, err := os.Stat(path.Join(container.RootfsPath(), "test")); err != nil {
 		if os.IsNotExist(err) {
 			utils.Debugf("Err: %s", err)
 			t.Fatalf("The test file has not been created")
 		}
 		t.Fatal(err)
 	}
+
+	if err := container.Unmount(); err != nil {
+		t.Fatalf("Unable to unmount container: %s", err)
+	}
 }
 
 func TestPostContainersKill(t *testing.T) {
@@ -686,7 +694,7 @@ func TestPostContainersKill(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
@@ -728,7 +736,7 @@ func TestPostContainersRestart(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
@@ -782,7 +790,7 @@ func TestPostContainersStart(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
@@ -834,7 +842,7 @@ func TestPostContainersStop(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
@@ -881,7 +889,7 @@ func TestPostContainersWait(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/sleep", "1"},
@@ -923,7 +931,7 @@ func TestPostContainersAttach(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
@@ -1012,7 +1020,7 @@ func TestPostContainersAttachStderr(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/sh", "-c", "/bin/cat >&2"},
@@ -1104,7 +1112,7 @@ func TestDeleteContainers(t *testing.T) {
 
 	srv := &Server{runtime: runtime}
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"touch", "/test"},
 	})
@@ -1142,7 +1150,8 @@ func TestOptionsRoute(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime, enableCors: true}
+	runtime.config.EnableCors = true
+	srv := &Server{runtime: runtime}
 
 	r := httptest.NewRecorder()
 	router, err := createRouter(srv, false)
@@ -1165,7 +1174,8 @@ func TestGetEnabledCors(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime, enableCors: true}
+	runtime.config.EnableCors = true
+	srv := &Server{runtime: runtime}
 
 	r := httptest.NewRecorder()
 
@@ -1292,7 +1302,7 @@ func TestPostContainersCopy(t *testing.T) {
 	srv := &Server{runtime: runtime}
 
 	// Create a container and remove a file
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"touch", "/test.txt"},

archive.go

@@ -80,20 +80,74 @@ func (compression *Compression) Extension() string {
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
 func Tar(path string, compression Compression) (io.Reader, error) {
-	return TarFilter(path, compression, nil)
+	return TarFilter(path, compression, nil, true, nil)
+}
+
+func escapeName(name string) string {
+	escaped := make([]byte, 0)
+	for i, c := range []byte(name) {
+		if i == 0 && c == '/' {
+			continue
+		}
+		// all printable chars except "-" which is 0x2d
+		if (0x20 <= c && c <= 0x7E) && c != 0x2d {
+			escaped = append(escaped, c)
+		} else {
+			escaped = append(escaped, fmt.Sprintf("\\%03o", c)...)
+		}
+	}
+	return string(escaped)
 }
 
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
-func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
+func TarFilter(path string, compression Compression, filter []string, recursive bool, createFiles []string) (io.Reader, error) {
+	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"}
 	if filter == nil {
 		filter = []string{"."}
 	}
-	for _, f := range filter {
-		args = append(args, "-c"+compression.Flag(), f)
+	args = append(args, "-c"+compression.Flag())
+
+	if !recursive {
+		args = append(args, "--no-recursion")
 	}
-	return CmdStream(exec.Command(args[0], args[1:]...))
+
+	files := ""
+	for _, f := range filter {
+		files = files + escapeName(f) + "\n"
+	}
+
+	tmpDir := ""
+
+	if createFiles != nil {
+		var err error // Can't use := here or we override the outer tmpDir
+		tmpDir, err = ioutil.TempDir("", "docker-tar")
+		if err != nil {
+			return nil, err
+		}
+
+		files = files + "-C" + tmpDir + "\n"
+		for _, f := range createFiles {
+			path := filepath.Join(tmpDir, f)
+			err := os.MkdirAll(filepath.Dir(path), 0600)
+			if err != nil {
+				return nil, err
+			}
+
+			if file, err := os.OpenFile(path, os.O_CREATE, 0600); err != nil {
+				return nil, err
+			} else {
+				file.Close()
+			}
+			files = files + escapeName(f) + "\n"
+		}
+	}
+
+	return CmdStream(exec.Command(args[0], args[1:]...), &files, func() {
+		if tmpDir != "" {
+			_ = os.RemoveAll(tmpDir)
+		}
+	})
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
@@ -140,7 +194,7 @@ func Untar(archive io.Reader, path string) error {
 // TarUntar aborts and returns the error.
 func TarUntar(src string, filter []string, dst string) error {
 	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
-	archive, err := TarFilter(src, Uncompressed, filter)
+	archive, err := TarFilter(src, Uncompressed, filter, true, nil)
 	if err != nil {
 		return err
 	}
@@ -227,13 +281,33 @@ func CopyFileWithTar(src, dst string) error {
 // CmdStream executes a command, and returns its stdout as a stream.
 // If the command fails to run or doesn't complete successfully, an error
 // will be returned, including anything written on stderr.
-func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
+func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) {
+	if input != nil {
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			if atEnd != nil {
+				atEnd()
+			}
+			return nil, err
+		}
+		// Write stdin if any
+		go func() {
+			_, _ = stdin.Write([]byte(*input))
+			stdin.Close()
+		}()
+	}
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	pipeR, pipeW := io.Pipe()
@@ -258,6 +332,9 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
 		} else {
 			pipeW.Close()
 		}
+		if atEnd != nil {
+			atEnd()
+		}
 	}()
 	// Run the command and return the pipe
 	if err := cmd.Start(); err != nil {

archive_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestCmdStreamLargeStderr(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -35,7 +35,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
 
 func TestCmdStreamBad(t *testing.T) {
 	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
-	out, err := CmdStream(badCmd)
+	out, err := CmdStream(badCmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -50,7 +50,7 @@ func TestCmdStreamBad(t *testing.T) {
 
 func TestCmdStreamGood(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

auth/auth_test.go

@@ -76,7 +76,7 @@ func TestCreateAccount(t *testing.T) {
 }
 
 func setupTempConfigFile() (*ConfigFile, error) {
-	root, err := ioutil.TempDir("", "docker-test")
+	root, err := ioutil.TempDir("", "docker-test-auth")
 	if err != nil {
 		return nil, err
 	}
@@ -101,6 +101,7 @@ func TestSameAuthDataPostSave(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(configFile.rootPath)
 
 	err = SaveConfig(configFile)
 	if err != nil {
@@ -127,6 +128,7 @@ func TestResolveAuthConfigIndexServer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(configFile.rootPath)
 
 	for _, registry := range []string{"", IndexServerAddress()} {
 		resolved := configFile.ResolveAuthConfig(registry)
@@ -141,6 +143,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(configFile.rootPath)
 
 	registryAuth := AuthConfig{
 		Username: "foo-user",

buildfile.go

@@ -176,6 +176,8 @@ func (b *buildFile) CmdEnv(args string) error {
 func (b *buildFile) CmdCmd(args string) error {
 	var cmd []string
 	if err := json.Unmarshal([]byte(args), &cmd); err != nil {
+		// If the unmarshal fails, it is not an error, we just use the
+		// args as a string.
 		utils.Debugf("Error unmarshalling: %s, setting cmd to /bin/sh -c", err)
 		cmd = []string{"/bin/sh", "-c", args}
 	}
@@ -187,6 +189,9 @@ func (b *buildFile) CmdCmd(args string) error {
 }
 
 func (b *buildFile) CmdExpose(args string) error {
+	if strings.Contains(args, ":") {
+		return fmt.Errorf("EXPOSE cannot be used to bind to a host ip or port")
+	}
 	ports := strings.Split(args, " ")
 	b.config.PortSpecs = append(ports, b.config.PortSpecs...)
 	return b.commit("", b.config.Cmd, fmt.Sprintf("EXPOSE %v", ports))
@@ -296,7 +301,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error {
 		}
 		// First try to unpack the source as an archive
 	} else if err := UntarPath(origPath, destPath); err != nil {
-		utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err)
+		utils.Debugf("[tar] Not a directory nor a tar archive. Copying as a file. Untar error from %s to %s: %s", origPath, destPath, err)
 		// If that fails, just copy it as a regular file
 		if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil {
 			return err
@@ -332,7 +337,7 @@ func (b *buildFile) CmdAdd(args string) error {
 
 	b.config.Image = b.image
 	// Create the container and start it
-	container, err := b.runtime.Create(b.config)
+	container, _, err := b.runtime.Create(b.config)
 	if err != nil {
 		return err
 	}
@@ -367,7 +372,7 @@ func (b *buildFile) run() (string, error) {
 	b.config.Image = b.image
 
 	// Create the container and start it
-	c, err := b.runtime.Create(b.config)
+	c, _, err := b.runtime.Create(b.config)
 	if err != nil {
 		return "", err
 	}
@@ -423,7 +428,7 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error {
 			}
 		}
 
-		container, err := b.runtime.Create(b.config)
+		container, _, err := b.runtime.Create(b.config)
 		if err != nil {
 			return err
 		}

buildfile_test.go

@@ -229,10 +229,7 @@ func TestBuild(t *testing.T) {
 
 func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image {
 	if srv == nil {
-		runtime, err := newTestRuntime()
-		if err != nil {
-			t.Fatal(err)
-		}
+		runtime := mkRuntime(t)
 		defer nuke(runtime)
 
 		srv = &Server{
@@ -370,10 +367,7 @@ func TestBuildEntrypoint(t *testing.T) {
 // testing #1405 - config.Cmd does not get cleaned up if
 // utilizing cache
 func TestBuildEntrypointRunCleanup(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{
@@ -402,10 +396,7 @@ func TestBuildEntrypointRunCleanup(t *testing.T) {
 }
 
 func TestBuildImageWithCache(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{
@@ -433,10 +424,7 @@ func TestBuildImageWithCache(t *testing.T) {
 }
 
 func TestBuildImageWithoutCache(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{
@@ -464,10 +452,7 @@ func TestBuildImageWithoutCache(t *testing.T) {
 }
 
 func TestForbiddenContextPath(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{
@@ -513,10 +498,7 @@ func TestForbiddenContextPath(t *testing.T) {
 }
 
 func TestBuildADDFileNotFound(t *testing.T) {
-	runtime, err := newTestRuntime()
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{

changes.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 )
 
 type ChangeType int
@@ -33,74 +34,283 @@ func (change *Change) String() string {
 	return fmt.Sprintf("%s %s", kind, change.Path)
 }
 
-func Changes(layers []string, rw string) ([]Change, error) {
+type FileInfo struct {
+	parent   *FileInfo
+	name     string
+	stat     syscall.Stat_t
+	children map[string]*FileInfo
+}
+
+func (root *FileInfo) LookUp(path string) *FileInfo {
+	parent := root
+	if path == "/" {
+		return root
+	}
+
+	pathElements := strings.Split(path, "/")
+	for _, elem := range pathElements {
+		if elem != "" {
+			child := parent.children[elem]
+			if child == nil {
+				return nil
+			}
+			parent = child
+		}
+	}
+	return parent
+}
+
+func (info *FileInfo) path() string {
+	if info.parent == nil {
+		return "/"
+	}
+	return filepath.Join(info.parent.path(), info.name)
+}
+
+func (info *FileInfo) unlink() {
+	if info.parent != nil {
+		delete(info.parent.children, info.name)
+	}
+}
+
+func (info *FileInfo) Remove(path string) bool {
+	child := info.LookUp(path)
+	if child != nil {
+		child.unlink()
+		return true
+	}
+	return false
+}
+
+func (info *FileInfo) isDir() bool {
+	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
+}
+
+func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
+	if oldInfo == nil {
+		// add
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeAdd,
+		}
+		*changes = append(*changes, change)
+	}
+
+	// We make a copy so we can modify it to detect additions
+	// also, we only recurse on the old dir if the new info is a directory
+	// otherwise any previous delete/change is considered recursive
+	oldChildren := make(map[string]*FileInfo)
+	if oldInfo != nil && info.isDir() {
+		for k, v := range oldInfo.children {
+			oldChildren[k] = v
+		}
+	}
+
+	for name, newChild := range info.children {
+		oldChild, _ := oldChildren[name]
+		if oldChild != nil {
+			// change?
+			oldStat := &oldChild.stat
+			newStat := &newChild.stat
+			// Note: We can't compare inode or ctime or blocksize here, because these change
+			// when copying a file into a container. However, that is not generally a problem
+			// because any content change will change mtime, and any status change should
+			// be visible when actually comparing the stat fields. The only time this
+			// breaks down is if some code intentionally hides a change by setting
+			// back mtime
+			oldMtime := syscall.NsecToTimeval(oldStat.Mtim.Nano())
+			newMtime := syscall.NsecToTimeval(oldStat.Mtim.Nano())
+			if oldStat.Mode != newStat.Mode ||
+				oldStat.Uid != newStat.Uid ||
+				oldStat.Gid != newStat.Gid ||
+				oldStat.Rdev != newStat.Rdev ||
+				// Don't look at size for dirs, its not a good measure of change
+				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
+				oldMtime.Sec != newMtime.Sec ||
+				oldMtime.Usec != newMtime.Usec {
+				change := Change{
+					Path: newChild.path(),
+					Kind: ChangeModify,
+				}
+				*changes = append(*changes, change)
+			}
+
+			// Remove from copy so we can detect deletions
+			delete(oldChildren, name)
+		}
+
+		newChild.addChanges(oldChild, changes)
+	}
+	for _, oldChild := range oldChildren {
+		// delete
+		change := Change{
+			Path: oldChild.path(),
+			Kind: ChangeDelete,
+		}
+		*changes = append(*changes, change)
+	}
+
+}
+
+func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
 	var changes []Change
-	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
+
+	info.addChanges(oldInfo, &changes)
+
+	return changes
+}
+
+func newRootFileInfo() *FileInfo {
+	root := &FileInfo{
+		name:     "/",
+		children: make(map[string]*FileInfo),
+	}
+	return root
+}
+
+func applyLayer(root *FileInfo, layer string) error {
+	err := filepath.Walk(layer, func(layerPath string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Skip root
+		if layerPath == layer {
+			return nil
+		}
+
+		// rebase path
+		relPath, err := filepath.Rel(layer, layerPath)
+		if err != nil {
+			return err
+		}
+		relPath = filepath.Join("/", relPath)
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", relPath); err != nil || matched {
+			if err != nil || !f.IsDir() {
+				return err
+			}
+			return filepath.SkipDir
+		}
+
+		var layerStat syscall.Stat_t
+		err = syscall.Lstat(layerPath, &layerStat)
+		if err != nil {
+			return err
+		}
+
+		file := filepath.Base(relPath)
+		// If there is a whiteout, then the file was removed
+		if strings.HasPrefix(file, ".wh.") {
+			originalFile := file[len(".wh."):]
+			deletePath := filepath.Join(filepath.Dir(relPath), originalFile)
+
+			root.Remove(deletePath)
+		} else {
+			// Added or changed file
+			existing := root.LookUp(relPath)
+			if existing != nil {
+				// Changed file
+				existing.stat = layerStat
+				if !existing.isDir() {
+					// Changed from dir to non-dir, delete all previous files
+					existing.children = make(map[string]*FileInfo)
+				}
+			} else {
+				// Added file
+				parent := root.LookUp(filepath.Dir(relPath))
+				if parent == nil {
+					return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
+				}
+
+				info := &FileInfo{
+					name:     filepath.Base(relPath),
+					children: make(map[string]*FileInfo),
+					parent:   parent,
+					stat:     layerStat,
+				}
+
+				parent.children[info.name] = info
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+func collectFileInfo(sourceDir string) (*FileInfo, error) {
+	root := newRootFileInfo()
+
+	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
 		// Rebase path
-		path, err = filepath.Rel(rw, path)
+		relPath, err := filepath.Rel(sourceDir, path)
 		if err != nil {
 			return err
 		}
-		path = filepath.Join("/", path)
+		relPath = filepath.Join("/", relPath)
 
-		// Skip root
-		if path == "/" {
+		if relPath == "/" {
 			return nil
 		}
 
-		// Skip AUFS metadata
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
+		parent := root.LookUp(filepath.Dir(relPath))
+		if parent == nil {
+			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
+		}
+
+		info := &FileInfo{
+			name:     filepath.Base(relPath),
+			children: make(map[string]*FileInfo),
+			parent:   parent,
+		}
+
+		if err := syscall.Lstat(path, &info.stat); err != nil {
 			return err
 		}
 
-		change := Change{
-			Path: path,
-		}
+		parent.children[info.name] = info
 
-		// Find out what kind of modification happened
-		file := filepath.Base(path)
-		// If there is a whiteout, then the file was removed
-		if strings.HasPrefix(file, ".wh.") {
-			originalFile := file[len(".wh."):]
-			change.Path = filepath.Join(filepath.Dir(path), originalFile)
-			change.Kind = ChangeDelete
-		} else {
-			// Otherwise, the file was added
-			change.Kind = ChangeAdd
-
-			// ...Unless it already existed in a top layer, in which case, it's a modification
-			for _, layer := range layers {
-				stat, err := os.Stat(filepath.Join(layer, path))
-				if err != nil && !os.IsNotExist(err) {
-					return err
-				}
-				if err == nil {
-					// The file existed in the top layer, so that's a modification
-
-					// However, if it's a directory, maybe it wasn't actually modified.
-					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
-					if stat.IsDir() && f.IsDir() {
-						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
-							// Both directories are the same, don't record the change
-							return nil
-						}
-					}
-					change.Kind = ChangeModify
-					break
-				}
-			}
-		}
-
-		// Record change
-		changes = append(changes, change)
 		return nil
 	})
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil {
 		return nil, err
 	}
-	return changes, nil
+	return root, nil
+}
+
+// Compare a directory with an array of layer directories it was based on and
+// generate an array of Change objects describing the changes
+func ChangesLayers(newDir string, layers []string) ([]Change, error) {
+	newRoot, err := collectFileInfo(newDir)
+	if err != nil {
+		return nil, err
+	}
+	oldRoot := newRootFileInfo()
+	for i := len(layers) - 1; i >= 0; i-- {
+		layer := layers[i]
+		if err = applyLayer(oldRoot, layer); err != nil {
+			return nil, err
+		}
+	}
+
+	return newRoot.Changes(oldRoot), nil
+}
+
+// Compare two directories and generate an array of Change objects describing the changes
+func ChangesDirs(newDir, oldDir string) ([]Change, error) {
+	oldRoot, err := collectFileInfo(oldDir)
+	if err != nil {
+		return nil, err
+	}
+	newRoot, err := collectFileInfo(newDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return newRoot.Changes(oldRoot), nil
 }

commands.go

@@ -23,6 +23,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"runtime"
 	"sort"
 	"strconv"
@@ -465,6 +466,11 @@ func (cli *DockerCli) CmdInfo(args ...string) error {
 
 	fmt.Fprintf(cli.out, "Containers: %d\n", out.Containers)
 	fmt.Fprintf(cli.out, "Images: %d\n", out.Images)
+	if out.DevmapperDataTotal != 0 {
+		fmt.Fprintf(cli.out, "Devmapper disk use: Data: %.1f/%.1f Metadata: %.1f/%.1f\n",
+			float64(out.DevmapperDataUsed)/(1024*1024), float64(out.DevmapperDataTotal)/(1024*1024),
+			float64(out.DevmapperMetadataUsed)/(1024*1024), float64(out.DevmapperMetadataTotal)/(1024*1024))
+	}
 	if out.Debug || os.Getenv("DEBUG") != "" {
 		fmt.Fprintf(cli.out, "Debug mode (server): %v\n", out.Debug)
 		fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "")
@@ -473,6 +479,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error {
 		fmt.Fprintf(cli.out, "LXC Version: %s\n", out.LXCVersion)
 		fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener)
 		fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion)
+		fmt.Fprintf(cli.out, "Devmapper pool: %s\n", out.DevmapperPool)
 	}
 
 	if len(out.IndexServerAddress) != 0 {
@@ -740,6 +747,8 @@ func (cli *DockerCli) CmdHistory(args ...string) error {
 func (cli *DockerCli) CmdRm(args ...string) error {
 	cmd := Subcmd("rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove one or more containers")
 	v := cmd.Bool("v", false, "Remove the volumes associated to the container")
+	link := cmd.Bool("link", false, "Remove the specified link and not the underlying container")
+
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -751,6 +760,9 @@ func (cli *DockerCli) CmdRm(args ...string) error {
 	if *v {
 		val.Set("v", "1")
 	}
+	if *link {
+		val.Set("link", "1")
+	}
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("DELETE", "/containers/"+name+"?"+val.Encode(), nil)
 		if err != nil {
@@ -985,25 +997,19 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 				out.Tag = "<none>"
 			}
 
+			if !*noTrunc {
+				out.ID = utils.TruncateID(out.ID)
+			}
+
 			if !*quiet {
-				fmt.Fprintf(w, "%s\t%s\t", out.Repository, out.Tag)
-				if *noTrunc {
-					fmt.Fprintf(w, "%s\t", out.ID)
-				} else {
-					fmt.Fprintf(w, "%s\t", utils.TruncateID(out.ID))
-				}
-				fmt.Fprintf(w, "%s ago\t", utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t", out.Repository, out.Tag, out.ID, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
 				if out.VirtualSize > 0 {
 					fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.Size), utils.HumanSize(out.VirtualSize))
 				} else {
 					fmt.Fprintf(w, "%s\n", utils.HumanSize(out.Size))
 				}
 			} else {
-				if *noTrunc {
-					fmt.Fprintln(w, out.ID)
-				} else {
-					fmt.Fprintln(w, utils.TruncateID(out.ID))
-				}
+				fmt.Fprintln(w, out.ID)
 			}
 		}
 
@@ -1017,10 +1023,10 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 func displayablePorts(ports []APIPort) string {
 	result := []string{}
 	for _, port := range ports {
-		if port.Type == "tcp" {
-			result = append(result, fmt.Sprintf("%d->%d", port.PublicPort, port.PrivatePort))
+		if port.IP == "" {
+			result = append(result, fmt.Sprintf("%d/%s", port.PublicPort, port.Type))
 		} else {
-			result = append(result, fmt.Sprintf("%d->%d/%s", port.PublicPort, port.PrivatePort, port.Type))
+			result = append(result, fmt.Sprintf("%s:%d->%d/%s", port.IP, port.PublicPort, port.PrivatePort, port.Type))
 		}
 	}
 	sort.Strings(result)
@@ -1073,7 +1079,7 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 	}
 	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
 	if !*quiet {
-		fmt.Fprint(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS")
+		fmt.Fprint(w, "ID\tIMAGE\tCOMMAND\tCREATED\tSTATUS\tPORTS\tNAMES")
 		if *size {
 			fmt.Fprintln(w, "\tSIZE")
 		} else {
@@ -1082,12 +1088,17 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 	}
 
 	for _, out := range outs {
+		if !*noTrunc {
+			out.ID = utils.TruncateID(out.ID)
+		}
 		if !*quiet {
-			if *noTrunc {
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\t%s\t", out.ID, out.Image, out.Command, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Status, displayablePorts(out.Ports))
-			} else {
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t%s\t%s\t", utils.TruncateID(out.ID), out.Image, utils.Trunc(out.Command, 20), utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Status, displayablePorts(out.Ports))
+			if !*noTrunc {
+				out.Command = utils.Trunc(out.Command, 20)
+				for i := 0; i < len(out.Names); i++ {
+					out.Names[i] = utils.Trunc(out.Names[i], 10)
+				}
 			}
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\t%s\t", out.ID, out.Image, out.Command, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Status, displayablePorts(out.Ports), strings.Join(out.Names, ","))
 			if *size {
 				if out.SizeRootFs > 0 {
 					fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.SizeRw), utils.HumanSize(out.SizeRootFs))
@@ -1098,11 +1109,7 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 				fmt.Fprint(w, "\n")
 			}
 		} else {
-			if *noTrunc {
-				fmt.Fprintln(w, out.ID)
-			} else {
-				fmt.Fprintln(w, utils.TruncateID(out.ID))
-			}
+			fmt.Fprintln(w, out.ID)
 		}
 	}
 
@@ -1112,6 +1119,64 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 	return nil
 }
 
+func (cli *DockerCli) CmdLs(args ...string) error {
+	cmd := Subcmd("ls", "", "List links for containers")
+	flAll := cmd.Bool("a", false, "Show all links")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	v := url.Values{}
+	if *flAll {
+		v.Set("all", "1")
+	}
+
+	body, _, err := cli.call("GET", "/containers/links?"+v.Encode(), nil)
+	if err != nil {
+		return err
+	}
+	var links []APILink
+	if err := json.Unmarshal(body, &links); err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
+	fmt.Fprintf(w, "NAME\tID\tIMAGE")
+	fmt.Fprintf(w, "\n")
+
+	sortLinks(links, func(i, j APILink) bool {
+		return len(i.Path) < len(j.Path)
+	})
+	for _, link := range links {
+		fmt.Fprintf(w, "%s\t%s\t%s", link.Path, utils.TruncateID(link.ContainerID), link.Image)
+		fmt.Fprintf(w, "\n")
+	}
+	w.Flush()
+
+	return nil
+}
+
+func (cli *DockerCli) CmdLink(args ...string) error {
+	cmd := Subcmd("link", "CURRENT_NAME NEW_NAME", "Link the container with a new name")
+	if err := cmd.Parse(args); err != nil {
+		return nil
+	}
+	if cmd.NArg() != 2 {
+		cmd.Usage()
+		return nil
+	}
+	body := map[string]string{
+		"currentName": cmd.Arg(0),
+		"newName":     cmd.Arg(1),
+	}
+
+	_, _, err := cli.call("POST", "/containers/link", body)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (cli *DockerCli) CmdCommit(args ...string) error {
 	cmd := Subcmd("commit", "[OPTIONS] CONTAINER [REPOSITORY [TAG]]", "Create a new image from a container's changes")
 	flComment := cmd.String("m", "", "Commit message")
@@ -1229,8 +1294,9 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
 		cmd.Usage()
 		return nil
 	}
+	name := cmd.Arg(0)
 
-	if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1&stderr=1", false, nil, cli.out, cli.err); err != nil {
+	if err := cli.hijack("POST", "/containers/"+name+"/attach?logs=1&stdout=1&stderr=1", false, nil, cli.out, cli.err); err != nil {
 		return err
 	}
 	return nil
@@ -1245,8 +1311,8 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 		cmd.Usage()
 		return nil
 	}
-
-	body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/json", nil)
+	name := cmd.Arg(0)
+	body, _, err := cli.call("GET", "/containers/"+name+"/json", nil)
 	if err != nil {
 		return err
 	}
@@ -1326,18 +1392,6 @@ func (cli *DockerCli) CmdSearch(args ...string) error {
 // Ports type - Used to parse multiple -p flags
 type ports []int
 
-// ListOpts type
-type ListOpts []string
-
-func (opts *ListOpts) String() string {
-	return fmt.Sprint(*opts)
-}
-
-func (opts *ListOpts) Set(value string) error {
-	*opts = append(*opts, value)
-	return nil
-}
-
 // AttachOpts stores arguments to 'docker run -a', eg. which streams to attach to
 type AttachOpts map[string]bool
 
@@ -1533,7 +1587,11 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	if config.AttachStdin || config.AttachStdout || config.AttachStderr {
 		if config.Tty {
 			if err := cli.monitorTtySize(runResult.ID); err != nil {
-				utils.Errorf("Error monitoring TTY size: %s\n", err)
+				// When running the test suite, there is no terminal, just pipes.
+				// Discard the error then.
+				if os.Getenv("TEST") != "1" {
+					utils.Errorf("Error monitoring TTY size: %s\n", err)
+				}
 			}
 		}
 
@@ -1642,6 +1700,10 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int,
 		params = bytes.NewBuffer(buf)
 	}
 
+	// fixme: refactor client to support redirect
+	re := regexp.MustCompile("/+")
+	path = re.ReplaceAllString(path, "/")
+
 	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), params)
 	if err != nil {
 		return nil, -1, err
@@ -1687,6 +1749,11 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h
 	if (method == "POST" || method == "PUT") && in == nil {
 		in = bytes.NewReader([]byte{})
 	}
+
+	// fixme: refactor client to support redirect
+	re := regexp.MustCompile("/+")
+	path = re.ReplaceAllString(path, "/")
+
 	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), in)
 	if err != nil {
 		return err
@@ -1743,6 +1810,9 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h
 }
 
 func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, stdout, stderr io.Writer) error {
+	// fixme: refactor client to support redirect
+	re := regexp.MustCompile("/+")
+	path = re.ReplaceAllString(path, "/")
 
 	req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil)
 	if err != nil {

config.go

@@ -0,0 +1,18 @@
+package docker
+
+import (
+	"net"
+)
+
+type DaemonConfig struct {
+	Pidfile                     string
+	GraphPath                   string
+	ProtoAddresses              []string
+	AutoRestart                 bool
+	EnableCors                  bool
+	Dns                         []string
+	EnableIptables              bool
+	BridgeIface                 string
+	DefaultIp                   net.IP
+	InterContainerCommunication bool
+}

container.go

@@ -16,7 +16,6 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -43,6 +42,7 @@ type Container struct {
 	ResolvConfPath string
 	HostnamePath   string
 	HostsPath      string
+	FilesystemType string
 
 	cmd       *exec.Cmd
 	stdout    *utils.WriteBroadcaster
@@ -58,6 +58,8 @@ type Container struct {
 	// Store rw/ro in a separate structure to preserve reverse-compatibility on-disk.
 	// Easier than migrating older container configs :)
 	VolumesRW map[string]bool
+
+	activeLinks map[string]*Link
 }
 
 type Config struct {
@@ -70,7 +72,8 @@ type Config struct {
 	AttachStdin     bool
 	AttachStdout    bool
 	AttachStderr    bool
-	PortSpecs       []string
+	PortSpecs       []string // Deprecated - Can be in the format of 8080/tcp
+	ExposedPorts    map[Port]struct{}
 	Tty             bool // Attach standard streams to a tty, including stdin if it is not closed.
 	OpenStdin       bool // Open stdin
 	StdinOnce       bool // If true, close stdin after the 1 attached client disconnects.
@@ -90,6 +93,8 @@ type HostConfig struct {
 	Binds           []string
 	ContainerIDFile string
 	LxcConf         []KeyValuePair
+	PortBindings    map[Port][]PortBinding
+	Links           []string
 }
 
 type BindMap struct {
@@ -107,6 +112,34 @@ type KeyValuePair struct {
 	Value string
 }
 
+type PortBinding struct {
+	HostIp   string
+	HostPort string
+}
+
+// 80/tcp
+type Port string
+
+func (p Port) Proto() string {
+	return strings.Split(string(p), "/")[1]
+}
+
+func (p Port) Port() string {
+	return strings.Split(string(p), "/")[0]
+}
+
+func (p Port) Int() int {
+	i, err := parsePort(p.Port())
+	if err != nil {
+		panic(err)
+	}
+	return i
+}
+
+func NewPort(proto, port string) Port {
+	return Port(fmt.Sprintf("%s/%s", port, proto))
+}
+
 func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
 	cmd := Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container")
 	if os.Getenv("TEST") != "" {
@@ -135,26 +168,32 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 
 	flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)")
 
-	var flPorts ListOpts
-	cmd.Var(&flPorts, "p", "Expose a container's port to the host (use 'docker port' to see the actual mapping)")
+	var flPublish utils.ListOpts
+	cmd.Var(&flPublish, "p", "Publish a container's port to the host (use 'docker port' to see the actual mapping)")
 
-	var flEnv ListOpts
+	var flExpose utils.ListOpts
+	cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host")
+
+	var flEnv utils.ListOpts
 	cmd.Var(&flEnv, "e", "Set environment variables")
 
-	var flDns ListOpts
+	var flDns utils.ListOpts
 	cmd.Var(&flDns, "dns", "Set custom dns servers")
 
 	flVolumes := NewPathOpts()
 	cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)")
 
-	var flVolumesFrom ListOpts
+	var flVolumesFrom utils.ListOpts
 	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container")
 
 	flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
 
-	var flLxcOpts ListOpts
+	var flLxcOpts utils.ListOpts
 	cmd.Var(&flLxcOpts, "lxc-conf", "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"")
 
+	var flLinks utils.ListOpts
+	cmd.Var(&flLinks, "link", "Add link to another container (containerid:alias)")
+
 	if err := cmd.Parse(args); err != nil {
 		return nil, nil, cmd, err
 	}
@@ -220,10 +259,28 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 		hostname = parts[0]
 		domainname = parts[1]
 	}
+
+	ports, portBindings, err := parsePortSpecs(flPublish)
+	if err != nil {
+		return nil, nil, cmd, err
+	}
+
+	// Merge in exposed ports to the map of published ports
+	for _, e := range flExpose {
+		if strings.Contains(e, ":") {
+			return nil, nil, cmd, fmt.Errorf("Invalid port format for -expose: %s", e)
+		}
+		p := NewPort(splitProtoPort(e))
+		if _, exists := ports[p]; !exists {
+			ports[p] = struct{}{}
+		}
+	}
+
 	config := &Config{
-		Hostname:        hostname,
+		Hostname:        *flHostname,
 		Domainname:      domainname,
-		PortSpecs:       flPorts,
+		PortSpecs:       nil, // Deprecated
+		ExposedPorts:    ports,
 		User:            *flUser,
 		Tty:             *flTty,
 		NetworkDisabled: !*flNetwork,
@@ -243,10 +300,13 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 		Privileged:      *flPrivileged,
 		WorkingDir:      *flWorkingDir,
 	}
+
 	hostConfig := &HostConfig{
 		Binds:           binds,
 		ContainerIDFile: *flContainerIDFile,
 		LxcConf:         lxcConf,
+		PortBindings:    portBindings,
+		Links:           flLinks,
 	}
 
 	if capabilities != nil && *flMemory > 0 && !capabilities.SwapLimit {
@@ -261,48 +321,55 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 	return config, hostConfig, cmd, nil
 }
 
-type PortMapping map[string]string
+type PortMapping map[string]string // Deprecated
 
 type NetworkSettings struct {
 	IPAddress   string
 	IPPrefixLen int
 	Gateway     string
 	Bridge      string
-	PortMapping map[string]PortMapping
+	PortMapping map[string]PortMapping // Deprecated
+	Ports       map[Port][]PortBinding
 }
 
-// returns a more easy to process description of the port mapping defined in the settings
 func (settings *NetworkSettings) PortMappingAPI() []APIPort {
 	var mapping []APIPort
-	for private, public := range settings.PortMapping["Tcp"] {
-		pubint, _ := strconv.ParseInt(public, 0, 0)
-		privint, _ := strconv.ParseInt(private, 0, 0)
-		mapping = append(mapping, APIPort{
-			PrivatePort: privint,
-			PublicPort:  pubint,
-			Type:        "tcp",
-		})
-	}
-	for private, public := range settings.PortMapping["Udp"] {
-		pubint, _ := strconv.ParseInt(public, 0, 0)
-		privint, _ := strconv.ParseInt(private, 0, 0)
-		mapping = append(mapping, APIPort{
-			PrivatePort: privint,
-			PublicPort:  pubint,
-			Type:        "udp",
-		})
+	for port, bindings := range settings.Ports {
+		p, _ := parsePort(port.Port())
+		if len(bindings) == 0 {
+			mapping = append(mapping, APIPort{
+				PublicPort: int64(p),
+				Type:       port.Proto(),
+			})
+			continue
+		}
+		for _, binding := range bindings {
+			p, _ := parsePort(port.Port())
+			h, _ := parsePort(binding.HostPort)
+			mapping = append(mapping, APIPort{
+				PrivatePort: int64(p),
+				PublicPort:  int64(h),
+				Type:        port.Proto(),
+				IP:          binding.HostIp,
+			})
+		}
 	}
 	return mapping
 }
 
 // Inject the io.Reader at the given path. Note: do not close the reader
 func (container *Container) Inject(file io.Reader, pth string) error {
-	// Make sure the directory exists
-	if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil {
+	if err := container.EnsureMounted(); err != nil {
 		return err
 	}
+
+	// Make sure the directory exists
+	if err := os.MkdirAll(path.Join(container.RootfsPath(), path.Dir(pth)), 0755); err != nil {
+		return err
+	}
+
 	// FIXME: Handle permissions/already existing dest
-	dest, err := os.Create(path.Join(container.rwPath(), pth))
+	dest, err := os.Create(path.Join(container.RootfsPath(), pth))
 	if err != nil {
 		return err
 	}
@@ -590,7 +657,7 @@ func (container *Container) Start(hostConfig *HostConfig) (err error) {
 	if container.runtime.networkManager.disabled {
 		container.Config.NetworkDisabled = true
 	} else {
-		if err := container.allocateNetwork(); err != nil {
+		if err := container.allocateNetwork(hostConfig); err != nil {
 			return err
 		}
 	}
@@ -753,6 +820,7 @@ func (container *Container) Start(hostConfig *HostConfig) (err error) {
 	}
 
 	params := []string{
+		"lxc-start",
 		"-n", container.ID,
 		"-f", container.lxcConfigPath(),
 		"--",
@@ -780,6 +848,46 @@ func (container *Container) Start(hostConfig *HostConfig) (err error) {
 		"-e", "container=lxc",
 		"-e", "HOSTNAME="+container.Config.Hostname,
 	)
+
+	// Init any links between the parent and children
+	runtime := container.runtime
+
+	children, err := runtime.Children(fmt.Sprintf("/%s", container.ID))
+	if err != nil {
+		return err
+	}
+
+	if len(children) > 0 {
+		container.activeLinks = make(map[string]*Link, len(children))
+
+		// If we encounter an error make sure that we rollback any network
+		// config and ip table changes
+		rollback := func() {
+			for _, link := range container.activeLinks {
+				link.Disable()
+			}
+			container.activeLinks = nil
+		}
+
+		for p, child := range children {
+			link, err := NewLink(container, child, p, runtime.networkManager.bridgeIface)
+			if err != nil {
+				rollback()
+				return err
+			}
+
+			container.activeLinks[link.Alias()] = link
+			if err := link.Enable(); err != nil {
+				rollback()
+				return err
+			}
+
+			for _, envVar := range link.ToEnv() {
+				params = append(params, "-e", envVar)
+			}
+		}
+	}
+
 	if container.Config.WorkingDir != "" {
 		workingDir := path.Clean(container.Config.WorkingDir)
 		utils.Debugf("[working dir] working dir is %s", workingDir)
@@ -801,7 +909,21 @@ func (container *Container) Start(hostConfig *HostConfig) (err error) {
 	params = append(params, "--", container.Path)
 	params = append(params, container.Args...)
 
-	container.cmd = exec.Command("lxc-start", params...)
+	if RootIsShared() {
+		// lxc-start really needs / to be private, or all kinds of stuff break
+		// What we really want is to clone into a new namespace and then
+		// mount / MS_REC|MS_PRIVATE, but since we can't really clone or fork
+		// without exec in go we have to do this horrible shell hack...
+		shellString :=
+			"mount --make-rprivate /; exec " +
+				utils.ShellQuoteArguments(params)
+
+		params = []string{
+			"unshare", "-m", "--", "/bin/sh", "-c", shellString,
+		}
+	}
+
+	container.cmd = exec.Command(params[0], params[1:]...)
 
 	// Setup logging of stdout and stderr to disk
 	if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil {
@@ -882,7 +1004,7 @@ func (container *Container) StderrPipe() (io.ReadCloser, error) {
 	return utils.NewBufReader(reader), nil
 }
 
-func (container *Container) allocateNetwork() error {
+func (container *Container) allocateNetwork(hostConfig *HostConfig) error {
 	if container.Config.NetworkDisabled {
 		return nil
 	}
@@ -909,36 +1031,59 @@ func (container *Container) allocateNetwork() error {
 		}
 	}
 
-	var portSpecs []string
-	if !container.State.Ghost {
-		portSpecs = container.Config.PortSpecs
-	} else {
-		for backend, frontend := range container.NetworkSettings.PortMapping["Tcp"] {
-			portSpecs = append(portSpecs, fmt.Sprintf("%s:%s/tcp", frontend, backend))
+	if container.Config.PortSpecs != nil {
+		utils.Debugf("Migrating port mappings for container: %s", strings.Join(container.Config.PortSpecs, ", "))
+		if err := migratePortMappings(container.Config); err != nil {
+			return err
 		}
-		for backend, frontend := range container.NetworkSettings.PortMapping["Udp"] {
-			portSpecs = append(portSpecs, fmt.Sprintf("%s:%s/udp", frontend, backend))
+		container.Config.PortSpecs = nil
+	}
+
+	portSpecs := make(map[Port]struct{})
+	bindings := make(map[Port][]PortBinding)
+
+	if !container.State.Ghost {
+		if container.Config.ExposedPorts != nil {
+			portSpecs = container.Config.ExposedPorts
+		}
+		if hostConfig.PortBindings != nil {
+			bindings = hostConfig.PortBindings
+		}
+	} else {
+		if container.NetworkSettings.Ports != nil {
+			for port, binding := range container.NetworkSettings.Ports {
+				portSpecs[port] = struct{}{}
+				bindings[port] = binding
+			}
 		}
 	}
 
-	container.NetworkSettings.PortMapping = make(map[string]PortMapping)
-	container.NetworkSettings.PortMapping["Tcp"] = make(PortMapping)
-	container.NetworkSettings.PortMapping["Udp"] = make(PortMapping)
-	for _, spec := range portSpecs {
-		nat, err := iface.AllocatePort(spec)
-		if err != nil {
-			iface.Release()
-			return err
+	container.NetworkSettings.PortMapping = nil
+
+	for port := range portSpecs {
+		binding := bindings[port]
+		for i := 0; i < len(binding); i++ {
+			b := binding[i]
+			nat, err := iface.AllocatePort(port, b)
+			if err != nil {
+				iface.Release()
+				return err
+			}
+			utils.Debugf("Allocate port: %s:%s->%s", nat.Binding.HostIp, port, nat.Binding.HostPort)
+			binding[i] = nat.Binding
 		}
-		proto := strings.Title(nat.Proto)
-		backend, frontend := strconv.Itoa(nat.Backend), strconv.Itoa(nat.Frontend)
-		container.NetworkSettings.PortMapping[proto][backend] = frontend
+		bindings[port] = binding
 	}
+	container.SaveHostConfig(hostConfig)
+
+	container.NetworkSettings.Ports = bindings
 	container.network = iface
+
 	container.NetworkSettings.Bridge = container.runtime.networkManager.bridgeIface
 	container.NetworkSettings.IPAddress = iface.IPNet.IP.String()
 	container.NetworkSettings.IPPrefixLen, _ = iface.IPNet.Mask.Size()
 	container.NetworkSettings.Gateway = iface.Gateway.String()
+
 	return nil
 }
 
@@ -951,7 +1096,7 @@ func (container *Container) releaseNetwork() {
 	container.NetworkSettings = &NetworkSettings{}
 }
 
-// FIXME: replace this with a control socket within docker-init
+// FIXME: replace this with a control socket within dockerinit
 func (container *Container) waitLxc() error {
 	for {
 		output, err := exec.Command("lxc-info", "-n", container.ID).CombinedOutput()
@@ -973,7 +1118,8 @@ func (container *Container) monitor(hostConfig *HostConfig) {
 	if container.cmd == nil {
 		utils.Debugf("monitor: waiting for container %s using waitLxc", container.ID)
 		if err := container.waitLxc(); err != nil {
-			utils.Errorf("monitor: while waiting for container %s, waitLxc had a problem: %s", container.ID, err)
+			// Discard the error as any signals or non 0 returns will generate an error
+			utils.Debugf("monitor: while waiting for container %s, waitLxc had a problem: %s", container.ShortID(), err)
 		}
 	} else {
 		utils.Debugf("monitor: waiting for container %s using cmd.Wait", container.ID)
@@ -1021,6 +1167,14 @@ func (container *Container) monitor(hostConfig *HostConfig) {
 
 func (container *Container) cleanup() {
 	container.releaseNetwork()
+
+	// Disable all active links
+	if container.activeLinks != nil {
+		for _, link := range container.activeLinks {
+			link.Disable()
+		}
+	}
+
 	if container.Config.OpenStdin {
 		if err := container.stdin.Close(); err != nil {
 			utils.Errorf("%s: Error close stdin: %s", container.ID, err)
@@ -1132,7 +1286,15 @@ func (container *Container) Resize(h, w int) error {
 }
 
 func (container *Container) ExportRw() (Archive, error) {
-	return Tar(container.rwPath(), Uncompressed)
+	if err := container.EnsureMounted(); err != nil {
+		return nil, err
+	}
+
+	image, err := container.GetImage()
+	if err != nil {
+		return nil, err
+	}
+	return image.ExportChanges(container.runtime, container.RootfsPath(), container.rwPath(), container.ID)
 }
 
 func (container *Container) RwChecksum() (string, error) {
@@ -1174,20 +1336,33 @@ func (container *Container) EnsureMounted() error {
 	return container.Mount()
 }
 
+func (container *Container) EnsureUnmounted() error {
+	if mounted, err := container.Mounted(); err != nil {
+		return err
+	} else if !mounted {
+		return nil
+	}
+	return container.Unmount()
+}
+
 func (container *Container) Mount() error {
 	image, err := container.GetImage()
 	if err != nil {
 		return err
 	}
-	return image.Mount(container.RootfsPath(), container.rwPath())
+	return image.Mount(container.runtime, container.RootfsPath(), container.rwPath(), container.ID)
 }
 
 func (container *Container) Changes() ([]Change, error) {
+	if err := container.EnsureMounted(); err != nil {
+		return nil, err
+	}
+
 	image, err := container.GetImage()
 	if err != nil {
 		return nil, err
 	}
-	return image.Changes(container.rwPath())
+	return image.Changes(container.runtime, container.RootfsPath(), container.rwPath(), container.ID)
 }
 
 func (container *Container) GetImage() (*Image, error) {
@@ -1198,11 +1373,20 @@ func (container *Container) GetImage() (*Image, error) {
 }
 
 func (container *Container) Mounted() (bool, error) {
-	return Mounted(container.RootfsPath())
+	image, err := container.GetImage()
+	if err != nil {
+		return false, err
+	}
+	return image.Mounted(container.runtime, container.RootfsPath(), container.rwPath())
 }
 
 func (container *Container) Unmount() error {
-	return Unmount(container.RootfsPath())
+	image, err := container.GetImage()
+	if err != nil {
+		return err
+	}
+	err = image.Unmount(container.runtime, container.RootfsPath(), container.ID)
+	return err
 }
 
 // ShortID returns a shorthand version of the container's id for convenience.
@@ -1290,5 +1474,11 @@ func (container *Container) Copy(resource string) (Archive, error) {
 		filter = []string{path.Base(basePath)}
 		basePath = path.Dir(basePath)
 	}
-	return TarFilter(basePath, Uncompressed, filter)
+	return TarFilter(basePath, Uncompressed, filter, true, nil)
+}
+
+// Returns true if the container exposes a certain port
+func (container *Container) Exposes(p Port) bool {
+	_, exists := container.Config.ExposedPorts[p]
+	return exists
 }

container_test.go

@@ -18,7 +18,7 @@ import (
 func TestIDFormat(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container1, err := runtime.Create(
+	container1, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"/bin/sh", "-c", "echo hello world"},
@@ -388,7 +388,7 @@ func TestRun(t *testing.T) {
 func TestOutput(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"echo", "-n", "foobar"},
@@ -407,11 +407,32 @@ func TestOutput(t *testing.T) {
 	}
 }
 
+func TestContainerNetwork(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	container, _, err := runtime.Create(
+		&Config{
+			Image: GetTestImage(runtime).ID,
+			Cmd:   []string{"ping", "-c", "1", "127.0.0.1"},
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container)
+	if err := container.Run(); err != nil {
+		t.Fatal(err)
+	}
+	if container.State.ExitCode != 0 {
+		t.Errorf("Unexpected ping 127.0.0.1 exit code %d (expected 0)", container.State.ExitCode)
+	}
+}
+
 func TestKillDifferentUser(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image:     GetTestImage(runtime).ID,
 		Cmd:       []string{"cat"},
 		OpenStdin: true,
@@ -471,7 +492,7 @@ func TestCreateVolume(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	c, err := runtime.Create(config)
+	c, _, err := runtime.Create(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -486,7 +507,7 @@ func TestCreateVolume(t *testing.T) {
 func TestKill(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"sleep", "2"},
 	},
@@ -530,7 +551,7 @@ func TestExitCode(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	trueContainer, err := runtime.Create(&Config{
+	trueContainer, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"/bin/true", ""},
 	})
@@ -545,7 +566,7 @@ func TestExitCode(t *testing.T) {
 		t.Errorf("Unexpected exit code %d (expected 0)", trueContainer.State.ExitCode)
 	}
 
-	falseContainer, err := runtime.Create(&Config{
+	falseContainer, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"/bin/false", ""},
 	})
@@ -564,7 +585,7 @@ func TestExitCode(t *testing.T) {
 func TestRestart(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"echo", "-n", "foobar"},
 	},
@@ -594,7 +615,7 @@ func TestRestart(t *testing.T) {
 func TestRestartStdin(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"cat"},
 
@@ -672,7 +693,7 @@ func TestUser(t *testing.T) {
 	defer nuke(runtime)
 
 	// Default user must be root
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 	},
@@ -690,7 +711,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a username
-	container, err = runtime.Create(&Config{
+	container, _, err = runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 
@@ -710,7 +731,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a UID
-	container, err = runtime.Create(&Config{
+	container, _, err = runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 
@@ -730,7 +751,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a different user by uid
-	container, err = runtime.Create(&Config{
+	container, _, err = runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 
@@ -752,7 +773,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Set a different user by username
-	container, err = runtime.Create(&Config{
+	container, _, err = runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 
@@ -772,7 +793,7 @@ func TestUser(t *testing.T) {
 	}
 
 	// Test an wrong username
-	container, err = runtime.Create(&Config{
+	container, _, err = runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"id"},
 
@@ -793,7 +814,7 @@ func TestMultipleContainers(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container1, err := runtime.Create(&Config{
+	container1, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"sleep", "2"},
 	},
@@ -803,7 +824,7 @@ func TestMultipleContainers(t *testing.T) {
 	}
 	defer runtime.Destroy(container1)
 
-	container2, err := runtime.Create(&Config{
+	container2, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"sleep", "2"},
 	},
@@ -847,7 +868,7 @@ func TestMultipleContainers(t *testing.T) {
 func TestStdin(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"cat"},
 
@@ -892,7 +913,7 @@ func TestStdin(t *testing.T) {
 func TestTty(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"cat"},
 
@@ -937,7 +958,7 @@ func TestTty(t *testing.T) {
 func TestEnv(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"env"},
 	},
@@ -986,7 +1007,7 @@ func TestEnv(t *testing.T) {
 func TestEntrypoint(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:      GetTestImage(runtime).ID,
 			Entrypoint: []string{"/bin/echo"},
@@ -1009,7 +1030,7 @@ func TestEntrypoint(t *testing.T) {
 func TestEntrypointNoCmd(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:      GetTestImage(runtime).ID,
 			Entrypoint: []string{"/bin/echo", "foobar"},
@@ -1060,7 +1081,7 @@ func TestLXCConfig(t *testing.T) {
 	cpuMin := 100
 	cpuMax := 10000
 	cpu := cpuMin + rand.Intn(cpuMax-cpuMin)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"/bin/true"},
 
@@ -1072,7 +1093,6 @@ func TestLXCConfig(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer runtime.Destroy(container)
 	container.generateLXCConfig(nil)
 	grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar")
 	grepFile(t, container.lxcConfigPath(),
@@ -1084,7 +1104,7 @@ func TestLXCConfig(t *testing.T) {
 func TestCustomLxcConfig(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"/bin/true"},
 
@@ -1115,7 +1135,7 @@ func BenchmarkRunSequencial(b *testing.B) {
 	runtime := mkRuntime(b)
 	defer nuke(runtime)
 	for i := 0; i < b.N; i++ {
-		container, err := runtime.Create(&Config{
+		container, _, err := runtime.Create(&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{"echo", "-n", "foo"},
 		},
@@ -1147,7 +1167,7 @@ func BenchmarkRunParallel(b *testing.B) {
 		complete := make(chan error)
 		tasks = append(tasks, complete)
 		go func(i int, complete chan error) {
-			container, err := runtime.Create(&Config{
+			container, _, err := runtime.Create(&Config{
 				Image: GetTestImage(runtime).ID,
 				Cmd:   []string{"echo", "-n", "foo"},
 			},
@@ -1189,7 +1209,7 @@ func BenchmarkRunParallel(b *testing.B) {
 }
 
 func tempDir(t *testing.T) string {
-	tmpDir, err := ioutil.TempDir("", "docker-test")
+	tmpDir, err := ioutil.TempDir("", "docker-test-container")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1297,7 +1317,7 @@ func TestBindMounts(t *testing.T) {
 func TestVolumesFromReadonlyMount(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(
+	container, _, err := runtime.Create(
 		&Config{
 			Image:   GetTestImage(runtime).ID,
 			Cmd:     []string{"/bin/echo", "-n", "foobar"},
@@ -1316,7 +1336,7 @@ func TestVolumesFromReadonlyMount(t *testing.T) {
 		t.Fail()
 	}
 
-	container2, err := runtime.Create(
+	container2, _, err := runtime.Create(
 		&Config{
 			Image:       GetTestImage(runtime).ID,
 			Cmd:         []string{"/bin/echo", "-n", "foobar"},
@@ -1352,7 +1372,7 @@ func TestRestartWithVolumes(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image:   GetTestImage(runtime).ID,
 		Cmd:     []string{"echo", "-n", "foobar"},
 		Volumes: map[string]struct{}{"/test": {}},
@@ -1395,7 +1415,7 @@ func TestVolumesFromWithVolumes(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image:   GetTestImage(runtime).ID,
 		Cmd:     []string{"sh", "-c", "echo -n bar > /test/foo"},
 		Volumes: map[string]struct{}{"/test": {}},
@@ -1422,7 +1442,7 @@ func TestVolumesFromWithVolumes(t *testing.T) {
 		t.Fail()
 	}
 
-	container2, err := runtime.Create(
+	container2, _, err := runtime.Create(
 		&Config{
 			Image:       GetTestImage(runtime).ID,
 			Cmd:         []string{"cat", "/test/foo"},
@@ -1463,7 +1483,7 @@ func TestOnlyLoopbackExistsWhenUsingDisableNetworkOption(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	c, err := runtime.Create(config)
+	c, _, err := runtime.Create(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1529,7 +1549,7 @@ func TestMultipleVolumesFrom(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image:   GetTestImage(runtime).ID,
 		Cmd:     []string{"sh", "-c", "echo -n bar > /test/foo"},
 		Volumes: map[string]struct{}{"/test": {}},
@@ -1556,7 +1576,7 @@ func TestMultipleVolumesFrom(t *testing.T) {
 		t.Fail()
 	}
 
-	container2, err := runtime.Create(
+	container2, _, err := runtime.Create(
 		&Config{
 			Image:   GetTestImage(runtime).ID,
 			Cmd:     []string{"sh", "-c", "echo -n bar > /other/foo"},
@@ -1577,7 +1597,7 @@ func TestMultipleVolumesFrom(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	container3, err := runtime.Create(
+	container3, _, err := runtime.Create(
 		&Config{
 			Image:       GetTestImage(runtime).ID,
 			Cmd:         []string{"/bin/echo", "-n", "foobar"},

contrib/mkimage-arch.sh

@@ -21,18 +21,20 @@ mkdir $ROOTFS
 #packages to ignore for space savings
 PKGIGNORE=linux,jfsutils,lvm2,cryptsetup,groff,man-db,man-pages,mdadm,pciutils,pcmciautils,reiserfsprogs,s-nail,xfsprogs
  
-expect -c "
+expect <<EOF
   set timeout 60
+  set send_slow {1 1}
   spawn pacstrap -c -d -G -i $ROOTFS base haveged --ignore $PKGIGNORE
   expect {
-    \"Install anyway\" { send "n\\r"; exp_continue }
-    \"(default=all)\" { send "\\r"; exp_continue }
-    \"Proceed with installation?\" { send "\\r"; exp_continue }
-    \"skip the above package\" {send "y\\r"; exp_continue }
-    \"checking\" { exp_continue }
-    \"loading\" { exp_continue }
-    \"installing\" { exp_continue }
-  }"
+    "Install anyway?" { send n\r; exp_continue }
+    "(default=all)" { send \r; exp_continue }
+    "Proceed with installation?" { send "\r"; exp_continue }
+    "skip the above package" {send "y\r"; exp_continue }
+    "checking" { exp_continue }
+    "loading" { exp_continue }
+    "installing" { exp_continue }
+  }
+EOF
 
 arch-chroot $ROOTFS /bin/sh -c "haveged -w 1024; pacman-key --init; pkill haveged; pacman -Rs --noconfirm haveged; pacman-key --populate archlinux"
 arch-chroot $ROOTFS /bin/sh -c "ln -s /usr/share/zoneinfo/UTC /etc/localtime"

devmapper/deviceset.go

@@ -0,0 +1,850 @@
+package devmapper
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var (
+	DefaultDataLoopbackSize     int64  = 100 * 1024 * 1024 * 1024
+	DefaultMetaDataLoopbackSize int64  = 2 * 1024 * 1024 * 1024
+	DefaultBaseFsSize           uint64 = 10 * 1024 * 1024 * 1024
+)
+
+type DevInfo struct {
+	Hash          string     `json:"-"`
+	DeviceId      int        `json:"device_id"`
+	Size          uint64     `json:"size"`
+	TransactionId uint64     `json:"transaction_id"`
+	Initialized   bool       `json:"initialized"`
+	devices       *DeviceSet `json:"-"`
+}
+
+type MetaData struct {
+	Devices map[string]*DevInfo `json:devices`
+}
+
+type DeviceSet struct {
+	MetaData
+	sync.Mutex
+	initialized      bool
+	root             string
+	devicePrefix     string
+	TransactionId    uint64
+	NewTransactionId uint64
+	nextFreeDevice   int
+	activeMounts     map[string]int
+}
+
+type DiskUsage struct {
+	Used  uint64
+	Total uint64
+}
+
+type Status struct {
+	PoolName         string
+	DataLoopback     string
+	MetadataLoopback string
+	Data             DiskUsage
+	Metadata         DiskUsage
+}
+
+func getDevName(name string) string {
+	return "/dev/mapper/" + name
+}
+
+func (info *DevInfo) Name() string {
+	hash := info.Hash
+	if hash == "" {
+		hash = "base"
+	}
+	return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
+}
+
+func (info *DevInfo) DevName() string {
+	return getDevName(info.Name())
+}
+
+func (devices *DeviceSet) loopbackDir() string {
+	return path.Join(devices.root, "devicemapper")
+}
+
+func (devices *DeviceSet) jsonFile() string {
+	return path.Join(devices.loopbackDir(), "json")
+}
+
+func (devices *DeviceSet) getPoolName() string {
+	return devices.devicePrefix + "-pool"
+}
+
+func (devices *DeviceSet) getPoolDevName() string {
+	return getDevName(devices.getPoolName())
+}
+
+func (devices *DeviceSet) hasImage(name string) bool {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	_, err := os.Stat(filename)
+	return err == nil
+}
+
+// ensureImage creates a sparse file of <size> bytes at the path
+// <root>/devicemapper/<name>.
+// If the file already exists, it does nothing.
+// Either way it returns the full path.
+func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
+		return "", err
+	}
+
+	if _, err := os.Stat(filename); err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		utils.Debugf("Creating loopback file %s for device-manage use", filename)
+		file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			return "", err
+		}
+
+		if err = file.Truncate(size); err != nil {
+			return "", err
+		}
+	}
+	return filename, nil
+}
+
+func (devices *DeviceSet) allocateDeviceId() int {
+	// TODO: Add smarter reuse of deleted devices
+	id := devices.nextFreeDevice
+	devices.nextFreeDevice = devices.nextFreeDevice + 1
+	return id
+}
+
+func (devices *DeviceSet) allocateTransactionId() uint64 {
+	devices.NewTransactionId = devices.NewTransactionId + 1
+	return devices.NewTransactionId
+}
+
+func (devices *DeviceSet) saveMetadata() error {
+	jsonData, err := json.Marshal(devices.MetaData)
+	if err != nil {
+		return fmt.Errorf("Error encoding metaadata to json: %s", err)
+	}
+	tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
+	if err != nil {
+		return fmt.Errorf("Error creating metadata file: %s", err)
+	}
+
+	n, err := tmpFile.Write(jsonData)
+	if err != nil {
+		return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
+	}
+	if n < len(jsonData) {
+		return io.ErrShortWrite
+	}
+	if err := tmpFile.Sync(); err != nil {
+		return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := os.Rename(tmpFile.Name(), devices.jsonFile()); err != nil {
+		return fmt.Errorf("Error committing metadata file", err)
+	}
+
+	if devices.NewTransactionId != devices.TransactionId {
+		if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
+			return fmt.Errorf("Error setting devmapper transition ID: %s", err)
+		}
+		devices.TransactionId = devices.NewTransactionId
+	}
+	return nil
+}
+
+func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
+	utils.Debugf("registerDevice(%v, %v)", id, hash)
+	info := &DevInfo{
+		Hash:          hash,
+		DeviceId:      id,
+		Size:          size,
+		TransactionId: devices.allocateTransactionId(),
+		Initialized:   false,
+		devices:       devices,
+	}
+
+	devices.Devices[hash] = info
+	if err := devices.saveMetadata(); err != nil {
+		// Try to remove unused device
+		delete(devices.Devices, hash)
+		return nil, err
+	}
+
+	return info, nil
+}
+
+func (devices *DeviceSet) activateDeviceIfNeeded(hash string) error {
+	utils.Debugf("activateDeviceIfNeeded(%v)", hash)
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
+		return nil
+	}
+
+	return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
+}
+
+func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
+	devname := info.DevName()
+
+	err := exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0,lazy_journal_init=0", devname).Run()
+	if err != nil {
+		err = exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0", devname).Run()
+	}
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) loadMetaData() error {
+	utils.Debugf("loadMetadata()")
+	defer utils.Debugf("loadMetadata END")
+	_, _, _, params, err := getStatus(devices.getPoolName())
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	devices.NewTransactionId = devices.TransactionId
+
+	jsonData, err := ioutil.ReadFile(devices.jsonFile())
+	if err != nil && !os.IsNotExist(err) {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	devices.MetaData.Devices = make(map[string]*DevInfo)
+	if jsonData != nil {
+		if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	for hash, d := range devices.Devices {
+		d.Hash = hash
+		d.devices = devices
+
+		if d.DeviceId >= devices.nextFreeDevice {
+			devices.nextFreeDevice = d.DeviceId + 1
+		}
+
+		// If the transaction id is larger than the actual one we lost the device due to some crash
+		if d.TransactionId > devices.TransactionId {
+			utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
+			delete(devices.Devices, hash)
+		}
+	}
+	return nil
+}
+
+func (devices *DeviceSet) setupBaseImage() error {
+	oldInfo := devices.Devices[""]
+	if oldInfo != nil && oldInfo.Initialized {
+		return nil
+	}
+
+	if oldInfo != nil && !oldInfo.Initialized {
+		utils.Debugf("Removing uninitialized base image")
+		if err := devices.removeDevice(""); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	utils.Debugf("Initializing base device-manager snapshot")
+
+	id := devices.allocateDeviceId()
+
+	// Create initial device
+	if err := createDevice(devices.getPoolDevName(), id); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize)
+	info, err := devices.registerDevice(id, "", DefaultBaseFsSize)
+	if err != nil {
+		_ = deleteDevice(devices.getPoolDevName(), id)
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Creating filesystem on base device-manager snapshot")
+
+	if err = devices.activateDeviceIfNeeded(""); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if err := devices.createFilesystem(info); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	info.Initialized = true
+	if err = devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func setCloseOnExec(name string) {
+	fileInfos, _ := ioutil.ReadDir("/proc/self/fd")
+	if fileInfos != nil {
+		for _, i := range fileInfos {
+			link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
+			if link == name {
+				fd, err := strconv.Atoi(i.Name())
+				if err == nil {
+					syscall.CloseOnExec(fd)
+				}
+			}
+		}
+	}
+}
+
+func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
+	if level >= 7 {
+		return // Ignore _LOG_DEBUG
+	}
+
+	utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
+}
+
+func major(device uint64) uint64 {
+	return (device >> 8) & 0xfff
+}
+
+func minor(device uint64) uint64 {
+	return (device & 0xff) | ((device >> 12) & 0xfff00)
+}
+
+func (devices *DeviceSet) initDevmapper() error {
+	logInit(devices)
+
+	// Make sure the sparse images exist in <root>/devicemapper/data and
+	// <root>/devicemapper/metadata
+
+	createdLoopback := !devices.hasImage("data") || !devices.hasImage("metadata")
+	data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (data): %s\n", err)
+		return err
+	}
+	metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (metadata): %s\n", err)
+		return err
+	}
+
+	// Set the device prefix from the device id and inode of the docker root dir
+
+	st, err := os.Stat(devices.root)
+	if err != nil {
+		return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
+	}
+	sysSt := st.Sys().(*syscall.Stat_t)
+	// "reg-" stands for "regular file".
+	// In the future we might use "dev-" for "device file", etc.
+	// docker-maj,min[-inode] stands for:
+	//	- Managed by docker
+	//	- The target of this device is at major <maj> and minor <min>
+	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
+	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
+	utils.Debugf("Generated prefix: %s", devices.devicePrefix)
+
+	// Check for the existence of the device <prefix>-pool
+	utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
+	info, err := getInfo(devices.getPoolName())
+	if info == nil {
+		utils.Debugf("Error device getInfo: %s", err)
+		return err
+	}
+
+	// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
+	// that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
+	// so we add this badhack to make sure it closes itself
+	setCloseOnExec("/dev/mapper/control")
+
+	// If the pool doesn't exist, create it
+	if info.Exists == 0 {
+		utils.Debugf("Pool doesn't exist. Creating it.")
+
+		dataFile, err := AttachLoopDevice(data)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer dataFile.Close()
+
+		metadataFile, err := AttachLoopDevice(metadata)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer metadataFile.Close()
+
+		if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// If we didn't just create the data or metadata image, we need to
+	// load the metadata from the existing file.
+	if !createdLoopback {
+		if err = devices.loadMetaData(); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// Setup the base image
+	if err := devices.setupBaseImage(); err != nil {
+		utils.Debugf("Error device setupBaseImage: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		utils.Debugf("Error init: %s\n", err)
+		return err
+	}
+
+	if devices.Devices[hash] != nil {
+		return fmt.Errorf("hash %s already exists", hash)
+	}
+
+	baseInfo := devices.Devices[baseHash]
+	if baseInfo == nil {
+		return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash)
+	}
+
+	deviceId := devices.allocateDeviceId()
+
+	if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
+		utils.Debugf("Error creating snap device: %s\n", err)
+		return err
+	}
+
+	if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
+		deleteDevice(devices.getPoolDevName(), deviceId)
+		utils.Debugf("Error registering device: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) removeDevice(hash string) error {
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("hash %s doesn't exists", hash)
+	}
+
+	devinfo, _ := getInfo(info.Name())
+	if devinfo != nil && devinfo.Exists != 0 {
+		if err := removeDevice(info.Name()); err != nil {
+			utils.Debugf("Error removing device: %s\n", err)
+			return err
+		}
+	}
+
+	if info.Initialized {
+		info.Initialized = false
+		if err := devices.saveMetadata(); err != nil {
+			utils.Debugf("Error saving meta data: %s\n", err)
+			return err
+		}
+	}
+
+	if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
+		utils.Debugf("Error deleting device: %s\n", err)
+		return err
+	}
+
+	devices.allocateTransactionId()
+	delete(devices.Devices, info.Hash)
+
+	if err := devices.saveMetadata(); err != nil {
+		devices.Devices[info.Hash] = info
+		utils.Debugf("Error saving meta data: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) RemoveDevice(hash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return devices.removeDevice(hash)
+}
+
+func (devices *DeviceSet) deactivateDevice(hash string) error {
+	utils.Debugf("[devmapper] deactivateDevice(%s)", hash)
+	defer utils.Debugf("[devmapper] deactivateDevice END")
+	var devname string
+	// FIXME: shouldn't we just register the pool into devices?
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	devinfo, err := getInfo(devname)
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	if devinfo.Exists != 0 {
+		if err := removeDevice(devname); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		if err := devices.waitRemove(hash); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// waitRemove blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is removed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitRemove(hash string) error {
+	utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash)
+	defer utils.Debugf("[deviceset %s] waitRemove END", devices.devicePrefix, hash)
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			// If there is an error we assume the device doesn't exist.
+			// The error might actually be something else, but we can't differentiate.
+			return nil
+		}
+		utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
+		if devinfo.Exists == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
+	}
+	return nil
+}
+
+// waitClose blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is closed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitClose(hash string) error {
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			return err
+		}
+		utils.Debugf("Waiting for unmount of %s: opencount=%d", devname, devinfo.OpenCount)
+		if devinfo.OpenCount == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to close", devname)
+	}
+	return nil
+}
+
+// byHash is a hack to allow looking up the deviceset's pool by the hash "pool".
+// FIXME: it seems probably cleaner to register the pool in devices.Devices,
+// but I am afraid of arcane implications deep in the devicemapper code,
+// so this will do.
+func (devices *DeviceSet) byHash(hash string) (devname string, err error) {
+	if hash == "pool" {
+		return devices.getPoolDevName(), nil
+	}
+	info := devices.Devices[hash]
+	if info == nil {
+		return "", fmt.Errorf("hash %s doesn't exists", hash)
+	}
+	return info.Name(), nil
+}
+
+func (devices *DeviceSet) Shutdown() error {
+	utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
+	defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
+	devices.Lock()
+	utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
+	defer devices.Unlock()
+
+	if !devices.initialized {
+		return nil
+	}
+
+	for path, count := range devices.activeMounts {
+		for i := count; i > 0; i-- {
+			if err := syscall.Unmount(path, 0); err != nil {
+				utils.Debugf("Shutdown unmounting %s, error: %s\n", path, err)
+			}
+		}
+		delete(devices.activeMounts, path)
+	}
+
+	for _, d := range devices.Devices {
+		if err := devices.waitClose(d.Hash); err != nil {
+			utils.Errorf("Warning: error waiting for device %s to unmount: %s\n", d.Hash, err)
+		}
+		if err := devices.deactivateDevice(d.Hash); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", d.Hash, err)
+		}
+	}
+
+	pool := devices.getPoolDevName()
+	if devinfo, err := getInfo(pool); err == nil && devinfo.Exists != 0 {
+		if err := devices.deactivateDevice("pool"); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", pool, err)
+		}
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		return fmt.Errorf("Error initializing devmapper: %s", err)
+	}
+
+	if err := devices.activateDeviceIfNeeded(hash); err != nil {
+		return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
+	}
+
+	info := devices.Devices[hash]
+
+	var flags uintptr = syscall.MS_MGC_VAL
+
+	if readOnly {
+		flags = flags | syscall.MS_RDONLY
+	}
+
+	err := syscall.Mount(info.DevName(), path, "ext4", flags, "discard")
+	if err != nil && err == syscall.EINVAL {
+		err = syscall.Mount(info.DevName(), path, "ext4", flags, "")
+	}
+	if err != nil {
+		return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
+	}
+
+	count := devices.activeMounts[path]
+	devices.activeMounts[path] = count + 1
+
+	return nil
+}
+
+func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error {
+	utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path)
+	defer utils.Debugf("[devmapper] UnmountDevice END")
+	devices.Lock()
+	defer devices.Unlock()
+
+	utils.Debugf("[devmapper] Unmount(%s)", path)
+	if err := syscall.Unmount(path, 0); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	utils.Debugf("[devmapper] Unmount done")
+	// Wait for the unmount to be effective,
+	// by watching the value of Info.OpenCount for the device
+	if err := devices.waitClose(hash); err != nil {
+		return err
+	}
+
+	if count := devices.activeMounts[path]; count > 1 {
+		devices.activeMounts[path] = count - 1
+	} else {
+		delete(devices.activeMounts, path)
+	}
+
+	if deactivate {
+		devices.deactivateDevice(hash)
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) HasDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		return false
+	}
+	return devices.Devices[hash] != nil
+}
+
+func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		return false
+	}
+
+	info := devices.Devices[hash]
+	return info != nil && info.Initialized
+}
+
+func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		return false
+	}
+
+	info := devices.Devices[hash]
+	if info == nil {
+		return false
+	}
+	devinfo, _ := getInfo(info.Name())
+	return devinfo != nil && devinfo.Exists != 0
+}
+
+func (devices *DeviceSet) SetInitialized(hash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.ensureInit(); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	info.Initialized = true
+	if err := devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) Status() *Status {
+	devices.Lock()
+	defer devices.Unlock()
+
+	status := &Status{}
+
+	if err := devices.ensureInit(); err != nil {
+		return status
+	}
+
+	status.PoolName = devices.getPoolName()
+	status.DataLoopback = path.Join(devices.loopbackDir(), "data")
+	status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
+
+	_, totalSizeInSectors, _, params, err := getStatus(devices.getPoolName())
+	if err == nil {
+		var transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64
+		if _, err := fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal); err == nil {
+			// Convert from blocks to bytes
+			blockSizeInSectors := totalSizeInSectors / dataTotal
+
+			status.Data.Used = dataUsed * blockSizeInSectors * 512
+			status.Data.Total = dataTotal * blockSizeInSectors * 512
+
+			// metadata blocks are always 4k
+			status.Metadata.Used = metadataUsed * 4096
+			status.Metadata.Total = metadataTotal * 4096
+		}
+	}
+
+	return status
+}
+
+func (devices *DeviceSet) ensureInit() error {
+	if !devices.initialized {
+		devices.initialized = true
+		if err := devices.initDevmapper(); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+	return nil
+}
+
+func NewDeviceSet(root string) *DeviceSet {
+	SetDevDir("/dev")
+
+	return &DeviceSet{
+		initialized:  false,
+		root:         root,
+		MetaData:     MetaData{Devices: make(map[string]*DevInfo)},
+		activeMounts: make(map[string]int),
+	}
+}

devmapper/devmapper.go

@@ -0,0 +1,709 @@
+package devmapper
+
+/*
+#cgo LDFLAGS: -L. -ldevmapper
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <libdevmapper.h>
+#include <linux/loop.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+
+#ifndef LOOP_CTL_GET_FREE
+#define LOOP_CTL_GET_FREE       0x4C82
+#endif
+
+// FIXME: this could easily be rewritten in go
+char*			attach_loop_device(const char *filename, int *loop_fd_out)
+{
+  struct loop_info64	loopinfo = {0};
+  struct stat		st;
+  char			buf[64];
+  int			i, loop_fd, fd, start_index;
+  char*			loopname;
+
+
+  *loop_fd_out = -1;
+
+  start_index = 0;
+  fd = open("/dev/loop-control", O_RDONLY);
+  if (fd >= 0) {
+    start_index = ioctl(fd, LOOP_CTL_GET_FREE);
+    close(fd);
+
+    if (start_index < 0)
+      start_index = 0;
+  }
+
+  fd = open(filename, O_RDWR);
+  if (fd < 0) {
+    perror("open");
+    return NULL;
+  }
+
+  loop_fd = -1;
+  for (i = start_index ; loop_fd < 0 ; i++ ) {
+    if (sprintf(buf, "/dev/loop%d", i) < 0) {
+	close(fd);
+	return NULL;
+    }
+
+    if (stat(buf, &st)) {
+      if (!S_ISBLK(st.st_mode)) {
+	 fprintf(stderr, "[error] Loopback device %s is not a block device.\n", buf);
+      } else if (errno == ENOENT) {
+	fprintf(stderr, "[error] There are no more loopback device available.\n");
+      } else {
+	fprintf(stderr, "[error] Unkown error trying to stat the loopback device %s (errno: %d).\n", buf, errno);
+      }
+      close(fd);
+      return NULL;
+    }
+
+    loop_fd = open(buf, O_RDWR);
+    if (loop_fd < 0 && errno == ENOENT) {
+      fprintf(stderr, "[error] The loopback device %s does not exists.\n", buf);
+      close(fd);
+      return NULL;
+    } else if (loop_fd < 0) {
+	fprintf(stderr, "[error] Unkown error openning the loopback device %s. (errno: %d)\n", buf, errno);
+	continue;
+    }
+
+    if (ioctl(loop_fd, LOOP_SET_FD, (void *)(size_t)fd) < 0) {
+      int errsv = errno;
+      close(loop_fd);
+      loop_fd = -1;
+      if (errsv != EBUSY) {
+        close(fd);
+        fprintf(stderr, "cannot set up loopback device %s: %s", buf, strerror(errsv));
+        return NULL;
+      }
+      continue;
+    }
+
+    close(fd);
+
+    strncpy((char*)loopinfo.lo_file_name, buf, LO_NAME_SIZE);
+    loopinfo.lo_offset = 0;
+    loopinfo.lo_flags = LO_FLAGS_AUTOCLEAR;
+
+    if (ioctl(loop_fd, LOOP_SET_STATUS64, &loopinfo) < 0) {
+      perror("ioctl LOOP_SET_STATUS64");
+      if (ioctl(loop_fd, LOOP_CLR_FD, 0) < 0) {
+        perror("ioctl LOOP_CLR_FD");
+      }
+      close(loop_fd);
+      fprintf (stderr, "cannot set up loopback device info");
+      return (NULL);
+    }
+
+    loopname = strdup(buf);
+    if (loopname == NULL) {
+      close(loop_fd);
+      return (NULL);
+    }
+
+    *loop_fd_out = loop_fd;
+    return (loopname);
+  }
+
+  return (NULL);
+}
+
+static int64_t	get_block_size(int fd)
+{
+  uint64_t	size;
+
+  if (ioctl(fd, BLKGETSIZE64, &size) == -1)
+    return -1;
+  return ((int64_t)size);
+}
+
+extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);
+
+static void
+log_cb(int level, const char *file, int line,
+       int dm_errno_or_class, const char *f, ...)
+{
+  char buffer[256];
+  va_list ap;
+
+  va_start(ap, f);
+  vsnprintf(buffer, 256, f, ap);
+  va_end(ap);
+
+  DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
+}
+
+static void
+log_with_errno_init ()
+{
+  dm_log_with_errno_init(log_cb);
+}
+
+*/
+import "C"
+
+import (
+	"errors"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"os"
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+type DevmapperLogger interface {
+	log(level int, file string, line int, dmError int, message string)
+}
+
+const (
+	DeviceCreate TaskType = iota
+	DeviceReload
+	DeviceRemove
+	DeviceRemoveAll
+	DeviceSuspend
+	DeviceResume
+	DeviceInfo
+	DeviceDeps
+	DeviceRename
+	DeviceVersion
+	DeviceStatus
+	DeviceTable
+	DeviceWaitevent
+	DeviceList
+	DeviceClear
+	DeviceMknodes
+	DeviceListVersions
+	DeviceTargetMsg
+	DeviceSetGeometry
+)
+
+const (
+	AddNodeOnResume AddNodeType = iota
+	AddNodeOnCreate
+)
+
+var (
+	ErrTaskRun              = errors.New("dm_task_run failed")
+	ErrTaskSetName          = errors.New("dm_task_set_name failed")
+	ErrTaskSetMessage       = errors.New("dm_task_set_message failed")
+	ErrTaskSetAddNode       = errors.New("dm_task_set_add_node failed")
+	ErrTaskSetRO            = errors.New("dm_task_set_ro failed")
+	ErrTaskAddTarget        = errors.New("dm_task_add_target failed")
+	ErrGetDriverVersion     = errors.New("dm_task_get_driver_version failed")
+	ErrAttachLoopbackDevice = errors.New("loopback mounting failed")
+	ErrGetBlockSize         = errors.New("Can't get block size")
+	ErrUdevWait             = errors.New("wait on udev cookie failed")
+	ErrSetDevDir            = errors.New("dm_set_dev_dir failed")
+	ErrGetLibraryVersion    = errors.New("dm_get_library_version failed")
+	ErrCreateRemoveTask     = errors.New("Can't create task of type DeviceRemove")
+	ErrRunRemoveDevice      = errors.New("running removeDevice failed")
+)
+
+type (
+	Task struct {
+		unmanaged *C.struct_dm_task
+	}
+	Info struct {
+		Exists        int
+		Suspended     int
+		LiveTable     int
+		InactiveTable int
+		OpenCount     int32
+		EventNr       uint32
+		Major         uint32
+		Minor         uint32
+		ReadOnly      int
+		TargetCount   int32
+	}
+	TaskType    int
+	AddNodeType int
+)
+
+func (t *Task) destroy() {
+	if t != nil {
+		C.dm_task_destroy(t.unmanaged)
+		runtime.SetFinalizer(t, nil)
+	}
+}
+
+func TaskCreate(tasktype TaskType) *Task {
+	c_task := C.dm_task_create(C.int(tasktype))
+	if c_task == nil {
+		return nil
+	}
+	task := &Task{unmanaged: c_task}
+	runtime.SetFinalizer(task, (*Task).destroy)
+	return task
+}
+
+func (t *Task) Run() error {
+	if res := C.dm_task_run(t.unmanaged); res != 1 {
+		return ErrTaskRun
+	}
+	return nil
+}
+
+func (t *Task) SetName(name string) error {
+	c_name := C.CString(name)
+	defer free(c_name)
+
+	if res := C.dm_task_set_name(t.unmanaged, c_name); res != 1 {
+		return ErrTaskSetName
+	}
+	return nil
+}
+
+func (t *Task) SetMessage(message string) error {
+	c_message := C.CString(message)
+	defer free(c_message)
+
+	if res := C.dm_task_set_message(t.unmanaged, c_message); res != 1 {
+		return ErrTaskSetMessage
+	}
+	return nil
+}
+
+func (t *Task) SetSector(sector uint64) error {
+	if res := C.dm_task_set_sector(t.unmanaged, C.uint64_t(sector)); res != 1 {
+		return ErrTaskSetAddNode
+	}
+	return nil
+}
+
+func (t *Task) SetCookie(cookie *uint32, flags uint16) error {
+	c_cookie := C.uint32_t(*cookie)
+	if res := C.dm_task_set_cookie(t.unmanaged, &c_cookie, C.uint16_t(flags)); res != 1 {
+		return ErrTaskSetAddNode
+	}
+	*cookie = uint32(c_cookie)
+	return nil
+}
+
+func (t *Task) SetAddNode(add_node AddNodeType) error {
+	if res := C.dm_task_set_add_node(t.unmanaged, C.dm_add_node_t(add_node)); res != 1 {
+		return ErrTaskSetAddNode
+	}
+	return nil
+}
+
+func (t *Task) SetRo() error {
+	if res := C.dm_task_set_ro(t.unmanaged); res != 1 {
+		return ErrTaskSetRO
+	}
+	return nil
+}
+
+func (t *Task) AddTarget(start uint64, size uint64, ttype string, params string) error {
+	c_ttype := C.CString(ttype)
+	defer free(c_ttype)
+
+	c_params := C.CString(params)
+	defer free(c_params)
+
+	if res := C.dm_task_add_target(t.unmanaged, C.uint64_t(start), C.uint64_t(size), c_ttype, c_params); res != 1 {
+		return ErrTaskAddTarget
+	}
+	return nil
+}
+
+func (t *Task) GetDriverVersion() (string, error) {
+	buffer := C.CString(string(make([]byte, 128)))
+	defer free(buffer)
+
+	if res := C.dm_task_get_driver_version(t.unmanaged, buffer, 128); res != 1 {
+		return "", ErrGetDriverVersion
+	}
+	return C.GoString(buffer), nil
+}
+
+func (t *Task) GetInfo() (*Info, error) {
+	c_info := C.struct_dm_info{}
+	if res := C.dm_task_get_info(t.unmanaged, &c_info); res != 1 {
+		return nil, ErrGetDriverVersion
+	}
+	return &Info{
+		Exists:        int(c_info.exists),
+		Suspended:     int(c_info.suspended),
+		LiveTable:     int(c_info.live_table),
+		InactiveTable: int(c_info.inactive_table),
+		OpenCount:     int32(c_info.open_count),
+		EventNr:       uint32(c_info.event_nr),
+		Major:         uint32(c_info.major),
+		Minor:         uint32(c_info.minor),
+		ReadOnly:      int(c_info.read_only),
+		TargetCount:   int32(c_info.target_count),
+	}, nil
+}
+
+func (t *Task) GetNextTarget(next uintptr) (uintptr, uint64, uint64, string, string) {
+	var (
+		c_start, c_length       C.uint64_t
+		c_target_type, c_params *C.char
+	)
+
+	nextp := C.dm_get_next_target(t.unmanaged, unsafe.Pointer(next), &c_start, &c_length, &c_target_type, &c_params)
+	return uintptr(nextp), uint64(c_start), uint64(c_length), C.GoString(c_target_type), C.GoString(c_params)
+}
+
+func AttachLoopDevice(filename string) (*os.File, error) {
+	c_filename := C.CString(filename)
+	defer free(c_filename)
+
+	var fd C.int
+	res := C.attach_loop_device(c_filename, &fd)
+	if res == nil {
+		if os.Getenv("DEBUG") != "" {
+			C.perror(C.CString(fmt.Sprintf("[debug] Error attach_loop_device(%s, %d)", filename, int(fd))))
+		}
+		return nil, ErrAttachLoopbackDevice
+	}
+	defer free(res)
+
+	return os.NewFile(uintptr(fd), C.GoString(res)), nil
+}
+
+func getBlockSize(fd uintptr) int {
+	var size uint64
+
+	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, C.BLKGETSIZE64, uintptr(unsafe.Pointer(&size))); err != 0 {
+		utils.Debugf("Error ioctl: %s", err)
+		return -1
+	}
+	return int(size)
+}
+
+func GetBlockDeviceSize(file *os.File) (uint64, error) {
+	if size := C.get_block_size(C.int(file.Fd())); size == -1 {
+		return 0, ErrGetBlockSize
+	} else {
+		return uint64(size), nil
+	}
+}
+
+func UdevWait(cookie uint32) error {
+	if res := C.dm_udev_wait(C.uint32_t(cookie)); res != 1 {
+		utils.Debugf("Failed to wait on udev cookie %d", cookie)
+		return ErrUdevWait
+	}
+	return nil
+}
+
+func LogInitVerbose(level int) {
+	C.dm_log_init_verbose(C.int(level))
+}
+
+var dmLogger DevmapperLogger = nil
+
+func logInit(logger DevmapperLogger) {
+	dmLogger = logger
+	C.log_with_errno_init()
+}
+
+func SetDevDir(dir string) error {
+	c_dir := C.CString(dir)
+	defer free(c_dir)
+
+	if res := C.dm_set_dev_dir(c_dir); res != 1 {
+		utils.Debugf("Error dm_set_dev_dir")
+		return ErrSetDevDir
+	}
+	return nil
+}
+
+func GetLibraryVersion() (string, error) {
+	buffer := C.CString(string(make([]byte, 128)))
+	defer free(buffer)
+
+	if res := C.dm_get_library_version(buffer, 128); res != 1 {
+		return "", ErrGetLibraryVersion
+	}
+	return C.GoString(buffer), nil
+}
+
+// Useful helper for cleanup
+func RemoveDevice(name string) error {
+	task := TaskCreate(DeviceRemove)
+	if task == nil {
+		return ErrCreateRemoveTask
+	}
+	if err := task.SetName(name); err != nil {
+		utils.Debugf("Can't set task name %s", name)
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return ErrRunRemoveDevice
+	}
+	return nil
+}
+
+func free(p *C.char) {
+	C.free(unsafe.Pointer(p))
+}
+
+// This is the programmatic example of "dmsetup create"
+func createPool(poolName string, dataFile *os.File, metadataFile *os.File) error {
+	task, err := createTask(DeviceCreate, poolName)
+	if task == nil {
+		return err
+	}
+
+	size, err := GetBlockDeviceSize(dataFile)
+	if err != nil {
+		return fmt.Errorf("Can't get data size")
+	}
+
+	params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768"
+	if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+
+	var cookie uint32 = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createTask(t TaskType, name string) (*Task, error) {
+	task := TaskCreate(t)
+	if task == nil {
+		return nil, fmt.Errorf("Can't create task of type %d", int(t))
+	}
+	if err := task.SetName(name); err != nil {
+		return nil, fmt.Errorf("Can't set task name %s", name)
+	}
+	return task, nil
+}
+
+func getInfo(name string) (*Info, error) {
+	task, err := createTask(DeviceInfo, name)
+	if task == nil {
+		return nil, err
+	}
+	if err := task.Run(); err != nil {
+		return nil, err
+	}
+	return task.GetInfo()
+}
+
+func getStatus(name string) (uint64, uint64, string, string, error) {
+	task, err := createTask(DeviceStatus, name)
+	if task == nil {
+		utils.Debugf("getStatus: Error createTask: %s", err)
+		return 0, 0, "", "", err
+	}
+	if err := task.Run(); err != nil {
+		utils.Debugf("getStatus: Error Run: %s", err)
+		return 0, 0, "", "", err
+	}
+
+	devinfo, err := task.GetInfo()
+	if err != nil {
+		utils.Debugf("getStatus: Error GetInfo: %s", err)
+		return 0, 0, "", "", err
+	}
+	if devinfo.Exists == 0 {
+		utils.Debugf("getStatus: Non existing device %s", name)
+		return 0, 0, "", "", fmt.Errorf("Non existing device %s", name)
+	}
+
+	_, start, length, target_type, params := task.GetNextTarget(0)
+	return start, length, target_type, params, nil
+}
+
+func setTransactionId(poolName string, oldId uint64, newId uint64) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("set_transaction_id %d %d", oldId, newId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running setTransactionId")
+	}
+	return nil
+}
+
+func suspendDevice(name string) error {
+	task, err := createTask(DeviceSuspend, name)
+	if task == nil {
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+	return nil
+}
+
+func resumeDevice(name string) error {
+	task, err := createTask(DeviceResume, name)
+	if task == nil {
+		return err
+	}
+
+	var cookie uint32 = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createDevice(poolName string, deviceId int) error {
+	utils.Debugf("[devmapper] createDevice(poolName=%v, deviceId=%v)", poolName, deviceId)
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_thin %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running createDevice")
+	}
+	return nil
+}
+
+func deleteDevice(poolName string, deviceId int) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("delete %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running deleteDevice")
+	}
+	return nil
+}
+
+func removeDevice(name string) error {
+	utils.Debugf("[devmapper] removeDevice START")
+	defer utils.Debugf("[devmapper] removeDevice END")
+	task, err := createTask(DeviceRemove, name)
+	if task == nil {
+		return err
+	}
+	if err = task.Run(); err != nil {
+		return fmt.Errorf("Error running removeDevice")
+	}
+	return nil
+}
+
+func activateDevice(poolName string, name string, deviceId int, size uint64) error {
+	task, err := createTask(DeviceCreate, name)
+	if task == nil {
+		return err
+	}
+
+	params := fmt.Sprintf("%s %d", poolName, deviceId)
+	if err := task.AddTarget(0, size/512, "thin", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+	if err := task.SetAddNode(AddNodeOnCreate); err != nil {
+		return fmt.Errorf("Can't add node")
+	}
+
+	var cookie uint32 = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func (devices *DeviceSet) createSnapDevice(poolName string, deviceId int, baseName string, baseDeviceId int) error {
+	devinfo, _ := getInfo(baseName)
+	doSuspend := devinfo != nil && devinfo.Exists != 0
+
+	if doSuspend {
+		if err := suspendDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_snap %d %d", deviceId, baseDeviceId)); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Error running DeviceCreate")
+	}
+
+	if doSuspend {
+		if err := resumeDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

devmapper/devmapper_log.go

@@ -0,0 +1,13 @@
+package devmapper
+
+import "C"
+
+// Due to the way cgo works this has to be in a separate file, as devmapper.go has
+// definitions in the cgo block, which is incompatible with using "//export"
+
+//export DevmapperLogCallback
+func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_class C.int, message *C.char) {
+	if dmLogger != nil {
+		dmLogger.log(int(level), C.GoString(file), int(line), int(dm_errno_or_class), C.GoString(message))
+	}
+}

devmapper/docker-device-tool/device_tool.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/devmapper"
+	"os"
+)
+
+func usage() {
+	fmt.Printf("Usage: %s [snap new-id base-id] | [remove id] | [mount id mountpoint]\n", os.Args[0])
+	os.Exit(1)
+}
+
+func main() {
+	devices := devmapper.NewDeviceSet("/var/lib/docker")
+
+	if len(os.Args) < 2 {
+		usage()
+	}
+
+	cmd := os.Args[1]
+	if cmd == "snap" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.AddDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "remove" {
+		if len(os.Args) < 3 {
+			usage()
+		}
+
+		err := devices.RemoveDevice(os.Args[2])
+		if err != nil {
+			fmt.Println("Can't remove device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "mount" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.MountDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Printf("Unknown command %s\n", cmd)
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		os.Exit(1)
+	}
+
+	return
+}

docker/docker.go

@@ -4,9 +4,11 @@ import (
 	"flag"
 	"fmt"
 	"github.com/dotcloud/docker"
+	"github.com/dotcloud/docker/sysinit"
 	"github.com/dotcloud/docker/utils"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
 	"os/signal"
 	"strconv"
@@ -22,7 +24,7 @@ var (
 func main() {
 	if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" {
 		// Running in init mode
-		docker.SysInit()
+		sysinit.SysInit()
 		return
 	}
 	// FIXME: Switch d and D ? (to be more sshd like)
@@ -35,9 +37,14 @@ func main() {
 	flGraphPath := flag.String("g", "/var/lib/docker", "Path to graph storage base dir.")
 	flEnableCors := flag.Bool("api-enable-cors", false, "Enable CORS requests in the remote api.")
 	flDns := flag.String("dns", "", "Set custom dns servers")
-	flHosts := docker.ListOpts{fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)}
+	flHosts := utils.ListOpts{fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)}
 	flag.Var(&flHosts, "H", "tcp://host:port to bind/connect to or unix://path/to/socket to use")
+	flEnableIptables := flag.Bool("iptables", true, "Disable iptables within docker")
+	flDefaultIp := flag.String("ip", "0.0.0.0", "Default ip address to use when binding a containers ports")
+	flInterContainerComm := flag.Bool("enable-container-comm", false, "Enable inter-container communication")
+
 	flag.Parse()
+
 	if *flVersion {
 		showVersion()
 		return
@@ -49,10 +56,9 @@ func main() {
 		flHosts[i] = utils.ParseHost(docker.DEFAULTHTTPHOST, docker.DEFAULTHTTPPORT, flHost)
 	}
 
+	bridge := docker.DefaultNetworkBridge
 	if *bridgeName != "" {
-		docker.NetworkBridgeIface = *bridgeName
-	} else {
-		docker.NetworkBridgeIface = docker.DefaultNetworkBridge
+		bridge = *bridgeName
 	}
 	if *flDebug {
 		os.Setenv("DEBUG", "1")
@@ -64,7 +70,26 @@ func main() {
 			flag.Usage()
 			return
 		}
-		if err := daemon(*pidfile, *flGraphPath, flHosts, *flAutoRestart, *flEnableCors, *flDns); err != nil {
+		var dns []string
+		if *flDns != "" {
+			dns = []string{*flDns}
+		}
+
+		ip := net.ParseIP(*flDefaultIp)
+
+		config := &docker.DaemonConfig{
+			Pidfile:                     *pidfile,
+			GraphPath:                   *flGraphPath,
+			AutoRestart:                 *flAutoRestart,
+			EnableCors:                  *flEnableCors,
+			Dns:                         dns,
+			EnableIptables:              *flEnableIptables,
+			BridgeIface:                 bridge,
+			ProtoAddresses:              flHosts,
+			DefaultIp:                   ip,
+			InterContainerCommunication: *flInterContainerComm,
+		}
+		if err := daemon(config); err != nil {
 			log.Fatal(err)
 			os.Exit(-1)
 		}
@@ -115,30 +140,26 @@ func removePidFile(pidfile string) {
 	}
 }
 
-func daemon(pidfile string, flGraphPath string, protoAddrs []string, autoRestart, enableCors bool, flDns string) error {
-	if err := createPidFile(pidfile); err != nil {
+func daemon(config *docker.DaemonConfig) error {
+	if err := createPidFile(config.Pidfile); err != nil {
 		log.Fatal(err)
 	}
-	defer removePidFile(pidfile)
+	defer removePidFile(config.Pidfile)
 
 	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM))
+	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)
 	go func() {
 		sig := <-c
 		log.Printf("Received signal '%v', exiting\n", sig)
-		removePidFile(pidfile)
+		removePidFile(config.Pidfile)
 		os.Exit(0)
 	}()
-	var dns []string
-	if flDns != "" {
-		dns = []string{flDns}
-	}
-	server, err := docker.NewServer(flGraphPath, autoRestart, enableCors, dns)
+	server, err := docker.NewServer(config)
 	if err != nil {
 		return err
 	}
-	chErrors := make(chan error, len(protoAddrs))
-	for _, protoAddr := range protoAddrs {
+	chErrors := make(chan error, len(config.ProtoAddresses))
+	for _, protoAddr := range config.ProtoAddresses {
 		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
 		if protoAddrParts[0] == "unix" {
 			syscall.Unlink(protoAddrParts[1])
@@ -154,7 +175,7 @@ func daemon(pidfile string, flGraphPath string, protoAddrs []string, autoRestart
 			chErrors <- docker.ListenAndServe(protoAddrParts[0], protoAddrParts[1], server, true)
 		}()
 	}
-	for i := 0; i < len(protoAddrs); i += 1 {
+	for i := 0; i < len(config.ProtoAddresses); i += 1 {
 		err := <-chErrors
 		if err != nil {
 			return err

dockerinit/dockerinit.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"github.com/dotcloud/docker/sysinit"
+)
+
+var (
+	GITCOMMIT string
+	VERSION   string
+)
+
+func main() {
+	// Running in init mode
+	sysinit.SysInit()
+	return
+}

docs/sources/api/index.rst

@@ -14,5 +14,5 @@ Your programs and scripts can access Docker's functionality via these interfaces
   registry_api
   index_api
   docker_remote_api
-
+  remote_api_client_libraries
 

docs/sources/commandline/cli.rst

@@ -4,11 +4,8 @@
 
 .. _cli:
 
-Overview
-======================
-
-Docker Usage
-~~~~~~~~~~~~~~~~~~
+Command Line Help
+-----------------
 
 To list available commands, either run ``docker`` with no parameters or execute
 ``docker help``::
@@ -21,71 +18,749 @@ To list available commands, either run ``docker`` with no parameters or execute
 
     ...
 
+.. _cli_attach:
+
+``attach``
+----------
+
+::
+
+    Usage: docker attach CONTAINER
+
+    Attach to a running container.
+
+You can detach from the container again (and leave it running) with
+``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of
+the Docker client when it quits.
+
+To stop a container, use ``docker stop``
+
+To kill the container, use ``docker kill``
+
+.. _cli_attach_examples:
+ 
+Examples:
+~~~~~~~~~
+
+.. code-block:: bash
+
+     $ ID=$(sudo docker run -d ubuntu /usr/bin/top -b)
+     $ sudo docker attach $ID
+     top - 02:05:52 up  3:05,  0 users,  load average: 0.01, 0.02, 0.05
+     Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
+     Cpu(s):  0.1%us,  0.2%sy,  0.0%ni, 99.7%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
+     Mem:    373572k total,   355560k used,    18012k free,    27872k buffers
+     Swap:   786428k total,        0k used,   786428k free,   221740k cached
+
+     PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
+      1 root      20   0 17200 1116  912 R    0  0.3   0:00.03 top                
+
+      top - 02:05:55 up  3:05,  0 users,  load average: 0.01, 0.02, 0.05
+      Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
+      Cpu(s):  0.0%us,  0.2%sy,  0.0%ni, 99.8%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
+      Mem:    373572k total,   355244k used,    18328k free,    27872k buffers
+      Swap:   786428k total,        0k used,   786428k free,   221776k cached
+
+        PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
+	    1 root      20   0 17208 1144  932 R    0  0.3   0:00.03 top                
 
 
-Available Commands
-~~~~~~~~~~~~~~~~~~
+      top - 02:05:58 up  3:06,  0 users,  load average: 0.01, 0.02, 0.05
+      Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
+      Cpu(s):  0.2%us,  0.3%sy,  0.0%ni, 99.5%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
+      Mem:    373572k total,   355780k used,    17792k free,    27880k buffers
+      Swap:   786428k total,        0k used,   786428k free,   221776k cached
 
-.. include:: command/attach.rst
+      PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
+           1 root      20   0 17208 1144  932 R    0  0.3   0:00.03 top                
+     ^C$ 
+     $ sudo docker stop $ID
 
-.. include:: command/build.rst
+.. _cli_build:
 
-.. include:: command/commit.rst
+``build``
+---------
 
-.. include:: command/cp.rst
+::
 
-.. include:: command/diff.rst
+    Usage: docker build [OPTIONS] PATH | URL | -
+    Build a new container image from the source code at PATH
+      -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success.
+      -q=false: Suppress verbose build output.
+      -no-cache: Do not use the cache when building the image.
+      -rm: Remove intermediate containers after a successful build
+    When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context
 
-.. include:: command/events.rst
+.. _cli_build_examples:
 
-.. include:: command/export.rst
+Examples:
+~~~~~~~~~
 
-.. include:: command/history.rst
+.. code-block:: bash
 
-.. include:: command/images.rst
+    sudo docker build .
 
-.. include:: command/import.rst
+This will read the ``Dockerfile`` from the current directory. It will
+also send any other files and directories found in the current
+directory to the ``docker`` daemon.
 
-.. include:: command/info.rst
+The contents of this directory would be used by ``ADD`` commands found
+within the ``Dockerfile``.  This will send a lot of data to the
+``docker`` daemon if the current directory contains a lot of data.  If
+the absolute path is provided instead of ``.`` then only the files and
+directories required by the ADD commands from the ``Dockerfile`` will be
+added to the context and transferred to the ``docker`` daemon.
 
-.. include:: command/insert.rst
+.. code-block:: bash
 
-.. include:: command/inspect.rst
+   sudo docker build -t vieux/apache:2.0 .
+
+This will build like the previous example, but it will then tag the
+resulting image. The repository name will be ``vieux/apache`` and the
+tag will be ``2.0``
+
+
+.. code-block:: bash
+
+    sudo docker build - < Dockerfile
+
+This will read a ``Dockerfile`` from *stdin* without context. Due to
+the lack of a context, no contents of any local directory will be sent
+to the ``docker`` daemon.  ``ADD`` doesn't work when running in this
+mode because the absence of the context provides no source files to
+copy to the container.
+
+
+.. code-block:: bash
+
+    sudo docker build github.com/creack/docker-firefox
+
+This will clone the Github repository and use it as context. The
+``Dockerfile`` at the root of the repository is used as
+``Dockerfile``.  Note that you can specify an arbitrary git repository
+by using the ``git://`` schema.
+
+
+.. _cli_commit:
+
+``commit``
+----------
+
+::
+
+    Usage: docker commit [OPTIONS] CONTAINER [REPOSITORY [TAG]]
+
+    Create a new image from a container's changes
+
+      -m="": Commit message
+      -author="": Author (eg. "John Hannibal Smith <hannibal@a-team.com>"
+      -run="": Configuration to be applied when the image is launched with `docker run`. 
+               (ex: '{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')
+
+Full -run example (multiline is ok within a single quote ``'``)
+
+::
+
+  $ sudo docker commit -run='
+  {
+      "Entrypoint" : null,
+      "Privileged" : false,
+      "User" : "",
+      "VolumesFrom" : "",
+      "Cmd" : ["cat", "-e", "/etc/resolv.conf"],
+      "Dns" : ["8.8.8.8", "8.8.4.4"],
+      "MemorySwap" : 0,
+      "AttachStdin" : false,
+      "AttachStderr" : false,
+      "CpuShares" : 0,
+      "OpenStdin" : false,
+      "Volumes" : null,
+      "Hostname" : "122612f45831",
+      "PortSpecs" : ["22", "80", "443"],
+      "Image" : "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc",
+      "Tty" : false,
+      "Env" : [
+         "HOME=/",
+         "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      ],
+      "StdinOnce" : false,
+      "Domainname" : "",
+      "WorkingDir" : "/",
+      "NetworkDisabled" : false,
+      "Memory" : 0,
+      "AttachStdout" : false
+  }' $CONTAINER_ID
+
+.. _cli_cp:
+
+``cp``
+------
+
+::
+
+    Usage: docker cp CONTAINER:RESOURCE HOSTPATH
+
+    Copy files/folders from the containers filesystem to the host
+    path.  Paths are relative to the root of the filesystem.
+
+.. _cli_diff:
+
+``diff``
+--------
+
+::
+
+    Usage: docker diff CONTAINER [OPTIONS]
+
+    Inspect changes on a container's filesystem
+
+.. _cli_events:
+
+``events``
+----------
+
+::
+
+    Usage: docker events
+
+    Get real time events from the server
+
+.. _cli_events_example:
+
+Examples
+~~~~~~~~
+
+You'll need two shells for this example.
+
+Shell 1: Listening for events
+.............................
+
+.. code-block:: bash
+    
+    $ sudo docker events
+
+Shell 2: Start and Stop a Container
+...................................
+
+.. code-block:: bash
+
+    $ sudo docker start 4386fb97867d
+    $ sudo docker stop 4386fb97867d
+
+Shell 1: (Again .. now showing events)
+......................................
+
+.. code-block:: bash
+
+    [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
+
+
+.. _cli_export:
+
+``export``
+----------
+
+::
+
+    Usage: docker export CONTAINER
+
+    Export the contents of a filesystem as a tar archive
+
+.. _cli_history:
+
+``history``
+-----------
+
+::
+
+    Usage: docker history [OPTIONS] IMAGE
+
+    Show the history of an image
+
+.. _cli_images:
+
+``images``
+----------
+
+::
+
+    Usage: docker images [OPTIONS] [NAME]
+
+    List images
+
+      -a=false: show all images
+      -q=false: only show numeric IDs
+      -viz=false: output in graphviz format
+
+Displaying images visually
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    sudo docker images -viz | dot -Tpng -o docker.png
+
+.. image:: docker_images.gif
+   :alt: Example inheritance graph of Docker images.
+
+.. _cli_import:
+
+``import``
+----------
+
+::
+
+    Usage: docker import URL|- [REPOSITORY [TAG]]
+
+    Create a new filesystem image from the contents of a tarball
+
+At this time, the URL must start with ``http`` and point to a single
+file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a
+root filesystem. If you would like to import from a local directory or
+archive, you can use the ``-`` parameter to take the data from
+standard in.
+
+Examples
+~~~~~~~~
+
+Import from a remote location
+.............................
+
+``$ sudo docker import http://example.com/exampleimage.tgz exampleimagerepo``
+
+Import from a local file
+........................
+
+Import to docker via pipe and standard in
+
+``$ cat exampleimage.tgz | sudo docker import - exampleimagelocal``
+
+Import from a local directory
+.............................
+
+``$ sudo tar -c . | docker import - exampleimagedir``
+
+Note the ``sudo`` in this example -- you must preserve the ownership
+of the files (especially root ownership) during the archiving with
+tar. If you are not root (or sudo) when you tar, then the ownerships
+might not get preserved.
+
+.. _cli_info:
+
+``info``
+--------
+
+::
+
+    Usage: docker info
+
+    Display system-wide information.
+
+.. _cli_insert:
+
+``insert``
+----------
+
+::
+
+    Usage: docker insert IMAGE URL PATH
+
+    Insert a file from URL in the IMAGE at PATH
+
+Examples
+~~~~~~~~
+
+Insert file from github
+.......................
+
+.. code-block:: bash
+
+    $ sudo docker insert 8283e18b24bc https://raw.github.com/metalivedev/django/master/postinstall /tmp/postinstall.sh
+
+.. _cli_inspect:
+
+``inspect``
+-----------
+
+::
+
+    Usage: docker inspect [OPTIONS] CONTAINER
+
+    Return low-level information on a container
+
+.. _cli_kill:
+
+``kill``
+--------
+
+::
+
+    Usage: docker kill [OPTIONS] CONTAINER [CONTAINER...]
+
+    Kill a running container
+
+.. _cli_link:
+
+``link``
+--------
+
+::
+
+    Usage: docker link CURRENT_NAME NEW_NAME
+
+    Link a container to a new name.
+
+
+Examples:
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ docker link /59669e088202c2ebe150b4346cb3301562d073b51261176a354a74e8f618bfbc /redis
+    $ docker ls
+    NAME                                                                      ID                                                                 IMAGE
+    /redis                                                                    59669e088202c2ebe150b4346cb3301562d073b51261176a354a74e8f618bfbc   crosbymichael/redis:latest
+    /59669e088202c2ebe150b4346cb3301562d073b51261176a354a74e8f618bfbc         59669e088202c2ebe150b4346cb3301562d073b51261176a354a74e8f618bfbc   crosbymichael/redis:latest
+
+
+This will create a new link for the existing name ``/59669e088202c2ebe150b4346cb3301562d073b51261176a354a74e8f618bfbc`` 
+with the new name ``/redis`` so that we can new reference the same container under the new name ``/redis``.
+
+.. _cli_login:
+
+``login``
+---------
+
+::
+
+    Usage: docker login [OPTIONS] [SERVER]
+
+    Register or Login to the docker registry server
+
+    -e="": email
+    -p="": password
+    -u="": username
+
+    If you want to login to a private registry you can
+    specify this by adding the server name.
+
+    example:
+    docker login localhost:8080
+
+
+.. _cli_logs:
+
+``logs``
+--------
+
+::
+
+    Usage: docker logs [OPTIONS] CONTAINER
+
+    Fetch the logs of a container
+
+
+.. _cli_port:
+
+``port``
+--------
+
+::
+
+    Usage: docker port [OPTIONS] CONTAINER PRIVATE_PORT
+
+    Lookup the public-facing port which is NAT-ed to PRIVATE_PORT
+
+
+.. _cli_ps:
+
+``ps``
+------
+
+::
+
+    Usage: docker ps [OPTIONS]
+
+    List containers
+
+      -a=false: Show all containers. Only running containers are shown by default.
+      -notrunc=false: Don't truncate output
+      -q=false: Only display numeric IDs
+
+.. _cli_pull:
+
+``pull``
+--------
+
+::
+
+    Usage: docker pull NAME
+
+    Pull an image or a repository from the registry
+
+
+.. _cli_push:
+
+``push``
+--------
+
+::
+
+    Usage: docker push NAME
+
+    Push an image or a repository to the registry
+
+
+.. _cli_restart:
+
+``restart``
+-----------
+
+::
+
+    Usage: docker restart [OPTIONS] NAME
+
+    Restart a running container
+
+.. _cli_rm:
+
+``rm``
+------
+
+::
+
+    Usage: docker rm [OPTIONS] CONTAINER
+
+    Remove one or more containers
+        -link="": Remove the link instead of the actual container
+ 
+
+Examples:
+~~~~~~~~~
+
+.. code-block:: bash
+
+    $ docker rm /redis
+    /redis
+
+
+This will remove the container referenced under the link ``/redis``.
+
+
+.. code-block:: bash
+
+    $ docker rm -link /webapp/redis
+    /webapp/redis
+
+
+This will remove the underlying link between ``/webapp`` and the ``/redis`` containers removing all
+network communication.
+
+.. _cli_rmi:
+
+``rmi``
+-------
+
+::
+
+    Usage: docker rmi IMAGE [IMAGE...]
+
+    Remove one or more images
+
+.. _cli_run:
+
+``run``
+-------
+
+::
+
+    Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...]
+
+    Run a command in a new container
+
+      -a=map[]: Attach to stdin, stdout or stderr
+      -c=0: CPU shares (relative weight)
+      -cidfile="": Write the container ID to the file
+      -d=false: Detached mode: Run container in the background, print new container id
+      -e=[]: Set environment variables
+      -h="": Container host name
+      -i=false: Keep stdin open even if not attached
+      -privileged=false: Give extended privileges to this container
+      -m=0: Memory limit (in bytes)
+      -n=true: Enable networking for this container
+      -p=[]: Map a network port to the container
+      -rm=false: Automatically remove the container when it exits (incompatible with -d)
+      -t=false: Allocate a pseudo-tty
+      -u="": Username or UID
+      -dns=[]: Set custom dns servers for the container
+      -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume.
+      -volumes-from="": Mount all volumes from the given container
+      -entrypoint="": Overwrite the default entrypoint set by the image
+      -w="": Working directory inside the container
+      -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
+      -expose=[]: Expose a port from the container without publishing it to your host
+      -link="": Add link to another container (containerid:alias)
+
+Examples
+--------
+
+.. code-block:: bash
+
+    sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test"
+
+This will create a container and print "test" to the console. The
+``cidfile`` flag makes docker attempt to create a new file and write the
+container ID to it. If the file exists already, docker will return an
+error. Docker will close this file when docker run exits.
+
+.. code-block:: bash
+
+   docker run mount -t tmpfs none /var/spool/squid
+
+This will *not* work, because by default, most potentially dangerous
+kernel capabilities are dropped; including ``cap_sys_admin`` (which is
+required to mount filesystems). However, the ``-privileged`` flag will
+allow it to run:
+
+.. code-block:: bash
+
+   docker run -privileged mount -t tmpfs none /var/spool/squid
+
+The ``-privileged`` flag gives *all* capabilities to the container,
+and it also lifts all the limitations enforced by the ``device``
+cgroup controller. In other words, the container can then do almost
+everything that the host can do. This flag exists to allow special
+use-cases, like running Docker within Docker.
+
+.. code-block:: bash
+
+   docker  run -w /path/to/dir/ -i -t  ubuntu pwd
+
+The ``-w`` lets the command being executed inside directory given, 
+here /path/to/dir/. If the path does not exists it is created inside the 
+container.
+
+.. code-block:: bash
+
+   docker  run  -v `pwd`:`pwd` -w `pwd` -i -t  ubuntu pwd
+
+The ``-v`` flag mounts the current working directory into the container. 
+The ``-w`` lets the command being executed inside the current 
+working directory, by changing into the directory to the value
+returned by ``pwd``. So this combination executes the command
+using the container, but inside the current working directory.
+
+.. code-block:: bash
+
+    docker run -p 127.0.0.0::80 ubuntu bash
+
+This the ``-p`` flag now allows you to bind a port to a specific
+interface of the host machine.  In this example port ``80`` of the 
+container will have a dynamically allocated port bound to 127.0.0.1 
+of the host.
+
+.. code-block:: bash
+
+    docker run -p 127.0.0.1:80:80 ubuntu bash
+
+This will bind port ``80`` of the container to port ``80`` on 127.0.0.1 of your
+host machine.
+
+.. code-block:: bash
+
+    docker run -expose 80 ubuntu bash
+
+This will expose port ``80`` of the container for use within a link
+without publishing the port to the host system's interfaces.  
+
+.. code-block:: bash
+
+    docker run -link /redis:redis ubuntu bash
+
+The ``-link`` flag will link the container named ``/redis`` into the 
+newly created container with the alias ``redis``.  The new container
+can access the network and environment of the redis container via
+environment variables.
+
+.. _cli_search:
+
+``search``
+----------
+
+::
+
+    Usage: docker search TERM
+
+    Searches for the TERM parameter on the Docker index and prints out
+    a list of repositories that match.
+
+.. _cli_start:
+
+``start``
+---------
+
+::
+
+    Usage: docker start [OPTIONS] NAME
+
+    Start a stopped container
+
+.. _cli_stop:
+
+``stop``
+--------
+
+::
+
+    Usage: docker stop [OPTIONS] CONTAINER [CONTAINER...]
+
+    Stop a running container
+
+      -t=10: Number of seconds to wait for the container to stop before killing it.
+
+.. _cli_tag:
+
+``tag``
+-------
+
+::
+
+    Usage: docker tag [OPTIONS] IMAGE REPOSITORY [TAG]
+
+    Tag an image into a repository
+
+      -f=false: Force
+
+.. _cli_top:
+
+``top``
+-------
+
+::
+
+    Usage: docker top CONTAINER
+
+    Lookup the running processes of a container
+
+.. _cli_version:
+
+``version``
+-----------
+
+Show the version of the docker client, daemon, and latest released version.
+
+
+.. _cli_wait:
+
+``wait``
+--------
+
+::
+
+    Usage: docker wait [OPTIONS] NAME
+
+    Block until a container stops, then print its exit code.
 
-.. include:: command/kill.rst
-
-.. include:: command/login.rst
-
-.. include:: command/logs.rst
-
-.. include:: command/port.rst
-
-.. include:: command/ps.rst
-
-.. include:: command/pull.rst
-
-.. include:: command/push.rst
-
-.. include:: command/restart.rst
-
-.. include:: command/rm.rst
-
-.. include:: command/rmi.rst
-
-.. include:: command/run.rst
-
-.. include:: command/search.rst
-
-.. include:: command/start.rst
-
-.. include:: command/stop.rst
-
-.. include:: command/tag.rst
-
-.. include:: command/top.rst
-
-.. include:: command/version.rst
-
-.. include:: command/wait.rst
 
 

docs/sources/commandline/command/attach.rst

@@ -1,59 +0,0 @@
-:title: Attach Command
-:description: Attach to a running container
-:keywords: attach, container, docker, documentation
-
-===========================================
-``attach`` -- Attach to a running container
-===========================================
-
-::
-
-    Usage: docker attach CONTAINER
-
-    Attach to a running container.
-
-You can detach from the container again (and leave it running) with
-``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of
-the Docker client when it quits.
-
-To stop a container, use ``docker stop``
-
-To kill the container, use ``docker kill``
- 
-Examples:
----------
-
-.. code-block:: bash
-
-     $ ID=$(sudo docker run -d ubuntu /usr/bin/top -b)
-     $ sudo docker attach $ID
-     top - 02:05:52 up  3:05,  0 users,  load average: 0.01, 0.02, 0.05
-     Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
-     Cpu(s):  0.1%us,  0.2%sy,  0.0%ni, 99.7%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
-     Mem:    373572k total,   355560k used,    18012k free,    27872k buffers
-     Swap:   786428k total,        0k used,   786428k free,   221740k cached
-
-     PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
-      1 root      20   0 17200 1116  912 R    0  0.3   0:00.03 top                
-
-      top - 02:05:55 up  3:05,  0 users,  load average: 0.01, 0.02, 0.05
-      Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
-      Cpu(s):  0.0%us,  0.2%sy,  0.0%ni, 99.8%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
-      Mem:    373572k total,   355244k used,    18328k free,    27872k buffers
-      Swap:   786428k total,        0k used,   786428k free,   221776k cached
-
-        PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
-	    1 root      20   0 17208 1144  932 R    0  0.3   0:00.03 top                
-
-
-      top - 02:05:58 up  3:06,  0 users,  load average: 0.01, 0.02, 0.05
-      Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
-      Cpu(s):  0.2%us,  0.3%sy,  0.0%ni, 99.5%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
-      Mem:    373572k total,   355780k used,    17792k free,    27880k buffers
-      Swap:   786428k total,        0k used,   786428k free,   221776k cached
-
-      PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND            
-           1 root      20   0 17208 1144  932 R    0  0.3   0:00.03 top                
-     ^C$ 
-     $ sudo docker stop $ID
-

docs/sources/commandline/command/build.rst

@@ -1,65 +0,0 @@
-:title: Build Command
-:description: Build a new image from the Dockerfile passed via stdin
-:keywords: build, docker, container, documentation
-
-================================================
-``build`` -- Build a container from a Dockerfile
-================================================
-
-::
-
-    Usage: docker build [OPTIONS] PATH | URL | -
-    Build a new container image from the source code at PATH
-      -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success.
-      -q=false: Suppress verbose build output.
-      -no-cache: Do not use the cache when building the image.
-      -rm: Remove intermediate containers after a successful build
-    When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context
-
-
-Examples
---------
-
-.. code-block:: bash
-
-    sudo docker build .
-
-This will read the ``Dockerfile`` from the current directory. It will
-also send any other files and directories found in the current
-directory to the ``docker`` daemon.
-
-The contents of this directory would be used by ``ADD`` commands found
-within the ``Dockerfile``.  This will send a lot of data to the
-``docker`` daemon if the current directory contains a lot of data.  If
-the absolute path is provided instead of ``.`` then only the files and
-directories required by the ADD commands from the ``Dockerfile`` will be
-added to the context and transferred to the ``docker`` daemon.
-
-.. code-block:: bash
-
-   sudo docker build -t vieux/apache:2.0 .
-
-This will build like the previous example, but it will then tag the
-resulting image. The repository name will be ``vieux/apache`` and the
-tag will be ``2.0``
-
-
-.. code-block:: bash
-
-    sudo docker build - < Dockerfile
-
-This will read a ``Dockerfile`` from *stdin* without context. Due to
-the lack of a context, no contents of any local directory will be sent
-to the ``docker`` daemon.  ``ADD`` doesn't work when running in this
-mode because the absence of the context provides no source files to
-copy to the container.
-
-
-.. code-block:: bash
-
-    sudo docker build github.com/creack/docker-firefox
-
-This will clone the Github repository and use it as context. The
-``Dockerfile`` at the root of the repository is used as
-``Dockerfile``.  Note that you can specify an arbitrary git repository
-by using the ``git://`` schema.

docs/sources/commandline/command/commit.rst

@@ -1,52 +0,0 @@
-:title: Commit Command
-:description: Create a new image from a container's changes
-:keywords: commit, docker, container, documentation
-
-===========================================================
-``commit`` -- Create a new image from a container's changes
-===========================================================
-
-::
-
-    Usage: docker commit [OPTIONS] CONTAINER [REPOSITORY [TAG]]
-
-    Create a new image from a container's changes
-
-      -m="": Commit message
-      -author="": Author (eg. "John Hannibal Smith <hannibal@a-team.com>"
-      -run="": Configuration to be applied when the image is launched with `docker run`. 
-               (ex: '{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')
-
-Full -run example (multiline is ok within a single quote ``'``)
-
-::
-
-  $ sudo docker commit -run='
-  {
-      "Entrypoint" : null,
-      "Privileged" : false,
-      "User" : "",
-      "VolumesFrom" : "",
-      "Cmd" : ["cat", "-e", "/etc/resolv.conf"],
-      "Dns" : ["8.8.8.8", "8.8.4.4"],
-      "MemorySwap" : 0,
-      "AttachStdin" : false,
-      "AttachStderr" : false,
-      "CpuShares" : 0,
-      "OpenStdin" : false,
-      "Volumes" : null,
-      "Hostname" : "122612f45831",
-      "PortSpecs" : ["22", "80", "443"],
-      "Image" : "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc",
-      "Tty" : false,
-      "Env" : [
-         "HOME=/",
-         "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-      ],
-      "StdinOnce" : false,
-      "Domainname" : "",
-      "WorkingDir" : "/",
-      "NetworkDisabled" : false,
-      "Memory" : 0,
-      "AttachStdout" : false
-  }' $CONTAINER_ID

docs/sources/commandline/command/cp.rst

@@ -1,14 +0,0 @@
-:title: Cp Command
-:description: Copy files/folders from the containers filesystem to the host path
-:keywords: cp, docker, container, documentation, copy
-
-============================================================================
-``cp`` -- Copy files/folders from the containers filesystem to the host path
-============================================================================
-
-::
-
-    Usage: docker cp CONTAINER:RESOURCE HOSTPATH
-
-    Copy files/folders from the containers filesystem to the host
-    path.  Paths are relative to the root of the filesystem.

docs/sources/commandline/command/diff.rst

@@ -1,13 +0,0 @@
-:title: Diff Command
-:description: Inspect changes on a container's filesystem
-:keywords: diff, docker, container, documentation
-
-=======================================================
-``diff`` -- Inspect changes on a container's filesystem
-=======================================================
-
-::
-
-    Usage: docker diff CONTAINER [OPTIONS]
-
-    Inspect changes on a container's filesystem

docs/sources/commandline/command/events.rst

@@ -1,34 +0,0 @@
-:title: Events Command
-:description: Get real time events from the server
-:keywords: events, docker, documentation
-
-=================================================================
-``events`` -- Get real time events from the server
-=================================================================
-
-::
-
-    Usage: docker events
-
-    Get real time events from the server
-
-Examples
---------
-
-Starting and stopping a container
-.................................
-
-.. code-block:: bash
-
-    $ sudo docker start 4386fb97867d
-    $ sudo docker stop 4386fb97867d
-
-In another shell
-
-.. code-block:: bash
-    
-    $ sudo docker events
-    [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start
-    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
-    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
-

docs/sources/commandline/command/export.rst

@@ -1,13 +0,0 @@
-:title: Export Command
-:description: Export the contents of a filesystem as a tar archive
-:keywords: export, docker, container, documentation
-
-=================================================================
-``export`` -- Stream the contents of a container as a tar archive
-=================================================================
-
-::
-
-    Usage: docker export CONTAINER
-
-    Export the contents of a filesystem as a tar archive

docs/sources/commandline/command/history.rst

@@ -1,13 +0,0 @@
-:title: History Command
-:description: Show the history of an image
-:keywords: history, docker, container, documentation
-
-===========================================
-``history`` -- Show the history of an image
-===========================================
-
-::
-
-    Usage: docker history [OPTIONS] IMAGE
-
-    Show the history of an image

docs/sources/commandline/command/images.rst

@@ -1,26 +0,0 @@
-:title: Images Command
-:description: List images
-:keywords: images, docker, container, documentation
-
-=========================
-``images`` -- List images
-=========================
-
-::
-
-    Usage: docker images [OPTIONS] [NAME]
-
-    List images
-
-      -a=false: show all images
-      -q=false: only show numeric IDs
-      -viz=false: output in graphviz format
-
-Displaying images visually
---------------------------
-
-::
-
-    sudo docker images -viz | dot -Tpng -o docker.png
-
-.. image:: https://docs.docker.io/en/latest/_static/docker_images.gif

docs/sources/commandline/command/import.rst

@@ -1,44 +0,0 @@
-:title: Import Command
-:description: Create a new filesystem image from the contents of a tarball
-:keywords: import, tarball, docker, url, documentation
-
-==========================================================================
-``import`` -- Create a new filesystem image from the contents of a tarball
-==========================================================================
-
-::
-
-    Usage: docker import URL|- [REPOSITORY [TAG]]
-
-    Create a new filesystem image from the contents of a tarball
-
-At this time, the URL must start with ``http`` and point to a single
-file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a
-root filesystem. If you would like to import from a local directory or
-archive, you can use the ``-`` parameter to take the data from
-standard in.
-
-Examples
---------
-
-Import from a remote location
-.............................
-
-``$ sudo docker import http://example.com/exampleimage.tgz exampleimagerepo``
-
-Import from a local file
-........................
-
-Import to docker via pipe and standard in
-
-``$ cat exampleimage.tgz | sudo docker import - exampleimagelocal``
-
-Import from a local directory
-.............................
-
-``$ sudo tar -c . | docker import - exampleimagedir``
-
-Note the ``sudo`` in this example -- you must preserve the ownership
-of the files (especially root ownership) during the archiving with
-tar. If you are not root (or sudo) when you tar, then the ownerships
-might not get preserved.

docs/sources/commandline/command/info.rst

@@ -1,13 +0,0 @@
-:title: Info Command
-:description: Display system-wide information.
-:keywords: info, docker, information, documentation
-
-===========================================
-``info`` -- Display system-wide information
-===========================================
-
-::
-
-    Usage: docker info
-
-    Display system-wide information.

docs/sources/commandline/command/insert.rst

@@ -1,23 +0,0 @@
-:title: Insert Command
-:description: Insert a file in an image
-:keywords: insert, image, docker, documentation
-
-==========================================================================
-``insert`` -- Insert a file in an image
-==========================================================================
-
-::
-
-    Usage: docker insert IMAGE URL PATH
-
-    Insert a file from URL in the IMAGE at PATH
-
-Examples
---------
-
-Insert file from github
-.......................
-
-.. code-block:: bash
-
-    $ sudo docker insert 8283e18b24bc https://raw.github.com/metalivedev/django/master/postinstall /tmp/postinstall.sh

docs/sources/commandline/command/inspect.rst

@@ -1,13 +0,0 @@
-:title: Inspect Command
-:description: Return low-level information on a container
-:keywords: inspect, container, docker, documentation
-
-==========================================================
-``inspect`` -- Return low-level information on a container
-==========================================================
-
-::
-
-    Usage: docker inspect [OPTIONS] CONTAINER
-
-    Return low-level information on a container

docs/sources/commandline/command/kill.rst

@@ -1,13 +0,0 @@
-:title: Kill Command
-:description: Kill a running container
-:keywords: kill, container, docker, documentation
-
-====================================
-``kill`` -- Kill a running container
-====================================
-
-::
-
-    Usage: docker kill [OPTIONS] CONTAINER [CONTAINER...]
-
-    Kill a running container

docs/sources/commandline/command/login.rst

@@ -1,24 +0,0 @@
-:title: Login Command
-:description: Register or Login to the docker registry server
-:keywords: login, docker, documentation
-
-============================================================
-``login`` -- Register or Login to the docker registry server
-============================================================
-
-::
-
-    Usage: docker login [OPTIONS] [SERVER]
-
-    Register or Login to the docker registry server
-
-    -e="": email
-    -p="": password
-    -u="": username
-
-    If you want to login to a private registry you can
-    specify this by adding the server name.
-
-    example:
-    docker login localhost:8080
-

docs/sources/commandline/command/logs.rst

@@ -1,13 +0,0 @@
-:title: Logs Command
-:description: Fetch the logs of a container
-:keywords: logs, container, docker, documentation
-
-=========================================
-``logs`` -- Fetch the logs of a container
-=========================================
-
-::
-
-    Usage: docker logs [OPTIONS] CONTAINER
-
-    Fetch the logs of a container

docs/sources/commandline/command/port.rst

@@ -1,13 +0,0 @@
-:title: Port Command
-:description: Lookup the public-facing port which is NAT-ed to PRIVATE_PORT
-:keywords: port, docker, container, documentation
-
-=========================================================================
-``port`` -- Lookup the public-facing port which is NAT-ed to PRIVATE_PORT
-=========================================================================
-
-::
-
-    Usage: docker port [OPTIONS] CONTAINER PRIVATE_PORT
-
-    Lookup the public-facing port which is NAT-ed to PRIVATE_PORT

docs/sources/commandline/command/ps.rst

@@ -1,17 +0,0 @@
-:title: Ps Command
-:description: List containers
-:keywords: ps, docker, documentation, container
-
-=========================
-``ps`` -- List containers
-=========================
-
-::
-
-    Usage: docker ps [OPTIONS]
-
-    List containers
-
-      -a=false: Show all containers. Only running containers are shown by default.
-      -notrunc=false: Don't truncate output
-      -q=false: Only display numeric IDs

docs/sources/commandline/command/pull.rst

@@ -1,13 +0,0 @@
-:title: Pull Command
-:description: Pull an image or a repository from the registry
-:keywords: pull, image, repo, repository, documentation, docker
-
-=========================================================================
-``pull`` -- Pull an image or a repository from the docker registry server
-=========================================================================
-
-::
-
-    Usage: docker pull NAME
-
-    Pull an image or a repository from the registry

docs/sources/commandline/command/push.rst

@@ -1,13 +0,0 @@
-:title: Push Command
-:description: Push an image or a repository to the registry
-:keywords: push, docker, image, repository, documentation, repo
-
-=======================================================================
-``push`` -- Push an image or a repository to the docker registry server
-=======================================================================
-
-::
-
-    Usage: docker push NAME
-
-    Push an image or a repository to the registry

docs/sources/commandline/command/restart.rst

@@ -1,13 +0,0 @@
-:title: Restart Command
-:description: Restart a running container
-:keywords: restart, container, docker, documentation
-
-==========================================
-``restart`` -- Restart a running container
-==========================================
-
-::
-
-    Usage: docker restart [OPTIONS] NAME
-
-    Restart a running container

docs/sources/commandline/command/rm.rst

@@ -1,13 +0,0 @@
-:title: Rm Command
-:description: Remove a container
-:keywords: remove, container, docker, documentation, rm
-
-============================
-``rm`` -- Remove a container
-============================
-
-::
-
-    Usage: docker rm [OPTIONS] CONTAINER
-
-    Remove one or more containers

docs/sources/commandline/command/rmi.rst

@@ -1,13 +0,0 @@
-:title: Rmi Command
-:description: Remove an image
-:keywords: rmi, remove, image, docker, documentation
-
-==========================
-``rmi`` -- Remove an image
-==========================
-
-::
-
-    Usage: docker rmi IMAGE [IMAGE...]
-
-    Remove one or more images

docs/sources/commandline/command/run.rst

@@ -1,85 +0,0 @@
-:title: Run Command
-:description: Run a command in a new container
-:keywords: run, container, docker, documentation 
-
-===========================================
-``run`` -- Run a command in a new container
-===========================================
-
-::
-
-    Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...]
-
-    Run a command in a new container
-
-      -a=map[]: Attach to stdin, stdout or stderr.
-      -c=0: CPU shares (relative weight)
-      -cidfile="": Write the container ID to the file
-      -d=false: Detached mode: Run container in the background, print new container id
-      -e=[]: Set environment variables
-      -h="": Container host name
-      -i=false: Keep stdin open even if not attached
-      -privileged=false: Give extended privileges to this container
-      -m=0: Memory limit (in bytes)
-      -n=true: Enable networking for this container
-      -p=[]: Map a network port to the container
-      -rm=false: Automatically remove the container when it exits (incompatible with -d)
-      -t=false: Allocate a pseudo-tty
-      -u="": Username or UID
-      -dns=[]: Set custom dns servers for the container
-      -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume.
-      -volumes-from="": Mount all volumes from the given container.
-      -entrypoint="": Overwrite the default entrypoint set by the image.
-      -w="": Working directory inside the container
-      -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
-
-Examples
---------
-
-.. code-block:: bash
-
-    sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test"
-
-This will create a container and print "test" to the console. The
-``cidfile`` flag makes docker attempt to create a new file and write the
-container ID to it. If the file exists already, docker will return an
-error. Docker will close this file when docker run exits.
-
-.. code-block:: bash
-
-   docker run mount -t tmpfs none /var/spool/squid
-
-This will *not* work, because by default, most potentially dangerous
-kernel capabilities are dropped; including ``cap_sys_admin`` (which is
-required to mount filesystems). However, the ``-privileged`` flag will
-allow it to run:
-
-.. code-block:: bash
-
-   docker run -privileged mount -t tmpfs none /var/spool/squid
-
-The ``-privileged`` flag gives *all* capabilities to the container,
-and it also lifts all the limitations enforced by the ``device``
-cgroup controller. In other words, the container can then do almost
-everything that the host can do. This flag exists to allow special
-use-cases, like running Docker within Docker.
-
-.. code-block:: bash
-
-   docker  run -w /path/to/dir/ -i -t  ubuntu pwd
-
-The ``-w`` lets the command being executed inside directory given, 
-here /path/to/dir/. If the path does not exists it is created inside the 
-container.
-
-.. code-block:: bash
-
-   docker  run  -v `pwd`:`pwd` -w `pwd` -i -t  ubuntu pwd
-
-The ``-v`` flag mounts the current working directory into the container. 
-The ``-w`` lets the command being executed inside the current 
-working directory, by changing into the directory to the value
-returned by ``pwd``. So this combination executes the command
-using the container, but inside the current working directory.
-
-

docs/sources/commandline/command/search.rst

@@ -1,14 +0,0 @@
-:title: Search Command
-:description: Searches for the TERM parameter on the Docker index and prints out a list of repositories that match.
-:keywords: search, docker, image, documentation 
-
-===================================================================
-``search`` -- Search for an image in the docker index
-===================================================================
-
-::
-
-    Usage: docker search TERM
-
-    Searches for the TERM parameter on the Docker index and prints out
-    a list of repositories that match.

docs/sources/commandline/command/start.rst

@@ -1,13 +0,0 @@
-:title: Start Command
-:description: Start a stopped container
-:keywords: start, docker, container, documentation
-
-======================================
-``start`` -- Start a stopped container
-======================================
-
-::
-
-    Usage: docker start [OPTIONS] NAME
-
-    Start a stopped container

docs/sources/commandline/command/stop.rst

@@ -1,15 +0,0 @@
-:title: Stop Command
-:description: Stop a running container
-:keywords: stop, container, docker, documentation
-
-====================================
-``stop`` -- Stop a running container
-====================================
-
-::
-
-    Usage: docker stop [OPTIONS] CONTAINER [CONTAINER...]
-
-    Stop a running container
-
-      -t=10: Number of seconds to wait for the container to stop before killing it.

docs/sources/commandline/command/tag.rst

@@ -1,15 +0,0 @@
-:title: Tag Command
-:description: Tag an image into a repository
-:keywords: tag, docker, image, repository, documentation, repo
-
-=========================================
-``tag`` -- Tag an image into a repository
-=========================================
-
-::
-
-    Usage: docker tag [OPTIONS] IMAGE REPOSITORY [TAG]
-
-    Tag an image into a repository
-
-      -f=false: Force

docs/sources/commandline/command/top.rst

@@ -1,13 +0,0 @@
-:title: Top Command
-:description: Lookup the running processes of a container
-:keywords: top, docker, container, documentation
-
-=======================================================
-``top`` -- Lookup the running processes of a container
-=======================================================
-
-::
-
-    Usage: docker top CONTAINER
-
-    Lookup the running processes of a container

docs/sources/commandline/command/version.rst

@@ -1,7 +0,0 @@
-:title: Version Command
-:description: 
-:keywords: version, docker, documentation
-
-==================================================
-``version`` -- Show the docker version information
-==================================================

docs/sources/commandline/command/wait.rst

@@ -1,13 +0,0 @@
-:title: Wait Command
-:description: Block until a container stops, then print its exit code.
-:keywords: wait, docker, container, documentation
-
-===================================================================
-``wait`` -- Block until a container stops, then print its exit code
-===================================================================
-
-::
-
-    Usage: docker wait [OPTIONS] NAME
-
-    Block until a container stops, then print its exit code.

docs/sources/commandline/index.rst

@@ -1,6 +1,6 @@
 :title: Commands
-:description: -- todo: change me
-:keywords: todo, commands, command line, help, docker, documentation
+:description: docker command line interface
+:keywords: commands, command line, help, docker
 
 
 Commands
@@ -12,34 +12,3 @@ Contents:
   :maxdepth: 1
 
   cli
-  attach  <command/attach>
-  build   <command/build>
-  commit  <command/commit>
-  cp      <command/cp>
-  diff    <command/diff>
-  events  <command/events>
-  export  <command/export>
-  history <command/history>
-  images  <command/images>
-  import  <command/import>
-  info    <command/info>
-  insert  <command/insert>
-  inspect <command/inspect>
-  kill    <command/kill>
-  login   <command/login>
-  logs    <command/logs>
-  port    <command/port>
-  ps      <command/ps>
-  pull    <command/pull>
-  push    <command/push>
-  restart <command/restart>
-  rm      <command/rm>
-  rmi     <command/rmi>
-  run     <command/run>
-  search  <command/search>
-  start   <command/start>
-  stop    <command/stop>
-  tag     <command/tag>
-  top     <command/top>
-  version <command/version>
-  wait    <command/wait>

docs/sources/examples/hello_world.rst

@@ -178,4 +178,4 @@ you could skip to any of the other examples:
 * :ref:`running_ssh_service`
 * :ref:`running_couchdb_service`
 * :ref:`postgresql_service`
-* :ref:`mongodb`
+* :ref:`mongodb_image`

docs/sources/examples/index.rst

@@ -1,6 +1,6 @@
 :title: Docker Examples
 :description: Examples on how to use Docker
-:keywords: docker, hello world, node, nodejs, python, couch, couchdb, redis, ssh, sshd, examples, postgresql
+:keywords: docker, hello world, node, nodejs, python, couch, couchdb, redis, ssh, sshd, examples, postgresql, link
 
 
 .. _example_list:
@@ -24,3 +24,4 @@ to more substantial services like you might find in production.
    postgresql_service
    mongodb
    running_riak_service
+   linking_into_redis

docs/sources/examples/linking_into_redis.rst

@@ -0,0 +1,132 @@
+:title: Linking to an Redis container
+:description: Running redis linked into your web app
+:keywords: docker, example, networking, redis, link
+
+.. _linking_redis:
+
+Linking Redis
+=============
+
+.. include:: example_header.inc
+
+Building a redis container to link as a child of our web application.
+
+Building the redis container
+----------------------------
+
+We will use a pre-build version of redis from the index under 
+the name ``crosbymichael/redis``.  If you are interested in the 
+Dockerfile that was used to build this container here it is.
+
+.. code-block:: bash
+    
+    # Build redis from source
+    # Make sure you have the redis source code checked out in
+    # the same directory as this Dockerfile
+    FROM ubuntu
+
+    RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list
+    RUN apt-get update
+    RUN apt-get upgrade -y
+
+    RUN apt-get install -y gcc make g++ build-essential libc6-dev tcl
+
+    ADD . /redis
+
+    RUN (cd /redis && make)
+    RUN (cd /redis && make test)
+
+    RUN mkdir -p /redis-data
+    VOLUME ["/redis-data"]
+    EXPOSE 6379
+
+    ENTRYPOINT ["/redis/src/redis-server"]
+    CMD ["--dir", "/redis-data"]
+
+
+We need to ``EXPOSE`` the default port of 6379 so that our link knows what ports 
+to connect to our redis container on.  If you do not expose any ports for the
+image then docker will not be able to establish the link between containers.
+
+
+Run the redis container
+-----------------------
+
+.. code-block:: bash
+    
+    docker run -d -e PASSWORD=docker crosbymichael/redis --requirepass docker
+ 
+This will run our redis container using the default port of 6379 and using
+as password to secure our service. Next we will link the redis container to 
+a new name using ``docker link``.
+
+
+Linking an existing container
+-----------------------------
+
+Docker will automatically create an initial link with the container's id but
+because the is long and not very user friendly we can link the container with
+a new name.
+
+.. code-block:: bash
+
+    docker link /39588b6a45100ef5b328b2c302ea085624f29e6cbab70f88be04793af02cec89 /redis
+
+Now we can reference our running redis service using the friendly name ``/redis``.  
+We can issue all the commands that you would expect; start, stop, attach, using the new name.
+
+Linking redis as a child
+------------------------
+
+Next we can start a new web application that has a dependency on redis and apply a link 
+to connect both containers.  If you noticed when running our redis service we did not use
+the ``-p`` option to publish the redis port to the host system.  Redis exposed port 6379
+but we did not publish the port.  This allows docker to prevent all network traffic to
+the redis container except when explicitly specified within a link.  This is a big win
+for security.  
+
+
+Now lets start our web application with a link into redis.
+
+.. code-block:: bash
+   
+    docker run -t -i -link /redis:db ubuntu bash
+
+    root@4c01db0b339c:/# env
+
+    HOSTNAME=4c01db0b339c
+    DB_NAME=/4c01db0b339cf19958731255a796ee072040a652f51652a4ade190ab8c27006f/db
+    TERM=xterm
+    DB_PORT=tcp://172.17.0.8:6379
+    DB_PORT_6379_TCP=tcp://172.17.0.8:6379
+    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+    PWD=/
+    DB_ENV_PASSWORD=dockerpass
+    SHLVL=1
+    HOME=/
+    container=lxc
+    _=/usr/bin/env
+    root@4c01db0b339c:/#
+
+
+When we inspect the environment of the linked container we can see a few extra environment 
+variables have been added.  When you specified ``-link /redis:db`` you are telling docker
+to link the container named ``/redis`` into this new container with the alias ``db``.  
+Environment variables are prefixed with the alias so that the parent container can access
+network and environment information from the child.
+
+.. code-block:: bash
+
+    # The name of the child container
+    DB_NAME=/4c01db0b339cf19958731255a796ee072040a652f51652a4ade190ab8c27006f/db
+    # The default protocol, ip, and port of the service running in the container
+    DB_PORT=tcp://172.17.0.8:6379
+    # A specific protocol, ip, and port of various services
+    DB_PORT_6379_TCP=tcp://172.17.0.8:6379
+    # Get environment variables of the container 
+    DB_ENV_PASSWORD=dockerpass
+
+
+Accessing the network information along with the environment of the child container allows
+us to easily connect to the redis service on the specific ip and port and use the password
+specified in the environment.  

docs/sources/index.rst

@@ -2,8 +2,6 @@
 :description: An overview of the Docker Documentation
 :keywords: containers, lxc, concepts, explanation
 
-.. image:: https://www.docker.io/static/img/linked/dockerlogo-horizontal.png
-
 Introduction
 ------------
 

gograph/MAINTAINERS

@@ -0,0 +1 @@
+Michael Crosby <michael@crosbymichael.com> (@crosbymichael)

gograph/gograph.go

@@ -0,0 +1,462 @@
+package gograph
+
+import (
+	_ "code.google.com/p/gosqlite/sqlite3"
+	"database/sql"
+	"fmt"
+	"os"
+	"path"
+)
+
+const (
+	createEntityTable = `
+    CREATE TABLE IF NOT EXISTS entity (
+        id text NOT NULL PRIMARY KEY
+    );`
+
+	createEdgeTable = `
+    CREATE TABLE IF NOT EXISTS edge (
+        "entity_id" text NOT NULL,
+        "parent_id" text NULL,
+        "name" text NOT NULL,
+        CONSTRAINT "parent_fk" FOREIGN KEY ("parent_id") REFERENCES "entity" ("id"),
+        CONSTRAINT "entity_fk" FOREIGN KEY ("entity_id") REFERENCES "entity" ("id")
+        );
+    `
+
+	createEdgeIndices = `
+    CREATE UNIQUE INDEX "name_parent_ix" ON "edge" (parent_id, name);
+    `
+)
+
+// Entity with a unique id
+type Entity struct {
+	id string
+}
+
+// An Edge connects two entities together
+type Edge struct {
+	EntityID string
+	Name     string
+	ParentID string
+}
+
+type Entities map[string]*Entity
+type Edges []*Edge
+
+type WalkFunc func(fullPath string, entity *Entity) error
+
+// Graph database for storing entities and their relationships
+type Database struct {
+	dbPath string
+}
+
+// Create a new graph database initialized with a root entity
+func NewDatabase(dbPath string) (*Database, error) {
+	db := &Database{dbPath}
+	if _, err := os.Stat(dbPath); err == nil {
+		return db, nil
+	}
+	conn, err := db.openConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	if _, err := conn.Exec(createEntityTable); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Exec(createEdgeTable); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Exec(createEdgeIndices); err != nil {
+		return nil, err
+	}
+
+	rollback := func() {
+		conn.Exec("ROLLBACK")
+	}
+
+	// Create root entities
+	if _, err := conn.Exec("BEGIN"); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Exec("INSERT INTO entity (id) VALUES (?);", "0"); err != nil {
+		rollback()
+		return nil, err
+	}
+
+	if _, err := conn.Exec("INSERT INTO edge (entity_id, name) VALUES(?,?);", "0", "/"); err != nil {
+		rollback()
+		return nil, err
+	}
+
+	if _, err := conn.Exec("COMMIT"); err != nil {
+		return nil, err
+	}
+	return db, nil
+}
+
+// Set the entity id for a given path
+func (db *Database) Set(fullPath, id string) (*Entity, error) {
+	conn, err := db.openConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	// FIXME: is rollback implicit when closing the connection?
+	rollback := func() {
+		conn.Exec("ROLLBACK")
+	}
+	// FIXME: use exclusive transactions to avoid race conditions
+	if _, err := conn.Exec("BEGIN"); err != nil {
+		return nil, err
+	}
+	var entityId string
+	if err := conn.QueryRow("SELECT id FROM entity WHERE id = ?;", id).Scan(&entityId); err != nil {
+		if err == sql.ErrNoRows {
+			if _, err := conn.Exec("INSERT INTO entity (id) VALUES(?);", id); err != nil {
+				rollback()
+				return nil, err
+			}
+		} else {
+			rollback()
+			return nil, err
+		}
+	}
+	e := &Entity{id}
+
+	parentPath, name := splitPath(fullPath)
+	if err := db.setEdge(conn, parentPath, name, e); err != nil {
+		rollback()
+		return nil, err
+	}
+
+	if _, err := conn.Exec("COMMIT"); err != nil {
+		return nil, err
+	}
+	return e, nil
+}
+
+func (db *Database) setEdge(conn *sql.DB, parentPath, name string, e *Entity) error {
+	parent, err := db.get(conn, parentPath)
+	if err != nil {
+		return err
+	}
+	if parent.id == e.id {
+		return fmt.Errorf("Cannot set self as child")
+	}
+
+	if _, err := conn.Exec("INSERT INTO edge (parent_id, name, entity_id) VALUES (?,?,?);", parent.id, name, e.id); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Return the root "/" entity for the database
+func (db *Database) RootEntity() *Entity {
+	return &Entity{
+		id: "0",
+	}
+}
+
+// Return the entity for a given path
+func (db *Database) Get(name string) *Entity {
+	conn, err := db.openConn()
+	if err != nil {
+		return nil
+	}
+	e, err := db.get(conn, name)
+	if err != nil {
+		return nil
+	}
+	return e
+}
+
+func (db *Database) get(conn *sql.DB, name string) (*Entity, error) {
+	e := db.RootEntity()
+	// We always know the root name so return it if
+	// it is requested
+	if name == "/" {
+		return e, nil
+	}
+
+	parts := split(name)
+	for i := 1; i < len(parts); i++ {
+		p := parts[i]
+
+		next := db.child(conn, e, p)
+		if next == nil {
+			return nil, fmt.Errorf("Cannot find child")
+		}
+		e = next
+	}
+	return e, nil
+
+}
+
+// List all entities by from the name
+// The key will be the full path of the entity
+func (db *Database) List(name string, depth int) Entities {
+	out := Entities{}
+	conn, err := db.openConn()
+	if err != nil {
+		return out
+	}
+	defer conn.Close()
+
+	for c := range db.children(conn, name, depth) {
+		out[c.FullPath] = c.Entity
+	}
+	return out
+}
+
+func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error {
+	conn, err := db.openConn()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	for c := range db.children(conn, name, depth) {
+		if err := walkFunc(c.FullPath, c.Entity); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Return the refrence count for a specified id
+func (db *Database) Refs(id string) int {
+	conn, err := db.openConn()
+	if err != nil {
+		return -1
+	}
+	defer conn.Close()
+
+	var count int
+	if err := conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil {
+		return 0
+	}
+	return count
+}
+
+// Return all the id's path references
+func (db *Database) RefPaths(id string) Edges {
+	refs := Edges{}
+	conn, err := db.openConn()
+	if err != nil {
+		return refs
+	}
+	defer conn.Close()
+
+	rows, err := conn.Query("SELECT name, parent_id FROM edge WHERE entity_id = ?;", id)
+	if err != nil {
+		return refs
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var name string
+		var parentId string
+		if err := rows.Scan(&name, &parentId); err != nil {
+			return refs
+		}
+		refs = append(refs, &Edge{
+			EntityID: id,
+			Name:     name,
+			ParentID: parentId,
+		})
+	}
+	return refs
+}
+
+// Delete the reference to an entity at a given path
+func (db *Database) Delete(name string) error {
+	if name == "/" {
+		return fmt.Errorf("Cannot delete root entity")
+	}
+	conn, err := db.openConn()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	parentPath, n := splitPath(name)
+	parent, err := db.get(conn, parentPath)
+	if err != nil {
+		return err
+	}
+
+	if _, err := conn.Exec("DELETE FROM edge WHERE parent_id = ? AND name LIKE ?;", parent.id, n+"%"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Remove the entity with the specified id
+// Walk the graph to make sure all references to the entity
+// are removed and return the number of references removed
+func (db *Database) Purge(id string) (int, error) {
+	conn, err := db.openConn()
+	if err != nil {
+		return -1, err
+	}
+	defer conn.Close()
+
+	rollback := func() {
+		conn.Exec("ROLLBACK")
+	}
+
+	if _, err := conn.Exec("BEGIN"); err != nil {
+		return -1, err
+	}
+
+	// Delete all edges
+	rows, err := conn.Exec("DELETE FROM edge WHERE entity_id = ?;", id)
+	if err != nil {
+		rollback()
+		return -1, err
+	}
+
+	changes, err := rows.RowsAffected()
+	if err != nil {
+		return -1, err
+	}
+
+	// Delete entity
+	if _, err := conn.Exec("DELETE FROM entity where id = ?;", id); err != nil {
+		rollback()
+		return -1, err
+	}
+
+	if _, err := conn.Exec("COMMIT"); err != nil {
+		return -1, err
+	}
+	return int(changes), nil
+}
+
+// Rename an edge for a given path
+func (db *Database) Rename(currentName, newName string) error {
+	parentPath, name := splitPath(currentName)
+	newParentPath, newEdgeName := splitPath(newName)
+
+	if parentPath != newParentPath {
+		return fmt.Errorf("Cannot rename when root paths do not match %s != %s", parentPath, newParentPath)
+	}
+
+	conn, err := db.openConn()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	parent, err := db.get(conn, parentPath)
+	if err != nil {
+		return err
+	}
+
+	rows, err := conn.Exec("UPDATE edge SET name = ? WHERE parent_id = ? AND name LIKE ?;", newEdgeName, parent.id, name+"%")
+	if err != nil {
+		return err
+	}
+	i, err := rows.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if i == 0 {
+		return fmt.Errorf("Cannot locate edge for %s %s", parent.id, name)
+	}
+	return nil
+}
+
+type WalkMeta struct {
+	Parent   *Entity
+	Entity   *Entity
+	FullPath string
+	Edge     *Edge
+}
+
+func (db *Database) children(conn *sql.DB, name string, depth int) <-chan WalkMeta {
+	out := make(chan WalkMeta)
+	e, err := db.get(conn, name)
+	if err != nil {
+		close(out)
+		return out
+	}
+
+	go func() {
+		rows, err := conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id)
+		if err != nil {
+			close(out)
+		}
+		defer rows.Close()
+
+		for rows.Next() {
+			var entityId, entityName string
+			if err := rows.Scan(&entityId, &entityName); err != nil {
+				// Log error
+				continue
+			}
+			child := &Entity{entityId}
+			edge := &Edge{
+				ParentID: e.id,
+				Name:     entityName,
+				EntityID: child.id,
+			}
+
+			meta := WalkMeta{
+				Parent:   e,
+				Entity:   child,
+				FullPath: path.Join(name, edge.Name),
+				Edge:     edge,
+			}
+
+			out <- meta
+			if depth == 0 {
+				continue
+			}
+			nDepth := depth
+			if depth != -1 {
+				nDepth -= 1
+			}
+			sc := db.children(conn, meta.FullPath, nDepth)
+			for c := range sc {
+				out <- c
+			}
+		}
+		close(out)
+	}()
+	return out
+}
+
+// Return the entity based on the parent path and name
+func (db *Database) child(conn *sql.DB, parent *Entity, name string) *Entity {
+	var id string
+	if err := conn.QueryRow("SELECT entity_id FROM edge WHERE parent_id = ? AND name LIKE ?;", parent.id, name+"%").Scan(&id); err != nil {
+		return nil
+	}
+	return &Entity{id}
+}
+
+func (db *Database) openConn() (*sql.DB, error) {
+	return sql.Open("sqlite3", db.dbPath)
+}
+
+// Return the id used to reference this entity
+func (e *Entity) ID() string {
+	return e.id
+}
+
+// Return the paths sorted by depth
+func (e Entities) Paths() []string {
+	out := make([]string, len(e))
+	var i int
+	for k := range e {
+		out[i] = k
+		i++
+	}
+	sortByDepth(out)
+
+	return out
+}

gograph/gograph_test.go

@@ -0,0 +1,464 @@
+package gograph
+
+import (
+	"os"
+	"path"
+	"strconv"
+	"testing"
+)
+
+func newTestDb(t *testing.T) *Database {
+	db, err := NewDatabase(path.Join(os.TempDir(), "sqlite.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return db
+}
+
+func destroyTestDb(db *Database) {
+	os.Remove(db.dbPath)
+}
+
+func TestNewDatabase(t *testing.T) {
+	db := newTestDb(t)
+	if db == nil {
+		t.Fatal("Database should not be nil")
+	}
+	defer destroyTestDb(db)
+}
+
+func TestCreateRootEnity(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+	root := db.RootEntity()
+	if root == nil {
+		t.Fatal("Root entity should not be nil")
+	}
+}
+
+func TestGetRootEntity(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	e := db.Get("/")
+	if e == nil {
+		t.Fatal("Entity should not be nil")
+	}
+	if e.ID() != "0" {
+		t.Fatalf("Enity id should be 0, got %s", e.ID())
+	}
+}
+
+func TestSetEntityWithDifferentName(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/test", "1")
+	if _, err := db.Set("/other", "1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSetDuplicateEntity(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	if _, err := db.Set("/foo", "42"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/foo", "43"); err == nil {
+		t.Fatalf("Creating an entry with a duplciate path did not cause an error")
+	}
+}
+
+func TestCreateChild(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	child, err := db.Set("/db", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if child == nil {
+		t.Fatal("Child should not be nil")
+	}
+	if child.ID() != "1" {
+		t.Fail()
+	}
+}
+
+func TestListAllRootChildren(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	for i := 1; i < 6; i++ {
+		a := strconv.Itoa(i)
+		if _, err := db.Set("/"+a, a); err != nil {
+			t.Fatal(err)
+		}
+	}
+	entries := db.List("/", -1)
+	if len(entries) != 5 {
+		t.Fatalf("Expect 5 entries for / got %d", len(entries))
+	}
+}
+
+func TestListAllSubChildren(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	_, err := db.Set("/webapp", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child2, err := db.Set("/db", "2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child4, err := db.Set("/logs", "4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/db/logs", child4.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child3, err := db.Set("/sentry", "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	entries := db.List("/webapp", 1)
+	if len(entries) != 3 {
+		t.Fatalf("Expect 3 entries for / got %d", len(entries))
+	}
+
+	entries = db.List("/webapp", 0)
+	if len(entries) != 2 {
+		t.Fatalf("Expect 2 entries for / got %d", len(entries))
+	}
+}
+
+func TestAddSelfAsChild(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	child, err := db.Set("/test", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/test/other", child.ID()); err == nil {
+		t.Fatal("Error should not be nil")
+	}
+}
+
+func TestAddChildToNonExistantRoot(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	if _, err := db.Set("/myapp", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := db.Set("/myapp/proxy/db", "2"); err == nil {
+		t.Fatal("Error should not be nil")
+	}
+}
+
+func TestWalkAll(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+	_, err := db.Set("/webapp", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child2, err := db.Set("/db", "2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child4, err := db.Set("/db/logs", "4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/logs", child4.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child3, err := db.Set("/sentry", "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child5, err := db.Set("/gograph", "5")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := db.Walk("/", func(p string, e *Entity) error {
+		t.Logf("Path: %s Entity: %s", p, e.ID())
+		return nil
+	}, -1); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetEntityByPath(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+	_, err := db.Set("/webapp", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child2, err := db.Set("/db", "2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child4, err := db.Set("/logs", "4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/db/logs", child4.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child3, err := db.Set("/sentry", "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child5, err := db.Set("/gograph", "5")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	entity := db.Get("/webapp/db/logs")
+	if entity == nil {
+		t.Fatal("Entity should not be nil")
+	}
+	if entity.ID() != "4" {
+		t.Fatalf("Expected to get entity with id 4, got %s", entity.ID())
+	}
+}
+
+func TestEnitiesPaths(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+	_, err := db.Set("/webapp", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child2, err := db.Set("/db", "2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child4, err := db.Set("/logs", "4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/db/logs", child4.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child3, err := db.Set("/sentry", "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child5, err := db.Set("/gograph", "5")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	out := db.List("/", -1)
+	for _, p := range out.Paths() {
+		t.Log(p)
+	}
+}
+
+func TestDeleteRootEntity(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	if err := db.Delete("/"); err == nil {
+		t.Fatal("Error should not be nil")
+	}
+}
+
+func TestDeleteEntity(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+	_, err := db.Set("/webapp", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child2, err := db.Set("/db", "2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	child4, err := db.Set("/logs", "4")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/db/logs", child4.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child3, err := db.Set("/sentry", "3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/sentry", child3.ID()); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/db", child2.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	child5, err := db.Set("/gograph", "5")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Set("/webapp/same-ref-diff-name", child5.ID()); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := db.Delete("/webapp/sentry"); err != nil {
+		t.Fatal(err)
+	}
+	entity := db.Get("/webapp/sentry")
+	if entity != nil {
+		t.Fatal("Entity /webapp/sentry should be nil")
+	}
+}
+
+func TestCountRefs(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/webapp", "1")
+
+	if db.Refs("1") != 1 {
+		t.Fatal("Expect reference count to be 1")
+	}
+
+	db.Set("/db", "2")
+	db.Set("/webapp/db", "2")
+	if db.Refs("2") != 2 {
+		t.Fatal("Expect reference count to be 2")
+	}
+}
+
+func TestPurgeId(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/webapp", "1")
+
+	if db.Refs("1") != 1 {
+		t.Fatal("Expect reference count to be 1")
+	}
+
+	db.Set("/db", "2")
+	db.Set("/webapp/db", "2")
+
+	count, err := db.Purge("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != 2 {
+		t.Fatal("Expected 2 references to be removed")
+	}
+}
+
+func TestRename(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/webapp", "1")
+
+	if db.Refs("1") != 1 {
+		t.Fatal("Expect reference count to be 1")
+	}
+
+	db.Set("/db", "2")
+	db.Set("/webapp/db", "2")
+
+	if db.Get("/webapp/db") == nil {
+		t.Fatal("Cannot find entity at path /webapp/db")
+	}
+
+	if err := db.Rename("/webapp/db", "/webapp/newdb"); err != nil {
+		t.Fatal(err)
+	}
+	if db.Get("/webapp/db") != nil {
+		t.Fatal("Entity should not exist at /webapp/db")
+	}
+	if db.Get("/webapp/newdb") == nil {
+		t.Fatal("Cannot find entity at path /webapp/newdb")
+	}
+
+}
+
+func TestCreateMultipleNames(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/db", "1")
+	if _, err := db.Set("/myapp", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	db.Walk("/", func(p string, e *Entity) error {
+		t.Logf("%s\n", p)
+		return nil
+	}, -1)
+}
+
+func TestRefPaths(t *testing.T) {
+	db := newTestDb(t)
+	defer destroyTestDb(db)
+
+	db.Set("/webapp", "1")
+
+	db.Set("/db", "2")
+	db.Set("/webapp/db", "2")
+
+	refs := db.RefPaths("2")
+	if len(refs) != 2 {
+		t.Fatalf("Expected reference count to be 2, got %d", len(refs))
+	}
+
+}

gograph/sort.go

@@ -0,0 +1,27 @@
+package gograph
+
+import "sort"
+
+type pathSorter struct {
+	paths []string
+	by    func(i, j string) bool
+}
+
+func sortByDepth(paths []string) {
+	s := &pathSorter{paths, func(i, j string) bool {
+		return pathDepth(i) > pathDepth(j)
+	}}
+	sort.Sort(s)
+}
+
+func (s *pathSorter) Len() int {
+	return len(s.paths)
+}
+
+func (s *pathSorter) Swap(i, j int) {
+	s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
+}
+
+func (s *pathSorter) Less(i, j int) bool {
+	return s.by(s.paths[i], s.paths[j])
+}

gograph/sort_test.go

@@ -0,0 +1,29 @@
+package gograph
+
+import (
+	"testing"
+)
+
+func TestSort(t *testing.T) {
+	paths := []string{
+		"/",
+		"/myreallylongname",
+		"/app/db",
+	}
+
+	sortByDepth(paths)
+
+	if len(paths) != 3 {
+		t.Fatalf("Expected 3 parts got %d", len(paths))
+	}
+
+	if paths[0] != "/app/db" {
+		t.Fatalf("Expected /app/db got %s", paths[0])
+	}
+	if paths[1] != "/myreallylongname" {
+		t.Fatalf("Expected /myreallylongname got %s", paths[1])
+	}
+	if paths[2] != "/" {
+		t.Fatalf("Expected / got %s", paths[2])
+	}
+}

gograph/utils.go

@@ -0,0 +1,32 @@
+package gograph
+
+import (
+	"path"
+	"strings"
+)
+
+// Split p on /
+func split(p string) []string {
+	return strings.Split(p, "/")
+}
+
+// Returns the depth or number of / in a given path
+func pathDepth(p string) int {
+	parts := split(p)
+	if len(parts) == 2 && parts[1] == "" {
+		return 1
+	}
+	return len(parts)
+}
+
+func splitPath(p string) (parent, name string) {
+	if p[0] != '/' {
+		p = "/" + p
+	}
+	parent, name = path.Split(p)
+	l := len(parent)
+	if parent[l-1] == '/' {
+		parent = parent[:l-1]
+	}
+	return
+}

graph_test.go

@@ -121,6 +121,9 @@ func TestRegister(t *testing.T) {
 }
 
 func TestMount(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
 	graph := tempGraph(t)
 	defer os.RemoveAll(graph.Root)
 	archive, err := fakeTar()
@@ -144,12 +147,12 @@ func TestMount(t *testing.T) {
 	if err := os.MkdirAll(rw, 0700); err != nil {
 		t.Fatal(err)
 	}
-	if err := image.Mount(rootfs, rw); err != nil {
+	if err := image.Mount(runtime, rootfs, rw, "testing"); err != nil {
 		t.Fatal(err)
 	}
 	// FIXME: test for mount contents
 	defer func() {
-		if err := Unmount(rootfs); err != nil {
+		if err := image.Unmount(runtime, rootfs, "testing"); err != nil {
 			t.Error(err)
 		}
 	}()

hack/PACKAGERS.md

@@ -32,14 +32,14 @@ the process.
 
 ## System build dependencies
 
-To build docker, you will need the following system dependencies
+To build docker, you will need the following system dependencies:
 
 * An amd64 machine
 * A recent version of git and mercurial
-* Go version 1.1.2
+* Go version 1.2rc1
+* A copy of libdevmapper.a (statically compiled), and associated headers
 * A clean checkout of the source must be added to a valid Go [workspace](http://golang.org/doc/code.html#Workspaces)
-under the path *src/github.com/dotcloud/docker*. See 
-
+under the path *src/github.com/dotcloud/docker*.
 
 ## Go dependencies
 
@@ -55,15 +55,13 @@ NOTE: if you''re not able to package the exact version (to the exact commit) of
 please get in touch so we can remediate! Who knows what discrepancies can be caused by even the
 slightest deviation. We promise to do our best to make everybody happy.
 
+## Disabling CGO for the net package
 
-## Disabling CGO
-
-Make sure to disable CGO on your system, and then recompile the standard library on the build
-machine:
+Make sure to disable CGO on your system for the net package using `-tags netgo`,
+and then recompile the standard library on the build machine:
 
 ```bash
-export CGO_ENABLED=0
-cd /tmp && echo 'package main' > t.go && go test -a -i -v
+go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
 ```
 
 ## Building Docker
@@ -71,7 +69,7 @@ cd /tmp && echo 'package main' > t.go && go test -a -i -v
 To build the docker binary, run the following command with the source checkout as the
 working directory:
 
-```
+```bash
 ./hack/make.sh binary
 ```
 
@@ -80,9 +78,9 @@ This will create a static binary under *./bundles/$VERSION/binary/docker-$VERSIO
 
 You are encouraged to use ./hack/make.sh without modification. If you must absolutely write
 your own script (are you really, really sure you need to? make.sh is really not that complicated),
-then please take care the respect the following:
+then please take care to respect the following:
 
-* In *./hack/make.sh*: $LDFLAGS, $VERSION and $GITCOMMIT
+* In *./hack/make.sh*: $LDFLAGS, $BUILDFLAGS, $VERSION and $GITCOMMIT
 * In *./hack/make/binary*: the exact build command to run
 
 You may be tempted to tweak these settings. In particular, being a rigorous maintainer, you may want
@@ -106,7 +104,6 @@ dependencies to be installed (see below).
 
 The test suite will also download a small test container, so you will need internet connectivity.
 
-
 ## Runtime dependencies
 
 To run properly, docker needs the following software to be installed at runtime:

hack/infrastructure/README.md

@@ -1,5 +1,83 @@
 # Docker project infrastructure
 
-This directory holds all information about the technical infrastructure of the docker project; servers, dns, email, and all the corresponding tools and configuration.
+This is an overview of the Docker infrastructure.
 
-Obviously credentials should not be stored in this repo, but how to obtain and use them should be documented here.
+**Note: obviously, credentials should not be stored in this repository.**
+However, when there are credentials, we should list how to obtain them
+(e.g. who has them).
+
+
+## Providers
+
+This should be the list of all the entities providing some kind of
+infrastructure service to the Docker project (either for free,
+or paid by dotCloud).
+
+
+Provider      | Service
+--------------|-------------------------------------------------
+AWS           | packages (S3 bucket), dotCloud PAAS, dev-env, ci
+CloudFlare    | cdn
+Digital Ocean | ci
+dotCloud PAAS | website, index, registry, ssl, blog
+DynECT        | dns (docker.io)            
+GitHub        | repository
+Linode        | stackbrew
+Mailgun       | outgoing e-mail            
+ReadTheDocs   | docs
+
+*Ordered-by: lexicographic*
+
+
+## URLs
+
+This should be the list of all the infrastructure-related URLs
+and which service is handling them.
+
+URL                                          | Service
+---------------------------------------------|---------------------------------
+ http://blog.docker.io/                      | blog
+*http://cdn-registry-1.docker.io/            | registry (pull)
+ http://debug.docker.io/                     | debug tool
+ http://docs.docker.io/                      | docsproxy (proxy to readthedocs)
+ http://docker-ci.dotcloud.com/              | ci
+ http://docker.io/                           | redirect to www.docker.io (dynect)
+ http://docker.readthedocs.org/              | docs
+*http://get.docker.io/                       | packages
+ https://github.com/dotcloud/docker          | repository
+*https://index.docker.io/                    | index
+ http://registry-1.docker.io/                | registry (push)
+ http://staging-docker-ci.dotcloud.com/      | ci
+*http://test.docker.io/                      | packages
+*http://www.docker.io/                       | website
+ http://? (internal URL, not for public use) | stackbrew
+
+*Ordered-by: lexicographic*
+
+**Note:** an asterisk in front of the URL means that it is cached by CloudFlare.
+
+
+## Services
+
+This should be the list of all services referenced above.
+
+Service             | Maintainer(s)      | How to update    | Source
+--------------------|--------------------|------------------|-------
+blog                | @jbarbier          | dotcloud push    | https://github.com/dotcloud/blog.docker.io
+cdn                 | @jpetazzo @samalba | cloudflare panel | N/A
+ci                  | @mzdaniel          | See [docker-ci]  | See [docker-ci]
+docs                | @metalivedev       | github webhook   | docker repo
+docsproxy           | @dhrp              | dotcloud push    | https://github.com/dotcloud/docker-docs-dotcloud-proxy
+index               | @kencochrane       | dotcloud push    | private
+packages            | @jpetazzo          | hack/release     | docker repo
+registry            | @samalba           | dotcloud push    | https://github.com/dotcloud/docker-registry
+repository (github) | N/A                | N/A              | N/A
+ssl (dotcloud)      | @jpetazzo          | dotcloud ops     | N/A
+ssl (cloudflare)    | @jpetazzo          | cloudflare panel | N/A
+stackbrew           | @shin-             | manual           | https://github.com/dotcloud/stackbrew/stackbrew
+website             | @dhrp              | dotcloud push    | https://github.com/dotcloud/www.docker.io
+
+*Ordered-by: lexicographic*
+
+
+[docker-ci]: docker-ci.rst

hack/infrastructure/overview.md

@@ -1,103 +0,0 @@
-# Docker Server Overview
-This is an overview of the Docker infrastructure
-
-## Docker Git Repo
-The Docker source code lives on github.com under the dotCloud account.
-https://github.com/dotcloud/docker
-
-## DNS
-We are using dyn.com for our DNS server for the docker.io domain. 
-It is using the dotCloud account.
-
-### DNS Redirect
-We have a DNS redirect in dyn.com that will automatically redirect 
-docker.io to www.docker.io
-
-## email
-Email is sent via  dotCloud account on MailGun.com
-
-## CDN
-We are using a CDN in front of some of the docker.io domains to help improve
-proformance. The CDN is Cloudflare, using a Pro account. 
-
-*This is currently disabled due to an issue with slow performance during pull
-in some regions of the world.*
-
-### CDN Domains
-- www.docker.io
-- test.docker.io
-- registry-1.docker.io
-- debug.docker.io
-- cdn-registry-1.docker.io
-
-## edge-docker.dotcloud.com
-All of the Docker applications that live on dotCloud go through their own
-load balancer, and this is where SSL is terminated as well.
-
-## www.docker.io
-This is hosted under the docker account on dotCloud's PaaS. 
-
-### Source Code
-The source code for the website lives here:
-https://github.com/dotcloud/www.docker.io
-
-## Docker Registry
-The registry is where the image data is store.
-
-### URL:
-- registry-1.docker.io
-- cdn-registry-1.docker.io
-
-There are two urls, one is behind a CDN the other isn't this is because when
-you pull, you pull from the CDN url, to help with pull speeds. We don't push
-through the CDN as well, because it doesn't help us, so we bypass it.
-
-### Data Store:
-The data store for the registry is using Amazon S3 in a bucket under the docker
-aws account.
-
-### Source Code
-The source code for the registry lives here: https://github.com/dotcloud/docker-registry
-
-### Hosted:
-Hosted on the Docker account on dotCloud's PaaS
-
-## index.docker.io
-This is the docker index, it stores all of the meta information about the 
-docker images, but all data is stored in the registry.
-
-### Source Code:
-Not available
-
-### Hosted:
-Hosted on the Docker account on dotCloud's PaaS
-
-## blog.docker.io
-This is a wordpress based Docker blog.
-
-### URL:
-http://blog.docker.io
-
-### Source Code:
-https://github.com/dotcloud/blog.docker.io
-
-## docs.docker.io
-This is where all of the documentation for docker lives.
-
-### Hosted:
-This website is hosted on ReadTheDocs.org.
-
-### Updates
-These docs get automatically updated when the Docker repo on github has
-new commits. It does this via a webhook.
-
-### Proxy:
-This is a simple dotcloud app, with its main and only purpose to forward
-http (and https) requests to docker.readthedocs.org.
-
-https://github.com/dotcloud/docker-docs-dotcloud-proxy
-
-## get.docker.io
-This is the docker repository where we store images
-
-TODO: need more here. jerome?

hack/make.sh

@@ -44,7 +44,8 @@ if [ -n "$(git status --porcelain)" ]; then
 fi
 
 # Use these flags when compiling the tests and final binary
-LDFLAGS="-X main.GITCOMMIT $GITCOMMIT -X main.VERSION $VERSION -d -w"
+LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-all"'
+BUILDFLAGS='-tags netgo'
 
 
 bundle() {

hack/make/binary

@@ -2,6 +2,6 @@
 
 DEST=$1
 
-if go build -o $DEST/docker-$VERSION -ldflags "$LDFLAGS" ./docker; then
+if go build -o $DEST/docker-$VERSION -ldflags "$LDFLAGS" $BUILDFLAGS ./docker; then
 	echo "Created binary: $DEST/docker-$VERSION"
 fi

hack/make/test

@@ -1,3 +1,5 @@
+#!/bin/sh
+
 DEST=$1
 
 set -e
@@ -14,7 +16,7 @@ bundle_test() {
 		for test_dir in $(find_test_dirs); do (
 			set -x
 			cd $test_dir
-			go test -v -ldflags "$LDFLAGS" $TESTFLAGS
+			go test -v -ldflags "$LDFLAGS" $BUILDFLAGS $TESTFLAGS
 		)  done
 	} 2>&1 | tee $DEST/test.log
 }
@@ -29,4 +31,26 @@ find_test_dirs() {
                sort -u
 }
 
+
+check_test_garbage() {
+	nGarbage=`find /tmp/ -name 'docker-*' | wc -l`
+	if [ $nGarbage -gt 0 ]; then
+		(
+		cat <<EOF
+
+Error: there are $nGarbage docker-related files in /tmp. Please clean them up and try again.
+--------------------------------------------------------------------------------------------
+EOF
+		find /tmp -name 'docker-*'
+		cat <<EOF
+--------------------------------------------------------------------------------------------
+If the error persists, your tests might leak temporary files. This is considered a test fail.
+EOF
+		exit 1
+		) 2>&1
+	fi
+}
+
+check_test_garbage
 bundle_test
+check_test_garbage

image.go

@@ -5,16 +5,16 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"github.com/dotcloud/docker/devmapper"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
@@ -61,6 +61,7 @@ func LoadImage(root string) (*Image, error) {
 	}
 
 	// Check that the filesystem layer exists
+	// FIXME: once an image is added into device mapper, the layer is no longer needed
 	if stat, err := os.Stat(layerPath(root)); err != nil {
 		if os.IsNotExist(err) {
 			return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.ID)
@@ -136,29 +137,8 @@ func jsonPath(root string) string {
 	return path.Join(root, "json")
 }
 
-func MountAUFS(ro []string, rw string, target string) error {
-	// FIXME: Now mount the layers
-	rwBranch := fmt.Sprintf("%v=rw", rw)
-	roBranches := ""
-	for _, layer := range ro {
-		roBranches += fmt.Sprintf("%v=ro+wh:", layer)
-	}
-	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)
-
-	branches += ",xino=/dev/shm/aufs.xino"
-
-	//if error, try to load aufs kernel module
-	if err := mount("none", target, "aufs", 0, branches); err != nil {
-		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
-		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
-			return fmt.Errorf("Unable to load the AUFS module")
-		}
-		log.Printf("...module loaded.")
-		if err := mount("none", target, "aufs", 0, branches); err != nil {
-			return fmt.Errorf("Unable to mount using aufs")
-		}
-	}
-	return nil
+func mountPath(root string) string {
+	return path.Join(root, "mount")
 }
 
 // TarLayer returns a tar archive of the image's filesystem layer.
@@ -170,35 +150,383 @@ func (image *Image) TarLayer(compression Compression) (Archive, error) {
 	return Tar(layerPath, compression)
 }
 
-func (image *Image) Mount(root, rw string) error {
-	if mounted, err := Mounted(root); err != nil {
-		return err
-	} else if mounted {
-		return fmt.Errorf("%s is already mounted", root)
-	}
-	layers, err := image.layers()
+type TimeUpdate struct {
+	path string
+	time []syscall.Timeval
+	mode uint32
+}
+
+func (image *Image) applyLayer(layer, target string) error {
+	var updateTimes []TimeUpdate
+	oldmask := syscall.Umask(0)
+	defer syscall.Umask(oldmask)
+	err := filepath.Walk(layer, func(srcPath string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Skip root
+		if srcPath == layer {
+			return nil
+		}
+
+		var srcStat syscall.Stat_t
+		err = syscall.Lstat(srcPath, &srcStat)
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(layer, srcPath)
+		if err != nil {
+			return err
+		}
+
+		targetPath := filepath.Join(target, relPath)
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match(".wh..wh.*", relPath); err != nil || matched {
+			if err != nil || !f.IsDir() {
+				return err
+			}
+			return filepath.SkipDir
+		}
+
+		// Find out what kind of modification happened
+		file := filepath.Base(srcPath)
+
+		// If there is a whiteout, then the file was removed
+		if strings.HasPrefix(file, ".wh.") {
+			originalFile := file[len(".wh."):]
+			deletePath := filepath.Join(filepath.Dir(targetPath), originalFile)
+
+			err = os.RemoveAll(deletePath)
+			if err != nil {
+				return err
+			}
+		} else {
+			var targetStat = &syscall.Stat_t{}
+			err := syscall.Lstat(targetPath, targetStat)
+			if err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+				targetStat = nil
+			}
+
+			if targetStat != nil && !(targetStat.Mode&syscall.S_IFDIR == syscall.S_IFDIR && srcStat.Mode&syscall.S_IFDIR == syscall.S_IFDIR) {
+				// Unless both src and dest are directories we remove the target and recreate it
+				// This is a bit wasteful in the case of only a mode change, but that is unlikely
+				// to matter much
+				err = os.RemoveAll(targetPath)
+				if err != nil {
+					return err
+				}
+				targetStat = nil
+			}
+
+			if f.IsDir() {
+				// Source is a directory
+				if targetStat == nil {
+					err = syscall.Mkdir(targetPath, srcStat.Mode&07777)
+					if err != nil {
+						return err
+					}
+				}
+			} else if srcStat.Mode&syscall.S_IFLNK == syscall.S_IFLNK {
+				// Source is symlink
+				link, err := os.Readlink(srcPath)
+				if err != nil {
+					return err
+				}
+
+				err = os.Symlink(link, targetPath)
+				if err != nil {
+					return err
+				}
+			} else if srcStat.Mode&syscall.S_IFBLK == syscall.S_IFBLK ||
+				srcStat.Mode&syscall.S_IFCHR == syscall.S_IFCHR ||
+				srcStat.Mode&syscall.S_IFIFO == syscall.S_IFIFO ||
+				srcStat.Mode&syscall.S_IFSOCK == syscall.S_IFSOCK {
+				// Source is special file
+				err = syscall.Mknod(targetPath, srcStat.Mode, int(srcStat.Rdev))
+				if err != nil {
+					return err
+				}
+			} else if srcStat.Mode&syscall.S_IFREG == syscall.S_IFREG {
+				// Source is regular file
+				fd, err := syscall.Open(targetPath, syscall.O_CREAT|syscall.O_WRONLY, srcStat.Mode&07777)
+				if err != nil {
+					return err
+				}
+				dstFile := os.NewFile(uintptr(fd), targetPath)
+				srcFile, err := os.Open(srcPath)
+				if err != nil {
+					_ = dstFile.Close()
+					return err
+				}
+				err = CopyFile(dstFile, srcFile)
+				_ = dstFile.Close()
+				_ = srcFile.Close()
+				if err != nil {
+					return err
+				}
+			} else {
+				return fmt.Errorf("Unknown type for file %s", srcPath)
+			}
+
+			err = syscall.Lchown(targetPath, int(srcStat.Uid), int(srcStat.Gid))
+			if err != nil {
+				return err
+			}
+
+			if srcStat.Mode&syscall.S_IFLNK != syscall.S_IFLNK {
+				err = syscall.Chmod(targetPath, srcStat.Mode&07777)
+				if err != nil {
+					return err
+				}
+			}
+
+			ts := []syscall.Timeval{
+				syscall.NsecToTimeval(srcStat.Atim.Nano()),
+				syscall.NsecToTimeval(srcStat.Mtim.Nano()),
+			}
+
+			u := TimeUpdate{
+				path: targetPath,
+				time: ts,
+				mode: srcStat.Mode,
+			}
+
+			// Delay time updates until all other changes done, or it is
+			// overwritten for directories (by child changes)
+			updateTimes = append(updateTimes, u)
+		}
+		return nil
+	})
 	if err != nil {
 		return err
 	}
+
+	// We do this in reverse order so that children are updated before parents
+	for i := len(updateTimes) - 1; i >= 0; i-- {
+		update := updateTimes[i]
+
+		O_PATH := 010000000 // Not in syscall yet
+		var err error
+		if update.mode&syscall.S_IFLNK == syscall.S_IFLNK {
+			// Update time on the symlink via O_PATH + futimes(), if supported by the kernel
+
+			fd, err := syscall.Open(update.path, syscall.O_RDWR|O_PATH|syscall.O_NOFOLLOW, 0600)
+			if err == syscall.EISDIR || err == syscall.ELOOP {
+				// O_PATH not supported by kernel, nothing to do, ignore
+			} else if err != nil {
+				return err
+			} else {
+				syscall.Futimes(fd, update.time)
+				syscall.Close(fd)
+			}
+		} else {
+			err = syscall.Utimes(update.path, update.time)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (image *Image) ensureImageDevice(devices *devmapper.DeviceSet) error {
+	if devices.HasInitializedDevice(image.ID) {
+		return nil
+	}
+
+	if image.Parent != "" && !devices.HasInitializedDevice(image.Parent) {
+		parentImg, err := image.GetParent()
+		if err != nil {
+			return fmt.Errorf("Error while getting parent image: %v", err)
+		}
+		err = parentImg.ensureImageDevice(devices)
+		if err != nil {
+			return err
+		}
+	}
+
+	root, err := image.root()
+	if err != nil {
+		return err
+	}
+
+	mountDir := mountPath(root)
+	if err := os.Mkdir(mountDir, 0600); err != nil && !os.IsExist(err) {
+		return err
+	}
+
+	mounted, err := Mounted(mountDir)
+	if err == nil && mounted {
+		utils.Debugf("Image %s is unexpectedly mounted, unmounting...", image.ID)
+		err = syscall.Unmount(mountDir, 0)
+		if err != nil {
+			return err
+		}
+	}
+
+	if devices.HasDevice(image.ID) {
+		utils.Debugf("Found non-initialized demove-mapper device for image %s, removing", image.ID)
+		err = devices.RemoveDevice(image.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	utils.Debugf("Creating device-mapper device for image id %s", image.ID)
+	if err := devices.AddDevice(image.ID, image.Parent); err != nil {
+		utils.Debugf("Error add device: %s", err)
+		return err
+	}
+
+	if err := devices.MountDevice(image.ID, mountDir, false); err != nil {
+		utils.Debugf("Error mounting device: %s", err)
+		devices.RemoveDevice(image.ID)
+		return err
+	}
+
+	if err = image.applyLayer(layerPath(root), mountDir); err != nil {
+		utils.Debugf("Error applying layer: %s", err)
+		devices.UnmountDevice(image.ID, mountDir, true)
+		devices.RemoveDevice(image.ID)
+		return err
+	}
+
+	// The docker init layer is conceptually above all other layers, so we apply
+	// it for every image. This is safe because the layer directory is the
+	// definition of the image, and the device-mapper device is just a cache
+	// of it instantiated. Diffs/commit compare the container device with the
+	// image device, which will then *not* pick up the init layer changes as
+	// part of the container changes
+	dockerinitLayer, err := image.getDockerInitLayer()
+	if err != nil {
+		devices.UnmountDevice(image.ID, mountDir, true)
+		devices.RemoveDevice(image.ID)
+		return err
+	}
+
+	if err := image.applyLayer(dockerinitLayer, mountDir); err != nil {
+		devices.UnmountDevice(image.ID, mountDir, true)
+		devices.RemoveDevice(image.ID)
+		return err
+	}
+
+	if err := devices.UnmountDevice(image.ID, mountDir, true); err != nil {
+		devices.RemoveDevice(image.ID)
+		return err
+	}
+
+	devices.SetInitialized(image.ID)
+
+	return nil
+}
+
+func (image *Image) Mounted(runtime *Runtime, root, rw string) (bool, error) {
+	return Mounted(root)
+}
+
+func (image *Image) Mount(runtime *Runtime, root, rw string, id string) error {
+	if mounted, _ := image.Mounted(runtime, root, rw); mounted {
+		return fmt.Errorf("%s is already mounted", root)
+	}
+
 	// Create the target directories if they don't exist
 	if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) {
 		return err
 	}
-	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
+
+	devices, err := runtime.GetDeviceSet()
+	if err != nil {
 		return err
 	}
-	if err := MountAUFS(layers, rw, root); err != nil {
+
+	if err := image.ensureImageDevice(devices); err != nil {
 		return err
 	}
+
+	if !devices.HasDevice(id) {
+		utils.Debugf("Creating device %s for container based on image %s", id, image.ID)
+		err = devices.AddDevice(id, image.ID)
+		if err != nil {
+			return err
+		}
+	}
+
+	utils.Debugf("Mounting container %s at %s for container", id, root)
+	if err := devices.MountDevice(id, root, false); err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func (image *Image) Changes(rw string) ([]Change, error) {
-	layers, err := image.layers()
+func (image *Image) Unmount(runtime *Runtime, root string, id string) error {
+	// Try to deactivate the device as generally there is no use for it anymore
+	devices, err := runtime.GetDeviceSet()
+	if err != nil {
+		return err
+	}
+
+	if err = devices.UnmountDevice(id, root, true); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (image *Image) Changes(runtime *Runtime, root, rw, id string) ([]Change, error) {
+	devices, err := runtime.GetDeviceSet()
 	if err != nil {
 		return nil, err
 	}
-	return Changes(layers, rw)
+
+	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+
+	wasActivated := devices.HasActivatedDevice(image.ID)
+
+	// We re-use rw for the temporary mount of the base image as its
+	// not used by device-mapper otherwise
+	err = devices.MountDevice(image.ID, rw, true)
+	if err != nil {
+		return nil, err
+	}
+
+	changes, err := ChangesDirs(root, rw)
+	devices.UnmountDevice(image.ID, rw, !wasActivated)
+	if err != nil {
+		return nil, err
+	}
+	return changes, nil
+}
+
+func (image *Image) ExportChanges(runtime *Runtime, root, rw, id string) (Archive, error) {
+	changes, err := image.Changes(runtime, root, rw, id)
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+
+	return TarFilter(root, Uncompressed, files, false, deletions)
 }
 
 func (image *Image) ShortID() string {

iptables/MAINTAINERS

@@ -0,0 +1 @@
+Michael Crosby <michael@crosbymichael.com> (@crosbymichael)

iptables/iptables.go

@@ -0,0 +1,105 @@
+package iptables
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+type Action string
+
+const (
+	Add    Action = "-A"
+	Delete Action = "-D"
+)
+
+var (
+	ErrIptablesNotFound = errors.New("Iptables not found")
+	nat                 = []string{"-t", "nat"}
+)
+
+type Chain struct {
+	Name   string
+	Bridge string
+}
+
+func NewChain(name, bridge string) (*Chain, error) {
+	if err := Raw("-t", "nat", "-N", name); err != nil {
+		return nil, err
+	}
+	chain := &Chain{
+		Name:   name,
+		Bridge: bridge,
+	}
+
+	if err := chain.Prerouting(Add, "-m", "addrtype", "--dst-type", "LOCAL"); err != nil {
+		return nil, fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
+	}
+	if err := chain.Output(Add, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8"); err != nil {
+		return nil, fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
+	}
+	return chain, nil
+}
+
+func RemoveExistingChain(name string) error {
+	chain := &Chain{
+		Name: name,
+	}
+	return chain.Remove()
+}
+
+func (c *Chain) Forward(action Action, ip net.IP, port int, proto, dest_addr string, dest_port int) error {
+	return Raw("-t", "nat", fmt.Sprint(action), c.Name,
+		"-p", proto,
+		"-d", ip.String(),
+		"--dport", strconv.Itoa(port),
+		"!", "-i", c.Bridge,
+		"-j", "DNAT",
+		"--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port)))
+}
+
+func (c *Chain) Prerouting(action Action, args ...string) error {
+	a := append(nat, fmt.Sprint(action), "PREROUTING")
+	if len(args) > 0 {
+		a = append(a, args...)
+	}
+	return Raw(append(a, "-j", c.Name)...)
+}
+
+func (c *Chain) Output(action Action, args ...string) error {
+	a := append(nat, fmt.Sprint(action), "OUTPUT")
+	if len(args) > 0 {
+		a = append(a, args...)
+	}
+	return Raw(append(a, "-j", c.Name)...)
+}
+
+func (c *Chain) Remove() error {
+	// Ignore errors - This could mean the chains were never set up
+	c.Prerouting(Delete, "-m", "addrtype", "--dst-type", "LOCAL")
+	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8")
+	c.Output(Delete, "-m", "addrtype", "--dst-type", "LOCAL") // Created in versions <= 0.1.6
+
+	c.Prerouting(Delete)
+	c.Output(Delete)
+
+	Raw("-t", "nat", "-F", c.Name)
+	Raw("-t", "nat", "-X", c.Name)
+
+	return nil
+}
+
+func Raw(args ...string) error {
+	path, err := exec.LookPath("iptables")
+	if err != nil {
+		return ErrIptablesNotFound
+	}
+	if err := exec.Command(path, args...).Run(); err != nil {
+		return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " "))
+	}
+	return nil
+
+}

iptables/iptables_test.go

@@ -0,0 +1,18 @@
+package iptables
+
+import (
+	"os"
+	"testing"
+)
+
+func TestIptables(t *testing.T) {
+	if err := Raw("-L"); err != nil {
+		t.Fatal(err)
+	}
+	path := os.Getenv("PATH")
+	os.Setenv("PATH", "")
+	defer os.Setenv("PATH", path)
+	if err := Raw("-L"); err == nil {
+		t.Fatal("Not finding iptables in the PATH should cause an error")
+	}
+}

links.go

@@ -0,0 +1,141 @@
+package docker
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/iptables"
+	"path"
+	"strings"
+)
+
+type Link struct {
+	ParentIP         string
+	ChildIP          string
+	Name             string
+	BridgeInterface  string
+	ChildEnvironment []string
+	Ports            []Port
+	IsEnabled        bool
+}
+
+func NewLink(parent, child *Container, name, bridgeInterface string) (*Link, error) {
+	if parent.ID == child.ID {
+		return nil, fmt.Errorf("Cannot link to self: %s == %s", parent.ID, child.ID)
+	}
+	if !child.State.Running {
+		return nil, fmt.Errorf("Cannot link to a non running container: %s AS %s", child.ID, name)
+	}
+
+	ports := make([]Port, len(child.Config.ExposedPorts))
+	var i int
+	for p := range child.Config.ExposedPorts {
+		ports[i] = p
+		i++
+	}
+
+	l := &Link{
+		BridgeInterface:  bridgeInterface,
+		Name:             name,
+		ChildIP:          child.NetworkSettings.IPAddress,
+		ParentIP:         parent.NetworkSettings.IPAddress,
+		ChildEnvironment: child.Config.Env,
+		Ports:            ports,
+	}
+	return l, nil
+
+}
+
+func (l *Link) Alias() string {
+	_, alias := path.Split(l.Name)
+	return alias
+}
+
+func (l *Link) ToEnv() []string {
+	env := []string{}
+	alias := strings.ToUpper(l.Alias())
+
+	if p := l.getDefaultPort(); p != nil {
+		env = append(env, fmt.Sprintf("%s_PORT=%s://%s:%s", alias, p.Proto(), l.ChildIP, p.Port()))
+	}
+
+	// Load exposed ports into the environment
+	for _, p := range l.Ports {
+		env = append(env, fmt.Sprintf("%s_PORT_%s_%s=%s://%s:%s", alias, p.Port(), strings.ToUpper(p.Proto()), p.Proto(), l.ChildIP, p.Port()))
+	}
+
+	// Load the linked container's name into the environment
+	env = append(env, fmt.Sprintf("%s_NAME=%s", alias, l.Name))
+
+	if l.ChildEnvironment != nil {
+		for _, v := range l.ChildEnvironment {
+			parts := strings.Split(v, "=")
+			if len(parts) != 2 {
+				continue
+			}
+			// Ignore a few variables that are added during docker build
+			if parts[0] == "HOME" || parts[0] == "PATH" {
+				continue
+			}
+			env = append(env, fmt.Sprintf("%s_ENV_%s=%s", alias, parts[0], parts[1]))
+		}
+	}
+	return env
+}
+
+// Default port rules
+func (l *Link) getDefaultPort() *Port {
+	var p Port
+	i := len(l.Ports)
+
+	if i == 0 {
+		return nil
+	} else if i > 1 {
+		sortPorts(l.Ports, func(ip, jp Port) bool {
+			// If the two ports have the same number, tcp takes priority
+			// Sort in desc order
+			return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && strings.ToLower(ip.Proto()) == "tcp")
+		})
+	}
+	p = l.Ports[0]
+	return &p
+}
+
+func (l *Link) Enable() error {
+	if err := l.toggle("-I", false); err != nil {
+		return err
+	}
+	l.IsEnabled = true
+	return nil
+}
+
+func (l *Link) Disable() {
+	// We do not care about errors here because the link may not
+	// exist in iptables
+	l.toggle("-D", true)
+
+	l.IsEnabled = false
+}
+
+func (l *Link) toggle(action string, ignoreErrors bool) error {
+	for _, p := range l.Ports {
+		if err := iptables.Raw(action, "FORWARD",
+			"-i", l.BridgeInterface, "-o", l.BridgeInterface,
+			"-p", p.Proto(),
+			"-s", l.ParentIP,
+			"--dport", p.Port(),
+			"-d", l.ChildIP,
+			"-j", "ACCEPT"); !ignoreErrors && err != nil {
+			return err
+		}
+
+		if err := iptables.Raw(action, "FORWARD",
+			"-i", l.BridgeInterface, "-o", l.BridgeInterface,
+			"-p", p.Proto(),
+			"-s", l.ChildIP,
+			"--sport", p.Port(),
+			"-d", l.ParentIP,
+			"-j", "ACCEPT"); !ignoreErrors && err != nil {
+			return err
+		}
+	}
+	return nil
+}

links_test.go

@@ -0,0 +1,104 @@
+package docker
+
+import (
+	"strings"
+	"testing"
+)
+
+func newMockLinkContainer(id string, ip string) *Container {
+	return &Container{
+		Config: &Config{},
+		ID:     id,
+		NetworkSettings: &NetworkSettings{
+			IPAddress: ip,
+		},
+	}
+}
+
+func TestLinkNew(t *testing.T) {
+	toID := GenerateID()
+	fromID := GenerateID()
+
+	from := newMockLinkContainer(fromID, "172.0.17.2")
+	from.Config.Env = []string{}
+	from.State = State{Running: true}
+	ports := make(map[Port]struct{})
+
+	ports[Port("6379/tcp")] = struct{}{}
+
+	from.Config.ExposedPorts = ports
+
+	to := newMockLinkContainer(toID, "172.0.17.3")
+
+	link, err := NewLink(to, from, "/db/docker", "172.0.17.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if link == nil {
+		t.FailNow()
+	}
+	if link.Name != "/db/docker" {
+		t.Fail()
+	}
+	if link.Alias() != "docker" {
+		t.Fail()
+	}
+	if link.ParentIP != "172.0.17.3" {
+		t.Fail()
+	}
+	if link.ChildIP != "172.0.17.2" {
+		t.Fail()
+	}
+	if link.BridgeInterface != "172.0.17.1" {
+		t.Fail()
+	}
+	for _, p := range link.Ports {
+		if p != Port("6379/tcp") {
+			t.Fail()
+		}
+	}
+}
+
+func TestLinkEnv(t *testing.T) {
+	toID := GenerateID()
+	fromID := GenerateID()
+
+	from := newMockLinkContainer(fromID, "172.0.17.2")
+	from.Config.Env = []string{"PASSWORD=gordon"}
+	from.State = State{Running: true}
+	ports := make(map[Port]struct{})
+
+	ports[Port("6379/tcp")] = struct{}{}
+
+	from.Config.ExposedPorts = ports
+
+	to := newMockLinkContainer(toID, "172.0.17.3")
+
+	link, err := NewLink(to, from, "/db/docker", "172.0.17.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rawEnv := link.ToEnv()
+	env := make(map[string]string, len(rawEnv))
+	for _, e := range rawEnv {
+		parts := strings.Split(e, "=")
+		if len(parts) != 2 {
+			t.FailNow()
+		}
+		env[parts[0]] = parts[1]
+	}
+	if env["DOCKER_PORT"] != "tcp://172.0.17.2:6379" {
+		t.Fatalf("Expected tcp://172.0.17.2:6379, got %s", env["DOCKER_PORT"])
+	}
+	if env["DOCKER_PORT_6379_TCP"] != "tcp://172.0.17.2:6379" {
+		t.Fatalf("Expected tcp://172.0.17.2:6379, got %s", env["DOCKER_PORT_6379_TCP"])
+	}
+	if env["DOCKER_NAME"] != "/db/docker" {
+		t.Fatalf("Expected /db/docker, got %s", env["DOCKER_NAME"])
+	}
+	if env["DOCKER_ENV_PASSWORD"] != "gordon" {
+		t.Fatalf("Expected gordon, got %s", env["DOCKER_ENV_PASSWORD"])
+	}
+}

lxc_template.go

@@ -92,7 +92,7 @@ lxc.mount.entry = devpts {{$ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,no
 #lxc.mount.entry = varlock {{$ROOTFS}}/var/lock tmpfs size=1024k,nosuid,nodev,noexec 0 0
 lxc.mount.entry = shm {{$ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0
 
-# Inject docker-init
+# Inject dockerinit
 lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/.dockerinit none bind,ro 0 0
 
 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container

mount.go

@@ -1,40 +1,11 @@
 package docker
 
 import (
-	"fmt"
-	"github.com/dotcloud/docker/utils"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"syscall"
-	"time"
 )
 
-func Unmount(target string) error {
-	if err := exec.Command("auplink", target, "flush").Run(); err != nil {
-		utils.Errorf("[warning]: couldn't run auplink before unmount: %s", err)
-	}
-	if err := syscall.Unmount(target, 0); err != nil {
-		return err
-	}
-	// Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint
-	// for some time. We'll just keep retrying until it succeeds.
-	for retries := 0; retries < 1000; retries++ {
-		err := os.Remove(target)
-		if err == nil {
-			// rm mntpoint succeeded
-			return nil
-		}
-		if os.IsNotExist(err) {
-			// mntpoint doesn't exist anymore. Success.
-			return nil
-		}
-		// fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err)
-		time.Sleep(10 * time.Millisecond)
-	}
-	return fmt.Errorf("Umount: Failed to umount %v", target)
-}
-
 func Mounted(mountpoint string) (bool, error) {
 	mntpoint, err := os.Stat(mountpoint)
 	if err != nil {

netlink/netlink.go

@@ -0,0 +1,548 @@
+package netlink
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"syscall"
+	"unsafe"
+)
+
+var nextSeqNr int
+
+func nativeEndian() binary.ByteOrder {
+	var x uint32 = 0x01020304
+	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
+		return binary.BigEndian
+	}
+	return binary.LittleEndian
+}
+
+func getSeq() int {
+	nextSeqNr = nextSeqNr + 1
+	return nextSeqNr
+}
+
+func getIpFamily(ip net.IP) int {
+	if len(ip) <= net.IPv4len {
+		return syscall.AF_INET
+	}
+	if ip.To4() != nil {
+		return syscall.AF_INET
+	}
+	return syscall.AF_INET6
+}
+
+type NetlinkRequestData interface {
+	ToWireFormat() []byte
+}
+
+type IfInfomsg struct {
+	syscall.IfInfomsg
+}
+
+func newIfInfomsg(family int) *IfInfomsg {
+	msg := &IfInfomsg{}
+	msg.Family = uint8(family)
+	msg.Type = uint16(0)
+	msg.Index = int32(0)
+	msg.Flags = uint32(0)
+	msg.Change = uint32(0)
+
+	return msg
+}
+
+func (msg *IfInfomsg) ToWireFormat() []byte {
+	native := nativeEndian()
+
+	len := syscall.SizeofIfInfomsg
+	b := make([]byte, len)
+	b[0] = msg.Family
+	b[1] = 0
+	native.PutUint16(b[2:4], msg.Type)
+	native.PutUint32(b[4:8], uint32(msg.Index))
+	native.PutUint32(b[8:12], msg.Flags)
+	native.PutUint32(b[12:16], msg.Change)
+	return b
+}
+
+type IfAddrmsg struct {
+	syscall.IfAddrmsg
+}
+
+func newIfAddrmsg(family int) *IfAddrmsg {
+	msg := &IfAddrmsg{}
+	msg.Family = uint8(family)
+	msg.Prefixlen = uint8(0)
+	msg.Flags = uint8(0)
+	msg.Scope = uint8(0)
+	msg.Index = uint32(0)
+
+	return msg
+}
+
+func (msg *IfAddrmsg) ToWireFormat() []byte {
+	native := nativeEndian()
+
+	len := syscall.SizeofIfAddrmsg
+	b := make([]byte, len)
+	b[0] = msg.Family
+	b[1] = msg.Prefixlen
+	b[2] = msg.Flags
+	b[3] = msg.Scope
+	native.PutUint32(b[4:8], msg.Index)
+	return b
+}
+
+type RtMsg struct {
+	syscall.RtMsg
+}
+
+func newRtMsg(family int) *RtMsg {
+	msg := &RtMsg{}
+	msg.Family = uint8(family)
+	msg.Table = syscall.RT_TABLE_MAIN
+	msg.Scope = syscall.RT_SCOPE_UNIVERSE
+	msg.Protocol = syscall.RTPROT_BOOT
+	msg.Type = syscall.RTN_UNICAST
+
+	return msg
+}
+
+func (msg *RtMsg) ToWireFormat() []byte {
+	native := nativeEndian()
+
+	len := syscall.SizeofRtMsg
+	b := make([]byte, len)
+	b[0] = msg.Family
+	b[1] = msg.Dst_len
+	b[2] = msg.Src_len
+	b[3] = msg.Tos
+	b[4] = msg.Table
+	b[5] = msg.Protocol
+	b[6] = msg.Scope
+	b[7] = msg.Type
+	native.PutUint32(b[8:12], msg.Flags)
+	return b
+}
+
+func rtaAlignOf(attrlen int) int {
+	return (attrlen + syscall.RTA_ALIGNTO - 1) & ^(syscall.RTA_ALIGNTO - 1)
+}
+
+type RtAttr struct {
+	syscall.RtAttr
+	Data []byte
+}
+
+func newRtAttr(attrType int, data []byte) *RtAttr {
+	attr := &RtAttr{}
+	attr.Type = uint16(attrType)
+	attr.Data = data
+
+	return attr
+}
+
+func (attr *RtAttr) ToWireFormat() []byte {
+	native := nativeEndian()
+
+	len := syscall.SizeofRtAttr + len(attr.Data)
+	b := make([]byte, rtaAlignOf(len))
+	native.PutUint16(b[0:2], uint16(len))
+	native.PutUint16(b[2:4], attr.Type)
+	for i, d := range attr.Data {
+		b[4+i] = d
+	}
+
+	return b
+}
+
+type NetlinkRequest struct {
+	syscall.NlMsghdr
+	Data []NetlinkRequestData
+}
+
+func (rr *NetlinkRequest) ToWireFormat() []byte {
+	native := nativeEndian()
+
+	length := rr.Len
+	dataBytes := make([][]byte, len(rr.Data))
+	for i, data := range rr.Data {
+		dataBytes[i] = data.ToWireFormat()
+		length = length + uint32(len(dataBytes[i]))
+	}
+	b := make([]byte, length)
+	native.PutUint32(b[0:4], length)
+	native.PutUint16(b[4:6], rr.Type)
+	native.PutUint16(b[6:8], rr.Flags)
+	native.PutUint32(b[8:12], rr.Seq)
+	native.PutUint32(b[12:16], rr.Pid)
+
+	i := 16
+	for _, data := range dataBytes {
+		for _, dataByte := range data {
+			b[i] = dataByte
+			i = i + 1
+		}
+	}
+	return b
+}
+
+func (rr *NetlinkRequest) AddData(data NetlinkRequestData) {
+	rr.Data = append(rr.Data, data)
+}
+
+func newNetlinkRequest(proto, flags int) *NetlinkRequest {
+	rr := &NetlinkRequest{}
+	rr.Len = uint32(syscall.NLMSG_HDRLEN)
+	rr.Type = uint16(proto)
+	rr.Flags = syscall.NLM_F_REQUEST | uint16(flags)
+	rr.Seq = uint32(getSeq())
+	return rr
+}
+
+type NetlinkSocket struct {
+	fd  int
+	lsa syscall.SockaddrNetlink
+}
+
+func getNetlinkSocket() (*NetlinkSocket, error) {
+	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_ROUTE)
+	if err != nil {
+		return nil, err
+	}
+	s := &NetlinkSocket{
+		fd: fd,
+	}
+	s.lsa.Family = syscall.AF_NETLINK
+	if err := syscall.Bind(fd, &s.lsa); err != nil {
+		syscall.Close(fd)
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (s *NetlinkSocket) Close() {
+	syscall.Close(s.fd)
+}
+
+func (s *NetlinkSocket) Send(request *NetlinkRequest) error {
+	if err := syscall.Sendto(s.fd, request.ToWireFormat(), 0, &s.lsa); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *NetlinkSocket) Recieve() ([]syscall.NetlinkMessage, error) {
+	rb := make([]byte, syscall.Getpagesize())
+	nr, _, err := syscall.Recvfrom(s.fd, rb, 0)
+	if err != nil {
+		return nil, err
+	}
+	if nr < syscall.NLMSG_HDRLEN {
+		return nil, fmt.Errorf("Got short response from netlink")
+	}
+	rb = rb[:nr]
+	return syscall.ParseNetlinkMessage(rb)
+}
+
+func (s *NetlinkSocket) GetPid() (uint32, error) {
+	lsa, err := syscall.Getsockname(s.fd)
+	if err != nil {
+		return 0, err
+	}
+	switch v := lsa.(type) {
+	case *syscall.SockaddrNetlink:
+		return v.Pid, nil
+	}
+	return 0, fmt.Errorf("Wrong socket type")
+}
+
+func (s *NetlinkSocket) HandleAck(seq uint32) error {
+	native := nativeEndian()
+
+	pid, err := s.GetPid()
+	if err != nil {
+		return err
+	}
+
+done:
+	for {
+		msgs, err := s.Recieve()
+		if err != nil {
+			return err
+		}
+		for _, m := range msgs {
+			if m.Header.Seq != seq {
+				return fmt.Errorf("Wrong Seq nr %d, expected %d", m.Header.Seq, seq)
+			}
+			if m.Header.Pid != pid {
+				return fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid)
+			}
+			if m.Header.Type == syscall.NLMSG_DONE {
+				break done
+			}
+			if m.Header.Type == syscall.NLMSG_ERROR {
+				error := int32(native.Uint32(m.Data[0:4]))
+				if error == 0 {
+					break done
+				}
+				return syscall.Errno(-error)
+			}
+		}
+	}
+
+	return nil
+}
+
+// Add a new default gateway. Identical to:
+// ip route add default via $ip
+func AddDefaultGw(ip net.IP) error {
+	s, err := getNetlinkSocket()
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	family := getIpFamily(ip)
+
+	wb := newNetlinkRequest(syscall.RTM_NEWROUTE, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK)
+
+	msg := newRtMsg(family)
+	wb.AddData(msg)
+
+	var ipData []byte
+	if family == syscall.AF_INET {
+		ipData = ip.To4()
+	} else {
+		ipData = ip.To16()
+	}
+
+	gateway := newRtAttr(syscall.RTA_GATEWAY, ipData)
+
+	wb.AddData(gateway)
+
+	if err := s.Send(wb); err != nil {
+		return err
+	}
+
+	return s.HandleAck(wb.Seq)
+
+}
+
+// Bring up a particular network interface
+func NetworkLinkUp(iface *net.Interface) error {
+	s, err := getNetlinkSocket()
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	wb := newNetlinkRequest(syscall.RTM_NEWLINK, syscall.NLM_F_ACK)
+
+	msg := newIfInfomsg(syscall.AF_UNSPEC)
+	msg.Change = syscall.IFF_UP
+	msg.Flags = syscall.IFF_UP
+	msg.Index = int32(iface.Index)
+	wb.AddData(msg)
+
+	if err := s.Send(wb); err != nil {
+		return err
+	}
+
+	return s.HandleAck(wb.Seq)
+}
+
+// Add an Ip address to an interface. This is identical to:
+// ip addr add $ip/$ipNet dev $iface
+func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error {
+	s, err := getNetlinkSocket()
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	family := getIpFamily(ip)
+
+	wb := newNetlinkRequest(syscall.RTM_NEWADDR, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK)
+
+	msg := newIfAddrmsg(family)
+	msg.Index = uint32(iface.Index)
+	prefixLen, _ := ipNet.Mask.Size()
+	msg.Prefixlen = uint8(prefixLen)
+	wb.AddData(msg)
+
+	var ipData []byte
+	if family == syscall.AF_INET {
+		ipData = ip.To4()
+	} else {
+		ipData = ip.To16()
+	}
+
+	localData := newRtAttr(syscall.IFA_LOCAL, ipData)
+	wb.AddData(localData)
+
+	addrData := newRtAttr(syscall.IFA_ADDRESS, ipData)
+	wb.AddData(addrData)
+
+	if err := s.Send(wb); err != nil {
+		return err
+	}
+
+	return s.HandleAck(wb.Seq)
+}
+
+func zeroTerminated(s string) []byte {
+	bytes := make([]byte, len(s)+1)
+	for i := 0; i < len(s); i++ {
+		bytes[i] = s[i]
+	}
+	bytes[len(s)] = 0
+	return bytes
+}
+
+func nonZeroTerminated(s string) []byte {
+	bytes := make([]byte, len(s))
+	for i := 0; i < len(s); i++ {
+		bytes[i] = s[i]
+	}
+	return bytes
+}
+
+// Add a new network link of a specified type. This is identical to
+// running: ip add link $name type $linkType
+func NetworkLinkAdd(name string, linkType string) error {
+	s, err := getNetlinkSocket()
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	wb := newNetlinkRequest(syscall.RTM_NEWLINK, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK)
+
+	msg := newIfInfomsg(syscall.AF_UNSPEC)
+	wb.AddData(msg)
+
+	nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name))
+	wb.AddData(nameData)
+
+	IFLA_INFO_KIND := 1
+
+	kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated(linkType))
+
+	infoData := newRtAttr(syscall.IFLA_LINKINFO, kindData.ToWireFormat())
+	wb.AddData(infoData)
+
+	if err := s.Send(wb); err != nil {
+		return err
+	}
+
+	return s.HandleAck(wb.Seq)
+}
+
+// Returns an array of IPNet for all the currently routed subnets on ipv4
+// This is similar to the first column of "ip route" output
+func NetworkGetRoutes() ([]*net.IPNet, error) {
+	native := nativeEndian()
+
+	s, err := getNetlinkSocket()
+	if err != nil {
+		return nil, err
+	}
+	defer s.Close()
+
+	wb := newNetlinkRequest(syscall.RTM_GETROUTE, syscall.NLM_F_DUMP)
+
+	msg := newIfInfomsg(syscall.AF_UNSPEC)
+	wb.AddData(msg)
+
+	if err := s.Send(wb); err != nil {
+		return nil, err
+	}
+
+	pid, err := s.GetPid()
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]*net.IPNet, 0)
+
+done:
+	for {
+		msgs, err := s.Recieve()
+		if err != nil {
+			return nil, err
+		}
+		for _, m := range msgs {
+			if m.Header.Seq != wb.Seq {
+				return nil, fmt.Errorf("Wrong Seq nr %d, expected 1", m.Header.Seq)
+			}
+			if m.Header.Pid != pid {
+				return nil, fmt.Errorf("Wrong pid %d, expected %d", m.Header.Pid, pid)
+			}
+			if m.Header.Type == syscall.NLMSG_DONE {
+				break done
+			}
+			if m.Header.Type == syscall.NLMSG_ERROR {
+				error := int32(native.Uint32(m.Data[0:4]))
+				if error == 0 {
+					break done
+				}
+				return nil, syscall.Errno(-error)
+			}
+			if m.Header.Type != syscall.RTM_NEWROUTE {
+				continue
+			}
+
+			var iface *net.Interface = nil
+			var ipNet *net.IPNet = nil
+
+			msg := (*RtMsg)(unsafe.Pointer(&m.Data[0:syscall.SizeofRtMsg][0]))
+
+			if msg.Flags&syscall.RTM_F_CLONED != 0 {
+				// Ignore cloned routes
+				continue
+			}
+
+			if msg.Table != syscall.RT_TABLE_MAIN {
+				// Ignore non-main tables
+				continue
+			}
+
+			if msg.Family != syscall.AF_INET {
+				// Ignore non-ipv4 routes
+				continue
+			}
+
+			if msg.Dst_len == 0 {
+				// Ignore default routes
+				continue
+			}
+
+			attrs, err := syscall.ParseNetlinkRouteAttr(&m)
+			if err != nil {
+				return nil, err
+			}
+			for _, attr := range attrs {
+				switch attr.Attr.Type {
+				case syscall.RTA_DST:
+					ip := attr.Value
+					ipNet = &net.IPNet{
+						IP:   ip,
+						Mask: net.CIDRMask(int(msg.Dst_len), 8*len(ip)),
+					}
+				case syscall.RTA_OIF:
+					index := int(native.Uint32(attr.Value[0:4]))
+					iface, _ = net.InterfaceByIndex(index)
+					_ = iface
+				}
+			}
+			if ipNet != nil {
+				res = append(res, ipNet)
+			}
+		}
+	}
+
+	return res, nil
+}

network.go

@@ -4,17 +4,16 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"github.com/dotcloud/docker/iptables"
+	"github.com/dotcloud/docker/netlink"
+	"github.com/dotcloud/docker/proxy"
 	"github.com/dotcloud/docker/utils"
 	"log"
 	"net"
-	"os/exec"
 	"strconv"
-	"strings"
 	"sync"
 )
 
-var NetworkBridgeIface string
-
 const (
 	DefaultNetworkBridge = "docker0"
 	DisableNetworkBridge = "none"
@@ -68,54 +67,10 @@ func networkSize(mask net.IPMask) int32 {
 	return int32(binary.BigEndian.Uint32(m)) + 1
 }
 
-//Wrapper around the ip command
-func ip(args ...string) (string, error) {
-	path, err := exec.LookPath("ip")
-	if err != nil {
-		return "", fmt.Errorf("command not found: ip")
-	}
-	output, err := exec.Command(path, args...).CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("ip failed: ip %v", strings.Join(args, " "))
-	}
-	return string(output), nil
-}
-
-// Wrapper around the iptables command
-func iptables(args ...string) error {
-	path, err := exec.LookPath("iptables")
-	if err != nil {
-		return fmt.Errorf("command not found: iptables")
-	}
-	if err := exec.Command(path, args...).Run(); err != nil {
-		return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " "))
-	}
-	return nil
-}
-
-func checkRouteOverlaps(routes string, dockerNetwork *net.IPNet) error {
-	utils.Debugf("Routes:\n\n%s", routes)
-	for _, line := range strings.Split(routes, "\n") {
-		if strings.Trim(line, "\r\n\t ") == "" || strings.Contains(line, "default") {
-			continue
-		}
-		_, network, err := net.ParseCIDR(strings.Split(line, " ")[0])
-		if err != nil {
-			// is this a mask-less IP address?
-			if ip := net.ParseIP(strings.Split(line, " ")[0]); ip == nil {
-				// fail only if it's neither a network nor a mask-less IP address
-				return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line)
-			} else {
-				_, network, err = net.ParseCIDR(ip.String() + "/32")
-				if err != nil {
-					return err
-				}
-			}
-		}
-		if err == nil && network != nil {
-			if networkOverlaps(dockerNetwork, network) {
-				return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork, line)
-			}
+func checkRouteOverlaps(networks []*net.IPNet, dockerNetwork *net.IPNet) error {
+	for _, network := range networks {
+		if networkOverlaps(dockerNetwork, network) {
+			return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork, network)
 		}
 	}
 	return nil
@@ -124,7 +79,7 @@ func checkRouteOverlaps(routes string, dockerNetwork *net.IPNet) error {
 // CreateBridgeIface creates a network bridge interface on the host system with the name `ifaceName`,
 // and attempts to configure it with an address which doesn't conflict with any other interface on the host.
 // If it can't find an address which doesn't conflict, it will return an error.
-func CreateBridgeIface(ifaceName string) error {
+func CreateBridgeIface(config *DaemonConfig) error {
 	addrs := []string{
 		// Here we don't follow the convention of using the 1st IP of the range for the gateway.
 		// This is to use the same gateway IPs as the /24 ranges, which predate the /16 ranges.
@@ -151,7 +106,7 @@ func CreateBridgeIface(ifaceName string) error {
 		if err != nil {
 			return err
 		}
-		routes, err := ip("route")
+		routes, err := netlink.NetworkGetRoutes()
 		if err != nil {
 			return err
 		}
@@ -163,23 +118,43 @@ func CreateBridgeIface(ifaceName string) error {
 		}
 	}
 	if ifaceAddr == "" {
-		return fmt.Errorf("Could not find a free IP address range for interface '%s'. Please configure its address manually and run 'docker -b %s'", ifaceName, ifaceName)
+		return fmt.Errorf("Could not find a free IP address range for interface '%s'. Please configure its address manually and run 'docker -b %s'", config.BridgeIface, config.BridgeIface)
 	}
-	utils.Debugf("Creating bridge %s with network %s", ifaceName, ifaceAddr)
+	utils.Debugf("Creating bridge %s with network %s", config.BridgeIface, ifaceAddr)
 
-	if output, err := ip("link", "add", ifaceName, "type", "bridge"); err != nil {
-		return fmt.Errorf("Error creating bridge: %s (output: %s)", err, output)
+	if err := netlink.NetworkLinkAdd(config.BridgeIface, "bridge"); err != nil {
+		return fmt.Errorf("Error creating bridge: %s", err)
+	}
+	iface, err := net.InterfaceByName(config.BridgeIface)
+	if err != nil {
+		return err
+	}
+	ipAddr, ipNet, err := net.ParseCIDR(ifaceAddr)
+	if err != nil {
+		return err
+	}
+	if netlink.NetworkLinkAddIp(iface, ipAddr, ipNet); err != nil {
+		return fmt.Errorf("Unable to add private network: %s", err)
+	}
+	if err := netlink.NetworkLinkUp(iface); err != nil {
+		return fmt.Errorf("Unable to start network bridge: %s", err)
 	}
 
-	if output, err := ip("addr", "add", ifaceAddr, "dev", ifaceName); err != nil {
-		return fmt.Errorf("Unable to add private network: %s (%s)", err, output)
-	}
-	if output, err := ip("link", "set", ifaceName, "up"); err != nil {
-		return fmt.Errorf("Unable to start network bridge: %s (%s)", err, output)
-	}
-	if err := iptables("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr,
-		"!", "-d", ifaceAddr, "-j", "MASQUERADE"); err != nil {
-		return fmt.Errorf("Unable to enable network bridge NAT: %s", err)
+	if config.EnableIptables {
+		if err := iptables.Raw("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr,
+			"!", "-d", ifaceAddr, "-j", "MASQUERADE"); err != nil {
+			return fmt.Errorf("Unable to enable network bridge NAT: %s", err)
+		}
+
+		if !config.InterContainerCommunication {
+			utils.Debugf("Disable inter-container communication")
+			if err := iptables.Raw("-A", "FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j", "DROP"); err != nil {
+				return fmt.Errorf("Unable to prevent intercontainer communication: %s", err)
+			}
+		} else {
+			utils.Debugf("Enable inter-container communication")
+			iptables.Raw("-D", "FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j", "DROP")
+		}
 	}
 	return nil
 }
@@ -216,58 +191,27 @@ func getIfaceAddr(name string) (net.Addr, error) {
 // It keeps track of all mappings and is able to unmap at will
 type PortMapper struct {
 	tcpMapping map[int]*net.TCPAddr
-	tcpProxies map[int]Proxy
+	tcpProxies map[int]proxy.Proxy
 	udpMapping map[int]*net.UDPAddr
-	udpProxies map[int]Proxy
+	udpProxies map[int]proxy.Proxy
+
+	iptables  *iptables.Chain
+	defaultIp net.IP
 }
 
-func (mapper *PortMapper) cleanup() error {
-	// Ignore errors - This could mean the chains were never set up
-	iptables("-t", "nat", "-D", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER")
-	iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8", "-j", "DOCKER")
-	iptables("-t", "nat", "-D", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER") // Created in versions <= 0.1.6
-	// Also cleanup rules created by older versions, or -X might fail.
-	iptables("-t", "nat", "-D", "PREROUTING", "-j", "DOCKER")
-	iptables("-t", "nat", "-D", "OUTPUT", "-j", "DOCKER")
-	iptables("-t", "nat", "-F", "DOCKER")
-	iptables("-t", "nat", "-X", "DOCKER")
-	mapper.tcpMapping = make(map[int]*net.TCPAddr)
-	mapper.tcpProxies = make(map[int]Proxy)
-	mapper.udpMapping = make(map[int]*net.UDPAddr)
-	mapper.udpProxies = make(map[int]Proxy)
-	return nil
-}
-
-func (mapper *PortMapper) setup() error {
-	if err := iptables("-t", "nat", "-N", "DOCKER"); err != nil {
-		return fmt.Errorf("Failed to create DOCKER chain: %s", err)
-	}
-	if err := iptables("-t", "nat", "-A", "PREROUTING", "-m", "addrtype", "--dst-type", "LOCAL", "-j", "DOCKER"); err != nil {
-		return fmt.Errorf("Failed to inject docker in PREROUTING chain: %s", err)
-	}
-	if err := iptables("-t", "nat", "-A", "OUTPUT", "-m", "addrtype", "--dst-type", "LOCAL", "!", "--dst", "127.0.0.0/8", "-j", "DOCKER"); err != nil {
-		return fmt.Errorf("Failed to inject docker in OUTPUT chain: %s", err)
-	}
-	return nil
-}
-
-func (mapper *PortMapper) iptablesForward(rule string, port int, proto string, dest_addr string, dest_port int) error {
-	return iptables("-t", "nat", rule, "DOCKER", "-p", proto, "--dport", strconv.Itoa(port),
-		"!", "-i", NetworkBridgeIface,
-		"-j", "DNAT", "--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port)))
-}
-
-func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error {
+func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error {
 	if _, isTCP := backendAddr.(*net.TCPAddr); isTCP {
 		backendPort := backendAddr.(*net.TCPAddr).Port
 		backendIP := backendAddr.(*net.TCPAddr).IP
-		if err := mapper.iptablesForward("-A", port, "tcp", backendIP.String(), backendPort); err != nil {
-			return err
+		if mapper.iptables != nil {
+			if err := mapper.iptables.Forward(iptables.Add, ip, port, "tcp", backendIP.String(), backendPort); err != nil {
+				return err
+			}
 		}
 		mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr)
-		proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr)
+		proxy, err := proxy.NewProxy(&net.TCPAddr{IP: ip, Port: port}, backendAddr)
 		if err != nil {
-			mapper.Unmap(port, "tcp")
+			mapper.Unmap(ip, port, "tcp")
 			return err
 		}
 		mapper.tcpProxies[port] = proxy
@@ -275,13 +219,15 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error {
 	} else {
 		backendPort := backendAddr.(*net.UDPAddr).Port
 		backendIP := backendAddr.(*net.UDPAddr).IP
-		if err := mapper.iptablesForward("-A", port, "udp", backendIP.String(), backendPort); err != nil {
-			return err
+		if mapper.iptables != nil {
+			if err := mapper.iptables.Forward(iptables.Add, ip, port, "udp", backendIP.String(), backendPort); err != nil {
+				return err
+			}
 		}
 		mapper.udpMapping[port] = backendAddr.(*net.UDPAddr)
-		proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr)
+		proxy, err := proxy.NewProxy(&net.UDPAddr{IP: ip, Port: port}, backendAddr)
 		if err != nil {
-			mapper.Unmap(port, "udp")
+			mapper.Unmap(ip, port, "udp")
 			return err
 		}
 		mapper.udpProxies[port] = proxy
@@ -290,7 +236,7 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error {
 	return nil
 }
 
-func (mapper *PortMapper) Unmap(port int, proto string) error {
+func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error {
 	if proto == "tcp" {
 		backendAddr, ok := mapper.tcpMapping[port]
 		if !ok {
@@ -300,8 +246,10 @@ func (mapper *PortMapper) Unmap(port int, proto string) error {
 			proxy.Close()
 			delete(mapper.tcpProxies, port)
 		}
-		if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
-			return err
+		if mapper.iptables != nil {
+			if err := mapper.iptables.Forward(iptables.Delete, ip, port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
+				return err
+			}
 		}
 		delete(mapper.tcpMapping, port)
 	} else {
@@ -313,21 +261,37 @@ func (mapper *PortMapper) Unmap(port int, proto string) error {
 			proxy.Close()
 			delete(mapper.udpProxies, port)
 		}
-		if err := mapper.iptablesForward("-D", port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
-			return err
+		if mapper.iptables != nil {
+			if err := mapper.iptables.Forward(iptables.Delete, ip, port, proto, backendAddr.IP.String(), backendAddr.Port); err != nil {
+				return err
+			}
 		}
 		delete(mapper.udpMapping, port)
 	}
 	return nil
 }
 
-func newPortMapper() (*PortMapper, error) {
-	mapper := &PortMapper{}
-	if err := mapper.cleanup(); err != nil {
+func newPortMapper(config *DaemonConfig) (*PortMapper, error) {
+	// We can always try removing the iptables
+	if err := iptables.RemoveExistingChain("DOCKER"); err != nil {
 		return nil, err
 	}
-	if err := mapper.setup(); err != nil {
-		return nil, err
+	var chain *iptables.Chain
+	if config.EnableIptables {
+		var err error
+		chain, err = iptables.NewChain("DOCKER", config.BridgeIface)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to create DOCKER chain: %s", err)
+		}
+	}
+
+	mapper := &PortMapper{
+		tcpMapping: make(map[int]*net.TCPAddr),
+		tcpProxies: make(map[int]proxy.Proxy),
+		udpMapping: make(map[int]*net.UDPAddr),
+		udpProxies: make(map[int]proxy.Proxy),
+		iptables:   chain,
+		defaultIp:  config.DefaultIp,
 	}
 	return mapper, nil
 }
@@ -519,40 +483,56 @@ type NetworkInterface struct {
 	disabled bool
 }
 
-// Allocate an external TCP port and map it to the interface
-func (iface *NetworkInterface) AllocatePort(spec string) (*Nat, error) {
+// Allocate an external port and map it to the interface
+func (iface *NetworkInterface) AllocatePort(port Port, binding PortBinding) (*Nat, error) {
 
 	if iface.disabled {
 		return nil, fmt.Errorf("Trying to allocate port for interface %v, which is disabled", iface) // FIXME
 	}
 
-	nat, err := parseNat(spec)
+	ip := iface.manager.portMapper.defaultIp
+
+	if binding.HostIp != "" {
+		ip = net.ParseIP(binding.HostIp)
+	} else {
+		binding.HostIp = ip.String()
+	}
+
+	nat := &Nat{
+		Port:    port,
+		Binding: binding,
+	}
+
+	containerPort, err := parsePort(port.Port())
 	if err != nil {
 		return nil, err
 	}
 
-	if nat.Proto == "tcp" {
-		extPort, err := iface.manager.tcpPortAllocator.Acquire(nat.Frontend)
+	hostPort, _ := parsePort(nat.Binding.HostPort)
+
+	if nat.Port.Proto() == "tcp" {
+		extPort, err := iface.manager.tcpPortAllocator.Acquire(hostPort)
 		if err != nil {
 			return nil, err
 		}
-		backend := &net.TCPAddr{IP: iface.IPNet.IP, Port: nat.Backend}
-		if err := iface.manager.portMapper.Map(extPort, backend); err != nil {
+
+		backend := &net.TCPAddr{IP: iface.IPNet.IP, Port: containerPort}
+		if err := iface.manager.portMapper.Map(ip, extPort, backend); err != nil {
 			iface.manager.tcpPortAllocator.Release(extPort)
 			return nil, err
 		}
-		nat.Frontend = extPort
+		nat.Binding.HostPort = strconv.Itoa(extPort)
 	} else {
-		extPort, err := iface.manager.udpPortAllocator.Acquire(nat.Frontend)
+		extPort, err := iface.manager.udpPortAllocator.Acquire(hostPort)
 		if err != nil {
 			return nil, err
 		}
-		backend := &net.UDPAddr{IP: iface.IPNet.IP, Port: nat.Backend}
-		if err := iface.manager.portMapper.Map(extPort, backend); err != nil {
+		backend := &net.UDPAddr{IP: iface.IPNet.IP, Port: containerPort}
+		if err := iface.manager.portMapper.Map(ip, extPort, backend); err != nil {
 			iface.manager.udpPortAllocator.Release(extPort)
 			return nil, err
 		}
-		nat.Frontend = extPort
+		nat.Binding.HostPort = strconv.Itoa(extPort)
 	}
 	iface.extPorts = append(iface.extPorts, nat)
 
@@ -560,83 +540,37 @@ func (iface *NetworkInterface) AllocatePort(spec string) (*Nat, error) {
 }
 
 type Nat struct {
-	Proto    string
-	Frontend int
-	Backend  int
+	Port    Port
+	Binding PortBinding
 }
 
-func parseNat(spec string) (*Nat, error) {
-	var nat Nat
-
-	if strings.Contains(spec, "/") {
-		specParts := strings.Split(spec, "/")
-		if len(specParts) != 2 {
-			return nil, fmt.Errorf("Invalid port format.")
-		}
-		proto := specParts[1]
-		spec = specParts[0]
-		if proto != "tcp" && proto != "udp" {
-			return nil, fmt.Errorf("Invalid port format: unknown protocol %v.", proto)
-		}
-		nat.Proto = proto
-	} else {
-		nat.Proto = "tcp"
-	}
-
-	if strings.Contains(spec, ":") {
-		specParts := strings.Split(spec, ":")
-		if len(specParts) != 2 {
-			return nil, fmt.Errorf("Invalid port format.")
-		}
-		// If spec starts with ':', external and internal ports must be the same.
-		// This might fail if the requested external port is not available.
-		var sameFrontend bool
-		if len(specParts[0]) == 0 {
-			sameFrontend = true
-		} else {
-			front, err := strconv.ParseUint(specParts[0], 10, 16)
-			if err != nil {
-				return nil, err
-			}
-			nat.Frontend = int(front)
-		}
-		back, err := strconv.ParseUint(specParts[1], 10, 16)
-		if err != nil {
-			return nil, err
-		}
-		nat.Backend = int(back)
-		if sameFrontend {
-			nat.Frontend = nat.Backend
-		}
-	} else {
-		port, err := strconv.ParseUint(spec, 10, 16)
-		if err != nil {
-			return nil, err
-		}
-		nat.Backend = int(port)
-	}
-
-	return &nat, nil
+func (n *Nat) String() string {
+	return fmt.Sprintf("%s:%d:%d/%s", n.Binding.HostIp, n.Binding.HostPort, n.Port.Port(), n.Port.Proto())
 }
 
 // Release: Network cleanup - release all resources
 func (iface *NetworkInterface) Release() {
-
 	if iface.disabled {
 		return
 	}
 
 	for _, nat := range iface.extPorts {
-		utils.Debugf("Unmaping %v/%v", nat.Proto, nat.Frontend)
-		if err := iface.manager.portMapper.Unmap(nat.Frontend, nat.Proto); err != nil {
-			log.Printf("Unable to unmap port %v/%v: %v", nat.Proto, nat.Frontend, err)
+		hostPort, err := parsePort(nat.Binding.HostPort)
+		if err != nil {
+			log.Printf("Unable to get host port: %s", err)
+			continue
 		}
-		if nat.Proto == "tcp" {
-			if err := iface.manager.tcpPortAllocator.Release(nat.Frontend); err != nil {
-				log.Printf("Unable to release port tcp/%v: %v", nat.Frontend, err)
+		ip := net.ParseIP(nat.Binding.HostIp)
+		utils.Debugf("Unmaping %s/%s", nat.Port.Proto, nat.Binding.HostPort)
+		if err := iface.manager.portMapper.Unmap(ip, hostPort, nat.Port.Proto()); err != nil {
+			log.Printf("Unable to unmap port %s: %s", nat, err)
+		}
+		if nat.Port.Proto() == "tcp" {
+			if err := iface.manager.tcpPortAllocator.Release(hostPort); err != nil {
+				log.Printf("Unable to release port %s", nat)
 			}
-		} else if err := iface.manager.udpPortAllocator.Release(nat.Frontend); err != nil {
-			log.Printf("Unable to release port udp/%v: %v", nat.Frontend, err)
+		} else if err := iface.manager.udpPortAllocator.Release(hostPort); err != nil {
+			log.Printf("Unable to release port %s: %s", nat, err)
 		}
 	}
 
@@ -704,22 +638,21 @@ func (manager *NetworkManager) Close() error {
 	return err3
 }
 
-func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
-
-	if bridgeIface == DisableNetworkBridge {
+func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) {
+	if config.BridgeIface == DisableNetworkBridge {
 		manager := &NetworkManager{
 			disabled: true,
 		}
 		return manager, nil
 	}
 
-	addr, err := getIfaceAddr(bridgeIface)
+	addr, err := getIfaceAddr(config.BridgeIface)
 	if err != nil {
 		// If the iface is not found, try to create it
-		if err := CreateBridgeIface(bridgeIface); err != nil {
+		if err := CreateBridgeIface(config); err != nil {
 			return nil, err
 		}
-		addr, err = getIfaceAddr(bridgeIface)
+		addr, err = getIfaceAddr(config.BridgeIface)
 		if err != nil {
 			return nil, err
 		}
@@ -737,13 +670,13 @@ func newNetworkManager(bridgeIface string) (*NetworkManager, error) {
 		return nil, err
 	}
 
-	portMapper, err := newPortMapper()
+	portMapper, err := newPortMapper(config)
 	if err != nil {
 		return nil, err
 	}
 
 	manager := &NetworkManager{
-		bridgeIface:      bridgeIface,
+		bridgeIface:      config.BridgeIface,
 		bridgeNetwork:    network,
 		ipAllocator:      ipAllocator,
 		tcpPortAllocator: tcpPortAllocator,

network_test.go

@@ -2,117 +2,9 @@ package docker
 
 import (
 	"net"
-	"os"
 	"testing"
 )
 
-func TestIptables(t *testing.T) {
-	if err := iptables("-L"); err != nil {
-		t.Fatal(err)
-	}
-	path := os.Getenv("PATH")
-	os.Setenv("PATH", "")
-	defer os.Setenv("PATH", path)
-	if err := iptables("-L"); err == nil {
-		t.Fatal("Not finding iptables in the PATH should cause an error")
-	}
-}
-
-func TestParseNat(t *testing.T) {
-	if nat, err := parseNat("4500"); err == nil {
-		if nat.Frontend != 0 || nat.Backend != 4500 || nat.Proto != "tcp" {
-			t.Errorf("-p 4500 should produce 0->4500/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat(":4501"); err == nil {
-		if nat.Frontend != 4501 || nat.Backend != 4501 || nat.Proto != "tcp" {
-			t.Errorf("-p :4501 should produce 4501->4501/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat("4502:4503"); err == nil {
-		if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" {
-			t.Errorf("-p 4502:4503 should produce 4502->4503/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat("4502:4503/tcp"); err == nil {
-		if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "tcp" {
-			t.Errorf("-p 4502:4503/tcp should produce 4502->4503/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat("4502:4503/udp"); err == nil {
-		if nat.Frontend != 4502 || nat.Backend != 4503 || nat.Proto != "udp" {
-			t.Errorf("-p 4502:4503/udp should produce 4502->4503/udp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat(":4503/udp"); err == nil {
-		if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "udp" {
-			t.Errorf("-p :4503/udp should produce 4503->4503/udp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat(":4503/tcp"); err == nil {
-		if nat.Frontend != 4503 || nat.Backend != 4503 || nat.Proto != "tcp" {
-			t.Errorf("-p :4503/tcp should produce 4503->4503/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat("4503/tcp"); err == nil {
-		if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "tcp" {
-			t.Errorf("-p 4503/tcp should produce 0->4503/tcp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if nat, err := parseNat("4503/udp"); err == nil {
-		if nat.Frontend != 0 || nat.Backend != 4503 || nat.Proto != "udp" {
-			t.Errorf("-p 4503/udp should produce 0->4503/udp, got %d->%d/%s",
-				nat.Frontend, nat.Backend, nat.Proto)
-		}
-	} else {
-		t.Fatal(err)
-	}
-
-	if _, err := parseNat("4503/tcpgarbage"); err == nil {
-		t.Fatal(err)
-	}
-
-	if _, err := parseNat("4503/tcp/udp"); err == nil {
-		t.Fatal(err)
-	}
-
-	if _, err := parseNat("4503/"); err == nil {
-		t.Fatal(err)
-	}
-}
-
 func TestPortAllocation(t *testing.T) {
 	allocator, err := newPortAllocator()
 	if err != nil {
@@ -385,12 +277,13 @@ func TestNetworkOverlaps(t *testing.T) {
 }
 
 func TestCheckRouteOverlaps(t *testing.T) {
-	routes := `default via 10.0.2.2 dev eth0
-10.0.2.0 dev eth0  proto kernel  scope link  src 10.0.2.15
-10.0.3.0/24 dev lxcbr0  proto kernel  scope link  src 10.0.3.1
-10.0.42.0/24 dev testdockbr0  proto kernel  scope link  src 10.0.42.1
-172.16.42.0/24 dev docker0  proto kernel  scope link  src 172.16.42.1
-192.168.142.0/24 dev eth1  proto kernel  scope link  src 192.168.142.142`
+	routesData := []string{"10.0.2.0/32", "10.0.3.0/24", "10.0.42.0/24", "172.16.42.0/24", "192.168.142.0/24"}
+
+	routes := []*net.IPNet{}
+	for _, addr := range routesData {
+		_, netX, _ := net.ParseCIDR(addr)
+		routes = append(routes, netX)
+	}
 
 	_, netX, _ := net.ParseCIDR("172.16.0.1/24")
 	if err := checkRouteOverlaps(routes, netX); err != nil {

proxy/MAINTAINERS

@@ -0,0 +1 @@
+Michael Crosby <michael@crosbymichael.com> (@crosbymichael)

proxy/network_proxy_test.go

@@ -1,4 +1,4 @@
-package docker
+package proxy
 
 import (
 	"bytes"

proxy/proxy.go

@@ -0,0 +1,29 @@
+package proxy
+
+import (
+	"fmt"
+	"net"
+)
+
+type Proxy interface {
+	// Start forwarding traffic back and forth the front and back-end
+	// addresses.
+	Run()
+	// Stop forwarding traffic and close both ends of the Proxy.
+	Close()
+	// Return the address on which the proxy is listening.
+	FrontendAddr() net.Addr
+	// Return the proxied address.
+	BackendAddr() net.Addr
+}
+
+func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
+	switch frontendAddr.(type) {
+	case *net.UDPAddr:
+		return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr))
+	case *net.TCPAddr:
+		return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr))
+	default:
+		panic(fmt.Errorf("Unsupported protocol"))
+	}
+}

proxy/tcp_proxy.go

@@ -0,0 +1,93 @@
+package proxy
+
+import (
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"log"
+	"net"
+	"syscall"
+)
+
+type TCPProxy struct {
+	listener     *net.TCPListener
+	frontendAddr *net.TCPAddr
+	backendAddr  *net.TCPAddr
+}
+
+func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
+	listener, err := net.ListenTCP("tcp", frontendAddr)
+	if err != nil {
+		return nil, err
+	}
+	// If the port in frontendAddr was 0 then ListenTCP will have a picked
+	// a port to listen on, hence the call to Addr to get that actual port:
+	return &TCPProxy{
+		listener:     listener,
+		frontendAddr: listener.Addr().(*net.TCPAddr),
+		backendAddr:  backendAddr,
+	}, nil
+}
+
+func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
+	backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
+	if err != nil {
+		log.Printf("Can't forward traffic to backend tcp/%v: %v\n", proxy.backendAddr, err.Error())
+		client.Close()
+		return
+	}
+
+	event := make(chan int64)
+	var broker = func(to, from *net.TCPConn) {
+		written, err := io.Copy(to, from)
+		if err != nil {
+			// If the socket we are writing to is shutdown with
+			// SHUT_WR, forward it to the other end of the pipe:
+			if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE {
+				from.CloseWrite()
+			}
+		}
+		to.CloseRead()
+		event <- written
+	}
+	utils.Debugf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr())
+	go broker(client, backend)
+	go broker(backend, client)
+
+	var transferred int64 = 0
+	for i := 0; i < 2; i++ {
+		select {
+		case written := <-event:
+			transferred += written
+		case <-quit:
+			// Interrupt the two brokers and "join" them.
+			client.Close()
+			backend.Close()
+			for ; i < 2; i++ {
+				transferred += <-event
+			}
+			goto done
+		}
+	}
+	client.Close()
+	backend.Close()
+done:
+	utils.Debugf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr())
+}
+
+func (proxy *TCPProxy) Run() {
+	quit := make(chan bool)
+	defer close(quit)
+	utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr)
+	for {
+		client, err := proxy.listener.Accept()
+		if err != nil {
+			utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error())
+			return
+		}
+		go proxy.clientLoop(client.(*net.TCPConn), quit)
+	}
+}
+
+func (proxy *TCPProxy) Close()                 { proxy.listener.Close() }
+func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
+func (proxy *TCPProxy) BackendAddr() net.Addr  { return proxy.backendAddr }

proxy/udp_proxy.go

@@ -1,10 +1,8 @@
-package docker
+package proxy
 
 import (
 	"encoding/binary"
-	"fmt"
 	"github.com/dotcloud/docker/utils"
-	"io"
 	"log"
 	"net"
 	"sync"
@@ -17,107 +15,6 @@ const (
 	UDPBufSize          = 2048
 )
 
-type Proxy interface {
-	// Start forwarding traffic back and forth the front and back-end
-	// addresses.
-	Run()
-	// Stop forwarding traffic and close both ends of the Proxy.
-	Close()
-	// Return the address on which the proxy is listening.
-	FrontendAddr() net.Addr
-	// Return the proxied address.
-	BackendAddr() net.Addr
-}
-
-type TCPProxy struct {
-	listener     *net.TCPListener
-	frontendAddr *net.TCPAddr
-	backendAddr  *net.TCPAddr
-}
-
-func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) {
-	listener, err := net.ListenTCP("tcp", frontendAddr)
-	if err != nil {
-		return nil, err
-	}
-	// If the port in frontendAddr was 0 then ListenTCP will have a picked
-	// a port to listen on, hence the call to Addr to get that actual port:
-	return &TCPProxy{
-		listener:     listener,
-		frontendAddr: listener.Addr().(*net.TCPAddr),
-		backendAddr:  backendAddr,
-	}, nil
-}
-
-func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) {
-	backend, err := net.DialTCP("tcp", nil, proxy.backendAddr)
-	if err != nil {
-		log.Printf("Can't forward traffic to backend tcp/%v: %v\n", proxy.backendAddr, err.Error())
-		client.Close()
-		return
-	}
-
-	event := make(chan int64)
-	var broker = func(to, from *net.TCPConn) {
-		written, err := io.Copy(to, from)
-		if err != nil {
-			// If the socket we are writing to is shutdown with
-			// SHUT_WR, forward it to the other end of the pipe:
-			if err, ok := err.(*net.OpError); ok && err.Err == syscall.EPIPE {
-				from.CloseWrite()
-			}
-		}
-		to.CloseRead()
-		event <- written
-	}
-	utils.Debugf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr())
-	go broker(client, backend)
-	go broker(backend, client)
-
-	var transferred int64 = 0
-	for i := 0; i < 2; i++ {
-		select {
-		case written := <-event:
-			transferred += written
-		case <-quit:
-			// Interrupt the two brokers and "join" them.
-			client.Close()
-			backend.Close()
-			for ; i < 2; i++ {
-				transferred += <-event
-			}
-			goto done
-		}
-	}
-	client.Close()
-	backend.Close()
-done:
-	utils.Debugf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr())
-}
-
-func (proxy *TCPProxy) Run() {
-	quit := make(chan bool)
-	defer close(quit)
-
-	utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr)
-	for {
-		client, err := proxy.listener.Accept()
-		if err != nil {
-			if utils.IsClosedError(err) {
-				utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr)
-			} else {
-				utils.Errorf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error())
-			}
-			return
-		}
-		go proxy.clientLoop(client.(*net.TCPConn), quit)
-	}
-}
-
-func (proxy *TCPProxy) Close()                 { proxy.listener.Close() }
-func (proxy *TCPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
-func (proxy *TCPProxy) BackendAddr() net.Addr  { return proxy.backendAddr }
-
 // A net.Addr where the IP is split into two fields so you can use it as a key
 // in a map:
 type connTrackKey struct {
@@ -253,14 +150,3 @@ func (proxy *UDPProxy) Close() {
 
 func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr }
 func (proxy *UDPProxy) BackendAddr() net.Addr  { return proxy.backendAddr }
-
-func NewProxy(frontendAddr, backendAddr net.Addr) (Proxy, error) {
-	switch frontendAddr.(type) {
-	case *net.UDPAddr:
-		return NewUDPProxy(frontendAddr.(*net.UDPAddr), backendAddr.(*net.UDPAddr))
-	case *net.TCPAddr:
-		return NewTCPProxy(frontendAddr.(*net.TCPAddr), backendAddr.(*net.TCPAddr))
-	default:
-		panic(fmt.Errorf("Unsupported protocol"))
-	}
-}

runtime.go

@@ -3,6 +3,8 @@ package docker
 import (
 	"container/list"
 	"fmt"
+	"github.com/dotcloud/docker/devmapper"
+	"github.com/dotcloud/docker/gograph"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -10,11 +12,16 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 )
 
+const (
+	DefaultFilesystemType = "devicemapper"
+)
+
 var defaultDns = []string{"8.8.8.8", "8.8.4.4"}
 
 type Capabilities struct {
@@ -24,7 +31,6 @@ type Capabilities struct {
 }
 
 type Runtime struct {
-	root           string
 	repository     string
 	containers     *list.List
 	networkManager *NetworkManager
@@ -33,16 +39,33 @@ type Runtime struct {
 	idIndex        *utils.TruncIndex
 	capabilities   *Capabilities
 	kernelVersion  *utils.KernelVersionInfo
-	autoRestart    bool
 	volumes        *Graph
 	srv            *Server
 	Dns            []string
+	deviceSet      *devmapper.DeviceSet
+	config         *DaemonConfig
+	containerGraph *gograph.Database
 }
 
 var sysInitPath string
 
 func init() {
-	sysInitPath = utils.SelfPath()
+	env := os.Getenv("_DOCKER_INIT_PATH")
+	if env != "" {
+		sysInitPath = env
+	} else {
+		selfPath := utils.SelfPath()
+
+		// If we have a separate dockerinit, use that, otherwise use the
+		// main docker binary
+		dir := filepath.Dir(selfPath)
+		dockerInitPath := filepath.Join(dir, "dockerinit")
+		if _, err := os.Stat(dockerInitPath); err != nil {
+			sysInitPath = selfPath
+		} else {
+			sysInitPath = dockerInitPath
+		}
+	}
 }
 
 // List returns an array of all containers registered in the runtime.
@@ -64,13 +87,28 @@ func (runtime *Runtime) getContainerElement(id string) *list.Element {
 	return nil
 }
 
+func (runtime *Runtime) GetDeviceSet() (*devmapper.DeviceSet, error) {
+	if runtime.deviceSet == nil {
+		return nil, fmt.Errorf("No device set available")
+	}
+	return runtime.deviceSet, nil
+}
+
 // Get looks for a container by the specified ID or name, and returns it.
 // If the container is not found, or if an error occurs, nil is returned.
 func (runtime *Runtime) Get(name string) *Container {
+	if name[0] != '/' {
+		name = "/" + name
+	}
+	if c, _ := runtime.GetByName(name); c != nil {
+		return c
+	}
+
 	id, err := runtime.idIndex.Get(name)
 	if err != nil {
 		return nil
 	}
+
 	e := runtime.getContainerElement(id)
 	if e == nil {
 		return nil
@@ -88,10 +126,9 @@ func (runtime *Runtime) containerRoot(id string) string {
 	return path.Join(runtime.repository, id)
 }
 
-// Load reads the contents of a container from disk and registers
-// it with Register.
+// Load reads the contents of a container from disk
 // This is typically done at startup.
-func (runtime *Runtime) Load(id string) (*Container, error) {
+func (runtime *Runtime) load(id string) (*Container, error) {
 	container := &Container{root: runtime.containerRoot(id)}
 	if err := container.FromDisk(); err != nil {
 		return nil, err
@@ -102,9 +139,6 @@ func (runtime *Runtime) Load(id string) (*Container, error) {
 	if container.State.Running {
 		container.State.Ghost = true
 	}
-	if err := runtime.Register(container); err != nil {
-		return nil, err
-	}
 	return container, nil
 }
 
@@ -149,11 +183,11 @@ func (runtime *Runtime) Register(container *Container) error {
 		}
 		if !strings.Contains(string(output), "RUNNING") {
 			utils.Debugf("Container %s was supposed to be running be is not.", container.ID)
-			if runtime.autoRestart {
+			if runtime.config.AutoRestart {
 				utils.Debugf("Restarting")
 				container.State.Ghost = false
 				container.State.setStopped(0)
-				hostConfig := &HostConfig{}
+				hostConfig, _ := container.ReadHostConfig()
 				if err := container.Start(hostConfig); err != nil {
 					return err
 				}
@@ -173,9 +207,9 @@ func (runtime *Runtime) Register(container *Container) error {
 	if !container.State.Running {
 		close(container.waitLock)
 	} else if !nomonitor {
-		container.allocateNetwork()
-		// hostConfig isn't needed here and can be nil
-		go container.monitor(nil)
+		hostConfig, _ := container.ReadHostConfig()
+		container.allocateNetwork(hostConfig)
+		go container.monitor(hostConfig)
 	}
 	return nil
 }
@@ -203,6 +237,7 @@ func (runtime *Runtime) Destroy(container *Container) error {
 	if err := container.Stop(3); err != nil {
 		return err
 	}
+
 	if mounted, err := container.Mounted(); err != nil {
 		return err
 	} else if mounted {
@@ -210,12 +245,35 @@ func (runtime *Runtime) Destroy(container *Container) error {
 			return fmt.Errorf("Unable to unmount container %v: %v", container.ID, err)
 		}
 	}
+
+	if _, err := runtime.containerGraph.Purge(container.ID); err != nil {
+		utils.Debugf("Unable to remove container from link graph: %s", err)
+	}
+
 	// Deregister the container before removing its directory, to avoid race conditions
 	runtime.idIndex.Delete(container.ID)
 	runtime.containers.Remove(element)
 	if err := os.RemoveAll(container.root); err != nil {
 		return fmt.Errorf("Unable to remove filesystem for %v: %v", container.ID, err)
 	}
+	if runtime.deviceSet.HasDevice(container.ID) {
+		if err := runtime.deviceSet.RemoveDevice(container.ID); err != nil {
+			return fmt.Errorf("Unable to remove device for %v: %v", container.ID, err)
+		}
+	}
+	return nil
+}
+
+func (runtime *Runtime) DeleteImage(id string) error {
+	err := runtime.graph.Delete(id)
+	if err != nil {
+		return err
+	}
+	if runtime.deviceSet.HasDevice(id) {
+		if err := runtime.deviceSet.RemoveDevice(id); err != nil {
+			return fmt.Errorf("Unable to remove device for %v: %v", id, err)
+		}
+	}
 	return nil
 }
 
@@ -228,9 +286,15 @@ func (runtime *Runtime) restore() error {
 	if err != nil {
 		return err
 	}
+
+	var (
+		containers          []*Container
+		containersToMigrate []*Container
+	)
+
 	for i, v := range dir {
 		id := v.Name()
-		container, err := runtime.Load(id)
+		container, err := runtime.load(id)
 		if i%21 == 0 && os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" {
 			fmt.Printf("\b%c", wheel[i%4])
 		}
@@ -239,10 +303,110 @@ func (runtime *Runtime) restore() error {
 			continue
 		}
 		utils.Debugf("Loaded container %v", container.ID)
+		containers = append(containers, container)
+
+		if container.FilesystemType != DefaultFilesystemType {
+			containersToMigrate = append(containersToMigrate, container)
+		}
+	}
+
+	// Migrate containers to the default filesystem type
+	if len(containersToMigrate) > 0 {
+		if err := migrateToDeviceMapper(runtime, containersToMigrate); err != nil {
+			return err
+		}
+	}
+
+	sortContainers(containers, func(i, j *Container) bool {
+		ic, _ := i.ReadHostConfig()
+		jc, _ := j.ReadHostConfig()
+
+		if ic == nil || ic.Links == nil {
+			return true
+		}
+		if jc == nil || jc.Links == nil {
+			return false
+		}
+		return len(ic.Links) < len(jc.Links)
+	})
+	for _, container := range containers {
+		if err := runtime.Register(container); err != nil {
+			utils.Debugf("Failed to register container %s: %s", container.ID, err)
+			continue
+		}
 	}
 	if os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" {
 		fmt.Printf("\bdone.\n")
 	}
+
+	return nil
+}
+
+func migrateToDeviceMapper(runtime *Runtime, containers []*Container) error {
+	var (
+		image    *Image
+		contents []os.FileInfo
+		err      error
+	)
+
+	fmt.Printf("Migrating %d containers to new storage backend\n", len(containers))
+	for _, container := range containers {
+		if container.State.Running {
+			fmt.Printf("WARNING - Cannot migrate %s because the container is running.  Please stop the container and relaunch the daemon!")
+			continue
+		}
+
+		fmt.Printf("Migrating %s\n", container.ID)
+
+		if contents, err = ioutil.ReadDir(container.rwPath()); err != nil {
+			if !os.IsNotExist(err) {
+				fmt.Printf("Error reading rw dir %s\n", err)
+			}
+			continue
+		}
+
+		if len(contents) == 0 {
+			fmt.Printf("Skipping migration of %s because rw layer contains no changes\n")
+			continue
+		}
+
+		if image, err = runtime.graph.Get(container.Image); err != nil {
+			fmt.Printf("Failed to fetch image %s\n", err)
+			continue
+		}
+
+		unmount := func() {
+			if err = image.Unmount(runtime, container.RootfsPath(), container.ID); err != nil {
+				fmt.Printf("Failed to unmount image %s\n", err)
+			}
+		}
+
+		if err = image.Mount(runtime, container.RootfsPath(), container.rwPath(), container.ID); err != nil {
+			fmt.Printf("Failed to mount image %s\n", err)
+			continue
+		}
+
+		if err = image.applyLayer(container.rwPath(), container.RootfsPath()); err != nil {
+			fmt.Printf("Failed to apply layer in storage backend %s\n", err)
+			unmount()
+			continue
+		}
+
+		unmount()
+
+		if err = os.RemoveAll(container.rwPath()); err != nil {
+			fmt.Printf("Failed to remove rw layer %s\n", err)
+		}
+
+		container.FilesystemType = DefaultFilesystemType
+		if err := container.ToDisk(); err != nil {
+			fmt.Printf("Failed to save filesystem type to disk %s\n", err)
+		}
+
+		fmt.Printf("Successful migration for %s\n", container.ID)
+	}
+	fmt.Printf("Migration complete\n")
+
 	return nil
 }
 
@@ -275,27 +439,44 @@ func (runtime *Runtime) UpdateCapabilities(quiet bool) {
 }
 
 // Create creates a new container from the given configuration.
-func (runtime *Runtime) Create(config *Config) (*Container, error) {
+func (runtime *Runtime) Create(config *Config) (*Container, []string, error) {
 	// Lookup image
 	img, err := runtime.repositories.LookupImage(config.Image)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
+	warnings := []string{}
 	if img.Config != nil {
+		if img.Config.PortSpecs != nil && warnings != nil {
+			for _, p := range img.Config.PortSpecs {
+				if strings.Contains(p, ":") {
+					warnings = append(warnings, "This image expects private ports to be mapped to public ports on your host. "+
+						"This has been deprecated and the public mappings will not be honored."+
+						"Use -p to publish the ports.")
+					break
+				}
+			}
+		}
 		if err := MergeConfig(config, img.Config); err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
 	if len(config.Entrypoint) != 0 && config.Cmd == nil {
 		config.Cmd = []string{}
 	} else if config.Cmd == nil || len(config.Cmd) == 0 {
-		return nil, fmt.Errorf("No command specified")
+		return nil, nil, fmt.Errorf("No command specified")
 	}
 
 	// Generate id
 	id := GenerateID()
+
+	// Set the default enitity in the graph
+	if _, err := runtime.containerGraph.Set(fmt.Sprintf("/%s", id), id); err != nil {
+		return nil, nil, err
+	}
+
 	// Generate default hostname
 	// FIXME: the lxc template no longer needs to set a default hostname
 	if config.Hostname == "" {
@@ -323,42 +504,43 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) {
 		Image:           img.ID, // Always use the resolved image id
 		NetworkSettings: &NetworkSettings{},
 		// FIXME: do we need to store this in the container?
-		SysInitPath: sysInitPath,
+		SysInitPath:    sysInitPath,
+		FilesystemType: DefaultFilesystemType,
 	}
 	container.root = runtime.containerRoot(container.ID)
 	// Step 1: create the container directory.
 	// This doubles as a barrier to avoid race conditions.
 	if err := os.Mkdir(container.root, 0700); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	resolvConf, err := utils.GetResolvConf()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	if len(config.Dns) == 0 && len(runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
+	if len(config.Dns) == 0 && len(runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
 		//"WARNING: Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns
-		runtime.Dns = defaultDns
+		runtime.config.Dns = defaultDns
 	}
 
 	// If custom dns exists, then create a resolv.conf for the container
-	if len(config.Dns) > 0 || len(runtime.Dns) > 0 {
+	if len(config.Dns) > 0 || len(runtime.config.Dns) > 0 {
 		var dns []string
 		if len(config.Dns) > 0 {
 			dns = config.Dns
 		} else {
-			dns = runtime.Dns
+			dns = runtime.config.Dns
 		}
 		container.ResolvConfPath = path.Join(container.root, "resolv.conf")
 		f, err := os.Create(container.ResolvConfPath)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		defer f.Close()
 		for _, dns := range dns {
 			if _, err := f.Write([]byte("nameserver " + dns + "\n")); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 	} else {
@@ -367,7 +549,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) {
 
 	// Step 2: save the container json
 	if err := container.ToDisk(); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Step 3: if hostname, build hostname and hosts files
@@ -397,9 +579,9 @@ ff02::2		ip6-allrouters
 
 	// Step 4: register the container
 	if err := runtime.Register(container); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return container, nil
+	return container, warnings, nil
 }
 
 // Commit creates a new filesystem image from the current state of a container.
@@ -429,13 +611,85 @@ func (runtime *Runtime) Commit(container *Container, repository, tag, comment, a
 	return img, nil
 }
 
-// FIXME: harmonize with NewGraph()
-func NewRuntime(flGraphPath string, autoRestart bool, dns []string) (*Runtime, error) {
-	runtime, err := NewRuntimeFromDirectory(flGraphPath, autoRestart)
+func (runtime *Runtime) GetByName(name string) (*Container, error) {
+	if id, err := runtime.idIndex.Get(name); err == nil {
+		name = id
+	}
+
+	entity := runtime.containerGraph.Get(name)
+	if entity == nil {
+		return nil, fmt.Errorf("Could not find entity for %s", name)
+	}
+	e := runtime.getContainerElement(entity.ID())
+	if e == nil {
+		return nil, fmt.Errorf("Could not find container for entity id %s", entity.ID())
+	}
+	return e.Value.(*Container), nil
+}
+
+func (runtime *Runtime) Children(name string) (map[string]*Container, error) {
+	children := make(map[string]*Container)
+
+	err := runtime.containerGraph.Walk(name, func(p string, e *gograph.Entity) error {
+		c := runtime.Get(e.ID())
+		if c == nil {
+			return fmt.Errorf("Could not get container for name %s and id %s", e.ID(), p)
+		}
+		children[p] = c
+		return nil
+	}, 0)
+
+	if err != nil {
+		return nil, err
+	}
+	return children, nil
+}
+
+func (runtime *Runtime) RenameLink(oldName, newName string) error {
+	if id, err := runtime.idIndex.Get(oldName); err == nil {
+		oldName = id
+	}
+	entity := runtime.containerGraph.Get(oldName)
+	if entity == nil {
+		return fmt.Errorf("Could not find entity for %s", oldName)
+	}
+
+	// This is not rename but adding a new link for the default name
+	// Strip the leading '/'
+	if strings.HasPrefix(entity.ID(), oldName[1:]) {
+		_, err := runtime.containerGraph.Set(newName, entity.ID())
+		return err
+	}
+	return runtime.containerGraph.Rename(oldName, newName)
+}
+
+func (runtime *Runtime) Link(parentName, childName, alias string) error {
+	if id, err := runtime.idIndex.Get(parentName); err == nil {
+		parentName = id
+	}
+	parent := runtime.containerGraph.Get(parentName)
+	if parent == nil {
+		return fmt.Errorf("Could not get container for %s", parentName)
+	}
+	if id, err := runtime.idIndex.Get(childName); err == nil {
+		childName = id
+	}
+	child := runtime.containerGraph.Get(childName)
+	if child == nil {
+		return fmt.Errorf("Could not get container for %s", childName)
+	}
+	cc := runtime.Get(child.ID())
+
+	_, err := runtime.containerGraph.Set(path.Join(parentName, alias), cc.ID)
+	return err
+}
+
+// FIXME: harmonize with NewGraph()
+func NewRuntime(config *DaemonConfig) (*Runtime, error) {
+	runtime, err := NewRuntimeFromDirectory(config)
 	if err != nil {
 		return nil, err
 	}
-	runtime.Dns = dns
 
 	if k, err := utils.GetKernelVersion(); err != nil {
 		log.Printf("WARNING: %s\n", err)
@@ -449,34 +703,42 @@ func NewRuntime(flGraphPath string, autoRestart bool, dns []string) (*Runtime, e
 	return runtime, nil
 }
 
-func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
-	runtimeRepo := path.Join(root, "containers")
+func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
+	runtimeRepo := path.Join(config.GraphPath, "containers")
 
 	if err := os.MkdirAll(runtimeRepo, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
 
-	g, err := NewGraph(path.Join(root, "graph"))
+	g, err := NewGraph(path.Join(config.GraphPath, "graph"))
 	if err != nil {
 		return nil, err
 	}
-	volumes, err := NewGraph(path.Join(root, "volumes"))
+	volumes, err := NewGraph(path.Join(config.GraphPath, "volumes"))
 	if err != nil {
 		return nil, err
 	}
-	repositories, err := NewTagStore(path.Join(root, "repositories"), g)
+	repositories, err := NewTagStore(path.Join(config.GraphPath, "repositories"), g)
 	if err != nil {
 		return nil, fmt.Errorf("Couldn't create Tag store: %s", err)
 	}
-	if NetworkBridgeIface == "" {
-		NetworkBridgeIface = DefaultNetworkBridge
+	if config.BridgeIface == "" {
+		config.BridgeIface = DefaultNetworkBridge
 	}
-	netManager, err := newNetworkManager(NetworkBridgeIface)
+	netManager, err := newNetworkManager(config)
 	if err != nil {
 		return nil, err
 	}
+
+	graph, err := gograph.NewDatabase(path.Join(config.GraphPath, "linkgraph.db"))
+	if err != nil {
+		return nil, err
+	}
+
+	// Initialize devicemapper deviceSet
+	deviceSet := devmapper.NewDeviceSet(config.GraphPath)
+
 	runtime := &Runtime{
-		root:           root,
 		repository:     runtimeRepo,
 		containers:     list.New(),
 		networkManager: netManager,
@@ -484,8 +746,10 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) {
 		repositories:   repositories,
 		idIndex:        utils.NewTruncIndex(),
 		capabilities:   &Capabilities{},
-		autoRestart:    autoRestart,
 		volumes:        volumes,
+		config:         config,
+		containerGraph: graph,
+		deviceSet:      deviceSet,
 	}
 
 	if err := runtime.restore(); err != nil {

runtime_test.go

@@ -3,11 +3,16 @@ package docker
 import (
 	"bytes"
 	"fmt"
+	"github.com/dotcloud/docker/devmapper"
+	"github.com/dotcloud/docker/sysinit"
 	"github.com/dotcloud/docker/utils"
 	"io"
+	"io/ioutil"
 	"log"
 	"net"
 	"os"
+	"path"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -24,6 +29,10 @@ const (
 	unitTestStoreBase     = "/var/lib/docker/unit-tests"
 	testDaemonAddr        = "127.0.0.1:4270"
 	testDaemonProto       = "tcp"
+
+	unitTestDMDataLoopbackSize     = 209715200 // 200MB
+	unitTestDMMetaDataLoopbackSize = 104857600 // 100MB
+	unitTestDMBaseFsSize           = 157286400 // 150MB
 )
 
 var (
@@ -43,7 +52,14 @@ func nuke(runtime *Runtime) error {
 	}
 	wg.Wait()
 	runtime.networkManager.Close()
-	return os.RemoveAll(runtime.root)
+
+	for _, container := range runtime.List() {
+		container.EnsureUnmounted()
+	}
+	if err := runtime.deviceSet.Shutdown(); err != nil {
+		utils.Debugf("Error shutting down devicemapper for runtime %s", runtime.config.GraphPath)
+	}
+	return os.RemoveAll(runtime.config.GraphPath)
 }
 
 func cleanup(runtime *Runtime) error {
@@ -57,7 +73,7 @@ func cleanup(runtime *Runtime) error {
 	}
 	for _, image := range images {
 		if image.ID != unitTestImageID {
-			runtime.graph.Delete(image.ID)
+			runtime.DeleteImage(image.ID)
 		}
 	}
 	return nil
@@ -72,12 +88,86 @@ func layerArchive(tarfile string) (io.Reader, error) {
 	return f, nil
 }
 
+// Remove any leftover device mapper devices from earlier runs of the unit tests
+func removeDev(name string) error {
+	path := filepath.Join("/dev/mapper", name)
+
+	if f, err := os.OpenFile(path, os.O_RDONLY, 07777); err != nil {
+		if er, ok := err.(*os.PathError); ok && er.Err == syscall.ENXIO {
+			// No device for this node, just remove it
+			return os.Remove(path)
+		}
+	} else {
+		f.Close()
+	}
+	if err := devmapper.RemoveDevice(name); err != nil {
+		return fmt.Errorf("Unable to remove device %s needed to get a freash unit test environment", name)
+	}
+	return nil
+}
+
+func cleanupDevMapper() error {
+	utils.Debugf("[devmapper cleanup] starting")
+	defer utils.Debugf("[devmapper cleanup] done")
+	filter := "docker-" + path.Base(unitTestStoreBase)
+	utils.Debugf("Filtering out %s\n", filter)
+	// Unmount any leftover mounts from previous unit test runs
+	if data, err := ioutil.ReadFile("/proc/mounts"); err != nil {
+		return err
+	} else {
+		for _, line := range strings.Split(string(data), "\n") {
+			if cols := strings.Split(line, " "); len(cols) >= 2 && strings.HasPrefix(cols[0], path.Join("/dev/mapper", filter)) {
+				utils.Debugf("[devmapper cleanup] Unmounting device: %s", cols[1])
+				if err := syscall.Unmount(cols[1], 0); err != nil {
+					return fmt.Errorf("Unable to unmount %s needed to get a freash unit test environment: %s", cols[1], err)
+				}
+			}
+		}
+	}
+
+	utils.Debugf("[devmapper cleanup] looking for leftover devices")
+	// Remove any leftover devmapper devices from previous unit run tests
+	infos, err := ioutil.ReadDir("/dev/mapper")
+	if err != nil {
+		// If the mapper file does not exist there is nothing to clean up
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	pools := []string{}
+	for _, info := range infos {
+		if name := info.Name(); strings.HasPrefix(name, filter+"-") {
+			if strings.HasSuffix(name, "-pool") {
+				pools = append(pools, name)
+			} else {
+				if err := removeDev(name); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	// We need to remove the pool last as the other devices block it
+	for _, pool := range pools {
+		utils.Debugf("[devmapper cleanup] Removing pool: %s", pool)
+		if err := removeDev(pool); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func init() {
 	os.Setenv("TEST", "1")
 
+	// Set unit-test specific values
+	devmapper.DefaultDataLoopbackSize = unitTestDMDataLoopbackSize
+	devmapper.DefaultMetaDataLoopbackSize = unitTestDMMetaDataLoopbackSize
+	devmapper.DefaultBaseFsSize = unitTestDMBaseFsSize
+
 	// Hack to run sys init during unit testing
 	if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" {
-		SysInit()
+		sysinit.SysInit()
 		return
 	}
 
@@ -85,47 +175,68 @@ func init() {
 		log.Fatal("docker tests need to be run as root")
 	}
 
-	NetworkBridgeIface = unitTestNetworkBridge
-
-	// Make it our Store root
-	if runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false); err != nil {
-		log.Fatalf("Unable to create a runtime for tests:", err)
-	} else {
-		globalRuntime = runtime
+	if err := cleanupDevMapper(); err != nil {
+		log.Fatalf("Unable to cleanup devmapper: %s", err)
 	}
 
-	// Cleanup any leftover container
-	for _, container := range globalRuntime.List() {
-		if err := globalRuntime.Destroy(container); err != nil {
-			log.Fatalf("Error destroying leftover container: %s", err)
-		}
+	// Setup the base runtime, which will be duplicated for each test.
+	// (no tests are run directly in the base)
+	setupBaseImage()
+
+	// Create the "global runtime" with a long-running daemon for integration tests
+	spawnGlobalDaemon()
+	startFds, startGoroutines = utils.GetTotalUsedFds(), runtime.NumGoroutine()
+}
+
+func setupBaseImage() {
+	config := &DaemonConfig{
+		GraphPath:   unitTestStoreBase,
+		AutoRestart: false,
+		BridgeIface: unitTestNetworkBridge,
+	}
+	runtime, err := NewRuntimeFromDirectory(config)
+	if err != nil {
+		log.Fatalf("Unable to create a runtime for tests:", err)
 	}
 
 	// Create the "Server"
 	srv := &Server{
-		runtime:     globalRuntime,
-		enableCors:  false,
+		runtime:     runtime,
 		pullingPool: make(map[string]struct{}),
 		pushingPool: make(map[string]struct{}),
 	}
+
 	// If the unit test is not found, try to download it.
-	if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
+	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
 		// Retrieve the Image
 		if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, nil, true); err != nil {
 			log.Fatalf("Unable to pull the test image:", err)
 		}
 	}
+}
+
+func spawnGlobalDaemon() {
+	if globalRuntime != nil {
+		utils.Debugf("Global runtime already exists. Skipping.")
+		return
+	}
+	globalRuntime = mkRuntime(log.New(os.Stderr, "", 0))
+	srv := &Server{
+		runtime:     globalRuntime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
+
 	// Spawn a Daemon
 	go func() {
+		utils.Debugf("Spawning global daemon for integration tests")
 		if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil {
 			log.Fatalf("Unable to spawn the test daemon:", err)
 		}
 	}()
-
 	// Give some time to ListenAndServer to actually start
+	// FIXME: use inmem transports instead of tcp
 	time.Sleep(time.Second)
-
-	startFds, startGoroutines = utils.GetTotalUsedFds(), runtime.NumGoroutine()
 }
 
 // FIXME: test that ImagePull(json=true) send correct json output
@@ -153,7 +264,7 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
 	}
 
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -162,12 +273,6 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	defer func() {
-		if err := runtime.Destroy(container); err != nil {
-			t.Error(err)
-		}
-	}()
-
 	// Make sure we can find the newly created container with List()
 	if len(runtime.List()) != 1 {
 		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
@@ -194,7 +299,7 @@ func TestRuntimeCreate(t *testing.T) {
 	}
 
 	// Make sure crete with bad parameters returns an error
-	_, err = runtime.Create(
+	_, _, err = runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 		},
@@ -203,7 +308,7 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Fatal("Builder.Create should throw an error when Cmd is missing")
 	}
 
-	_, err = runtime.Create(
+	_, _, err = runtime.Create(
 		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{},
@@ -218,29 +323,18 @@ func TestRuntimeCreate(t *testing.T) {
 		Cmd:       []string{"/bin/ls"},
 		PortSpecs: []string{"80"},
 	}
-	container, err = runtime.Create(config)
+	container, _, err = runtime.Create(config)
 
-	image, err := runtime.Commit(container, "testrepo", "testtag", "", "", config)
+	_, err = runtime.Commit(container, "testrepo", "testtag", "", "", config)
 	if err != nil {
 		t.Error(err)
 	}
-
-	_, err = runtime.Create(
-		&Config{
-			Image:     image.ID,
-			PortSpecs: []string{"80000:80"},
-		},
-	)
-	if err == nil {
-		t.Fatal("Builder.Create should throw an error when PortSpecs is invalid")
-	}
-
 }
 
 func TestDestroy(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
-	container, err := runtime.Create(&Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -314,6 +408,7 @@ func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container,
 	port := 5554
 	var container *Container
 	var strPort string
+	var p Port
 	for {
 		port += 1
 		strPort = strconv.Itoa(port)
@@ -326,22 +421,33 @@ func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container,
 			t.Fatal(fmt.Errorf("Unknown protocol %v", proto))
 		}
 		t.Log("Trying port", strPort)
-		container, err = runtime.Create(&Config{
-			Image:     GetTestImage(runtime).ID,
-			Cmd:       []string{"sh", "-c", cmd},
-			PortSpecs: []string{fmt.Sprintf("%s/%s", strPort, proto)},
+		ep := make(map[Port]struct{}, 1)
+		p = Port(fmt.Sprintf("%s/%s", strPort, proto))
+		ep[p] = struct{}{}
+
+		container, _, err = runtime.Create(&Config{
+			Image:        GetTestImage(runtime).ID,
+			Cmd:          []string{"sh", "-c", cmd},
+			PortSpecs:    []string{fmt.Sprintf("%s/%s", strPort, proto)},
+			ExposedPorts: ep,
 		})
-		if container != nil {
-			break
-		}
 		if err != nil {
 			nuke(runtime)
 			t.Fatal(err)
 		}
+
+		if container != nil {
+			break
+		}
 		t.Logf("Port %v already in use", strPort)
 	}
 
-	hostConfig := &HostConfig{}
+	hostConfig := &HostConfig{
+		PortBindings: make(map[Port][]PortBinding),
+	}
+	hostConfig.PortBindings[p] = []PortBinding{
+		{},
+	}
 	if err := container.Start(hostConfig); err != nil {
 		nuke(runtime)
 		t.Fatal(err)
@@ -356,7 +462,7 @@ func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container,
 	// Even if the state is running, lets give some time to lxc to spawn the process
 	container.WaitTimeout(500 * time.Millisecond)
 
-	strPort = container.NetworkSettings.PortMapping[strings.Title(proto)][strPort]
+	strPort = container.NetworkSettings.Ports[p][0].HostPort
 	return runtime, container, strPort
 }
 
@@ -488,7 +594,8 @@ func TestRestore(t *testing.T) {
 
 	// Here are are simulating a docker restart - that is, reloading all containers
 	// from scratch
-	runtime2, err := NewRuntimeFromDirectory(runtime1.root, false)
+	runtime1.config.AutoRestart = false
+	runtime2, err := NewRuntimeFromDirectory(runtime1.config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -515,3 +622,283 @@ func TestRestore(t *testing.T) {
 	}
 	container2.State.Running = false
 }
+
+func TestContainerCreatedWithDefaultFilesystemType(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	container, _, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
+	defer runtime.Destroy(container)
+
+	if container.FilesystemType != DefaultFilesystemType {
+		t.Fatalf("Container filesystem type should be %s but got %s", DefaultFilesystemType, container.FilesystemType)
+	}
+}
+
+func TestReloadContainerLinks(t *testing.T) {
+	runtime1 := mkRuntime(t)
+	defer nuke(runtime1)
+	// Create a container with one instance of docker
+	container1, _, _ := mkContainer(runtime1, []string{"_", "ls", "-al"}, t)
+	defer runtime1.Destroy(container1)
+
+	// Create a second container meant to be killed
+	container2, _, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/cat"}, t)
+	defer runtime1.Destroy(container2)
+
+	// Start the container non blocking
+	hostConfig := &HostConfig{}
+	if err := container2.Start(hostConfig); err != nil {
+		t.Fatal(err)
+	}
+	h1 := &HostConfig{}
+	// Add a link to container 2
+	h1.Links = []string{utils.TruncateID(container2.ID) + ":first"}
+	if err := container1.Start(h1); err != nil {
+		t.Fatal(err)
+	}
+
+	if !container2.State.Running {
+		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
+	}
+
+	if !container1.State.Running {
+		t.Fatalf("Container %s should appear as running bu isn't", container1.ID)
+	}
+
+	if len(runtime1.List()) != 2 {
+		t.Errorf("Expected 2 container, %v found", len(runtime1.List()))
+	}
+
+	if !container2.State.Running {
+		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
+	}
+
+	// Here are are simulating a docker restart - that is, reloading all containers
+	// from scratch
+	runtime1.config.AutoRestart = true
+	runtime2, err := NewRuntimeFromDirectory(runtime1.config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer nuke(runtime2)
+	if len(runtime2.List()) != 2 {
+		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
+	}
+	runningCount := 0
+	for _, c := range runtime2.List() {
+		if c.State.Running {
+			t.Logf("Running container found: %v (%v)", c.ID, c.Path)
+			runningCount++
+		}
+	}
+	if runningCount != 2 {
+		t.Fatalf("Expected 2 container alive, %d found", runningCount)
+	}
+
+	// Make sure container 2 ( the child of container 1 ) was registered and started first
+	// with the runtime
+	first := runtime2.containers.Front()
+	if first.Value.(*Container).ID != container2.ID {
+		t.Fatalf("Container 2 %s should be registered first in the runtime", container2.ID)
+	}
+
+	t.Logf("Number of links: %d", runtime2.containerGraph.Refs("engine"))
+	// Verify that the link is still registered in the runtime
+	entity := runtime2.containerGraph.Get(fmt.Sprintf("/%s", container1.ID))
+	if entity == nil {
+		t.Fatal("Entity should not be nil")
+	}
+}
+
+func TestDefaultContainerName(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{runtime: runtime}
+
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err := srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	container := runtime.Get(shortId)
+	containerID := container.ID
+
+	paths := runtime.containerGraph.RefPaths(containerID)
+	if paths == nil || len(paths) == 0 {
+		t.Fatalf("Could not find edges for %s", containerID)
+	}
+	edge := paths[0]
+	if edge.ParentID != "0" {
+		t.Fatalf("Expected engine got %s", edge.ParentID)
+	}
+	if edge.EntityID != containerID {
+		t.Fatalf("Expected %s got %s", containerID, edge.EntityID)
+	}
+	if edge.Name != containerID {
+		t.Fatalf("Expected %s got %s", containerID, edge.Name)
+	}
+}
+
+func TestDefaultContainerRename(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{runtime: runtime}
+
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err := srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	container := runtime.Get(shortId)
+	containerID := container.ID
+
+	if err := runtime.RenameLink(fmt.Sprintf("/%s", containerID), "/webapp"); err != nil {
+		t.Fatal(err)
+	}
+
+	webapp, err := runtime.GetByName("/webapp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if webapp.ID != container.ID {
+		t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID)
+	}
+}
+
+func TestLinkChildContainer(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{runtime: runtime}
+
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err := srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	container := runtime.Get(shortId)
+
+	if err := runtime.RenameLink(fmt.Sprintf("/%s", container.ID), "/webapp"); err != nil {
+		t.Fatal(err)
+	}
+
+	webapp, err := runtime.GetByName("/webapp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if webapp.ID != container.ID {
+		t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID)
+	}
+
+	config, _, _, err = ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err = srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	childContainer := runtime.Get(shortId)
+	if err := runtime.RenameLink(fmt.Sprintf("/%s", childContainer.ID), "/db"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := runtime.Link("/webapp", "/db", "db"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the child by it's new name
+	db, err := runtime.GetByName("/webapp/db")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if db.ID != childContainer.ID {
+		t.Fatalf("Expect db id to match container id: %s != %s", db.ID, childContainer.ID)
+	}
+}
+
+func TestGetAllChildren(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{runtime: runtime}
+
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err := srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	container := runtime.Get(shortId)
+
+	if err := runtime.RenameLink(fmt.Sprintf("/%s", container.ID), "/webapp"); err != nil {
+		t.Fatal(err)
+	}
+
+	webapp, err := runtime.GetByName("/webapp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if webapp.ID != container.ID {
+		t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID)
+	}
+
+	config, _, _, err = ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	shortId, _, err = srv.ContainerCreate(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	childContainer := runtime.Get(shortId)
+	if err := runtime.RenameLink(fmt.Sprintf("/%s", childContainer.ID), "/db"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := runtime.Link("/webapp", "/db", "db"); err != nil {
+		t.Fatal(err)
+	}
+
+	children, err := runtime.Children("/webapp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if children == nil {
+		t.Fatal("Children should not be nil")
+	}
+	if len(children) == 0 {
+		t.Fatal("Children should not be empty")
+	}
+
+	for key, value := range children {
+		if key != "/webapp/db" {
+			t.Fatalf("Expected /webapp/db got %s", key)
+		}
+		if value.ID != childContainer.ID {
+			t.Fatalf("Expected id %s got %s", childContainer.ID, value.ID)
+		}
+	}
+}

server.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/dotcloud/docker/auth"
+	"github.com/dotcloud/docker/gograph"
 	"github.com/dotcloud/docker/registry"
 	"github.com/dotcloud/docker/utils"
 	"io"
@@ -103,7 +104,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error {
 }
 
 func (srv *Server) ImagesSearch(term string) ([]APISearch, error) {
-	r, err := registry.NewRegistry(srv.runtime.root, nil, srv.HTTPRequestFactory(nil))
+	r, err := registry.NewRegistry(srv.runtime.config.GraphPath, nil, srv.HTTPRequestFactory(nil))
 	if err != nil {
 		return nil, err
 	}
@@ -140,7 +141,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.
 		return "", err
 	}
 
-	c, err := srv.runtime.Create(config)
+	c, _, err := srv.runtime.Create(config)
 	if err != nil {
 		return "", err
 	}
@@ -268,19 +269,26 @@ func (srv *Server) DockerInfo() *APIInfo {
 		kernelVersion = kv.String()
 	}
 
+	devSetInfo := srv.runtime.deviceSet.Status()
+
 	return &APIInfo{
-		Containers:         len(srv.runtime.List()),
-		Images:             imgcount,
-		MemoryLimit:        srv.runtime.capabilities.MemoryLimit,
-		SwapLimit:          srv.runtime.capabilities.SwapLimit,
-		IPv4Forwarding:     !srv.runtime.capabilities.IPv4ForwardingDisabled,
-		Debug:              os.Getenv("DEBUG") != "",
-		NFd:                utils.GetTotalUsedFds(),
-		NGoroutines:        runtime.NumGoroutine(),
-		LXCVersion:         lxcVersion,
-		NEventsListener:    len(srv.events),
-		KernelVersion:      kernelVersion,
-		IndexServerAddress: auth.IndexServerAddress(),
+		Containers:             len(srv.runtime.List()),
+		Images:                 imgcount,
+		MemoryLimit:            srv.runtime.capabilities.MemoryLimit,
+		SwapLimit:              srv.runtime.capabilities.SwapLimit,
+		IPv4Forwarding:         !srv.runtime.capabilities.IPv4ForwardingDisabled,
+		Debug:                  os.Getenv("DEBUG") != "",
+		NFd:                    utils.GetTotalUsedFds(),
+		NGoroutines:            runtime.NumGoroutine(),
+		LXCVersion:             lxcVersion,
+		NEventsListener:        len(srv.events),
+		KernelVersion:          kernelVersion,
+		IndexServerAddress:     auth.IndexServerAddress(),
+		DevmapperPool:          devSetInfo.PoolName,
+		DevmapperDataUsed:      devSetInfo.Data.Used,
+		DevmapperDataTotal:     devSetInfo.Data.Total,
+		DevmapperMetadataUsed:  devSetInfo.Metadata.Used,
+		DevmapperMetadataTotal: devSetInfo.Metadata.Total,
 	}
 }
 
@@ -358,7 +366,7 @@ func (srv *Server) ContainerChanges(name string) ([]Change, error) {
 func (srv *Server) Containers(all, size bool, n int, since, before string) []APIContainers {
 	var foundBefore bool
 	var displayed int
-	retContainers := []APIContainers{}
+	out := []APIContainers{}
 
 	for _, container := range srv.runtime.List() {
 		if !container.State.Running && !all && n == -1 && since == "" && before == "" {
@@ -380,23 +388,35 @@ func (srv *Server) Containers(all, size bool, n int, since, before string) []API
 			break
 		}
 		displayed++
-
-		c := APIContainers{
-			ID: container.ID,
-		}
-		c.Image = srv.runtime.repositories.ImageName(container.Image)
-		c.Command = fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " "))
-		c.Created = container.Created.Unix()
-		c.Status = container.State.String()
-		c.Ports = container.NetworkSettings.PortMappingAPI()
-		if size {
-			c.SizeRw, c.SizeRootFs = container.GetSize()
-		}
-		retContainers = append(retContainers, c)
+		c := createAPIContainer(container, size, srv.runtime)
+		out = append(out, c)
 	}
-	return retContainers
+	return out
 }
 
+func createAPIContainer(container *Container, size bool, runtime *Runtime) APIContainers {
+	c := APIContainers{
+		ID: container.ID,
+	}
+	names := []string{}
+	runtime.containerGraph.Walk("/", func(p string, e *gograph.Entity) error {
+		if e.ID() == container.ID {
+			names = append(names, p)
+		}
+		return nil
+	}, -1)
+	c.Names = names
+
+	c.Image = runtime.repositories.ImageName(container.Image)
+	c.Command = fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " "))
+	c.Created = container.Created.Unix()
+	c.Status = container.State.String()
+	c.Ports = container.NetworkSettings.PortMappingAPI()
+	if size {
+		c.SizeRw, c.SizeRootFs = container.GetSize()
+	}
+	return c
+}
 func (srv *Server) ContainerCommit(name, repo, tag, author, comment string, config *Config) (string, error) {
 	container := srv.runtime.Get(name)
 	if container == nil {
@@ -635,7 +655,7 @@ func (srv *Server) poolRemove(kind, key string) error {
 }
 
 func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, metaHeaders map[string][]string, parallel bool) error {
-	r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory(metaHeaders))
+	r, err := registry.NewRegistry(srv.runtime.config.GraphPath, authConfig, srv.HTTPRequestFactory(metaHeaders))
 	if err != nil {
 		return err
 	}
@@ -844,7 +864,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo
 
 	out = utils.NewWriteFlusher(out)
 	img, err := srv.runtime.graph.Get(localName)
-	r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory(metaHeaders))
+	r, err2 := registry.NewRegistry(srv.runtime.config.GraphPath, authConfig, srv.HTTPRequestFactory(metaHeaders))
 	if err2 != nil {
 		return err2
 	}
@@ -909,10 +929,9 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write
 	return nil
 }
 
-func (srv *Server) ContainerCreate(config *Config) (string, error) {
-
+func (srv *Server) ContainerCreate(config *Config) (string, []string, error) {
 	if config.Memory != 0 && config.Memory < 524288 {
-		return "", fmt.Errorf("Memory limit must be given in bytes (minimum 524288 bytes)")
+		return "", nil, fmt.Errorf("Memory limit must be given in bytes (minimum 524288 bytes)")
 	}
 
 	if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit {
@@ -922,7 +941,7 @@ func (srv *Server) ContainerCreate(config *Config) (string, error) {
 	if config.Memory > 0 && !srv.runtime.capabilities.SwapLimit {
 		config.MemorySwap = -1
 	}
-	container, err := srv.runtime.Create(config)
+	container, buildWarnings, err := srv.runtime.Create(config)
 	if err != nil {
 		if srv.runtime.graph.IsNotExist(err) {
 
@@ -931,12 +950,12 @@ func (srv *Server) ContainerCreate(config *Config) (string, error) {
 				tag = DEFAULTTAG
 			}
 
-			return "", fmt.Errorf("No such image: %s (tag: %s)", config.Image, tag)
+			return "", nil, fmt.Errorf("No such image: %s (tag: %s)", config.Image, tag)
 		}
-		return "", err
+		return "", nil, err
 	}
 	srv.LogEvent("create", container.ShortID(), srv.runtime.repositories.ImageName(container.Image))
-	return container.ShortID(), nil
+	return container.ShortID(), buildWarnings, nil
 }
 
 func (srv *Server) ContainerRestart(name string, t int) error {
@@ -951,7 +970,34 @@ func (srv *Server) ContainerRestart(name string, t int) error {
 	return nil
 }
 
-func (srv *Server) ContainerDestroy(name string, removeVolume bool) error {
+func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool) error {
+	if removeLink {
+		p := name
+		if p[0] != '/' {
+			p = "/" + p
+		}
+		parent, n := path.Split(p)
+		l := len(parent)
+		if parent[l-1] == '/' {
+			parent = parent[:l-1]
+		}
+
+		pe := srv.runtime.containerGraph.Get(parent)
+		parentContainer := srv.runtime.Get(pe.ID())
+
+		if parentContainer != nil && parentContainer.activeLinks != nil {
+			if link, exists := parentContainer.activeLinks[n]; exists {
+				link.Disable()
+			} else {
+				utils.Debugf("Could not find active link for %s", name)
+			}
+		}
+
+		if err := srv.runtime.containerGraph.Delete(name); err != nil {
+			return err
+		}
+		return nil
+	}
 	if container := srv.runtime.Get(name); container != nil {
 		if container.State.Running {
 			return fmt.Errorf("Impossible to remove a running container, please stop it first")
@@ -996,6 +1042,14 @@ func (srv *Server) ContainerDestroy(name string, removeVolume bool) error {
 
 var ErrImageReferenced = errors.New("Image referenced by a repository")
 
+func (srv *Server) getChildImages(id string) ([]*Image, error) {
+	byParents, err := srv.runtime.graph.ByParent()
+	if err != nil {
+		return nil, err
+	}
+	return byParents[id], nil
+}
+
 func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error {
 	// If the image is referenced by a repo, do not delete
 	if len(srv.runtime.repositories.ByID()[id]) != 0 {
@@ -1028,7 +1082,7 @@ func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error {
 		if err := srv.runtime.repositories.DeleteAll(id); err != nil {
 			return err
 		}
-		err := srv.runtime.graph.Delete(id)
+		err := srv.runtime.DeleteImage(id)
 		if err != nil {
 			return err
 		}
@@ -1101,8 +1155,35 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) {
 	if err != nil {
 		return nil, fmt.Errorf("No such image: %s", name)
 	}
+	images := make(map[string]bool)
+	images[img.ID] = true
+
+	children, err := srv.getChildImages(img.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, i := range children {
+		images[i.ID] = true
+	}
+
+	// Check for any containers referencing the image or children of the image
+	referencedContainers := []string{}
+
+	for e := srv.runtime.containers.Front(); e != nil; e = e.Next() {
+		c := e.Value.(*Container)
+		if images[c.Image] {
+			referencedContainers = append(referencedContainers, c.ID)
+		}
+	}
+
+	if len(referencedContainers) > 0 {
+		return nil, fmt.Errorf("Cannot delete image with existing containers.  Please remove %s before deleting image.",
+			strings.Join(referencedContainers, ", "))
+	}
+
 	if !autoPrune {
-		if err := srv.runtime.graph.Delete(img.ID); err != nil {
+		if err := srv.runtime.DeleteImage(img.ID); err != nil {
 			return nil, fmt.Errorf("Error deleting image %s: %s", name, err)
 		}
 		return nil, nil
@@ -1143,14 +1224,32 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error)
 }
 
 func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error {
-	if container := srv.runtime.Get(name); container != nil {
-		if err := container.Start(hostConfig); err != nil {
-			return fmt.Errorf("Error starting container %s: %s", name, err)
-		}
-		srv.LogEvent("start", container.ShortID(), srv.runtime.repositories.ImageName(container.Image))
-	} else {
+	runtime := srv.runtime
+	container := runtime.Get(name)
+	if container == nil {
 		return fmt.Errorf("No such container: %s", name)
 	}
+
+	// Register links
+	if hostConfig != nil && hostConfig.Links != nil {
+		for _, l := range hostConfig.Links {
+			parts, err := parseLink(l)
+			if err != nil {
+				return err
+			}
+
+			childName := parts["name"]
+			if err := runtime.Link(fmt.Sprintf("/%s", container.ID), childName, parts["alias"]); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := container.Start(hostConfig); err != nil {
+		return fmt.Errorf("Error starting container %s: %s", name, err)
+	}
+	srv.LogEvent("start", container.ShortID(), runtime.repositories.ImageName(container.Image))
+
 	return nil
 }
 
@@ -1302,35 +1401,30 @@ func (srv *Server) ContainerCopy(name string, resource string, out io.Writer) er
 
 }
 
-func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (*Server, error) {
+func NewServer(config *DaemonConfig) (*Server, error) {
 	if runtime.GOARCH != "amd64" {
 		log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH)
 	}
-	runtime, err := NewRuntime(flGraphPath, autoRestart, dns)
+	runtime, err := NewRuntime(config)
 	if err != nil {
 		return nil, err
 	}
-	srv := &Server{
+	runtime.srv = &Server{
 		runtime:     runtime,
-		enableCors:  enableCors,
 		pullingPool: make(map[string]struct{}),
 		pushingPool: make(map[string]struct{}),
 		events:      make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events
 		listeners:   make(map[string]chan utils.JSONMessage),
 		reqFactory:  nil,
 	}
-	runtime.srv = srv
-	return srv, nil
+	return runtime.srv, nil
 }
 
 func (srv *Server) HTTPRequestFactory(metaHeaders map[string][]string) *utils.HTTPRequestFactory {
 	if srv.reqFactory == nil {
-		ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...)
-		md := &utils.HTTPMetaHeadersDecorator{
-			Headers: metaHeaders,
-		}
-		factory := utils.NewHTTPRequestFactory(ud, md)
-		srv.reqFactory = factory
+		srv.reqFactory = utils.NewHTTPRequestFactory(
+			utils.NewHTTPUserAgentDecorator(srv.versionInfos()...),
+			&utils.HTTPMetaHeadersDecorator{Headers: metaHeaders})
 	}
 	return srv.reqFactory
 }
@@ -1350,7 +1444,6 @@ func (srv *Server) LogEvent(action, id, from string) {
 type Server struct {
 	sync.Mutex
 	runtime     *Runtime
-	enableCors  bool
 	pullingPool map[string]struct{}
 	pushingPool map[string]struct{}
 	events      []utils.JSONMessage

server_test.go

@@ -89,7 +89,7 @@ func TestCreateRm(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	id, err := srv.ContainerCreate(config)
+	id, _, err := srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -98,7 +98,7 @@ func TestCreateRm(t *testing.T) {
 		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
 	}
 
-	if err = srv.ContainerDestroy(id, true); err != nil {
+	if err = srv.ContainerDestroy(id, true, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -119,7 +119,7 @@ func TestCreateRmVolumes(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	id, err := srv.ContainerCreate(config)
+	id, _, err := srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -138,7 +138,7 @@ func TestCreateRmVolumes(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = srv.ContainerDestroy(id, true); err != nil {
+	if err = srv.ContainerDestroy(id, true, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -158,7 +158,7 @@ func TestCommit(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	id, err := srv.ContainerCreate(config)
+	id, _, err := srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -179,7 +179,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	id, err := srv.ContainerCreate(config)
+	id, _, err := srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -214,7 +214,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 	}
 
 	// FIXME: this failed once with a race condition ("Unable to remove filesystem for xxx: directory not empty")
-	if err = srv.ContainerDestroy(id, true); err != nil {
+	if err = srv.ContainerDestroy(id, true, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -230,7 +230,7 @@ func TestRunWithTooLowMemoryLimit(t *testing.T) {
 	srv := &Server{runtime: runtime}
 	defer nuke(runtime)
 	// Try to create a container with a memory limit of 1 byte less than the minimum allowed limit.
-	_, err = srv.ContainerCreate(
+	_, _, err = srv.ContainerCreate(
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Memory:    524287,
@@ -351,6 +351,7 @@ func TestPools(t *testing.T) {
 
 func TestLogEvent(t *testing.T) {
 	runtime := mkRuntime(t)
+	defer nuke(runtime)
 	srv := &Server{
 		runtime:   runtime,
 		events:    make([]utils.JSONMessage, 0, 64),
@@ -401,7 +402,7 @@ func TestRmi(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	containerID, err := srv.ContainerCreate(config)
+	containerID, _, err := srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -422,7 +423,7 @@ func TestRmi(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	containerID, err = srv.ContainerCreate(config)
+	containerID, _, err = srv.ContainerCreate(config)
 	if err != nil {
 		t.Fatal(err)
 	}

sorter.go

@@ -34,3 +34,72 @@ func sortImagesByCreationAndTag(images []APIImages) {
 
 	sort.Sort(sorter)
 }
+
+type portSorter struct {
+	ports []Port
+	by    func(i, j Port) bool
+}
+
+func (s *portSorter) Len() int {
+	return len(s.ports)
+}
+
+func (s *portSorter) Swap(i, j int) {
+	s.ports[i], s.ports[j] = s.ports[j], s.ports[i]
+}
+
+func (s *portSorter) Less(i, j int) bool {
+	ip := s.ports[i]
+	jp := s.ports[j]
+
+	return s.by(ip, jp)
+}
+
+func sortPorts(ports []Port, predicate func(i, j Port) bool) {
+	s := &portSorter{ports, predicate}
+	sort.Sort(s)
+}
+
+type containerSorter struct {
+	containers []*Container
+	by         func(i, j *Container) bool
+}
+
+func (s *containerSorter) Len() int {
+	return len(s.containers)
+}
+
+func (s *containerSorter) Swap(i, j int) {
+	s.containers[i], s.containers[j] = s.containers[j], s.containers[i]
+}
+
+func (s *containerSorter) Less(i, j int) bool {
+	return s.by(s.containers[i], s.containers[j])
+}
+
+func sortContainers(containers []*Container, predicate func(i, j *Container) bool) {
+	s := &containerSorter{containers, predicate}
+	sort.Sort(s)
+}
+
+type apiLinkSorter struct {
+	links []APILink
+	by    func(i, j APILink) bool
+}
+
+func (s *apiLinkSorter) Len() int {
+	return len(s.links)
+}
+
+func (s *apiLinkSorter) Swap(i, j int) {
+	s.links[i], s.links[j] = s.links[j], s.links[i]
+}
+
+func (s *apiLinkSorter) Less(i, j int) bool {
+	return s.by(s.links[i], s.links[j])
+}
+
+func sortLinks(links []APILink, predicate func(i, j APILink) bool) {
+	s := &apiLinkSorter{links, predicate}
+	sort.Sort(s)
+}

sorter_test.go

@@ -1,6 +1,7 @@
 package docker
 
 import (
+	"fmt"
 	"testing"
 )
 
@@ -55,3 +56,38 @@ func TestServerListOrderedImagesByCreationDateAndTag(t *testing.T) {
 		t.Error("Expected []APIImges to be ordered by most recent creation date and tag name.")
 	}
 }
+
+func TestSortUniquePorts(t *testing.T) {
+	ports := []Port{
+		Port("6379/tcp"),
+		Port("22/tcp"),
+	}
+
+	sortPorts(ports, func(ip, jp Port) bool {
+		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
+	})
+
+	first := ports[0]
+	if fmt.Sprint(first) != "22/tcp" {
+		t.Log(fmt.Sprint(first))
+		t.Fail()
+	}
+}
+
+func TestSortSamePortWithDifferentProto(t *testing.T) {
+	ports := []Port{
+		Port("8888/tcp"),
+		Port("8888/udp"),
+		Port("6379/tcp"),
+		Port("6379/udp"),
+	}
+
+	sortPorts(ports, func(ip, jp Port) bool {
+		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
+	})
+
+	first := ports[0]
+	if fmt.Sprint(first) != "6379/tcp" {
+		t.Fail()
+	}
+}

state.go

@@ -9,12 +9,12 @@ import (
 
 type State struct {
 	sync.Mutex
-	Running   bool
-	Pid       int
-	ExitCode  int
-	StartedAt time.Time
+	Running    bool
+	Pid        int
+	ExitCode   int
+	StartedAt  time.Time
 	FinishedAt time.Time
-	Ghost     bool
+	Ghost      bool
 }
 
 // String returns a human-readable description of the state

sysinit/sysinit.go

@@ -1,10 +1,12 @@
-package docker
+package sysinit
 
 import (
 	"flag"
 	"fmt"
+	"github.com/dotcloud/docker/netlink"
 	"github.com/dotcloud/docker/utils"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"strconv"
@@ -17,7 +19,14 @@ func setupNetworking(gw string) {
 	if gw == "" {
 		return
 	}
-	if _, err := ip("route", "add", "default", "via", gw); err != nil {
+
+	ip := net.ParseIP(gw)
+	if ip == nil {
+		log.Fatalf("Unable to set up networking, %s is not a valid IP", gw)
+		return
+	}
+
+	if err := netlink.AddDefaultGw(ip); err != nil {
 		log.Fatalf("Unable to set up networking: %v", err)
 	}
 }
@@ -58,7 +67,7 @@ func changeUser(u string) {
 }
 
 // Clear environment pollution introduced by lxc-start
-func cleanupEnv(env ListOpts) {
+func cleanupEnv(env utils.ListOpts) {
 	os.Clearenv()
 	for _, kv := range env {
 		parts := strings.SplitN(kv, "=", 2)
@@ -86,14 +95,14 @@ func executeProgram(name string, args []string) {
 // up the environment before running the actual process
 func SysInit() {
 	if len(os.Args) <= 1 {
-		fmt.Println("You should not invoke docker-init manually")
+		fmt.Println("You should not invoke dockerinit manually")
 		os.Exit(1)
 	}
 	var u = flag.String("u", "", "username or uid")
 	var gw = flag.String("g", "", "gateway address")
 	var workdir = flag.String("w", "", "workdir")
 
-	var flEnv ListOpts
+	var flEnv utils.ListOpts
 	flag.Var(&flEnv, "e", "Set environment variables")
 
 	flag.Parse()

utils.go

@@ -1,8 +1,35 @@
 package docker
 
+/*
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+
+// See linux.git/fs/btrfs/ioctl.h
+#define BTRFS_IOCTL_MAGIC 0x94
+#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int)
+
+int
+btrfs_reflink(int fd_out, int fd_in)
+{
+  int res;
+  res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in);
+  if (res < 0)
+    return errno;
+  return 0;
+}
+
+*/
+import "C"
 import (
 	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"strconv"
 	"strings"
+	"syscall"
 )
 
 // Compare two Config struct. Do not compare the "Image" nor "Hostname" fields
@@ -27,6 +54,7 @@ func CompareConfig(a, b *Config) bool {
 		len(a.Dns) != len(b.Dns) ||
 		len(a.Env) != len(b.Env) ||
 		len(a.PortSpecs) != len(b.PortSpecs) ||
+		len(a.ExposedPorts) != len(b.ExposedPorts) ||
 		len(a.Entrypoint) != len(b.Entrypoint) ||
 		len(a.Volumes) != len(b.Volumes) {
 		return false
@@ -52,6 +80,11 @@ func CompareConfig(a, b *Config) bool {
 			return false
 		}
 	}
+	for k := range a.ExposedPorts {
+		if _, exists := b.ExposedPorts[k]; !exists {
+			return false
+		}
+	}
 	for i := 0; i < len(a.Entrypoint); i++ {
 		if a.Entrypoint[i] != b.Entrypoint[i] {
 			return false
@@ -78,26 +111,38 @@ func MergeConfig(userConf, imageConf *Config) error {
 	if userConf.CpuShares == 0 {
 		userConf.CpuShares = imageConf.CpuShares
 	}
-	if userConf.PortSpecs == nil || len(userConf.PortSpecs) == 0 {
-		userConf.PortSpecs = imageConf.PortSpecs
-	} else {
-		for _, imagePortSpec := range imageConf.PortSpecs {
-			found := false
-			imageNat, err := parseNat(imagePortSpec)
-			if err != nil {
-				return err
+	if userConf.ExposedPorts == nil || len(userConf.ExposedPorts) == 0 {
+		userConf.ExposedPorts = imageConf.ExposedPorts
+	}
+
+	if userConf.PortSpecs != nil && len(userConf.PortSpecs) > 0 {
+		if userConf.ExposedPorts == nil {
+			userConf.ExposedPorts = make(map[Port]struct{})
+		}
+		ports, _, err := parsePortSpecs(userConf.PortSpecs)
+		if err != nil {
+			return err
+		}
+		for port := range ports {
+			if _, exists := userConf.ExposedPorts[port]; !exists {
+				userConf.ExposedPorts[port] = struct{}{}
 			}
-			for _, userPortSpec := range userConf.PortSpecs {
-				userNat, err := parseNat(userPortSpec)
-				if err != nil {
-					return err
-				}
-				if imageNat.Proto == userNat.Proto && imageNat.Backend == userNat.Backend {
-					found = true
-				}
-			}
-			if !found {
-				userConf.PortSpecs = append(userConf.PortSpecs, imagePortSpec)
+		}
+		userConf.PortSpecs = nil
+	}
+	if imageConf.PortSpecs != nil && len(imageConf.PortSpecs) > 0 {
+		utils.Debugf("Migrating image port specs to containter: %s", strings.Join(imageConf.PortSpecs, ", "))
+		if userConf.ExposedPorts == nil {
+			userConf.ExposedPorts = make(map[Port]struct{})
+		}
+
+		ports, _, err := parsePortSpecs(imageConf.PortSpecs)
+		if err != nil {
+			return err
+		}
+		for port := range ports {
+			if _, exists := userConf.ExposedPorts[port]; !exists {
+				userConf.ExposedPorts[port] = struct{}{}
 			}
 		}
 	}
@@ -155,7 +200,7 @@ func MergeConfig(userConf, imageConf *Config) error {
 	return nil
 }
 
-func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) {
+func parseLxcConfOpts(opts utils.ListOpts) ([]KeyValuePair, error) {
 	out := make([]KeyValuePair, len(opts))
 	for i, o := range opts {
 		k, v, err := parseLxcOpt(o)
@@ -174,3 +219,131 @@ func parseLxcOpt(opt string) (string, string, error) {
 	}
 	return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil
 }
+
+func RootIsShared() bool {
+	if data, err := ioutil.ReadFile("/proc/self/mountinfo"); err == nil {
+		for _, line := range strings.Split(string(data), "\n") {
+			cols := strings.Split(line, " ")
+			if len(cols) >= 6 && cols[4] == "/" {
+				return strings.HasPrefix(cols[6], "shared")
+			}
+		}
+	}
+
+	// No idea, probably safe to assume so
+	return true
+}
+
+func BtrfsReflink(fd_out, fd_in uintptr) error {
+	res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in))
+	if res != 0 {
+		return syscall.Errno(res)
+	}
+	return nil
+}
+
+func CopyFile(dstFile, srcFile *os.File) error {
+	err := BtrfsReflink(dstFile.Fd(), srcFile.Fd())
+	if err == nil {
+		return nil
+	}
+
+	// Fall back to normal copy
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}
+
+// We will receive port specs in the format of ip:public:private/proto and these need to be
+// parsed in the internal types
+func parsePortSpecs(ports []string) (map[Port]struct{}, map[Port][]PortBinding, error) {
+	exposedPorts := make(map[Port]struct{}, len(ports))
+	bindings := make(map[Port][]PortBinding)
+
+	for _, rawPort := range ports {
+		proto := "tcp"
+		if i := strings.LastIndex(rawPort, "/"); i != -1 {
+			proto = rawPort[i+1:]
+			rawPort = rawPort[:i]
+		}
+		if !strings.Contains(rawPort, ":") {
+			rawPort = fmt.Sprintf("::%s", rawPort)
+		} else if len(strings.Split(rawPort, ":")) == 2 {
+			rawPort = fmt.Sprintf(":%s", rawPort)
+		}
+
+		parts, err := utils.PartParser("ip:hostPort:containerPort", rawPort)
+		if err != nil {
+			return nil, nil, err
+		}
+		containerPort := parts["containerPort"]
+		rawIp := parts["ip"]
+		hostPort := parts["hostPort"]
+
+		if containerPort == "" {
+			return nil, nil, fmt.Errorf("No port specified: %s<empty>", rawPort)
+		}
+
+		port := NewPort(proto, containerPort)
+		if _, exists := exposedPorts[port]; !exists {
+			exposedPorts[port] = struct{}{}
+		}
+
+		binding := PortBinding{
+			HostIp:   rawIp,
+			HostPort: hostPort,
+		}
+		bslice, exists := bindings[port]
+		if !exists {
+			bslice = []PortBinding{}
+		}
+		bindings[port] = append(bslice, binding)
+	}
+	return exposedPorts, bindings, nil
+}
+
+// Splits a port in the format of port/proto
+func splitProtoPort(rawPort string) (string, string) {
+	parts := strings.Split(rawPort, "/")
+	l := len(parts)
+	if l == 0 {
+		return "", ""
+	}
+	if l == 1 {
+		return "tcp", rawPort
+	}
+	return parts[0], parts[1]
+}
+
+func parsePort(rawPort string) (int, error) {
+	port, err := strconv.ParseUint(rawPort, 10, 16)
+	if err != nil {
+		return 0, err
+	}
+	return int(port), nil
+}
+
+func migratePortMappings(config *Config) error {
+	if config.PortSpecs != nil {
+		// We don't have to worry about migrating the bindings to the host
+		// This is our breaking change
+		ports, _, err := parsePortSpecs(config.PortSpecs)
+		if err != nil {
+			return err
+		}
+		config.PortSpecs = nil
+
+		if config.ExposedPorts == nil {
+			config.ExposedPorts = make(map[Port]struct{}, len(ports))
+		}
+		for k, v := range ports {
+			config.ExposedPorts[k] = v
+		}
+	}
+	return nil
+}
+
+// Links come in the format of
+// name:alias
+func parseLink(rawLink string) (map[string]string, error) {
+	return utils.PartParser("name:alias", rawLink)
+}