Merge pull request #32 from shykes/shykes-0.6.5-dm-plugin

Merge bug fix from dotcloud/docker master

AUTHORS

@@ -44,7 +44,6 @@ Daniel Nordberg <dnordberg@gmail.com>
 Daniel Robinson <gottagetmac@gmail.com>
 Daniel Von Fange <daniel@leancoder.com>
 Daniel YC Lin <dlin.tw@gmail.com>
-Darren Coxall <darren@darrencoxall.com>
 David Calavera <david.calavera@gmail.com>
 David Sissitka <me@dsissitka.com>
 Deni Bertovic <deni@kset.org>
@@ -121,7 +120,6 @@ Marko Mikulicic <mmikulicic@gmail.com>
 Markus Fix <lispmeister@gmail.com>
 Martin Redmond <martin@tinychat.com>
 Matt Apperson <me@mattapperson.com>
-Mathieu Le Marec - Pasquet <kiorky@cryptelium.net>
 Matt Bachmann <bachmann.matt@gmail.com>
 Matthew Mueller <mattmuelle@gmail.com>
 Maxim Treskin <zerthurd@gmail.com>

CHANGELOG.md

@@ -1,43 +1,5 @@
 # Changelog
 
-## 0.6.7 (2013-11-21)
-
-#### Runtime
-
-* Improved stability, fixes some race conditons
-* Skip the volumes mounted when deleting the volumes of container.
-* Fix layer size computation: handle hard links correctly
-* Use the work Path for docker cp CONTAINER:PATH
-* Fix tmp dir never cleanup
-* Speedup docker ps
-* More informative error message on name collisions
-* Fix nameserver regex
-* Always return long id's
-* Fix container restart race condition
-* Keep published ports on docker stop;docker start
-* Fix container networking on Fedora
-* Correctly express "any address" to iptables
-* Fix network setup when reconnecting to ghost container
-* Prevent deletion if image is used by a running container
-* Lock around read operations in graph
-
-#### RemoteAPI
-
-* Return full ID on docker rmi
-
-#### Client
-
-+ Add -tree option to images
-+ Offline image transfer
-* Exit with status 2 on usage error and display usage on stderr
-* Do not forward SIGCHLD to container
-* Use string timestamp for docker events -since
-
-#### Other
-
-* Update to go 1.2rc5
-+ Add /etc/default/docker support to upstart
-
 ## 0.6.6 (2013-11-06)
 
 #### Runtime
@@ -55,7 +17,6 @@
 + Prevent DNS server conflicts in CreateBridgeIface
 + Validate bind mounts on the server side
 + Use parent image config in docker build
-* Fix regression in /etc/hosts
 
 #### Client
 

Dockerfile

@@ -23,7 +23,7 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-docker-version 0.6.1
+docker-version	0.6.1
 from	ubuntu:12.04
 maintainer	Solomon Hykes <solomon@dotcloud.com>
 
@@ -33,22 +33,23 @@ run	apt-get update
 run	apt-get install -y -q curl
 run	apt-get install -y -q git
 run	apt-get install -y -q mercurial
-run apt-get install -y -q build-essential libsqlite3-dev
+run	apt-get install -y -q build-essential libsqlite3-dev
 
 # Install Go
-run	curl -s https://go.googlecode.com/files/go1.2rc5.src.tar.gz | tar -v -C /usr/local -xz
+run	curl -s https://go.googlecode.com/files/go1.2rc4.src.tar.gz | tar -v -C /usr/local -xz
 env	PATH	/usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin
 env	GOPATH	/go:/go/src/github.com/dotcloud/docker/vendor
-run cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
+run	cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
 
 # Ubuntu stuff
 run	apt-get install -y -q ruby1.9.3 rubygems libffi-dev
 run	gem install --no-rdoc --no-ri fpm
 run	apt-get install -y -q reprepro dpkg-sig
 
+# Install s3cmd 1.0.1 (earlier versions don't support env variables in the config)
 run	apt-get install -y -q python-pip
-run	pip install s3cmd==1.1.0-beta3
-run	pip install python-magic==0.4.6
+run	pip install s3cmd
+run	pip install python-magic
 run	/bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg
 
 # Runtime dependencies
@@ -56,11 +57,20 @@ run	apt-get install -y -q iptables
 run	apt-get install -y -q lxc
 run	apt-get install -y -q aufs-tools
 
+# Get lvm2 source for compiling statically
+run	git clone git://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout v2_02_103
+# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
+# note: we can't use "git clone -b" above because it requires at least git 1.7.10 to be able to use that on a tag instead of a branch and we only have 1.7.9.5
+
+# Compile and install lvm2
+run	cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
+
 volume	/var/lib/docker
 workdir	/go/src/github.com/dotcloud/docker
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
-entrypoint ["hack/dind"]
+entrypoint	["hack/dind"]
 
 # Upload docker source
-add	.       /go/src/github.com/dotcloud/docker
+add	.	/go/src/github.com/dotcloud/docker

VERSION

@@ -1 +1 @@
-0.6.7
+0.6.6-dev

Vagrantfile

@@ -70,7 +70,7 @@ SCRIPT
 # trigger dkms to build the virtualbox guest module install.
 $vbox_script = <<VBOX_SCRIPT + $script
 # Install the VirtualBox guest additions if they aren't already installed.
-if [ ! -d /opt/VBoxGuestAdditions-4.3.2/ ]; then
+if [ ! -d /opt/VBoxGuestAdditions-4.2.12/ ]; then
     # Update remote package metadata.  'apt-get update' is idempotent.
     apt-get update -q
 
@@ -79,9 +79,9 @@ if [ ! -d /opt/VBoxGuestAdditions-4.3.2/ ]; then
     apt-get install -q -y linux-headers-generic-lts-raring dkms
 
     echo 'Downloading VBox Guest Additions...'
-    wget -cq http://dlc.sun.com.edgesuite.net/virtualbox/4.3.2/VBoxGuestAdditions_4.3.2.iso
+    wget -cq http://dlc.sun.com.edgesuite.net/virtualbox/4.2.12/VBoxGuestAdditions_4.2.12.iso
 
-    mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.3.2.iso /mnt
+    mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.2.12.iso /mnt
     /mnt/VBoxLinuxAdditions.run --nox11
     umount /mnt
 fi

api.go

@@ -149,12 +149,13 @@ func postContainersKill(srv *Server, version float64, w http.ResponseWriter, r *
 
 	signal := 0
 	if r != nil {
-		if s := r.Form.Get("signal"); s != "" {
-			s, err := strconv.Atoi(s)
-			if err != nil {
+		s := r.Form.Get("signal")
+		if s != "" {
+			if s, err := strconv.Atoi(s); err != nil {
 				return err
+			} else {
+				signal = s
 			}
-			signal = s
 		}
 	}
 	if err := srv.ContainerKill(name, signal); err != nil {
@@ -200,8 +201,9 @@ func getImagesJSON(srv *Server, version float64, w http.ResponseWriter, r *http.
 		}
 
 		return writeJSON(w, http.StatusOK, outs2)
+	} else {
+		return writeJSON(w, http.StatusOK, outs)
 	}
-	return writeJSON(w, http.StatusOK, outs)
 }
 
 func getImagesViz(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
@@ -314,10 +316,13 @@ func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *ht
 	if err := parseForm(r); err != nil {
 		return err
 	}
-	procsStr, err := srv.ContainerTop(vars["name"], r.Form.Get("ps_args"))
+	name := vars["name"]
+	ps_args := r.Form.Get("ps_args")
+	procsStr, err := srv.ContainerTop(name, ps_args)
 	if err != nil {
 		return err
 	}
+
 	return writeJSON(w, http.StatusOK, procsStr)
 }
 
@@ -345,12 +350,13 @@ func getContainersJSON(srv *Server, version float64, w http.ResponseWriter, r *h
 	if version < 1.5 {
 		outs2 := []APIContainersOld{}
 		for _, ctnr := range outs {
-			outs2 = append(outs2, *ctnr.ToLegacy())
+			outs2 = append(outs2, ctnr.ToLegacy())
 		}
 
 		return writeJSON(w, http.StatusOK, outs2)
+	} else {
+		return writeJSON(w, http.StatusOK, outs)
 	}
-	return writeJSON(w, http.StatusOK, outs)
 }
 
 func postImagesTag(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
@@ -534,18 +540,6 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http
 	return nil
 }
 
-func getImagesGet(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	name := vars["name"]
-	if version > 1.0 {
-		w.Header().Set("Content-Type", "application/x-tar")
-	}
-	return srv.ImageExport(name, w)
-}
-
-func postImagesLoad(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	return srv.ImageLoad(r.Body)
-}
-
 func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := parseForm(r); err != nil {
 		return nil
@@ -646,10 +640,12 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R
 	if imgs != nil {
 		if len(imgs) != 0 {
 			return writeJSON(w, http.StatusOK, imgs)
+		} else {
+			return fmt.Errorf("Conflict, %s wasn't deleted", name)
 		}
-		return fmt.Errorf("Conflict, %s wasn't deleted", name)
+	} else {
+		w.WriteHeader(http.StatusNoContent)
 	}
-	w.WriteHeader(http.StatusNoContent)
 	return nil
 }
 
@@ -659,6 +655,9 @@ func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r
 	}
 	name := vars["name"]
 	job := srv.Eng.Job("start", name)
+	if err := job.ImportEnv(HostConfig{}); err != nil {
+		return fmt.Errorf("Couldn't initialize host configuration")
+	}
 	// allow a nil body for backwards compatibility
 	if r.Body != nil {
 		if matchesContentType(r.Header.Get("Content-Type"), "application/json") {
@@ -931,7 +930,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ
 		if err != nil {
 			return err
 		}
-		c, err := MkBuildContext(string(dockerFile), nil)
+		c, err := mkBuildContext(string(dockerFile), nil)
 		if err != nil {
 			return err
 		}
@@ -979,7 +978,7 @@ func postContainersCopy(srv *Server, version float64, w http.ResponseWriter, r *
 	}
 
 	if copyData.Resource == "" {
-		return fmt.Errorf("Path cannot be empty")
+		return fmt.Errorf("Resource cannot be empty")
 	}
 	if copyData.Resource[0] == '/' {
 		copyData.Resource = copyData.Resource[1:]
@@ -1048,7 +1047,6 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/images/json":                    getImagesJSON,
 			"/images/viz":                     getImagesViz,
 			"/images/search":                  getImagesSearch,
-			"/images/{name:.*}/get":           getImagesGet,
 			"/images/{name:.*}/history":       getImagesHistory,
 			"/images/{name:.*}/json":          getImagesByName,
 			"/containers/ps":                  getContainersJSON,
@@ -1065,7 +1063,6 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/build":                        postBuild,
 			"/images/create":                postImagesCreate,
 			"/images/{name:.*}/insert":      postImagesInsert,
-			"/images/load":                  postImagesLoad,
 			"/images/{name:.*}/push":        postImagesPush,
 			"/images/{name:.*}/tag":         postImagesTag,
 			"/containers/create":            postContainersCreate,
@@ -1111,20 +1108,6 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 	return r, nil
 }
 
-// ServeRequest processes a single http request to the docker remote api.
-// FIXME: refactor this to be part of Server and not require re-creating a new
-// router each time. This requires first moving ListenAndServe into Server.
-func ServeRequest(srv *Server, apiversion float64, w http.ResponseWriter, req *http.Request) error {
-	router, err := createRouter(srv, false)
-	if err != nil {
-		return err
-	}
-	// Insert APIVERSION into the request as a convenience
-	req.URL.Path = fmt.Sprintf("/v%g%s", apiversion, req.URL.Path)
-	router.ServeHTTP(w, req)
-	return nil
-}
-
 func ListenAndServe(proto, addr string, srv *Server, logging bool) error {
 	log.Printf("Listening for HTTP on %s (%s)\n", addr, proto)
 

api_params.go

@@ -2,147 +2,149 @@ package docker
 
 import "strings"
 
-type (
-	APIHistory struct {
-		ID        string   `json:"Id"`
-		Tags      []string `json:",omitempty"`
-		Created   int64
-		CreatedBy string `json:",omitempty"`
-		Size      int64
-	}
+type APIHistory struct {
+	ID        string   `json:"Id"`
+	Tags      []string `json:",omitempty"`
+	Created   int64
+	CreatedBy string `json:",omitempty"`
+	Size      int64
+}
 
-	APIImages struct {
-		ID          string   `json:"Id"`
-		RepoTags    []string `json:",omitempty"`
-		Created     int64
-		Size        int64
-		VirtualSize int64
-		ParentId    string `json:",omitempty"`
-	}
+type APIImages struct {
+	ID          string   `json:"Id"`
+	RepoTags    []string `json:",omitempty"`
+	Created     int64
+	Size        int64
+	VirtualSize int64
+	ParentId    string `json:",omitempty"`
+}
 
-	APIImagesOld struct {
-		Repository  string `json:",omitempty"`
-		Tag         string `json:",omitempty"`
-		ID          string `json:"Id"`
-		Created     int64
-		Size        int64
-		VirtualSize int64
-	}
+type APIImagesOld struct {
+	Repository  string `json:",omitempty"`
+	Tag         string `json:",omitempty"`
+	ID          string `json:"Id"`
+	Created     int64
+	Size        int64
+	VirtualSize int64
+}
 
-	APIInfo struct {
-		Debug              bool
-		Containers         int
-		Images             int
-		NFd                int    `json:",omitempty"`
-		NGoroutines        int    `json:",omitempty"`
-		MemoryLimit        bool   `json:",omitempty"`
-		SwapLimit          bool   `json:",omitempty"`
-		IPv4Forwarding     bool   `json:",omitempty"`
-		LXCVersion         string `json:",omitempty"`
-		NEventsListener    int    `json:",omitempty"`
-		KernelVersion      string `json:",omitempty"`
-		IndexServerAddress string `json:",omitempty"`
-	}
+func (self *APIImages) ToLegacy() []APIImagesOld {
 
-	APITop struct {
-		Titles    []string
-		Processes [][]string
-	}
-
-	APIRmi struct {
-		Deleted  string `json:",omitempty"`
-		Untagged string `json:",omitempty"`
-	}
-
-	APIContainers struct {
-		ID         string `json:"Id"`
-		Image      string
-		Command    string
-		Created    int64
-		Status     string
-		Ports      []APIPort
-		SizeRw     int64
-		SizeRootFs int64
-		Names      []string
-	}
-
-	APIContainersOld struct {
-		ID         string `json:"Id"`
-		Image      string
-		Command    string
-		Created    int64
-		Status     string
-		Ports      string
-		SizeRw     int64
-		SizeRootFs int64
-	}
-
-	APIID struct {
-		ID string `json:"Id"`
-	}
-
-	APIRun struct {
-		ID       string   `json:"Id"`
-		Warnings []string `json:",omitempty"`
-	}
-
-	APIPort struct {
-		PrivatePort int64
-		PublicPort  int64
-		Type        string
-		IP          string
-	}
-
-	APIVersion struct {
-		Version   string
-		GitCommit string `json:",omitempty"`
-		GoVersion string `json:",omitempty"`
-	}
-
-	APIWait struct {
-		StatusCode int
-	}
-
-	APIAuth struct {
-		Status string
-	}
-
-	APIImageConfig struct {
-		ID string `json:"Id"`
-		*Config
-	}
-
-	APICopy struct {
-		Resource string
-		HostPath string
-	}
-)
-
-func (api APIImages) ToLegacy() []APIImagesOld {
 	outs := []APIImagesOld{}
-	for _, repotag := range api.RepoTags {
+	for _, repotag := range self.RepoTags {
+
 		components := strings.SplitN(repotag, ":", 2)
+
 		outs = append(outs, APIImagesOld{
-			ID:          api.ID,
+			ID:          self.ID,
 			Repository:  components[0],
 			Tag:         components[1],
-			Created:     api.Created,
-			Size:        api.Size,
-			VirtualSize: api.VirtualSize,
+			Created:     self.Created,
+			Size:        self.Size,
+			VirtualSize: self.VirtualSize,
 		})
 	}
+
 	return outs
 }
 
-func (api APIContainers) ToLegacy() *APIContainersOld {
-	return &APIContainersOld{
-		ID:         api.ID,
-		Image:      api.Image,
-		Command:    api.Command,
-		Created:    api.Created,
-		Status:     api.Status,
-		Ports:      displayablePorts(api.Ports),
-		SizeRw:     api.SizeRw,
-		SizeRootFs: api.SizeRootFs,
+type APIInfo struct {
+	Debug              bool
+	Containers         int
+	Images             int
+	NFd                int    `json:",omitempty"`
+	NGoroutines        int    `json:",omitempty"`
+	MemoryLimit        bool   `json:",omitempty"`
+	SwapLimit          bool   `json:",omitempty"`
+	IPv4Forwarding     bool   `json:",omitempty"`
+	LXCVersion         string `json:",omitempty"`
+	NEventsListener    int    `json:",omitempty"`
+	KernelVersion      string `json:",omitempty"`
+	IndexServerAddress string `json:",omitempty"`
+}
+
+type APITop struct {
+	Titles    []string
+	Processes [][]string
+}
+
+type APIRmi struct {
+	Deleted  string `json:",omitempty"`
+	Untagged string `json:",omitempty"`
+}
+
+type APIContainers struct {
+	ID         string `json:"Id"`
+	Image      string
+	Command    string
+	Created    int64
+	Status     string
+	Ports      []APIPort
+	SizeRw     int64
+	SizeRootFs int64
+	Names      []string
+}
+
+func (self *APIContainers) ToLegacy() APIContainersOld {
+	return APIContainersOld{
+		ID:         self.ID,
+		Image:      self.Image,
+		Command:    self.Command,
+		Created:    self.Created,
+		Status:     self.Status,
+		Ports:      displayablePorts(self.Ports),
+		SizeRw:     self.SizeRw,
+		SizeRootFs: self.SizeRootFs,
 	}
 }
+
+type APIContainersOld struct {
+	ID         string `json:"Id"`
+	Image      string
+	Command    string
+	Created    int64
+	Status     string
+	Ports      string
+	SizeRw     int64
+	SizeRootFs int64
+}
+
+type APIID struct {
+	ID string `json:"Id"`
+}
+
+type APIRun struct {
+	ID       string   `json:"Id"`
+	Warnings []string `json:",omitempty"`
+}
+
+type APIPort struct {
+	PrivatePort int64
+	PublicPort  int64
+	Type        string
+	IP          string
+}
+
+type APIVersion struct {
+	Version   string
+	GitCommit string `json:",omitempty"`
+	GoVersion string `json:",omitempty"`
+}
+
+type APIWait struct {
+	StatusCode int
+}
+
+type APIAuth struct {
+	Status string
+}
+
+type APIImageConfig struct {
+	ID string `json:"Id"`
+	*Config
+}
+
+type APICopy struct {
+	Resource string
+	HostPath string
+}

api_unit_test.go

@@ -1,19 +0,0 @@
-package docker
-
-import (
-	"testing"
-)
-
-func TestJsonContentType(t *testing.T) {
-	if !matchesContentType("application/json", "application/json") {
-		t.Fail()
-	}
-
-	if !matchesContentType("application/json; charset=utf-8", "application/json") {
-		t.Fail()
-	}
-
-	if matchesContentType("dockerapplication/json", "application/json") {
-		t.Fail()
-	}
-}

archive/archive.go

@@ -15,7 +15,15 @@ import (
 
 type Archive io.Reader
 
-type Compression uint32
+type Compression int
+
+type TarOptions struct {
+	Includes    []string
+	Excludes    []string
+	Recursive   bool
+	Compression Compression
+	CreateFiles []string
+}
 
 const (
 	Uncompressed Compression = iota
@@ -80,20 +88,78 @@ func (compression *Compression) Extension() string {
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
 func Tar(path string, compression Compression) (io.Reader, error) {
-	return TarFilter(path, compression, nil)
+	return TarFilter(path, &TarOptions{Recursive: true, Compression: compression})
+}
+
+func escapeName(name string) string {
+	escaped := make([]byte, 0)
+	for i, c := range []byte(name) {
+		if i == 0 && c == '/' {
+			continue
+		}
+		// all printable chars except "-" which is 0x2d
+		if (0x20 <= c && c <= 0x7E) && c != 0x2d {
+			escaped = append(escaped, c)
+		} else {
+			escaped = append(escaped, fmt.Sprintf("\\%03o", c)...)
+		}
+	}
+	return string(escaped)
 }
 
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
-func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
-	if filter == nil {
-		filter = []string{"."}
+func TarFilter(path string, options *TarOptions) (io.Reader, error) {
+	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"}
+	if options.Includes == nil {
+		options.Includes = []string{"."}
 	}
-	for _, f := range filter {
-		args = append(args, "-c"+compression.Flag(), f)
+	args = append(args, "-c"+options.Compression.Flag())
+
+	for _, exclude := range options.Excludes {
+		args = append(args, fmt.Sprintf("--exclude=%s", exclude))
 	}
-	return CmdStream(exec.Command(args[0], args[1:]...))
+
+	if !options.Recursive {
+		args = append(args, "--no-recursion")
+	}
+
+	files := ""
+	for _, f := range options.Includes {
+		files = files + escapeName(f) + "\n"
+	}
+
+	tmpDir := ""
+
+	if options.CreateFiles != nil {
+		var err error // Can't use := here or we override the outer tmpDir
+		tmpDir, err = ioutil.TempDir("", "docker-tar")
+		if err != nil {
+			return nil, err
+		}
+
+		files = files + "-C" + tmpDir + "\n"
+		for _, f := range options.CreateFiles {
+			path := filepath.Join(tmpDir, f)
+			err := os.MkdirAll(filepath.Dir(path), 0600)
+			if err != nil {
+				return nil, err
+			}
+
+			if file, err := os.OpenFile(path, os.O_CREATE, 0600); err != nil {
+				return nil, err
+			} else {
+				file.Close()
+			}
+			files = files + escapeName(f) + "\n"
+		}
+	}
+
+	return CmdStream(exec.Command(args[0], args[1:]...), &files, func() {
+		if tmpDir != "" {
+			_ = os.RemoveAll(tmpDir)
+		}
+	})
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
@@ -101,7 +167,7 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, path string) error {
+func Untar(archive io.Reader, path string, options *TarOptions) error {
 	if archive == nil {
 		return fmt.Errorf("Empty archive")
 	}
@@ -109,22 +175,28 @@ func Untar(archive io.Reader, path string) error {
 	buf := make([]byte, 10)
 	totalN := 0
 	for totalN < 10 {
-		n, err := archive.Read(buf[totalN:])
-		if err != nil {
+		if n, err := archive.Read(buf[totalN:]); err != nil {
 			if err == io.EOF {
 				return fmt.Errorf("Tarball too short")
 			}
 			return err
+		} else {
+			totalN += n
+			utils.Debugf("[tar autodetect] n: %d", n)
 		}
-		totalN += n
-		utils.Debugf("[tar autodetect] n: %d", n)
 	}
-
 	compression := DetectCompression(buf)
 
 	utils.Debugf("Archive compression detected: %s", compression.Extension())
+	args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()}
 
-	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
+	if options != nil {
+		for _, exclude := range options.Excludes {
+			args = append(args, fmt.Sprintf("--exclude=%s", exclude))
+		}
+	}
+
+	cmd := exec.Command("tar", args...)
 	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
@@ -141,11 +213,11 @@ func Untar(archive io.Reader, path string) error {
 // TarUntar aborts and returns the error.
 func TarUntar(src string, filter []string, dst string) error {
 	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
-	archive, err := TarFilter(src, Uncompressed, filter)
+	archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed, Includes: filter, Recursive: true})
 	if err != nil {
 		return err
 	}
-	return Untar(archive, dst)
+	return Untar(archive, dst, nil)
 }
 
 // UntarPath is a convenience function which looks for an archive
@@ -153,7 +225,7 @@ func TarUntar(src string, filter []string, dst string) error {
 func UntarPath(src, dst string) error {
 	if archive, err := os.Open(src); err != nil {
 		return err
-	} else if err := Untar(archive, dst); err != nil {
+	} else if err := Untar(archive, dst, nil); err != nil {
 		return err
 	}
 	return nil
@@ -222,19 +294,39 @@ func CopyFileWithTar(src, dst string) error {
 		return err
 	}
 	tw.Close()
-	return Untar(buf, filepath.Dir(dst))
+	return Untar(buf, filepath.Dir(dst), nil)
 }
 
 // CmdStream executes a command, and returns its stdout as a stream.
 // If the command fails to run or doesn't complete successfully, an error
 // will be returned, including anything written on stderr.
-func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
+func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) {
+	if input != nil {
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			if atEnd != nil {
+				atEnd()
+			}
+			return nil, err
+		}
+		// Write stdin if any
+		go func() {
+			_, _ = stdin.Write([]byte(*input))
+			stdin.Close()
+		}()
+	}
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	pipeR, pipeW := io.Pipe()
@@ -259,6 +351,9 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
 		} else {
 			pipeW.Close()
 		}
+		if atEnd != nil {
+			atEnd()
+		}
 	}()
 	// Run the command and return the pipe
 	if err := cmd.Start(); err != nil {

archive/archive_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestCmdStreamLargeStderr(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -35,7 +35,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
 
 func TestCmdStreamBad(t *testing.T) {
 	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
-	out, err := CmdStream(badCmd)
+	out, err := CmdStream(badCmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -50,7 +50,7 @@ func TestCmdStreamBad(t *testing.T) {
 
 func TestCmdStreamGood(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +83,7 @@ func tarUntar(t *testing.T, origin string, compression Compression) error {
 		return err
 	}
 	defer os.RemoveAll(tmp)
-	if err := Untar(archive, tmp); err != nil {
+	if err := Untar(archive, tmp, nil); err != nil {
 		return err
 	}
 	if _, err := os.Stat(tmp); err != nil {

archive/changes.go

@@ -0,0 +1,301 @@
+package archive
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+type ChangeType int
+
+const (
+	ChangeModify = iota
+	ChangeAdd
+	ChangeDelete
+)
+
+type Change struct {
+	Path string
+	Kind ChangeType
+}
+
+func (change *Change) String() string {
+	var kind string
+	switch change.Kind {
+	case ChangeModify:
+		kind = "C"
+	case ChangeAdd:
+		kind = "A"
+	case ChangeDelete:
+		kind = "D"
+	}
+	return fmt.Sprintf("%s %s", kind, change.Path)
+}
+
+func Changes(layers []string, rw string) ([]Change, error) {
+	var changes []Change
+	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(rw, path)
+		if err != nil {
+			return err
+		}
+		path = filepath.Join("/", path)
+
+		// Skip root
+		if path == "/" {
+			return nil
+		}
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
+			return err
+		}
+
+		change := Change{
+			Path: path,
+		}
+
+		// Find out what kind of modification happened
+		file := filepath.Base(path)
+		// If there is a whiteout, then the file was removed
+		if strings.HasPrefix(file, ".wh.") {
+			originalFile := file[len(".wh."):]
+			change.Path = filepath.Join(filepath.Dir(path), originalFile)
+			change.Kind = ChangeDelete
+		} else {
+			// Otherwise, the file was added
+			change.Kind = ChangeAdd
+
+			// ...Unless it already existed in a top layer, in which case, it's a modification
+			for _, layer := range layers {
+				stat, err := os.Stat(filepath.Join(layer, path))
+				if err != nil && !os.IsNotExist(err) {
+					return err
+				}
+				if err == nil {
+					// The file existed in the top layer, so that's a modification
+
+					// However, if it's a directory, maybe it wasn't actually modified.
+					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
+					if stat.IsDir() && f.IsDir() {
+						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
+							// Both directories are the same, don't record the change
+							return nil
+						}
+					}
+					change.Kind = ChangeModify
+					break
+				}
+			}
+		}
+
+		// Record change
+		changes = append(changes, change)
+		return nil
+	})
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	return changes, nil
+}
+
+type FileInfo struct {
+	parent   *FileInfo
+	name     string
+	stat     syscall.Stat_t
+	children map[string]*FileInfo
+}
+
+func (root *FileInfo) LookUp(path string) *FileInfo {
+	parent := root
+	if path == "/" {
+		return root
+	}
+
+	pathElements := strings.Split(path, "/")
+	for _, elem := range pathElements {
+		if elem != "" {
+			child := parent.children[elem]
+			if child == nil {
+				return nil
+			}
+			parent = child
+		}
+	}
+	return parent
+}
+
+func (info *FileInfo) path() string {
+	if info.parent == nil {
+		return "/"
+	}
+	return filepath.Join(info.parent.path(), info.name)
+}
+
+func (info *FileInfo) isDir() bool {
+	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
+}
+
+func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
+	if oldInfo == nil {
+		// add
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeAdd,
+		}
+		*changes = append(*changes, change)
+	}
+
+	// We make a copy so we can modify it to detect additions
+	// also, we only recurse on the old dir if the new info is a directory
+	// otherwise any previous delete/change is considered recursive
+	oldChildren := make(map[string]*FileInfo)
+	if oldInfo != nil && info.isDir() {
+		for k, v := range oldInfo.children {
+			oldChildren[k] = v
+		}
+	}
+
+	for name, newChild := range info.children {
+		oldChild, _ := oldChildren[name]
+		if oldChild != nil {
+			// change?
+			oldStat := &oldChild.stat
+			newStat := &newChild.stat
+			// Note: We can't compare inode or ctime or blocksize here, because these change
+			// when copying a file into a container. However, that is not generally a problem
+			// because any content change will change mtime, and any status change should
+			// be visible when actually comparing the stat fields. The only time this
+			// breaks down is if some code intentionally hides a change by setting
+			// back mtime
+			if oldStat.Mode != newStat.Mode ||
+				oldStat.Uid != newStat.Uid ||
+				oldStat.Gid != newStat.Gid ||
+				oldStat.Rdev != newStat.Rdev ||
+				// Don't look at size for dirs, its not a good measure of change
+				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
+				oldStat.Mtim != newStat.Mtim {
+				change := Change{
+					Path: newChild.path(),
+					Kind: ChangeModify,
+				}
+				*changes = append(*changes, change)
+			}
+
+			// Remove from copy so we can detect deletions
+			delete(oldChildren, name)
+		}
+
+		newChild.addChanges(oldChild, changes)
+	}
+	for _, oldChild := range oldChildren {
+		// delete
+		change := Change{
+			Path: oldChild.path(),
+			Kind: ChangeDelete,
+		}
+		*changes = append(*changes, change)
+	}
+
+}
+
+func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
+	var changes []Change
+
+	info.addChanges(oldInfo, &changes)
+
+	return changes
+}
+
+func newRootFileInfo() *FileInfo {
+	root := &FileInfo{
+		name:     "/",
+		children: make(map[string]*FileInfo),
+	}
+	return root
+}
+
+func collectFileInfo(sourceDir string) (*FileInfo, error) {
+	root := newRootFileInfo()
+
+	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		relPath, err := filepath.Rel(sourceDir, path)
+		if err != nil {
+			return err
+		}
+		relPath = filepath.Join("/", relPath)
+
+		if relPath == "/" {
+			return nil
+		}
+
+		parent := root.LookUp(filepath.Dir(relPath))
+		if parent == nil {
+			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
+		}
+
+		info := &FileInfo{
+			name:     filepath.Base(relPath),
+			children: make(map[string]*FileInfo),
+			parent:   parent,
+		}
+
+		if err := syscall.Lstat(path, &info.stat); err != nil {
+			return err
+		}
+
+		parent.children[info.name] = info
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return root, nil
+}
+
+// Compare two directories and generate an array of Change objects describing the changes
+func ChangesDirs(newDir, oldDir string) ([]Change, error) {
+	oldRoot, err := collectFileInfo(oldDir)
+	if err != nil {
+		return nil, err
+	}
+	newRoot, err := collectFileInfo(newDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return newRoot.Changes(oldRoot), nil
+}
+
+func ExportChanges(root, rw string) (Archive, error) {
+	changes, err := ChangesDirs(root, rw)
+	if err != nil {
+		return nil, err
+	}
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+	return TarFilter(root, &TarOptions{Compression: Uncompressed, Recursive: false, Includes: files, CreateFiles: deletions})
+}

archive/diff.go

@@ -0,0 +1,63 @@
+package archive
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ApplyLayer parses a diff in the standard layer format from `layer`, and
+// applies it to the directory `dest`.
+func ApplyLayer(dest string, layer Archive) error {
+	// Poor man's diff applyer in 2 steps:
+
+	// Step 1: untar everything in place
+	if err := Untar(layer, dest, nil); err != nil {
+		return err
+	}
+
+	// Step 2: walk for whiteouts and apply them, removing them in the process
+	err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error {
+		if err != nil {
+			if os.IsNotExist(err) {
+				// This happens in the case of whiteouts in parent dir removing a directory
+				// We just ignore it
+				return filepath.SkipDir
+			}
+			return err
+		}
+
+		// Rebase path
+		path, err := filepath.Rel(dest, fullPath)
+		if err != nil {
+			return err
+		}
+		path = filepath.Join("/", path)
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil {
+			return err
+		} else if matched {
+			log.Printf("Removing aufs metadata %s", fullPath)
+			_ = os.RemoveAll(fullPath)
+		}
+
+		filename := filepath.Base(path)
+		if strings.HasPrefix(filename, ".wh.") {
+			rmTargetName := filename[len(".wh."):]
+			rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName)
+			// Remove the file targeted by the whiteout
+			log.Printf("Removing whiteout target %s", rmTargetPath)
+			_ = os.Remove(rmTargetPath)
+			// Remove the whiteout itself
+			log.Printf("Removing whiteout %s", fullPath)
+			_ = os.RemoveAll(fullPath)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

aufs/aufs.go

@@ -0,0 +1,316 @@
+/*
+
+aufs driver directory structure
+
+.
+├── layers // Metadata of layers
+│   ├── 1
+│   ├── 2
+│   └── 3
+├── diffs  // Content of the layer
+│   ├── 1  // Contains layers that need to be mounted for the id
+│   ├── 2
+│   └── 3
+└── mnt    // Mount points for the rw layers to be mounted
+    ├── 1
+    ├── 2
+    └── 3
+
+*/
+
+package aufs
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
+	"github.com/dotcloud/docker/utils"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+)
+
+func init() {
+	graphdriver.Register("aufs", Init)
+}
+
+type AufsDriver struct {
+	root string
+}
+
+// New returns a new AUFS driver.
+// An error is returned if AUFS is not supported.
+func Init(root string) (graphdriver.Driver, error) {
+	// Try to load the aufs kernel module
+	if err := supportsAufs(); err != nil {
+		return nil, err
+	}
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	// Create the root aufs driver dir and return
+	// if it already exists
+	// If not populate the dir structure
+	if err := os.MkdirAll(root, 0755); err != nil {
+		if os.IsExist(err) {
+			return &AufsDriver{root}, nil
+		}
+		return nil, err
+	}
+
+	for _, p := range paths {
+		if err := os.MkdirAll(path.Join(root, p), 0755); err != nil {
+			return nil, err
+		}
+	}
+	return &AufsDriver{root}, nil
+}
+
+// Return a nil error if the kernel supports aufs
+// We cannot modprobe because inside dind modprobe fails
+// to run
+func supportsAufs() error {
+	// We can try to modprobe aufs first before looking at
+	// proc/filesystems for when aufs is supported
+	exec.Command("modprobe", "aufs").Run()
+
+	f, err := os.Open("/proc/filesystems")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		if strings.Contains(s.Text(), "aufs") {
+			return nil
+		}
+	}
+	return fmt.Errorf("AUFS was not found in /proc/filesystems")
+}
+
+func (a *AufsDriver) rootPath() string {
+	return a.root
+}
+
+func (a *AufsDriver) String() string {
+	return "aufs"
+}
+
+// Three folders are created for each id
+// mnt, layers, and diff
+func (a *AufsDriver) Create(id, parent string) error {
+	if err := a.createDirsFor(id); err != nil {
+		return err
+	}
+	// Write the layers metadata
+	f, err := os.Create(path.Join(a.rootPath(), "layers", id))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if parent != "" {
+		ids, err := getParentIds(a.rootPath(), parent)
+		if err != nil {
+			return err
+		}
+
+		if _, err := fmt.Fprintln(f, parent); err != nil {
+			return err
+		}
+		for _, i := range ids {
+			if _, err := fmt.Fprintln(f, i); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (a *AufsDriver) createDirsFor(id string) error {
+	paths := []string{
+		"mnt",
+		"diff",
+	}
+
+	for _, p := range paths {
+		if err := os.MkdirAll(path.Join(a.rootPath(), p, id), 0755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Unmount and remove the dir information
+func (a *AufsDriver) Remove(id string) error {
+	// Make sure the dir is umounted first
+	if err := a.unmount(id); err != nil {
+		return err
+	}
+	tmpDirs := []string{
+		"mnt",
+		"diff",
+	}
+
+	// Remove the dirs atomically
+	for _, p := range tmpDirs {
+		// We need to use a temp dir in the same dir as the driver so Rename
+		// does not fall back to the slow copy if /tmp and the driver dir
+		// are on different devices
+		tmp := path.Join(a.rootPath(), "tmp", p, id)
+		if err := os.MkdirAll(tmp, 0755); err != nil {
+			return err
+		}
+		realPath := path.Join(a.rootPath(), p, id)
+		if err := os.Rename(realPath, tmp); err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmp)
+	}
+
+	// Remove the layers file for the id
+	return os.Remove(path.Join(a.rootPath(), "layers", id))
+}
+
+// Return the rootfs path for the id
+// This will mount the dir at it's given path
+func (a *AufsDriver) Get(id string) (string, error) {
+	ids, err := getParentIds(a.rootPath(), id)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		ids = []string{}
+	}
+
+	// If a dir does not have a parent ( no layers )do not try to mount
+	// just return the diff path to the data
+	out := path.Join(a.rootPath(), "diff", id)
+	if len(ids) > 0 {
+		out = path.Join(a.rootPath(), "mnt", id)
+		if err := a.mount(id); err != nil {
+			return "", err
+		}
+	}
+	return out, nil
+}
+
+// Returns an archive of the contents for the id
+func (a *AufsDriver) Diff(id string) (archive.Archive, error) {
+	return archive.TarFilter(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
+		Recursive:   true,
+		Compression: archive.Uncompressed,
+	})
+}
+
+func (a *AufsDriver) ApplyDiff(id string, diff archive.Archive) error {
+	return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil)
+}
+
+// Returns the size of the contents for the id
+func (a *AufsDriver) Size(id string) (int64, error) {
+	return utils.TreeSize(path.Join(a.rootPath(), "diff", id))
+}
+
+func (a *AufsDriver) Changes(id string) ([]archive.Change, error) {
+	layers, err := a.getParentLayerPaths(id)
+	if err != nil {
+		return nil, err
+	}
+	return archive.Changes(layers, path.Join(a.rootPath(), "diff", id))
+}
+
+func (a *AufsDriver) getParentLayerPaths(id string) ([]string, error) {
+	parentIds, err := getParentIds(a.rootPath(), id)
+	if err != nil {
+		return nil, err
+	}
+	if len(parentIds) == 0 {
+		return nil, fmt.Errorf("Dir %s does not have any parent layers", id)
+	}
+	layers := make([]string, len(parentIds))
+
+	// Get the diff paths for all the parent ids
+	for i, p := range parentIds {
+		layers[i] = path.Join(a.rootPath(), "diff", p)
+	}
+	return layers, nil
+}
+
+func (a *AufsDriver) mount(id string) error {
+	// If the id is mounted or we get an error return
+	if mounted, err := a.mounted(id); err != nil || mounted {
+		return err
+	}
+
+	var (
+		target = path.Join(a.rootPath(), "mnt", id)
+		rw     = path.Join(a.rootPath(), "diff", id)
+	)
+
+	layers, err := a.getParentLayerPaths(id)
+	if err != nil {
+		return err
+	}
+
+	if err := a.aufsMount(layers, rw, target); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *AufsDriver) unmount(id string) error {
+	if mounted, err := a.mounted(id); err != nil || !mounted {
+		return err
+	}
+	target := path.Join(a.rootPath(), "mnt", id)
+	return Unmount(target)
+}
+
+func (a *AufsDriver) mounted(id string) (bool, error) {
+	target := path.Join(a.rootPath(), "mnt", id)
+	return Mounted(target)
+}
+
+// During cleanup aufs needs to unmount all mountpoints
+func (a *AufsDriver) Cleanup() error {
+	ids, err := loadIds(path.Join(a.rootPath(), "layers"))
+	if err != nil {
+		return err
+	}
+	for _, id := range ids {
+		if err := a.unmount(id); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *AufsDriver) aufsMount(ro []string, rw, target string) error {
+	rwBranch := fmt.Sprintf("%v=rw", rw)
+	roBranches := ""
+	for _, layer := range ro {
+		roBranches += fmt.Sprintf("%v=ro+wh:", layer)
+	}
+	branches := fmt.Sprintf("br:%v:%v,xino=/dev/shm/aufs.xino", rwBranch, roBranches)
+
+	//if error, try to load aufs kernel module
+	if err := mount("none", target, "aufs", 0, branches); err != nil {
+		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
+		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
+			return fmt.Errorf("Unable to load the AUFS module")
+		}
+		log.Printf("...module loaded.")
+		if err := mount("none", target, "aufs", 0, branches); err != nil {
+			return fmt.Errorf("Unable to mount using aufs %s", err)
+		}
+	}
+	return nil
+}

aufs/aufs_test.go

@@ -0,0 +1,517 @@
+package aufs
+
+import (
+	"github.com/dotcloud/docker/archive"
+	"os"
+	"path"
+	"testing"
+)
+
+var (
+	tmp = path.Join(os.TempDir(), "aufs-tests", "aufs")
+)
+
+func newDriver(t *testing.T) *AufsDriver {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	d, err := Init(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d.(*AufsDriver)
+}
+
+func TestNewAufsDriver(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	d, err := Init(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if d == nil {
+		t.Fatalf("Driver should not be nil")
+	}
+}
+
+func TestAufsString(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if d.String() != "aufs" {
+		t.Fatalf("Expected aufs got %s", d.String())
+	}
+}
+
+func TestCreateDirStructure(t *testing.T) {
+	newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	paths := []string{
+		"mnt",
+		"layers",
+		"diff",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// We should be able to create two drivers with the same dir structure
+func TestNewDriverFromExistingDir(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := Init(tmp); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Init(tmp); err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll(tmp)
+}
+
+func TestCreateNewDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCreateNewDirStructure(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p, "1")); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRemoveImage(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Remove("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
+			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
+		}
+	}
+}
+
+func TestGetWithoutParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := path.Join(tmp, "diff", "1")
+	if diffPath != expected {
+		t.Fatalf("Expected path %s got %s", expected, diffPath)
+	}
+}
+
+func TestCleanupWithNoDirs(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCleanupWithDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMountedFalseResponse(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	response, err := d.mounted("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response != false {
+		t.Fatalf("Response if dir id 1 is mounted should be false")
+	}
+}
+
+func TestMountedTrueReponse(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+	defer d.Cleanup()
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	response, err := d.mounted("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response != true {
+		t.Fatalf("Response if dir id 2 is mounted should be true")
+	}
+}
+
+func TestMountWithParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPath, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mntPath == "" {
+		t.Fatal("mntPath should not be empty string")
+	}
+
+	expected := path.Join(tmp, "mnt", "2")
+	if mntPath != expected {
+		t.Fatalf("Expected %s got %s", expected, mntPath)
+	}
+}
+
+func TestRemoveMountedDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPath, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mntPath == "" {
+		t.Fatal("mntPath should not be empty string")
+	}
+
+	mounted, err := d.mounted("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !mounted {
+		t.Fatalf("Dir id 2 should be mounted")
+	}
+
+	if err := d.Remove("2"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCreateWithInvalidParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", "docker"); err == nil {
+		t.Fatalf("Error should not be nil with parent does not exist")
+	}
+}
+
+func TestGetDiff(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Truncate(size); err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+
+	a, err := d.Diff("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a == nil {
+		t.Fatalf("Archive should not be nil")
+	}
+}
+
+func TestChanges(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPoint, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a file to save in the mountpoint
+	f, err := os.Create(path.Join(mntPoint, "test.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := f.WriteString("testline"); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err := d.Changes("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(changes) != 1 {
+		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
+	}
+	change := changes[0]
+
+	expectedPath := "/test.txt"
+	if change.Path != expectedPath {
+		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
+	}
+
+	if change.Kind != archive.ChangeAdd {
+		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
+	}
+
+	if err := d.Create("3", "2"); err != nil {
+		t.Fatal(err)
+	}
+	mntPoint, err = d.Get("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a file to save in the mountpoint
+	f, err = os.Create(path.Join(mntPoint, "test2.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := f.WriteString("testline"); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err = d.Changes("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(changes) != 1 {
+		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
+	}
+	change = changes[0]
+
+	expectedPath = "/test2.txt"
+	if change.Path != expectedPath {
+		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
+	}
+
+	if change.Kind != archive.ChangeAdd {
+		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
+	}
+}
+
+func TestDiffSize(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Truncate(size)
+	s, err := f.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	size = s.Size()
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	diffSize, err := d.Size("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diffSize != size {
+		t.Fatalf("Expected size to be %d got %d", size, diffSize)
+	}
+}
+
+func TestApplyDiff(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+	defer d.Cleanup()
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Truncate(size)
+	f.Close()
+
+	diff, err := d.Diff("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Create("2", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("3", "2"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.ApplyDiff("3", diff); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that the file is in the mount point for id 3
+
+	mountPoint, err := d.Get("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path.Join(mountPoint, "test_file")); err != nil {
+		t.Fatal(err)
+	}
+}

aufs/dirs.go

@@ -0,0 +1,46 @@
+package aufs
+
+import (
+	"bufio"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+// Return all the directories
+func loadIds(root string) ([]string, error) {
+	dirs, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+	out := []string{}
+	for _, d := range dirs {
+		if !d.IsDir() {
+			out = append(out, d.Name())
+		}
+	}
+	return out, nil
+}
+
+// Read the layers file for the current id and return all the
+// layers represented by new lines in the file
+//
+// If there are no lines in the file then the id has no parent
+// and an empty slice is returned.
+func getParentIds(root, id string) ([]string, error) {
+	f, err := os.Open(path.Join(root, "layers", id))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	out := []string{}
+	s := bufio.NewScanner(f)
+
+	for s.Scan() {
+		if t := s.Text(); t != "" {
+			out = append(out, s.Text())
+		}
+	}
+	return out, s.Err()
+}

aufs/mount.go

@@ -1,13 +1,11 @@
-package docker
+package aufs
 
 import (
-	"fmt"
 	"github.com/dotcloud/docker/utils"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"syscall"
-	"time"
 )
 
 func Unmount(target string) error {
@@ -17,22 +15,7 @@ func Unmount(target string) error {
 	if err := syscall.Unmount(target, 0); err != nil {
 		return err
 	}
-	// Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint
-	// for some time. We'll just keep retrying until it succeeds.
-	for retries := 0; retries < 1000; retries++ {
-		err := os.Remove(target)
-		if err == nil {
-			// rm mntpoint succeeded
-			return nil
-		}
-		if os.IsNotExist(err) {
-			// mntpoint doesn't exist anymore. Success.
-			return nil
-		}
-		// fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err)
-		time.Sleep(10 * time.Millisecond)
-	}
-	return fmt.Errorf("Umount: Failed to umount %v", target)
+	return nil
 }
 
 func Mounted(mountpoint string) (bool, error) {
@@ -49,5 +32,6 @@ func Mounted(mountpoint string) (bool, error) {
 	}
 	mntpointSt := mntpoint.Sys().(*syscall.Stat_t)
 	parentSt := parent.Sys().(*syscall.Stat_t)
+
 	return mntpointSt.Dev != parentSt.Dev, nil
 }

aufs/mount_darwin.go

@@ -1,4 +1,4 @@
-package docker
+package aufs
 
 import "errors"
 

aufs/mount_linux.go

@@ -1,4 +1,4 @@
-package docker
+package aufs
 
 import "syscall"
 

auth/auth.go

@@ -196,9 +196,10 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e
 		if loginAgainstOfficialIndex {
 			return "", fmt.Errorf("Login: Your account hasn't been activated. " +
 				"Please check your e-mail for a confirmation link.")
+		} else {
+			return "", fmt.Errorf("Login: Your account hasn't been activated. " +
+				"Please see the documentation of the registry " + serverAddress + " for instructions how to activate it.")
 		}
-		return "", fmt.Errorf("Login: Your account hasn't been activated. " +
-			"Please see the documentation of the registry " + serverAddress + " for instructions how to activate it.")
 	} else if reqStatusCode == 400 {
 		if string(reqBody) == "\"Username or email already exists\"" {
 			req, err := factory.NewRequest("GET", serverAddress+"users/", nil)

auth/auth_test.go

@@ -1,8 +1,11 @@
 package auth
 
 import (
+	"crypto/rand"
+	"encoding/hex"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 )
 
@@ -26,6 +29,52 @@ func TestEncodeAuth(t *testing.T) {
 	}
 }
 
+func TestLogin(t *testing.T) {
+	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
+	defer os.Setenv("DOCKER_INDEX_URL", "")
+	authConfig := &AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"}
+	status, err := Login(authConfig, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status != "Login Succeeded" {
+		t.Fatalf("Expected status \"Login Succeeded\", found \"%s\" instead", status)
+	}
+}
+
+func TestCreateAccount(t *testing.T) {
+	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
+	defer os.Setenv("DOCKER_INDEX_URL", "")
+	tokenBuffer := make([]byte, 16)
+	_, err := rand.Read(tokenBuffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	token := hex.EncodeToString(tokenBuffer)[:12]
+	username := "ut" + token
+	authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"}
+	status, err := Login(authConfig, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedStatus := "Account created. Please use the confirmation link we sent" +
+		" to your e-mail to activate it."
+	if status != expectedStatus {
+		t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status)
+	}
+
+	status, err = Login(authConfig, nil)
+	if err == nil {
+		t.Fatalf("Expected error but found nil instead")
+	}
+
+	expectedError := "Login: Account is not Active"
+
+	if !strings.Contains(err.Error(), expectedError) {
+		t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err)
+	}
+}
+
 func setupTempConfigFile() (*ConfigFile, error) {
 	root, err := ioutil.TempDir("", "docker-test-auth")
 	if err != nil {

buildfile.go

@@ -476,7 +476,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	if err := archive.Untar(context, name); err != nil {
+	if err := archive.Untar(context, name, nil); err != nil {
 		return "", err
 	}
 	defer os.RemoveAll(name)

buildfile_test.go

@@ -2,9 +2,7 @@ package docker
 
 import (
 	"fmt"
-	"github.com/dotcloud/docker"
 	"github.com/dotcloud/docker/archive"
-	"github.com/dotcloud/docker/engine"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -16,7 +14,7 @@ import (
 // mkTestContext generates a build context from the contents of the provided dockerfile.
 // This context is suitable for use as an argument to BuildFile.Build()
 func mkTestContext(dockerfile string, files [][2]string, t *testing.T) archive.Archive {
-	context, err := docker.MkBuildContext(dockerfile, files)
+	context, err := mkBuildContext(dockerfile, files)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -230,15 +228,17 @@ func TestBuild(t *testing.T) {
 	}
 }
 
-func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, useCache bool) *docker.Image {
-	if eng == nil {
-		eng = NewTestEngine(t)
-		runtime := mkRuntimeFromEngine(eng, t)
-		// FIXME: we might not need runtime, why not simply nuke
-		// the engine?
+func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image {
+	if srv == nil {
+		runtime := mkRuntime(t)
 		defer nuke(runtime)
+
+		srv = &Server{
+			runtime:     runtime,
+			pullingPool: make(map[string]struct{}),
+			pushingPool: make(map[string]struct{}),
+		}
 	}
-	srv := mkServerFromEngine(eng, t)
 
 	httpServer, err := mkTestingFileServer(context.remoteFiles)
 	if err != nil {
@@ -252,17 +252,10 @@ func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, u
 	}
 	port := httpServer.URL[idx+1:]
 
-	iIP := eng.Hack_GetGlobalVar("httpapi.bridgeIP")
-	if iIP == nil {
-		t.Fatal("Legacy bridgeIP field not set in engine")
-	}
-	ip, ok := iIP.(net.IP)
-	if !ok {
-		panic("Legacy bridgeIP field in engine does not cast to net.IP")
-	}
+	ip := srv.runtime.networkManager.bridgeNetwork.IP
 	dockerfile := constructDockerfile(context.dockerfile, ip, port)
 
-	buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, useCache, false)
+	buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache, false)
 	id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t))
 	if err != nil {
 		t.Fatal(err)
@@ -375,14 +368,20 @@ func TestBuildEntrypoint(t *testing.T) {
 // testing #1405 - config.Cmd does not get cleaned up if
 // utilizing cache
 func TestBuildEntrypointRunCleanup(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	img := buildImage(testContextTemplate{`
         from {IMAGE}
         run echo "hello"
         `,
-		nil, nil}, t, eng, true)
+		nil, nil}, t, srv, true)
 
 	img = buildImage(testContextTemplate{`
         from {IMAGE}
@@ -390,7 +389,7 @@ func TestBuildEntrypointRunCleanup(t *testing.T) {
         add foo /foo
         entrypoint ["/bin/echo"]
         `,
-		[][2]string{{"foo", "HEYO"}}, nil}, t, eng, true)
+		[][2]string{{"foo", "HEYO"}}, nil}, t, srv, true)
 
 	if len(img.Config.Cmd) != 0 {
 		t.Fail()
@@ -398,8 +397,14 @@ func TestBuildEntrypointRunCleanup(t *testing.T) {
 }
 
 func TestBuildImageWithCache(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	template := testContextTemplate{`
         from {IMAGE}
@@ -407,11 +412,11 @@ func TestBuildImageWithCache(t *testing.T) {
         `,
 		nil, nil}
 
-	img := buildImage(template, t, eng, true)
+	img := buildImage(template, t, srv, true)
 	imageId := img.ID
 
 	img = nil
-	img = buildImage(template, t, eng, true)
+	img = buildImage(template, t, srv, true)
 
 	if imageId != img.ID {
 		t.Logf("Image ids should match: %s != %s", imageId, img.ID)
@@ -420,8 +425,14 @@ func TestBuildImageWithCache(t *testing.T) {
 }
 
 func TestBuildImageWithoutCache(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	template := testContextTemplate{`
         from {IMAGE}
@@ -429,11 +440,11 @@ func TestBuildImageWithoutCache(t *testing.T) {
         `,
 		nil, nil}
 
-	img := buildImage(template, t, eng, true)
+	img := buildImage(template, t, srv, true)
 	imageId := img.ID
 
 	img = nil
-	img = buildImage(template, t, eng, false)
+	img = buildImage(template, t, srv, false)
 
 	if imageId == img.ID {
 		t.Logf("Image ids should not match: %s == %s", imageId, img.ID)
@@ -442,9 +453,14 @@ func TestBuildImageWithoutCache(t *testing.T) {
 }
 
 func TestForbiddenContextPath(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
-	srv := mkServerFromEngine(eng, t)
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	context := testContextTemplate{`
         from {IMAGE}
@@ -465,17 +481,10 @@ func TestForbiddenContextPath(t *testing.T) {
 	}
 	port := httpServer.URL[idx+1:]
 
-	iIP := eng.Hack_GetGlobalVar("httpapi.bridgeIP")
-	if iIP == nil {
-		t.Fatal("Legacy bridgeIP field not set in engine")
-	}
-	ip, ok := iIP.(net.IP)
-	if !ok {
-		panic("Legacy bridgeIP field in engine does not cast to net.IP")
-	}
+	ip := srv.runtime.networkManager.bridgeNetwork.IP
 	dockerfile := constructDockerfile(context.dockerfile, ip, port)
 
-	buildfile := docker.NewBuildFile(srv, ioutil.Discard, false, true, false)
+	buildfile := NewBuildFile(srv, ioutil.Discard, false, true, false)
 	_, err = buildfile.Build(mkTestContext(dockerfile, context.files, t))
 
 	if err == nil {
@@ -490,8 +499,14 @@ func TestForbiddenContextPath(t *testing.T) {
 }
 
 func TestBuildADDFileNotFound(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	context := testContextTemplate{`
         from {IMAGE}
@@ -511,17 +526,10 @@ func TestBuildADDFileNotFound(t *testing.T) {
 	}
 	port := httpServer.URL[idx+1:]
 
-	iIP := eng.Hack_GetGlobalVar("httpapi.bridgeIP")
-	if iIP == nil {
-		t.Fatal("Legacy bridgeIP field not set in engine")
-	}
-	ip, ok := iIP.(net.IP)
-	if !ok {
-		panic("Legacy bridgeIP field in engine does not cast to net.IP")
-	}
+	ip := srv.runtime.networkManager.bridgeNetwork.IP
 	dockerfile := constructDockerfile(context.dockerfile, ip, port)
 
-	buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, false, true, false)
+	buildfile := NewBuildFile(srv, ioutil.Discard, false, true, false)
 	_, err = buildfile.Build(mkTestContext(dockerfile, context.files, t))
 
 	if err == nil {
@@ -536,20 +544,26 @@ func TestBuildADDFileNotFound(t *testing.T) {
 }
 
 func TestBuildInheritance(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
 
 	img := buildImage(testContextTemplate{`
             from {IMAGE}
             expose 4243
             `,
-		nil, nil}, t, eng, true)
+		nil, nil}, t, srv, true)
 
 	img2 := buildImage(testContextTemplate{fmt.Sprintf(`
             from %s
             entrypoint ["/bin/echo"]
             `, img.ID),
-		nil, nil}, t, eng, true)
+		nil, nil}, t, srv, true)
 
 	// from child
 	if img2.Config.Entrypoint[0] != "/bin/echo" {

changes.go

@@ -1,106 +0,0 @@
-package docker
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-type ChangeType int
-
-const (
-	ChangeModify = iota
-	ChangeAdd
-	ChangeDelete
-)
-
-type Change struct {
-	Path string
-	Kind ChangeType
-}
-
-func (change *Change) String() string {
-	var kind string
-	switch change.Kind {
-	case ChangeModify:
-		kind = "C"
-	case ChangeAdd:
-		kind = "A"
-	case ChangeDelete:
-		kind = "D"
-	}
-	return fmt.Sprintf("%s %s", kind, change.Path)
-}
-
-func Changes(layers []string, rw string) ([]Change, error) {
-	var changes []Change
-	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Rebase path
-		path, err = filepath.Rel(rw, path)
-		if err != nil {
-			return err
-		}
-		path = filepath.Join("/", path)
-
-		// Skip root
-		if path == "/" {
-			return nil
-		}
-
-		// Skip AUFS metadata
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
-			return err
-		}
-
-		change := Change{
-			Path: path,
-		}
-
-		// Find out what kind of modification happened
-		file := filepath.Base(path)
-		// If there is a whiteout, then the file was removed
-		if strings.HasPrefix(file, ".wh.") {
-			originalFile := file[len(".wh."):]
-			change.Path = filepath.Join(filepath.Dir(path), originalFile)
-			change.Kind = ChangeDelete
-		} else {
-			// Otherwise, the file was added
-			change.Kind = ChangeAdd
-
-			// ...Unless it already existed in a top layer, in which case, it's a modification
-			for _, layer := range layers {
-				stat, err := os.Stat(filepath.Join(layer, path))
-				if err != nil && !os.IsNotExist(err) {
-					return err
-				}
-				if err == nil {
-					// The file existed in the top layer, so that's a modification
-
-					// However, if it's a directory, maybe it wasn't actually modified.
-					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
-					if stat.IsDir() && f.IsDir() {
-						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
-							// Both directories are the same, don't record the change
-							return nil
-						}
-					}
-					change.Kind = ChangeModify
-					break
-				}
-			}
-		}
-
-		// Record change
-		changes = append(changes, change)
-		return nil
-	})
-	if err != nil && !os.IsNotExist(err) {
-		return nil, err
-	}
-	return changes, nil
-}

commands.go

@@ -22,7 +22,6 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
-	"path"
 	"path/filepath"
 	"reflect"
 	"regexp"
@@ -93,7 +92,6 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 		{"insert", "Insert a file in an image"},
 		{"inspect", "Return low-level information on a container"},
 		{"kill", "Kill a running container"},
-		{"load", "Load an image from a tar archive"},
 		{"login", "Register or Login to the docker registry server"},
 		{"logs", "Fetch the logs of a container"},
 		{"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"},
@@ -104,7 +102,6 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 		{"rm", "Remove one or more containers"},
 		{"rmi", "Remove one or more images"},
 		{"run", "Run a command in a new container"},
-		{"save", "Save an image to a tar archive"},
 		{"search", "Search for an image in the docker index"},
 		{"start", "Start a stopped container"},
 		{"stop", "Stop a running container"},
@@ -120,7 +117,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 }
 
 func (cli *DockerCli) CmdInsert(args ...string) error {
-	cmd := cli.Subcmd("insert", "IMAGE URL PATH", "Insert a file from URL in the IMAGE at PATH")
+	cmd := Subcmd("insert", "IMAGE URL PATH", "Insert a file from URL in the IMAGE at PATH")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -138,7 +135,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
 
 // mkBuildContext returns an archive of an empty context with the contents
 // of `dockerfile` at the path ./Dockerfile
-func MkBuildContext(dockerfile string, files [][2]string) (archive.Archive, error) {
+func mkBuildContext(dockerfile string, files [][2]string) (archive.Archive, error) {
 	buf := new(bytes.Buffer)
 	tw := tar.NewWriter(buf)
 	files = append(files, [2]string{"Dockerfile", dockerfile})
@@ -162,7 +159,7 @@ func MkBuildContext(dockerfile string, files [][2]string) (archive.Archive, erro
 }
 
 func (cli *DockerCli) CmdBuild(args ...string) error {
-	cmd := cli.Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH")
+	cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH")
 	tag := cmd.String("t", "", "Repository name (and optionally a tag) to be applied to the resulting image in case of success")
 	suppressOutput := cmd.Bool("q", false, "Suppress verbose build output")
 	noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image")
@@ -188,7 +185,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 		if err != nil {
 			return err
 		}
-		context, err = MkBuildContext(string(dockerfile), nil)
+		context, err = mkBuildContext(string(dockerfile), nil)
 	} else if utils.IsURL(cmd.Arg(0)) || utils.IsGIT(cmd.Arg(0)) {
 		isRemote = true
 	} else {
@@ -260,7 +257,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error {
 
 // 'docker login': login / register a user to registry service.
 func (cli *DockerCli) CmdLogin(args ...string) error {
-	cmd := cli.Subcmd("login", "[OPTIONS] [SERVER]", "Register or Login to a docker registry server, if no server is specified \""+auth.IndexServerAddress()+"\" is the default.")
+	cmd := Subcmd("login", "[OPTIONS] [SERVER]", "Register or Login to a docker registry server, if no server is specified \""+auth.IndexServerAddress()+"\" is the default.")
 
 	var username, password, email string
 
@@ -368,7 +365,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error {
 
 // 'docker wait': block until a container stops
 func (cli *DockerCli) CmdWait(args ...string) error {
-	cmd := cli.Subcmd("wait", "CONTAINER [CONTAINER...]", "Block until a container stops, then print its exit code.")
+	cmd := Subcmd("wait", "CONTAINER [CONTAINER...]", "Block until a container stops, then print its exit code.")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -391,7 +388,7 @@ func (cli *DockerCli) CmdWait(args ...string) error {
 
 // 'docker version': show version information
 func (cli *DockerCli) CmdVersion(args ...string) error {
-	cmd := cli.Subcmd("version", "", "Show the docker version information.")
+	cmd := Subcmd("version", "", "Show the docker version information.")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -442,7 +439,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error {
 
 // 'docker info': display system-wide information.
 func (cli *DockerCli) CmdInfo(args ...string) error {
-	cmd := cli.Subcmd("info", "", "Display system-wide information")
+	cmd := Subcmd("info", "", "Display system-wide information")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -494,7 +491,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error {
 }
 
 func (cli *DockerCli) CmdStop(args ...string) error {
-	cmd := cli.Subcmd("stop", "[OPTIONS] CONTAINER [CONTAINER...]", "Stop a running container (Send SIGTERM, and then SIGKILL after grace period)")
+	cmd := Subcmd("stop", "[OPTIONS] CONTAINER [CONTAINER...]", "Stop a running container (Send SIGTERM, and then SIGKILL after grace period)")
 	nSeconds := cmd.Int("t", 10, "Number of seconds to wait for the container to stop before killing it.")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -521,7 +518,7 @@ func (cli *DockerCli) CmdStop(args ...string) error {
 }
 
 func (cli *DockerCli) CmdRestart(args ...string) error {
-	cmd := cli.Subcmd("restart", "[OPTIONS] CONTAINER [CONTAINER...]", "Restart a running container")
+	cmd := Subcmd("restart", "[OPTIONS] CONTAINER [CONTAINER...]", "Restart a running container")
 	nSeconds := cmd.Int("t", 10, "Number of seconds to try to stop for before killing the container. Once killed it will then be restarted. Default=10")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -552,9 +549,6 @@ func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal {
 	utils.CatchAll(sigc)
 	go func() {
 		for s := range sigc {
-			if s == syscall.SIGCHLD {
-				continue
-			}
 			if _, _, err := cli.call("POST", fmt.Sprintf("/containers/%s/kill?signal=%d", cid, s), nil); err != nil {
 				utils.Debugf("Error sending signal: %s", err)
 			}
@@ -564,7 +558,7 @@ func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal {
 }
 
 func (cli *DockerCli) CmdStart(args ...string) error {
-	cmd := cli.Subcmd("start", "CONTAINER [CONTAINER...]", "Restart a stopped container")
+	cmd := Subcmd("start", "CONTAINER [CONTAINER...]", "Restart a stopped container")
 	attach := cmd.Bool("a", false, "Attach container's stdout/stderr and forward all signals to the process")
 	openStdin := cmd.Bool("i", false, "Attach container's stdin")
 	if err := cmd.Parse(args); err != nil {
@@ -576,7 +570,6 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 	}
 
 	var cErr chan error
-	var tty bool
 	if *attach || *openStdin {
 		if cmd.NArg() > 1 {
 			return fmt.Errorf("Impossible to start and attach multiple containers at once.")
@@ -593,13 +586,17 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 			return err
 		}
 
-		tty = container.Config.Tty
-
 		if !container.Config.Tty {
 			sigc := cli.forwardAllSignals(cmd.Arg(0))
 			defer utils.StopCatch(sigc)
 		}
 
+		if container.Config.Tty && cli.isTerminal {
+			if err := cli.monitorTtySize(cmd.Arg(0)); err != nil {
+				return err
+			}
+		}
+
 		var in io.ReadCloser
 
 		v := url.Values{}
@@ -637,20 +634,14 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 		}
 		return encounteredError
 	}
-
 	if *openStdin || *attach {
-		if tty && cli.isTerminal {
-			if err := cli.monitorTtySize(cmd.Arg(0)); err != nil {
-				utils.Errorf("Error monitoring TTY size: %s\n", err)
-			}
-		}
 		return <-cErr
 	}
 	return nil
 }
 
 func (cli *DockerCli) CmdInspect(args ...string) error {
-	cmd := cli.Subcmd("inspect", "CONTAINER|IMAGE [CONTAINER|IMAGE...]", "Return low-level information on a container/image")
+	cmd := Subcmd("inspect", "CONTAINER|IMAGE [CONTAINER|IMAGE...]", "Return low-level information on a container/image")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -701,7 +692,7 @@ func (cli *DockerCli) CmdInspect(args ...string) error {
 }
 
 func (cli *DockerCli) CmdTop(args ...string) error {
-	cmd := cli.Subcmd("top", "CONTAINER [ps OPTIONS]", "Lookup the running processes of a container")
+	cmd := Subcmd("top", "CONTAINER [ps OPTIONS]", "Lookup the running processes of a container")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -733,7 +724,7 @@ func (cli *DockerCli) CmdTop(args ...string) error {
 }
 
 func (cli *DockerCli) CmdPort(args ...string) error {
-	cmd := cli.Subcmd("port", "CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT")
+	cmd := Subcmd("port", "CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -775,7 +766,7 @@ func (cli *DockerCli) CmdPort(args ...string) error {
 
 // 'docker rmi IMAGE' removes all images with the name IMAGE
 func (cli *DockerCli) CmdRmi(args ...string) error {
-	cmd := cli.Subcmd("rmi", "IMAGE [IMAGE...]", "Remove one or more images")
+	cmd := Subcmd("rmi", "IMAGE [IMAGE...]", "Remove one or more images")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -811,7 +802,7 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 }
 
 func (cli *DockerCli) CmdHistory(args ...string) error {
-	cmd := cli.Subcmd("history", "[OPTIONS] IMAGE", "Show the history of an image")
+	cmd := Subcmd("history", "[OPTIONS] IMAGE", "Show the history of an image")
 	quiet := cmd.Bool("q", false, "only show numeric IDs")
 	noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
 
@@ -868,7 +859,7 @@ func (cli *DockerCli) CmdHistory(args ...string) error {
 }
 
 func (cli *DockerCli) CmdRm(args ...string) error {
-	cmd := cli.Subcmd("rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove one or more containers")
+	cmd := Subcmd("rm", "[OPTIONS] CONTAINER [CONTAINER...]", "Remove one or more containers")
 	v := cmd.Bool("v", false, "Remove the volumes associated to the container")
 	link := cmd.Bool("link", false, "Remove the specified link and not the underlying container")
 
@@ -902,7 +893,7 @@ func (cli *DockerCli) CmdRm(args ...string) error {
 
 // 'docker kill NAME' kills a running container
 func (cli *DockerCli) CmdKill(args ...string) error {
-	cmd := cli.Subcmd("kill", "CONTAINER [CONTAINER...]", "Kill a running container (send SIGKILL)")
+	cmd := Subcmd("kill", "CONTAINER [CONTAINER...]", "Kill a running container (send SIGKILL)")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -924,7 +915,7 @@ func (cli *DockerCli) CmdKill(args ...string) error {
 }
 
 func (cli *DockerCli) CmdImport(args ...string) error {
-	cmd := cli.Subcmd("import", "URL|- [REPOSITORY[:TAG]]", "Create a new filesystem image from the contents of a tarball(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).")
+	cmd := Subcmd("import", "URL|- [REPOSITORY[:TAG]]", "Create a new filesystem image from the contents of a tarball(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).")
 
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -958,7 +949,7 @@ func (cli *DockerCli) CmdImport(args ...string) error {
 }
 
 func (cli *DockerCli) CmdPush(args ...string) error {
-	cmd := cli.Subcmd("push", "NAME", "Push an image or a repository to the registry")
+	cmd := Subcmd("push", "NAME", "Push an image or a repository to the registry")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1020,7 +1011,7 @@ func (cli *DockerCli) CmdPush(args ...string) error {
 }
 
 func (cli *DockerCli) CmdPull(args ...string) error {
-	cmd := cli.Subcmd("pull", "NAME", "Pull an image or a repository from the registry")
+	cmd := Subcmd("pull", "NAME", "Pull an image or a repository from the registry")
 	tag := cmd.String("t", "", "Download tagged image in repository")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -1080,9 +1071,9 @@ func (cli *DockerCli) CmdPull(args ...string) error {
 }
 
 func (cli *DockerCli) CmdImages(args ...string) error {
-	cmd := cli.Subcmd("images", "[OPTIONS] [NAME]", "List images")
+	cmd := Subcmd("images", "[OPTIONS] [NAME]", "List images")
 	quiet := cmd.Bool("q", false, "only show numeric IDs")
-	all := cmd.Bool("a", false, "show all images (by default filter out the intermediate images used to build)")
+	all := cmd.Bool("a", false, "show all images")
 	noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
 	flViz := cmd.Bool("viz", false, "output graph in graphviz format")
 	flTree := cmd.Bool("tree", false, "output graph in tree format")
@@ -1128,16 +1119,18 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 		}
 
 		var outs []APIImages
-		err = json.Unmarshal(body, &outs)
-		if err != nil {
+		if err := json.Unmarshal(body, &outs); err != nil {
 			return err
 		}
 
-		var startImageArg = cmd.Arg(0)
-		var startImage APIImages
+		var (
+			startImageArg = cmd.Arg(0)
+			startImage    APIImages
+
+			roots    []APIImages
+			byParent = make(map[string][]APIImages)
+		)
 
-		var roots []APIImages
-		var byParent = make(map[string][]APIImages)
 		for _, image := range outs {
 			if image.ParentId == "" {
 				roots = append(roots, image)
@@ -1277,7 +1270,7 @@ func displayablePorts(ports []APIPort) string {
 }
 
 func (cli *DockerCli) CmdPs(args ...string) error {
-	cmd := cli.Subcmd("ps", "[OPTIONS]", "List containers")
+	cmd := Subcmd("ps", "[OPTIONS]", "List containers")
 	quiet := cmd.Bool("q", false, "Only display numeric IDs")
 	size := cmd.Bool("s", false, "Display sizes")
 	all := cmd.Bool("a", false, "Show all containers. Only running containers are shown by default.")
@@ -1366,10 +1359,10 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 }
 
 func (cli *DockerCli) CmdCommit(args ...string) error {
-	cmd := cli.Subcmd("commit", "[OPTIONS] CONTAINER [REPOSITORY[:TAG]]", "Create a new image from a container's changes")
+	cmd := Subcmd("commit", "[OPTIONS] CONTAINER [REPOSITORY[:TAG]]", "Create a new image from a container's changes")
 	flComment := cmd.String("m", "", "Commit message")
 	flAuthor := cmd.String("author", "", "Author (eg. \"John Hannibal Smith <hannibal@a-team.com>\"")
-	flConfig := cmd.String("run", "", "Config automatically applied when the image is run. "+`(ex: -run='{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')`)
+	flConfig := cmd.String("run", "", "Config automatically applied when the image is run. "+`(ex: {"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')`)
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1418,7 +1411,7 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 }
 
 func (cli *DockerCli) CmdEvents(args ...string) error {
-	cmd := cli.Subcmd("events", "[OPTIONS]", "Get real time events from the server")
+	cmd := Subcmd("events", "[OPTIONS]", "Get real time events from the server")
 	since := cmd.String("since", "", "Show previously created events and then stream.")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -1451,7 +1444,7 @@ func (cli *DockerCli) CmdEvents(args ...string) error {
 }
 
 func (cli *DockerCli) CmdExport(args ...string) error {
-	cmd := cli.Subcmd("export", "CONTAINER", "Export the contents of a filesystem as a tar archive")
+	cmd := Subcmd("export", "CONTAINER", "Export the contents of a filesystem as a tar archive")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1468,7 +1461,7 @@ func (cli *DockerCli) CmdExport(args ...string) error {
 }
 
 func (cli *DockerCli) CmdDiff(args ...string) error {
-	cmd := cli.Subcmd("diff", "CONTAINER", "Inspect changes on a container's filesystem")
+	cmd := Subcmd("diff", "CONTAINER", "Inspect changes on a container's filesystem")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1494,7 +1487,7 @@ func (cli *DockerCli) CmdDiff(args ...string) error {
 }
 
 func (cli *DockerCli) CmdLogs(args ...string) error {
-	cmd := cli.Subcmd("logs", "CONTAINER", "Fetch the logs of a container")
+	cmd := Subcmd("logs", "CONTAINER", "Fetch the logs of a container")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1521,7 +1514,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error {
 }
 
 func (cli *DockerCli) CmdAttach(args ...string) error {
-	cmd := cli.Subcmd("attach", "[OPTIONS] CONTAINER", "Attach to a running container")
+	cmd := Subcmd("attach", "[OPTIONS] CONTAINER", "Attach to a running container")
 	noStdin := cmd.Bool("nostdin", false, "Do not attach stdin")
 	proxy := cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)")
 	if err := cmd.Parse(args); err != nil {
@@ -1543,7 +1536,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 		return err
 	}
 
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		return fmt.Errorf("Impossible to attach to a stopped container, start it first")
 	}
 
@@ -1576,7 +1569,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error {
 }
 
 func (cli *DockerCli) CmdSearch(args ...string) error {
-	cmd := cli.Subcmd("search", "TERM", "Search the docker index for images")
+	cmd := Subcmd("search", "TERM", "Search the docker index for images")
 	noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
 	trusted := cmd.Bool("trusted", false, "Only show trusted builds")
 	stars := cmd.Int("stars", 0, "Only displays with at least xxx stars")
@@ -1688,7 +1681,7 @@ func (opts PathOpts) Set(val string) error {
 }
 
 func (cli *DockerCli) CmdTag(args ...string) error {
-	cmd := cli.Subcmd("tag", "[OPTIONS] IMAGE REPOSITORY[:TAG]", "Tag an image into a repository")
+	cmd := Subcmd("tag", "[OPTIONS] IMAGE REPOSITORY[:TAG]", "Tag an image into a repository")
 	force := cmd.Bool("f", false, "Force")
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -1721,223 +1714,8 @@ func (cli *DockerCli) CmdTag(args ...string) error {
 	return nil
 }
 
-//FIXME Only used in tests
-func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
-	cmd := flag.NewFlagSet("run", flag.ContinueOnError)
-	cmd.SetOutput(ioutil.Discard)
-	cmd.Usage = nil
-	return parseRun(cmd, args, capabilities)
-}
-
-func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
-
-	flHostname := cmd.String("h", "", "Container host name")
-	flWorkingDir := cmd.String("w", "", "Working directory inside the container")
-	flUser := cmd.String("u", "", "Username or UID")
-	flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id")
-	flAttach := NewAttachOpts()
-	cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.")
-	flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached")
-	flTty := cmd.Bool("t", false, "Allocate a pseudo-tty")
-	flMemoryString := cmd.String("m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
-	flContainerIDFile := cmd.String("cidfile", "", "Write the container ID to the file")
-	flNetwork := cmd.Bool("n", true, "Enable networking for this container")
-	flPrivileged := cmd.Bool("privileged", false, "Give extended privileges to this container")
-	flAutoRemove := cmd.Bool("rm", false, "Automatically remove the container when it exits (incompatible with -d)")
-	cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)")
-	cmd.String("name", "", "Assign a name to the container")
-	flPublishAll := cmd.Bool("P", false, "Publish all exposed ports to the host interfaces")
-
-	if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit {
-		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n")
-		*flMemoryString = ""
-	}
-
-	flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)")
-
-	var flPublish utils.ListOpts
-	cmd.Var(&flPublish, "p", "Publish a container's port to the host (use 'docker port' to see the actual mapping)")
-
-	var flExpose utils.ListOpts
-	cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host")
-
-	var flEnv utils.ListOpts
-	cmd.Var(&flEnv, "e", "Set environment variables")
-
-	var flDns utils.ListOpts
-	cmd.Var(&flDns, "dns", "Set custom dns servers")
-
-	flVolumes := NewPathOpts()
-	cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)")
-
-	var flVolumesFrom utils.ListOpts
-	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)")
-
-	flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
-
-	var flLxcOpts utils.ListOpts
-	cmd.Var(&flLxcOpts, "lxc-conf", "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"")
-
-	var flLinks utils.ListOpts
-	cmd.Var(&flLinks, "link", "Add link to another container (name:alias)")
-
-	if err := cmd.Parse(args); err != nil {
-		return nil, nil, cmd, err
-	}
-	if *flDetach && len(flAttach) > 0 {
-		return nil, nil, cmd, ErrConflictAttachDetach
-	}
-	if *flWorkingDir != "" && !path.IsAbs(*flWorkingDir) {
-		return nil, nil, cmd, ErrInvalidWorikingDirectory
-	}
-	if *flDetach && *flAutoRemove {
-		return nil, nil, cmd, ErrConflictDetachAutoRemove
-	}
-
-	// If neither -d or -a are set, attach to everything by default
-	if len(flAttach) == 0 && !*flDetach {
-		if !*flDetach {
-			flAttach.Set("stdout")
-			flAttach.Set("stderr")
-			if *flStdin {
-				flAttach.Set("stdin")
-			}
-		}
-	}
-
-	envs := []string{}
-
-	for _, env := range flEnv {
-		arr := strings.Split(env, "=")
-		if len(arr) > 1 {
-			envs = append(envs, env)
-		} else {
-			v := os.Getenv(env)
-			envs = append(envs, env+"="+v)
-		}
-	}
-
-	var flMemory int64
-
-	if *flMemoryString != "" {
-		parsedMemory, err := utils.RAMInBytes(*flMemoryString)
-
-		if err != nil {
-			return nil, nil, cmd, err
-		}
-
-		flMemory = parsedMemory
-	}
-
-	var binds []string
-
-	// add any bind targets to the list of container volumes
-	for bind := range flVolumes {
-		arr := strings.Split(bind, ":")
-		if len(arr) > 1 {
-			if arr[0] == "/" {
-				return nil, nil, cmd, fmt.Errorf("Invalid bind mount: source can't be '/'")
-			}
-			dstDir := arr[1]
-			flVolumes[dstDir] = struct{}{}
-			binds = append(binds, bind)
-			delete(flVolumes, bind)
-		}
-	}
-
-	parsedArgs := cmd.Args()
-	runCmd := []string{}
-	entrypoint := []string{}
-	image := ""
-	if len(parsedArgs) >= 1 {
-		image = cmd.Arg(0)
-	}
-	if len(parsedArgs) > 1 {
-		runCmd = parsedArgs[1:]
-	}
-	if *flEntrypoint != "" {
-		entrypoint = []string{*flEntrypoint}
-	}
-
-	var lxcConf []KeyValuePair
-	lxcConf, err := parseLxcConfOpts(flLxcOpts)
-	if err != nil {
-		return nil, nil, cmd, err
-	}
-
-	hostname := *flHostname
-	domainname := ""
-
-	parts := strings.SplitN(hostname, ".", 2)
-	if len(parts) > 1 {
-		hostname = parts[0]
-		domainname = parts[1]
-	}
-
-	ports, portBindings, err := parsePortSpecs(flPublish)
-	if err != nil {
-		return nil, nil, cmd, err
-	}
-
-	// Merge in exposed ports to the map of published ports
-	for _, e := range flExpose {
-		if strings.Contains(e, ":") {
-			return nil, nil, cmd, fmt.Errorf("Invalid port format for -expose: %s", e)
-		}
-		p := NewPort(splitProtoPort(e))
-		if _, exists := ports[p]; !exists {
-			ports[p] = struct{}{}
-		}
-	}
-
-	config := &Config{
-		Hostname:        hostname,
-		Domainname:      domainname,
-		PortSpecs:       nil, // Deprecated
-		ExposedPorts:    ports,
-		User:            *flUser,
-		Tty:             *flTty,
-		NetworkDisabled: !*flNetwork,
-		OpenStdin:       *flStdin,
-		Memory:          flMemory,
-		CpuShares:       *flCpuShares,
-		AttachStdin:     flAttach.Get("stdin"),
-		AttachStdout:    flAttach.Get("stdout"),
-		AttachStderr:    flAttach.Get("stderr"),
-		Env:             envs,
-		Cmd:             runCmd,
-		Dns:             flDns,
-		Image:           image,
-		Volumes:         flVolumes,
-		VolumesFrom:     strings.Join(flVolumesFrom, ","),
-		Entrypoint:      entrypoint,
-		WorkingDir:      *flWorkingDir,
-	}
-
-	hostConfig := &HostConfig{
-		Binds:           binds,
-		ContainerIDFile: *flContainerIDFile,
-		LxcConf:         lxcConf,
-		Privileged:      *flPrivileged,
-		PortBindings:    portBindings,
-		Links:           flLinks,
-		PublishAllPorts: *flPublishAll,
-	}
-
-	if capabilities != nil && flMemory > 0 && !capabilities.SwapLimit {
-		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n")
-		config.MemorySwap = -1
-	}
-
-	// When allocating stdin in attached mode, close stdin at client disconnect
-	if config.OpenStdin && config.AttachStdin {
-		config.StdinOnce = true
-	}
-	return config, hostConfig, cmd, nil
-}
-
 func (cli *DockerCli) CmdRun(args ...string) error {
-	config, hostConfig, cmd, err := parseRun(cli.Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container"), args, nil)
+	config, hostConfig, cmd, err := ParseRun(args, nil)
 	if err != nil {
 		return err
 	}
@@ -2148,7 +1926,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 }
 
 func (cli *DockerCli) CmdCp(args ...string) error {
-	cmd := cli.Subcmd("cp", "CONTAINER:PATH HOSTPATH", "Copy files/folders from the PATH to the HOSTPATH")
+	cmd := Subcmd("cp", "CONTAINER:RESOURCE HOSTPATH", "Copy files/folders from the RESOURCE to the HOSTPATH")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -2162,7 +1940,7 @@ func (cli *DockerCli) CmdCp(args ...string) error {
 	info := strings.Split(cmd.Arg(0), ":")
 
 	if len(info) != 2 {
-		return fmt.Errorf("Error: Path not specified")
+		return fmt.Errorf("Error: Resource not specified")
 	}
 
 	copyData.Resource = info[1]
@@ -2175,48 +1953,13 @@ func (cli *DockerCli) CmdCp(args ...string) error {
 
 	if statusCode == 200 {
 		r := bytes.NewReader(data)
-		if err := archive.Untar(r, copyData.HostPath); err != nil {
+		if err := archive.Untar(r, copyData.HostPath, nil); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (cli *DockerCli) CmdSave(args ...string) error {
-	cmd := cli.Subcmd("save", "IMAGE DESTINATION", "Save an image to a tar archive")
-	if err := cmd.Parse(args); err != nil {
-		return err
-	}
-
-	if cmd.NArg() != 1 {
-		cmd.Usage()
-		return nil
-	}
-
-	image := cmd.Arg(0)
-	if err := cli.stream("GET", "/images/"+image+"/get", nil, cli.out, nil); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (cli *DockerCli) CmdLoad(args ...string) error {
-	cmd := cli.Subcmd("load", "SOURCE", "Load an image from a tar archive")
-	if err := cmd.Parse(args); err != nil {
-		return err
-	}
-
-	if cmd.NArg() != 0 {
-		cmd.Usage()
-		return nil
-	}
-
-	if err := cli.stream("POST", "/images/load", cli.in, cli.out, nil); err != nil {
-		return err
-	}
-	return nil
-}
-
 func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, error) {
 	var params io.Reader
 	if data != nil {
@@ -2329,9 +2072,10 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h
 
 	if matchesContentType(resp.Header.Get("Content-Type"), "application/json") {
 		return utils.DisplayJSONMessagesStream(resp.Body, out)
-	}
-	if _, err := io.Copy(out, resp.Body); err != nil {
-		return err
+	} else {
+		if _, err := io.Copy(out, resp.Body); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -2467,12 +2211,12 @@ func (cli *DockerCli) monitorTtySize(id string) error {
 	return nil
 }
 
-func (cli *DockerCli) Subcmd(name, signature, description string) *flag.FlagSet {
+func Subcmd(name, signature, description string) *flag.FlagSet {
 	flags := flag.NewFlagSet(name, flag.ContinueOnError)
 	flags.Usage = func() {
-		fmt.Fprintf(cli.err, "\nUsage: docker %s %s\n\n%s\n\n", name, signature, description)
+		// FIXME: use custom stdout or return error
+		fmt.Fprintf(os.Stdout, "\nUsage: docker %s %s\n\n%s\n\n", name, signature, description)
 		flags.PrintDefaults()
-		os.Exit(2)
 	}
 	return flags
 }
@@ -2517,7 +2261,7 @@ func getExitCode(cli *DockerCli, containerId string) (bool, int, error) {
 	if err := json.Unmarshal(body, c); err != nil {
 		return false, -1, err
 	}
-	return c.State.IsRunning(), c.State.GetExitCode(), nil
+	return c.State.Running, c.State.ExitCode, nil
 }
 
 func NewDockerCli(in io.ReadCloser, out, err io.Writer, proto, addr string) *DockerCli {

commands_test.go

@@ -3,8 +3,6 @@ package docker
 import (
 	"bufio"
 	"fmt"
-	"github.com/dotcloud/docker"
-	"github.com/dotcloud/docker/engine"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -68,8 +66,8 @@ func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error
 func TestRunHostname(t *testing.T) {
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -113,8 +111,8 @@ func TestRunHostname(t *testing.T) {
 func TestRunWorkdir(t *testing.T) {
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -158,8 +156,8 @@ func TestRunWorkdir(t *testing.T) {
 func TestRunWorkdirExists(t *testing.T) {
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -203,8 +201,8 @@ func TestRunExit(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c1 := make(chan struct{})
 	go func() {
@@ -256,8 +254,8 @@ func TestRunDisconnect(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c1 := make(chan struct{})
 	go func() {
@@ -289,7 +287,7 @@ func TestRunDisconnect(t *testing.T) {
 	setTimeout(t, "Waiting for /bin/cat to exit timed out", 2*time.Second, func() {
 		container := globalRuntime.List()[0]
 		container.Wait()
-		if container.State.IsRunning() {
+		if container.State.Running {
 			t.Fatalf("/bin/cat is still running after closing stdin")
 		}
 	})
@@ -301,8 +299,8 @@ func TestRunDisconnectTty(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c1 := make(chan struct{})
 	go func() {
@@ -319,7 +317,7 @@ func TestRunDisconnectTty(t *testing.T) {
 		for {
 			// Client disconnect after run -i should keep stdin out in TTY mode
 			l := globalRuntime.List()
-			if len(l) == 1 && l[0].State.IsRunning() {
+			if len(l) == 1 && l[0].State.Running {
 				break
 			}
 			time.Sleep(10 * time.Millisecond)
@@ -329,7 +327,7 @@ func TestRunDisconnectTty(t *testing.T) {
 	// Client disconnect after run -i should keep stdin out in TTY mode
 	container := globalRuntime.List()[0]
 
-	setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() {
+	setTimeout(t, "Read/Write assertion timed out", 2000*time.Second, func() {
 		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
 			t.Fatal(err)
 		}
@@ -345,7 +343,7 @@ func TestRunDisconnectTty(t *testing.T) {
 
 	// Give some time to monitor to do his thing
 	container.WaitTimeout(500 * time.Millisecond)
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		t.Fatalf("/bin/cat should  still be running after closing stdin (tty mode)")
 	}
 }
@@ -358,8 +356,8 @@ func TestRunAttachStdin(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	ch := make(chan struct{})
 	go func() {
@@ -422,8 +420,8 @@ func TestRunDetach(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	ch := make(chan struct{})
 	go func() {
@@ -454,7 +452,7 @@ func TestRunDetach(t *testing.T) {
 	})
 
 	time.Sleep(500 * time.Millisecond)
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		t.Fatal("The detached container should be still running")
 	}
 
@@ -468,8 +466,8 @@ func TestAttachDetach(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	ch := make(chan struct{})
 	go func() {
@@ -479,7 +477,7 @@ func TestAttachDetach(t *testing.T) {
 		}
 	}()
 
-	var container *docker.Container
+	var container *Container
 
 	setTimeout(t, "Reading container's id timed out", 10*time.Second, func() {
 		buf := make([]byte, 1024)
@@ -500,7 +498,7 @@ func TestAttachDetach(t *testing.T) {
 
 	stdin, stdinPipe = io.Pipe()
 	stdout, stdoutPipe = io.Pipe()
-	cli = docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	cli = NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
 
 	ch = make(chan struct{})
 	go func() {
@@ -534,7 +532,7 @@ func TestAttachDetach(t *testing.T) {
 	})
 
 	time.Sleep(500 * time.Millisecond)
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		t.Fatal("The detached container should be still running")
 	}
 
@@ -548,8 +546,8 @@ func TestAttachDetachTruncatedID(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	go stdout.Read(make([]byte, 1024))
 	setTimeout(t, "Starting container timed out", 2*time.Second, func() {
@@ -562,7 +560,7 @@ func TestAttachDetachTruncatedID(t *testing.T) {
 
 	stdin, stdinPipe = io.Pipe()
 	stdout, stdoutPipe = io.Pipe()
-	cli = docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	cli = NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
 
 	ch := make(chan struct{})
 	go func() {
@@ -596,7 +594,7 @@ func TestAttachDetachTruncatedID(t *testing.T) {
 	})
 
 	time.Sleep(500 * time.Millisecond)
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		t.Fatal("The detached container should be still running")
 	}
 
@@ -610,8 +608,8 @@ func TestAttachDisconnect(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	go func() {
 		// Start a process in daemon mode
@@ -629,7 +627,7 @@ func TestAttachDisconnect(t *testing.T) {
 	setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() {
 		for {
 			l := globalRuntime.List()
-			if len(l) == 1 && l[0].State.IsRunning() {
+			if len(l) == 1 && l[0].State.Running {
 				break
 			}
 			time.Sleep(10 * time.Millisecond)
@@ -665,7 +663,7 @@ func TestAttachDisconnect(t *testing.T) {
 	// We closed stdin, expect /bin/cat to still be running
 	// Wait a little bit to make sure container.monitor() did his thing
 	err := container.WaitTimeout(500 * time.Millisecond)
-	if err == nil || !container.State.IsRunning() {
+	if err == nil || !container.State.Running {
 		t.Fatalf("/bin/cat is not running after closing stdin")
 	}
 
@@ -679,8 +677,8 @@ func TestAttachDisconnect(t *testing.T) {
 func TestRunAutoRemove(t *testing.T) {
 	t.Skip("Fixme. Skipping test for now, race condition")
 	stdout, stdoutPipe := io.Pipe()
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -714,8 +712,8 @@ func TestRunAutoRemove(t *testing.T) {
 }
 
 func TestCmdLogs(t *testing.T) {
-	cli := docker.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	if err := cli.CmdRun(unitTestImageID, "sh", "-c", "ls -l"); err != nil {
 		t.Fatal(err)
@@ -732,8 +730,8 @@ func TestCmdLogs(t *testing.T) {
 // Expected behaviour: using / as a bind mount source should throw an error
 func TestRunErrorBindMountRootSource(t *testing.T) {
 
-	cli := docker.NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -751,8 +749,8 @@ func TestRunErrorBindMountRootSource(t *testing.T) {
 // Expected behaviour: error out when attempting to bind mount non-existing source paths
 func TestRunErrorBindNonExistingSource(t *testing.T) {
 
-	cli := docker.NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {
@@ -770,10 +768,11 @@ func TestRunErrorBindNonExistingSource(t *testing.T) {
 func TestImagesViz(t *testing.T) {
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
-	image := buildTestImages(t, globalEngine)
+	srv := &Server{runtime: globalRuntime}
+	image := buildTestImages(t, srv)
 
 	c := make(chan struct{})
 	go func() {
@@ -820,10 +819,11 @@ func TestImagesViz(t *testing.T) {
 func TestImagesTree(t *testing.T) {
 	stdout, stdoutPipe := io.Pipe()
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
-	image := buildTestImages(t, globalEngine)
+	srv := &Server{runtime: globalRuntime}
+	image := buildTestImages(t, srv)
 
 	c := make(chan struct{})
 	go func() {
@@ -840,13 +840,12 @@ func TestImagesTree(t *testing.T) {
 			t.Fatal(err)
 		}
 		cmdOutput := string(cmdOutputBytes)
-
 		regexpStrings := []string{
 			fmt.Sprintf("└─%s Size: (\\d+.\\d+ MB) \\(virtual \\d+.\\d+ MB\\) Tags: %s:latest", unitTestImageIDShort, unitTestImageName),
-			"(?m)^  └─[0-9a-f]+",
-			"(?m)^    └─[0-9a-f]+",
-			"(?m)^      └─[0-9a-f]+",
-			fmt.Sprintf("        └─%s Size: \\d+ B \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)),
+			"(?m)   └─[0-9a-f]+.*",
+			"(?m)    └─[0-9a-f]+.*",
+			"(?m)      └─[0-9a-f]+.*",
+			fmt.Sprintf("(?m)^        └─%s Size: \\d+.\\d+ MB \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)),
 		}
 
 		compiledRegexps := []*regexp.Regexp{}
@@ -867,7 +866,7 @@ func TestImagesTree(t *testing.T) {
 	})
 }
 
-func buildTestImages(t *testing.T, eng *engine.Engine) *docker.Image {
+func buildTestImages(t *testing.T, srv *Server) *Image {
 
 	var testBuilder = testContextTemplate{
 		`
@@ -880,9 +879,9 @@ run    [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]
 		nil,
 		nil,
 	}
-	image := buildImage(testBuilder, t, eng, true)
+	image := buildImage(testBuilder, t, srv, true)
 
-	err := mkServerFromEngine(eng, t).ContainerTag(image.ID, "test", "latest", false)
+	err := srv.ContainerTag(image.ID, "test", "latest", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -902,8 +901,8 @@ func TestRunCidFile(t *testing.T) {
 	}
 	tmpCidFile := path.Join(tmpDir, "cid")
 
-	cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
-	defer cleanup(globalEngine, t)
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
 
 	c := make(chan struct{})
 	go func() {

config_test.go

@@ -1,149 +0,0 @@
-package docker
-
-import (
-	"testing"
-)
-
-func TestCompareConfig(t *testing.T) {
-	volumes1 := make(map[string]struct{})
-	volumes1["/test1"] = struct{}{}
-	config1 := Config{
-		Dns:         []string{"1.1.1.1", "2.2.2.2"},
-		PortSpecs:   []string{"1111:1111", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "11111111",
-		Volumes:     volumes1,
-	}
-	config2 := Config{
-		Dns:         []string{"0.0.0.0", "2.2.2.2"},
-		PortSpecs:   []string{"1111:1111", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "11111111",
-		Volumes:     volumes1,
-	}
-	config3 := Config{
-		Dns:         []string{"1.1.1.1", "2.2.2.2"},
-		PortSpecs:   []string{"0000:0000", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "11111111",
-		Volumes:     volumes1,
-	}
-	config4 := Config{
-		Dns:         []string{"1.1.1.1", "2.2.2.2"},
-		PortSpecs:   []string{"0000:0000", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "22222222",
-		Volumes:     volumes1,
-	}
-	volumes2 := make(map[string]struct{})
-	volumes2["/test2"] = struct{}{}
-	config5 := Config{
-		Dns:         []string{"1.1.1.1", "2.2.2.2"},
-		PortSpecs:   []string{"0000:0000", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "11111111",
-		Volumes:     volumes2,
-	}
-	if CompareConfig(&config1, &config2) {
-		t.Fatalf("CompareConfig should return false, Dns are different")
-	}
-	if CompareConfig(&config1, &config3) {
-		t.Fatalf("CompareConfig should return false, PortSpecs are different")
-	}
-	if CompareConfig(&config1, &config4) {
-		t.Fatalf("CompareConfig should return false, VolumesFrom are different")
-	}
-	if CompareConfig(&config1, &config5) {
-		t.Fatalf("CompareConfig should return false, Volumes are different")
-	}
-	if !CompareConfig(&config1, &config1) {
-		t.Fatalf("CompareConfig should return true")
-	}
-}
-
-func TestMergeConfig(t *testing.T) {
-	volumesImage := make(map[string]struct{})
-	volumesImage["/test1"] = struct{}{}
-	volumesImage["/test2"] = struct{}{}
-	configImage := &Config{
-		Dns:         []string{"1.1.1.1", "2.2.2.2"},
-		PortSpecs:   []string{"1111:1111", "2222:2222"},
-		Env:         []string{"VAR1=1", "VAR2=2"},
-		VolumesFrom: "1111",
-		Volumes:     volumesImage,
-	}
-
-	volumesUser := make(map[string]struct{})
-	volumesUser["/test3"] = struct{}{}
-	configUser := &Config{
-		Dns:       []string{"3.3.3.3"},
-		PortSpecs: []string{"3333:2222", "3333:3333"},
-		Env:       []string{"VAR2=3", "VAR3=3"},
-		Volumes:   volumesUser,
-	}
-
-	if err := MergeConfig(configUser, configImage); err != nil {
-		t.Error(err)
-	}
-
-	if len(configUser.Dns) != 3 {
-		t.Fatalf("Expected 3 dns, 1.1.1.1, 2.2.2.2 and 3.3.3.3, found %d", len(configUser.Dns))
-	}
-	for _, dns := range configUser.Dns {
-		if dns != "1.1.1.1" && dns != "2.2.2.2" && dns != "3.3.3.3" {
-			t.Fatalf("Expected 1.1.1.1 or 2.2.2.2 or 3.3.3.3, found %s", dns)
-		}
-	}
-
-	if len(configUser.ExposedPorts) != 3 {
-		t.Fatalf("Expected 3 ExposedPorts, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts))
-	}
-	for portSpecs := range configUser.ExposedPorts {
-		if portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" {
-			t.Fatalf("Expected 1111 or 2222 or 3333, found %s", portSpecs)
-		}
-	}
-	if len(configUser.Env) != 3 {
-		t.Fatalf("Expected 3 env var, VAR1=1, VAR2=3 and VAR3=3, found %d", len(configUser.Env))
-	}
-	for _, env := range configUser.Env {
-		if env != "VAR1=1" && env != "VAR2=3" && env != "VAR3=3" {
-			t.Fatalf("Expected VAR1=1 or VAR2=3 or VAR3=3, found %s", env)
-		}
-	}
-
-	if len(configUser.Volumes) != 3 {
-		t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes))
-	}
-	for v := range configUser.Volumes {
-		if v != "/test1" && v != "/test2" && v != "/test3" {
-			t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v)
-		}
-	}
-
-	if configUser.VolumesFrom != "1111" {
-		t.Fatalf("Expected VolumesFrom to be 1111, found %s", configUser.VolumesFrom)
-	}
-
-	ports, _, err := parsePortSpecs([]string{"0000"})
-	if err != nil {
-		t.Error(err)
-	}
-	configImage2 := &Config{
-		ExposedPorts: ports,
-	}
-
-	if err := MergeConfig(configUser, configImage2); err != nil {
-		t.Error(err)
-	}
-
-	if len(configUser.ExposedPorts) != 4 {
-		t.Fatalf("Expected 4 ExposedPorts, 0000, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts))
-	}
-	for portSpecs := range configUser.ExposedPorts {
-		if portSpecs.Port() != "0000" && portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" {
-			t.Fatalf("Expected 0000 or 1111 or 2222 or 3333, found %s", portSpecs)
-		}
-	}
-
-}

container.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"flag"
 	"fmt"
 	"github.com/dotcloud/docker/archive"
 	"github.com/dotcloud/docker/term"
@@ -19,15 +20,13 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 )
 
 type Container struct {
-	sync.Mutex
-
-	root string
+	root   string // Path to the "home" of the container, including metadata.
+	rootfs string // Path to the root filesystem of the container.
 
 	ID string
 
@@ -158,6 +157,218 @@ func NewPort(proto, port string) Port {
 	return Port(fmt.Sprintf("%s/%s", port, proto))
 }
 
+func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, *flag.FlagSet, error) {
+	cmd := Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container")
+	if os.Getenv("TEST") != "" {
+		cmd.SetOutput(ioutil.Discard)
+		cmd.Usage = nil
+	}
+
+	flHostname := cmd.String("h", "", "Container host name")
+	flWorkingDir := cmd.String("w", "", "Working directory inside the container")
+	flUser := cmd.String("u", "", "Username or UID")
+	flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id")
+	flAttach := NewAttachOpts()
+	cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.")
+	flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached")
+	flTty := cmd.Bool("t", false, "Allocate a pseudo-tty")
+	flMemoryString := cmd.String("m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
+	flContainerIDFile := cmd.String("cidfile", "", "Write the container ID to the file")
+	flNetwork := cmd.Bool("n", true, "Enable networking for this container")
+	flPrivileged := cmd.Bool("privileged", false, "Give extended privileges to this container")
+	flAutoRemove := cmd.Bool("rm", false, "Automatically remove the container when it exits (incompatible with -d)")
+	cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)")
+	cmd.String("name", "", "Assign a name to the container")
+	flPublishAll := cmd.Bool("P", false, "Publish all exposed ports to the host interfaces")
+
+	if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit {
+		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n")
+		*flMemoryString = ""
+	}
+
+	flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)")
+
+	var flPublish utils.ListOpts
+	cmd.Var(&flPublish, "p", "Publish a container's port to the host (use 'docker port' to see the actual mapping)")
+
+	var flExpose utils.ListOpts
+	cmd.Var(&flExpose, "expose", "Expose a port from the container without publishing it to your host")
+
+	var flEnv utils.ListOpts
+	cmd.Var(&flEnv, "e", "Set environment variables")
+
+	var flDns utils.ListOpts
+	cmd.Var(&flDns, "dns", "Set custom dns servers")
+
+	flVolumes := NewPathOpts()
+	cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)")
+
+	var flVolumesFrom utils.ListOpts
+	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)")
+
+	flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
+
+	var flLxcOpts utils.ListOpts
+	cmd.Var(&flLxcOpts, "lxc-conf", "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"")
+
+	var flLinks utils.ListOpts
+	cmd.Var(&flLinks, "link", "Add link to another container (name:alias)")
+
+	if err := cmd.Parse(args); err != nil {
+		return nil, nil, cmd, err
+	}
+	if *flDetach && len(flAttach) > 0 {
+		return nil, nil, cmd, ErrConflictAttachDetach
+	}
+	if *flWorkingDir != "" && !path.IsAbs(*flWorkingDir) {
+		return nil, nil, cmd, ErrInvalidWorikingDirectory
+	}
+	if *flDetach && *flAutoRemove {
+		return nil, nil, cmd, ErrConflictDetachAutoRemove
+	}
+
+	// If neither -d or -a are set, attach to everything by default
+	if len(flAttach) == 0 && !*flDetach {
+		if !*flDetach {
+			flAttach.Set("stdout")
+			flAttach.Set("stderr")
+			if *flStdin {
+				flAttach.Set("stdin")
+			}
+		}
+	}
+
+	envs := []string{}
+
+	for _, env := range flEnv {
+		arr := strings.Split(env, "=")
+		if len(arr) > 1 {
+			envs = append(envs, env)
+		} else {
+			v := os.Getenv(env)
+			envs = append(envs, env+"="+v)
+		}
+	}
+
+	var flMemory int64
+
+	if *flMemoryString != "" {
+		parsedMemory, err := utils.RAMInBytes(*flMemoryString)
+
+		if err != nil {
+			return nil, nil, cmd, err
+		}
+
+		flMemory = parsedMemory
+	}
+
+	var binds []string
+
+	// add any bind targets to the list of container volumes
+	for bind := range flVolumes {
+		arr := strings.Split(bind, ":")
+		if len(arr) > 1 {
+			if arr[0] == "/" {
+				return nil, nil, cmd, fmt.Errorf("Invalid bind mount: source can't be '/'")
+			}
+			dstDir := arr[1]
+			flVolumes[dstDir] = struct{}{}
+			binds = append(binds, bind)
+			delete(flVolumes, bind)
+		}
+	}
+
+	parsedArgs := cmd.Args()
+	runCmd := []string{}
+	entrypoint := []string{}
+	image := ""
+	if len(parsedArgs) >= 1 {
+		image = cmd.Arg(0)
+	}
+	if len(parsedArgs) > 1 {
+		runCmd = parsedArgs[1:]
+	}
+	if *flEntrypoint != "" {
+		entrypoint = []string{*flEntrypoint}
+	}
+
+	var lxcConf []KeyValuePair
+	lxcConf, err := parseLxcConfOpts(flLxcOpts)
+	if err != nil {
+		return nil, nil, cmd, err
+	}
+
+	hostname := *flHostname
+	domainname := ""
+
+	parts := strings.SplitN(hostname, ".", 2)
+	if len(parts) > 1 {
+		hostname = parts[0]
+		domainname = parts[1]
+	}
+
+	ports, portBindings, err := parsePortSpecs(flPublish)
+	if err != nil {
+		return nil, nil, cmd, err
+	}
+
+	// Merge in exposed ports to the map of published ports
+	for _, e := range flExpose {
+		if strings.Contains(e, ":") {
+			return nil, nil, cmd, fmt.Errorf("Invalid port format for -expose: %s", e)
+		}
+		p := NewPort(splitProtoPort(e))
+		if _, exists := ports[p]; !exists {
+			ports[p] = struct{}{}
+		}
+	}
+
+	config := &Config{
+		Hostname:        hostname,
+		Domainname:      domainname,
+		PortSpecs:       nil, // Deprecated
+		ExposedPorts:    ports,
+		User:            *flUser,
+		Tty:             *flTty,
+		NetworkDisabled: !*flNetwork,
+		OpenStdin:       *flStdin,
+		Memory:          flMemory,
+		CpuShares:       *flCpuShares,
+		AttachStdin:     flAttach.Get("stdin"),
+		AttachStdout:    flAttach.Get("stdout"),
+		AttachStderr:    flAttach.Get("stderr"),
+		Env:             envs,
+		Cmd:             runCmd,
+		Dns:             flDns,
+		Image:           image,
+		Volumes:         flVolumes,
+		VolumesFrom:     strings.Join(flVolumesFrom, ","),
+		Entrypoint:      entrypoint,
+		WorkingDir:      *flWorkingDir,
+	}
+
+	hostConfig := &HostConfig{
+		Binds:           binds,
+		ContainerIDFile: *flContainerIDFile,
+		LxcConf:         lxcConf,
+		Privileged:      *flPrivileged,
+		PortBindings:    portBindings,
+		Links:           flLinks,
+		PublishAllPorts: *flPublishAll,
+	}
+
+	if capabilities != nil && flMemory > 0 && !capabilities.SwapLimit {
+		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n")
+		config.MemorySwap = -1
+	}
+
+	// When allocating stdin in attached mode, close stdin at client disconnect
+	if config.OpenStdin && config.AttachStdin {
+		config.StdinOnce = true
+	}
+	return config, hostConfig, cmd, nil
+}
+
 type PortMapping map[string]string // Deprecated
 
 type NetworkSettings struct {
@@ -196,8 +407,13 @@ func (settings *NetworkSettings) PortMappingAPI() []APIPort {
 
 // Inject the io.Reader at the given path. Note: do not close the reader
 func (container *Container) Inject(file io.Reader, pth string) error {
+	if err := container.EnsureMounted(); err != nil {
+		return fmt.Errorf("inject: error mounting container %s: %s", container.ID, err)
+	}
+
 	// Return error if path exists
-	if _, err := os.Stat(path.Join(container.rwPath(), pth)); err == nil {
+	destPath := path.Join(container.RootfsPath(), pth)
+	if _, err := os.Stat(destPath); err == nil {
 		// Since err is nil, the path could be stat'd and it exists
 		return fmt.Errorf("%s exists", pth)
 	} else if !os.IsNotExist(err) {
@@ -208,14 +424,16 @@ func (container *Container) Inject(file io.Reader, pth string) error {
 	}
 
 	// Make sure the directory exists
-	if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil {
+	if err := os.MkdirAll(path.Join(container.RootfsPath(), path.Dir(pth)), 0755); err != nil {
 		return err
 	}
 
-	dest, err := os.Create(path.Join(container.rwPath(), pth))
+	dest, err := os.Create(destPath)
 	if err != nil {
 		return err
 	}
+	defer dest.Close()
+
 	if _, err := io.Copy(dest, file); err != nil {
 		return err
 	}
@@ -490,10 +708,9 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 }
 
 func (container *Container) Start() (err error) {
-	container.Lock()
-	defer container.Unlock()
-
-	if container.State.IsRunning() {
+	container.State.Lock()
+	defer container.State.Unlock()
+	if container.State.Running {
 		return fmt.Errorf("The container %s is already running.", container.ID)
 	}
 	defer func() {
@@ -607,6 +824,7 @@ func (container *Container) Start() (err error) {
 		}
 	}
 
+	volumesDriver := container.runtime.volumes.driver
 	// Create the requested volumes if they don't exist
 	for volPath := range container.Config.Volumes {
 		volPath = path.Clean(volPath)
@@ -626,13 +844,17 @@ func (container *Container) Start() (err error) {
 			}
 			// Otherwise create an directory in $ROOT/volumes/ and use that
 		} else {
-			c, err := container.runtime.volumes.Create(nil, container, "", "", nil)
+
+			// Do not pass a container as the parameter for the volume creation.
+			// The graph driver using the container's information ( Image ) to
+			// create the parent.
+			c, err := container.runtime.volumes.Create(nil, nil, "", "", nil)
 			if err != nil {
 				return err
 			}
-			srcPath, err = c.layer()
+			srcPath, err = volumesDriver.Get(c.ID)
 			if err != nil {
-				return err
+				return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err)
 			}
 			srcRW = true // RW by default
 		}
@@ -822,7 +1044,7 @@ func (container *Container) Start() (err error) {
 	}
 	// FIXME: save state on disk *first*, then converge
 	// this way disk state is used as a journal, eg. we can restore after crash etc.
-	container.State.SetRunning(container.cmd.Process.Pid)
+	container.State.setRunning(container.cmd.Process.Pid)
 
 	// Init the lock
 	container.waitLock = make(chan struct{})
@@ -830,14 +1052,14 @@ func (container *Container) Start() (err error) {
 	container.ToDisk()
 	go container.monitor()
 
-	defer utils.Debugf("Container running: %v", container.State.IsRunning())
+	defer utils.Debugf("Container running: %v", container.State.Running)
 	// We wait for the container to be fully running.
 	// Timeout after 5 seconds. In case of broken pipe, just retry.
 	// Note: The container can run and finish correctly before
 	//       the end of this loop
 	for now := time.Now(); time.Since(now) < 5*time.Second; {
 		// If the container dies while waiting for it, just return
-		if !container.State.IsRunning() {
+		if !container.State.Running {
 			return nil
 		}
 		output, err := exec.Command("lxc-info", "-s", "-n", container.ID).CombinedOutput()
@@ -854,11 +1076,11 @@ func (container *Container) Start() (err error) {
 		if strings.Contains(string(output), "RUNNING") {
 			return nil
 		}
-		utils.Debugf("Waiting for the container to start (running: %v): %s", container.State.IsRunning(), bytes.TrimSpace(output))
+		utils.Debugf("Waiting for the container to start (running: %v): %s", container.State.Running, bytes.TrimSpace(output))
 		time.Sleep(50 * time.Millisecond)
 	}
 
-	if container.State.IsRunning() {
+	if container.State.Running {
 		return ErrContainerStartTimeout
 	}
 	return ErrContainerStart
@@ -939,12 +1161,11 @@ func (container *Container) allocateNetwork() error {
 		return nil
 	}
 
-	var (
-		iface *NetworkInterface
-		err   error
-	)
-	if container.State.IsGhost() {
-		if manager := container.runtime.networkManager; manager.disabled {
+	var iface *NetworkInterface
+	var err error
+	if container.State.Ghost {
+		manager := container.runtime.networkManager
+		if manager.disabled {
 			iface = &NetworkInterface{disabled: true}
 		} else {
 			iface = &NetworkInterface{
@@ -980,12 +1201,10 @@ func (container *Container) allocateNetwork() error {
 		}
 	}
 
-	var (
-		portSpecs = make(map[Port]struct{})
-		bindings  = make(map[Port][]PortBinding)
-	)
+	portSpecs := make(map[Port]struct{})
+	bindings := make(map[Port][]PortBinding)
 
-	if !container.State.IsGhost() {
+	if !container.State.Ghost {
 		if container.Config.ExposedPorts != nil {
 			portSpecs = container.Config.ExposedPorts
 		}
@@ -1081,6 +1300,9 @@ func (container *Container) monitor() {
 		exitCode = container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
 	}
 
+	// Report status back
+	container.State.setStopped(exitCode)
+
 	if container.runtime != nil && container.runtime.srv != nil {
 		container.runtime.srv.LogEvent("die", container.ID, container.runtime.repositories.ImageName(container.Image))
 	}
@@ -1093,9 +1315,6 @@ func (container *Container) monitor() {
 		container.stdin, container.stdinPipe = io.Pipe()
 	}
 
-	// Report status back
-	container.State.SetStopped(exitCode)
-
 	// Release the lock
 	close(container.waitLock)
 
@@ -1144,10 +1363,10 @@ func (container *Container) cleanup() {
 }
 
 func (container *Container) kill(sig int) error {
-	container.Lock()
-	defer container.Unlock()
+	container.State.Lock()
+	defer container.State.Unlock()
 
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		return nil
 	}
 
@@ -1160,7 +1379,7 @@ func (container *Container) kill(sig int) error {
 }
 
 func (container *Container) Kill() error {
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		return nil
 	}
 
@@ -1185,7 +1404,7 @@ func (container *Container) Kill() error {
 }
 
 func (container *Container) Stop(seconds int) error {
-	if !container.State.IsRunning() {
+	if !container.State.Running {
 		return nil
 	}
 
@@ -1219,7 +1438,7 @@ func (container *Container) Restart(seconds int) error {
 // Wait blocks until the container stops running, then returns its exit code.
 func (container *Container) Wait() int {
 	<-container.waitLock
-	return container.State.GetExitCode()
+	return container.State.ExitCode
 }
 
 func (container *Container) Resize(h, w int) error {
@@ -1231,15 +1450,14 @@ func (container *Container) Resize(h, w int) error {
 }
 
 func (container *Container) ExportRw() (archive.Archive, error) {
-	return archive.Tar(container.rwPath(), archive.Uncompressed)
-}
-
-func (container *Container) RwChecksum() (string, error) {
-	rwData, err := archive.Tar(container.rwPath(), archive.Xz)
-	if err != nil {
-		return "", err
+	if err := container.EnsureMounted(); err != nil {
+		return nil, err
 	}
-	return utils.HashData(rwData)
+	if container.runtime == nil {
+		return nil, fmt.Errorf("Can't load storage driver for unregistered container %s", container.ID)
+	}
+
+	return container.runtime.Diff(container)
 }
 
 func (container *Container) Export() (archive.Archive, error) {
@@ -1265,28 +1483,17 @@ func (container *Container) WaitTimeout(timeout time.Duration) error {
 }
 
 func (container *Container) EnsureMounted() error {
-	if mounted, err := container.Mounted(); err != nil {
-		return err
-	} else if mounted {
-		return nil
-	}
+	// FIXME: EnsureMounted is deprecated because drivers are now responsible
+	// for re-entrant mounting in their Get() method.
 	return container.Mount()
 }
 
 func (container *Container) Mount() error {
-	image, err := container.GetImage()
-	if err != nil {
-		return err
-	}
-	return image.Mount(container.RootfsPath(), container.rwPath())
+	return container.runtime.Mount(container)
 }
 
-func (container *Container) Changes() ([]Change, error) {
-	image, err := container.GetImage()
-	if err != nil {
-		return nil, err
-	}
-	return image.Changes(container.rwPath())
+func (container *Container) Changes() ([]archive.Change, error) {
+	return container.runtime.Changes(container)
 }
 
 func (container *Container) GetImage() (*Image, error) {
@@ -1296,18 +1503,8 @@ func (container *Container) GetImage() (*Image, error) {
 	return container.runtime.graph.Get(container.Image)
 }
 
-func (container *Container) Mounted() (bool, error) {
-	return Mounted(container.RootfsPath())
-}
-
 func (container *Container) Unmount() error {
-	if _, err := os.Stat(container.RootfsPath()); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	return Unmount(container.RootfsPath())
+	return container.runtime.Unmount(container)
 }
 
 func (container *Container) logPath(name string) string {
@@ -1336,11 +1533,7 @@ func (container *Container) lxcConfigPath() string {
 
 // This method must be exported to be used from the lxc template
 func (container *Container) RootfsPath() string {
-	return path.Join(container.root, "rootfs")
-}
-
-func (container *Container) rwPath() string {
-	return path.Join(container.root, "rw")
+	return container.rootfs
 }
 
 func validateID(id string) error {
@@ -1352,47 +1545,30 @@ func validateID(id string) error {
 
 // GetSize, return real size, virtual size
 func (container *Container) GetSize() (int64, int64) {
-	var sizeRw, sizeRootfs int64
-	data := make(map[uint64]bool)
+	var (
+		sizeRw, sizeRootfs int64
+		err                error
+		driver             = container.runtime.driver
+	)
 
-	filepath.Walk(container.rwPath(), func(path string, fileInfo os.FileInfo, err error) error {
-		if fileInfo == nil {
-			return nil
-		}
-		size := fileInfo.Size()
-		if size == 0 {
-			return nil
-		}
+	sizeRw, err = driver.Size(container.ID)
+	if err != nil {
+		utils.Errorf("Warning: driver %s couldn't return diff size of container %s: %s", driver, container.ID, err)
+		// FIXME: GetSize should return an error. Not changing it now in case
+		// there is a side-effect.
+		sizeRw = -1
+	}
 
-		inode := fileInfo.Sys().(*syscall.Stat_t).Ino
-		if _, entryExists := data[inode]; entryExists {
-			return nil
-		}
-		data[inode] = false
-
-		sizeRw += size
-		return nil
-	})
-
-	data = make(map[uint64]bool)
-	_, err := os.Stat(container.RootfsPath())
+	if err := container.EnsureMounted(); err != nil {
+		utils.Errorf("Warning: failed to compute size of container rootfs %s: %s", container.ID, err)
+		return sizeRw, sizeRootfs
+	}
+	_, err = os.Stat(container.RootfsPath())
 	if err == nil {
 		filepath.Walk(container.RootfsPath(), func(path string, fileInfo os.FileInfo, err error) error {
-			if fileInfo == nil {
-				return nil
+			if fileInfo != nil {
+				sizeRootfs += fileInfo.Size()
 			}
-			size := fileInfo.Size()
-			if size == 0 {
-				return nil
-			}
-
-			inode := fileInfo.Sys().(*syscall.Stat_t).Ino
-			if _, entryExists := data[inode]; entryExists {
-				return nil
-			}
-			data[inode] = false
-
-			sizeRootfs += size
 			return nil
 		})
 	}
@@ -1417,7 +1593,11 @@ func (container *Container) Copy(resource string) (archive.Archive, error) {
 		filter = []string{path.Base(basePath)}
 		basePath = path.Dir(basePath)
 	}
-	return archive.TarFilter(basePath, archive.Uncompressed, filter)
+	return archive.TarFilter(basePath, &archive.TarOptions{
+		Compression: archive.Uncompressed,
+		Includes:    filter,
+		Recursive:   true,
+	})
 }
 
 // Returns true if the container exposes a certain port

container_unit_test.go

@@ -1,161 +0,0 @@
-package docker
-
-import (
-	"testing"
-)
-
-func TestParseLxcConfOpt(t *testing.T) {
-	opts := []string{"lxc.utsname=docker", "lxc.utsname = docker "}
-
-	for _, o := range opts {
-		k, v, err := parseLxcOpt(o)
-		if err != nil {
-			t.FailNow()
-		}
-		if k != "lxc.utsname" {
-			t.Fail()
-		}
-		if v != "docker" {
-			t.Fail()
-		}
-	}
-}
-
-func TestParseNetworkOptsPrivateOnly(t *testing.T) {
-	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::80"})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(ports) != 1 {
-		t.Logf("Expected 1 got %d", len(ports))
-		t.FailNow()
-	}
-	if len(bindings) != 1 {
-		t.Logf("Expected 1 got %d", len(bindings))
-		t.FailNow()
-	}
-	for k := range ports {
-		if k.Proto() != "tcp" {
-			t.Logf("Expected tcp got %s", k.Proto())
-			t.Fail()
-		}
-		if k.Port() != "80" {
-			t.Logf("Expected 80 got %s", k.Port())
-			t.Fail()
-		}
-		b, exists := bindings[k]
-		if !exists {
-			t.Log("Binding does not exist")
-			t.FailNow()
-		}
-		if len(b) != 1 {
-			t.Logf("Expected 1 got %d", len(b))
-			t.FailNow()
-		}
-		s := b[0]
-		if s.HostPort != "" {
-			t.Logf("Expected \"\" got %s", s.HostPort)
-			t.Fail()
-		}
-		if s.HostIp != "192.168.1.100" {
-			t.Fail()
-		}
-	}
-}
-
-func TestParseNetworkOptsPublic(t *testing.T) {
-	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100:8080:80"})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(ports) != 1 {
-		t.Logf("Expected 1 got %d", len(ports))
-		t.FailNow()
-	}
-	if len(bindings) != 1 {
-		t.Logf("Expected 1 got %d", len(bindings))
-		t.FailNow()
-	}
-	for k := range ports {
-		if k.Proto() != "tcp" {
-			t.Logf("Expected tcp got %s", k.Proto())
-			t.Fail()
-		}
-		if k.Port() != "80" {
-			t.Logf("Expected 80 got %s", k.Port())
-			t.Fail()
-		}
-		b, exists := bindings[k]
-		if !exists {
-			t.Log("Binding does not exist")
-			t.FailNow()
-		}
-		if len(b) != 1 {
-			t.Logf("Expected 1 got %d", len(b))
-			t.FailNow()
-		}
-		s := b[0]
-		if s.HostPort != "8080" {
-			t.Logf("Expected 8080 got %s", s.HostPort)
-			t.Fail()
-		}
-		if s.HostIp != "192.168.1.100" {
-			t.Fail()
-		}
-	}
-}
-
-func TestParseNetworkOptsUdp(t *testing.T) {
-	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::6000/udp"})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(ports) != 1 {
-		t.Logf("Expected 1 got %d", len(ports))
-		t.FailNow()
-	}
-	if len(bindings) != 1 {
-		t.Logf("Expected 1 got %d", len(bindings))
-		t.FailNow()
-	}
-	for k := range ports {
-		if k.Proto() != "udp" {
-			t.Logf("Expected udp got %s", k.Proto())
-			t.Fail()
-		}
-		if k.Port() != "6000" {
-			t.Logf("Expected 6000 got %s", k.Port())
-			t.Fail()
-		}
-		b, exists := bindings[k]
-		if !exists {
-			t.Log("Binding does not exist")
-			t.FailNow()
-		}
-		if len(b) != 1 {
-			t.Logf("Expected 1 got %d", len(b))
-			t.FailNow()
-		}
-		s := b[0]
-		if s.HostPort != "" {
-			t.Logf("Expected \"\" got %s", s.HostPort)
-			t.Fail()
-		}
-		if s.HostIp != "192.168.1.100" {
-			t.Fail()
-		}
-	}
-}
-
-func TestGetFullName(t *testing.T) {
-	name, err := getFullName("testing")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if name != "/testing" {
-		t.Fatalf("Expected /testing got %s", name)
-	}
-	if _, err := getFullName(""); err == nil {
-		t.Fatal("Error should not be nil")
-	}
-}

contrib/mkimage-debootstrap.sh

@@ -192,7 +192,7 @@ if [ "$justTar" ]; then
 	sudo tar --numeric-owner -caf "$repo" .
 else
 	# create the image (and tag $repo:$suite)
-	sudo tar --numeric-owner -c . | $docker import - $repo:$suite
+	sudo tar --numeric-owner -c . | $docker import - $repo $suite
 	
 	# test the image
 	$docker run -i -t $repo:$suite echo success
@@ -202,25 +202,25 @@ else
 			Debian)
 				if [ "$suite" = "$debianStable" -o "$suite" = 'stable' ] && [ -r etc/debian_version ]; then
 					# tag latest
-					$docker tag $repo:$suite $repo:latest
+					$docker tag $repo:$suite $repo latest
 					
 					if [ -r etc/debian_version ]; then
 						# tag the specific debian release version (which is only reasonable to tag on debian stable)
 						ver=$(cat etc/debian_version)
-						$docker tag $repo:$suite $repo:$ver
+						$docker tag $repo:$suite $repo $ver
 					fi
 				fi
 				;;
 			Ubuntu)
 				if [ "$suite" = "$ubuntuLatestLTS" ]; then
 					# tag latest
-					$docker tag $repo:$suite $repo:latest
+					$docker tag $repo:$suite $repo latest
 				fi
 				if [ -r etc/lsb-release ]; then
 					lsbRelease="$(. etc/lsb-release && echo "$DISTRIB_RELEASE")"
 					if [ "$lsbRelease" ]; then
 						# tag specific Ubuntu version number, if available (12.04, etc.)
-						$docker tag $repo:$suite $repo:$lsbRelease
+						$docker tag $repo:$suite $repo $lsbRelease
 					fi
 				fi
 				;;

devmapper/deviceset.go

@@ -0,0 +1,803 @@
+package devmapper
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var (
+	DefaultDataLoopbackSize     int64  = 100 * 1024 * 1024 * 1024
+	DefaultMetaDataLoopbackSize int64  = 2 * 1024 * 1024 * 1024
+	DefaultBaseFsSize           uint64 = 10 * 1024 * 1024 * 1024
+)
+
+type DevInfo struct {
+	Hash          string     `json:"-"`
+	DeviceId      int        `json:"device_id"`
+	Size          uint64     `json:"size"`
+	TransactionId uint64     `json:"transaction_id"`
+	Initialized   bool       `json:"initialized"`
+	devices       *DeviceSet `json:"-"`
+}
+
+type MetaData struct {
+	Devices map[string]*DevInfo `json:devices`
+}
+
+type DeviceSet struct {
+	MetaData
+	sync.Mutex
+	root             string
+	devicePrefix     string
+	TransactionId    uint64
+	NewTransactionId uint64
+	nextFreeDevice   int
+	activeMounts     map[string]int
+}
+
+type DiskUsage struct {
+	Used  uint64
+	Total uint64
+}
+
+type Status struct {
+	PoolName         string
+	DataLoopback     string
+	MetadataLoopback string
+	Data             DiskUsage
+	Metadata         DiskUsage
+}
+
+func getDevName(name string) string {
+	return "/dev/mapper/" + name
+}
+
+func (info *DevInfo) Name() string {
+	hash := info.Hash
+	if hash == "" {
+		hash = "base"
+	}
+	return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
+}
+
+func (info *DevInfo) DevName() string {
+	return getDevName(info.Name())
+}
+
+func (devices *DeviceSet) loopbackDir() string {
+	return path.Join(devices.root, "devicemapper")
+}
+
+func (devices *DeviceSet) jsonFile() string {
+	return path.Join(devices.loopbackDir(), "json")
+}
+
+func (devices *DeviceSet) getPoolName() string {
+	return devices.devicePrefix + "-pool"
+}
+
+func (devices *DeviceSet) getPoolDevName() string {
+	return getDevName(devices.getPoolName())
+}
+
+func (devices *DeviceSet) hasImage(name string) bool {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	_, err := os.Stat(filename)
+	return err == nil
+}
+
+// ensureImage creates a sparse file of <size> bytes at the path
+// <root>/devicemapper/<name>.
+// If the file already exists, it does nothing.
+// Either way it returns the full path.
+func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
+		return "", err
+	}
+
+	if _, err := os.Stat(filename); err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		utils.Debugf("Creating loopback file %s for device-manage use", filename)
+		file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			return "", err
+		}
+		defer file.Close()
+
+		if err = file.Truncate(size); err != nil {
+			return "", err
+		}
+	}
+	return filename, nil
+}
+
+func (devices *DeviceSet) allocateDeviceId() int {
+	// TODO: Add smarter reuse of deleted devices
+	id := devices.nextFreeDevice
+	devices.nextFreeDevice = devices.nextFreeDevice + 1
+	return id
+}
+
+func (devices *DeviceSet) allocateTransactionId() uint64 {
+	devices.NewTransactionId = devices.NewTransactionId + 1
+	return devices.NewTransactionId
+}
+
+func (devices *DeviceSet) saveMetadata() error {
+	jsonData, err := json.Marshal(devices.MetaData)
+	if err != nil {
+		return fmt.Errorf("Error encoding metaadata to json: %s", err)
+	}
+	tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
+	if err != nil {
+		return fmt.Errorf("Error creating metadata file: %s", err)
+	}
+
+	n, err := tmpFile.Write(jsonData)
+	if err != nil {
+		return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
+	}
+	if n < len(jsonData) {
+		return io.ErrShortWrite
+	}
+	if err := tmpFile.Sync(); err != nil {
+		return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := os.Rename(tmpFile.Name(), devices.jsonFile()); err != nil {
+		return fmt.Errorf("Error committing metadata file", err)
+	}
+
+	if devices.NewTransactionId != devices.TransactionId {
+		if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
+			return fmt.Errorf("Error setting devmapper transition ID: %s", err)
+		}
+		devices.TransactionId = devices.NewTransactionId
+	}
+	return nil
+}
+
+func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
+	utils.Debugf("registerDevice(%v, %v)", id, hash)
+	info := &DevInfo{
+		Hash:          hash,
+		DeviceId:      id,
+		Size:          size,
+		TransactionId: devices.allocateTransactionId(),
+		Initialized:   false,
+		devices:       devices,
+	}
+
+	devices.Devices[hash] = info
+	if err := devices.saveMetadata(); err != nil {
+		// Try to remove unused device
+		delete(devices.Devices, hash)
+		return nil, err
+	}
+
+	return info, nil
+}
+
+func (devices *DeviceSet) activateDeviceIfNeeded(hash string) error {
+	utils.Debugf("activateDeviceIfNeeded(%v)", hash)
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
+		return nil
+	}
+
+	return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
+}
+
+func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
+	devname := info.DevName()
+
+	err := exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0,lazy_journal_init=0", devname).Run()
+	if err != nil {
+		err = exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0", devname).Run()
+	}
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) loadMetaData() error {
+	utils.Debugf("loadMetadata()")
+	defer utils.Debugf("loadMetadata END")
+	_, _, _, params, err := getStatus(devices.getPoolName())
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	devices.NewTransactionId = devices.TransactionId
+
+	jsonData, err := ioutil.ReadFile(devices.jsonFile())
+	if err != nil && !os.IsNotExist(err) {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	devices.MetaData.Devices = make(map[string]*DevInfo)
+	if jsonData != nil {
+		if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	for hash, d := range devices.Devices {
+		d.Hash = hash
+		d.devices = devices
+
+		if d.DeviceId >= devices.nextFreeDevice {
+			devices.nextFreeDevice = d.DeviceId + 1
+		}
+
+		// If the transaction id is larger than the actual one we lost the device due to some crash
+		if d.TransactionId > devices.TransactionId {
+			utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
+			delete(devices.Devices, hash)
+		}
+	}
+	return nil
+}
+
+func (devices *DeviceSet) setupBaseImage() error {
+	oldInfo := devices.Devices[""]
+	if oldInfo != nil && oldInfo.Initialized {
+		return nil
+	}
+
+	if oldInfo != nil && !oldInfo.Initialized {
+		utils.Debugf("Removing uninitialized base image")
+		if err := devices.removeDevice(""); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	utils.Debugf("Initializing base device-manager snapshot")
+
+	id := devices.allocateDeviceId()
+
+	// Create initial device
+	if err := createDevice(devices.getPoolDevName(), id); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize)
+	info, err := devices.registerDevice(id, "", DefaultBaseFsSize)
+	if err != nil {
+		_ = deleteDevice(devices.getPoolDevName(), id)
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Creating filesystem on base device-manager snapshot")
+
+	if err = devices.activateDeviceIfNeeded(""); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if err := devices.createFilesystem(info); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	info.Initialized = true
+	if err = devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func setCloseOnExec(name string) {
+	fileInfos, _ := ioutil.ReadDir("/proc/self/fd")
+	if fileInfos != nil {
+		for _, i := range fileInfos {
+			link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
+			if link == name {
+				fd, err := strconv.Atoi(i.Name())
+				if err == nil {
+					syscall.CloseOnExec(fd)
+				}
+			}
+		}
+	}
+}
+
+func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
+	if level >= 7 {
+		return // Ignore _LOG_DEBUG
+	}
+
+	utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
+}
+
+func major(device uint64) uint64 {
+	return (device >> 8) & 0xfff
+}
+
+func minor(device uint64) uint64 {
+	return (device & 0xff) | ((device >> 12) & 0xfff00)
+}
+
+func (devices *DeviceSet) initDevmapper() error {
+	logInit(devices)
+
+	// Make sure the sparse images exist in <root>/devicemapper/data and
+	// <root>/devicemapper/metadata
+
+	createdLoopback := !devices.hasImage("data") || !devices.hasImage("metadata")
+	data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (data): %s\n", err)
+		return err
+	}
+	metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (metadata): %s\n", err)
+		return err
+	}
+
+	// Set the device prefix from the device id and inode of the docker root dir
+
+	st, err := os.Stat(devices.root)
+	if err != nil {
+		return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
+	}
+	sysSt := st.Sys().(*syscall.Stat_t)
+	// "reg-" stands for "regular file".
+	// In the future we might use "dev-" for "device file", etc.
+	// docker-maj,min[-inode] stands for:
+	//	- Managed by docker
+	//	- The target of this device is at major <maj> and minor <min>
+	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
+	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
+	utils.Debugf("Generated prefix: %s", devices.devicePrefix)
+
+	// Check for the existence of the device <prefix>-pool
+	utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
+	info, err := getInfo(devices.getPoolName())
+	if info == nil {
+		utils.Debugf("Error device getInfo: %s", err)
+		return err
+	}
+
+	// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
+	// that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
+	// so we add this badhack to make sure it closes itself
+	setCloseOnExec("/dev/mapper/control")
+
+	// If the pool doesn't exist, create it
+	if info.Exists == 0 {
+		utils.Debugf("Pool doesn't exist. Creating it.")
+
+		dataFile, err := AttachLoopDevice(data)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer dataFile.Close()
+
+		metadataFile, err := AttachLoopDevice(metadata)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer metadataFile.Close()
+
+		if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// If we didn't just create the data or metadata image, we need to
+	// load the metadata from the existing file.
+	if !createdLoopback {
+		if err = devices.loadMetaData(); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// Setup the base image
+	if err := devices.setupBaseImage(); err != nil {
+		utils.Debugf("Error device setupBaseImage: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if devices.Devices[hash] != nil {
+		return fmt.Errorf("hash %s already exists", hash)
+	}
+
+	baseInfo := devices.Devices[baseHash]
+	if baseInfo == nil {
+		return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash)
+	}
+
+	deviceId := devices.allocateDeviceId()
+
+	if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
+		utils.Debugf("Error creating snap device: %s\n", err)
+		return err
+	}
+
+	if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
+		deleteDevice(devices.getPoolDevName(), deviceId)
+		utils.Debugf("Error registering device: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) removeDevice(hash string) error {
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("hash %s doesn't exists", hash)
+	}
+
+	devinfo, _ := getInfo(info.Name())
+	if devinfo != nil && devinfo.Exists != 0 {
+		if err := removeDevice(info.Name()); err != nil {
+			utils.Debugf("Error removing device: %s\n", err)
+			return err
+		}
+	}
+
+	if info.Initialized {
+		info.Initialized = false
+		if err := devices.saveMetadata(); err != nil {
+			utils.Debugf("Error saving meta data: %s\n", err)
+			return err
+		}
+	}
+
+	if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
+		utils.Debugf("Error deleting device: %s\n", err)
+		return err
+	}
+
+	devices.allocateTransactionId()
+	delete(devices.Devices, info.Hash)
+
+	if err := devices.saveMetadata(); err != nil {
+		devices.Devices[info.Hash] = info
+		utils.Debugf("Error saving meta data: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) RemoveDevice(hash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	return devices.removeDevice(hash)
+}
+
+func (devices *DeviceSet) deactivateDevice(hash string) error {
+	utils.Debugf("[devmapper] deactivateDevice(%s)", hash)
+	defer utils.Debugf("[devmapper] deactivateDevice END")
+	var devname string
+	// FIXME: shouldn't we just register the pool into devices?
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	devinfo, err := getInfo(devname)
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	if devinfo.Exists != 0 {
+		if err := removeDevice(devname); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		if err := devices.waitRemove(hash); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// waitRemove blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is removed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitRemove(hash string) error {
+	utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash)
+	defer utils.Debugf("[deviceset %s] waitRemove END", devices.devicePrefix, hash)
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			// If there is an error we assume the device doesn't exist.
+			// The error might actually be something else, but we can't differentiate.
+			return nil
+		}
+		utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
+		if devinfo.Exists == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
+	}
+	return nil
+}
+
+// waitClose blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is closed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitClose(hash string) error {
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			return err
+		}
+		utils.Debugf("Waiting for unmount of %s: opencount=%d", devname, devinfo.OpenCount)
+		if devinfo.OpenCount == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to close", devname)
+	}
+	return nil
+}
+
+// byHash is a hack to allow looking up the deviceset's pool by the hash "pool".
+// FIXME: it seems probably cleaner to register the pool in devices.Devices,
+// but I am afraid of arcane implications deep in the devicemapper code,
+// so this will do.
+func (devices *DeviceSet) byHash(hash string) (devname string, err error) {
+	if hash == "pool" {
+		return devices.getPoolDevName(), nil
+	}
+	info := devices.Devices[hash]
+	if info == nil {
+		return "", fmt.Errorf("hash %s doesn't exists", hash)
+	}
+	return info.Name(), nil
+}
+
+func (devices *DeviceSet) Shutdown() error {
+	utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
+	defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
+	devices.Lock()
+	utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
+	defer devices.Unlock()
+
+	for path, count := range devices.activeMounts {
+		for i := count; i > 0; i-- {
+			if err := syscall.Unmount(path, 0); err != nil {
+				utils.Debugf("Shutdown unmounting %s, error: %s\n", path, err)
+			}
+		}
+		delete(devices.activeMounts, path)
+	}
+
+	for _, d := range devices.Devices {
+		if err := devices.waitClose(d.Hash); err != nil {
+			utils.Errorf("Warning: error waiting for device %s to unmount: %s\n", d.Hash, err)
+		}
+		if err := devices.deactivateDevice(d.Hash); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", d.Hash, err)
+		}
+	}
+
+	pool := devices.getPoolDevName()
+	if devinfo, err := getInfo(pool); err == nil && devinfo.Exists != 0 {
+		if err := devices.deactivateDevice("pool"); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", pool, err)
+		}
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.activateDeviceIfNeeded(hash); err != nil {
+		return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
+	}
+
+	info := devices.Devices[hash]
+
+	var flags uintptr = syscall.MS_MGC_VAL
+
+	if readOnly {
+		flags = flags | syscall.MS_RDONLY
+	}
+
+	err := syscall.Mount(info.DevName(), path, "ext4", flags, "discard")
+	if err != nil && err == syscall.EINVAL {
+		err = syscall.Mount(info.DevName(), path, "ext4", flags, "")
+	}
+	if err != nil {
+		return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
+	}
+
+	count := devices.activeMounts[path]
+	devices.activeMounts[path] = count + 1
+
+	return devices.setInitialized(hash)
+}
+
+func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error {
+	utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path)
+	defer utils.Debugf("[devmapper] UnmountDevice END")
+	devices.Lock()
+	defer devices.Unlock()
+
+	utils.Debugf("[devmapper] Unmount(%s)", path)
+	if err := syscall.Unmount(path, 0); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	utils.Debugf("[devmapper] Unmount done")
+	// Wait for the unmount to be effective,
+	// by watching the value of Info.OpenCount for the device
+	if err := devices.waitClose(hash); err != nil {
+		return err
+	}
+
+	if count := devices.activeMounts[path]; count > 1 {
+		devices.activeMounts[path] = count - 1
+	} else {
+		delete(devices.activeMounts, path)
+	}
+
+	if deactivate {
+		devices.deactivateDevice(hash)
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) HasDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	return devices.Devices[hash] != nil
+}
+
+func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	info := devices.Devices[hash]
+	return info != nil && info.Initialized
+}
+
+func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	info := devices.Devices[hash]
+	if info == nil {
+		return false
+	}
+	devinfo, _ := getInfo(info.Name())
+	return devinfo != nil && devinfo.Exists != 0
+}
+
+func (devices *DeviceSet) setInitialized(hash string) error {
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	info.Initialized = true
+	if err := devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) Status() *Status {
+	devices.Lock()
+	defer devices.Unlock()
+
+	status := &Status{}
+
+	status.PoolName = devices.getPoolName()
+	status.DataLoopback = path.Join(devices.loopbackDir(), "data")
+	status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
+
+	_, totalSizeInSectors, _, params, err := getStatus(devices.getPoolName())
+	if err == nil {
+		var transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64
+		if _, err := fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal); err == nil {
+			// Convert from blocks to bytes
+			blockSizeInSectors := totalSizeInSectors / dataTotal
+
+			status.Data.Used = dataUsed * blockSizeInSectors * 512
+			status.Data.Total = dataTotal * blockSizeInSectors * 512
+
+			// metadata blocks are always 4k
+			status.Metadata.Used = metadataUsed * 4096
+			status.Metadata.Total = metadataTotal * 4096
+		}
+	}
+
+	return status
+}
+
+func NewDeviceSet(root string) (*DeviceSet, error) {
+	SetDevDir("/dev")
+
+	devices := &DeviceSet{
+		root:         root,
+		MetaData:     MetaData{Devices: make(map[string]*DevInfo)},
+		activeMounts: make(map[string]int),
+	}
+
+	if err := devices.initDevmapper(); err != nil {
+		return nil, err
+	}
+
+	return devices, nil
+}

devmapper/devmapper.go

@@ -0,0 +1,505 @@
+package devmapper
+
+import (
+	"errors"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"os"
+	"runtime"
+)
+
+type DevmapperLogger interface {
+	log(level int, file string, line int, dmError int, message string)
+}
+
+const (
+	DeviceCreate TaskType = iota
+	DeviceReload
+	DeviceRemove
+	DeviceRemoveAll
+	DeviceSuspend
+	DeviceResume
+	DeviceInfo
+	DeviceDeps
+	DeviceRename
+	DeviceVersion
+	DeviceStatus
+	DeviceTable
+	DeviceWaitevent
+	DeviceList
+	DeviceClear
+	DeviceMknodes
+	DeviceListVersions
+	DeviceTargetMsg
+	DeviceSetGeometry
+)
+
+const (
+	AddNodeOnResume AddNodeType = iota
+	AddNodeOnCreate
+)
+
+var (
+	ErrTaskRun              = errors.New("dm_task_run failed")
+	ErrTaskSetName          = errors.New("dm_task_set_name failed")
+	ErrTaskSetMessage       = errors.New("dm_task_set_message failed")
+	ErrTaskSetAddNode       = errors.New("dm_task_set_add_node failed")
+	ErrTaskSetRo            = errors.New("dm_task_set_ro failed")
+	ErrTaskAddTarget        = errors.New("dm_task_add_target failed")
+	ErrTaskSetSector        = errors.New("dm_task_set_sector failed")
+	ErrTaskGetInfo          = errors.New("dm_task_get_info failed")
+	ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed")
+	ErrTaskSetCookie        = errors.New("dm_task_set_cookie failed")
+	ErrNilCookie            = errors.New("cookie ptr can't be nil")
+	ErrAttachLoopbackDevice = errors.New("loopback mounting failed")
+	ErrGetBlockSize         = errors.New("Can't get block size")
+	ErrUdevWait             = errors.New("wait on udev cookie failed")
+	ErrSetDevDir            = errors.New("dm_set_dev_dir failed")
+	ErrGetLibraryVersion    = errors.New("dm_get_library_version failed")
+	ErrCreateRemoveTask     = errors.New("Can't create task of type DeviceRemove")
+	ErrRunRemoveDevice      = errors.New("running removeDevice failed")
+	ErrInvalidAddNode       = errors.New("Invalide AddNoce type")
+)
+
+type (
+	Task struct {
+		unmanaged *CDmTask
+	}
+	Info struct {
+		Exists        int
+		Suspended     int
+		LiveTable     int
+		InactiveTable int
+		OpenCount     int32
+		EventNr       uint32
+		Major         uint32
+		Minor         uint32
+		ReadOnly      int
+		TargetCount   int32
+	}
+	TaskType    int
+	AddNodeType int
+)
+
+func (t *Task) destroy() {
+	if t != nil {
+		DmTaskDestory(t.unmanaged)
+		runtime.SetFinalizer(t, nil)
+	}
+}
+
+func TaskCreate(tasktype TaskType) *Task {
+	Ctask := DmTaskCreate(int(tasktype))
+	if Ctask == nil {
+		return nil
+	}
+	task := &Task{unmanaged: Ctask}
+	runtime.SetFinalizer(task, (*Task).destroy)
+	return task
+}
+
+func (t *Task) Run() error {
+	if res := DmTaskRun(t.unmanaged); res != 1 {
+		return ErrTaskRun
+	}
+	return nil
+}
+
+func (t *Task) SetName(name string) error {
+	if res := DmTaskSetName(t.unmanaged, name); res != 1 {
+		return ErrTaskSetName
+	}
+	return nil
+}
+
+func (t *Task) SetMessage(message string) error {
+	if res := DmTaskSetMessage(t.unmanaged, message); res != 1 {
+		return ErrTaskSetMessage
+	}
+	return nil
+}
+
+func (t *Task) SetSector(sector uint64) error {
+	if res := DmTaskSetSector(t.unmanaged, sector); res != 1 {
+		return ErrTaskSetSector
+	}
+	return nil
+}
+
+func (t *Task) SetCookie(cookie *uint, flags uint16) error {
+	if cookie == nil {
+		return ErrNilCookie
+	}
+	if res := DmTaskSetCookie(t.unmanaged, cookie, flags); res != 1 {
+		return ErrTaskSetCookie
+	}
+	return nil
+}
+
+func (t *Task) SetAddNode(addNode AddNodeType) error {
+	if addNode != AddNodeOnResume && addNode != AddNodeOnCreate {
+		return ErrInvalidAddNode
+	}
+	if res := DmTaskSetAddNode(t.unmanaged, addNode); res != 1 {
+		return ErrTaskSetAddNode
+	}
+	return nil
+}
+
+func (t *Task) SetRo() error {
+	if res := DmTaskSetRo(t.unmanaged); res != 1 {
+		return ErrTaskSetRo
+	}
+	return nil
+}
+
+func (t *Task) AddTarget(start, size uint64, ttype, params string) error {
+	if res := DmTaskAddTarget(t.unmanaged, start, size,
+		ttype, params); res != 1 {
+		return ErrTaskAddTarget
+	}
+	return nil
+}
+
+func (t *Task) GetInfo() (*Info, error) {
+	info := &Info{}
+	if res := DmTaskGetInfo(t.unmanaged, info); res != 1 {
+		return nil, ErrTaskGetInfo
+	}
+	return info, nil
+}
+
+func (t *Task) GetNextTarget(next uintptr) (nextPtr uintptr, start uint64,
+	length uint64, targetType string, params string) {
+
+	return DmGetNextTarget(t.unmanaged, next, &start, &length,
+			&targetType, &params),
+		start, length, targetType, params
+}
+
+func AttachLoopDevice(filename string) (*os.File, error) {
+	var fd int
+	res := DmAttachLoopDevice(filename, &fd)
+	if res == "" {
+		return nil, ErrAttachLoopbackDevice
+	}
+	return os.NewFile(uintptr(fd), res), nil
+}
+
+func UdevWait(cookie uint) error {
+	if res := DmUdevWait(cookie); res != 1 {
+		utils.Debugf("Failed to wait on udev cookie %d", cookie)
+		return ErrUdevWait
+	}
+	return nil
+}
+
+func LogInitVerbose(level int) {
+	DmLogInitVerbose(level)
+}
+
+var dmLogger DevmapperLogger = nil
+
+func logInit(logger DevmapperLogger) {
+	dmLogger = logger
+	LogWithErrnoInit()
+}
+
+func SetDevDir(dir string) error {
+	if res := DmSetDevDir(dir); res != 1 {
+		utils.Debugf("Error dm_set_dev_dir")
+		return ErrSetDevDir
+	}
+	return nil
+}
+
+func GetLibraryVersion() (string, error) {
+	var version string
+	if res := DmGetLibraryVersion(&version); res != 1 {
+		return "", ErrGetLibraryVersion
+	}
+	return version, nil
+}
+
+// Useful helper for cleanup
+func RemoveDevice(name string) error {
+	task := TaskCreate(DeviceRemove)
+	if task == nil {
+		return ErrCreateRemoveTask
+	}
+	if err := task.SetName(name); err != nil {
+		utils.Debugf("Can't set task name %s", name)
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return ErrRunRemoveDevice
+	}
+	return nil
+}
+
+func GetBlockDeviceSize(file *os.File) (uint64, error) {
+	size, errno := DmGetBlockSize(file.Fd())
+	if size == -1 || errno != 0 {
+		return 0, ErrGetBlockSize
+	}
+	return uint64(size), nil
+}
+
+// This is the programmatic example of "dmsetup create"
+func createPool(poolName string, dataFile *os.File, metadataFile *os.File) error {
+	task, err := createTask(DeviceCreate, poolName)
+	if task == nil {
+		return err
+	}
+
+	size, err := GetBlockDeviceSize(dataFile)
+	if err != nil {
+		return fmt.Errorf("Can't get data size")
+	}
+
+	params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768"
+	if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate (createPool)")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createTask(t TaskType, name string) (*Task, error) {
+	task := TaskCreate(t)
+	if task == nil {
+		return nil, fmt.Errorf("Can't create task of type %d", int(t))
+	}
+	if err := task.SetName(name); err != nil {
+		return nil, fmt.Errorf("Can't set task name %s", name)
+	}
+	return task, nil
+}
+
+func getInfo(name string) (*Info, error) {
+	task, err := createTask(DeviceInfo, name)
+	if task == nil {
+		return nil, err
+	}
+	if err := task.Run(); err != nil {
+		return nil, err
+	}
+	return task.GetInfo()
+}
+
+func getStatus(name string) (uint64, uint64, string, string, error) {
+	task, err := createTask(DeviceStatus, name)
+	if task == nil {
+		utils.Debugf("getStatus: Error createTask: %s", err)
+		return 0, 0, "", "", err
+	}
+	if err := task.Run(); err != nil {
+		utils.Debugf("getStatus: Error Run: %s", err)
+		return 0, 0, "", "", err
+	}
+
+	devinfo, err := task.GetInfo()
+	if err != nil {
+		utils.Debugf("getStatus: Error GetInfo: %s", err)
+		return 0, 0, "", "", err
+	}
+	if devinfo.Exists == 0 {
+		utils.Debugf("getStatus: Non existing device %s", name)
+		return 0, 0, "", "", fmt.Errorf("Non existing device %s", name)
+	}
+
+	_, start, length, target_type, params := task.GetNextTarget(0)
+	return start, length, target_type, params, nil
+}
+
+func setTransactionId(poolName string, oldId uint64, newId uint64) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("set_transaction_id %d %d", oldId, newId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running setTransactionId")
+	}
+	return nil
+}
+
+func suspendDevice(name string) error {
+	task, err := createTask(DeviceSuspend, name)
+	if task == nil {
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+	return nil
+}
+
+func resumeDevice(name string) error {
+	task, err := createTask(DeviceResume, name)
+	if task == nil {
+		return err
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createDevice(poolName string, deviceId int) error {
+	utils.Debugf("[devmapper] createDevice(poolName=%v, deviceId=%v)", poolName, deviceId)
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_thin %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running createDevice")
+	}
+	return nil
+}
+
+func deleteDevice(poolName string, deviceId int) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("delete %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running deleteDevice")
+	}
+	return nil
+}
+
+func removeDevice(name string) error {
+	utils.Debugf("[devmapper] removeDevice START")
+	defer utils.Debugf("[devmapper] removeDevice END")
+	task, err := createTask(DeviceRemove, name)
+	if task == nil {
+		return err
+	}
+	if err = task.Run(); err != nil {
+		return fmt.Errorf("Error running removeDevice")
+	}
+	return nil
+}
+
+func activateDevice(poolName string, name string, deviceId int, size uint64) error {
+	task, err := createTask(DeviceCreate, name)
+	if task == nil {
+		return err
+	}
+
+	params := fmt.Sprintf("%s %d", poolName, deviceId)
+	if err := task.AddTarget(0, size/512, "thin", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+	if err := task.SetAddNode(AddNodeOnCreate); err != nil {
+		return fmt.Errorf("Can't add node")
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate (activateDevice)")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func (devices *DeviceSet) createSnapDevice(poolName string, deviceId int, baseName string, baseDeviceId int) error {
+	devinfo, _ := getInfo(baseName)
+	doSuspend := devinfo != nil && devinfo.Exists != 0
+
+	if doSuspend {
+		if err := suspendDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_snap %d %d", deviceId, baseDeviceId)); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Error running DeviceCreate (createSnapDevice)")
+	}
+
+	if doSuspend {
+		if err := resumeDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

devmapper/devmapper_log.go

@@ -0,0 +1,13 @@
+package devmapper
+
+import "C"
+
+// Due to the way cgo works this has to be in a separate file, as devmapper.go has
+// definitions in the cgo block, which is incompatible with using "//export"
+
+//export DevmapperLogCallback
+func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_class C.int, message *C.char) {
+	if dmLogger != nil {
+		dmLogger.log(int(level), C.GoString(file), int(line), int(dm_errno_or_class), C.GoString(message))
+	}
+}

devmapper/devmapper_test.go

@@ -0,0 +1,285 @@
+package devmapper
+
+import (
+	"syscall"
+	"testing"
+)
+
+func TestTaskCreate(t *testing.T) {
+	// Test success
+	taskCreate(t, DeviceInfo)
+
+	// Test Failure
+	DmTaskCreate = dmTaskCreateFail
+	defer func() { DmTaskCreate = dmTaskCreateFct }()
+	if task := TaskCreate(-1); task != nil {
+		t.Fatalf("An error should have occured while creating an invalid task.")
+	}
+}
+
+func TestTaskRun(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	// Perform the RUN
+	if err := task.Run(); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure we don't have error with GetInfo
+	if _, err := task.GetInfo(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskRun = dmTaskRunFail
+	defer func() { DmTaskRun = dmTaskRunFct }()
+
+	task = taskCreate(t, DeviceInfo)
+	// Perform the RUN
+	if err := task.Run(); err != ErrTaskRun {
+		t.Fatalf("An error should have occured while running task.")
+	}
+	// Make sure GetInfo also fails
+	if _, err := task.GetInfo(); err != ErrTaskGetInfo {
+		t.Fatalf("GetInfo should fail if task.Run() failed.")
+	}
+}
+
+func TestTaskSetName(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetName("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetName = dmTaskSetNameFail
+	defer func() { DmTaskSetName = dmTaskSetNameFct }()
+
+	if err := task.SetName("test"); err != ErrTaskSetName {
+		t.Fatalf("An error should have occured while runnign SetName.")
+	}
+}
+
+func TestTaskSetMessage(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetMessage("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetMessage = dmTaskSetMessageFail
+	defer func() { DmTaskSetMessage = dmTaskSetMessageFct }()
+
+	if err := task.SetMessage("test"); err != ErrTaskSetMessage {
+		t.Fatalf("An error should have occured while runnign SetMessage.")
+	}
+}
+
+func TestTaskSetSector(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetSector(128); err != nil {
+		t.Fatal(err)
+	}
+
+	DmTaskSetSector = dmTaskSetSectorFail
+	defer func() { DmTaskSetSector = dmTaskSetSectorFct }()
+
+	// Test failure
+	if err := task.SetSector(0); err != ErrTaskSetSector {
+		t.Fatalf("An error should have occured while running SetSector.")
+	}
+}
+
+func TestTaskSetCookie(t *testing.T) {
+	var (
+		cookie uint = 0
+		task        = taskCreate(t, DeviceInfo)
+	)
+
+	// Test success
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	if err := task.SetCookie(nil, 0); err != ErrNilCookie {
+		t.Fatalf("An error should have occured while running SetCookie with nil cookie.")
+	}
+
+	DmTaskSetCookie = dmTaskSetCookieFail
+	defer func() { DmTaskSetCookie = dmTaskSetCookieFct }()
+
+	if err := task.SetCookie(&cookie, 0); err != ErrTaskSetCookie {
+		t.Fatalf("An error should have occured while running SetCookie.")
+	}
+}
+
+func TestTaskSetAddNode(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetAddNode(0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	if err := task.SetAddNode(-1); err != ErrInvalidAddNode {
+		t.Fatalf("An error should have occured running SetAddNode with wrong node.")
+	}
+
+	DmTaskSetAddNode = dmTaskSetAddNodeFail
+	defer func() { DmTaskSetAddNode = dmTaskSetAddNodeFct }()
+
+	if err := task.SetAddNode(0); err != ErrTaskSetAddNode {
+		t.Fatalf("An error should have occured running SetAddNode.")
+	}
+}
+
+func TestTaskSetRo(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetRo(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetRo = dmTaskSetRoFail
+	defer func() { DmTaskSetRo = dmTaskSetRoFct }()
+
+	if err := task.SetRo(); err != ErrTaskSetRo {
+		t.Fatalf("An error should have occured running SetRo.")
+	}
+}
+
+func TestTaskAddTarget(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.AddTarget(0, 128, "thinp", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskAddTarget = dmTaskAddTargetFail
+	defer func() { DmTaskAddTarget = dmTaskAddTargetFct }()
+
+	if err := task.AddTarget(0, 128, "thinp", ""); err != ErrTaskAddTarget {
+		t.Fatalf("An error should have occured running AddTarget.")
+	}
+}
+
+// func TestTaskGetInfo(t *testing.T) {
+// 	task := taskCreate(t, DeviceInfo)
+
+// 	// Test success
+// 	if _, err := task.GetInfo(); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	// Test failure
+// 	DmTaskGetInfo = dmTaskGetInfoFail
+// 	defer func() { DmTaskGetInfo = dmTaskGetInfoFct }()
+
+// 	if _, err := task.GetInfo(); err != ErrTaskGetInfo {
+// 		t.Fatalf("An error should have occured running GetInfo.")
+// 	}
+// }
+
+// func TestTaskGetNextTarget(t *testing.T) {
+// 	task := taskCreate(t, DeviceInfo)
+
+// 	if next, _, _, _, _ := task.GetNextTarget(0); next == 0 {
+// 		t.Fatalf("The next target should not be 0.")
+// 	}
+// }
+
+/// Utils
+func taskCreate(t *testing.T, taskType TaskType) *Task {
+	task := TaskCreate(taskType)
+	if task == nil {
+		t.Fatalf("Error creating task")
+	}
+	return task
+}
+
+/// Failure function replacement
+func dmTaskCreateFail(t int) *CDmTask {
+	return nil
+}
+
+func dmTaskRunFail(task *CDmTask) int {
+	return -1
+}
+
+func dmTaskSetNameFail(task *CDmTask, name string) int {
+	return -1
+}
+
+func dmTaskSetMessageFail(task *CDmTask, message string) int {
+	return -1
+}
+
+func dmTaskSetSectorFail(task *CDmTask, sector uint64) int {
+	return -1
+}
+
+func dmTaskSetCookieFail(task *CDmTask, cookie *uint, flags uint16) int {
+	return -1
+}
+
+func dmTaskSetAddNodeFail(task *CDmTask, addNode AddNodeType) int {
+	return -1
+}
+
+func dmTaskSetRoFail(task *CDmTask) int {
+	return -1
+}
+
+func dmTaskAddTargetFail(task *CDmTask,
+	start, size uint64, ttype, params string) int {
+	return -1
+}
+
+func dmTaskGetDriverVersionFail(task *CDmTask, version *string) int {
+	return -1
+}
+
+func dmTaskGetInfoFail(task *CDmTask, info *Info) int {
+	return -1
+}
+
+func dmGetNextTargetFail(task *CDmTask, next uintptr, start, length *uint64,
+	target, params *string) uintptr {
+	return 0
+}
+
+func dmAttachLoopDeviceFail(filename string, fd *int) string {
+	return ""
+}
+
+func sysGetBlockSizeFail(fd uintptr, size *uint64) syscall.Errno {
+	return 1
+}
+
+func dmGetBlockSizeFail(fd uintptr) int64 {
+	return -1
+}
+
+func dmUdevWaitFail(cookie uint) int {
+	return -1
+}
+
+func dmSetDevDirFail(dir string) int {
+	return -1
+}
+
+func dmGetLibraryVersionFail(version *string) int {
+	return -1
+}

devmapper/devmapper_wrapper.go

@@ -0,0 +1,330 @@
+package devmapper
+
+/*
+#cgo LDFLAGS: -L. -ldevmapper
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <libdevmapper.h>
+#include <linux/loop.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+
+#ifndef LOOP_CTL_GET_FREE
+#define LOOP_CTL_GET_FREE       0x4C82
+#endif
+
+// FIXME: this could easily be rewritten in go
+char*			attach_loop_device(const char *filename, int *loop_fd_out)
+{
+  struct loop_info64	loopinfo = {0};
+  struct stat		st;
+  char			buf[64];
+  int			i, loop_fd, fd, start_index;
+  char*			loopname;
+
+
+  *loop_fd_out = -1;
+
+  start_index = 0;
+  fd = open("/dev/loop-control", O_RDONLY);
+  if (fd >= 0) {
+    start_index = ioctl(fd, LOOP_CTL_GET_FREE);
+    close(fd);
+
+    if (start_index < 0)
+      start_index = 0;
+  }
+
+  fd = open(filename, O_RDWR);
+  if (fd < 0) {
+    perror("open");
+    return NULL;
+  }
+
+  loop_fd = -1;
+  for (i = start_index ; loop_fd < 0 ; i++ ) {
+    if (sprintf(buf, "/dev/loop%d", i) < 0) {
+	close(fd);
+	return NULL;
+    }
+
+    if (stat(buf, &st)) {
+      if (!S_ISBLK(st.st_mode)) {
+	 fprintf(stderr, "[error] Loopback device %s is not a block device.\n", buf);
+      } else if (errno == ENOENT) {
+	fprintf(stderr, "[error] There are no more loopback device available.\n");
+      } else {
+	fprintf(stderr, "[error] Unkown error trying to stat the loopback device %s (errno: %d).\n", buf, errno);
+      }
+      close(fd);
+      return NULL;
+    }
+
+    loop_fd = open(buf, O_RDWR);
+    if (loop_fd < 0 && errno == ENOENT) {
+      fprintf(stderr, "[error] The loopback device %s does not exists.\n", buf);
+      close(fd);
+      return NULL;
+    } else if (loop_fd < 0) {
+	fprintf(stderr, "[error] Unkown error openning the loopback device %s. (errno: %d)\n", buf, errno);
+	continue;
+    }
+
+    if (ioctl(loop_fd, LOOP_SET_FD, (void *)(size_t)fd) < 0) {
+      int errsv = errno;
+      close(loop_fd);
+      loop_fd = -1;
+      if (errsv != EBUSY) {
+        close(fd);
+        fprintf(stderr, "cannot set up loopback device %s: %s", buf, strerror(errsv));
+        return NULL;
+      }
+      continue;
+    }
+
+    close(fd);
+
+    strncpy((char*)loopinfo.lo_file_name, buf, LO_NAME_SIZE);
+    loopinfo.lo_offset = 0;
+    loopinfo.lo_flags = LO_FLAGS_AUTOCLEAR;
+
+    if (ioctl(loop_fd, LOOP_SET_STATUS64, &loopinfo) < 0) {
+      perror("ioctl LOOP_SET_STATUS64");
+      if (ioctl(loop_fd, LOOP_CLR_FD, 0) < 0) {
+        perror("ioctl LOOP_CLR_FD");
+      }
+      close(loop_fd);
+      fprintf (stderr, "cannot set up loopback device info");
+      return (NULL);
+    }
+
+    loopname = strdup(buf);
+    if (loopname == NULL) {
+      close(loop_fd);
+      return (NULL);
+    }
+
+    *loop_fd_out = loop_fd;
+    return (loopname);
+  }
+
+  return (NULL);
+}
+
+extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);
+
+static void	log_cb(int level, const char *file, int line,
+		       int dm_errno_or_class, const char *f, ...)
+{
+  char buffer[256];
+  va_list ap;
+
+  va_start(ap, f);
+  vsnprintf(buffer, 256, f, ap);
+  va_end(ap);
+
+  DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
+}
+
+static void	log_with_errno_init()
+{
+  dm_log_with_errno_init(log_cb);
+}
+
+*/
+import "C"
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+type (
+	CDmTask C.struct_dm_task
+)
+
+var (
+	DmTaskDestory       = dmTaskDestroyFct
+	DmTaskCreate        = dmTaskCreateFct
+	DmTaskRun           = dmTaskRunFct
+	DmTaskSetName       = dmTaskSetNameFct
+	DmTaskSetMessage    = dmTaskSetMessageFct
+	DmTaskSetSector     = dmTaskSetSectorFct
+	DmTaskSetCookie     = dmTaskSetCookieFct
+	DmTaskSetAddNode    = dmTaskSetAddNodeFct
+	DmTaskSetRo         = dmTaskSetRoFct
+	DmTaskAddTarget     = dmTaskAddTargetFct
+	DmTaskGetInfo       = dmTaskGetInfoFct
+	DmGetNextTarget     = dmGetNextTargetFct
+	DmGetBlockSize      = dmGetBlockSizeFct
+	DmAttachLoopDevice  = dmAttachLoopDeviceFct
+	DmUdevWait          = dmUdevWaitFct
+	DmLogInitVerbose    = dmLogInitVerboseFct
+	DmSetDevDir         = dmSetDevDirFct
+	DmGetLibraryVersion = dmGetLibraryVersionFct
+	LogWithErrnoInit    = logWithErrnoInitFct
+	GetBlockSize        = getBlockSizeFct
+)
+
+func free(p *C.char) {
+	C.free(unsafe.Pointer(p))
+}
+
+func dmTaskDestroyFct(task *CDmTask) {
+	C.dm_task_destroy((*C.struct_dm_task)(task))
+}
+
+func dmTaskCreateFct(taskType int) *CDmTask {
+	return (*CDmTask)(C.dm_task_create(C.int(taskType)))
+}
+
+func dmTaskRunFct(task *CDmTask) int {
+	return int(C.dm_task_run((*C.struct_dm_task)(task)))
+}
+
+func dmTaskSetNameFct(task *CDmTask, name string) int {
+	Cname := C.CString(name)
+	defer free(Cname)
+
+	return int(C.dm_task_set_name((*C.struct_dm_task)(task),
+		Cname))
+}
+
+func dmTaskSetMessageFct(task *CDmTask, message string) int {
+	Cmessage := C.CString(message)
+	defer free(Cmessage)
+
+	return int(C.dm_task_set_message((*C.struct_dm_task)(task),
+		Cmessage))
+}
+
+func dmTaskSetSectorFct(task *CDmTask, sector uint64) int {
+	return int(C.dm_task_set_sector((*C.struct_dm_task)(task),
+		C.uint64_t(sector)))
+}
+
+func dmTaskSetCookieFct(task *CDmTask, cookie *uint, flags uint16) int {
+	cCookie := C.uint32_t(*cookie)
+	defer func() {
+		*cookie = uint(cCookie)
+	}()
+	return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie,
+		C.uint16_t(flags)))
+}
+
+func dmTaskSetAddNodeFct(task *CDmTask, addNode AddNodeType) int {
+	return int(C.dm_task_set_add_node((*C.struct_dm_task)(task),
+		C.dm_add_node_t(addNode)))
+}
+
+func dmTaskSetRoFct(task *CDmTask) int {
+	return int(C.dm_task_set_ro((*C.struct_dm_task)(task)))
+}
+
+func dmTaskAddTargetFct(task *CDmTask,
+	start, size uint64, ttype, params string) int {
+
+	Cttype := C.CString(ttype)
+	defer free(Cttype)
+
+	Cparams := C.CString(params)
+	defer free(Cparams)
+
+	return int(C.dm_task_add_target((*C.struct_dm_task)(task),
+		C.uint64_t(start), C.uint64_t(size), Cttype, Cparams))
+}
+
+func dmGetBlockSizeFct(fd uintptr) (int64, syscall.Errno) {
+	var size int64
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, C.BLKGETSIZE64,
+		uintptr(unsafe.Pointer(&size)))
+	return size, err
+}
+
+func dmTaskGetInfoFct(task *CDmTask, info *Info) int {
+	Cinfo := C.struct_dm_info{}
+	defer func() {
+		info.Exists = int(Cinfo.exists)
+		info.Suspended = int(Cinfo.suspended)
+		info.LiveTable = int(Cinfo.live_table)
+		info.InactiveTable = int(Cinfo.inactive_table)
+		info.OpenCount = int32(Cinfo.open_count)
+		info.EventNr = uint32(Cinfo.event_nr)
+		info.Major = uint32(Cinfo.major)
+		info.Minor = uint32(Cinfo.minor)
+		info.ReadOnly = int(Cinfo.read_only)
+		info.TargetCount = int32(Cinfo.target_count)
+	}()
+	return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
+}
+
+func dmGetNextTargetFct(task *CDmTask, next uintptr, start, length *uint64,
+	target, params *string) uintptr {
+
+	var (
+		Cstart, Clength      C.uint64_t
+		CtargetType, Cparams *C.char
+	)
+	defer func() {
+		*start = uint64(Cstart)
+		*length = uint64(Clength)
+		*target = C.GoString(CtargetType)
+		*params = C.GoString(Cparams)
+	}()
+	nextp := C.dm_get_next_target((*C.struct_dm_task)(task),
+		unsafe.Pointer(next), &Cstart, &Clength, &CtargetType, &Cparams)
+	return uintptr(nextp)
+}
+
+func dmAttachLoopDeviceFct(filename string, fd *int) string {
+	cFilename := C.CString(filename)
+	defer free(cFilename)
+
+	var cFd C.int
+	defer func() {
+		*fd = int(cFd)
+	}()
+
+	ret := C.attach_loop_device(cFilename, &cFd)
+	defer free(ret)
+	return C.GoString(ret)
+}
+
+func getBlockSizeFct(fd uintptr, size *uint64) syscall.Errno {
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, C.BLKGETSIZE64,
+		uintptr(unsafe.Pointer(&size)))
+	return err
+}
+
+func dmUdevWaitFct(cookie uint) int {
+	return int(C.dm_udev_wait(C.uint32_t(cookie)))
+}
+
+func dmLogInitVerboseFct(level int) {
+	C.dm_log_init_verbose(C.int(level))
+}
+
+func logWithErrnoInitFct() {
+	C.log_with_errno_init()
+}
+
+func dmSetDevDirFct(dir string) int {
+	Cdir := C.CString(dir)
+	defer free(Cdir)
+
+	return int(C.dm_set_dev_dir(Cdir))
+}
+
+func dmGetLibraryVersionFct(version *string) int {
+	buffer := C.CString(string(make([]byte, 128)))
+	defer free(buffer)
+	defer func() {
+		*version = C.GoString(buffer)
+	}()
+	return int(C.dm_get_library_version(buffer, 128))
+}

devmapper/docker-device-tool/device_tool.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/devmapper"
+	"os"
+)
+
+func usage() {
+	fmt.Printf("Usage: %s [snap new-id base-id] | [remove id] | [mount id mountpoint]\n", os.Args[0])
+	os.Exit(1)
+}
+
+func main() {
+	devices := devmapper.NewDeviceSet("/var/lib/docker")
+
+	if len(os.Args) < 2 {
+		usage()
+	}
+
+	cmd := os.Args[1]
+	if cmd == "snap" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.AddDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "remove" {
+		if len(os.Args) < 3 {
+			usage()
+		}
+
+		err := devices.RemoveDevice(os.Args[2])
+		if err != nil {
+			fmt.Println("Can't remove device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "mount" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.MountDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Printf("Unknown command %s\n", cmd)
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		os.Exit(1)
+	}
+
+	return
+}

devmapper/driver.go

@@ -0,0 +1,77 @@
+package devmapper
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/graphdriver"
+	"os"
+	"path"
+)
+
+func init() {
+	graphdriver.Register("devicemapper", Init)
+}
+
+// Placeholder interfaces, to be replaced
+// at integration.
+
+// End of placeholder interfaces.
+
+type Driver struct {
+	*DeviceSet
+	home string
+}
+
+func Init(home string) (graphdriver.Driver, error) {
+	deviceSet, err := NewDeviceSet(home)
+	if err != nil {
+		return nil, err
+	}
+	d := &Driver{
+		DeviceSet: deviceSet,
+		home:      home,
+	}
+	return d, nil
+}
+
+func (d *Driver) String() string {
+	return "devicemapper"
+}
+
+func (d *Driver) Cleanup() error {
+	return d.DeviceSet.Shutdown()
+}
+
+func (d *Driver) Create(id string, parent string) error {
+	return d.DeviceSet.AddDevice(id, parent)
+}
+
+func (d *Driver) Remove(id string) error {
+	return d.DeviceSet.RemoveDevice(id)
+}
+
+func (d *Driver) Get(id string) (string, error) {
+	mp := path.Join(d.home, "mnt", id)
+	if err := d.mount(id, mp); err != nil {
+		return "", err
+	}
+	return mp, nil
+}
+
+func (d *Driver) Size(id string) (int64, error) {
+	return -1, fmt.Errorf("Not implemented")
+}
+
+func (d *Driver) mount(id, mountPoint string) error {
+	// Create the target directories if they don't exist
+	if err := os.MkdirAll(mountPoint, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+	// If mountpoint is already mounted, do nothing
+	if mounted, err := Mounted(mountPoint); err != nil {
+		return fmt.Errorf("Error checking mountpoint: %s", err)
+	} else if mounted {
+		return nil
+	}
+	// Mount the device
+	return d.DeviceSet.MountDevice(id, mountPoint, false)
+}

devmapper/driver_test.go

@@ -0,0 +1,301 @@
+package devmapper
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func init() {
+	// Reduce the size the the base fs and loopback for the tests
+	DefaultDataLoopbackSize = 300 * 1024 * 1024
+	DefaultMetaDataLoopbackSize = 200 * 1024 * 1024
+	DefaultBaseFsSize = 300 * 1024 * 1024
+
+}
+
+func mkTestDirectory(t *testing.T) string {
+	dir, err := ioutil.TempDir("", "docker-test-devmapper-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func newDriver(t *testing.T) *Driver {
+	home := mkTestDirectory(t)
+	d, err := Init(home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d.(*Driver)
+}
+
+func cleanup(d *Driver) {
+	d.Cleanup()
+	os.RemoveAll(d.home)
+}
+
+func TestInit(t *testing.T) {
+	home := mkTestDirectory(t)
+	defer os.RemoveAll(home)
+	driver, err := Init(home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := driver.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	id := "foo"
+	if err := driver.Create(id, ""); err != nil {
+		t.Fatal(err)
+	}
+	dir, err := driver.Get(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st, err := os.Stat(dir); err != nil {
+		t.Fatal(err)
+	} else if !st.IsDir() {
+		t.Fatalf("Get(%V) did not return a directory", id)
+	}
+}
+
+func TestDriverName(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if d.String() != "devicemapper" {
+		t.Fatalf("Expected driver name to be devicemapper got %s", d.String())
+	}
+}
+
+func TestDriverCreate(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestDriverRemove(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Remove("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCleanup(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(d.home)
+
+	mountPoints := make([]string, 2)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	// Mount the id
+	p, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountPoints[0] = p
+
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	p, err = d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountPoints[1] = p
+
+	// Ensure that all the mount points are currently mounted
+	for _, p := range mountPoints {
+		if mounted, err := Mounted(p); err != nil {
+			t.Fatal(err)
+		} else if !mounted {
+			t.Fatalf("Expected %s to be mounted", p)
+		}
+	}
+
+	// Ensure that devices are active
+	for _, p := range []string{"1", "2"} {
+		if !d.HasActivatedDevice(p) {
+			t.Fatalf("Expected %s to have an active device", p)
+		}
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that all the mount points are no longer mounted
+	for _, p := range mountPoints {
+		if mounted, err := Mounted(p); err != nil {
+			t.Fatal(err)
+		} else if mounted {
+			t.Fatalf("Expected %s to not be mounted", p)
+		}
+	}
+
+	// Ensure that devices are no longer activated
+	for _, p := range []string{"1", "2"} {
+		if d.HasActivatedDevice(p) {
+			t.Fatalf("Expected %s not be an active device", p)
+		}
+	}
+}
+
+func TestNotMounted(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	mounted, err := Mounted(path.Join(d.home, "mnt", "1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mounted {
+		t.Fatal("Id 1 should not be mounted")
+	}
+}
+
+func TestMounted(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	mounted, err := Mounted(path.Join(d.home, "mnt", "1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !mounted {
+		t.Fatal("Id 1 should be mounted")
+	}
+}
+
+func TestInitCleanedDriver(t *testing.T) {
+	d := newDriver(t)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+
+	driver, err := Init(d.home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d = driver.(*Driver)
+	defer cleanup(d)
+
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMountMountedDriver(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// Perform get on same id to ensure that it will
+	// not be mounted twice
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetReturnsValidDevice(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if !d.HasDevice("1") {
+		t.Fatalf("Expected id 1 to be in device set")
+	}
+
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	if !d.HasActivatedDevice("1") {
+		t.Fatalf("Expected id 1 to be activated")
+	}
+
+	if !d.HasInitializedDevice("1") {
+		t.Fatalf("Expected id 1 to be initialized")
+	}
+}
+
+func TestDriverGetSize(t *testing.T) {
+	t.Skipf("Size is currently not implemented")
+
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	mountPoint, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(mountPoint, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Truncate(size); err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+
+	diffSize, err := d.Size("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diffSize != size {
+		t.Fatalf("Expected size %d got %d", size, diffSize)
+	}
+}

devmapper/mount.go

@@ -0,0 +1,27 @@
+package devmapper
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// FIXME: this is copy-pasted from the aufs driver.
+// It should be moved into the core.
+
+func Mounted(mountpoint string) (bool, error) {
+	mntpoint, err := os.Stat(mountpoint)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	parent, err := os.Stat(filepath.Join(mountpoint, ".."))
+	if err != nil {
+		return false, err
+	}
+	mntpointSt := mntpoint.Sys().(*syscall.Stat_t)
+	parentSt := parent.Sys().(*syscall.Stat_t)
+	return mntpointSt.Dev != parentSt.Dev, nil
+}

docs/Makefile

@@ -31,7 +31,7 @@ help:
 #	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 #	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 #	@echo "  text       to make text files"
-	@echo "  man        to make a manual page"
+#	@echo "  man        to make manual pages"
 #	@echo "  texinfo    to make Texinfo files"
 #	@echo "  info       to make Texinfo files and run them through makeinfo"
 #	@echo "  gettext    to make PO message catalogs"

docs/requirements.txt

@@ -1,2 +1,2 @@
 Sphinx==1.1.3
-sphinxcontrib-httpdomain==1.1.9
+sphinxcontrib-httpdomain==1.1.8

docs/sources/api/docker_remote_api_v1.7.rst

@@ -13,12 +13,9 @@ Docker Remote API v1.7
 1. Brief introduction
 =====================
 
-- The Remote API has replaced rcli
-- The daemon listens on ``unix:///var/run/docker.sock``, but you can
-  :ref:`bind_docker`.
-- The API tends to be REST, but for some complex commands, like
-  ``attach`` or ``pull``, the HTTP connection is hijacked to transport
-  ``stdout, stdin`` and ``stderr``
+- The Remote API is replacing rcli
+- Default port in the docker daemon is 4243
+- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr
 
 2. Endpoints
 ============
@@ -693,12 +690,10 @@ Create an image
         :query repo: repository
 	:query tag: tag
 	:query registry: the registry to pull from
-	:reqheader X-Registry-Auth: base64-encoded AuthConfig object
         :statuscode 200: no error
         :statuscode 500: server error
 
 
-
 Insert a file in an image
 *************************
 
@@ -840,16 +835,18 @@ Push an image on the registry
     HTTP/1.1 200 OK
     Content-Type: application/json
 
-    {"status":"Pushing..."}
-    {"status":"Pushing", "progress":"1/? (n/a)"}
-    {"error":"Invalid..."}
-    ...
+   {"status":"Pushing..."}
+   {"status":"Pushing", "progress":"1/? (n/a)"}
+   {"error":"Invalid..."}
+   ...
+
+	The ``X-Registry-Auth`` header can be used to include a
+	base64-encoded AuthConfig object.
 
    :query registry: the registry you wan to push, optional
-   :reqheader X-Registry-Auth: include a base64-encoded AuthConfig object.
    :statuscode 200: no error
-   :statuscode 404: no such image
-   :statuscode 500: server error
+        :statuscode 404: no such image
+        :statuscode 500: server error
 
 
 Tag an image into a repository
@@ -962,9 +959,9 @@ Search images
 	   ...
 	   ]
 
-	:query term: term to search
-	:statuscode 200: no error
-	:statuscode 500: server error
+	   :query term: term to search
+	   :statuscode 200: no error
+	   :statuscode 500: server error
 
 
 2.3 Misc
@@ -994,22 +991,18 @@ Build an image from Dockerfile via stdin
       {{ STREAM }}
 
 
-   The stream must be a tar archive compressed with one of the
-   following algorithms: identity (no compression), gzip, bzip2,
-   xz. 
+       The stream must be a tar archive compressed with one of the following algorithms:
+       identity (no compression), gzip, bzip2, xz. The archive must include a file called
+       `Dockerfile` at its root. It may include any number of other files, which will be
+       accessible in the build context (See the ADD build command).
 
-   The archive must include a file called ``Dockerfile`` at its
-   root. It may include any number of other files, which will be
-   accessible in the build context (See the :ref:`ADD build command
-   <dockerbuilder>`).
-
-   :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success
-   :query q: suppress verbose build output
-   :query nocache: do not use the cache when building the image
-   :reqheader Content-type: should be set to ``"application/tar"``.
-   :statuscode 200: no error
-   :statuscode 500: server error
+       The Content-type header should be set to "application/tar".
 
+	:query t: repository name (and optionally a tag) to be applied to the resulting image in case of success
+	:query q: suppress verbose build output
+    :query nocache: do not use the cache when building the image
+	:statuscode 200: no error
+    :statuscode 500: server error
 
 
 Check auth configuration
@@ -1171,53 +1164,6 @@ Monitor Docker's events
         :statuscode 200: no error
         :statuscode 500: server error
 
-Get a tarball containing all images and tags in a repository
-************************************************************
-
-.. http:get:: /images/(name)/get
-
-  Get a tarball containing all images and metadata for the repository specified by ``name``.
-
-  **Example request**
-
-  .. sourcecode:: http
-  
-           GET /images/ubuntu/get
-
-       **Example response**:
-
-       .. sourcecode:: http
-
-          HTTP/1.1 200 OK
-    Content-Type: application/x-tar
-
-    Binary data stream
-        :statuscode 200: no error
-        :statuscode 500: server error
-
-Load a tarball with a set of images and tags into docker
-********************************************************
-
-.. http:post:: /images/load
-
-  Load a set of images and tags into the docker repository.
-
-  **Example request**
-
-  .. sourcecode:: http
-
-           POST /images/load
-
-         Tarball in body
-
-       **Example response**:
-
-       .. sourcecode:: http
-
-          HTTP/1.1 200 OK
-
-        :statuscode 200: no error
-        :statuscode 500: server error
 
 3. Going further
 ================

docs/sources/api/remote_api_client_libraries.rst

@@ -1,4 +1,4 @@
-:title: Remote API Client Libraries
+:title: Registry API
 :description: Various client libraries available to use with the Docker remote API
 :keywords: API, Docker, index, registry, REST, documentation, clients, Python, Ruby, Javascript, Erlang, Go
 

docs/sources/commandline/cli.rst

@@ -88,65 +88,31 @@ Examples:
 
     Usage: docker build [OPTIONS] PATH | URL | -
     Build a new container image from the source code at PATH
-      -t="": Repository name (and optionally a tag) to be applied 
-             to the resulting image in case of success.
+      -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success.
       -q=false: Suppress verbose build output.
       -no-cache: Do not use the cache when building the image.
       -rm: Remove intermediate containers after a successful build
-
-The files at PATH or URL are called the "context" of the build. The
-build process may refer to any of the files in the context, for
-example when using an :ref:`ADD <dockerfile_add>` instruction.  When a
-single ``Dockerfile`` is given as URL, then no context is set.  When a
-git repository is set as URL, then the repository is used as the
-context
+    When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context
 
 .. _cli_build_examples:
 
-.. seealso:: :ref:`dockerbuilder`.
-
 Examples:
 ~~~~~~~~~
 
 .. code-block:: bash
 
     sudo docker build .
-    Uploading context 10240 bytes
-    Step 1 : FROM busybox
-    Pulling repository busybox
-     ---> e9aa60c60128MB/2.284 MB (100%) endpoint: https://cdn-registry-1.docker.io/v1/
-    Step 2 : RUN ls -lh /
-     ---> Running in 9c9e81692ae9
-    total 24
-    drwxr-xr-x    2 root     root        4.0K Mar 12  2013 bin
-    drwxr-xr-x    5 root     root        4.0K Oct 19 00:19 dev
-    drwxr-xr-x    2 root     root        4.0K Oct 19 00:19 etc
-    drwxr-xr-x    2 root     root        4.0K Nov 15 23:34 lib
-    lrwxrwxrwx    1 root     root           3 Mar 12  2013 lib64 -> lib
-    dr-xr-xr-x  116 root     root           0 Nov 15 23:34 proc
-    lrwxrwxrwx    1 root     root           3 Mar 12  2013 sbin -> bin
-    dr-xr-xr-x   13 root     root           0 Nov 15 23:34 sys
-    drwxr-xr-x    2 root     root        4.0K Mar 12  2013 tmp
-    drwxr-xr-x    2 root     root        4.0K Nov 15 23:34 usr
-     ---> b35f4035db3f
-    Step 3 : CMD echo Hello World
-     ---> Running in 02071fceb21b
-     ---> f52f38b7823e
-    Successfully built f52f38b7823e
 
-This example specifies that the PATH is ``.``, and so all the files in
-the local directory get tar'd and sent to the Docker daemon.  The PATH
-specifies where to find the files for the "context" of the build on
-the Docker daemon. Remember that the daemon could be running on a
-remote machine and that no parsing of the Dockerfile happens at the
-client side (where you're running ``docker build``). That means that
-*all* the files at PATH get sent, not just the ones listed to
-:ref:`ADD <dockerfile_add>` in the ``Dockerfile``.
-
-The transfer of context from the local machine to the Docker daemon is
-what the ``docker`` client means when you see the "Uploading context"
-message.
+This will read the ``Dockerfile`` from the current directory. It will
+also send any other files and directories found in the current
+directory to the ``docker`` daemon.
 
+The contents of this directory would be used by ``ADD`` commands found
+within the ``Dockerfile``.  This will send a lot of data to the
+``docker`` daemon if the current directory contains a lot of data.  If
+the absolute path is provided instead of ``.`` then only the files and
+directories required by the ADD commands from the ``Dockerfile`` will be
+added to the context and transferred to the ``docker`` daemon.
 
 .. code-block:: bash
 
@@ -163,15 +129,16 @@ tag will be ``2.0``
 
 This will read a ``Dockerfile`` from *stdin* without context. Due to
 the lack of a context, no contents of any local directory will be sent
-to the ``docker`` daemon.  Since there is no context, a Dockerfile
-``ADD`` only works if it refers to a remote URL.
+to the ``docker`` daemon.  ``ADD`` doesn't work when running in this
+mode because the absence of the context provides no source files to
+copy to the container.
 
 .. code-block:: bash
 
     sudo docker build github.com/creack/docker-firefox
 
-This will clone the Github repository and use the cloned repository as
-context. The ``Dockerfile`` at the root of the repository is used as
+This will clone the Github repository and use it as context. The
+``Dockerfile`` at the root of the repository is used as
 ``Dockerfile``.  Note that you can specify an arbitrary git repository
 by using the ``git://`` schema.
 
@@ -190,7 +157,7 @@ by using the ``git://`` schema.
       -m="": Commit message
       -author="": Author (eg. "John Hannibal Smith <hannibal@a-team.com>"
       -run="": Configuration to be applied when the image is launched with `docker run`.
-               (ex: -run='{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')
+               (ex: '{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}')
 
 Simple commit of an existing container
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -206,7 +173,7 @@ Simple commit of an existing container
 	$ docker images | head
 	REPOSITORY                        TAG                 ID                  CREATED             SIZE
 	SvenDowideit/testimage            version3            f5283438590d        16 seconds ago      204.2 MB (virtual 335.7 MB)
-	
+	S
 
 Full -run example
 .................
@@ -252,15 +219,10 @@ Full -run example
 
 ::
 
-    Usage: docker cp CONTAINER:PATH HOSTPATH
+    Usage: docker cp CONTAINER:RESOURCE HOSTPATH
 
     Copy files/folders from the containers filesystem to the host
     path.  Paths are relative to the root of the filesystem.
-    
-.. code-block:: bash
-
-    $ sudo docker cp 7bb0e258aefe:/etc/debian_version .
-    $ sudo docker cp blue_frog:/etc/hosts .
 
 .. _cli_diff:
 
@@ -269,33 +231,9 @@ Full -run example
 
 ::
 
-    Usage: docker diff CONTAINER
- 
-    List the changed files and directories in a container's filesystem
+    Usage: docker diff CONTAINER [OPTIONS]
 
-There are 3 events that are listed in the 'diff':
-
-1. ```A``` - Add
-2. ```D``` - Delete
-3. ```C``` - Change
-
-for example:
-
-.. code-block:: bash
-
-	$ sudo docker diff 7bb0e258aefe
-
-	C /dev
-	A /dev/kmsg
-	C /etc
-	A /etc/mtab
-	A /go
-	A /go/src
-	A /go/src/github.com
-	A /go/src/github.com/dotcloud
-	A /go/src/github.com/dotcloud/docker
-	A /go/src/github.com/dotcloud/docker/.git
-	....
+    Inspect changes on a container's filesystem
 
 .. _cli_events:
 
@@ -385,40 +323,6 @@ Show events in the past from a specified time
       -notrunc=false: Don't truncate output
       -q=false: only show numeric IDs
 
-To see how the docker:latest image was built:
-
-.. code-block:: bash
-
-	$ docker history docker
-	ID                  CREATED             CREATED BY
-	docker:latest       19 hours ago        /bin/sh -c #(nop) ADD . in /go/src/github.com/dotcloud/docker
-	cf5f2467662d        2 weeks ago         /bin/sh -c #(nop) ENTRYPOINT ["hack/dind"]
-	3538fbe372bf        2 weeks ago         /bin/sh -c #(nop) WORKDIR /go/src/github.com/dotcloud/docker
-	7450f65072e5        2 weeks ago         /bin/sh -c #(nop) VOLUME /var/lib/docker
-	b79d62b97328        2 weeks ago         /bin/sh -c apt-get install -y -q lxc
-	36714852a550        2 weeks ago         /bin/sh -c apt-get install -y -q iptables
-	8c4c706df1d6        2 weeks ago         /bin/sh -c /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEYn' > /.s3cfg
-	b89989433c48        2 weeks ago         /bin/sh -c pip install python-magic
-	a23e640d85b5        2 weeks ago         /bin/sh -c pip install s3cmd
-	41f54fec7e79        2 weeks ago         /bin/sh -c apt-get install -y -q python-pip
-	d9bc04add907        2 weeks ago         /bin/sh -c apt-get install -y -q reprepro dpkg-sig
-	e74f4760fa70        2 weeks ago         /bin/sh -c gem install --no-rdoc --no-ri fpm
-	1e43224726eb        2 weeks ago         /bin/sh -c apt-get install -y -q ruby1.9.3 rubygems libffi-dev
-	460953ae9d7f        2 weeks ago         /bin/sh -c #(nop) ENV GOPATH=/go:/go/src/github.com/dotcloud/docker/vendor
-	8b63eb1d666b        2 weeks ago         /bin/sh -c #(nop) ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/goroot/bin
-	3087f3bcedf2        2 weeks ago         /bin/sh -c #(nop) ENV GOROOT=/goroot
-	635840d198e5        2 weeks ago         /bin/sh -c cd /goroot/src && ./make.bash
-	439f4a0592ba        2 weeks ago         /bin/sh -c curl -s https://go.googlecode.com/files/go1.1.2.src.tar.gz | tar -v -C / -xz && mv /go /goroot
-	13967ed36e93        2 weeks ago         /bin/sh -c #(nop) ENV CGO_ENABLED=0
-	bf7424458437        2 weeks ago         /bin/sh -c apt-get install -y -q build-essential
-	a89ec997c3bf        2 weeks ago         /bin/sh -c apt-get install -y -q mercurial
-	b9f165c6e749        2 weeks ago         /bin/sh -c apt-get install -y -q git
-	17a64374afa7        2 weeks ago         /bin/sh -c apt-get install -y -q curl
-	d5e85dc5b1d8        2 weeks ago         /bin/sh -c apt-get update
-	13e642467c11        2 weeks ago         /bin/sh -c echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
-	ae6dde92a94e        2 weeks ago         /bin/sh -c #(nop) MAINTAINER Solomon Hykes <solomon@dotcloud.com>
-	ubuntu:12.04        6 months ago 
-
 .. _cli_images:
 
 ``images``
@@ -430,52 +334,18 @@ To see how the docker:latest image was built:
 
     List images
 
-      -a=false: show all images (by default filter out the intermediate images used to build)
+      -a=false: show all images
       -notrunc=false: Don't truncate output
       -q=false: only show numeric IDs
       -tree=false: output graph in tree format
       -viz=false: output graph in graphviz format
-      
-Listing the most recently created images
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code-block:: bash
-
-	$ sudo docker images | head
-	REPOSITORY                    TAG                 IMAGE ID            CREATED             SIZE
-	<none>                        <none>              77af4d6b9913        19 hours ago        30.53 MB (virtual 1.089 GB)
-	committest                    latest              b6fa739cedf5        19 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              78a85c484f71        19 hours ago        30.53 MB (virtual 1.089 GB)
-	docker                        latest              30557a29d5ab        20 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              0124422dd9f9        20 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              18ad6fad3402        22 hours ago        23.68 MB (virtual 1.082 GB)
-	<none>                        <none>              f9f1e26352f0        23 hours ago        30.46 MB (virtual 1.089 GB)
-	tryout                        latest              2629d1fa0b81        23 hours ago        16.4 kB (virtual 131.5 MB)
-	<none>                        <none>              5ed6274db6ce        24 hours ago        30.44 MB (virtual 1.089 GB)
-
-Listing the full length image IDs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. code-block:: bash
-
-	$ sudo docker images -notrunc | head
-	REPOSITORY                    TAG                 IMAGE ID                                                           CREATED             SIZE
-	<none>                        <none>              77af4d6b9913e693e8d0b4b294fa62ade6054e6b2f1ffb617ac955dd63fb0182   19 hours ago        30.53 MB (virtual 1.089 GB)
-	committest                    latest              b6fa739cedf5ea12a620a439402b6004d057da800f91c7524b5086a5e4749c9f   19 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              78a85c484f71509adeaace20e72e941f6bdd2b25b4c75da8693efd9f61a37921   19 hours ago        30.53 MB (virtual 1.089 GB)
-	docker                        latest              30557a29d5abc51e5f1d5b472e79b7e296f595abcf19fe6b9199dbbc809c6ff4   20 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              0124422dd9f9cf7ef15c0617cda3931ee68346455441d66ab8bdc5b05e9fdce5   20 hours ago        30.53 MB (virtual 1.089 GB)
-	<none>                        <none>              18ad6fad340262ac2a636efd98a6d1f0ea775ae3d45240d3418466495a19a81b   22 hours ago        23.68 MB (virtual 1.082 GB)
-	<none>                        <none>              f9f1e26352f0a3ba6a0ff68167559f64f3e21ff7ada60366e2d44a04befd1d3a   23 hours ago        30.46 MB (virtual 1.089 GB)
-	tryout                        latest              2629d1fa0b81b222fca63371ca16cbf6a0772d07759ff80e8d1369b926940074   23 hours ago        16.4 kB (virtual 131.5 MB)
-	<none>                        <none>              5ed6274db6ceb2397844896966ea239290555e74ef307030ebb01ff91b1914df   24 hours ago        30.44 MB (virtual 1.089 GB)
 
 Displaying images visually
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: bash
+::
 
-    $ sudo docker images -viz | dot -Tpng -o docker.png
+    sudo docker images -viz | dot -Tpng -o docker.png
 
 .. image:: docker_images.gif
    :alt: Example inheritance graph of Docker images.
@@ -484,9 +354,9 @@ Displaying images visually
 Displaying image hierarchy
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. code-block:: bash
+::
 
-    $ sudo docker images -tree
+    sudo docker images -tree
 
     |─8dbd9e392a96 Size: 131.5 MB (virtual 131.5 MB) Tags: ubuntu:12.04,ubuntu:latest,ubuntu:precise
     └─27cf78414709 Size: 180.1 MB (virtual 180.1 MB)
@@ -565,21 +435,6 @@ might not get preserved.
 
     Display system-wide information.
 
-.. code-block:: bash
-
-	$ sudo docker info
-	Containers: 292
-	Images: 194
-	Debug mode (server): false
-	Debug mode (client): false
-	Fds: 22
-	Goroutines: 67
-	LXC Version: 0.9.0
-	EventsListeners: 115
-	Kernel Version: 3.8.0-33-generic
-	WARNING: No swap limit support
-
-
 .. _cli_insert:
 
 ``insert``
@@ -631,18 +486,6 @@ Known Issues (kill)
 * :issue:`197` indicates that ``docker kill`` may leave directories
   behind and make it difficult to remove the container.
 
-.. _cli_load:
-
-``load``
---------
-
-::
-
-    Usage: docker load < repository.tar
-
-    Loads a tarred repository from the standard input stream.
-    Restores both images and tags.
-
 .. _cli_login:
 
 ``login``
@@ -929,24 +772,6 @@ id may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in
 read-only or read-write mode, respectively. By default, the volumes are mounted
 in the same mode (rw or ro) as the reference container.
 
-Known Issues (run -volumes-from)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* :issue:`2702`: "lxc-start: Permission denied - failed to mount"
-  could indicate a permissions problem with AppArmor. Please see the
-  issue for a workaround.
-
-.. _cli_save:
-
-``save``
-
-::
-
-    Usage: docker save image > repository.tar
-
-    Streams a tarred repository to the standard output stream.
-    Contains all parent layers, and all tags + versions.
-
 .. _cli_search:
 
 ``search``

docs/sources/conf.py

@@ -235,7 +235,7 @@ latex_documents = [
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('commandline/cli', 'docker', u'Docker Documentation',
+    ('toctree', 'docker', u'Docker Documentation',
      [u'Team Docker'], 1)
 ]
 

docs/sources/examples/couchdb_data_volumes.rst

@@ -20,7 +20,7 @@ Note that we're marking ``/var/lib/couchdb`` as a data volume.
 
 .. code-block:: bash
 
-    COUCH1=$(sudo docker run -d -p 5984 -v /var/lib/couchdb shykes/couchdb:2013-05-03)
+    COUCH1=$(sudo docker run -d -v /var/lib/couchdb shykes/couchdb:2013-05-03)
 
 Add data to the first database
 ------------------------------
@@ -31,7 +31,7 @@ replace ``localhost`` with the public IP of your Docker host.
 .. code-block:: bash
 
     HOST=localhost
-    URL="http://$HOST:$(sudo docker port $COUCH1 5984 | grep -Po '\d+$')/_utils/"
+    URL="http://$HOST:$(sudo docker port $COUCH1 5984)/_utils/"
     echo "Navigate to $URL in your browser, and use the couch interface to add data"
 
 Create second database
@@ -41,7 +41,7 @@ This time, we're requesting shared access to ``$COUCH1``'s volumes.
 
 .. code-block:: bash
 
-    COUCH2=$(sudo docker run -d -p 5984 -volumes-from $COUCH1 shykes/couchdb:2013-05-03)
+    COUCH2=$(sudo docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03)
 
 Browse data on the second database
 ----------------------------------
@@ -49,7 +49,7 @@ Browse data on the second database
 .. code-block:: bash
 
     HOST=localhost
-    URL="http://$HOST:$(sudo docker port $COUCH2 5984 | grep -Po '\d+$')/_utils/"
+    URL="http://$HOST:$(sudo docker port $COUCH2 5984)/_utils/"
     echo "Navigate to $URL in your browser. You should see the same data as in the first database"'!'
 
 Congratulations, you are now running two Couchdb containers, completely

docs/sources/examples/example_header.inc

@@ -1,7 +1,4 @@
 
 .. note::
     
-    * This example assumes you have Docker running in daemon mode. For
-      more information please see :ref:`running_examples`.
-    * **If you don't like sudo** then see :ref:`dockergroup`
-
+    This example assumes you have Docker running in daemon mode. For more information please see :ref:`running_examples`

docs/sources/examples/hello_world.rst

@@ -127,12 +127,10 @@ Check the logs make sure it is working correctly.
 
     sudo docker attach $CONTAINER_ID
 
-Attach to the container to see the results in real-time.
+Attach to the container to see the results in realtime.
 
 - **"docker attach**" This will allow us to attach to a background
   process to see what is going on.
-- **"-sig-proxy=true"** Proxify all received signal to the process
-  (even in non-tty mode)
 - **$CONTAINER_ID** The Id of the container we want to attach too.
 
 Exit from the container attachment by pressing Control-C.

docs/sources/examples/python_web_app.rst

@@ -39,12 +39,11 @@ container. The ``BUILD_JOB`` environment variable will be set with the new conta
 
 .. code-block:: bash
 
-    sudo docker attach -sig-proxy=false $BUILD_JOB
+    sudo docker attach $BUILD_JOB
     [...]
 
 While this container is running, we can attach to the new container to
-see what is going on. The flag ``-sig-proxy`` set as ``false`` allows you to connect and
-disconnect (Ctrl-C) to it without stopping the container.
+see what is going on. You can use Ctrl-C to disconnect.
 
 .. code-block:: bash
 

docs/sources/examples/running_redis_service.rst

@@ -29,7 +29,7 @@ with your own user name.
 
 .. code-block:: bash
 
-    sudo docker build -t <your username>/redis .
+    sudo docker build -t <your username>/redis
 
 Run the service
 ---------------
@@ -82,7 +82,7 @@ of our ``redis`` container.
     DB_PORT_6379_TCP_ADDR=172.17.0.33
     DB_PORT_6379_TCP_PROTO=tcp
 
-We can see that we've got a small list of environment variables prefixed with ``DB``.
+We can see that we've got a small list of environmental varaibles prefixed with ``DB``. 
 The ``DB`` comes from the link alias specified when we launched the container. Let's use 
 the ``DB_PORT_6379_TCP_ADDR`` variable to connect to our Redis container.
 

docs/sources/examples/running_ssh_service.rst

@@ -12,9 +12,9 @@ SSH Daemon Service
 
 **Video:**
 
-I've created a little screencast to show how to create an SSHd service
+I've create a little screencast to show how to create a SSHd service
 and connect to it. It is something like 11 minutes and not entirely
-smooth, but it gives you a good idea.
+smooth, but gives you a good idea.
 
 .. note::
    This screencast was created before Docker version 0.5.2, so the

docs/sources/installation/amazon.rst

@@ -22,10 +22,12 @@ Amazon QuickStart
 
 1. **Choose an image:**
 
-   * Launch the `Create Instance Wizard` <https://console.aws.amazon.com/ec2/v2/home?#LaunchInstanceWizard:> menu on your AWS Console
-   * Select "Community AMIs" option and serch for ``amd64 precise`` (click enter to search)
-   * If you choose a EBS enabled AMI you will be able to launch a `t1.micro` instance (more info on `pricing` <http://aws.amazon.com/en/ec2/pricing/> )
-   * When you click select you'll be taken to the instance setup, and you're one click away from having your Ubuntu VM up and running.
+   * Open http://cloud-images.ubuntu.com/locator/ec2/
+   * Enter ``amd64 precise`` in the search field (it will search as you
+     type)
+   * Pick an image by clicking on the image name. *An EBS-enabled
+     image will let you use a t1.micro instance.* Clicking on the image 
+     name will take you to your AWS Console.
 
 2. **Tell CloudInit to install Docker:**
 

docs/sources/installation/index.rst

@@ -19,12 +19,11 @@ Contents:
 
    ubuntulinux
    binaries
-   security
-   upgrading
-   kernel
    vagrant
    windows
    amazon
    rackspace
    archlinux
    gentoolinux
+   upgrading
+   kernel

docs/sources/installation/kernel.rst

@@ -25,7 +25,6 @@ If you cannot or do not want to use the "official" kernels,
 here is some technical background about the features (both optional and
 mandatory) that docker needs to run successfully.
 
-
 Linux version 3.8 or above
 --------------------------
 
@@ -40,15 +39,6 @@ The symptoms include:
 - kernel crash causing the machine to freeze for a few minutes, or even
   completely.
 
-Additionally, kernels prior 3.4 did not implement ``reboot_pid_ns``,
-which means that the ``reboot()`` syscall could reboot the host machine,
-instead of terminating the container. To work around that problem,
-LXC userland tools (since version 0.8) automatically drop the ``SYS_BOOT``
-capability when necessary. Still, if you run a pre-3.4 kernel with pre-0.8
-LXC tools, be aware that containers can reboot the whole host! This is
-not something that Docker wants to address in the short term, since you
-shouldn't use kernels prior 3.8 with Docker anyway.
-
 While it is still possible to use older kernels for development, it is
 really not advised to do so.
 

docs/sources/installation/security.rst

@@ -1,267 +0,0 @@
-:title: Docker Security
-:description: Review of the Docker Daemon attack surface
-:keywords: Docker, Docker documentation, security
-
-.. _dockersecurity:
-
-Docker Security
-===============
-
-  *Adapted from* `Containers & Docker: How Secure are They? <blogsecurity>`_
-
-There are three major areas to consider when reviewing Docker security:
-
-* the intrinsic security of containers, as implemented by kernel
-  namespaces and cgroups;
-* the attack surface of the Docker daemon itself;
-* the "hardening" security features of the kernel and how they
-  interact with containers.
-
-Kernel Namespaces
------------------
-
-Docker containers are essentially LXC containers, and they come with
-the same security features. When you start a container with ``docker
-run``, behind the scenes Docker uses ``lxc-start`` to execute the
-Docker container. This creates a set of namespaces and control groups
-for the container. Those namespaces and control groups are not created
-by Docker itself, but by ``lxc-start``. This means that as the LXC
-userland tools evolve (and provide additional namespaces and isolation
-features), Docker will automatically make use of them.
-
-**Namespaces provide the first and most straightforward form of
-isolation**: processes running within a container cannot see, and even
-less affect, processes running in another container, or in the host
-system.
-
-**Each container also gets its own network stack**, meaning that a
-container doesn’t get a privileged access to the sockets or interfaces
-of another container. Of course, if the host system is setup
-accordingly, containers can interact with each other through their
-respective network interfaces — just like they can interact with
-external hosts. When you specify public ports for your containers or
-use :ref:`links <working_with_links_names>` then IP traffic is allowed
-between containers. They can ping each other, send/receive UDP
-packets, and establish TCP connections, but that can be restricted if
-necessary. From a network architecture point of view, all containers
-on a given Docker host are sitting on bridge interfaces. This means
-that they are just like physical machines connected through a common
-Ethernet switch; no more, no less.
-
-How mature is the code providing kernel namespaces and private
-networking?  Kernel namespaces were introduced `between kernel version
-2.6.15 and 2.6.26
-<http://lxc.sourceforge.net/index.php/about/kernel-namespaces/>`_. This
-means that since July 2008 (date of the 2.6.26 release, now 5 years
-ago), namespace code has been exercised and scrutinized on a large
-number of production systems. And there is more: the design and
-inspiration for the namespaces code are even older. Namespaces are
-actually an effort to reimplement the features of `OpenVZ
-<http://en.wikipedia.org/wiki/OpenVZ>`_ in such a way that they could
-be merged within the mainstream kernel. And OpenVZ was initially
-released in 2005, so both the design and the implementation are
-pretty mature.
-
-Control Groups
---------------
-
-Control Groups are the other key component of Linux Containers. They
-implement resource accounting and limiting. They provide a lot of very
-useful metrics, but they also help to ensure that each container gets
-its fair share of memory, CPU, disk I/O; and, more importantly, that a
-single container cannot bring the system down by exhausting one of
-those resources.
-
-So while they do not play a role in preventing one container from
-accessing or affecting the data and processes of another container,
-they are essential to fend off some denial-of-service attacks. They
-are particularly important on multi-tenant platforms, like public and
-private PaaS, to guarantee a consistent uptime (and performance) even
-when some applications start to misbehave.
-
-Control Groups have been around for a while as well: the code was
-started in 2006, and initially merged in kernel 2.6.24.
-
-Docker Daemon Attack Surface
-----------------------------
-
-Running containers (and applications) with Docker implies running the
-Docker daemon. This daemon currently requires root privileges, and you
-should therefore be aware of some important details.
-
-First of all, **only trusted users should be allowed to control your
-Docker daemon**. This is a direct consequence of some powerful Docker
-features. Specifically, Docker allows you to share a directory between
-the Docker host and a guest container; and it allows you to do so
-without limiting the access rights of the container. This means that
-you can start a container where the ``/host`` directory will be the
-``/`` directory on your host; and the container will be able to alter
-your host filesystem without any restriction. This sounds crazy? Well,
-you have to know that **all virtualization systems allowing filesystem
-resource sharing behave the same way**. Nothing prevents you from
-sharing your root filesystem (or even your root block device) with a
-virtual machine.
-
-This has a strong security implication: if you instrument Docker from
-e.g. a web server to provision containers through an API, you should
-be even more careful than usual with parameter checking, to make sure
-that a malicious user cannot pass crafted parameters causing Docker to
-create arbitrary containers.
-
-For this reason, the REST API endpoint (used by the Docker CLI to
-communicate with the Docker daemon) changed in Docker 0.5.2, and now
-uses a UNIX socket instead of a TCP socket bound on 127.0.0.1 (the
-latter being prone to cross-site-scripting attacks if you happen to
-run Docker directly on your local machine, outside of a VM). You can
-then use traditional UNIX permission checks to limit access to the
-control socket.
-
-You can also expose the REST API over HTTP if you explicitly decide
-so. However, if you do that, being aware of the abovementioned
-security implication, you should ensure that it will be reachable
-only from a trusted network or VPN; or protected with e.g. ``stunnel``
-and client SSL certificates.
-
-Recent improvements in Linux namespaces will soon allow to run
-full-featured containers without root privileges, thanks to the new
-user namespace. This is covered in detail `here
-<http://s3hh.wordpress.com/2013/07/19/creating-and-using-containers-without-privilege/>`_. Moreover,
-this will solve the problem caused by sharing filesystems between host
-and guest, since the user namespace allows users within containers
-(including the root user) to be mapped to other users in the host
-system.
-
-The end goal for Docker is therefore to implement two additional
-security improvements:
-
-* map the root user of a container to a non-root user of the Docker
-  host, to mitigate the effects of a container-to-host privilege
-  escalation;
-* allow the Docker daemon to run without root privileges, and delegate
-  operations requiring those privileges to well-audited sub-processes,
-  each with its own (very limited) scope: virtual network setup,
-  filesystem management, etc.
-
-Finally, if you run Docker on a server, it is recommended to run
-exclusively Docker in the server, and move all other services within
-containers controlled by Docker. Of course, it is fine to keep your
-favorite admin tools (probably at least an SSH server), as well as
-existing monitoring/supervision processes (e.g. NRPE, collectd, etc).
-
-Linux Kernel Capabilities
--------------------------
-
-By default, Docker starts containers with a very restricted set of
-capabilities. What does that mean?
-
-Capabilities turn the binary "root/non-root" dichotomy into a
-fine-grained access control system. Processes (like web servers) that
-just need to bind on a port below 1024 do not have to run as root:
-they can just be granted the ``net_bind_service`` capability
-instead. And there are many other capabilities, for almost all the
-specific areas where root privileges are usually needed.
-
-This means a lot for container security; let’s see why!
-
-Your average server (bare metal or virtual machine) needs to run a
-bunch of processes as root. Those typically include SSH, cron,
-syslogd; hardware management tools (to e.g. load modules), network
-configuration tools (to handle e.g. DHCP, WPA, or VPNs), and much
-more. A container is very different, because almost all of those tasks
-are handled by the infrastructure around the container:
-
-* SSH access will typically be managed by a single server running in
-  the Docker host;
-* ``cron``, when necessary, should run as a user process, dedicated
-  and tailored for the app that needs its scheduling service, rather
-  than as a platform-wide facility;
-* log management will also typically be handed to Docker, or by
-  third-party services like Loggly or Splunk;
-* hardware management is irrelevant, meaning that you never need to
-  run ``udevd`` or equivalent daemons within containers;
-* network management happens outside of the containers, enforcing
-  separation of concerns as much as possible, meaning that a container
-  should never need to perform ``ifconfig``, ``route``, or ip commands
-  (except when a container is specifically engineered to behave like a
-  router or firewall, of course).
-
-This means that in most cases, containers will not need "real" root
-privileges *at all*. And therefore, containers can run with a reduced
-capability set; meaning that "root" within a container has much less
-privileges than the real "root". For instance, it is possible to:
-
-* deny all "mount" operations;
-* deny access to raw sockets (to prevent packet spoofing);
-* deny access to some filesystem operations, like creating new device
-  nodes, changing the owner of files, or altering attributes
-  (including the immutable flag);
-* deny module loading;
-* and many others.
-
-This means that even if an intruder manages to escalate to root within
-a container, it will be much harder to do serious damage, or to
-escalate to the host.
-
-This won't affect regular web apps; but malicious users will find that
-the arsenal at their disposal has shrunk considerably! You can see
-`the list of dropped capabilities in the Docker code
-<https://github.com/dotcloud/docker/blob/v0.5.0/lxc_template.go#L97>`_,
-and a full list of available capabilities in `Linux manpages
-<http://man7.org/linux/man-pages/man7/capabilities.7.html>`_.
-
-Of course, you can always enable extra capabilities if you really need
-them (for instance, if you want to use a FUSE-based filesystem), but
-by default, Docker containers will be locked down to ensure maximum
-safety.
-
-Other Kernel Security Features
-------------------------------
-
-Capabilities are just one of the many security features provided by
-modern Linux kernels. It is also possible to leverage existing,
-well-known systems like TOMOYO, AppArmor, SELinux, GRSEC, etc. with
-Docker.
-
-While Docker currently only enables capabilities, it doesn't interfere
-with the other systems. This means that there are many different ways
-to harden a Docker host. Here are a few examples.
-
-* You can run a kernel with GRSEC and PAX. This will add many safety
-  checks, both at compile-time and run-time; it will also defeat many
-  exploits, thanks to techniques like address randomization. It
-  doesn’t require Docker-specific configuration, since those security
-  features apply system-wide, independently of containers.
-* If your distribution comes with security model templates for LXC
-  containers, you can use them out of the box. For instance, Ubuntu
-  comes with AppArmor templates for LXC, and those templates provide
-  an extra safety net (even though it overlaps greatly with
-  capabilities).
-* You can define your own policies using your favorite access control
-  mechanism. Since Docker containers are standard LXC containers,
-  there is nothing “magic” or specific to Docker.
-
-Just like there are many third-party tools to augment Docker
-containers with e.g. special network topologies or shared filesystems,
-you can expect to see tools to harden existing Docker containers
-without affecting Docker’s core.
-
-Conclusions
------------
-
-Docker containers are, by default, quite secure; especially if you
-take care of running your processes inside the containers as
-non-privileged users (i.e. non root).
-
-You can add an extra layer of safety by enabling Apparmor, SELinux,
-GRSEC, or your favorite hardening solution.
-
-Last but not least, if you see interesting security features in other
-containerization systems, you will be able to implement them as well
-with Docker, since everything is provided by the kernel anyway.
-
-For more context and especially for comparisons with VMs and other
-container systems, please also see the `original blog post
-<blogsecurity>`_.
-
-.. _blogsecurity: http://blog.docker.io/2013/08/containers-docker-how-secure-are-they/
-

docs/sources/use/basics.rst

@@ -34,11 +34,13 @@ This will find the ``ubuntu`` image by name in the :ref:`Central Index
 <searching_central_index>` and download it from the top-level Central 
 Repository to a local image cache.
 
-.. NOTE:: When the image has successfully downloaded, you will see a
-   12 character hash ``539c0211cd76: Download complete`` which is the
-   short form of the image ID. These short image IDs are the first 12
-   characters of the full image ID - which can be found using ``docker
-   inspect`` or ``docker images -notrunc=true``
+.. NOTE:: When the image has successfully downloaded, you will see a 12 
+character hash ``539c0211cd76: Download complete`` which is the short 
+form of the image ID. These short image IDs are the first 12 characters 
+of the full image ID - which can be found using ``docker inspect`` or 
+``docker images -notrunc=true``
+
+.. _dockergroup:
 
 Running an interactive shell
 ----------------------------
@@ -51,38 +53,33 @@ Running an interactive shell
   # use the escape sequence Ctrl-p + Ctrl-q
   sudo docker run -i -t ubuntu /bin/bash
 
-.. _dockergroup:
 
-sudo and the docker Group
--------------------------
+Why ``sudo``?
+-------------
 
 The ``docker`` daemon always runs as root, and since ``docker``
 version 0.5.2, ``docker`` binds to a Unix socket instead of a TCP
 port. By default that Unix socket is owned by the user *root*, and so,
 by default, you can access it with ``sudo``.
 
-Starting in version 0.5.3, if you (or your Docker installer) create a
-Unix group called *docker* and add users to it, then the ``docker``
-daemon will make the ownership of the Unix socket read/writable by the
-*docker* group when the daemon starts. The ``docker`` daemon must
-always run as root, but if you run the ``docker`` client as a user in
-the *docker* group then you don't need to add ``sudo`` to all the
-client commands.
-
-**Example:**
+Starting in version 0.5.3, if you create a Unix group called *docker*
+and add users to it, then the ``docker`` daemon will make the
+ownership of the Unix socket read/writable by the *docker* group when
+the daemon starts. The ``docker`` daemon must always run as root, but
+if you run the ``docker`` client as a user in the *docker* group then
+you don't need to add ``sudo`` to all the client commands.
 
 .. code-block:: bash
 
-  # Add the docker group if it doesn't already exist.
+  # Add the docker group
   sudo groupadd docker
 
-  # Add the user "ubuntu" to the docker group.
-  # Change the user name to match your preferred user.
+  # Add the ubuntu user to the docker group
   # You may have to logout and log back in again for
-  # this to take effect.
+  # this to take effect
   sudo gpasswd -a ubuntu docker
 
-  # Restart the docker daemon.
+  # Restart the docker daemon
   sudo service docker restart
 
 .. _bind_docker:
@@ -90,7 +87,7 @@ client commands.
 Bind Docker to another host/port or a Unix socket
 -------------------------------------------------
 
-.. warning:: Changing the default ``docker`` daemon binding to a TCP
+.. DANGER:: Changing the default ``docker`` daemon binding to a TCP
    port or Unix *docker* user group will increase your security risks
    by allowing non-root users to potentially gain *root* access on the
    host (`e.g. #1369

docs/sources/use/builder.rst

@@ -15,39 +15,27 @@ commit them along the way, giving you a final image.
 
 .. contents:: Table of Contents
 
-.. _dockerfile_usage:
-
 1. Usage
 ========
 
-To :ref:`build <cli_build>` an image from a source repository, create
-a description file called ``Dockerfile`` at the root of your
-repository. This file will describe the steps to assemble the image.
+To build an image from a source repository, create a description file
+called ``Dockerfile`` at the root of your repository. This file will
+describe the steps to assemble the image.
 
 Then call ``docker build`` with the path of your source repository as
-argument (for example, ``.``):
+argument:
 
     ``sudo docker build .``
 
-The path to the source repository defines where to find the *context*
-of the build. The build is run by the Docker daemon, not by the CLI,
-so the whole context must be transferred to the daemon. The Docker CLI
-reports "Uploading context" when the context is sent to the daemon.
-
 You can specify a repository and tag at which to save the new image if the
 build succeeds:
 
     ``sudo docker build -t shykes/myapp .``
 
-The Docker daemon will run your steps one-by-one, committing the
-result if necessary, before finally outputting the ID of your new
-image. The Docker daemon will automatically clean up the context you
-sent.
+Docker will run your steps one-by-one, committing the result if necessary,
+before finally outputting the ID of your new image.
 
-When you're done with your build, you're ready to look into
-:ref:`image_push`.
-
-.. _dockerfile_format:
+When you're done with your build, you're ready to look into :ref:`image_push`.
 
 2. Format
 =========
@@ -75,16 +63,12 @@ allows statements like:
     # Comment
     RUN echo 'we are running some # of cool things'
 
-.. _dockerfile_instructions:
-
 3. Instructions
 ===============
 
 Here is the set of instructions you can use in a ``Dockerfile`` for
 building images.
 
-.. _dockerfile_from:
-
 3.1 FROM
 --------
 
@@ -110,8 +94,6 @@ output by the commit before each new ``FROM`` command.
 If no ``tag`` is given to the ``FROM`` instruction, ``latest`` is
 assumed. If the used tag does not exist, an error will be returned.
 
-.. _dockerfile_maintainer:
-
 3.2 MAINTAINER
 --------------
 
@@ -120,8 +102,6 @@ assumed. If the used tag does not exist, an error will be returned.
 The ``MAINTAINER`` instruction allows you to set the *Author* field of
 the generated images.
 
-.. _dockerfile_run:
-
 3.3 RUN
 -------
 
@@ -144,7 +124,7 @@ Known Issues (RUN)
   ``rm`` a file, for example. The issue describes a workaround.
 * :issue:`2424` Locale will not be set automatically.
 
-.. _dockerfile_cmd:
+
 
 3.4 CMD
 -------
@@ -189,7 +169,7 @@ array:
 
 If you would like your container to run the same executable every
 time, then you should consider using ``ENTRYPOINT`` in combination
-with ``CMD``. See :ref:`dockerfile_entrypoint`.
+with ``CMD``. See :ref:`entrypoint_def`.
 
 If the user specifies arguments to ``docker run`` then they will
 override the default specified in CMD.
@@ -199,8 +179,6 @@ override the default specified in CMD.
     command and commits the result; ``CMD`` does not execute anything at
     build time, but specifies the intended command for the image.
 
-.. _dockerfile_expose:
-
 3.5 EXPOSE
 ----------
 
@@ -211,8 +189,6 @@ functionally equivalent to running ``docker commit -run '{"PortSpecs":
 ["<port>", "<port2>"]}'`` outside the builder. Refer to
 :ref:`port_redirection` for detailed information.
 
-.. _dockerfile_env:
-
 3.6 ENV
 -------
 
@@ -227,8 +203,6 @@ with ``<key>=<value>``
     The environment variables will persist when a container is run
     from the resulting image.
 
-.. _dockerfile_add:
-
 3.7 ADD
 -------
 
@@ -289,7 +263,7 @@ The copy obeys the following rules:
 * If ``<dest>`` doesn't exist, it is created along with all missing
   directories in its path.
 
-.. _dockerfile_entrypoint:
+.. _entrypoint_def:
 
 3.8 ENTRYPOINT
 --------------
@@ -338,18 +312,14 @@ this optional but default, you could use a CMD:
     CMD ["-l", "-"]
     ENTRYPOINT ["/usr/bin/wc"]
 
-.. _dockerfile_volume:
 
 3.9 VOLUME
 ----------
 
     ``VOLUME ["/data"]``
 
-The ``VOLUME`` instruction will create a mount point with the specified name and mark it 
-as holding externally mounted volumes from native host or other containers. For more information/examples 
-and mounting instructions via docker client, refer to :ref:`volume_def` documentation. 
-
-.. _dockerfile_user:
+The ``VOLUME`` instruction will add one or more new volumes to any
+container created from the image.
 
 3.10 USER
 ---------
@@ -359,8 +329,6 @@ and mounting instructions via docker client, refer to :ref:`volume_def` document
 The ``USER`` instruction sets the username or UID to use when running
 the image.
 
-.. _dockerfile_workdir:
-
 3.11 WORKDIR
 ------------
 
@@ -369,7 +337,6 @@ the image.
 The ``WORKDIR`` instruction sets the working directory in which
 the command given by ``CMD`` is executed.
 
-.. _dockerfile_examples:
 
 4. Dockerfile Examples
 ======================

docs/sources/use/host_integration.rst

@@ -29,32 +29,14 @@ Here are a few sample scripts for systemd and upstart to integrate with docker.
 Sample Upstart Script
 ---------------------
 
-In this example we've already created a container to run Redis with an id of
-0a7e070b698b.  To create an upstart script for our container, we create a file
-named ``/etc/init/redis.conf`` and place the following into it:
-
 .. code-block:: bash
 
    description "Redis container"
    author "Me"
-   start on filesystem and started docker
+   start on filesystem and started lxc-net and started docker
    stop on runlevel [!2345]
    respawn
-   script
-     # Wait for docker to finish starting up first.
-     FILE=/var/run/docker.sock
-     while [ ! -e $FILE ] ; do
-       inotifywait -t 2 -e create $(dirname $FILE)
-     done
-     /usr/bin/docker start -a 0a7e070b698b
-   end script
-
-Next, we have to configure docker so that it's run with the option ``-r=false``.
-Run the following command:
-
-.. code-block:: bash
-
-   $ sudo sh -c "echo 'DOCKER_OPTS=\"-r=false\"' > /etc/default/docker"
+   exec docker start -a 0a7e070b698b
 
 
 Sample systemd Script

docs/sources/use/working_with_volumes.rst

@@ -30,60 +30,44 @@ Each container can have zero or more data volumes.
 Getting Started
 ...............
 
-Using data volumes is as simple as adding a new flag: ``-v``. The
-parameter ``-v`` can be used more than once in order to create more
-volumes within the new container. The example below shows the
-instruction to create a container with two new volumes::
+
+
+Using data volumes is as simple as adding a new flag: ``-v``. The parameter ``-v`` can be used more than once in order to create more volumes within the new container. The example below shows the instruction to create a container with two new volumes::
 
   docker run -v /var/volume1 -v /var/volume2 shykes/couchdb
 
-For a Dockerfile, the VOLUME instruction will add one or more new
-volumes to any container created from the image::
+For a Dockerfile, the VOLUME instruction will add one or more new volumes to any container created from the image::
 
   VOLUME ["/var/volume1", "/var/volume2"]
 
 
-Mount Volumes from an Existing Container:
------------------------------------------
+Create a new container using existing volumes from an existing container:
+---------------------------------------------------------------------------
 
-The command below creates a new container which is runnning as daemon
-``-d`` and with one volume ``/var/lib/couchdb``::
+
+The command below creates a new container which is runnning as daemon ``-d`` and with one volume ``/var/lib/couchdb``::
 
   COUCH1=$(sudo docker run -d -v /var/lib/couchdb shykes/couchdb:2013-05-03)
 
-From the container id of that previous container ``$COUCH1`` it's
-possible to create new container sharing the same volume using the
-parameter ``-volumes-from container_id``::
+From the container id of that previous container ``$COUCH1`` it's possible to create new container sharing the same volume using the parameter ``-volumes-from container_id``::
 
   COUCH2=$(sudo docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03)
 
 Now, the second container has the all the information from the first volume.
 
 
-Mount a Host Directory as a Container Volume:
----------------------------------------------
-
-::
+Create a new container which mounts a host directory into it:
+-------------------------------------------------------------
 
   -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro].
   If "host-dir" is missing, then docker creates a new volume.
 
-This is not available for a Dockerfile due the portability and sharing
-purpose of it. The [host-dir] volumes is something 100% host dependent
-and will break on any other machine.
+  This is not available for a Dockerfile due the portability and sharing purpose of it. The [host-dir] volumes is something    100% host dependent and will break on any other machine.
 
 For example::
 
   sudo docker run -v /var/logs:/var/host_logs:ro shykes/couchdb:2013-05-03
 
-The command above mounts the host directory ``/var/logs`` into the
-container with read only permissions as ``/var/host_logs``.
+The command above mounts the host directory ``/var/logs`` into the container with read only permissions as ``/var/host_logs``.
 
 .. versionadded:: v0.5.0
-
-Known Issues
-............
-
-* :issue:`2702`: "lxc-start: Permission denied - failed to mount"
-  could indicate a permissions problem with AppArmor. Please see the
-  issue for a workaround.

docs/sources/use/workingwithrepository.rst

@@ -177,15 +177,6 @@ you can push and pull it like any other repository, but it will
 there will be no user name checking performed. Your registry will
 function completely independently from the Central Index.
 
-.. raw:: html
-
-   <iframe width="640" height="360"
-   src="//www.youtube.com/embed/CAewZCBT4PI?rel=0" frameborder="0"
-   allowfullscreen></iframe>
-
-.. seealso:: `Docker Blog: How to use your own registry 
-   <http://blog.docker.io/2013/07/how-to-use-your-own-registry/>`_
-
 Authentication file
 -------------------
 

engine/engine_test.go

@@ -1,55 +0,0 @@
-package engine
-
-import (
-	"testing"
-)
-
-func TestRegister(t *testing.T) {
-	if err := Register("dummy1", nil); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := Register("dummy1", nil); err == nil {
-		t.Fatalf("Expecting error, got none")
-	}
-
-	eng := newTestEngine(t)
-
-	//Should fail because globan handlers are copied
-	//at the engine creation
-	if err := eng.Register("dummy1", nil); err == nil {
-		t.Fatalf("Expecting error, got none")
-	}
-
-	if err := eng.Register("dummy2", nil); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := eng.Register("dummy2", nil); err == nil {
-		t.Fatalf("Expecting error, got none")
-	}
-}
-
-func TestJob(t *testing.T) {
-	eng := newTestEngine(t)
-	job1 := eng.Job("dummy1", "--level=awesome")
-
-	if job1.handler != nil {
-		t.Fatalf("job1.handler should be empty")
-	}
-
-	h := func(j *Job) string {
-		return j.Name
-	}
-
-	eng.Register("dummy2", h)
-	job2 := eng.Job("dummy2", "--level=awesome")
-
-	if job2.handler == nil {
-		t.Fatalf("job2.handler shouldn't be nil")
-	}
-
-	if job2.handler(job2) != job2.Name {
-		t.Fatalf("handler dummy2 was not found in job2")
-	}
-}

engine/env_test.go

@@ -23,101 +23,7 @@ func TestSetenv(t *testing.T) {
 	if val := job.Getenv("foo"); val != "bar" {
 		t.Fatalf("Getenv returns incorrect value: %s", val)
 	}
-
-	job.Setenv("bar", "")
-	if val := job.Getenv("bar"); val != "" {
-		t.Fatalf("Getenv returns incorrect value: %s", val)
-	}
 	if val := job.Getenv("nonexistent"); val != "" {
 		t.Fatalf("Getenv returns incorrect value: %s", val)
 	}
 }
-
-func TestSetenvBool(t *testing.T) {
-	job := mkJob(t, "dummy")
-	job.SetenvBool("foo", true)
-	if val := job.GetenvBool("foo"); !val {
-		t.Fatalf("GetenvBool returns incorrect value: %b", val)
-	}
-
-	job.SetenvBool("bar", false)
-	if val := job.GetenvBool("bar"); val {
-		t.Fatalf("GetenvBool returns incorrect value: %b", val)
-	}
-
-	if val := job.GetenvBool("nonexistent"); val {
-		t.Fatalf("GetenvBool returns incorrect value: %b", val)
-	}
-}
-
-func TestSetenvInt(t *testing.T) {
-	job := mkJob(t, "dummy")
-
-	job.SetenvInt("foo", -42)
-	if val := job.GetenvInt("foo"); val != -42 {
-		t.Fatalf("GetenvInt returns incorrect value: %d", val)
-	}
-
-	job.SetenvInt("bar", 42)
-	if val := job.GetenvInt("bar"); val != 42 {
-		t.Fatalf("GetenvInt returns incorrect value: %d", val)
-	}
-	if val := job.GetenvInt("nonexistent"); val != -1 {
-		t.Fatalf("GetenvInt returns incorrect value: %d", val)
-	}
-}
-
-func TestSetenvList(t *testing.T) {
-	job := mkJob(t, "dummy")
-
-	job.SetenvList("foo", []string{"bar"})
-	if val := job.GetenvList("foo"); len(val) != 1 || val[0] != "bar" {
-		t.Fatalf("GetenvList returns incorrect value: %v", val)
-	}
-
-	job.SetenvList("bar", nil)
-	if val := job.GetenvList("bar"); val != nil {
-		t.Fatalf("GetenvList returns incorrect value: %v", val)
-	}
-	if val := job.GetenvList("nonexistent"); val != nil {
-		t.Fatalf("GetenvList returns incorrect value: %v", val)
-	}
-}
-
-func TestImportEnv(t *testing.T) {
-	type dummy struct {
-		DummyInt         int
-		DummyStringArray []string
-	}
-
-	job := mkJob(t, "dummy")
-	if err := job.ImportEnv(&dummy{42, []string{"foo", "bar"}}); err != nil {
-		t.Fatal(err)
-	}
-
-	dmy := dummy{}
-	if err := job.ExportEnv(&dmy); err != nil {
-		t.Fatal(err)
-	}
-
-	if dmy.DummyInt != 42 {
-		t.Fatalf("Expected 42, got %d", dmy.DummyInt)
-	}
-
-	if len(dmy.DummyStringArray) != 2 || dmy.DummyStringArray[0] != "foo" || dmy.DummyStringArray[1] != "bar" {
-		t.Fatalf("Expected {foo, bar}, got %v", dmy.DummyStringArray)
-	}
-
-}
-
-func TestEnviron(t *testing.T) {
-	job := mkJob(t, "dummy")
-	job.Setenv("foo", "bar")
-	val, exists := job.Environ()["foo"]
-	if !exists {
-		t.Fatalf("foo not found in the environ")
-	}
-	if val != "bar" {
-		t.Fatalf("bar not found in the environ")
-	}
-}

engine/job.go

@@ -205,12 +205,8 @@ func (job *Job) SetenvInt(key string, value int64) {
 	job.Setenv(key, fmt.Sprintf("%d", value))
 }
 
-// Returns nil if key not found
 func (job *Job) GetenvList(key string) []string {
 	sval := job.Getenv(key)
-	if sval == "" {
-		return nil
-	}
 	l := make([]string, 0, 1)
 	if err := json.Unmarshal([]byte(sval), &l); err != nil {
 		l = append(l, sval)
@@ -218,7 +214,7 @@ func (job *Job) GetenvList(key string) []string {
 	return l
 }
 
-func (job *Job) SetenvJson(key string, value interface{}) error {
+func (job *Job) SetenvList(key string, value []string) error {
 	sval, err := json.Marshal(value)
 	if err != nil {
 		return err
@@ -227,10 +223,6 @@ func (job *Job) SetenvJson(key string, value interface{}) error {
 	return nil
 }
 
-func (job *Job) SetenvList(key string, value []string) error {
-	return job.SetenvJson(key, value)
-}
-
 func (job *Job) Setenv(key, value string) {
 	job.env = append(job.env, key+"="+value)
 }
@@ -238,7 +230,7 @@ func (job *Job) Setenv(key, value string) {
 // DecodeEnv decodes `src` as a json dictionary, and adds
 // each decoded key-value pair to the environment.
 //
-// If `src` cannot be decoded as a json dictionary, an error
+// If `text` cannot be decoded as a json dictionary, an error
 // is returned.
 func (job *Job) DecodeEnv(src io.Reader) error {
 	m := make(map[string]interface{})

engine/utils.go

@@ -11,6 +11,10 @@ import (
 
 var globalTestID string
 
+func init() {
+	Register("dummy", func(job *Job) string { return "" })
+}
+
 func newTestEngine(t *testing.T) *Engine {
 	// Use the caller function name as a prefix.
 	// This helps trace temp directories back to their test.

graph.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -17,11 +18,12 @@ import (
 type Graph struct {
 	Root    string
 	idIndex *utils.TruncIndex
+	driver  graphdriver.Driver
 }
 
 // NewGraph instantiates a new graph at the given root path in the filesystem.
 // `root` will be created if it doesn't exist.
-func NewGraph(root string) (*Graph, error) {
+func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
 	abspath, err := filepath.Abs(root)
 	if err != nil {
 		return nil, err
@@ -30,9 +32,11 @@ func NewGraph(root string) (*Graph, error) {
 	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
+
 	graph := &Graph{
 		Root:    abspath,
 		idIndex: utils.NewTruncIndex(),
+		driver:  driver,
 	}
 	if err := graph.restore(); err != nil {
 		return nil, err
@@ -78,16 +82,22 @@ func (graph *Graph) Get(name string) (*Image, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Check that the filesystem layer exists
+	rootfs, err := graph.driver.Get(img.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+	}
 	if img.ID != id {
 		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
 	}
 	img.graph = graph
 	if img.Size == 0 {
-		root, err := img.root()
+		size, err := utils.TreeSize(rootfs)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("Error computing size of rootfs %s: %s", img.ID, err)
 		}
-		if err := StoreSize(img, root); err != nil {
+		img.Size = size
+		if err := img.SaveSize(graph.imageRoot(id)); err != nil {
 			return nil, err
 		}
 	}
@@ -131,14 +141,24 @@ func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Im
 	if err != nil {
 		return fmt.Errorf("Mktemp failed: %s", err)
 	}
-	if err := StoreImage(img, jsonData, layerData, tmp); err != nil {
+
+	// Create root filesystem in the driver
+	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
+		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	// Mount the root filesystem so we can apply the diff/layer
+	rootfs, err := graph.driver.Get(img.ID)
+	if err != nil {
+		return fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	img.graph = graph
+	if err := StoreImage(img, jsonData, layerData, tmp, rootfs); err != nil {
 		return err
 	}
 	// Commit
 	if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
 		return err
 	}
-	img.graph = graph
 	graph.idIndex.Add(img.ID)
 	return nil
 }
@@ -152,7 +172,7 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression,
 	if err != nil {
 		return nil, err
 	}
-	tmp, err := graph.tmp()
+	tmp, err := graph.Mktemp("")
 	if err != nil {
 		return nil, err
 	}
@@ -160,42 +180,25 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression,
 	if err != nil {
 		return nil, err
 	}
-	return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root)
+	return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp)
 }
 
 // Mktemp creates a temporary sub-directory inside the graph's filesystem.
 func (graph *Graph) Mktemp(id string) (string, error) {
-	if id == "" {
-		id = GenerateID()
+	dir := path.Join(graph.Root, "_tmp", GenerateID())
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return "", err
 	}
-	tmp, err := graph.tmp()
-	if err != nil {
-		return "", fmt.Errorf("Couldn't create temp: %s", err)
-	}
-	if tmp.Exists(id) {
-		return "", fmt.Errorf("Image %s already exists", id)
-	}
-	return tmp.imageRoot(id), nil
+	return dir, nil
 }
 
-// getDockerInitLayer returns the path of a layer containing a mountpoint suitable
+// setupInitLayer populates a directory with mountpoints suitable
 // for bind-mounting dockerinit into the container. The mountpoint is simply an
 // empty file at /.dockerinit
 //
 // This extra layer is used by all containers as the top-most ro layer. It protects
 // the container from unwanted side-effects on the rw layer.
-func (graph *Graph) getDockerInitLayer() (string, error) {
-	tmp, err := graph.tmp()
-	if err != nil {
-		return "", err
-	}
-	initLayer := tmp.imageRoot("_dockerinit")
-	if err := os.Mkdir(initLayer, 0755); err != nil && !os.IsExist(err) {
-		// If directory already existed, keep going.
-		// For all other errors, abort.
-		return "", err
-	}
-
+func setupInitLayer(initLayer string) error {
 	for pth, typ := range map[string]string{
 		"/dev/pts":         "dir",
 		"/dev/shm":         "dir",
@@ -214,31 +217,27 @@ func (graph *Graph) getDockerInitLayer() (string, error) {
 				switch typ {
 				case "dir":
 					if err := os.MkdirAll(path.Join(initLayer, pth), 0755); err != nil {
-						return "", err
+						return err
 					}
 				case "file":
 					if err := os.MkdirAll(path.Join(initLayer, path.Dir(pth)), 0755); err != nil {
-						return "", err
+						return err
 					}
-					f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755)
-					if err != nil {
-						return "", err
+
+					if f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755); err != nil {
+						return err
+					} else {
+						f.Close()
 					}
-					f.Close()
 				}
 			} else {
-				return "", err
+				return err
 			}
 		}
 	}
 
 	// Layer is ready to use, if it wasn't before.
-	return initLayer, nil
-}
-
-func (graph *Graph) tmp() (*Graph, error) {
-	// Changed to _tmp from :tmp:, because it messed with ":" separators in aufs branch syntax...
-	return NewGraph(path.Join(graph.Root, "_tmp"))
+	return nil
 }
 
 // Check if given error is "not empty".
@@ -270,6 +269,9 @@ func (graph *Graph) Delete(name string) error {
 	if err != nil {
 		return err
 	}
+	// Remove rootfs data from the driver
+	graph.driver.Remove(id)
+	// Remove the trashed image directory
 	return os.RemoveAll(tmp)
 }
 

graph_test.go

@@ -5,10 +5,12 @@ import (
 	"bytes"
 	"errors"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"testing"
 	"time"
 )
@@ -120,6 +122,41 @@ func TestRegister(t *testing.T) {
 	}
 }
 
+func TestMount(t *testing.T) {
+	graph := tempGraph(t)
+	defer os.RemoveAll(graph.Root)
+	archive, err := fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	image, err := graph.Create(archive, nil, "Testing", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmp, err := ioutil.TempDir("", "docker-test-graph-mount-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	rootfs := path.Join(tmp, "rootfs")
+	if err := os.MkdirAll(rootfs, 0700); err != nil {
+		t.Fatal(err)
+	}
+	rw := path.Join(tmp, "rw")
+	if err := os.MkdirAll(rw, 0700); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := graph.driver.Get(image.ID); err != nil {
+		t.Fatal(err)
+	}
+	// FIXME: test for mount contents
+	defer func() {
+		if err := graph.driver.Cleanup(); err != nil {
+			t.Error(err)
+		}
+	}()
+}
+
 // Test that an image can be deleted by its shorthand prefix
 func TestDeletePrefix(t *testing.T) {
 	graph := tempGraph(t)
@@ -259,7 +296,11 @@ func tempGraph(t *testing.T) *Graph {
 	if err != nil {
 		t.Fatal(err)
 	}
-	graph, err := NewGraph(tmp)
+	backend, err := graphdriver.New(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph, err := NewGraph(tmp, backend)
 	if err != nil {
 		t.Fatal(err)
 	}

graphdriver/driver.go

@@ -0,0 +1,86 @@
+package graphdriver
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/utils"
+	"os"
+	"path"
+)
+
+type InitFunc func(root string) (Driver, error)
+
+type Driver interface {
+	Create(id, parent string) error
+	Remove(id string) error
+
+	Get(id string) (dir string, err error)
+	Size(id string) (bytes int64, err error)
+
+	Cleanup() error
+}
+
+type Differ interface {
+	Diff(id string) (archive.Archive, error)
+	Changes(id string) ([]archive.Change, error)
+	ApplyDiff(id string, diff archive.Archive) error
+}
+
+var (
+	// All registred drivers
+	drivers map[string]InitFunc
+	// Slice of drivers that should be used in an order
+	priority = []string{
+		"aufs",
+		"devicemapper",
+		"dummy",
+	}
+)
+
+func init() {
+	drivers = make(map[string]InitFunc)
+}
+
+func Register(name string, initFunc InitFunc) error {
+	if _, exists := drivers[name]; exists {
+		return fmt.Errorf("Name already registered %s", name)
+	}
+	drivers[name] = initFunc
+
+	return nil
+}
+
+func getDriver(name, home string) (Driver, error) {
+	if initFunc, exists := drivers[name]; exists {
+		return initFunc(path.Join(home, name))
+	}
+	return nil, fmt.Errorf("No such driver: %s", name)
+}
+
+func New(root string) (Driver, error) {
+	var driver Driver
+	var lastError error
+	// Use environment variable DOCKER_DRIVER to force a choice of driver
+	if name := os.Getenv("DOCKER_DRIVER"); name != "" {
+		return getDriver(name, root)
+	}
+	// Check for priority drivers first
+	for _, name := range priority {
+		driver, lastError = getDriver(name, root)
+		if lastError != nil {
+			utils.Debugf("Error loading driver %s: %s", name, lastError)
+			continue
+		}
+		return driver, nil
+	}
+
+	// Check all registered drivers if no priority driver is found
+	for _, initFunc := range drivers {
+		driver, lastError = initFunc(root)
+		if lastError != nil {
+			continue
+		}
+		return driver, nil
+	}
+	return nil, lastError
+}

graphdriver/dummy/driver.go

@@ -0,0 +1,78 @@
+package dummy
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
+	"os"
+	"path"
+)
+
+func init() {
+	graphdriver.Register("dummy", Init)
+}
+
+func Init(home string) (graphdriver.Driver, error) {
+	d := &Driver{
+		home: home,
+	}
+	return d, nil
+}
+
+type Driver struct {
+	home string
+}
+
+func (d *Driver) String() string {
+	return "dummy"
+}
+
+func (d *Driver) Cleanup() error {
+	return nil
+}
+
+func (d *Driver) Create(id string, parent string) error {
+	dir := d.dir(id)
+	if err := os.MkdirAll(path.Dir(dir), 0700); err != nil {
+		return err
+	}
+	if err := os.Mkdir(dir, 0700); err != nil {
+		return err
+	}
+	if parent == "" {
+		return nil
+	}
+	parentDir, err := d.Get(parent)
+	if err != nil {
+		return fmt.Errorf("%s: %s", parent, err)
+	}
+	if err := archive.CopyWithTar(parentDir, dir); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *Driver) dir(id string) string {
+	return path.Join(d.home, "dir", path.Base(id))
+}
+
+func (d *Driver) Remove(id string) error {
+	if _, err := os.Stat(d.dir(id)); err != nil {
+		return err
+	}
+	return os.RemoveAll(d.dir(id))
+}
+
+func (d *Driver) Get(id string) (string, error) {
+	dir := d.dir(id)
+	if st, err := os.Stat(dir); err != nil {
+		return "", err
+	} else if !st.IsDir() {
+		return "", fmt.Errorf("%s: not a directory", dir)
+	}
+	return dir, nil
+}
+
+func (d *Driver) Size(id string) (int64, error) {
+	return -1, fmt.Errorf("Not implemented")
+}

http_test.go

@@ -1,51 +0,0 @@
-package docker
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-func TestGetBoolParam(t *testing.T) {
-	if ret, err := getBoolParam("true"); err != nil || !ret {
-		t.Fatalf("true -> true, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam("True"); err != nil || !ret {
-		t.Fatalf("True -> true, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam("1"); err != nil || !ret {
-		t.Fatalf("1 -> true, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam(""); err != nil || ret {
-		t.Fatalf("\"\" -> false, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam("false"); err != nil || ret {
-		t.Fatalf("false -> false, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam("0"); err != nil || ret {
-		t.Fatalf("0 -> false, nil | got %t %s", ret, err)
-	}
-	if ret, err := getBoolParam("faux"); err == nil || ret {
-		t.Fatalf("faux -> false, err | got %t %s", ret, err)
-	}
-}
-
-func TesthttpError(t *testing.T) {
-	r := httptest.NewRecorder()
-
-	httpError(r, fmt.Errorf("No such method"))
-	if r.Code != http.StatusNotFound {
-		t.Fatalf("Expected %d, got %d", http.StatusNotFound, r.Code)
-	}
-
-	httpError(r, fmt.Errorf("This accound hasn't been activated"))
-	if r.Code != http.StatusForbidden {
-		t.Fatalf("Expected %d, got %d", http.StatusForbidden, r.Code)
-	}
-
-	httpError(r, fmt.Errorf("Some error"))
-	if r.Code != http.StatusInternalServerError {
-		t.Fatalf("Expected %d, got %d", http.StatusInternalServerError, r.Code)
-	}
-}

image.go

@@ -6,17 +6,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
-	"os/exec"
 	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 	"time"
 )
 
@@ -55,166 +52,88 @@ func LoadImage(root string) (*Image, error) {
 			return nil, err
 		}
 	} else {
-		size, err := strconv.Atoi(string(buf))
-		if err != nil {
+		if size, err := strconv.Atoi(string(buf)); err != nil {
 			return nil, err
+		} else {
+			img.Size = int64(size)
 		}
-		img.Size = int64(size)
 	}
 
-	// Check that the filesystem layer exists
-	if stat, err := os.Stat(layerPath(root)); err != nil {
-		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.ID)
-		}
-		return nil, err
-	} else if !stat.IsDir() {
-		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.ID, layerPath(root))
-	}
 	return img, nil
 }
 
-func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root string) error {
-	// Check that root doesn't already exist
-	if _, err := os.Stat(root); err == nil {
-		return fmt.Errorf("Image %s already exists", img.ID)
-	} else if !os.IsNotExist(err) {
-		return err
-	}
+func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root, rootfs string) error {
 	// Store the layer
-	layer := layerPath(root)
+	layer := rootfs
 	if err := os.MkdirAll(layer, 0755); err != nil {
 		return err
 	}
 
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		start := time.Now()
-		utils.Debugf("Start untar layer")
-		if err := archive.Untar(layerData, layer); err != nil {
-			return err
+		if differ, ok := img.graph.driver.(graphdriver.Differ); ok {
+			if err := differ.ApplyDiff(img.ID, layerData); err != nil {
+				return err
+			}
+		} else {
+			start := time.Now()
+			utils.Debugf("Start untar layer")
+			if err := archive.ApplyLayer(layer, layerData); err != nil {
+				return err
+			}
+			utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 		}
-		utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 	}
 
 	// If raw json is provided, then use it
 	if jsonData != nil {
 		return ioutil.WriteFile(jsonPath(root), jsonData, 0600)
+	} else { // Otherwise, unmarshal the image
+		jsonData, err := json.Marshal(img)
+		if err != nil {
+			return err
+		}
+		if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
+			return err
+		}
 	}
-	// Otherwise, unmarshal the image
-	jsonData, err := json.Marshal(img)
+	// Compute and save the size of the rootfs
+	size, err := utils.TreeSize(rootfs)
 	if err != nil {
-		return err
+		return fmt.Errorf("Error computing size of rootfs %s: %s", img.ID, err)
 	}
-	if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
+	img.Size = size
+	if err := img.SaveSize(root); err != nil {
 		return err
 	}
 
-	return StoreSize(img, root)
-}
-
-func StoreSize(img *Image, root string) error {
-	layer := layerPath(root)
-	data := make(map[uint64]bool)
-
-	var totalSize int64
-	filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error {
-		size := fileInfo.Size()
-		if size == 0 {
-			return nil
-		}
-
-		inode := fileInfo.Sys().(*syscall.Stat_t).Ino
-		if _, entryExists := data[inode]; entryExists {
-			return nil
-		}
-		data[inode] = false
-
-		totalSize += size
-		return nil
-	})
-	img.Size = totalSize
-
-	if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil {
-		return nil
-	}
-
 	return nil
 }
 
-func layerPath(root string) string {
-	return path.Join(root, "layer")
+// SaveSize stores the current `size` value of `img` in the directory `root`.
+func (img *Image) SaveSize(root string) error {
+	if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(img.Size))), 0600); err != nil {
+		return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err)
+	}
+	return nil
 }
 
 func jsonPath(root string) string {
 	return path.Join(root, "json")
 }
 
-func MountAUFS(ro []string, rw string, target string) error {
-	// FIXME: Now mount the layers
-	rwBranch := fmt.Sprintf("%v=rw", rw)
-	roBranches := ""
-	for _, layer := range ro {
-		roBranches += fmt.Sprintf("%v=ro+wh:", layer)
-	}
-	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)
-
-	branches += ",xino=/dev/shm/aufs.xino"
-
-	//if error, try to load aufs kernel module
-	if err := mount("none", target, "aufs", 0, branches); err != nil {
-		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
-		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
-			return fmt.Errorf("Unable to load the AUFS module")
-		}
-		log.Printf("...module loaded.")
-		if err := mount("none", target, "aufs", 0, branches); err != nil {
-			return fmt.Errorf("Unable to mount using aufs")
-		}
-	}
-	return nil
-}
-
 // TarLayer returns a tar archive of the image's filesystem layer.
-func (img *Image) TarLayer(compression archive.Compression) (archive.Archive, error) {
-	layerPath, err := img.layer()
+func (image *Image) TarLayer(compression archive.Compression) (archive.Archive, error) {
+	if image.graph == nil {
+		return nil, fmt.Errorf("Can't load storage driver for unregistered image %s", image.ID)
+	}
+	layerPath, err := image.graph.driver.Get(image.ID)
 	if err != nil {
 		return nil, err
 	}
 	return archive.Tar(layerPath, compression)
 }
 
-func (img *Image) Mount(root, rw string) error {
-	if mounted, err := Mounted(root); err != nil {
-		return err
-	} else if mounted {
-		return fmt.Errorf("%s is already mounted", root)
-	}
-	layers, err := img.layers()
-	if err != nil {
-		return err
-	}
-	// Create the target directories if they don't exist
-	if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-	if err := MountAUFS(layers, rw, root); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (img *Image) Changes(rw string) ([]Change, error) {
-	layers, err := img.layers()
-	if err != nil {
-		return nil, err
-	}
-	return Changes(layers, rw)
-}
-
 func ValidateID(id string) error {
 	if id == "" {
 		return fmt.Errorf("Image id can't be empty")
@@ -250,40 +169,6 @@ func (img *Image) History() ([]*Image, error) {
 	return parents, nil
 }
 
-// layers returns all the filesystem layers needed to mount an image
-// FIXME: @shykes refactor this function with the new error handling
-//        (I'll do it if I have time tonight, I focus on the rest)
-func (img *Image) layers() ([]string, error) {
-	var (
-		list []string
-		e    error
-	)
-	if err := img.WalkHistory(
-		func(img *Image) (err error) {
-			if layer, err := img.layer(); err != nil {
-				e = err
-			} else if layer != "" {
-				list = append(list, layer)
-			}
-			return err
-		},
-	); err != nil {
-		return nil, err
-	} else if e != nil { // Did an error occur inside the handler?
-		return nil, e
-	}
-	if len(list) == 0 {
-		return nil, fmt.Errorf("No layer found for image %s\n", img.ID)
-	}
-
-	// Inject the dockerinit layer (empty place-holder for mount-binding dockerinit)
-	dockerinitLayer, err := img.getDockerInitLayer()
-	if err != nil {
-		return nil, err
-	}
-	return append([]string{dockerinitLayer}, list...), nil
-}
-
 func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
 	currentImg := img
 	for currentImg != nil {
@@ -310,13 +195,6 @@ func (img *Image) GetParent() (*Image, error) {
 	return img.graph.Get(img.Parent)
 }
 
-func (img *Image) getDockerInitLayer() (string, error) {
-	if img.graph == nil {
-		return "", fmt.Errorf("Can't lookup dockerinit layer of unregistered image")
-	}
-	return img.graph.getDockerInitLayer()
-}
-
 func (img *Image) root() (string, error) {
 	if img.graph == nil {
 		return "", fmt.Errorf("Can't lookup root of unregistered image")
@@ -324,15 +202,6 @@ func (img *Image) root() (string, error) {
 	return img.graph.imageRoot(img.ID), nil
 }
 
-// Return the path of an image's layer
-func (img *Image) layer() (string, error) {
-	root, err := img.root()
-	if err != nil {
-		return "", err
-	}
-	return layerPath(root), nil
-}
-
 func (img *Image) getParentsSize(size int64) int64 {
 	parentImage, err := img.GetParent()
 	if err != nil || parentImage == nil {

integration/auth_test.go

@@ -1,61 +0,0 @@
-package docker
-
-import (
-	"crypto/rand"
-	"encoding/hex"
-	"github.com/dotcloud/docker/auth"
-	"os"
-	"strings"
-	"testing"
-)
-
-// FIXME: these tests have an external dependency on a staging index hosted
-// on the docker.io infrastructure. That dependency should be removed.
-// - Unit tests should have no side-effect dependencies.
-// - Integration tests should have side-effects limited to the host environment being tested.
-
-func TestLogin(t *testing.T) {
-	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
-	defer os.Setenv("DOCKER_INDEX_URL", "")
-	authConfig := &auth.AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"}
-	status, err := auth.Login(authConfig, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if status != "Login Succeeded" {
-		t.Fatalf("Expected status \"Login Succeeded\", found \"%s\" instead", status)
-	}
-}
-
-func TestCreateAccount(t *testing.T) {
-	os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com")
-	defer os.Setenv("DOCKER_INDEX_URL", "")
-	tokenBuffer := make([]byte, 16)
-	_, err := rand.Read(tokenBuffer)
-	if err != nil {
-		t.Fatal(err)
-	}
-	token := hex.EncodeToString(tokenBuffer)[:12]
-	username := "ut" + token
-	authConfig := &auth.AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"}
-	status, err := auth.Login(authConfig, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	expectedStatus := "Account created. Please use the confirmation link we sent" +
-		" to your e-mail to activate it."
-	if status != expectedStatus {
-		t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status)
-	}
-
-	status, err = auth.Login(authConfig, nil)
-	if err == nil {
-		t.Fatalf("Expected error but found nil instead")
-	}
-
-	expectedError := "Login: Account is not Active"
-
-	if !strings.Contains(err.Error(), expectedError) {
-		t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err)
-	}
-}

integration/graph_test.go

@@ -1,57 +0,0 @@
-package docker
-
-import (
-	"github.com/dotcloud/docker"
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-)
-
-func TestMount(t *testing.T) {
-	graph := tempGraph(t)
-	defer os.RemoveAll(graph.Root)
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	image, err := graph.Create(archive, nil, "Testing", "", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	tmp, err := ioutil.TempDir("", "docker-test-graph-mount-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-	rootfs := path.Join(tmp, "rootfs")
-	if err := os.MkdirAll(rootfs, 0700); err != nil {
-		t.Fatal(err)
-	}
-	rw := path.Join(tmp, "rw")
-	if err := os.MkdirAll(rw, 0700); err != nil {
-		t.Fatal(err)
-	}
-	if err := image.Mount(rootfs, rw); err != nil {
-		t.Fatal(err)
-	}
-	// FIXME: test for mount contents
-	defer func() {
-		if err := docker.Unmount(rootfs); err != nil {
-			t.Error(err)
-		}
-	}()
-}
-
-//FIXME: duplicate
-func tempGraph(t *testing.T) *docker.Graph {
-	tmp, err := ioutil.TempDir("", "docker-graph-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	graph, err := docker.NewGraph(tmp)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return graph
-}

integration/iptables_test.go

@@ -1,22 +0,0 @@
-package docker
-
-import (
-	"github.com/dotcloud/docker/iptables"
-	"os"
-	"testing"
-)
-
-// FIXME: this test should be a unit test.
-// For example by mocking os/exec to make sure iptables is not actually called.
-
-func TestIptables(t *testing.T) {
-	if _, err := iptables.Raw("-L"); err != nil {
-		t.Fatal(err)
-	}
-	path := os.Getenv("PATH")
-	os.Setenv("PATH", "")
-	defer os.Setenv("PATH", path)
-	if _, err := iptables.Raw("-L"); err == nil {
-		t.Fatal("Not finding iptables in the PATH should cause an error")
-	}
-}

integration/sorter_test.go

@@ -1,63 +0,0 @@
-package docker
-
-import (
-	"github.com/dotcloud/docker"
-	"github.com/dotcloud/docker/utils"
-	"io/ioutil"
-	"testing"
-	"time"
-)
-
-func TestServerListOrderedImagesByCreationDate(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
-	srv := mkServerFromEngine(eng, t)
-
-	if err := generateImage("", srv); err != nil {
-		t.Fatal(err)
-	}
-
-	images, err := srv.Images(true, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if images[0].Created < images[1].Created {
-		t.Error("Expected []APIImges to be ordered by most recent creation date.")
-	}
-}
-
-func TestServerListOrderedImagesByCreationDateAndTag(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
-	srv := mkServerFromEngine(eng, t)
-
-	err := generateImage("bar", srv)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	time.Sleep(time.Second)
-
-	err = generateImage("zed", srv)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	images, err := srv.Images(true, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if images[0].RepoTags[0] != "repo:zed" && images[0].RepoTags[0] != "repo:bar" {
-		t.Errorf("Expected []APIImges to be ordered by most recent creation date. %s", images)
-	}
-}
-
-func generateImage(name string, srv *docker.Server) error {
-	archive, err := fakeTar()
-	if err != nil {
-		return err
-	}
-	return srv.ImageImport("-", "repo", name, archive, ioutil.Discard, utils.NewStreamFormatter(true))
-}

integration/utils_test.go

@@ -1,328 +0,0 @@
-package docker
-
-import (
-	"archive/tar"
-	"bytes"
-	"fmt"
-	"github.com/dotcloud/docker"
-	"github.com/dotcloud/docker/engine"
-	"github.com/dotcloud/docker/utils"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path"
-	"strings"
-	"testing"
-	"time"
-)
-
-// This file contains utility functions for docker's unit test suite.
-// It has to be named XXX_test.go, apparently, in other to access private functions
-// from other XXX_test.go functions.
-
-// Create a temporary runtime suitable for unit testing.
-// Call t.Fatal() at the first error.
-func mkRuntime(f utils.Fataler) *docker.Runtime {
-	root, err := newTestDirectory(unitTestStoreBase)
-	if err != nil {
-		f.Fatal(err)
-	}
-	config := &docker.DaemonConfig{
-		Root:        root,
-		AutoRestart: false,
-	}
-	r, err := docker.NewRuntimeFromDirectory(config)
-	if err != nil {
-		f.Fatal(err)
-	}
-	r.UpdateCapabilities(true)
-	return r
-}
-
-func createNamedTestContainer(eng *engine.Engine, config *docker.Config, f utils.Fataler, name string) (shortId string) {
-	job := eng.Job("create", name)
-	if err := job.ImportEnv(config); err != nil {
-		f.Fatal(err)
-	}
-	job.StdoutParseString(&shortId)
-	if err := job.Run(); err != nil {
-		f.Fatal(err)
-	}
-	return
-}
-
-func createTestContainer(eng *engine.Engine, config *docker.Config, f utils.Fataler) (shortId string) {
-	return createNamedTestContainer(eng, config, f, "")
-}
-
-func startContainer(eng *engine.Engine, id string, t utils.Fataler) {
-	job := eng.Job("start", id)
-	if err := job.Run(); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func containerRun(eng *engine.Engine, id string, t utils.Fataler) {
-	startContainer(eng, id, t)
-	containerWait(eng, id, t)
-}
-
-func containerFileExists(eng *engine.Engine, id, dir string, t utils.Fataler) bool {
-	c := getContainer(eng, id, t)
-	if err := c.EnsureMounted(); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := os.Stat(path.Join(c.RootfsPath(), dir)); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-		t.Fatal(err)
-	}
-	return true
-}
-
-func containerAttach(eng *engine.Engine, id string, t utils.Fataler) (io.WriteCloser, io.ReadCloser) {
-	c := getContainer(eng, id, t)
-	i, err := c.StdinPipe()
-	if err != nil {
-		t.Fatal(err)
-	}
-	o, err := c.StdoutPipe()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return i, o
-}
-
-func containerWait(eng *engine.Engine, id string, t utils.Fataler) int {
-	return getContainer(eng, id, t).Wait()
-}
-
-func containerWaitTimeout(eng *engine.Engine, id string, t utils.Fataler) error {
-	return getContainer(eng, id, t).WaitTimeout(500 * time.Millisecond)
-}
-
-func containerKill(eng *engine.Engine, id string, t utils.Fataler) {
-	if err := getContainer(eng, id, t).Kill(); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func containerRunning(eng *engine.Engine, id string, t utils.Fataler) bool {
-	return getContainer(eng, id, t).State.IsRunning()
-}
-
-func containerAssertExists(eng *engine.Engine, id string, t utils.Fataler) {
-	getContainer(eng, id, t)
-}
-
-func containerAssertNotExists(eng *engine.Engine, id string, t utils.Fataler) {
-	runtime := mkRuntimeFromEngine(eng, t)
-	if c := runtime.Get(id); c != nil {
-		t.Fatal(fmt.Errorf("Container %s should not exist", id))
-	}
-}
-
-// assertHttpNotError expect the given response to not have an error.
-// Otherwise the it causes the test to fail.
-func assertHttpNotError(r *httptest.ResponseRecorder, t utils.Fataler) {
-	// Non-error http status are [200, 400)
-	if r.Code < http.StatusOK || r.Code >= http.StatusBadRequest {
-		t.Fatal(fmt.Errorf("Unexpected http error: %v", r.Code))
-	}
-}
-
-// assertHttpError expect the given response to have an error.
-// Otherwise the it causes the test to fail.
-func assertHttpError(r *httptest.ResponseRecorder, t utils.Fataler) {
-	// Non-error http status are [200, 400)
-	if !(r.Code < http.StatusOK || r.Code >= http.StatusBadRequest) {
-		t.Fatal(fmt.Errorf("Unexpected http success code: %v", r.Code))
-	}
-}
-
-func getContainer(eng *engine.Engine, id string, t utils.Fataler) *docker.Container {
-	runtime := mkRuntimeFromEngine(eng, t)
-	c := runtime.Get(id)
-	if c == nil {
-		t.Fatal(fmt.Errorf("No such container: %s", id))
-	}
-	return c
-}
-
-func mkServerFromEngine(eng *engine.Engine, t utils.Fataler) *docker.Server {
-	iSrv := eng.Hack_GetGlobalVar("httpapi.server")
-	if iSrv == nil {
-		panic("Legacy server field not set in engine")
-	}
-	srv, ok := iSrv.(*docker.Server)
-	if !ok {
-		panic("Legacy server field in engine does not cast to *docker.Server")
-	}
-	return srv
-}
-
-func mkRuntimeFromEngine(eng *engine.Engine, t utils.Fataler) *docker.Runtime {
-	iRuntime := eng.Hack_GetGlobalVar("httpapi.runtime")
-	if iRuntime == nil {
-		panic("Legacy runtime field not set in engine")
-	}
-	runtime, ok := iRuntime.(*docker.Runtime)
-	if !ok {
-		panic("Legacy runtime field in engine does not cast to *docker.Runtime")
-	}
-	return runtime
-}
-
-func NewTestEngine(t utils.Fataler) *engine.Engine {
-	root, err := newTestDirectory(unitTestStoreBase)
-	if err != nil {
-		t.Fatal(err)
-	}
-	eng, err := engine.New(root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Load default plugins
-	// (This is manually copied and modified from main() until we have a more generic plugin system)
-	job := eng.Job("initapi")
-	job.Setenv("Root", root)
-	job.SetenvBool("AutoRestart", false)
-	// TestGetEnabledCors and TestOptionsRoute require EnableCors=true
-	job.SetenvBool("EnableCors", true)
-	if err := job.Run(); err != nil {
-		t.Fatal(err)
-	}
-	return eng
-}
-
-func newTestDirectory(templateDir string) (dir string, err error) {
-	return utils.TestDirectory(templateDir)
-}
-
-func getCallerName(depth int) string {
-	return utils.GetCallerName(depth)
-}
-
-// Write `content` to the file at path `dst`, creating it if necessary,
-// as well as any missing directories.
-// The file is truncated if it already exists.
-// Call t.Fatal() at the first error.
-func writeFile(dst, content string, t *testing.T) {
-	// Create subdirectories if necessary
-	if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) {
-		t.Fatal(err)
-	}
-	f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Write content (truncate if it exists)
-	if _, err := io.Copy(f, strings.NewReader(content)); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// Return the contents of file at path `src`.
-// Call t.Fatal() at the first error (including if the file doesn't exist)
-func readFile(src string, t *testing.T) (content string) {
-	f, err := os.Open(src)
-	if err != nil {
-		t.Fatal(err)
-	}
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return string(data)
-}
-
-// Create a test container from the given runtime `r` and run arguments `args`.
-// If the image name is "_", (eg. []string{"-i", "-t", "_", "bash"}, it is
-// dynamically replaced by the current test image.
-// The caller is responsible for destroying the container.
-// Call t.Fatal() at the first error.
-func mkContainer(r *docker.Runtime, args []string, t *testing.T) (*docker.Container, *docker.HostConfig, error) {
-	config, hc, _, err := docker.ParseRun(args, nil)
-	defer func() {
-		if err != nil && t != nil {
-			t.Fatal(err)
-		}
-	}()
-	if err != nil {
-		return nil, nil, err
-	}
-	if config.Image == "_" {
-		config.Image = GetTestImage(r).ID
-	}
-	c, _, err := r.Create(config, "")
-	if err != nil {
-		return nil, nil, err
-	}
-	// NOTE: hostConfig is ignored.
-	// If `args` specify privileged mode, custom lxc conf, external mount binds,
-	// port redirects etc. they will be ignored.
-	// This is because the correct way to set these things is to pass environment
-	// to the `start` job.
-	// FIXME: this helper function should be deprecated in favor of calling
-	// `create` and `start` jobs directly.
-	return c, hc, nil
-}
-
-// Create a test container, start it, wait for it to complete, destroy it,
-// and return its standard output as a string.
-// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image.
-// If t is not nil, call t.Fatal() at the first error. Otherwise return errors normally.
-func runContainer(eng *engine.Engine, r *docker.Runtime, args []string, t *testing.T) (output string, err error) {
-	defer func() {
-		if err != nil && t != nil {
-			t.Fatal(err)
-		}
-	}()
-	container, hc, err := mkContainer(r, args, t)
-	if err != nil {
-		return "", err
-	}
-	defer r.Destroy(container)
-	stdout, err := container.StdoutPipe()
-	if err != nil {
-		return "", err
-	}
-	defer stdout.Close()
-
-	job := eng.Job("start", container.ID)
-	if err := job.ImportEnv(hc); err != nil {
-		return "", err
-	}
-	if err := job.Run(); err != nil {
-		return "", err
-	}
-
-	container.Wait()
-	data, err := ioutil.ReadAll(stdout)
-	if err != nil {
-		return "", err
-	}
-	output = string(data)
-	return
-}
-
-// FIXME: this is duplicated from graph_test.go in the docker package.
-func fakeTar() (io.Reader, error) {
-	content := []byte("Hello world!\n")
-	buf := new(bytes.Buffer)
-	tw := tar.NewWriter(buf)
-	for _, name := range []string{"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres/postgres.conf"} {
-		hdr := new(tar.Header)
-		hdr.Size = int64(len(content))
-		hdr.Name = name
-		if err := tw.WriteHeader(hdr); err != nil {
-			return nil, err
-		}
-		tw.Write([]byte(content))
-	}
-	tw.Close()
-	return buf, nil
-}

iptables/iptables_test.go

@@ -0,0 +1,18 @@
+package iptables
+
+import (
+	"os"
+	"testing"
+)
+
+func TestIptables(t *testing.T) {
+	if _, err := Raw("-L"); err != nil {
+		t.Fatal(err)
+	}
+	path := os.Getenv("PATH")
+	os.Setenv("PATH", "")
+	defer os.Setenv("PATH", path)
+	if _, err := Raw("-L"); err == nil {
+		t.Fatal("Not finding iptables in the PATH should cause an error")
+	}
+}

links.go

@@ -21,7 +21,7 @@ func NewLink(parent, child *Container, name, bridgeInterface string) (*Link, err
 	if parent.ID == child.ID {
 		return nil, fmt.Errorf("Cannot link to self: %s == %s", parent.ID, child.ID)
 	}
-	if !child.State.IsRunning() {
+	if !child.State.Running {
 		return nil, fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, name)
 	}
 

lxc_template.go

@@ -120,7 +120,7 @@ lxc.aa_profile = unconfined
 #  (Note: 'lxc.cap.keep' is coming soon and should replace this under the
 #         security principle 'deny all unless explicitly permitted', see
 #         http://sourceforge.net/mailarchive/message.php?msg_id=31054627 )
-lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod setpcap sys_admin sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config
+lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config
 {{end}}
 
 # limits

lxc_template_unit_test.go

@@ -1,102 +0,0 @@
-package docker
-
-import (
-	"bufio"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestLXCConfig(t *testing.T) {
-	root, err := ioutil.TempDir("", "TestLXCConfig")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(root)
-	// Memory is allocated randomly for testing
-	rand.Seed(time.Now().UTC().UnixNano())
-	memMin := 33554432
-	memMax := 536870912
-	mem := memMin + rand.Intn(memMax-memMin)
-	// CPU shares as well
-	cpuMin := 100
-	cpuMax := 10000
-	cpu := cpuMin + rand.Intn(cpuMax-cpuMin)
-	container := &Container{
-		root: root,
-		Config: &Config{
-			Hostname:        "foobar",
-			Memory:          int64(mem),
-			CpuShares:       int64(cpu),
-			NetworkDisabled: true,
-		},
-		hostConfig: &HostConfig{
-			Privileged: false,
-		},
-	}
-	if err := container.generateLXCConfig(); err != nil {
-		t.Fatal(err)
-	}
-	grepFile(t, container.lxcConfigPath(), "lxc.utsname = foobar")
-	grepFile(t, container.lxcConfigPath(),
-		fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem))
-	grepFile(t, container.lxcConfigPath(),
-		fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2))
-}
-
-func TestCustomLxcConfig(t *testing.T) {
-	root, err := ioutil.TempDir("", "TestCustomLxcConfig")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(root)
-	container := &Container{
-		root: root,
-		Config: &Config{
-			Hostname:        "foobar",
-			NetworkDisabled: true,
-		},
-		hostConfig: &HostConfig{
-			Privileged: false,
-			LxcConf: []KeyValuePair{
-				{
-					Key:   "lxc.utsname",
-					Value: "docker",
-				},
-				{
-					Key:   "lxc.cgroup.cpuset.cpus",
-					Value: "0,1",
-				},
-			},
-		},
-	}
-	if err := container.generateLXCConfig(); err != nil {
-		t.Fatal(err)
-	}
-	grepFile(t, container.lxcConfigPath(), "lxc.utsname = docker")
-	grepFile(t, container.lxcConfigPath(), "lxc.cgroup.cpuset.cpus = 0,1")
-}
-
-func grepFile(t *testing.T, path string, pattern string) {
-	f, err := os.Open(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-	r := bufio.NewReader(f)
-	var (
-		line string
-	)
-	err = nil
-	for err == nil {
-		line, err = r.ReadString('\n')
-		if strings.Contains(line, pattern) == true {
-			return
-		}
-	}
-	t.Fatalf("grepFile: pattern \"%s\" not found in \"%s\"", pattern, path)
-}

runtime.go

@@ -1,11 +1,16 @@
 package docker
 
 import (
-	_ "code.google.com/p/gosqlite/sqlite3" // registers sqlite
+	_ "code.google.com/p/gosqlite/sqlite3"
 	"container/list"
 	"database/sql"
 	"fmt"
+	"github.com/dotcloud/docker/archive"
+	_ "github.com/dotcloud/docker/aufs"
+	_ "github.com/dotcloud/docker/devmapper"
 	"github.com/dotcloud/docker/gograph"
+	"github.com/dotcloud/docker/graphdriver"
+	_ "github.com/dotcloud/docker/graphdriver/dummy"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -13,9 +18,9 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 )
 
@@ -40,6 +45,7 @@ type Runtime struct {
 	srv            *Server
 	config         *DaemonConfig
 	containerGraph *gograph.Database
+	driver         graphdriver.Driver
 }
 
 // List returns an array of all containers registered in the runtime.
@@ -100,8 +106,8 @@ func (runtime *Runtime) load(id string) (*Container, error) {
 	if container.ID != id {
 		return container, fmt.Errorf("Container %s is stored at %s", container.ID, id)
 	}
-	if container.State.IsRunning() {
-		container.State.SetGhost(true)
+	if container.State.Running {
+		container.State.Ghost = true
 	}
 	return container, nil
 }
@@ -118,6 +124,16 @@ func (runtime *Runtime) Register(container *Container) error {
 		return err
 	}
 
+	// Get the root filesystem from the driver
+	rootfs, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return fmt.Errorf("Error getting container filesystem %s from driver %s: %s", container.ID, runtime.driver, err)
+	}
+	container.rootfs = rootfs
+
+	// init the wait lock
+	container.waitLock = make(chan struct{})
+
 	container.runtime = runtime
 
 	// Attach to stdout and stderr
@@ -133,10 +149,14 @@ func (runtime *Runtime) Register(container *Container) error {
 	runtime.containers.PushBack(container)
 	runtime.idIndex.Add(container.ID)
 
+	// When we actually restart, Start() do the monitoring.
+	// However, when we simply 'reattach', we have to restart a monitor
+	nomonitor := false
+
 	// FIXME: if the container is supposed to be running but is not, auto restart it?
 	//        if so, then we need to restart monitor and init a new lock
 	// If the container is supposed to be running, make sure of it
-	if container.State.IsRunning() {
+	if container.State.Running {
 		output, err := exec.Command("lxc-info", "-n", container.ID).CombinedOutput()
 		if err != nil {
 			return err
@@ -145,30 +165,30 @@ func (runtime *Runtime) Register(container *Container) error {
 			utils.Debugf("Container %s was supposed to be running be is not.", container.ID)
 			if runtime.config.AutoRestart {
 				utils.Debugf("Restarting")
-				container.State.SetGhost(false)
-				container.State.SetStopped(0)
+				container.State.Ghost = false
+				container.State.setStopped(0)
 				if err := container.Start(); err != nil {
 					return err
 				}
+				nomonitor = true
 			} else {
 				utils.Debugf("Marking as stopped")
-				container.State.SetStopped(-127)
+				container.State.setStopped(-127)
 				if err := container.ToDisk(); err != nil {
 					return err
 				}
 			}
-		} else {
-			utils.Debugf("Reconnecting to container %v", container.ID)
-
-			if err := container.allocateNetwork(); err != nil {
-				return err
-			}
-
-			container.waitLock = make(chan struct{})
-
-			go container.monitor()
 		}
 	}
+
+	// If the container is not running or just has been flagged not running
+	// then close the wait lock chan (will be reset upon start)
+	if !container.State.Running {
+		close(container.waitLock)
+	} else if !nomonitor {
+		container.allocateNetwork()
+		go container.monitor()
+	}
 	return nil
 }
 
@@ -216,12 +236,8 @@ func (runtime *Runtime) Destroy(container *Container) error {
 		return err
 	}
 
-	if mounted, err := container.Mounted(); err != nil {
-		return err
-	} else if mounted {
-		if err := container.Unmount(); err != nil {
-			return fmt.Errorf("Unable to unmount container %v: %v", container.ID, err)
-		}
+	if err := runtime.driver.Remove(container.ID); err != nil {
+		return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", runtime.driver, container.ID, err)
 	}
 
 	if _, err := runtime.containerGraph.Purge(container.ID); err != nil {
@@ -395,8 +411,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin
 	// Set the enitity in the graph using the default name specified
 	if _, err := runtime.containerGraph.Set(name, id); err != nil {
 		if strings.HasSuffix(err.Error(), "name are not unique") {
-			conflictingContainer, _ := runtime.GetByName(name)
-			return nil, nil, fmt.Errorf("Conflict, The name %s is already assigned to %s. You have to delete (or rename) that container to be able to assign %s to a container again.", name, utils.TruncateID(conflictingContainer.ID), name)
+			return nil, nil, fmt.Errorf("Conflict, %s already exists.", name)
 		}
 		return nil, nil, err
 	}
@@ -439,6 +454,21 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin
 		return nil, nil, err
 	}
 
+	initID := fmt.Sprintf("%s-init", container.ID)
+	if err := runtime.driver.Create(initID, img.ID); err != nil {
+		return nil, nil, err
+	}
+	initPath, err := runtime.driver.Get(initID)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := setupInitLayer(initPath); err != nil {
+		return nil, nil, err
+	}
+
+	if err := runtime.driver.Create(container.ID, initID); err != nil {
+		return nil, nil, err
+	}
 	resolvConf, err := utils.GetResolvConf()
 	if err != nil {
 		return nil, nil, err
@@ -511,12 +541,7 @@ func (runtime *Runtime) Commit(container *Container, repository, tag, comment, a
 	return img, nil
 }
 
-// FIXME: this is deprecated by the getFullName *function*
 func (runtime *Runtime) getFullName(name string) (string, error) {
-	return getFullName(name)
-}
-
-func getFullName(name string) (string, error) {
 	if name == "" {
 		return "", fmt.Errorf("Container name cannot be empty")
 	}
@@ -584,6 +609,13 @@ func NewRuntime(config *DaemonConfig) (*Runtime, error) {
 }
 
 func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
+	// Load storage driver
+	driver, err := graphdriver.New(config.Root)
+	if err != nil {
+		return nil, err
+	}
+	utils.Debugf("Using graph driver %s", driver)
+
 	runtimeRepo := path.Join(config.Root, "containers")
 
 	if err := os.MkdirAll(runtimeRepo, 0700); err != nil && !os.IsExist(err) {
@@ -593,11 +625,11 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 	if err := linkLxcStart(config.Root); err != nil {
 		return nil, err
 	}
-	g, err := NewGraph(path.Join(config.Root, "graph"))
+	g, err := NewGraph(path.Join(config.Root, "graph"), driver)
 	if err != nil {
 		return nil, err
 	}
-	volumes, err := NewGraph(path.Join(config.Root, "volumes"))
+	volumes, err := NewGraph(path.Join(config.Root, "volumes"), driver)
 	if err != nil {
 		return nil, err
 	}
@@ -642,6 +674,7 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 		volumes:        volumes,
 		config:         config,
 		containerGraph: graph,
+		driver:         driver,
 	}
 
 	if err := runtime.restore(); err != nil {
@@ -652,26 +685,79 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 
 func (runtime *Runtime) Close() error {
 	runtime.networkManager.Close()
+	runtime.driver.Cleanup()
 	return runtime.containerGraph.Close()
 }
 
-// Nuke kills all containers then removes all content
-// from the content root, including images, volumes and
-// container filesystems.
-// Again: this will remove your entire docker runtime!
-func (runtime *Runtime) Nuke() error {
-	var wg sync.WaitGroup
-	for _, container := range runtime.List() {
-		wg.Add(1)
-		go func(c *Container) {
-			c.Kill()
-			wg.Done()
-		}(container)
+func (runtime *Runtime) Mount(container *Container) error {
+	dir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, runtime.driver, err)
 	}
-	wg.Wait()
-	runtime.Close()
+	if container.rootfs == "" {
+		container.rootfs = dir
+	} else if container.rootfs != dir {
+		return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
+			runtime.driver, container.ID, container.rootfs, dir)
+	}
+	return nil
+}
 
-	return os.RemoveAll(runtime.config.Root)
+func (runtime *Runtime) Unmount(container *Container) error {
+	// FIXME: Unmount is deprecated because drivers are responsible for mounting
+	// and unmounting when necessary. Use driver.Remove() instead.
+	return nil
+}
+
+func (runtime *Runtime) Changes(container *Container) ([]archive.Change, error) {
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Changes(container.ID)
+	}
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	initDir, err := runtime.driver.Get(container.ID + "-init")
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container init rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	return archive.ChangesDirs(cDir, initDir)
+}
+
+func (runtime *Runtime) Diff(container *Container) (archive.Archive, error) {
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Diff(container.ID)
+	}
+
+	changes, err := runtime.Changes(container)
+	if err != nil {
+		return nil, err
+	}
+
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == archive.ChangeModify || change.Kind == archive.ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == archive.ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+	// FIXME: Why do we create whiteout files inside Tar code ?
+	return archive.TarFilter(cDir, &archive.TarOptions{
+		Compression: archive.Uncompressed,
+		Includes:    files,
+		Recursive:   false,
+		CreateFiles: deletions,
+	})
 }
 
 func linkLxcStart(root string) error {
@@ -691,14 +777,6 @@ func linkLxcStart(root string) error {
 	return os.Symlink(sourcePath, targetPath)
 }
 
-// FIXME: this is a convenience function for integration tests
-// which need direct access to runtime.graph.
-// Once the tests switch to using engine and jobs, this method
-// can go away.
-func (runtime *Runtime) Graph() *Graph {
-	return runtime.graph
-}
-
 // History is a convenience type for storing a list of containers,
 // ordered by creation date.
 type History []*Container

runtime_test.go

@@ -3,7 +3,6 @@ package docker
 import (
 	"bytes"
 	"fmt"
-	"github.com/dotcloud/docker"
 	"github.com/dotcloud/docker/engine"
 	"github.com/dotcloud/docker/sysinit"
 	"github.com/dotcloud/docker/utils"
@@ -16,6 +15,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -32,33 +32,42 @@ const (
 )
 
 var (
-	// FIXME: globalRuntime is deprecated by globalEngine. All tests should be converted.
-	globalRuntime   *docker.Runtime
-	globalEngine    *engine.Engine
+	globalRuntime   *Runtime
 	startFds        int
 	startGoroutines int
 )
 
-// FIXME: nuke() is deprecated by Runtime.Nuke()
-func nuke(runtime *docker.Runtime) error {
-	return runtime.Nuke()
+func nuke(runtime *Runtime) error {
+	if nonuke := os.Getenv("NONUKE"); nonuke != "" {
+		return nil
+	}
+	var wg sync.WaitGroup
+	for _, container := range runtime.List() {
+		wg.Add(1)
+		go func(c *Container) {
+			c.Kill()
+			wg.Done()
+		}(container)
+	}
+	wg.Wait()
+	runtime.Close()
+
+	os.Remove(filepath.Join(runtime.config.Root, "linkgraph.db"))
+	return os.RemoveAll(runtime.config.Root)
 }
 
-// FIXME: cleanup and nuke are redundant.
-func cleanup(eng *engine.Engine, t *testing.T) error {
-	runtime := mkRuntimeFromEngine(eng, t)
+func cleanup(runtime *Runtime) error {
 	for _, container := range runtime.List() {
 		container.Kill()
 		runtime.Destroy(container)
 	}
-	srv := mkServerFromEngine(eng, t)
-	images, err := srv.Images(true, "")
+	images, err := runtime.graph.Map()
 	if err != nil {
 		return err
 	}
 	for _, image := range images {
 		if image.ID != unitTestImageID {
-			srv.ImageDelete(image.ID, false)
+			runtime.graph.Delete(image.ID)
 		}
 	}
 	return nil
@@ -127,9 +136,10 @@ func setupBaseImage() {
 		log.Fatalf("Unable to create a runtime for tests:", err)
 	}
 	srv := mkServerFromEngine(eng, log.New(os.Stderr, "", 0))
+	runtime := srv.runtime
 
 	// If the unit test is not found, try to download it.
-	if img, err := srv.ImageInspect(unitTestImageName); err != nil || img.ID != unitTestImageID {
+	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
 		// Retrieve the Image
 		if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, nil, true); err != nil {
 			log.Fatalf("Unable to pull the test image: %s", err)
@@ -144,8 +154,8 @@ func spawnGlobalDaemon() {
 	}
 	t := log.New(os.Stderr, "", 0)
 	eng := NewTestEngine(t)
-	globalEngine = eng
-	globalRuntime = mkRuntimeFromEngine(eng, t)
+	srv := mkServerFromEngine(eng, t)
+	globalRuntime = srv.runtime
 
 	// Spawn a Daemon
 	go func() {
@@ -167,8 +177,8 @@ func spawnGlobalDaemon() {
 
 // FIXME: test that ImagePull(json=true) send correct json output
 
-func GetTestImage(runtime *docker.Runtime) *docker.Image {
-	imgs, err := runtime.Graph().Map()
+func GetTestImage(runtime *Runtime) *Image {
+	imgs, err := runtime.graph.Map()
 	if err != nil {
 		log.Fatalf("Unable to get the test image:", err)
 	}
@@ -177,7 +187,7 @@ func GetTestImage(runtime *docker.Runtime) *docker.Image {
 			return image
 		}
 	}
-	log.Fatalf("Test image %v not found in %s: %s", unitTestImageID, runtime.Graph().Root, imgs)
+	log.Fatalf("Test image %v not found in %s: %s", unitTestImageID, runtime.graph.Root, imgs)
 	return nil
 }
 
@@ -190,7 +200,7 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Errorf("Expected 0 containers, %v found", len(runtime.List()))
 	}
 
-	container, _, err := runtime.Create(&docker.Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"ls", "-al"},
 	},
@@ -231,25 +241,13 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Errorf("Exists() returned false for a newly created container")
 	}
 
-	// Test that conflict error displays correct details
-	testContainer, _, _ := runtime.Create(
-		&docker.Config{
-			Image: GetTestImage(runtime).ID,
-			Cmd:   []string{"ls", "-al"},
-		},
-		"conflictname",
-	)
-	if _, _, err := runtime.Create(&docker.Config{Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}}, testContainer.Name); err == nil || !strings.Contains(err.Error(), utils.TruncateID(testContainer.ID)) {
-		t.Fatalf("Name conflict error doesn't include the correct short id. Message was: %s", err.Error())
-	}
-
 	// Make sure create with bad parameters returns an error
-	if _, _, err = runtime.Create(&docker.Config{Image: GetTestImage(runtime).ID}, ""); err == nil {
+	if _, _, err = runtime.Create(&Config{Image: GetTestImage(runtime).ID}, ""); err == nil {
 		t.Fatal("Builder.Create should throw an error when Cmd is missing")
 	}
 
 	if _, _, err := runtime.Create(
-		&docker.Config{
+		&Config{
 			Image: GetTestImage(runtime).ID,
 			Cmd:   []string{},
 		},
@@ -258,7 +256,7 @@ func TestRuntimeCreate(t *testing.T) {
 		t.Fatal("Builder.Create should throw an error when Cmd is empty")
 	}
 
-	config := &docker.Config{
+	config := &Config{
 		Image:     GetTestImage(runtime).ID,
 		Cmd:       []string{"/bin/ls"},
 		PortSpecs: []string{"80"},
@@ -271,7 +269,7 @@ func TestRuntimeCreate(t *testing.T) {
 	}
 
 	// test expose 80:8000
-	container, warnings, err := runtime.Create(&docker.Config{
+	container, warnings, err := runtime.Create(&Config{
 		Image:     GetTestImage(runtime).ID,
 		Cmd:       []string{"ls", "-al"},
 		PortSpecs: []string{"80:8000"},
@@ -290,7 +288,7 @@ func TestDestroy(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container, _, err := runtime.Create(&docker.Config{
+	container, _, err := runtime.Create(&Config{
 		Image: GetTestImage(runtime).ID,
 		Cmd:   []string{"ls", "-al"},
 	}, "")
@@ -317,6 +315,12 @@ func TestDestroy(t *testing.T) {
 		t.Errorf("Unable to get newly created container")
 	}
 
+	// Make sure the container root directory does not exist anymore
+	_, err = os.Stat(container.root)
+	if err == nil || !os.IsNotExist(err) {
+		t.Errorf("Container root directory still exists after destroy")
+	}
+
 	// Test double destroy
 	if err := runtime.Destroy(container); err == nil {
 		// It should have failed
@@ -328,13 +332,13 @@ func TestGet(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
-	container1, _, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
+	container1, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
 	defer runtime.Destroy(container1)
 
-	container2, _, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
+	container2, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
 	defer runtime.Destroy(container2)
 
-	container3, _, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
+	container3, _ := mkContainer(runtime, []string{"_", "ls", "-al"}, t)
 	defer runtime.Destroy(container3)
 
 	if runtime.Get(container1.ID) != container1 {
@@ -351,21 +355,15 @@ func TestGet(t *testing.T) {
 
 }
 
-func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *docker.Container, string) {
+func startEchoServerContainer(t *testing.T, proto string) (*Runtime, *Container, string) {
 	var (
-		err     error
-		id      string
-		strPort string
-		eng     = NewTestEngine(t)
-		runtime = mkRuntimeFromEngine(eng, t)
-		port    = 5554
-		p       docker.Port
+		err       error
+		container *Container
+		strPort   string
+		runtime   = mkRuntime(t)
+		port      = 5554
+		p         Port
 	)
-	defer func() {
-		if err != nil {
-			runtime.Nuke()
-		}
-	}()
 
 	for {
 		port += 1
@@ -378,48 +376,40 @@ func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *doc
 		} else {
 			t.Fatal(fmt.Errorf("Unknown protocol %v", proto))
 		}
-		ep := make(map[docker.Port]struct{}, 1)
-		p = docker.Port(fmt.Sprintf("%s/%s", strPort, proto))
+		ep := make(map[Port]struct{}, 1)
+		p = Port(fmt.Sprintf("%s/%s", strPort, proto))
 		ep[p] = struct{}{}
 
-		jobCreate := eng.Job("create")
-		jobCreate.Setenv("Image", unitTestImageID)
-		jobCreate.SetenvList("Cmd", []string{"sh", "-c", cmd})
-		jobCreate.SetenvList("PortSpecs", []string{fmt.Sprintf("%s/%s", strPort, proto)})
-		jobCreate.SetenvJson("ExposedPorts", ep)
-		jobCreate.StdoutParseString(&id)
-		if err := jobCreate.Run(); err != nil {
+		container, _, err = runtime.Create(&Config{
+			Image:        GetTestImage(runtime).ID,
+			Cmd:          []string{"sh", "-c", cmd},
+			PortSpecs:    []string{fmt.Sprintf("%s/%s", strPort, proto)},
+			ExposedPorts: ep,
+		}, "")
+		if err != nil {
+			nuke(runtime)
 			t.Fatal(err)
 		}
-		// FIXME: this relies on the undocumented behavior of runtime.Create
-		// which will return a nil error AND container if the exposed ports
-		// are invalid. That behavior should be fixed!
-		if id != "" {
+
+		if container != nil {
 			break
 		}
 		t.Logf("Port %v already in use, trying another one", strPort)
-
 	}
 
-	jobStart := eng.Job("start", id)
-	portBindings := make(map[docker.Port][]docker.PortBinding)
-	portBindings[p] = []docker.PortBinding{
+	container.hostConfig = &HostConfig{
+		PortBindings: make(map[Port][]PortBinding),
+	}
+	container.hostConfig.PortBindings[p] = []PortBinding{
 		{},
 	}
-	if err := jobStart.SetenvJson("PortsBindings", portBindings); err != nil {
+	if err := container.Start(); err != nil {
+		nuke(runtime)
 		t.Fatal(err)
 	}
-	if err := jobStart.Run(); err != nil {
-		t.Fatal(err)
-	}
-
-	container := runtime.Get(id)
-	if container == nil {
-		t.Fatalf("Couldn't fetch test container %s", id)
-	}
 
 	setTimeout(t, "Waiting for the container to be started timed out", 2*time.Second, func() {
-		for !container.State.IsRunning() {
+		for !container.State.Running {
 			time.Sleep(10 * time.Millisecond)
 		}
 	})
@@ -517,15 +507,14 @@ func TestAllocateUDPPortLocalhost(t *testing.T) {
 }
 
 func TestRestore(t *testing.T) {
-	eng := NewTestEngine(t)
-	runtime1 := mkRuntimeFromEngine(eng, t)
-	defer runtime1.Nuke()
+	runtime1 := mkRuntime(t)
+	defer nuke(runtime1)
 	// Create a container with one instance of docker
-	container1, _, _ := mkContainer(runtime1, []string{"_", "ls", "-al"}, t)
+	container1, _ := mkContainer(runtime1, []string{"_", "ls", "-al"}, t)
 	defer runtime1.Destroy(container1)
 
 	// Create a second container meant to be killed
-	container2, _, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/cat"}, t)
+	container2, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/cat"}, t)
 	defer runtime1.Destroy(container2)
 
 	// Start the container non blocking
@@ -533,7 +522,7 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !container2.State.IsRunning() {
+	if !container2.State.Running {
 		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
 	}
 
@@ -543,7 +532,7 @@ func TestRestore(t *testing.T) {
 	if err := container2.WaitTimeout(2 * time.Second); err != nil {
 		t.Fatal(err)
 	}
-	container2.State.SetRunning(42)
+	container2.State.Running = true
 	container2.ToDisk()
 
 	if len(runtime1.List()) != 2 {
@@ -553,31 +542,24 @@ func TestRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !container2.State.IsRunning() {
+	if !container2.State.Running {
 		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
 	}
 
 	// Here are are simulating a docker restart - that is, reloading all containers
 	// from scratch
-	root := eng.Root()
-	eng, err := engine.New(root)
+	runtime1.config.AutoRestart = false
+	runtime2, err := NewRuntimeFromDirectory(runtime1.config)
 	if err != nil {
 		t.Fatal(err)
 	}
-	job := eng.Job("initapi")
-	job.Setenv("Root", eng.Root())
-	job.SetenvBool("Autorestart", false)
-	if err := job.Run(); err != nil {
-		t.Fatal(err)
-	}
-
-	runtime2 := mkRuntimeFromEngine(eng, t)
+	defer nuke(runtime2)
 	if len(runtime2.List()) != 2 {
 		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
 	}
 	runningCount := 0
 	for _, c := range runtime2.List() {
-		if c.State.IsRunning() {
+		if c.State.Running {
 			t.Errorf("Running container found: %v (%v)", c.ID, c.Path)
 			runningCount++
 		}
@@ -592,35 +574,18 @@ func TestRestore(t *testing.T) {
 	if err := container3.Run(); err != nil {
 		t.Fatal(err)
 	}
-	container2.State.SetStopped(0)
+	container2.State.Running = false
 }
 
 func TestReloadContainerLinks(t *testing.T) {
-	// FIXME: here we don't use NewTestEngine because it calls initapi with Autorestart=false,
-	// and we want to set it to true.
-	root, err := newTestDirectory(unitTestStoreBase)
-	if err != nil {
-		t.Fatal(err)
-	}
-	eng, err := engine.New(root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	job := eng.Job("initapi")
-	job.Setenv("Root", eng.Root())
-	job.SetenvBool("Autorestart", true)
-	if err := job.Run(); err != nil {
-		t.Fatal(err)
-	}
-
-	runtime1 := mkRuntimeFromEngine(eng, t)
+	runtime1 := mkRuntime(t)
 	defer nuke(runtime1)
 	// Create a container with one instance of docker
-	container1, _, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/sh"}, t)
+	container1, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/sh"}, t)
 	defer runtime1.Destroy(container1)
 
 	// Create a second container meant to be killed
-	container2, _, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/cat"}, t)
+	container2, _ := mkContainer(runtime1, []string{"-i", "_", "/bin/cat"}, t)
 	defer runtime1.Destroy(container2)
 
 	// Start the container non blocking
@@ -628,9 +593,7 @@ func TestReloadContainerLinks(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Add a link to container 2
-	// FIXME @shykes: setting hostConfig.Links seems redundant with calling RegisterLink().
-	// Why do we need it @crosbymichael?
-	// container1.hostConfig.Links = []string{"/" + container2.ID + ":first"}
+	container1.hostConfig.Links = []string{"/" + container2.ID + ":first"}
 	if err := runtime1.RegisterLink(container1, container2, "first"); err != nil {
 		t.Fatal(err)
 	}
@@ -638,11 +601,11 @@ func TestReloadContainerLinks(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if !container2.State.IsRunning() {
+	if !container2.State.Running {
 		t.Fatalf("Container %v should appear as running but isn't", container2.ID)
 	}
 
-	if !container1.State.IsRunning() {
+	if !container1.State.Running {
 		t.Fatalf("Container %s should appear as running but isn't", container1.ID)
 	}
 
@@ -652,24 +615,18 @@ func TestReloadContainerLinks(t *testing.T) {
 
 	// Here are are simulating a docker restart - that is, reloading all containers
 	// from scratch
-	eng, err = engine.New(root)
+	runtime1.config.AutoRestart = true
+	runtime2, err := NewRuntimeFromDirectory(runtime1.config)
 	if err != nil {
 		t.Fatal(err)
 	}
-	job = eng.Job("initapi")
-	job.Setenv("Root", eng.Root())
-	job.SetenvBool("Autorestart", false)
-	if err := job.Run(); err != nil {
-		t.Fatal(err)
-	}
-
-	runtime2 := mkRuntimeFromEngine(eng, t)
+	defer nuke(runtime2)
 	if len(runtime2.List()) != 2 {
 		t.Errorf("Expected 2 container, %v found", len(runtime2.List()))
 	}
 	runningCount := 0
 	for _, c := range runtime2.List() {
-		if c.State.IsRunning() {
+		if c.State.Running {
 			runningCount++
 		}
 	}
@@ -677,32 +634,27 @@ func TestReloadContainerLinks(t *testing.T) {
 		t.Fatalf("Expected 2 container alive, %d found", runningCount)
 	}
 
-	// FIXME: we no longer test if containers were registered in the right order,
-	// because there is no public
 	// Make sure container 2 ( the child of container 1 ) was registered and started first
 	// with the runtime
-	//
-	containers := runtime2.List()
-	if len(containers) == 0 {
-		t.Fatalf("Runtime has no containers")
-	}
-	first := containers[0]
-	if first.ID != container2.ID {
+	first := runtime2.containers.Front()
+	if first.Value.(*Container).ID != container2.ID {
 		t.Fatalf("Container 2 %s should be registered first in the runtime", container2.ID)
 	}
 
 	// Verify that the link is still registered in the runtime
-	if c := runtime2.Get(container1.Name); c == nil {
-		t.Fatal("Named container is no longer registered after restart")
+	entity := runtime2.containerGraph.Get(container1.Name)
+	if entity == nil {
+		t.Fatal("Entity should not be nil")
 	}
 }
 
 func TestDefaultContainerName(t *testing.T) {
 	eng := NewTestEngine(t)
-	runtime := mkRuntimeFromEngine(eng, t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -714,19 +666,29 @@ func TestDefaultContainerName(t *testing.T) {
 		t.Fatalf("Expect /some_name got %s", container.Name)
 	}
 
-	if c := runtime.Get("/some_name"); c == nil {
-		t.Fatalf("Couldn't retrieve test container as /some_name")
-	} else if c.ID != containerID {
-		t.Fatalf("Container /some_name has ID %s instead of %s", c.ID, containerID)
+	paths := runtime.containerGraph.RefPaths(containerID)
+	if paths == nil || len(paths) == 0 {
+		t.Fatalf("Could not find edges for %s", containerID)
+	}
+	edge := paths[0]
+	if edge.ParentID != "0" {
+		t.Fatalf("Expected engine got %s", edge.ParentID)
+	}
+	if edge.EntityID != containerID {
+		t.Fatalf("Expected %s got %s", containerID, edge.EntityID)
+	}
+	if edge.Name != "some_name" {
+		t.Fatalf("Expected some_name got %s", edge.Name)
 	}
 }
 
 func TestRandomContainerName(t *testing.T) {
 	eng := NewTestEngine(t)
-	runtime := mkRuntimeFromEngine(eng, t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -738,19 +700,29 @@ func TestRandomContainerName(t *testing.T) {
 		t.Fatalf("Expected not empty container name")
 	}
 
-	if c := runtime.Get(container.Name); c == nil {
-		log.Fatalf("Could not lookup container %s by its name", container.Name)
-	} else if c.ID != containerID {
-		log.Fatalf("Looking up container name %s returned id %s instead of %s", container.Name, c.ID, containerID)
+	paths := runtime.containerGraph.RefPaths(containerID)
+	if paths == nil || len(paths) == 0 {
+		t.Fatalf("Could not find edges for %s", containerID)
+	}
+	edge := paths[0]
+	if edge.ParentID != "0" {
+		t.Fatalf("Expected engine got %s", edge.ParentID)
+	}
+	if edge.EntityID != containerID {
+		t.Fatalf("Expected %s got %s", containerID, edge.EntityID)
+	}
+	if edge.Name == "" {
+		t.Fatalf("Expected not empty container name")
 	}
 }
 
 func TestLinkChildContainer(t *testing.T) {
 	eng := NewTestEngine(t)
-	runtime := mkRuntimeFromEngine(eng, t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -766,7 +738,7 @@ func TestLinkChildContainer(t *testing.T) {
 		t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID)
 	}
 
-	config, _, _, err = docker.ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
+	config, _, _, err = ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -789,10 +761,11 @@ func TestLinkChildContainer(t *testing.T) {
 
 func TestGetAllChildren(t *testing.T) {
 	eng := NewTestEngine(t)
-	runtime := mkRuntimeFromEngine(eng, t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -808,7 +781,7 @@ func TestGetAllChildren(t *testing.T) {
 		t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID)
 	}
 
-	config, _, _, err = docker.ParseRun([]string{unitTestImageID, "echo test"}, nil)
+	config, _, _, err = ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -840,3 +813,19 @@ func TestGetAllChildren(t *testing.T) {
 		}
 	}
 }
+
+func TestGetFullName(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	name, err := runtime.getFullName("testing")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if name != "/testing" {
+		t.Fatalf("Expected /testing got %s", name)
+	}
+	if _, err := runtime.getFullName(""); err == nil {
+		t.Fatal("Error should not be nil")
+	}
+}

server.go

@@ -62,8 +62,6 @@ func jobInitApi(job *engine.Job) string {
 		os.Exit(0)
 	}()
 	job.Eng.Hack_SetGlobalVar("httpapi.server", srv)
-	job.Eng.Hack_SetGlobalVar("httpapi.runtime", srv.runtime)
-	job.Eng.Hack_SetGlobalVar("httpapi.bridgeIP", srv.runtime.networkManager.bridgeNetwork.IP)
 	if err := job.Eng.Register("create", srv.ContainerCreate); err != nil {
 		return err.Error()
 	}
@@ -197,222 +195,6 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error {
 	return fmt.Errorf("No such container: %s", name)
 }
 
-// ImageExport exports all images with the given tag. All versions
-// containing the same tag are exported. The resulting output is an
-// uncompressed tar ball.
-// name is the set of tags to export.
-// out is the writer where the images are written to.
-func (srv *Server) ImageExport(name string, out io.Writer) error {
-	// get image json
-	tempdir, err := ioutil.TempDir("", "docker-export-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tempdir)
-
-	utils.Debugf("Serializing %s", name)
-
-	rootRepo, err := srv.runtime.repositories.Get(name)
-	if err != nil {
-		return err
-	}
-	if rootRepo != nil {
-		for _, id := range rootRepo {
-			image, err := srv.ImageInspect(id)
-			if err != nil {
-				return err
-			}
-
-			if err := srv.exportImage(image, tempdir); err != nil {
-				return err
-			}
-		}
-
-		// write repositories
-		rootRepoMap := map[string]Repository{}
-		rootRepoMap[name] = rootRepo
-		rootRepoJson, _ := json.Marshal(rootRepoMap)
-
-		if err := ioutil.WriteFile(path.Join(tempdir, "repositories"), rootRepoJson, os.ModeAppend); err != nil {
-			return err
-		}
-	} else {
-		image, err := srv.ImageInspect(name)
-		if err != nil {
-			return err
-		}
-		if err := srv.exportImage(image, tempdir); err != nil {
-			return err
-		}
-	}
-
-	fs, err := archive.Tar(tempdir, archive.Uncompressed)
-	if err != nil {
-		return err
-	}
-
-	if _, err := io.Copy(out, fs); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (srv *Server) exportImage(image *Image, tempdir string) error {
-	for i := image; i != nil; {
-		// temporary directory
-		tmpImageDir := path.Join(tempdir, i.ID)
-		if err := os.Mkdir(tmpImageDir, os.ModeDir); err != nil {
-			return err
-		}
-
-		var version = "1.0"
-		var versionBuf = []byte(version)
-
-		if err := ioutil.WriteFile(path.Join(tmpImageDir, "VERSION"), versionBuf, os.ModeAppend); err != nil {
-			return err
-		}
-
-		// serialize json
-		b, err := json.Marshal(i)
-		if err != nil {
-			return err
-		}
-		if err := ioutil.WriteFile(path.Join(tmpImageDir, "json"), b, os.ModeAppend); err != nil {
-			return err
-		}
-
-		// serialize filesystem
-		fs, err := archive.Tar(path.Join(srv.runtime.graph.Root, i.ID, "layer"), archive.Uncompressed)
-		if err != nil {
-			return err
-		}
-
-		fsTar, err := os.Create(path.Join(tmpImageDir, "layer.tar"))
-		if err != nil {
-			return err
-		}
-		if _, err = io.Copy(fsTar, fs); err != nil {
-			return err
-		}
-		fsTar.Close()
-
-		// find parent
-		if i.Parent != "" {
-			i, err = srv.ImageInspect(i.Parent)
-			if err != nil {
-				return err
-			}
-		} else {
-			i = nil
-		}
-	}
-	return nil
-}
-
-// Loads a set of images into the repository. This is the complementary of ImageExport.
-// The input stream is an uncompressed tar ball containing images and metadata.
-func (srv *Server) ImageLoad(in io.Reader) error {
-	tmpImageDir, err := ioutil.TempDir("", "docker-import-")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpImageDir)
-
-	var (
-		repoTarFile = path.Join(tmpImageDir, "repo.tar")
-		repoDir     = path.Join(tmpImageDir, "repo")
-	)
-
-	tarFile, err := os.Create(repoTarFile)
-	if err != nil {
-		return err
-	}
-	if _, err := io.Copy(tarFile, in); err != nil {
-		return err
-	}
-	tarFile.Close()
-
-	repoFile, err := os.Open(repoTarFile)
-	if err != nil {
-		return err
-	}
-	if err := os.Mkdir(repoDir, os.ModeDir); err != nil {
-		return err
-	}
-	if err := archive.Untar(repoFile, repoDir); err != nil {
-		return err
-	}
-
-	dirs, err := ioutil.ReadDir(repoDir)
-	if err != nil {
-		return err
-	}
-
-	for _, d := range dirs {
-		if d.IsDir() {
-			if err := srv.recursiveLoad(d.Name(), tmpImageDir); err != nil {
-				return err
-			}
-		}
-	}
-
-	repositoriesJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", "repositories"))
-	if err == nil {
-		repositories := map[string]Repository{}
-		if err := json.Unmarshal(repositoriesJson, &repositories); err != nil {
-			return err
-		}
-
-		for imageName, tagMap := range repositories {
-			for tag, address := range tagMap {
-				if err := srv.runtime.repositories.Set(imageName, tag, address, true); err != nil {
-					return err
-				}
-			}
-		}
-	} else if !os.IsNotExist(err) {
-		return err
-	}
-
-	return nil
-}
-
-func (srv *Server) recursiveLoad(address, tmpImageDir string) error {
-	if _, err := srv.ImageInspect(address); err != nil {
-		utils.Debugf("Loading %s", address)
-
-		imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json"))
-		if err != nil {
-			return err
-			utils.Debugf("Error reading json", err)
-		}
-
-		layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar"))
-		if err != nil {
-			utils.Debugf("Error reading embedded tar", err)
-			return err
-		}
-		img, err := NewImgJSON(imageJson)
-		if err != nil {
-			utils.Debugf("Error unmarshalling json", err)
-			return err
-		}
-		if img.Parent != "" {
-			if !srv.runtime.graph.Exists(img.Parent) {
-				if err := srv.recursiveLoad(img.Parent, tmpImageDir); err != nil {
-					return err
-				}
-			}
-		}
-		if err := srv.runtime.graph.Register(imageJson, layer, img); err != nil {
-			return err
-		}
-	}
-	utils.Debugf("Completed processing %s", address)
-
-	return nil
-}
-
 func (srv *Server) ImagesSearch(term string) ([]registry.SearchResult, error) {
 	r, err := registry.NewRegistry(srv.runtime.config.Root, nil, srv.HTTPRequestFactory(nil))
 	if err != nil {
@@ -638,9 +420,9 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) {
 
 }
 
-func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) {
+func (srv *Server) ContainerTop(name, ps_args string) (*APITop, error) {
 	if container := srv.runtime.Get(name); container != nil {
-		output, err := exec.Command("lxc-ps", "--name", container.ID, "--", psArgs).CombinedOutput()
+		output, err := exec.Command("lxc-ps", "--name", container.ID, "--", ps_args).CombinedOutput()
 		if err != nil {
 			return nil, fmt.Errorf("lxc-ps: %s (%s)", err, output)
 		}
@@ -675,7 +457,7 @@ func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) {
 	return nil, fmt.Errorf("No such container: %s", name)
 }
 
-func (srv *Server) ContainerChanges(name string) ([]Change, error) {
+func (srv *Server) ContainerChanges(name string) ([]archive.Change, error) {
 	if container := srv.runtime.Get(name); container != nil {
 		return container.Changes()
 	}
@@ -687,21 +469,18 @@ func (srv *Server) Containers(all, size bool, n int, since, before string) []API
 	var displayed int
 	out := []APIContainers{}
 
-	names := map[string][]string{}
-	srv.runtime.containerGraph.Walk("/", func(p string, e *gograph.Entity) error {
-		names[e.ID()] = append(names[e.ID()], p)
-		return nil
-	}, -1)
-
 	for _, container := range srv.runtime.List() {
-		if !container.State.IsRunning() && !all && n == -1 && since == "" && before == "" {
+		if !container.State.Running && !all && n == -1 && since == "" && before == "" {
 			continue
 		}
-		if before != "" && !foundBefore {
+		if before != "" {
 			if container.ID == before || utils.TruncateID(container.ID) == before {
 				foundBefore = true
+				continue
+			}
+			if !foundBefore {
+				continue
 			}
-			continue
 		}
 		if displayed == n {
 			break
@@ -710,17 +489,25 @@ func (srv *Server) Containers(all, size bool, n int, since, before string) []API
 			break
 		}
 		displayed++
-		c := createAPIContainer(names[container.ID], container, size, srv.runtime)
+		c := createAPIContainer(container, size, srv.runtime)
 		out = append(out, c)
 	}
 	return out
 }
 
-func createAPIContainer(names []string, container *Container, size bool, runtime *Runtime) APIContainers {
+func createAPIContainer(container *Container, size bool, runtime *Runtime) APIContainers {
 	c := APIContainers{
 		ID: container.ID,
 	}
+	names := []string{}
+	runtime.containerGraph.Walk("/", func(p string, e *gograph.Entity) error {
+		if e.ID() == container.ID {
+			names = append(names, p)
+		}
+		return nil
+	}, -1)
 	c.Names = names
+
 	c.Image = runtime.repositories.ImageName(container.Image)
 	c.Command = fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " "))
 	c.Created = container.Created.Unix()
@@ -743,7 +530,6 @@ func (srv *Server) ContainerCommit(name, repo, tag, author, comment string, conf
 	return img.ID, err
 }
 
-// FIXME: this should be called ImageTag
 func (srv *Server) ContainerTag(name, repo, tag string, force bool) error {
 	if err := srv.runtime.repositories.Set(repo, tag, name, force); err != nil {
 		return err
@@ -756,11 +542,12 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
 	if err != nil {
 		return err
 	}
-	out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependent layers"))
+	out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling", "dependend layers"))
 	// FIXME: Try to stream the images?
 	// FIXME: Launch the getRemoteImage() in goroutines
 
-	for _, id := range history {
+	for i := len(history) - 1; i >= 0; i-- {
+		id := history[i]
 
 		// ensure no two downloads of the same layer happen at the same time
 		if err := srv.poolAdd("pull", "layer:"+id); err != nil {
@@ -773,13 +560,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
 			out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "metadata"))
 			imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token)
 			if err != nil {
-				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers"))
+				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
 				// FIXME: Keep going in case of error?
 				return err
 			}
 			img, err := NewImgJSON(imgJSON)
 			if err != nil {
-				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers"))
+				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
 				return fmt.Errorf("Failed to parse json: %s", err)
 			}
 
@@ -787,12 +574,12 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin
 			out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling", "fs layer"))
 			layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token)
 			if err != nil {
-				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependent layers"))
+				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "pulling dependend layers"))
 				return err
 			}
 			defer layer.Close()
 			if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil {
-				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependent layers"))
+				out.Write(sf.FormatProgress(utils.TruncateID(id), "Error", "downloading dependend layers"))
 				return err
 			}
 		}
@@ -1102,13 +889,12 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName
 					out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID))
 					continue
 				}
-				checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf)
-				if err != nil {
+				if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil {
 					// FIXME: Continue on error?
 					return err
+				} else {
+					elem.Checksum = checksum
 				}
-				elem.Checksum = checksum
-
 				if err := pushTags(); err != nil {
 					return err
 				}
@@ -1148,15 +934,13 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID,
 	if err != nil {
 		return "", fmt.Errorf("Failed to generate layer archive: %s", err)
 	}
-	defer os.RemoveAll(layerData.Name())
 
 	// Send the layer
-	checksum, err = r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, false), ep, token, jsonRaw)
-	if err != nil {
+	if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, false), ep, token, jsonRaw); err != nil {
 		return "", err
+	} else {
+		imgData.Checksum = checksum
 	}
-	imgData.Checksum = checksum
-
 	out.Write(sf.FormatStatus("", ""))
 
 	// Send the checksum
@@ -1279,12 +1063,7 @@ func (srv *Server) ContainerCreate(job *engine.Job) string {
 		return err.Error()
 	}
 	srv.LogEvent("create", container.ID, srv.runtime.repositories.ImageName(container.Image))
-	// FIXME: this is necessary because runtime.Create might return a nil container
-	// with a non-nil error. This should not happen! Once it's fixed we
-	// can remove this workaround.
-	if container != nil {
-		job.Printf("%s\n", container.ID)
-	}
+	job.Printf("%s\n", container.ID)
 	for _, warning := range buildWarnings {
 		job.Errorf("%s\n", warning)
 	}
@@ -1339,28 +1118,13 @@ func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool)
 	}
 
 	if container != nil {
-		if container.State.IsRunning() {
+		if container.State.Running {
 			return fmt.Errorf("Impossible to remove a running container, please stop it first")
 		}
 		volumes := make(map[string]struct{})
-
-		binds := make(map[string]struct{})
-
-		for _, bind := range container.hostConfig.Binds {
-			splitBind := strings.Split(bind, ":")
-			source := splitBind[0]
-			binds[source] = struct{}{}
-		}
-
 		// Store all the deleted containers volumes
 		for _, volumeId := range container.Volumes {
-
-			// Skip the volumes mounted from external
-			if _, exists := binds[volumeId]; exists {
-				continue
-			}
-
-			volumeId = strings.TrimSuffix(volumeId, "/layer")
+			volumeId = strings.TrimRight(volumeId, "/layer")
 			volumeId = filepath.Base(volumeId)
 			volumes[volumeId] = struct{}{}
 		}
@@ -1433,8 +1197,8 @@ func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi) error {
 		if err != nil {
 			return err
 		}
-		*imgs = append(*imgs, APIRmi{Deleted: id})
-		srv.LogEvent("delete", id, "")
+		*imgs = append(*imgs, APIRmi{Deleted: utils.TruncateID(id)})
+		srv.LogEvent("delete", utils.TruncateID(id), "")
 		return nil
 	}
 	return nil
@@ -1516,7 +1280,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) {
 
 	// Prevent deletion if image is used by a running container
 	for _, container := range srv.runtime.List() {
-		if container.State.IsRunning() {
+		if container.State.Running {
 			parent, err := srv.runtime.repositories.LookupImage(container.Image)
 			if err != nil {
 				return nil, err
@@ -1738,7 +1502,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std
 
 	//stream
 	if stream {
-		if container.State.IsGhost() {
+		if container.State.Ghost {
 			return fmt.Errorf("Impossible to attach to a ghost container")
 		}
 
@@ -1837,7 +1601,7 @@ func (srv *Server) HTTPRequestFactory(metaHeaders map[string][]string) *utils.HT
 	return srv.reqFactory
 }
 
-func (srv *Server) LogEvent(action, id, from string) *utils.JSONMessage {
+func (srv *Server) LogEvent(action, id, from string) {
 	now := time.Now().Unix()
 	jm := utils.JSONMessage{Status: action, ID: id, From: from, Time: now}
 	srv.events = append(srv.events, jm)
@@ -1847,7 +1611,6 @@ func (srv *Server) LogEvent(action, id, from string) *utils.JSONMessage {
 		default:
 		}
 	}
-	return &jm
 }
 
 type Server struct {

server_test.go

@@ -1,31 +1,32 @@
 package docker
 
 import (
-	"github.com/dotcloud/docker"
 	"github.com/dotcloud/docker/utils"
 	"io/ioutil"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestContainerTagImageDelete(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
 
-	srv := mkServerFromEngine(eng, t)
+	srv := &Server{runtime: runtime}
 
 	initialImages, err := srv.Images(false, "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := srv.ContainerTag(unitTestImageName, "utest", "tag1", false); err != nil {
+
+	if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := srv.ContainerTag(unitTestImageName, "utest/docker", "tag2", false); err != nil {
+	if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
-	if err := srv.ContainerTag(unitTestImageName, "utest:5000/docker", "tag3", false); err != nil {
+	if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -81,43 +82,46 @@ func TestContainerTagImageDelete(t *testing.T) {
 func TestCreateRm(t *testing.T) {
 	eng := NewTestEngine(t)
 	srv := mkServerFromEngine(eng, t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	runtime := srv.runtime
+	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	id := createTestContainer(eng, config, t)
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 1 {
-		t.Errorf("Expected 1 container, %v found", len(c))
+	if len(runtime.List()) != 1 {
+		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
 	}
 
 	if err = srv.ContainerDestroy(id, true, false); err != nil {
 		t.Fatal(err)
 	}
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 0 {
-		t.Errorf("Expected 0 container, %v found", len(c))
+	if len(runtime.List()) != 0 {
+		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
 	}
 
 }
 
 func TestCreateRmVolumes(t *testing.T) {
 	eng := NewTestEngine(t)
-	srv := mkServerFromEngine(eng, t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
 
-	config, hostConfig, _, err := docker.ParseRun([]string{"-v", "/srv", unitTestImageID, "echo", "test"}, nil)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
+	defer nuke(runtime)
+
+	config, hostConfig, _, err := ParseRun([]string{"-v", "/srv", GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	id := createTestContainer(eng, config, t)
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 1 {
-		t.Errorf("Expected 1 container, %v found", len(c))
+	if len(runtime.List()) != 1 {
+		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
 	}
 
 	job := eng.Job("start", id)
@@ -137,17 +141,18 @@ func TestCreateRmVolumes(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 0 {
-		t.Errorf("Expected 0 container, %v found", len(c))
+	if len(runtime.List()) != 0 {
+		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
 	}
 }
 
 func TestCommit(t *testing.T) {
 	eng := NewTestEngine(t)
 	srv := mkServerFromEngine(eng, t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	runtime := srv.runtime
+	defer nuke(runtime)
 
-	config, _, _, err := docker.ParseRun([]string{unitTestImageID, "/bin/cat"}, nil)
+	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -162,17 +167,18 @@ func TestCommit(t *testing.T) {
 func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 	eng := NewTestEngine(t)
 	srv := mkServerFromEngine(eng, t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	runtime := srv.runtime
+	defer nuke(runtime)
 
-	config, hostConfig, _, err := docker.ParseRun([]string{"-i", unitTestImageID, "/bin/cat"}, nil)
+	config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	id := createTestContainer(eng, config, t)
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 1 {
-		t.Errorf("Expected 1 container, %v found", len(c))
+	if len(runtime.List()) != 1 {
+		t.Errorf("Expected 1 container, %v found", len(runtime.List()))
 	}
 
 	job := eng.Job("start", id)
@@ -208,18 +214,21 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if c := srv.Containers(true, false, -1, "", ""); len(c) != 0 {
-		t.Errorf("Expected 0 container, %v found", len(c))
+	if len(runtime.List()) != 0 {
+		t.Errorf("Expected 0 container, %v found", len(runtime.List()))
 	}
+
 }
 
 func TestRunWithTooLowMemoryLimit(t *testing.T) {
 	eng := NewTestEngine(t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
+	defer nuke(runtime)
 
 	// Try to create a container with a memory limit of 1 byte less than the minimum allowed limit.
 	job := eng.Job("create")
-	job.Setenv("Image", unitTestImageID)
+	job.Setenv("Image", GetTestImage(runtime).ID)
 	job.Setenv("Memory", "524287")
 	job.Setenv("CpuShares", "1000")
 	job.SetenvList("Cmd", []string{"/bin/cat"})
@@ -230,17 +239,163 @@ func TestRunWithTooLowMemoryLimit(t *testing.T) {
 	}
 }
 
+func TestContainerTop(t *testing.T) {
+	t.Skip("Fixme. Skipping test for now. Reported error: 'server_test.go:236: Expected 2 processes, found 1.'")
+
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	srv := &Server{runtime: runtime}
+
+	c, _ := mkContainer(runtime, []string{"_", "/bin/sh", "-c", "sleep 2"}, t)
+	c, err := mkContainer(runtime, []string{"_", "/bin/sh", "-c", "sleep 2"}, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer runtime.Destroy(c)
+	if err := c.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Give some time to the process to start
+	c.WaitTimeout(500 * time.Millisecond)
+
+	if !c.State.Running {
+		t.Errorf("Container should be running")
+	}
+	procs, err := srv.ContainerTop(c.ID, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(procs.Processes) != 2 {
+		t.Fatalf("Expected 2 processes, found %d.", len(procs.Processes))
+	}
+
+	pos := -1
+	for i := 0; i < len(procs.Titles); i++ {
+		if procs.Titles[i] == "CMD" {
+			pos = i
+			break
+		}
+	}
+
+	if pos == -1 {
+		t.Fatalf("Expected CMD, not found.")
+	}
+
+	if procs.Processes[0][pos] != "sh" && procs.Processes[0][pos] != "busybox" {
+		t.Fatalf("Expected `busybox` or `sh`, found %s.", procs.Processes[0][pos])
+	}
+
+	if procs.Processes[1][pos] != "sh" && procs.Processes[1][pos] != "busybox" {
+		t.Fatalf("Expected `busybox` or `sh`, found %s.", procs.Processes[1][pos])
+	}
+}
+
+func TestPools(t *testing.T) {
+	runtime := mkRuntime(t)
+	srv := &Server{
+		runtime:     runtime,
+		pullingPool: make(map[string]struct{}),
+		pushingPool: make(map[string]struct{}),
+	}
+	defer nuke(runtime)
+
+	err := srv.poolAdd("pull", "test1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolAdd("pull", "test2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolAdd("push", "test1")
+	if err == nil || err.Error() != "pull test1 is already in progress" {
+		t.Fatalf("Expected `pull test1 is already in progress`")
+	}
+	err = srv.poolAdd("pull", "test1")
+	if err == nil || err.Error() != "pull test1 is already in progress" {
+		t.Fatalf("Expected `pull test1 is already in progress`")
+	}
+	err = srv.poolAdd("wait", "test3")
+	if err == nil || err.Error() != "Unknown pool type" {
+		t.Fatalf("Expected `Unknown pool type`")
+	}
+
+	err = srv.poolRemove("pull", "test2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolRemove("pull", "test2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolRemove("pull", "test1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolRemove("push", "test1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = srv.poolRemove("wait", "test3")
+	if err == nil || err.Error() != "Unknown pool type" {
+		t.Fatalf("Expected `Unknown pool type`")
+	}
+}
+
+func TestLogEvent(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{
+		runtime:   runtime,
+		events:    make([]utils.JSONMessage, 0, 64),
+		listeners: make(map[string]chan utils.JSONMessage),
+	}
+
+	srv.LogEvent("fakeaction", "fakeid", "fakeimage")
+
+	listener := make(chan utils.JSONMessage)
+	srv.Lock()
+	srv.listeners["test"] = listener
+	srv.Unlock()
+
+	srv.LogEvent("fakeaction2", "fakeid", "fakeimage")
+
+	if len(srv.events) != 2 {
+		t.Fatalf("Expected 2 events, found %d", len(srv.events))
+	}
+	go func() {
+		time.Sleep(200 * time.Millisecond)
+		srv.LogEvent("fakeaction3", "fakeid", "fakeimage")
+		time.Sleep(200 * time.Millisecond)
+		srv.LogEvent("fakeaction4", "fakeid", "fakeimage")
+	}()
+
+	setTimeout(t, "Listening for events timed out", 2*time.Second, func() {
+		for i := 2; i < 4; i++ {
+			event := <-listener
+			if event != srv.events[i] {
+				t.Fatalf("Event received it different than expected")
+			}
+		}
+	})
+}
+
 func TestRmi(t *testing.T) {
 	eng := NewTestEngine(t)
 	srv := mkServerFromEngine(eng, t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
+	runtime := srv.runtime
+	defer nuke(runtime)
 
 	initialImages, err := srv.Images(false, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	config, hostConfig, _, err := docker.ParseRun([]string{unitTestImageID, "echo", "test"}, nil)
+	config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -256,10 +411,6 @@ func TestRmi(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := srv.ContainerWait(containerID); err != nil {
-		t.Fatal(err)
-	}
-
 	imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil)
 	if err != nil {
 		t.Fatal(err)
@@ -281,10 +432,6 @@ func TestRmi(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := srv.ContainerWait(containerID); err != nil {
-		t.Fatal(err)
-	}
-
 	_, err = srv.ContainerCommit(containerID, "test", "", "", "", nil)
 	if err != nil {
 		t.Fatal(err)
@@ -324,19 +471,19 @@ func TestRmi(t *testing.T) {
 }
 
 func TestImagesFilter(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer nuke(mkRuntimeFromEngine(eng, t))
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
 
-	srv := mkServerFromEngine(eng, t)
+	srv := &Server{runtime: runtime}
 
-	if err := srv.ContainerTag(unitTestImageName, "utest", "tag1", false); err != nil {
+	if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := srv.ContainerTag(unitTestImageName, "utest/docker", "tag2", false); err != nil {
+	if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
-	if err := srv.ContainerTag(unitTestImageName, "utest:5000/docker", "tag3", false); err != nil {
+	if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil {
 		t.Fatal(err)
 	}
 
@@ -378,9 +525,9 @@ func TestImagesFilter(t *testing.T) {
 }
 
 func TestImageInsert(t *testing.T) {
-	eng := NewTestEngine(t)
-	defer mkRuntimeFromEngine(eng, t).Nuke()
-	srv := mkServerFromEngine(eng, t)
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	srv := &Server{runtime: runtime}
 	sf := utils.NewStreamFormatter(true)
 
 	// bad image name fails
@@ -389,12 +536,12 @@ func TestImageInsert(t *testing.T) {
 	}
 
 	// bad url fails
-	if err := srv.ImageInsert(unitTestImageID, "http://bad_host_name_that_will_totally_fail.com/", "/foo", ioutil.Discard, sf); err == nil {
+	if err := srv.ImageInsert(GetTestImage(runtime).ID, "http://bad_host_name_that_will_totally_fail.com/", "/foo", ioutil.Discard, sf); err == nil {
 		t.Fatal("expected an error and got none")
 	}
 
 	// success returns nil
-	if err := srv.ImageInsert(unitTestImageID, "https://www.docker.io/static/img/docker-top-logo.png", "/foo", ioutil.Discard, sf); err != nil {
+	if err := srv.ImageInsert(GetTestImage(runtime).ID, "https://www.docker.io/static/img/docker-top-logo.png", "/foo", ioutil.Discard, sf); err != nil {
 		t.Fatalf("expected no error, but got %v", err)
 	}
 }

server_unit_test.go

@@ -1,109 +0,0 @@
-package docker
-
-import (
-	"github.com/dotcloud/docker/utils"
-	"testing"
-	"time"
-)
-
-func TestPools(t *testing.T) {
-	srv := &Server{
-		pullingPool: make(map[string]struct{}),
-		pushingPool: make(map[string]struct{}),
-	}
-
-	err := srv.poolAdd("pull", "test1")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolAdd("pull", "test2")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolAdd("push", "test1")
-	if err == nil || err.Error() != "pull test1 is already in progress" {
-		t.Fatalf("Expected `pull test1 is already in progress`")
-	}
-	err = srv.poolAdd("pull", "test1")
-	if err == nil || err.Error() != "pull test1 is already in progress" {
-		t.Fatalf("Expected `pull test1 is already in progress`")
-	}
-	err = srv.poolAdd("wait", "test3")
-	if err == nil || err.Error() != "Unknown pool type" {
-		t.Fatalf("Expected `Unknown pool type`")
-	}
-
-	err = srv.poolRemove("pull", "test2")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolRemove("pull", "test2")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolRemove("pull", "test1")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolRemove("push", "test1")
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = srv.poolRemove("wait", "test3")
-	if err == nil || err.Error() != "Unknown pool type" {
-		t.Fatalf("Expected `Unknown pool type`")
-	}
-}
-
-func TestLogEvent(t *testing.T) {
-	srv := &Server{
-		events:    make([]utils.JSONMessage, 0, 64),
-		listeners: make(map[string]chan utils.JSONMessage),
-	}
-
-	srv.LogEvent("fakeaction", "fakeid", "fakeimage")
-
-	listener := make(chan utils.JSONMessage)
-	srv.Lock()
-	srv.listeners["test"] = listener
-	srv.Unlock()
-
-	srv.LogEvent("fakeaction2", "fakeid", "fakeimage")
-
-	if len(srv.events) != 2 {
-		t.Fatalf("Expected 2 events, found %d", len(srv.events))
-	}
-	go func() {
-		time.Sleep(200 * time.Millisecond)
-		srv.LogEvent("fakeaction3", "fakeid", "fakeimage")
-		time.Sleep(200 * time.Millisecond)
-		srv.LogEvent("fakeaction4", "fakeid", "fakeimage")
-	}()
-
-	setTimeout(t, "Listening for events timed out", 2*time.Second, func() {
-		for i := 2; i < 4; i++ {
-			event := <-listener
-			if event != srv.events[i] {
-				t.Fatalf("Event received it different than expected")
-			}
-		}
-	})
-}
-
-// FIXME: this is duplicated from integration/commands_test.go
-func setTimeout(t *testing.T, msg string, d time.Duration, f func()) {
-	c := make(chan bool)
-
-	// Make sure we are not too long
-	go func() {
-		time.Sleep(d)
-		c <- true
-	}()
-	go func() {
-		f()
-		c <- false
-	}()
-	if <-c && msg != "" {
-		t.Fatal(msg)
-	}
-}

sorter_test.go

@@ -0,0 +1,111 @@
+package docker
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestServerListOrderedImagesByCreationDate(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	archive, err := fakeTar()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = runtime.graph.Create(archive, nil, "Testing", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srv := &Server{runtime: runtime}
+
+	images, err := srv.Images(true, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if images[0].Created < images[1].Created {
+		t.Error("Expected []APIImges to be ordered by most recent creation date.")
+	}
+}
+
+func TestServerListOrderedImagesByCreationDateAndTag(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	err := generateImage("bar", runtime)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Second)
+
+	err = generateImage("zed", runtime)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srv := &Server{runtime: runtime}
+	images, err := srv.Images(true, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if images[0].RepoTags[0] != "repo:zed" && images[0].RepoTags[0] != "repo:bar" {
+		t.Errorf("Expected []APIImges to be ordered by most recent creation date. %s", images)
+	}
+}
+
+func generateImage(name string, runtime *Runtime) error {
+
+	archive, err := fakeTar()
+	if err != nil {
+		return err
+	}
+	image, err := runtime.graph.Create(archive, nil, "Testing", "", nil)
+	if err != nil {
+		return err
+	}
+
+	srv := &Server{runtime: runtime}
+	srv.ContainerTag(image.ID, "repo", name, false)
+
+	return nil
+}
+
+func TestSortUniquePorts(t *testing.T) {
+	ports := []Port{
+		Port("6379/tcp"),
+		Port("22/tcp"),
+	}
+
+	sortPorts(ports, func(ip, jp Port) bool {
+		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
+	})
+
+	first := ports[0]
+	if fmt.Sprint(first) != "22/tcp" {
+		t.Log(fmt.Sprint(first))
+		t.Fail()
+	}
+}
+
+func TestSortSamePortWithDifferentProto(t *testing.T) {
+	ports := []Port{
+		Port("8888/tcp"),
+		Port("8888/udp"),
+		Port("6379/tcp"),
+		Port("6379/udp"),
+	}
+
+	sortPorts(ports, func(ip, jp Port) bool {
+		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
+	})
+
+	first := ports[0]
+	if fmt.Sprint(first) != "6379/tcp" {
+		t.Fail()
+	}
+}

sorter_unit_test.go

@@ -1,41 +0,0 @@
-package docker
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestSortUniquePorts(t *testing.T) {
-	ports := []Port{
-		Port("6379/tcp"),
-		Port("22/tcp"),
-	}
-
-	sortPorts(ports, func(ip, jp Port) bool {
-		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
-	})
-
-	first := ports[0]
-	if fmt.Sprint(first) != "22/tcp" {
-		t.Log(fmt.Sprint(first))
-		t.Fail()
-	}
-}
-
-func TestSortSamePortWithDifferentProto(t *testing.T) {
-	ports := []Port{
-		Port("8888/tcp"),
-		Port("8888/udp"),
-		Port("6379/tcp"),
-		Port("6379/udp"),
-	}
-
-	sortPorts(ports, func(ip, jp Port) bool {
-		return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp")
-	})
-
-	first := ports[0]
-	if fmt.Sprint(first) != "6379/tcp" {
-		t.Fail()
-	}
-}

state.go

@@ -8,7 +8,7 @@ import (
 )
 
 type State struct {
-	sync.RWMutex
+	sync.Mutex
 	Running    bool
 	Pid        int
 	ExitCode   int
@@ -19,9 +19,6 @@ type State struct {
 
 // String returns a human-readable description of the state
 func (s *State) String() string {
-	s.RLock()
-	defer s.RUnlock()
-
 	if s.Running {
 		if s.Ghost {
 			return fmt.Sprintf("Ghost")
@@ -31,38 +28,7 @@ func (s *State) String() string {
 	return fmt.Sprintf("Exit %d", s.ExitCode)
 }
 
-func (s *State) IsRunning() bool {
-	s.RLock()
-	defer s.RUnlock()
-
-	return s.Running
-}
-
-func (s *State) IsGhost() bool {
-	s.RLock()
-	defer s.RUnlock()
-
-	return s.Ghost
-}
-
-func (s *State) GetExitCode() int {
-	s.RLock()
-	defer s.RUnlock()
-
-	return s.ExitCode
-}
-
-func (s *State) SetGhost(val bool) {
-	s.Lock()
-	defer s.Unlock()
-
-	s.Ghost = val
-}
-
-func (s *State) SetRunning(pid int) {
-	s.Lock()
-	defer s.Unlock()
-
+func (s *State) setRunning(pid int) {
 	s.Running = true
 	s.Ghost = false
 	s.ExitCode = 0
@@ -70,10 +36,7 @@ func (s *State) SetRunning(pid int) {
 	s.StartedAt = time.Now()
 }
 
-func (s *State) SetStopped(exitCode int) {
-	s.Lock()
-	defer s.Unlock()
-
+func (s *State) setStopped(exitCode int) {
 	s.Running = false
 	s.Pid = 0
 	s.FinishedAt = time.Now()

tags_test.go

@@ -0,0 +1,46 @@
+package docker
+
+import (
+	"testing"
+)
+
+func TestLookupImage(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + DEFAULTTAG); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + "fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := runtime.repositories.LookupImage("fail:fail"); err == nil {
+		t.Errorf("Expected error, none found")
+	} else if img != nil {
+		t.Errorf("Expected 0 image, 1 found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageID); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+
+	if img, err := runtime.repositories.LookupImage(unitTestImageName + ":" + unitTestImageID); err != nil {
+		t.Fatal(err)
+	} else if img == nil {
+		t.Errorf("Expected 1 image, none found")
+	}
+}

tags_unit_test.go

@@ -1,80 +0,0 @@
-package docker
-
-import (
-	"github.com/dotcloud/docker/utils"
-	"os"
-	"path"
-	"testing"
-)
-
-const (
-	testImageName string = "myapp"
-	testImageID   string = "foo"
-)
-
-func mkTestTagStore(root string, t *testing.T) *TagStore {
-	graph, err := NewGraph(root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	store, err := NewTagStore(path.Join(root, "tags"), graph)
-	if err != nil {
-		t.Fatal(err)
-	}
-	archive, err := fakeTar()
-	if err != nil {
-		t.Fatal(err)
-	}
-	img := &Image{ID: testImageID}
-	if err := graph.Register(nil, archive, img); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.Set(testImageName, "", testImageID, false); err != nil {
-		t.Fatal(err)
-	}
-	return store
-}
-
-func TestLookupImage(t *testing.T) {
-	tmp, err := utils.TestDirectory("")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-	store := mkTestTagStore(tmp, t)
-
-	if img, err := store.LookupImage(testImageName); err != nil {
-		t.Fatal(err)
-	} else if img == nil {
-		t.Errorf("Expected 1 image, none found")
-	}
-	if img, err := store.LookupImage(testImageName + ":" + DEFAULTTAG); err != nil {
-		t.Fatal(err)
-	} else if img == nil {
-		t.Errorf("Expected 1 image, none found")
-	}
-
-	if img, err := store.LookupImage(testImageName + ":" + "fail"); err == nil {
-		t.Errorf("Expected error, none found")
-	} else if img != nil {
-		t.Errorf("Expected 0 image, 1 found")
-	}
-
-	if img, err := store.LookupImage("fail:fail"); err == nil {
-		t.Errorf("Expected error, none found")
-	} else if img != nil {
-		t.Errorf("Expected 0 image, 1 found")
-	}
-
-	if img, err := store.LookupImage(testImageID); err != nil {
-		t.Fatal(err)
-	} else if img == nil {
-		t.Errorf("Expected 1 image, none found")
-	}
-
-	if img, err := store.LookupImage(testImageName + ":" + testImageID); err != nil {
-		t.Fatal(err)
-	} else if img == nil {
-		t.Errorf("Expected 1 image, none found")
-	}
-}

utils.go

@@ -1,14 +1,43 @@
 package docker
 
+/*
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+
+// See linux.git/fs/btrfs/ioctl.h
+#define BTRFS_IOCTL_MAGIC 0x94
+#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int)
+
+int
+btrfs_reflink(int fd_out, int fd_in)
+{
+  int res;
+  res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in);
+  if (res < 0)
+    return errno;
+  return 0;
+}
+
+*/
+import "C"
 import (
 	"fmt"
+	"github.com/dotcloud/docker/archive"
 	"github.com/dotcloud/docker/namesgenerator"
 	"github.com/dotcloud/docker/utils"
+	"io"
 	"io/ioutil"
+	"os"
 	"strconv"
 	"strings"
+	"syscall"
 )
 
+type Change struct {
+	archive.Change
+}
+
 // Compare two Config struct. Do not compare the "Image" nor "Hostname" fields
 // If OpenStdin is set, then it differs
 func CompareConfig(a, b *Config) bool {
@@ -305,6 +334,14 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error {
 	return nil
 }
 
+func BtrfsReflink(fd_out, fd_in uintptr) error {
+	res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in))
+	if res != 0 {
+		return syscall.Errno(res)
+	}
+	return nil
+}
+
 // Links come in the format of
 // name:alias
 func parseLink(rawLink string) (map[string]string, error) {
@@ -337,3 +374,14 @@ func (c *checker) Exists(name string) bool {
 func generateRandomName(runtime *Runtime) (string, error) {
 	return namesgenerator.GenerateRandomName(&checker{runtime})
 }
+
+func CopyFile(dstFile, srcFile *os.File) error {
+	err := BtrfsReflink(dstFile.Fd(), srcFile.Fd())
+	if err == nil {
+		return nil
+	}
+
+	// Fall back to normal copy
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}

utils/fs.go

@@ -0,0 +1,19 @@
+package utils
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// TreeSize walks a directory tree and returns its total size in bytes.
+func TreeSize(dir string) (size int64, err error) {
+	err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, e error) error {
+		// Ignore directory sizes
+		if fileInfo.IsDir() {
+			return nil
+		}
+		size += fileInfo.Size()
+		return nil
+	})
+	return
+}

utils/http.go

@@ -81,12 +81,12 @@ func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator {
 	return ret
 }
 
-func (h *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) {
+func (self *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) {
 	if req == nil {
 		return req, nil
 	}
 
-	userAgent := appendVersions(req.UserAgent(), h.versions...)
+	userAgent := appendVersions(req.UserAgent(), self.versions...)
 	if len(userAgent) > 0 {
 		req.Header.Set("User-Agent", userAgent)
 	}
@@ -97,11 +97,11 @@ type HTTPMetaHeadersDecorator struct {
 	Headers map[string][]string
 }
 
-func (h *HTTPMetaHeadersDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) {
-	if h.Headers == nil {
+func (self *HTTPMetaHeadersDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) {
+	if self.Headers == nil {
 		return req, nil
 	}
-	for k, v := range h.Headers {
+	for k, v := range self.Headers {
 		req.Header[k] = v
 	}
 	return req, nil
@@ -114,25 +114,25 @@ type HTTPRequestFactory struct {
 }
 
 func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory {
-	return &HTTPRequestFactory{
-		decorators: d,
-	}
+	ret := new(HTTPRequestFactory)
+	ret.decorators = d
+	return ret
 }
 
 // NewRequest() creates a new *http.Request,
 // applies all decorators in the HTTPRequestFactory on the request,
 // then applies decorators provided by d on the request.
-func (h *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) {
+func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) {
 	req, err := http.NewRequest(method, urlStr, body)
 	if err != nil {
 		return nil, err
 	}
 
 	// By default, a nil factory should work.
-	if h == nil {
+	if self == nil {
 		return req, nil
 	}
-	for _, dec := range h.decorators {
+	for _, dec := range self.decorators {
 		req, err = dec.ChangeRequest(req)
 		if err != nil {
 			return nil, err

utils/utils.go

@@ -1123,7 +1123,7 @@ func (graph *DependencyGraph) GenerateTraversalMap() ([][]string, error) {
 	for len(processed) < len(graph.nodes) {
 		// Use a temporary buffer for processed nodes, otherwise
 		// nodes that depend on each other could end up in the same round.
-		tmpProcessed := []*DependencyNode{}
+		tmp_processed := []*DependencyNode{}
 		for _, node := range graph.nodes {
 			// If the node has more dependencies than what we have cleared,
 			// it won't be valid for this round.
@@ -1137,7 +1137,7 @@ func (graph *DependencyGraph) GenerateTraversalMap() ([][]string, error) {
 			// It's not been processed yet and has 0 deps. Add it!
 			// (this is a shortcut for what we're doing below)
 			if node.Degree() == 0 {
-				tmpProcessed = append(tmpProcessed, node)
+				tmp_processed = append(tmp_processed, node)
 				continue
 			}
 			// If at least one dep hasn't been processed yet, we can't
@@ -1151,17 +1151,17 @@ func (graph *DependencyGraph) GenerateTraversalMap() ([][]string, error) {
 			}
 			// All deps have already been processed. Add it!
 			if ok {
-				tmpProcessed = append(tmpProcessed, node)
+				tmp_processed = append(tmp_processed, node)
 			}
 		}
-		Debugf("Round %d: found %d available nodes", len(result), len(tmpProcessed))
+		Debugf("Round %d: found %d available nodes", len(result), len(tmp_processed))
 		// If no progress has been made this round,
 		// that means we have circular dependencies.
-		if len(tmpProcessed) == 0 {
+		if len(tmp_processed) == 0 {
 			return nil, fmt.Errorf("Could not find a solution to this dependency graph")
 		}
 		round := []string{}
-		for _, nd := range tmpProcessed {
+		for _, nd := range tmp_processed {
 			round = append(round, nd.id)
 			processed[nd] = true
 		}
@@ -1242,40 +1242,3 @@ func PartParser(template, data string) (map[string]string, error) {
 	}
 	return out, nil
 }
-
-var globalTestID string
-
-// TestDirectory creates a new temporary directory and returns its path.
-// The contents of directory at path `templateDir` is copied into the
-// new directory.
-func TestDirectory(templateDir string) (dir string, err error) {
-	if globalTestID == "" {
-		globalTestID = RandomString()[:4]
-	}
-	prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, GetCallerName(2))
-	if prefix == "" {
-		prefix = "docker-test-"
-	}
-	dir, err = ioutil.TempDir("", prefix)
-	if err = os.Remove(dir); err != nil {
-		return
-	}
-	if templateDir != "" {
-		if err = CopyDirectory(templateDir, dir); err != nil {
-			return
-		}
-	}
-	return
-}
-
-// GetCallerName introspects the call stack and returns the name of the
-// function `depth` levels down in the stack.
-func GetCallerName(depth int) string {
-	// Use the caller function name as a prefix.
-	// This helps trace temp directories back to their test.
-	pc, _, _, _ := runtime.Caller(depth + 1)
-	callerLongName := runtime.FuncForPC(pc).Name()
-	parts := strings.Split(callerLongName, ".")
-	callerShortName := parts[len(parts)-1]
-	return callerShortName
-}

utils_test.go

@@ -0,0 +1,493 @@
+package docker
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/engine"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// This file contains utility functions for docker's unit test suite.
+// It has to be named XXX_test.go, apparently, in other to access private functions
+// from other XXX_test.go functions.
+
+var globalTestID string
+
+// Create a temporary runtime suitable for unit testing.
+// Call t.Fatal() at the first error.
+func mkRuntime(f utils.Fataler) *Runtime {
+	root, err := newTestDirectory(unitTestStoreBase)
+	if err != nil {
+		f.Fatal(err)
+	}
+	config := &DaemonConfig{
+		Root:        root,
+		AutoRestart: false,
+	}
+	r, err := NewRuntimeFromDirectory(config)
+	if err != nil {
+		f.Fatal(err)
+	}
+	r.UpdateCapabilities(true)
+	return r
+}
+
+func createNamedTestContainer(eng *engine.Engine, config *Config, f utils.Fataler, name string) (shortId string) {
+	job := eng.Job("create", name)
+	if err := job.ImportEnv(config); err != nil {
+		f.Fatal(err)
+	}
+	job.StdoutParseString(&shortId)
+	if err := job.Run(); err != nil {
+		f.Fatal(err)
+	}
+	return
+}
+
+func createTestContainer(eng *engine.Engine, config *Config, f utils.Fataler) (shortId string) {
+	return createNamedTestContainer(eng, config, f, "")
+}
+
+func mkServerFromEngine(eng *engine.Engine, t utils.Fataler) *Server {
+	iSrv := eng.Hack_GetGlobalVar("httpapi.server")
+	if iSrv == nil {
+		panic("Legacy server field not set in engine")
+	}
+	srv, ok := iSrv.(*Server)
+	if !ok {
+		panic("Legacy server field in engine does not cast to *Server")
+	}
+	return srv
+}
+
+func NewTestEngine(t utils.Fataler) *engine.Engine {
+	root, err := newTestDirectory(unitTestStoreBase)
+	if err != nil {
+		t.Fatal(err)
+	}
+	eng, err := engine.New(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Load default plugins
+	// (This is manually copied and modified from main() until we have a more generic plugin system)
+	job := eng.Job("initapi")
+	job.Setenv("Root", root)
+	job.SetenvBool("AutoRestart", false)
+	if err := job.Run(); err != nil {
+		t.Fatal(err)
+	}
+	return eng
+}
+
+func newTestDirectory(templateDir string) (dir string, err error) {
+	if globalTestID == "" {
+		globalTestID = GenerateID()[:4]
+	}
+	prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, getCallerName(2))
+	if prefix == "" {
+		prefix = "docker-test-"
+	}
+	dir, err = ioutil.TempDir("", prefix)
+	if err = os.Remove(dir); err != nil {
+		return
+	}
+	if err = utils.CopyDirectory(templateDir, dir); err != nil {
+		return
+	}
+	return
+}
+
+func getCallerName(depth int) string {
+	// Use the caller function name as a prefix.
+	// This helps trace temp directories back to their test.
+	pc, _, _, _ := runtime.Caller(depth + 1)
+	callerLongName := runtime.FuncForPC(pc).Name()
+	parts := strings.Split(callerLongName, ".")
+	callerShortName := parts[len(parts)-1]
+	return callerShortName
+}
+
+// Write `content` to the file at path `dst`, creating it if necessary,
+// as well as any missing directories.
+// The file is truncated if it already exists.
+// Call t.Fatal() at the first error.
+func writeFile(dst, content string, t *testing.T) {
+	// Create subdirectories if necessary
+	if err := os.MkdirAll(path.Dir(dst), 0700); err != nil && !os.IsExist(err) {
+		t.Fatal(err)
+	}
+	f, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Write content (truncate if it exists)
+	if _, err := io.Copy(f, strings.NewReader(content)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Return the contents of file at path `src`.
+// Call t.Fatal() at the first error (including if the file doesn't exist)
+func readFile(src string, t *testing.T) (content string) {
+	f, err := os.Open(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(data)
+}
+
+// Create a test container from the given runtime `r` and run arguments `args`.
+// If the image name is "_", (eg. []string{"-i", "-t", "_", "bash"}, it is
+// dynamically replaced by the current test image.
+// The caller is responsible for destroying the container.
+// Call t.Fatal() at the first error.
+func mkContainer(r *Runtime, args []string, t *testing.T) (*Container, error) {
+	config, hostConfig, _, err := ParseRun(args, nil)
+	defer func() {
+		if err != nil && t != nil {
+			t.Fatal(err)
+		}
+	}()
+	if err != nil {
+		return nil, err
+	}
+	if config.Image == "_" {
+		config.Image = GetTestImage(r).ID
+	}
+	c, _, err := r.Create(config, "")
+	if err != nil {
+		return nil, err
+	}
+	c.hostConfig = hostConfig
+	return c, nil
+}
+
+// Create a test container, start it, wait for it to complete, destroy it,
+// and return its standard output as a string.
+// The image name (eg. the XXX in []string{"-i", "-t", "XXX", "bash"}, is dynamically replaced by the current test image.
+// If t is not nil, call t.Fatal() at the first error. Otherwise return errors normally.
+func runContainer(r *Runtime, args []string, t *testing.T) (output string, err error) {
+	defer func() {
+		if err != nil && t != nil {
+			t.Fatal(err)
+		}
+	}()
+	container, err := mkContainer(r, args, t)
+	if err != nil {
+		return "", err
+	}
+	defer r.Destroy(container)
+	stdout, err := container.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+	defer stdout.Close()
+	if err := container.Start(); err != nil {
+		return "", err
+	}
+	container.Wait()
+	data, err := ioutil.ReadAll(stdout)
+	if err != nil {
+		return "", err
+	}
+	output = string(data)
+	return
+}
+
+func TestCompareConfig(t *testing.T) {
+	volumes1 := make(map[string]struct{})
+	volumes1["/test1"] = struct{}{}
+	config1 := Config{
+		Dns:         []string{"1.1.1.1", "2.2.2.2"},
+		PortSpecs:   []string{"1111:1111", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "11111111",
+		Volumes:     volumes1,
+	}
+	config2 := Config{
+		Dns:         []string{"0.0.0.0", "2.2.2.2"},
+		PortSpecs:   []string{"1111:1111", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "11111111",
+		Volumes:     volumes1,
+	}
+	config3 := Config{
+		Dns:         []string{"1.1.1.1", "2.2.2.2"},
+		PortSpecs:   []string{"0000:0000", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "11111111",
+		Volumes:     volumes1,
+	}
+	config4 := Config{
+		Dns:         []string{"1.1.1.1", "2.2.2.2"},
+		PortSpecs:   []string{"0000:0000", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "22222222",
+		Volumes:     volumes1,
+	}
+	volumes2 := make(map[string]struct{})
+	volumes2["/test2"] = struct{}{}
+	config5 := Config{
+		Dns:         []string{"1.1.1.1", "2.2.2.2"},
+		PortSpecs:   []string{"0000:0000", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "11111111",
+		Volumes:     volumes2,
+	}
+	if CompareConfig(&config1, &config2) {
+		t.Fatalf("CompareConfig should return false, Dns are different")
+	}
+	if CompareConfig(&config1, &config3) {
+		t.Fatalf("CompareConfig should return false, PortSpecs are different")
+	}
+	if CompareConfig(&config1, &config4) {
+		t.Fatalf("CompareConfig should return false, VolumesFrom are different")
+	}
+	if CompareConfig(&config1, &config5) {
+		t.Fatalf("CompareConfig should return false, Volumes are different")
+	}
+	if !CompareConfig(&config1, &config1) {
+		t.Fatalf("CompareConfig should return true")
+	}
+}
+
+func TestMergeConfig(t *testing.T) {
+	volumesImage := make(map[string]struct{})
+	volumesImage["/test1"] = struct{}{}
+	volumesImage["/test2"] = struct{}{}
+	configImage := &Config{
+		Dns:         []string{"1.1.1.1", "2.2.2.2"},
+		PortSpecs:   []string{"1111:1111", "2222:2222"},
+		Env:         []string{"VAR1=1", "VAR2=2"},
+		VolumesFrom: "1111",
+		Volumes:     volumesImage,
+	}
+
+	volumesUser := make(map[string]struct{})
+	volumesUser["/test3"] = struct{}{}
+	configUser := &Config{
+		Dns:       []string{"3.3.3.3"},
+		PortSpecs: []string{"3333:2222", "3333:3333"},
+		Env:       []string{"VAR2=3", "VAR3=3"},
+		Volumes:   volumesUser,
+	}
+
+	if err := MergeConfig(configUser, configImage); err != nil {
+		t.Error(err)
+	}
+
+	if len(configUser.Dns) != 3 {
+		t.Fatalf("Expected 3 dns, 1.1.1.1, 2.2.2.2 and 3.3.3.3, found %d", len(configUser.Dns))
+	}
+	for _, dns := range configUser.Dns {
+		if dns != "1.1.1.1" && dns != "2.2.2.2" && dns != "3.3.3.3" {
+			t.Fatalf("Expected 1.1.1.1 or 2.2.2.2 or 3.3.3.3, found %s", dns)
+		}
+	}
+
+	if len(configUser.ExposedPorts) != 3 {
+		t.Fatalf("Expected 3 ExposedPorts, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts))
+	}
+	for portSpecs := range configUser.ExposedPorts {
+		if portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" {
+			t.Fatalf("Expected 1111 or 2222 or 3333, found %s", portSpecs)
+		}
+	}
+	if len(configUser.Env) != 3 {
+		t.Fatalf("Expected 3 env var, VAR1=1, VAR2=3 and VAR3=3, found %d", len(configUser.Env))
+	}
+	for _, env := range configUser.Env {
+		if env != "VAR1=1" && env != "VAR2=3" && env != "VAR3=3" {
+			t.Fatalf("Expected VAR1=1 or VAR2=3 or VAR3=3, found %s", env)
+		}
+	}
+
+	if len(configUser.Volumes) != 3 {
+		t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes))
+	}
+	for v := range configUser.Volumes {
+		if v != "/test1" && v != "/test2" && v != "/test3" {
+			t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v)
+		}
+	}
+
+	if configUser.VolumesFrom != "1111" {
+		t.Fatalf("Expected VolumesFrom to be 1111, found %s", configUser.VolumesFrom)
+	}
+
+	ports, _, err := parsePortSpecs([]string{"0000"})
+	if err != nil {
+		t.Error(err)
+	}
+	configImage2 := &Config{
+		ExposedPorts: ports,
+	}
+
+	if err := MergeConfig(configUser, configImage2); err != nil {
+		t.Error(err)
+	}
+
+	if len(configUser.ExposedPorts) != 4 {
+		t.Fatalf("Expected 4 ExposedPorts, 0000, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts))
+	}
+	for portSpecs := range configUser.ExposedPorts {
+		if portSpecs.Port() != "0000" && portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" {
+			t.Fatalf("Expected 0000 or 1111 or 2222 or 3333, found %s", portSpecs)
+		}
+	}
+
+}
+
+func TestParseLxcConfOpt(t *testing.T) {
+	opts := []string{"lxc.utsname=docker", "lxc.utsname = docker "}
+
+	for _, o := range opts {
+		k, v, err := parseLxcOpt(o)
+		if err != nil {
+			t.FailNow()
+		}
+		if k != "lxc.utsname" {
+			t.Fail()
+		}
+		if v != "docker" {
+			t.Fail()
+		}
+	}
+}
+
+func TestParseNetworkOptsPrivateOnly(t *testing.T) {
+	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::80"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ports) != 1 {
+		t.Logf("Expected 1 got %d", len(ports))
+		t.FailNow()
+	}
+	if len(bindings) != 1 {
+		t.Logf("Expected 1 got %d", len(bindings))
+		t.FailNow()
+	}
+	for k := range ports {
+		if k.Proto() != "tcp" {
+			t.Logf("Expected tcp got %s", k.Proto())
+			t.Fail()
+		}
+		if k.Port() != "80" {
+			t.Logf("Expected 80 got %s", k.Port())
+			t.Fail()
+		}
+		b, exists := bindings[k]
+		if !exists {
+			t.Log("Binding does not exist")
+			t.FailNow()
+		}
+		if len(b) != 1 {
+			t.Logf("Expected 1 got %d", len(b))
+			t.FailNow()
+		}
+		s := b[0]
+		if s.HostPort != "" {
+			t.Logf("Expected \"\" got %s", s.HostPort)
+			t.Fail()
+		}
+		if s.HostIp != "192.168.1.100" {
+			t.Fail()
+		}
+	}
+}
+
+func TestParseNetworkOptsPublic(t *testing.T) {
+	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100:8080:80"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ports) != 1 {
+		t.Logf("Expected 1 got %d", len(ports))
+		t.FailNow()
+	}
+	if len(bindings) != 1 {
+		t.Logf("Expected 1 got %d", len(bindings))
+		t.FailNow()
+	}
+	for k := range ports {
+		if k.Proto() != "tcp" {
+			t.Logf("Expected tcp got %s", k.Proto())
+			t.Fail()
+		}
+		if k.Port() != "80" {
+			t.Logf("Expected 80 got %s", k.Port())
+			t.Fail()
+		}
+		b, exists := bindings[k]
+		if !exists {
+			t.Log("Binding does not exist")
+			t.FailNow()
+		}
+		if len(b) != 1 {
+			t.Logf("Expected 1 got %d", len(b))
+			t.FailNow()
+		}
+		s := b[0]
+		if s.HostPort != "8080" {
+			t.Logf("Expected 8080 got %s", s.HostPort)
+			t.Fail()
+		}
+		if s.HostIp != "192.168.1.100" {
+			t.Fail()
+		}
+	}
+}
+
+func TestParseNetworkOptsUdp(t *testing.T) {
+	ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::6000/udp"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ports) != 1 {
+		t.Logf("Expected 1 got %d", len(ports))
+		t.FailNow()
+	}
+	if len(bindings) != 1 {
+		t.Logf("Expected 1 got %d", len(bindings))
+		t.FailNow()
+	}
+	for k := range ports {
+		if k.Proto() != "udp" {
+			t.Logf("Expected udp got %s", k.Proto())
+			t.Fail()
+		}
+		if k.Port() != "6000" {
+			t.Logf("Expected 6000 got %s", k.Port())
+			t.Fail()
+		}
+		b, exists := bindings[k]
+		if !exists {
+			t.Log("Binding does not exist")
+			t.FailNow()
+		}
+		if len(b) != 1 {
+			t.Logf("Expected 1 got %d", len(b))
+			t.FailNow()
+		}
+		s := b[0]
+		if s.HostPort != "" {
+			t.Logf("Expected \"\" got %s", s.HostPort)
+			t.Fail()
+		}
+		if s.HostIp != "192.168.1.100" {
+			t.Fail()
+		}
+	}
+}