Merge pull request #32 from shykes/shykes-0.6.5-dm-plugin

Merge bug fix from dotcloud/docker master

.gitignore

@@ -18,3 +18,4 @@ bundles/
 .hg/
 .git/
 vendor/pkg/
+pyenv

AUTHORS

@@ -94,6 +94,7 @@ Jonathan Rudenberg <jonathan@titanous.com>
 Joost Cassee <joost@cassee.net>
 Jordan Arentsen <blissdev@gmail.com>
 Joseph Anthony Pasquale Holsten <joseph@josephholsten.com>
+Josh Poimboeuf <jpoimboe@redhat.com>
 Julien Barbier <write0@gmail.com>
 Jérôme Petazzoni <jerome.petazzoni@dotcloud.com>
 Karan Lyons <karan@karanlyons.com>
@@ -165,6 +166,7 @@ Sridatta Thatipamala <sthatipamala@gmail.com>
 Sridhar Ratnakumar <sridharr@activestate.com>
 Steeve Morin <steeve.morin@gmail.com>
 Stefan Praszalowicz <stefan@greplin.com>
+Sven Dowideit <SvenDowideit@home.org.au>
 Thatcher Peskens <thatcher@dotcloud.com>
 Thermionix <bond711@gmail.com>
 Thijs Terlouw <thijsterlouw@gmail.com>

CHANGELOG.md

@@ -17,7 +17,6 @@
 + Prevent DNS server conflicts in CreateBridgeIface
 + Validate bind mounts on the server side
 + Use parent image config in docker build
-* Fix regression in /etc/hosts
 
 #### Client
 

CONTRIBUTING.md

@@ -1,11 +1,14 @@
 # Contributing to Docker
 
-Want to hack on Docker? Awesome! Here are instructions to get you started. They are probably not perfect, please let us know if anything feels
-wrong or incomplete.
+Want to hack on Docker? Awesome! Here are instructions to get you
+started. They are probably not perfect, please let us know if anything
+feels wrong or incomplete.
 
 ## Build Environment
 
-For instructions on setting up your development environment, please see our dedicated [dev environment setup docs](http://docs.docker.io/en/latest/contributing/devenvironment/).
+For instructions on setting up your development environment, please
+see our dedicated [dev environment setup
+docs](http://docs.docker.io/en/latest/contributing/devenvironment/).
 
 ## Contribution guidelines
 

Dockerfile

@@ -23,7 +23,7 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-docker-version 0.6.1
+docker-version	0.6.1
 from	ubuntu:12.04
 maintainer	Solomon Hykes <solomon@dotcloud.com>
 
@@ -33,13 +33,13 @@ run	apt-get update
 run	apt-get install -y -q curl
 run	apt-get install -y -q git
 run	apt-get install -y -q mercurial
-run apt-get install -y -q build-essential libsqlite3-dev
+run	apt-get install -y -q build-essential libsqlite3-dev
 
 # Install Go
-run	curl -s https://go.googlecode.com/files/go1.2rc3.src.tar.gz | tar -v -C /usr/local -xz
+run	curl -s https://go.googlecode.com/files/go1.2rc4.src.tar.gz | tar -v -C /usr/local -xz
 env	PATH	/usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin
 env	GOPATH	/go:/go/src/github.com/dotcloud/docker/vendor
-run cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
+run	cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std
 
 # Ubuntu stuff
 run	apt-get install -y -q ruby1.9.3 rubygems libffi-dev
@@ -57,11 +57,20 @@ run	apt-get install -y -q iptables
 run	apt-get install -y -q lxc
 run	apt-get install -y -q aufs-tools
 
+# Get lvm2 source for compiling statically
+run	git clone git://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout v2_02_103
+# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
+# note: we can't use "git clone -b" above because it requires at least git 1.7.10 to be able to use that on a tag instead of a branch and we only have 1.7.9.5
+
+# Compile and install lvm2
+run	cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper
+# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
+
 volume	/var/lib/docker
 workdir	/go/src/github.com/dotcloud/docker
 
 # Wrap all commands in the "docker-in-docker" script to allow nested containers
-entrypoint ["hack/dind"]
+entrypoint	["hack/dind"]
 
 # Upload docker source
-add	.       /go/src/github.com/dotcloud/docker
+add	.	/go/src/github.com/dotcloud/docker

NOTICE

@@ -8,35 +8,12 @@ by Keith Rarick, licensed under the MIT License.
 
 The following is courtesy of our legal counsel:
 
-Transfers of Docker shall be in accordance with applicable export
-controls of any country and all other applicable legal requirements.
-Docker shall not be distributed or downloaded to or in Cuba, Iran,
-North Korea, Sudan or Syria and shall not be distributed or downloaded
-to any person on the Denied Persons List administered by the U.S.
-Department of Commerce.
 
-What does that mean? 
-Here is a further explanation from our legal counsel:
+Use and transfer of Docker may be subject to certain restrictions by the
+United States and other governments.  
+It is your responsibility to ensure that your use and/or transfer does not
+violate applicable laws. 
 
-Like all software products that utilize cryptography, the export and
-use of Docker is subject to the U.S. Commerce Department's Export
-Administration Regulations (EAR) because it uses or contains
-cryptography (see
-http://www.bis.doc.gov/index.php/policy-guidance/encryption).  Certain
-free and open source software projects have a lightweight set of
-requirements, which can generally be met by providing email notice to
-the appropriate U.S. government agencies that their source code is
-available on a publicly available repository and making the
-appropriate statements in the README.
+For more information, please see http://www.bis.doc.gov
 
-The restrictions of the EAR apply to certain denied locations
-(currently Iran, Sudan, Syria, North Korea, or Cuba) and those
-individuals on the Denied Persons List, which is available here:
-http://www.bis.doc.gov/index.php/policy-guidance/lists-of-parties-of-concern/denied-persons-list.
-If you are incorporating Docker into a new open source project, the
-EAR restrictions apply to your incorporation of Docker into your
-project in the same manner as other cryptography-enabled projects,
-such as OpenSSL, almost all Linux distributions, etc.
-
-For more information, see http://www.apache.org/dev/crypto.html and/or
-seek legal counsel.
+See also http://www.apache.org/dev/crypto.html and/or seek legal counsel.

README.md

@@ -193,10 +193,9 @@ wrong or incomplete.
 *Brought to you courtesy of our legal counsel. For more context,
 please see the Notice document.*
 
-Transfers of Docker shall be in accordance with applicable export controls 
-of any country and all other applicable legal requirements. Without limiting the 
-foregoing, Docker shall not be distributed or downloaded to any individual or 
-location if such distribution or download would violate the applicable US 
-government export regulations. 
+Use and transfer of Docker may be subject to certain restrictions by the
+United States and other governments.  
+It is your responsibility to ensure that your use and/or transfer does not
+violate applicable laws. 
 
 For more information, please see http://www.bis.doc.gov

VERSION

@@ -1 +1 @@
-0.6.6
+0.6.6-dev

Vagrantfile

@@ -4,65 +4,135 @@
 BOX_NAME = ENV['BOX_NAME'] || "ubuntu"
 BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box"
 VF_BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64_vmware_fusion.box"
+AWS_BOX_URI = ENV['BOX_URI'] || "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
 AWS_REGION = ENV['AWS_REGION'] || "us-east-1"
-AWS_AMI    = ENV['AWS_AMI']    || "ami-d0f89fb9"
+AWS_AMI = ENV['AWS_AMI'] || "ami-69f5a900"
+AWS_INSTANCE_TYPE = ENV['AWS_INSTANCE_TYPE'] || 't1.micro'
+
 FORWARD_DOCKER_PORTS = ENV['FORWARD_DOCKER_PORTS']
 
+SSH_PRIVKEY_PATH = ENV["SSH_PRIVKEY_PATH"]
+
+# A script to upgrade from the 12.04 kernel to the raring backport kernel (3.8)
+# and install docker.
+$script = <<SCRIPT
+# The username to add to the docker group will be passed as the first argument
+# to the script.  If nothing is passed, default to "vagrant".
+user="$1"
+if [ -z "$user" ]; then
+    user=vagrant
+fi
+
+# Adding an apt gpg key is idempotent.
+wget -q -O - https://get.docker.io/gpg | apt-key add -
+
+# Creating the docker.list file is idempotent, but it may overrite desired
+# settings if it already exists.  This could be solved with md5sum but it
+# doesn't seem worth it.
+echo 'deb http://get.docker.io/ubuntu docker main' > \
+    /etc/apt/sources.list.d/docker.list
+
+# Update remote package metadata.  'apt-get update' is idempotent.
+apt-get update -q
+
+# Install docker.  'apt-get install' is idempotent.
+apt-get install -q -y lxc-docker
+
+usermod -a -G docker "$user"
+
+tmp=`mktemp -q` && {
+    # Only install the backport kernel, don't bother upgrade if the backport is
+    # already installed.  We want parse the output of apt so we need to save it
+    # with 'tee'.  NOTE: The installation of the kernel will trigger dkms to
+    # install vboxguest if needed.
+    apt-get install -q -y --no-upgrade linux-image-generic-lts-raring | \
+        tee "$tmp"
+
+    # Parse the number of installed packages from the output
+    NUM_INST=`awk '$2 == "upgraded," && $4 == "newly" { print $3 }' "$tmp"`
+    rm "$tmp"
+}
+
+# If the number of installed packages is greater than 0, we want to reboot (the
+# backport kernel was installed but is not running).
+if [ "$NUM_INST" -gt 0 ];
+then
+    echo "Rebooting down to activate new kernel."
+    echo "/vagrant will not be mounted.  Use 'vagrant halt' followed by"
+    echo "'vagrant up' to ensure /vagrant is mounted."
+    shutdown -r now
+fi
+SCRIPT
+
+# We need to install the virtualbox guest additions *before* we do the normal
+# docker installation.  As such this script is prepended to the common docker
+# install script above.  This allows the install of the backport kernel to
+# trigger dkms to build the virtualbox guest module install.
+$vbox_script = <<VBOX_SCRIPT + $script
+# Install the VirtualBox guest additions if they aren't already installed.
+if [ ! -d /opt/VBoxGuestAdditions-4.2.12/ ]; then
+    # Update remote package metadata.  'apt-get update' is idempotent.
+    apt-get update -q
+
+    # Kernel Headers and dkms are required to build the vbox guest kernel
+    # modules.
+    apt-get install -q -y linux-headers-generic-lts-raring dkms
+
+    echo 'Downloading VBox Guest Additions...'
+    wget -cq http://dlc.sun.com.edgesuite.net/virtualbox/4.2.12/VBoxGuestAdditions_4.2.12.iso
+
+    mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.2.12.iso /mnt
+    /mnt/VBoxLinuxAdditions.run --nox11
+    umount /mnt
+fi
+VBOX_SCRIPT
+
 Vagrant::Config.run do |config|
   # Setup virtual machine box. This VM configuration code is always executed.
   config.vm.box = BOX_NAME
   config.vm.box_url = BOX_URI
 
-  config.ssh.forward_agent = true
-
-  # Provision docker and new kernel if deployment was not done.
-  # It is assumed Vagrant can successfully launch the provider instance.
-  if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
-    # Add lxc-docker package
-    pkg_cmd = "wget -q -O - https://get.docker.io/gpg | apt-key add -;" \
-      "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list;" \
-      "apt-get update -qq; apt-get install -q -y --force-yes lxc-docker; "
-    # Add Ubuntu raring backported kernel
-    pkg_cmd << "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; "
-    # Add guest additions if local vbox VM. As virtualbox is the default provider,
-    # it is assumed it won't be explicitly stated.
-    if ENV["VAGRANT_DEFAULT_PROVIDER"].nil? && ARGV.none? { |arg| arg.downcase.start_with?("--provider") }
-      pkg_cmd << "apt-get install -q -y linux-headers-generic-lts-raring dkms; " \
-        "echo 'Downloading VBox Guest Additions...'; " \
-        "wget -q http://dlc.sun.com.edgesuite.net/virtualbox/4.2.12/VBoxGuestAdditions_4.2.12.iso; "
-      # Prepare the VM to add guest additions after reboot
-      pkg_cmd << "echo -e 'mount -o loop,ro /home/vagrant/VBoxGuestAdditions_4.2.12.iso /mnt\n" \
-        "echo yes | /mnt/VBoxLinuxAdditions.run\numount /mnt\n" \
-          "rm /root/guest_additions.sh; ' > /root/guest_additions.sh; " \
-        "chmod 700 /root/guest_additions.sh; " \
-        "sed -i -E 's#^exit 0#[ -x /root/guest_additions.sh ] \\&\\& /root/guest_additions.sh#' /etc/rc.local; " \
-        "echo 'Installation of VBox Guest Additions is proceeding in the background.'; " \
-        "echo '\"vagrant reload\" can be used in about 2 minutes to activate the new guest additions.'; "
-    end
-    # Add vagrant user to the docker group
-    pkg_cmd << "usermod -a -G docker vagrant; "
-    # Activate new kernel
-    pkg_cmd << "shutdown -r +1; "
-    config.vm.provision :shell, :inline => pkg_cmd
+  # Use the specified private key path if it is specified and not empty.
+  if SSH_PRIVKEY_PATH
+      config.ssh.private_key_path = SSH_PRIVKEY_PATH
   end
+
+  config.ssh.forward_agent = true
 end
 
-
 # Providers were added on Vagrant >= 1.1.0
+#
+# NOTE: The vagrant "vm.provision" appends its arguments to a list and executes
+# them in order.  If you invoke "vm.provision :shell, :inline => $script"
+# twice then vagrant will run the script two times.  Unfortunately when you use
+# providers and the override argument to set up provisioners (like the vbox
+# guest extensions) they 1) don't replace the other provisioners (they append
+# to the end of the list) and 2) you can't control the order the provisioners
+# are executed (you can only append to the list).  If you want the virtualbox
+# only script to run before the other script, you have to jump through a lot of
+# hoops.
+#
+# Here is my only repeatable solution: make one script that is common ($script)
+# and another script that is the virtual box guest *prepended* to the common
+# script.  Only ever use "vm.provision" *one time* per provider.  That means
+# every single provider has an override, and every single one configures
+# "vm.provision".  Much saddness, but such is life.
 Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
   config.vm.provider :aws do |aws, override|
-    aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
-    aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+    username = "ubuntu"
+    override.vm.box_url = AWS_BOX_URI
+    override.vm.provision :shell, :inline => $script, :args => username
+    aws.access_key_id = ENV["AWS_ACCESS_KEY"]
+    aws.secret_access_key = ENV["AWS_SECRET_KEY"]
     aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
-    override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"]
-    override.ssh.username = "ubuntu"
+    override.ssh.username = username
     aws.region = AWS_REGION
     aws.ami    = AWS_AMI
-    aws.instance_type = "t1.micro"
+    aws.instance_type = AWS_INSTANCE_TYPE
   end
 
-  config.vm.provider :rackspace do |rs|
-    config.ssh.private_key_path = ENV["RS_PRIVATE_KEY"]
+  config.vm.provider :rackspace do |rs, override|
+    override.vm.provision :shell, :inline => $script
     rs.username = ENV["RS_USERNAME"]
     rs.api_key  = ENV["RS_API_KEY"]
     rs.public_key_path = ENV["RS_PUBLIC_KEY"]
@@ -71,20 +141,25 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
   end
 
   config.vm.provider :vmware_fusion do |f, override|
-    override.vm.box = BOX_NAME
     override.vm.box_url = VF_BOX_URI
     override.vm.synced_folder ".", "/vagrant", disabled: true
+    override.vm.provision :shell, :inline => $script
     f.vmx["displayName"] = "docker"
   end
 
-  config.vm.provider :virtualbox do |vb|
-    config.vm.box = BOX_NAME
-    config.vm.box_url = BOX_URI
+  config.vm.provider :virtualbox do |vb, override|
+    override.vm.provision :shell, :inline => $vbox_script
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
     vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
   end
 end
 
+# If this is a version 1 config, virtualbox is the only option.  A version 2
+# config would have already been set in the above provider section.
+Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config|
+  config.vm.provision :shell, :inline => $vbox_script
+end
+
 if !FORWARD_DOCKER_PORTS.nil?
   Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config|
     (49000..49900).each do |port|

api.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	APIVERSION        = 1.6
+	APIVERSION        = 1.7
 	DEFAULTHTTPHOST   = "127.0.0.1"
 	DEFAULTHTTPPORT   = 4243
 	DEFAULTUNIXSOCKET = "/var/run/docker.sock"
@@ -61,7 +61,10 @@ func parseMultipartForm(r *http.Request) error {
 
 func httpError(w http.ResponseWriter, err error) {
 	statusCode := http.StatusInternalServerError
-	if strings.HasPrefix(err.Error(), "No such") {
+	// FIXME: this is brittle and should not be necessary.
+	// If we need to differentiate between different possible error types, we should
+	// create appropriate error types with clearly defined meaning.
+	if strings.Contains(err.Error(), "No such") {
 		statusCode = http.StatusNotFound
 	} else if strings.HasPrefix(err.Error(), "Bad parameter") {
 		statusCode = http.StatusBadRequest
@@ -191,10 +194,24 @@ func getImagesJSON(srv *Server, version float64, w http.ResponseWriter, r *http.
 		return err
 	}
 
-	return writeJSON(w, http.StatusOK, outs)
+	if version < 1.7 {
+		outs2 := []APIImagesOld{}
+		for _, ctnr := range outs {
+			outs2 = append(outs2, ctnr.ToLegacy()...)
+		}
+
+		return writeJSON(w, http.StatusOK, outs2)
+	} else {
+		return writeJSON(w, http.StatusOK, outs)
+	}
 }
 
 func getImagesViz(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	if version > 1.6 {
+		w.WriteHeader(http.StatusNotFound)
+		return fmt.Errorf("This is now implemented in the client.")
+	}
+
 	if err := srv.ImagesViz(w); err != nil {
 		return err
 	}
@@ -465,15 +482,16 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht
 		w.Header().Set("Content-Type", "application/json")
 	}
 	sf := utils.NewStreamFormatter(version > 1.0)
-	imgID, err := srv.ImageInsert(name, url, path, w, sf)
+	err := srv.ImageInsert(name, url, path, w, sf)
 	if err != nil {
 		if sf.Used() {
 			w.Write(sf.FormatError(err))
 			return nil
 		}
+		return err
 	}
 
-	return writeJSON(w, http.StatusOK, &APIID{ID: imgID})
+	return nil
 }
 
 func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
@@ -526,43 +544,36 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r
 	if err := parseForm(r); err != nil {
 		return nil
 	}
-	config := &Config{}
 	out := &APIRun{}
-	name := r.Form.Get("name")
-
-	if err := json.NewDecoder(r.Body).Decode(config); err != nil {
+	job := srv.Eng.Job("create", r.Form.Get("name"))
+	if err := job.DecodeEnv(r.Body); err != nil {
 		return err
 	}
-
 	resolvConf, err := utils.GetResolvConf()
 	if err != nil {
 		return err
 	}
-
-	if !config.NetworkDisabled && len(config.Dns) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
+	if !job.GetenvBool("NetworkDisabled") && len(job.Getenv("Dns")) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) {
 		out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns))
-		config.Dns = defaultDns
+		job.SetenvList("Dns", defaultDns)
 	}
-
-	id, warnings, err := srv.ContainerCreate(config, name)
-	if err != nil {
+	// Read container ID from the first line of stdout
+	job.StdoutParseString(&out.ID)
+	// Read warnings from stderr
+	job.StderrParseLines(&out.Warnings, 0)
+	if err := job.Run(); err != nil {
 		return err
 	}
-	out.ID = id
-	for _, warning := range warnings {
-		out.Warnings = append(out.Warnings, warning)
-	}
-
-	if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit {
+	if job.GetenvInt("Memory") > 0 && !srv.runtime.capabilities.MemoryLimit {
 		log.Println("WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.")
 		out.Warnings = append(out.Warnings, "Your kernel does not support memory limit capabilities. Limitation discarded.")
 	}
-	if config.Memory > 0 && !srv.runtime.capabilities.SwapLimit {
+	if job.GetenvInt("Memory") > 0 && !srv.runtime.capabilities.SwapLimit {
 		log.Println("WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.")
 		out.Warnings = append(out.Warnings, "Your kernel does not support memory swap capabilities. Limitation discarded.")
 	}
 
-	if !config.NetworkDisabled && srv.runtime.capabilities.IPv4ForwardingDisabled {
+	if !job.GetenvBool("NetworkDisabled") && srv.runtime.capabilities.IPv4ForwardingDisabled {
 		log.Println("Warning: IPv4 forwarding is disabled.")
 		out.Warnings = append(out.Warnings, "IPv4 forwarding is disabled.")
 	}
@@ -639,26 +650,23 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R
 }
 
 func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	var hostConfig *HostConfig
-	// allow a nil body for backwards compatibility
-	if r.Body != nil {
-		if matchesContentType(r.Header.Get("Content-Type"), "application/json") {
-			hostConfig = &HostConfig{}
-			if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil {
-				return err
-			}
-		}
-	}
-
 	if vars == nil {
 		return fmt.Errorf("Missing parameter")
 	}
 	name := vars["name"]
-	// Register any links from the host config before starting the container
-	if err := srv.RegisterLinks(name, hostConfig); err != nil {
-		return err
+	job := srv.Eng.Job("start", name)
+	if err := job.ImportEnv(HostConfig{}); err != nil {
+		return fmt.Errorf("Couldn't initialize host configuration")
 	}
-	if err := srv.ContainerStart(name, hostConfig); err != nil {
+	// allow a nil body for backwards compatibility
+	if r.Body != nil {
+		if matchesContentType(r.Header.Get("Content-Type"), "application/json") {
+			if err := job.DecodeEnv(r.Body); err != nil {
+				return err
+			}
+		}
+	}
+	if err := job.Run(); err != nil {
 		return err
 	}
 	w.WriteHeader(http.StatusNoContent)

api_params.go

@@ -1,5 +1,7 @@
 package docker
 
+import "strings"
+
 type APIHistory struct {
 	ID        string   `json:"Id"`
 	Tags      []string `json:",omitempty"`
@@ -9,6 +11,15 @@ type APIHistory struct {
 }
 
 type APIImages struct {
+	ID          string   `json:"Id"`
+	RepoTags    []string `json:",omitempty"`
+	Created     int64
+	Size        int64
+	VirtualSize int64
+	ParentId    string `json:",omitempty"`
+}
+
+type APIImagesOld struct {
 	Repository  string `json:",omitempty"`
 	Tag         string `json:",omitempty"`
 	ID          string `json:"Id"`
@@ -17,6 +28,26 @@ type APIImages struct {
 	VirtualSize int64
 }
 
+func (self *APIImages) ToLegacy() []APIImagesOld {
+
+	outs := []APIImagesOld{}
+	for _, repotag := range self.RepoTags {
+
+		components := strings.SplitN(repotag, ":", 2)
+
+		outs = append(outs, APIImagesOld{
+			ID:          self.ID,
+			Repository:  components[0],
+			Tag:         components[1],
+			Created:     self.Created,
+			Size:        self.Size,
+			VirtualSize: self.VirtualSize,
+		})
+	}
+
+	return outs
+}
+
 type APIInfo struct {
 	Debug              bool
 	Containers         int

api_test.go

@@ -184,7 +184,7 @@ func TestGetImagesJSON(t *testing.T) {
 
 	found := false
 	for _, img := range images {
-		if img.Repository == unitTestImageName {
+		if strings.Contains(img.RepoTags[0], unitTestImageName) {
 			found = true
 			break
 		}
@@ -275,31 +275,6 @@ func TestGetImagesJSON(t *testing.T) {
 	}
 }
 
-func TestGetImagesViz(t *testing.T) {
-	runtime := mkRuntime(t)
-	defer nuke(runtime)
-
-	srv := &Server{runtime: runtime}
-
-	r := httptest.NewRecorder()
-	if err := getImagesViz(srv, APIVERSION, r, nil, nil); err != nil {
-		t.Fatal(err)
-	}
-
-	if r.Code != http.StatusOK {
-		t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code)
-	}
-
-	reader := bufio.NewReader(r.Body)
-	line, err := reader.ReadString('\n')
-	if err != nil {
-		t.Fatal(err)
-	}
-	if line != "digraph docker {\n" {
-		t.Errorf("Expected digraph docker {\n, %s found", line)
-	}
-}
-
 func TestGetImagesHistory(t *testing.T) {
 	runtime := mkRuntime(t)
 	defer nuke(runtime)
@@ -634,11 +609,11 @@ func TestPostCommit(t *testing.T) {
 }
 
 func TestPostContainersCreate(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime}
-
 	configJSON, err := json.Marshal(&Config{
 		Image:  GetTestImage(runtime).ID,
 		Memory: 33554432,
@@ -675,7 +650,7 @@ func TestPostContainersCreate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := os.Stat(path.Join(container.rwPath(), "test")); err != nil {
+	if _, err := os.Stat(path.Join(container.RootfsPath(), "test")); err != nil {
 		if os.IsNotExist(err) {
 			utils.Debugf("Err: %s", err)
 			t.Fatalf("The test file has not been created")
@@ -781,27 +756,23 @@ func TestPostContainersRestart(t *testing.T) {
 }
 
 func TestPostContainersStart(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
 
-	srv := &Server{runtime: runtime}
-
-	container, _, err := runtime.Create(
+	id := createTestContainer(
+		eng,
 		&Config{
 			Image:     GetTestImage(runtime).ID,
 			Cmd:       []string{"/bin/cat"},
 			OpenStdin: true,
 		},
-		"",
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer runtime.Destroy(container)
+		t)
 
 	hostConfigJSON, err := json.Marshal(&HostConfig{})
 
-	req, err := http.NewRequest("POST", "/containers/"+container.ID+"/start", bytes.NewReader(hostConfigJSON))
+	req, err := http.NewRequest("POST", "/containers/"+id+"/start", bytes.NewReader(hostConfigJSON))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -809,22 +780,26 @@ func TestPostContainersStart(t *testing.T) {
 	req.Header.Set("Content-Type", "application/json")
 
 	r := httptest.NewRecorder()
-	if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil {
+	if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": id}); err != nil {
 		t.Fatal(err)
 	}
 	if r.Code != http.StatusNoContent {
 		t.Fatalf("%d NO CONTENT expected, received %d\n", http.StatusNoContent, r.Code)
 	}
 
+	container := runtime.Get(id)
+	if container == nil {
+		t.Fatalf("Container %s was not created", id)
+	}
 	// Give some time to the process to start
+	// FIXME: use Wait once it's available as a job
 	container.WaitTimeout(500 * time.Millisecond)
-
 	if !container.State.Running {
 		t.Errorf("Container should be running")
 	}
 
 	r = httptest.NewRecorder()
-	if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err == nil {
+	if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": id}); err == nil {
 		t.Fatalf("A running container should be able to be started")
 	}
 
@@ -1138,7 +1113,7 @@ func TestDeleteContainers(t *testing.T) {
 		t.Fatalf("The container as not been deleted")
 	}
 
-	if _, err := os.Stat(path.Join(container.rwPath(), "test")); err == nil {
+	if _, err := os.Stat(path.Join(container.RootfsPath(), "test")); err == nil {
 		t.Fatalf("The test file has not been deleted")
 	}
 }
@@ -1226,7 +1201,7 @@ func TestDeleteImages(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(images) != len(initialImages)+1 {
+	if len(images[0].RepoTags) != len(initialImages[0].RepoTags)+1 {
 		t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images))
 	}
 
@@ -1265,17 +1240,11 @@ func TestDeleteImages(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if len(images) != len(initialImages) {
+	if len(images[0].RepoTags) != len(initialImages[0].RepoTags) {
 		t.Errorf("Expected %d image, %d found", len(initialImages), len(images))
 	}
 
-	/*	if c := runtime.Get(container.Id); c != nil {
-			t.Fatalf("The container as not been deleted")
-		}
-
-		if _, err := os.Stat(path.Join(container.rwPath(), "test")); err == nil {
-			t.Fatalf("The test file has not been deleted")
-		} */
+	// FIXME: check that container has been deleted, and its filesystem too
 }
 
 func TestJsonContentType(t *testing.T) {

archive/archive.go

@@ -15,7 +15,15 @@ import (
 
 type Archive io.Reader
 
-type Compression uint32
+type Compression int
+
+type TarOptions struct {
+	Includes    []string
+	Excludes    []string
+	Recursive   bool
+	Compression Compression
+	CreateFiles []string
+}
 
 const (
 	Uncompressed Compression = iota
@@ -80,20 +88,78 @@ func (compression *Compression) Extension() string {
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
 func Tar(path string, compression Compression) (io.Reader, error) {
-	return TarFilter(path, compression, nil)
+	return TarFilter(path, &TarOptions{Recursive: true, Compression: compression})
+}
+
+func escapeName(name string) string {
+	escaped := make([]byte, 0)
+	for i, c := range []byte(name) {
+		if i == 0 && c == '/' {
+			continue
+		}
+		// all printable chars except "-" which is 0x2d
+		if (0x20 <= c && c <= 0x7E) && c != 0x2d {
+			escaped = append(escaped, c)
+		} else {
+			escaped = append(escaped, fmt.Sprintf("\\%03o", c)...)
+		}
+	}
+	return string(escaped)
 }
 
 // Tar creates an archive from the directory at `path`, only including files whose relative
 // paths are included in `filter`. If `filter` is nil, then all files are included.
-func TarFilter(path string, compression Compression, filter []string) (io.Reader, error) {
-	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path}
-	if filter == nil {
-		filter = []string{"."}
+func TarFilter(path string, options *TarOptions) (io.Reader, error) {
+	args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"}
+	if options.Includes == nil {
+		options.Includes = []string{"."}
 	}
-	for _, f := range filter {
-		args = append(args, "-c"+compression.Flag(), f)
+	args = append(args, "-c"+options.Compression.Flag())
+
+	for _, exclude := range options.Excludes {
+		args = append(args, fmt.Sprintf("--exclude=%s", exclude))
 	}
-	return CmdStream(exec.Command(args[0], args[1:]...))
+
+	if !options.Recursive {
+		args = append(args, "--no-recursion")
+	}
+
+	files := ""
+	for _, f := range options.Includes {
+		files = files + escapeName(f) + "\n"
+	}
+
+	tmpDir := ""
+
+	if options.CreateFiles != nil {
+		var err error // Can't use := here or we override the outer tmpDir
+		tmpDir, err = ioutil.TempDir("", "docker-tar")
+		if err != nil {
+			return nil, err
+		}
+
+		files = files + "-C" + tmpDir + "\n"
+		for _, f := range options.CreateFiles {
+			path := filepath.Join(tmpDir, f)
+			err := os.MkdirAll(filepath.Dir(path), 0600)
+			if err != nil {
+				return nil, err
+			}
+
+			if file, err := os.OpenFile(path, os.O_CREATE, 0600); err != nil {
+				return nil, err
+			} else {
+				file.Close()
+			}
+			files = files + escapeName(f) + "\n"
+		}
+	}
+
+	return CmdStream(exec.Command(args[0], args[1:]...), &files, func() {
+		if tmpDir != "" {
+			_ = os.RemoveAll(tmpDir)
+		}
+	})
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,
@@ -101,7 +167,7 @@ func TarFilter(path string, compression Compression, filter []string) (io.Reader
 // The archive may be compressed with one of the following algorithms:
 //  identity (uncompressed), gzip, bzip2, xz.
 // FIXME: specify behavior when target path exists vs. doesn't exist.
-func Untar(archive io.Reader, path string) error {
+func Untar(archive io.Reader, path string, options *TarOptions) error {
 	if archive == nil {
 		return fmt.Errorf("Empty archive")
 	}
@@ -122,8 +188,15 @@ func Untar(archive io.Reader, path string) error {
 	compression := DetectCompression(buf)
 
 	utils.Debugf("Archive compression detected: %s", compression.Extension())
+	args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()}
 
-	cmd := exec.Command("tar", "--numeric-owner", "-f", "-", "-C", path, "-x"+compression.Flag())
+	if options != nil {
+		for _, exclude := range options.Excludes {
+			args = append(args, fmt.Sprintf("--exclude=%s", exclude))
+		}
+	}
+
+	cmd := exec.Command("tar", args...)
 	cmd.Stdin = io.MultiReader(bytes.NewReader(buf), archive)
 	// Hardcode locale environment for predictable outcome regardless of host configuration.
 	//   (see https://github.com/dotcloud/docker/issues/355)
@@ -140,11 +213,11 @@ func Untar(archive io.Reader, path string) error {
 // TarUntar aborts and returns the error.
 func TarUntar(src string, filter []string, dst string) error {
 	utils.Debugf("TarUntar(%s %s %s)", src, filter, dst)
-	archive, err := TarFilter(src, Uncompressed, filter)
+	archive, err := TarFilter(src, &TarOptions{Compression: Uncompressed, Includes: filter, Recursive: true})
 	if err != nil {
 		return err
 	}
-	return Untar(archive, dst)
+	return Untar(archive, dst, nil)
 }
 
 // UntarPath is a convenience function which looks for an archive
@@ -152,7 +225,7 @@ func TarUntar(src string, filter []string, dst string) error {
 func UntarPath(src, dst string) error {
 	if archive, err := os.Open(src); err != nil {
 		return err
-	} else if err := Untar(archive, dst); err != nil {
+	} else if err := Untar(archive, dst, nil); err != nil {
 		return err
 	}
 	return nil
@@ -221,19 +294,39 @@ func CopyFileWithTar(src, dst string) error {
 		return err
 	}
 	tw.Close()
-	return Untar(buf, filepath.Dir(dst))
+	return Untar(buf, filepath.Dir(dst), nil)
 }
 
 // CmdStream executes a command, and returns its stdout as a stream.
 // If the command fails to run or doesn't complete successfully, an error
 // will be returned, including anything written on stderr.
-func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
+func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) {
+	if input != nil {
+		stdin, err := cmd.StdinPipe()
+		if err != nil {
+			if atEnd != nil {
+				atEnd()
+			}
+			return nil, err
+		}
+		// Write stdin if any
+		go func() {
+			_, _ = stdin.Write([]byte(*input))
+			stdin.Close()
+		}()
+	}
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
+		if atEnd != nil {
+			atEnd()
+		}
 		return nil, err
 	}
 	pipeR, pipeW := io.Pipe()
@@ -258,6 +351,9 @@ func CmdStream(cmd *exec.Cmd) (io.Reader, error) {
 		} else {
 			pipeW.Close()
 		}
+		if atEnd != nil {
+			atEnd()
+		}
 	}()
 	// Run the command and return the pipe
 	if err := cmd.Start(); err != nil {

archive/archive_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestCmdStreamLargeStderr(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -35,7 +35,7 @@ func TestCmdStreamLargeStderr(t *testing.T) {
 
 func TestCmdStreamBad(t *testing.T) {
 	badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1")
-	out, err := CmdStream(badCmd)
+	out, err := CmdStream(badCmd, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to start command: %s", err)
 	}
@@ -50,7 +50,7 @@ func TestCmdStreamBad(t *testing.T) {
 
 func TestCmdStreamGood(t *testing.T) {
 	cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0")
-	out, err := CmdStream(cmd)
+	out, err := CmdStream(cmd, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +83,7 @@ func tarUntar(t *testing.T, origin string, compression Compression) error {
 		return err
 	}
 	defer os.RemoveAll(tmp)
-	if err := Untar(archive, tmp); err != nil {
+	if err := Untar(archive, tmp, nil); err != nil {
 		return err
 	}
 	if _, err := os.Stat(tmp); err != nil {

archive/changes.go

@@ -0,0 +1,301 @@
+package archive
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+type ChangeType int
+
+const (
+	ChangeModify = iota
+	ChangeAdd
+	ChangeDelete
+)
+
+type Change struct {
+	Path string
+	Kind ChangeType
+}
+
+func (change *Change) String() string {
+	var kind string
+	switch change.Kind {
+	case ChangeModify:
+		kind = "C"
+	case ChangeAdd:
+		kind = "A"
+	case ChangeDelete:
+		kind = "D"
+	}
+	return fmt.Sprintf("%s %s", kind, change.Path)
+}
+
+func Changes(layers []string, rw string) ([]Change, error) {
+	var changes []Change
+	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		path, err = filepath.Rel(rw, path)
+		if err != nil {
+			return err
+		}
+		path = filepath.Join("/", path)
+
+		// Skip root
+		if path == "/" {
+			return nil
+		}
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
+			return err
+		}
+
+		change := Change{
+			Path: path,
+		}
+
+		// Find out what kind of modification happened
+		file := filepath.Base(path)
+		// If there is a whiteout, then the file was removed
+		if strings.HasPrefix(file, ".wh.") {
+			originalFile := file[len(".wh."):]
+			change.Path = filepath.Join(filepath.Dir(path), originalFile)
+			change.Kind = ChangeDelete
+		} else {
+			// Otherwise, the file was added
+			change.Kind = ChangeAdd
+
+			// ...Unless it already existed in a top layer, in which case, it's a modification
+			for _, layer := range layers {
+				stat, err := os.Stat(filepath.Join(layer, path))
+				if err != nil && !os.IsNotExist(err) {
+					return err
+				}
+				if err == nil {
+					// The file existed in the top layer, so that's a modification
+
+					// However, if it's a directory, maybe it wasn't actually modified.
+					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
+					if stat.IsDir() && f.IsDir() {
+						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
+							// Both directories are the same, don't record the change
+							return nil
+						}
+					}
+					change.Kind = ChangeModify
+					break
+				}
+			}
+		}
+
+		// Record change
+		changes = append(changes, change)
+		return nil
+	})
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	return changes, nil
+}
+
+type FileInfo struct {
+	parent   *FileInfo
+	name     string
+	stat     syscall.Stat_t
+	children map[string]*FileInfo
+}
+
+func (root *FileInfo) LookUp(path string) *FileInfo {
+	parent := root
+	if path == "/" {
+		return root
+	}
+
+	pathElements := strings.Split(path, "/")
+	for _, elem := range pathElements {
+		if elem != "" {
+			child := parent.children[elem]
+			if child == nil {
+				return nil
+			}
+			parent = child
+		}
+	}
+	return parent
+}
+
+func (info *FileInfo) path() string {
+	if info.parent == nil {
+		return "/"
+	}
+	return filepath.Join(info.parent.path(), info.name)
+}
+
+func (info *FileInfo) isDir() bool {
+	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
+}
+
+func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
+	if oldInfo == nil {
+		// add
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeAdd,
+		}
+		*changes = append(*changes, change)
+	}
+
+	// We make a copy so we can modify it to detect additions
+	// also, we only recurse on the old dir if the new info is a directory
+	// otherwise any previous delete/change is considered recursive
+	oldChildren := make(map[string]*FileInfo)
+	if oldInfo != nil && info.isDir() {
+		for k, v := range oldInfo.children {
+			oldChildren[k] = v
+		}
+	}
+
+	for name, newChild := range info.children {
+		oldChild, _ := oldChildren[name]
+		if oldChild != nil {
+			// change?
+			oldStat := &oldChild.stat
+			newStat := &newChild.stat
+			// Note: We can't compare inode or ctime or blocksize here, because these change
+			// when copying a file into a container. However, that is not generally a problem
+			// because any content change will change mtime, and any status change should
+			// be visible when actually comparing the stat fields. The only time this
+			// breaks down is if some code intentionally hides a change by setting
+			// back mtime
+			if oldStat.Mode != newStat.Mode ||
+				oldStat.Uid != newStat.Uid ||
+				oldStat.Gid != newStat.Gid ||
+				oldStat.Rdev != newStat.Rdev ||
+				// Don't look at size for dirs, its not a good measure of change
+				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
+				oldStat.Mtim != newStat.Mtim {
+				change := Change{
+					Path: newChild.path(),
+					Kind: ChangeModify,
+				}
+				*changes = append(*changes, change)
+			}
+
+			// Remove from copy so we can detect deletions
+			delete(oldChildren, name)
+		}
+
+		newChild.addChanges(oldChild, changes)
+	}
+	for _, oldChild := range oldChildren {
+		// delete
+		change := Change{
+			Path: oldChild.path(),
+			Kind: ChangeDelete,
+		}
+		*changes = append(*changes, change)
+	}
+
+}
+
+func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
+	var changes []Change
+
+	info.addChanges(oldInfo, &changes)
+
+	return changes
+}
+
+func newRootFileInfo() *FileInfo {
+	root := &FileInfo{
+		name:     "/",
+		children: make(map[string]*FileInfo),
+	}
+	return root
+}
+
+func collectFileInfo(sourceDir string) (*FileInfo, error) {
+	root := newRootFileInfo()
+
+	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		relPath, err := filepath.Rel(sourceDir, path)
+		if err != nil {
+			return err
+		}
+		relPath = filepath.Join("/", relPath)
+
+		if relPath == "/" {
+			return nil
+		}
+
+		parent := root.LookUp(filepath.Dir(relPath))
+		if parent == nil {
+			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
+		}
+
+		info := &FileInfo{
+			name:     filepath.Base(relPath),
+			children: make(map[string]*FileInfo),
+			parent:   parent,
+		}
+
+		if err := syscall.Lstat(path, &info.stat); err != nil {
+			return err
+		}
+
+		parent.children[info.name] = info
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return root, nil
+}
+
+// Compare two directories and generate an array of Change objects describing the changes
+func ChangesDirs(newDir, oldDir string) ([]Change, error) {
+	oldRoot, err := collectFileInfo(oldDir)
+	if err != nil {
+		return nil, err
+	}
+	newRoot, err := collectFileInfo(newDir)
+	if err != nil {
+		return nil, err
+	}
+
+	return newRoot.Changes(oldRoot), nil
+}
+
+func ExportChanges(root, rw string) (Archive, error) {
+	changes, err := ChangesDirs(root, rw)
+	if err != nil {
+		return nil, err
+	}
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+	return TarFilter(root, &TarOptions{Compression: Uncompressed, Recursive: false, Includes: files, CreateFiles: deletions})
+}

archive/diff.go

@@ -0,0 +1,63 @@
+package archive
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ApplyLayer parses a diff in the standard layer format from `layer`, and
+// applies it to the directory `dest`.
+func ApplyLayer(dest string, layer Archive) error {
+	// Poor man's diff applyer in 2 steps:
+
+	// Step 1: untar everything in place
+	if err := Untar(layer, dest, nil); err != nil {
+		return err
+	}
+
+	// Step 2: walk for whiteouts and apply them, removing them in the process
+	err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error {
+		if err != nil {
+			if os.IsNotExist(err) {
+				// This happens in the case of whiteouts in parent dir removing a directory
+				// We just ignore it
+				return filepath.SkipDir
+			}
+			return err
+		}
+
+		// Rebase path
+		path, err := filepath.Rel(dest, fullPath)
+		if err != nil {
+			return err
+		}
+		path = filepath.Join("/", path)
+
+		// Skip AUFS metadata
+		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil {
+			return err
+		} else if matched {
+			log.Printf("Removing aufs metadata %s", fullPath)
+			_ = os.RemoveAll(fullPath)
+		}
+
+		filename := filepath.Base(path)
+		if strings.HasPrefix(filename, ".wh.") {
+			rmTargetName := filename[len(".wh."):]
+			rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName)
+			// Remove the file targeted by the whiteout
+			log.Printf("Removing whiteout target %s", rmTargetPath)
+			_ = os.Remove(rmTargetPath)
+			// Remove the whiteout itself
+			log.Printf("Removing whiteout %s", fullPath)
+			_ = os.RemoveAll(fullPath)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}

aufs/aufs.go

@@ -0,0 +1,316 @@
+/*
+
+aufs driver directory structure
+
+.
+├── layers // Metadata of layers
+│   ├── 1
+│   ├── 2
+│   └── 3
+├── diffs  // Content of the layer
+│   ├── 1  // Contains layers that need to be mounted for the id
+│   ├── 2
+│   └── 3
+└── mnt    // Mount points for the rw layers to be mounted
+    ├── 1
+    ├── 2
+    └── 3
+
+*/
+
+package aufs
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
+	"github.com/dotcloud/docker/utils"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+)
+
+func init() {
+	graphdriver.Register("aufs", Init)
+}
+
+type AufsDriver struct {
+	root string
+}
+
+// New returns a new AUFS driver.
+// An error is returned if AUFS is not supported.
+func Init(root string) (graphdriver.Driver, error) {
+	// Try to load the aufs kernel module
+	if err := supportsAufs(); err != nil {
+		return nil, err
+	}
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	// Create the root aufs driver dir and return
+	// if it already exists
+	// If not populate the dir structure
+	if err := os.MkdirAll(root, 0755); err != nil {
+		if os.IsExist(err) {
+			return &AufsDriver{root}, nil
+		}
+		return nil, err
+	}
+
+	for _, p := range paths {
+		if err := os.MkdirAll(path.Join(root, p), 0755); err != nil {
+			return nil, err
+		}
+	}
+	return &AufsDriver{root}, nil
+}
+
+// Return a nil error if the kernel supports aufs
+// We cannot modprobe because inside dind modprobe fails
+// to run
+func supportsAufs() error {
+	// We can try to modprobe aufs first before looking at
+	// proc/filesystems for when aufs is supported
+	exec.Command("modprobe", "aufs").Run()
+
+	f, err := os.Open("/proc/filesystems")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		if strings.Contains(s.Text(), "aufs") {
+			return nil
+		}
+	}
+	return fmt.Errorf("AUFS was not found in /proc/filesystems")
+}
+
+func (a *AufsDriver) rootPath() string {
+	return a.root
+}
+
+func (a *AufsDriver) String() string {
+	return "aufs"
+}
+
+// Three folders are created for each id
+// mnt, layers, and diff
+func (a *AufsDriver) Create(id, parent string) error {
+	if err := a.createDirsFor(id); err != nil {
+		return err
+	}
+	// Write the layers metadata
+	f, err := os.Create(path.Join(a.rootPath(), "layers", id))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if parent != "" {
+		ids, err := getParentIds(a.rootPath(), parent)
+		if err != nil {
+			return err
+		}
+
+		if _, err := fmt.Fprintln(f, parent); err != nil {
+			return err
+		}
+		for _, i := range ids {
+			if _, err := fmt.Fprintln(f, i); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (a *AufsDriver) createDirsFor(id string) error {
+	paths := []string{
+		"mnt",
+		"diff",
+	}
+
+	for _, p := range paths {
+		if err := os.MkdirAll(path.Join(a.rootPath(), p, id), 0755); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Unmount and remove the dir information
+func (a *AufsDriver) Remove(id string) error {
+	// Make sure the dir is umounted first
+	if err := a.unmount(id); err != nil {
+		return err
+	}
+	tmpDirs := []string{
+		"mnt",
+		"diff",
+	}
+
+	// Remove the dirs atomically
+	for _, p := range tmpDirs {
+		// We need to use a temp dir in the same dir as the driver so Rename
+		// does not fall back to the slow copy if /tmp and the driver dir
+		// are on different devices
+		tmp := path.Join(a.rootPath(), "tmp", p, id)
+		if err := os.MkdirAll(tmp, 0755); err != nil {
+			return err
+		}
+		realPath := path.Join(a.rootPath(), p, id)
+		if err := os.Rename(realPath, tmp); err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmp)
+	}
+
+	// Remove the layers file for the id
+	return os.Remove(path.Join(a.rootPath(), "layers", id))
+}
+
+// Return the rootfs path for the id
+// This will mount the dir at it's given path
+func (a *AufsDriver) Get(id string) (string, error) {
+	ids, err := getParentIds(a.rootPath(), id)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		ids = []string{}
+	}
+
+	// If a dir does not have a parent ( no layers )do not try to mount
+	// just return the diff path to the data
+	out := path.Join(a.rootPath(), "diff", id)
+	if len(ids) > 0 {
+		out = path.Join(a.rootPath(), "mnt", id)
+		if err := a.mount(id); err != nil {
+			return "", err
+		}
+	}
+	return out, nil
+}
+
+// Returns an archive of the contents for the id
+func (a *AufsDriver) Diff(id string) (archive.Archive, error) {
+	return archive.TarFilter(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
+		Recursive:   true,
+		Compression: archive.Uncompressed,
+	})
+}
+
+func (a *AufsDriver) ApplyDiff(id string, diff archive.Archive) error {
+	return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil)
+}
+
+// Returns the size of the contents for the id
+func (a *AufsDriver) Size(id string) (int64, error) {
+	return utils.TreeSize(path.Join(a.rootPath(), "diff", id))
+}
+
+func (a *AufsDriver) Changes(id string) ([]archive.Change, error) {
+	layers, err := a.getParentLayerPaths(id)
+	if err != nil {
+		return nil, err
+	}
+	return archive.Changes(layers, path.Join(a.rootPath(), "diff", id))
+}
+
+func (a *AufsDriver) getParentLayerPaths(id string) ([]string, error) {
+	parentIds, err := getParentIds(a.rootPath(), id)
+	if err != nil {
+		return nil, err
+	}
+	if len(parentIds) == 0 {
+		return nil, fmt.Errorf("Dir %s does not have any parent layers", id)
+	}
+	layers := make([]string, len(parentIds))
+
+	// Get the diff paths for all the parent ids
+	for i, p := range parentIds {
+		layers[i] = path.Join(a.rootPath(), "diff", p)
+	}
+	return layers, nil
+}
+
+func (a *AufsDriver) mount(id string) error {
+	// If the id is mounted or we get an error return
+	if mounted, err := a.mounted(id); err != nil || mounted {
+		return err
+	}
+
+	var (
+		target = path.Join(a.rootPath(), "mnt", id)
+		rw     = path.Join(a.rootPath(), "diff", id)
+	)
+
+	layers, err := a.getParentLayerPaths(id)
+	if err != nil {
+		return err
+	}
+
+	if err := a.aufsMount(layers, rw, target); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (a *AufsDriver) unmount(id string) error {
+	if mounted, err := a.mounted(id); err != nil || !mounted {
+		return err
+	}
+	target := path.Join(a.rootPath(), "mnt", id)
+	return Unmount(target)
+}
+
+func (a *AufsDriver) mounted(id string) (bool, error) {
+	target := path.Join(a.rootPath(), "mnt", id)
+	return Mounted(target)
+}
+
+// During cleanup aufs needs to unmount all mountpoints
+func (a *AufsDriver) Cleanup() error {
+	ids, err := loadIds(path.Join(a.rootPath(), "layers"))
+	if err != nil {
+		return err
+	}
+	for _, id := range ids {
+		if err := a.unmount(id); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *AufsDriver) aufsMount(ro []string, rw, target string) error {
+	rwBranch := fmt.Sprintf("%v=rw", rw)
+	roBranches := ""
+	for _, layer := range ro {
+		roBranches += fmt.Sprintf("%v=ro+wh:", layer)
+	}
+	branches := fmt.Sprintf("br:%v:%v,xino=/dev/shm/aufs.xino", rwBranch, roBranches)
+
+	//if error, try to load aufs kernel module
+	if err := mount("none", target, "aufs", 0, branches); err != nil {
+		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
+		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
+			return fmt.Errorf("Unable to load the AUFS module")
+		}
+		log.Printf("...module loaded.")
+		if err := mount("none", target, "aufs", 0, branches); err != nil {
+			return fmt.Errorf("Unable to mount using aufs %s", err)
+		}
+	}
+	return nil
+}

aufs/aufs_test.go

@@ -0,0 +1,517 @@
+package aufs
+
+import (
+	"github.com/dotcloud/docker/archive"
+	"os"
+	"path"
+	"testing"
+)
+
+var (
+	tmp = path.Join(os.TempDir(), "aufs-tests", "aufs")
+)
+
+func newDriver(t *testing.T) *AufsDriver {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	d, err := Init(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d.(*AufsDriver)
+}
+
+func TestNewAufsDriver(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	d, err := Init(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if d == nil {
+		t.Fatalf("Driver should not be nil")
+	}
+}
+
+func TestAufsString(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if d.String() != "aufs" {
+		t.Fatalf("Expected aufs got %s", d.String())
+	}
+}
+
+func TestCreateDirStructure(t *testing.T) {
+	newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	paths := []string{
+		"mnt",
+		"layers",
+		"diff",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// We should be able to create two drivers with the same dir structure
+func TestNewDriverFromExistingDir(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := Init(tmp); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Init(tmp); err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll(tmp)
+}
+
+func TestCreateNewDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCreateNewDirStructure(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p, "1")); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestRemoveImage(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Remove("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	paths := []string{
+		"mnt",
+		"diff",
+		"layers",
+	}
+
+	for _, p := range paths {
+		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
+			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
+		}
+	}
+}
+
+func TestGetWithoutParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := path.Join(tmp, "diff", "1")
+	if diffPath != expected {
+		t.Fatalf("Expected path %s got %s", expected, diffPath)
+	}
+}
+
+func TestCleanupWithNoDirs(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCleanupWithDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMountedFalseResponse(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	response, err := d.mounted("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response != false {
+		t.Fatalf("Response if dir id 1 is mounted should be false")
+	}
+}
+
+func TestMountedTrueReponse(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+	defer d.Cleanup()
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	response, err := d.mounted("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response != true {
+		t.Fatalf("Response if dir id 2 is mounted should be true")
+	}
+}
+
+func TestMountWithParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPath, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mntPath == "" {
+		t.Fatal("mntPath should not be empty string")
+	}
+
+	expected := path.Join(tmp, "mnt", "2")
+	if mntPath != expected {
+		t.Fatalf("Expected %s got %s", expected, mntPath)
+	}
+}
+
+func TestRemoveMountedDir(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPath, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mntPath == "" {
+		t.Fatal("mntPath should not be empty string")
+	}
+
+	mounted, err := d.mounted("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !mounted {
+		t.Fatalf("Dir id 2 should be mounted")
+	}
+
+	if err := d.Remove("2"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCreateWithInvalidParent(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", "docker"); err == nil {
+		t.Fatalf("Error should not be nil with parent does not exist")
+	}
+}
+
+func TestGetDiff(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Truncate(size); err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+
+	a, err := d.Diff("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a == nil {
+		t.Fatalf("Archive should not be nil")
+	}
+}
+
+func TestChanges(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	defer func() {
+		if err := d.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	mntPoint, err := d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a file to save in the mountpoint
+	f, err := os.Create(path.Join(mntPoint, "test.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := f.WriteString("testline"); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err := d.Changes("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(changes) != 1 {
+		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
+	}
+	change := changes[0]
+
+	expectedPath := "/test.txt"
+	if change.Path != expectedPath {
+		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
+	}
+
+	if change.Kind != archive.ChangeAdd {
+		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
+	}
+
+	if err := d.Create("3", "2"); err != nil {
+		t.Fatal(err)
+	}
+	mntPoint, err = d.Get("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a file to save in the mountpoint
+	f, err = os.Create(path.Join(mntPoint, "test2.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := f.WriteString("testline"); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	changes, err = d.Changes("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(changes) != 1 {
+		t.Fatalf("Dir 2 should have one change from parent got %d", len(changes))
+	}
+	change = changes[0]
+
+	expectedPath = "/test2.txt"
+	if change.Path != expectedPath {
+		t.Fatalf("Expected path %s got %s", expectedPath, change.Path)
+	}
+
+	if change.Kind != archive.ChangeAdd {
+		t.Fatalf("Change kind should be ChangeAdd got %s", change.Kind)
+	}
+}
+
+func TestDiffSize(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Truncate(size)
+	s, err := f.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+	size = s.Size()
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	diffSize, err := d.Size("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diffSize != size {
+		t.Fatalf("Expected size to be %d got %d", size, diffSize)
+	}
+}
+
+func TestApplyDiff(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(tmp)
+	defer d.Cleanup()
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	diffPath, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Add a file to the diff path with a fixed size
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(diffPath, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Truncate(size)
+	f.Close()
+
+	diff, err := d.Diff("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Create("2", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.Create("3", "2"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.ApplyDiff("3", diff); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that the file is in the mount point for id 3
+
+	mountPoint, err := d.Get("3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path.Join(mountPoint, "test_file")); err != nil {
+		t.Fatal(err)
+	}
+}

aufs/dirs.go

@@ -0,0 +1,46 @@
+package aufs
+
+import (
+	"bufio"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+// Return all the directories
+func loadIds(root string) ([]string, error) {
+	dirs, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+	out := []string{}
+	for _, d := range dirs {
+		if !d.IsDir() {
+			out = append(out, d.Name())
+		}
+	}
+	return out, nil
+}
+
+// Read the layers file for the current id and return all the
+// layers represented by new lines in the file
+//
+// If there are no lines in the file then the id has no parent
+// and an empty slice is returned.
+func getParentIds(root, id string) ([]string, error) {
+	f, err := os.Open(path.Join(root, "layers", id))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	out := []string{}
+	s := bufio.NewScanner(f)
+
+	for s.Scan() {
+		if t := s.Text(); t != "" {
+			out = append(out, s.Text())
+		}
+	}
+	return out, s.Err()
+}

aufs/mount.go

@@ -1,13 +1,11 @@
-package docker
+package aufs
 
 import (
-	"fmt"
 	"github.com/dotcloud/docker/utils"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"syscall"
-	"time"
 )
 
 func Unmount(target string) error {
@@ -17,22 +15,7 @@ func Unmount(target string) error {
 	if err := syscall.Unmount(target, 0); err != nil {
 		return err
 	}
-	// Even though we just unmounted the filesystem, AUFS will prevent deleting the mntpoint
-	// for some time. We'll just keep retrying until it succeeds.
-	for retries := 0; retries < 1000; retries++ {
-		err := os.Remove(target)
-		if err == nil {
-			// rm mntpoint succeeded
-			return nil
-		}
-		if os.IsNotExist(err) {
-			// mntpoint doesn't exist anymore. Success.
-			return nil
-		}
-		// fmt.Printf("(%v) Remove %v returned: %v\n", retries, target, err)
-		time.Sleep(10 * time.Millisecond)
-	}
-	return fmt.Errorf("Umount: Failed to umount %v", target)
+	return nil
 }
 
 func Mounted(mountpoint string) (bool, error) {
@@ -49,5 +32,6 @@ func Mounted(mountpoint string) (bool, error) {
 	}
 	mntpointSt := mntpoint.Sys().(*syscall.Stat_t)
 	parentSt := parent.Sys().(*syscall.Stat_t)
+
 	return mntpointSt.Dev != parentSt.Dev, nil
 }

aufs/mount_darwin.go

@@ -1,4 +1,4 @@
-package docker
+package aufs
 
 import "errors"
 

aufs/mount_linux.go

@@ -1,4 +1,4 @@
-package docker
+package aufs
 
 import "syscall"
 

buildfile.go

@@ -476,7 +476,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	if err := archive.Untar(context, name); err != nil {
+	if err := archive.Untar(context, name, nil); err != nil {
 		return "", err
 	}
 	defer os.RemoveAll(name)

buildfile_test.go

@@ -544,10 +544,7 @@ func TestBuildADDFileNotFound(t *testing.T) {
 }
 
 func TestBuildInheritance(t *testing.T) {
-	runtime, err := newTestRuntime("")
-	if err != nil {
-		t.Fatal(err)
-	}
+	runtime := mkRuntime(t)
 	defer nuke(runtime)
 
 	srv := &Server{

changes.go

@@ -1,106 +0,0 @@
-package docker
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-type ChangeType int
-
-const (
-	ChangeModify = iota
-	ChangeAdd
-	ChangeDelete
-)
-
-type Change struct {
-	Path string
-	Kind ChangeType
-}
-
-func (change *Change) String() string {
-	var kind string
-	switch change.Kind {
-	case ChangeModify:
-		kind = "C"
-	case ChangeAdd:
-		kind = "A"
-	case ChangeDelete:
-		kind = "D"
-	}
-	return fmt.Sprintf("%s %s", kind, change.Path)
-}
-
-func Changes(layers []string, rw string) ([]Change, error) {
-	var changes []Change
-	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Rebase path
-		path, err = filepath.Rel(rw, path)
-		if err != nil {
-			return err
-		}
-		path = filepath.Join("/", path)
-
-		// Skip root
-		if path == "/" {
-			return nil
-		}
-
-		// Skip AUFS metadata
-		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
-			return err
-		}
-
-		change := Change{
-			Path: path,
-		}
-
-		// Find out what kind of modification happened
-		file := filepath.Base(path)
-		// If there is a whiteout, then the file was removed
-		if strings.HasPrefix(file, ".wh.") {
-			originalFile := file[len(".wh."):]
-			change.Path = filepath.Join(filepath.Dir(path), originalFile)
-			change.Kind = ChangeDelete
-		} else {
-			// Otherwise, the file was added
-			change.Kind = ChangeAdd
-
-			// ...Unless it already existed in a top layer, in which case, it's a modification
-			for _, layer := range layers {
-				stat, err := os.Stat(filepath.Join(layer, path))
-				if err != nil && !os.IsNotExist(err) {
-					return err
-				}
-				if err == nil {
-					// The file existed in the top layer, so that's a modification
-
-					// However, if it's a directory, maybe it wasn't actually modified.
-					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
-					if stat.IsDir() && f.IsDir() {
-						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() {
-							// Both directories are the same, don't record the change
-							return nil
-						}
-					}
-					change.Kind = ChangeModify
-					break
-				}
-			}
-		}
-
-		// Record change
-		changes = append(changes, change)
-		return nil
-	})
-	if err != nil && !os.IsNotExist(err) {
-		return nil, err
-	}
-	return changes, nil
-}

commands.go

@@ -130,10 +130,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error {
 	v.Set("url", cmd.Arg(1))
 	v.Set("path", cmd.Arg(2))
 
-	if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out, nil); err != nil {
-		return err
-	}
-	return nil
+	return cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out, nil)
 }
 
 // mkBuildContext returns an archive of an empty context with the contents
@@ -376,15 +373,17 @@ func (cli *DockerCli) CmdWait(args ...string) error {
 		cmd.Usage()
 		return nil
 	}
+	var encounteredError error
 	for _, name := range cmd.Args() {
 		status, err := waitForExit(cli, name)
 		if err != nil {
-			fmt.Fprintf(cli.err, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to wait one or more containers")
 		} else {
 			fmt.Fprintf(cli.out, "%d\n", status)
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 // 'docker version': show version information
@@ -505,15 +504,17 @@ func (cli *DockerCli) CmdStop(args ...string) error {
 	v := url.Values{}
 	v.Set("t", strconv.Itoa(*nSeconds))
 
+	var encounteredError error
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("POST", "/containers/"+name+"/stop?"+v.Encode(), nil)
 		if err != nil {
 			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to stop one or more containers")
 		} else {
 			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 func (cli *DockerCli) CmdRestart(args ...string) error {
@@ -530,15 +531,17 @@ func (cli *DockerCli) CmdRestart(args ...string) error {
 	v := url.Values{}
 	v.Set("t", strconv.Itoa(*nSeconds))
 
+	var encounteredError error
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("POST", "/containers/"+name+"/restart?"+v.Encode(), nil)
 		if err != nil {
 			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to  restart one or more containers")
 		} else {
 			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal {
@@ -772,15 +775,19 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 		return nil
 	}
 
+	var encounteredError error
 	for _, name := range cmd.Args() {
 		body, _, err := cli.call("DELETE", "/images/"+name, nil)
 		if err != nil {
-			fmt.Fprintf(cli.err, "%s", err)
+			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to remove one or more images")
 		} else {
 			var outs []APIRmi
 			err = json.Unmarshal(body, &outs)
 			if err != nil {
-				return err
+				fmt.Fprintf(cli.err, "%s\n", err)
+				encounteredError = fmt.Errorf("Error: failed to remove one or more images")
+				continue
 			}
 			for _, out := range outs {
 				if out.Deleted != "" {
@@ -791,7 +798,7 @@ func (cli *DockerCli) CmdRmi(args ...string) error {
 			}
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 func (cli *DockerCli) CmdHistory(args ...string) error {
@@ -870,15 +877,18 @@ func (cli *DockerCli) CmdRm(args ...string) error {
 	if *link {
 		val.Set("link", "1")
 	}
+
+	var encounteredError error
 	for _, name := range cmd.Args() {
 		_, _, err := cli.call("DELETE", "/containers/"+name+"?"+val.Encode(), nil)
 		if err != nil {
 			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to remove one or more containers")
 		} else {
 			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 // 'docker kill NAME' kills a running container
@@ -892,15 +902,16 @@ func (cli *DockerCli) CmdKill(args ...string) error {
 		return nil
 	}
 
+	var encounteredError error
 	for _, name := range args {
-		_, _, err := cli.call("POST", "/containers/"+name+"/kill", nil)
-		if err != nil {
+		if _, _, err := cli.call("POST", "/containers/"+name+"/kill", nil); err != nil {
 			fmt.Fprintf(cli.err, "%s\n", err)
+			encounteredError = fmt.Errorf("Error: failed to kill one or more containers")
 		} else {
 			fmt.Fprintf(cli.out, "%s\n", name)
 		}
 	}
-	return nil
+	return encounteredError
 }
 
 func (cli *DockerCli) CmdImport(args ...string) error {
@@ -913,8 +924,16 @@ func (cli *DockerCli) CmdImport(args ...string) error {
 		cmd.Usage()
 		return nil
 	}
-	src := cmd.Arg(0)
-	repository, tag := utils.ParseRepositoryTag(cmd.Arg(1))
+
+	var src, repository, tag string
+
+	if cmd.NArg() == 3 {
+		fmt.Fprintf(cli.err, "[DEPRECATED] The format 'URL|- [REPOSITORY [TAG]]' as been deprecated. Please use URL|- [REPOSITORY[:TAG]]\n")
+		src, repository, tag = cmd.Arg(0), cmd.Arg(1), cmd.Arg(2)
+	} else {
+		src = cmd.Arg(0)
+		repository, tag = utils.ParseRepositoryTag(cmd.Arg(1))
+	}
 	v := url.Values{}
 	v.Set("repo", repository)
 	v.Set("tag", tag)
@@ -1057,6 +1076,7 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 	all := cmd.Bool("a", false, "show all images")
 	noTrunc := cmd.Bool("notrunc", false, "Don't truncate output")
 	flViz := cmd.Bool("viz", false, "output graph in graphviz format")
+	flTree := cmd.Bool("tree", false, "output graph in tree format")
 
 	if err := cmd.Parse(args); err != nil {
 		return nil
@@ -1067,11 +1087,79 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 	}
 
 	if *flViz {
-		body, _, err := cli.call("GET", "/images/viz", false)
+		body, _, err := cli.call("GET", "/images/json?all=1", nil)
 		if err != nil {
 			return err
 		}
-		fmt.Fprintf(cli.out, "%s", body)
+
+		var outs []APIImages
+		err = json.Unmarshal(body, &outs)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(cli.out, "digraph docker {\n")
+
+		for _, image := range outs {
+			if image.ParentId == "" {
+				fmt.Fprintf(cli.out, " base -> \"%s\" [style=invis]\n", utils.TruncateID(image.ID))
+			} else {
+				fmt.Fprintf(cli.out, " \"%s\" -> \"%s\"\n", utils.TruncateID(image.ParentId), utils.TruncateID(image.ID))
+			}
+			if image.RepoTags[0] != "<none>:<none>" {
+				fmt.Fprintf(cli.out, " \"%s\" [label=\"%s\\n%s\",shape=box,fillcolor=\"paleturquoise\",style=\"filled,rounded\"];\n", utils.TruncateID(image.ID), utils.TruncateID(image.ID), strings.Join(image.RepoTags, "\\n"))
+			}
+		}
+
+		fmt.Fprintf(cli.out, " base [style=invisible]\n}\n")
+	} else if *flTree {
+		body, _, err := cli.call("GET", "/images/json?all=1", nil)
+		if err != nil {
+			return err
+		}
+
+		var outs []APIImages
+		if err := json.Unmarshal(body, &outs); err != nil {
+			return err
+		}
+
+		var (
+			startImageArg = cmd.Arg(0)
+			startImage    APIImages
+
+			roots    []APIImages
+			byParent = make(map[string][]APIImages)
+		)
+
+		for _, image := range outs {
+			if image.ParentId == "" {
+				roots = append(roots, image)
+			} else {
+				if children, exists := byParent[image.ParentId]; exists {
+					byParent[image.ParentId] = append(children, image)
+				} else {
+					byParent[image.ParentId] = []APIImages{image}
+				}
+			}
+
+			if startImageArg != "" {
+				if startImageArg == image.ID || startImageArg == utils.TruncateID(image.ID) {
+					startImage = image
+				}
+
+				for _, repotag := range image.RepoTags {
+					if repotag == startImageArg {
+						startImage = image
+					}
+				}
+			}
+		}
+
+		if startImageArg != "" {
+			WalkTree(cli, noTrunc, []APIImages{startImage}, byParent, "")
+		} else {
+			WalkTree(cli, noTrunc, roots, byParent, "")
+		}
 	} else {
 		v := url.Values{}
 		if cmd.NArg() == 1 {
@@ -1098,26 +1186,24 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 		}
 
 		for _, out := range outs {
-			if out.Repository == "" {
-				out.Repository = "<none>"
-			}
-			if out.Tag == "" {
-				out.Tag = "<none>"
-			}
+			for _, repotag := range out.RepoTags {
 
-			if !*noTrunc {
-				out.ID = utils.TruncateID(out.ID)
-			}
+				repo, tag := utils.ParseRepositoryTag(repotag)
 
-			if !*quiet {
-				fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t", out.Repository, out.Tag, out.ID, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
-				if out.VirtualSize > 0 {
-					fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.Size), utils.HumanSize(out.VirtualSize))
-				} else {
-					fmt.Fprintf(w, "%s\n", utils.HumanSize(out.Size))
+				if !*noTrunc {
+					out.ID = utils.TruncateID(out.ID)
+				}
+
+				if !*quiet {
+					fmt.Fprintf(w, "%s\t%s\t%s\t%s ago\t", repo, tag, out.ID, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))))
+					if out.VirtualSize > 0 {
+						fmt.Fprintf(w, "%s (virtual %s)\n", utils.HumanSize(out.Size), utils.HumanSize(out.VirtualSize))
+					} else {
+						fmt.Fprintf(w, "%s\n", utils.HumanSize(out.Size))
+					}
+				} else {
+					fmt.Fprintln(w, out.ID)
 				}
-			} else {
-				fmt.Fprintln(w, out.ID)
 			}
 		}
 
@@ -1128,6 +1214,48 @@ func (cli *DockerCli) CmdImages(args ...string) error {
 	return nil
 }
 
+func WalkTree(cli *DockerCli, noTrunc *bool, images []APIImages, byParent map[string][]APIImages, prefix string) {
+	if len(images) > 1 {
+		length := len(images)
+		for index, image := range images {
+			if index+1 == length {
+				PrintTreeNode(cli, noTrunc, image, prefix+"└─")
+				if subimages, exists := byParent[image.ID]; exists {
+					WalkTree(cli, noTrunc, subimages, byParent, prefix+"  ")
+				}
+			} else {
+				PrintTreeNode(cli, noTrunc, image, prefix+"|─")
+				if subimages, exists := byParent[image.ID]; exists {
+					WalkTree(cli, noTrunc, subimages, byParent, prefix+"| ")
+				}
+			}
+		}
+	} else {
+		for _, image := range images {
+			PrintTreeNode(cli, noTrunc, image, prefix+"└─")
+			if subimages, exists := byParent[image.ID]; exists {
+				WalkTree(cli, noTrunc, subimages, byParent, prefix+"  ")
+			}
+		}
+	}
+}
+
+func PrintTreeNode(cli *DockerCli, noTrunc *bool, image APIImages, prefix string) {
+	var imageID string
+	if *noTrunc {
+		imageID = image.ID
+	} else {
+		imageID = utils.TruncateID(image.ID)
+	}
+
+	fmt.Fprintf(cli.out, "%s%s Size: %s (virtual %s)", prefix, imageID, utils.HumanSize(image.Size), utils.HumanSize(image.VirtualSize))
+	if image.RepoTags[0] != "<none>:<none>" {
+		fmt.Fprintf(cli.out, " Tags: %s\n", strings.Join(image.RepoTags, ", "))
+	} else {
+		fmt.Fprint(cli.out, "\n")
+	}
+}
+
 func displayablePorts(ports []APIPort) string {
 	result := []string{}
 	for _, port := range ports {
@@ -1238,8 +1366,16 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
-	name := cmd.Arg(0)
-	repository, tag := utils.ParseRepositoryTag(cmd.Arg(1))
+
+	var name, repository, tag string
+
+	if cmd.NArg() == 3 {
+		fmt.Fprintf(cli.err, "[DEPRECATED] The format 'CONTAINER [REPOSITORY [TAG]]' as been deprecated. Please use CONTAINER [REPOSITORY[:TAG]]\n")
+		name, repository, tag = cmd.Arg(0), cmd.Arg(1), cmd.Arg(2)
+	} else {
+		name = cmd.Arg(0)
+		repository, tag = utils.ParseRepositoryTag(cmd.Arg(1))
+	}
 
 	if name == "" {
 		cmd.Usage()
@@ -1276,7 +1412,7 @@ func (cli *DockerCli) CmdCommit(args ...string) error {
 
 func (cli *DockerCli) CmdEvents(args ...string) error {
 	cmd := Subcmd("events", "[OPTIONS]", "Get real time events from the server")
-	since := cmd.String("since", "", "Show events previously created (used for polling).")
+	since := cmd.String("since", "", "Show previously created events and then stream.")
 	if err := cmd.Parse(args); err != nil {
 		return nil
 	}
@@ -1288,7 +1424,17 @@ func (cli *DockerCli) CmdEvents(args ...string) error {
 
 	v := url.Values{}
 	if *since != "" {
-		v.Set("since", *since)
+		loc := time.FixedZone(time.Now().Zone())
+		format := "2006-01-02 15:04:05 -0700 MST"
+		if len(*since) < len(format) {
+			format = format[:len(*since)]
+		}
+
+		if t, err := time.ParseInLocation(format, *since, loc); err == nil {
+			v.Set("since", strconv.FormatInt(t.Unix(), 10))
+		} else {
+			v.Set("since", *since)
+		}
 	}
 
 	if err := cli.stream("GET", "/events?"+v.Encode(), nil, cli.out, nil); err != nil {
@@ -1545,9 +1691,16 @@ func (cli *DockerCli) CmdTag(args ...string) error {
 		return nil
 	}
 
-	v := url.Values{}
-	repository, tag := utils.ParseRepositoryTag(cmd.Arg(1))
+	var repository, tag string
 
+	if cmd.NArg() == 3 {
+		fmt.Fprintf(cli.err, "[DEPRECATED] The format 'IMAGE [REPOSITORY [TAG]]' as been deprecated. Please use IMAGE [REPOSITORY[:TAG]]\n")
+		repository, tag = cmd.Arg(1), cmd.Arg(2)
+	} else {
+		repository, tag = utils.ParseRepositoryTag(cmd.Arg(1))
+	}
+
+	v := url.Values{}
 	v.Set("repo", repository)
 	v.Set("tag", tag)
 
@@ -1800,7 +1953,7 @@ func (cli *DockerCli) CmdCp(args ...string) error {
 
 	if statusCode == 200 {
 		r := bytes.NewReader(data)
-		if err := archive.Untar(r, copyData.HostPath); err != nil {
+		if err := archive.Untar(r, copyData.HostPath, nil); err != nil {
 			return err
 		}
 	}
@@ -1858,7 +2011,7 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int,
 		if len(body) == 0 {
 			return nil, resp.StatusCode, fmt.Errorf("Error: %s", http.StatusText(resp.StatusCode))
 		}
-		return nil, resp.StatusCode, fmt.Errorf("Error: %s", body)
+		return nil, resp.StatusCode, fmt.Errorf("Error: %s", bytes.TrimSpace(body))
 	}
 	return body, resp.StatusCode, nil
 }
@@ -1914,7 +2067,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h
 		if len(body) == 0 {
 			return fmt.Errorf("Error :%s", http.StatusText(resp.StatusCode))
 		}
-		return fmt.Errorf("Error: %s", body)
+		return fmt.Errorf("Error: %s", bytes.TrimSpace(body))
 	}
 
 	if matchesContentType(resp.Header.Get("Content-Type"), "application/json") {

commands_test.go

@@ -6,6 +6,9 @@ import (
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -380,8 +383,8 @@ func TestRunAttachStdin(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		if cmdOutput != container.ShortID()+"\n" {
-			t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortID()+"\n", cmdOutput)
+		if cmdOutput != container.ID+"\n" {
+			t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ID+"\n", cmdOutput)
 		}
 	})
 
@@ -458,7 +461,7 @@ func TestRunDetach(t *testing.T) {
 	})
 }
 
-// TestAttachDetach checks that attach in tty mode can be detached
+// TestAttachDetach checks that attach in tty mode can be detached using the long container ID
 func TestAttachDetach(t *testing.T) {
 	stdin, stdinPipe := io.Pipe()
 	stdout, stdoutPipe := io.Pipe()
@@ -485,8 +488,8 @@ func TestAttachDetach(t *testing.T) {
 
 		container = globalRuntime.List()[0]
 
-		if strings.Trim(string(buf[:n]), " \r\n") != container.ShortID() {
-			t.Fatalf("Wrong ID received. Expect %s, received %s", container.ShortID(), buf[:n])
+		if strings.Trim(string(buf[:n]), " \r\n") != container.ID {
+			t.Fatalf("Wrong ID received. Expect %s, received %s", container.ID, buf[:n])
 		}
 	})
 	setTimeout(t, "Starting container timed out", 10*time.Second, func() {
@@ -500,7 +503,69 @@ func TestAttachDetach(t *testing.T) {
 	ch = make(chan struct{})
 	go func() {
 		defer close(ch)
-		if err := cli.CmdAttach(container.ShortID()); err != nil {
+		if err := cli.CmdAttach(container.ID); err != nil {
+			if err != io.ErrClosedPipe {
+				t.Fatal(err)
+			}
+		}
+	}()
+
+	setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() {
+		if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil {
+			if err != io.ErrClosedPipe {
+				t.Fatal(err)
+			}
+		}
+	})
+
+	setTimeout(t, "Escape sequence timeout", 5*time.Second, func() {
+		stdinPipe.Write([]byte{16, 17})
+		if err := stdinPipe.Close(); err != nil {
+			t.Fatal(err)
+		}
+	})
+	closeWrap(stdin, stdinPipe, stdout, stdoutPipe)
+
+	// wait for CmdRun to return
+	setTimeout(t, "Waiting for CmdAttach timed out", 15*time.Second, func() {
+		<-ch
+	})
+
+	time.Sleep(500 * time.Millisecond)
+	if !container.State.Running {
+		t.Fatal("The detached container should be still running")
+	}
+
+	setTimeout(t, "Waiting for container to die timedout", 5*time.Second, func() {
+		container.Kill()
+	})
+}
+
+// TestAttachDetachTruncatedID checks that attach in tty mode can be detached
+func TestAttachDetachTruncatedID(t *testing.T) {
+	stdin, stdinPipe := io.Pipe()
+	stdout, stdoutPipe := io.Pipe()
+
+	cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
+	go stdout.Read(make([]byte, 1024))
+	setTimeout(t, "Starting container timed out", 2*time.Second, func() {
+		if err := cli.CmdRun("-i", "-t", "-d", unitTestImageID, "cat"); err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	container := globalRuntime.List()[0]
+
+	stdin, stdinPipe = io.Pipe()
+	stdout, stdoutPipe = io.Pipe()
+	cli = NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+
+	ch := make(chan struct{})
+	go func() {
+		defer close(ch)
+		if err := cli.CmdAttach(utils.TruncateID(container.ID)); err != nil {
 			if err != io.ErrClosedPipe {
 				t.Fatal(err)
 			}
@@ -699,3 +764,179 @@ func TestRunErrorBindNonExistingSource(t *testing.T) {
 		<-c
 	})
 }
+
+func TestImagesViz(t *testing.T) {
+	stdout, stdoutPipe := io.Pipe()
+
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
+	srv := &Server{runtime: globalRuntime}
+	image := buildTestImages(t, srv)
+
+	c := make(chan struct{})
+	go func() {
+		defer close(c)
+		if err := cli.CmdImages("-viz"); err != nil {
+			t.Fatal(err)
+		}
+		stdoutPipe.Close()
+	}()
+
+	setTimeout(t, "Reading command output time out", 2*time.Second, func() {
+		cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout))
+		if err != nil {
+			t.Fatal(err)
+		}
+		cmdOutput := string(cmdOutputBytes)
+
+		regexpStrings := []string{
+			"digraph docker {",
+			fmt.Sprintf("base -> \"%s\" \\[style=invis]", unitTestImageIDShort),
+			fmt.Sprintf("label=\"%s\\\\n%s:latest\"", unitTestImageIDShort, unitTestImageName),
+			fmt.Sprintf("label=\"%s\\\\n%s:%s\"", utils.TruncateID(image.ID), "test", "latest"),
+			"base \\[style=invisible]",
+		}
+
+		compiledRegexps := []*regexp.Regexp{}
+		for _, regexpString := range regexpStrings {
+			regexp, err := regexp.Compile(regexpString)
+			if err != nil {
+				fmt.Println("Error in regex string: ", err)
+				return
+			}
+			compiledRegexps = append(compiledRegexps, regexp)
+		}
+
+		for _, regexp := range compiledRegexps {
+			if !regexp.MatchString(cmdOutput) {
+				t.Fatalf("images -viz content '%s' did not match regexp '%s'", cmdOutput, regexp)
+			}
+		}
+	})
+}
+
+func TestImagesTree(t *testing.T) {
+	stdout, stdoutPipe := io.Pipe()
+
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
+	srv := &Server{runtime: globalRuntime}
+	image := buildTestImages(t, srv)
+
+	c := make(chan struct{})
+	go func() {
+		defer close(c)
+		if err := cli.CmdImages("-tree"); err != nil {
+			t.Fatal(err)
+		}
+		stdoutPipe.Close()
+	}()
+
+	setTimeout(t, "Reading command output time out", 2*time.Second, func() {
+		cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout))
+		if err != nil {
+			t.Fatal(err)
+		}
+		cmdOutput := string(cmdOutputBytes)
+		regexpStrings := []string{
+			fmt.Sprintf("└─%s Size: (\\d+.\\d+ MB) \\(virtual \\d+.\\d+ MB\\) Tags: %s:latest", unitTestImageIDShort, unitTestImageName),
+			"(?m)   └─[0-9a-f]+.*",
+			"(?m)    └─[0-9a-f]+.*",
+			"(?m)      └─[0-9a-f]+.*",
+			fmt.Sprintf("(?m)^        └─%s Size: \\d+.\\d+ MB \\(virtual \\d+.\\d+ MB\\) Tags: test:latest", utils.TruncateID(image.ID)),
+		}
+
+		compiledRegexps := []*regexp.Regexp{}
+		for _, regexpString := range regexpStrings {
+			regexp, err := regexp.Compile(regexpString)
+			if err != nil {
+				fmt.Println("Error in regex string: ", err)
+				return
+			}
+			compiledRegexps = append(compiledRegexps, regexp)
+		}
+
+		for _, regexp := range compiledRegexps {
+			if !regexp.MatchString(cmdOutput) {
+				t.Fatalf("images -tree content '%s' did not match regexp '%s'", cmdOutput, regexp)
+			}
+		}
+	})
+}
+
+func buildTestImages(t *testing.T, srv *Server) *Image {
+
+	var testBuilder = testContextTemplate{
+		`
+from   {IMAGE}
+run    sh -c 'echo root:testpass > /tmp/passwd'
+run    mkdir -p /var/run/sshd
+run    [ "$(cat /tmp/passwd)" = "root:testpass" ]
+run    [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]
+`,
+		nil,
+		nil,
+	}
+	image := buildImage(testBuilder, t, srv, true)
+
+	err := srv.ContainerTag(image.ID, "test", "latest", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return image
+}
+
+// #2098 - Docker cidFiles only contain short version of the containerId
+//sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test"
+// TestRunCidFile tests that run -cidfile returns the longid
+func TestRunCidFile(t *testing.T) {
+	stdout, stdoutPipe := io.Pipe()
+
+	tmpDir, err := ioutil.TempDir("", "TestRunCidFile")
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpCidFile := path.Join(tmpDir, "cid")
+
+	cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr)
+	defer cleanup(globalRuntime)
+
+	c := make(chan struct{})
+	go func() {
+		defer close(c)
+		if err := cli.CmdRun("-cidfile", tmpCidFile, unitTestImageID, "ls"); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	defer os.RemoveAll(tmpDir)
+	setTimeout(t, "Reading command output time out", 2*time.Second, func() {
+		cmdOutput, err := bufio.NewReader(stdout).ReadString('\n')
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(cmdOutput) < 1 {
+			t.Fatalf("'ls' should return something , not '%s'", cmdOutput)
+		}
+		//read the tmpCidFile
+		buffer, err := ioutil.ReadFile(tmpCidFile)
+		if err != nil {
+			t.Fatal(err)
+		}
+		id := string(buffer)
+
+		if len(id) != len("2bf44ea18873287bd9ace8a4cb536a7cbe134bed67e805fdf2f58a57f69b320c") {
+			t.Fatalf("-cidfile should be a long id, not '%s'", id)
+		}
+		//test that its a valid cid? (though the container is gone..)
+		//remove the file and dir.
+	})
+
+	setTimeout(t, "CmdRun timed out", 5*time.Second, func() {
+		<-c
+	})
+
+}

config.go

@@ -1,15 +1,14 @@
 package docker
 
 import (
-	"net"
 	"github.com/dotcloud/docker/engine"
+	"net"
 )
 
 // FIXME: separate runtime configuration from http api configuration
 type DaemonConfig struct {
 	Pidfile                     string
 	Root                        string
-	ProtoAddresses              []string
 	AutoRestart                 bool
 	EnableCors                  bool
 	Dns                         []string
@@ -36,7 +35,6 @@ func ConfigFromJob(job *engine.Job) *DaemonConfig {
 	} else {
 		config.BridgeIface = DefaultNetworkBridge
 	}
-	config.ProtoAddresses = job.GetenvList("ProtoAddresses")
 	config.DefaultIp = net.ParseIP(job.Getenv("DefaultIp"))
 	config.InterContainerCommunication = job.GetenvBool("InterContainerCommunication")
 	return &config

container.go

@@ -25,7 +25,8 @@ import (
 )
 
 type Container struct {
-	root string
+	root   string // Path to the "home" of the container, including metadata.
+	rootfs string // Path to the root filesystem of the container.
 
 	ID string
 
@@ -133,7 +134,11 @@ type PortBinding struct {
 type Port string
 
 func (p Port) Proto() string {
-	return strings.Split(string(p), "/")[1]
+	parts := strings.Split(string(p), "/")
+	if len(parts) == 1 {
+		return "tcp"
+	}
+	return parts[1]
 }
 
 func (p Port) Port() string {
@@ -167,7 +172,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 	cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.")
 	flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached")
 	flTty := cmd.Bool("t", false, "Allocate a pseudo-tty")
-	flMemory := cmd.Int64("m", 0, "Memory limit (in bytes)")
+	flMemoryString := cmd.String("m", "", "Memory limit (format: <number><optional unit>, where unit = b, k, m or g)")
 	flContainerIDFile := cmd.String("cidfile", "", "Write the container ID to the file")
 	flNetwork := cmd.Bool("n", true, "Enable networking for this container")
 	flPrivileged := cmd.Bool("privileged", false, "Give extended privileges to this container")
@@ -176,9 +181,9 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 	cmd.String("name", "", "Assign a name to the container")
 	flPublishAll := cmd.Bool("P", false, "Publish all exposed ports to the host interfaces")
 
-	if capabilities != nil && *flMemory > 0 && !capabilities.MemoryLimit {
+	if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit {
 		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n")
-		*flMemory = 0
+		*flMemoryString = ""
 	}
 
 	flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)")
@@ -199,7 +204,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 	cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)")
 
 	var flVolumesFrom utils.ListOpts
-	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container")
+	cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)")
 
 	flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image")
 
@@ -245,6 +250,18 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 		}
 	}
 
+	var flMemory int64
+
+	if *flMemoryString != "" {
+		parsedMemory, err := utils.RAMInBytes(*flMemoryString)
+
+		if err != nil {
+			return nil, nil, cmd, err
+		}
+
+		flMemory = parsedMemory
+	}
+
 	var binds []string
 
 	// add any bind targets to the list of container volumes
@@ -315,7 +332,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 		Tty:             *flTty,
 		NetworkDisabled: !*flNetwork,
 		OpenStdin:       *flStdin,
-		Memory:          *flMemory,
+		Memory:          flMemory,
 		CpuShares:       *flCpuShares,
 		AttachStdin:     flAttach.Get("stdin"),
 		AttachStdout:    flAttach.Get("stdout"),
@@ -340,7 +357,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig,
 		PublishAllPorts: *flPublishAll,
 	}
 
-	if capabilities != nil && *flMemory > 0 && !capabilities.SwapLimit {
+	if capabilities != nil && flMemory > 0 && !capabilities.SwapLimit {
 		//fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n")
 		config.MemorySwap = -1
 	}
@@ -390,26 +407,33 @@ func (settings *NetworkSettings) PortMappingAPI() []APIPort {
 
 // Inject the io.Reader at the given path. Note: do not close the reader
 func (container *Container) Inject(file io.Reader, pth string) error {
+	if err := container.EnsureMounted(); err != nil {
+		return fmt.Errorf("inject: error mounting container %s: %s", container.ID, err)
+	}
+
 	// Return error if path exists
-	if _, err := os.Stat(path.Join(container.rwPath(), pth)); err == nil {
+	destPath := path.Join(container.RootfsPath(), pth)
+	if _, err := os.Stat(destPath); err == nil {
 		// Since err is nil, the path could be stat'd and it exists
 		return fmt.Errorf("%s exists", pth)
-	} else if ! os.IsNotExist(err) {
+	} else if !os.IsNotExist(err) {
 		// Expect err might be that the file doesn't exist, so
-		// if it's some other error, return that. 
+		// if it's some other error, return that.
 
 		return err
 	}
 
 	// Make sure the directory exists
-	if err := os.MkdirAll(path.Join(container.rwPath(), path.Dir(pth)), 0755); err != nil {
+	if err := os.MkdirAll(path.Join(container.RootfsPath(), path.Dir(pth)), 0755); err != nil {
 		return err
 	}
 
-	dest, err := os.Create(path.Join(container.rwPath(), pth))
+	dest, err := os.Create(destPath)
 	if err != nil {
 		return err
 	}
+	defer dest.Close()
+
 	if _, err := io.Copy(dest, file); err != nil {
 		return err
 	}
@@ -686,24 +710,25 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s
 func (container *Container) Start() (err error) {
 	container.State.Lock()
 	defer container.State.Unlock()
+	if container.State.Running {
+		return fmt.Errorf("The container %s is already running.", container.ID)
+	}
 	defer func() {
 		if err != nil {
 			container.cleanup()
 		}
 	}()
-
-	if container.State.Running {
-		return fmt.Errorf("The container %s is already running.", container.ID)
-	}
 	if err := container.EnsureMounted(); err != nil {
 		return err
 	}
 	if container.runtime.networkManager.disabled {
 		container.Config.NetworkDisabled = true
+		container.buildHostnameAndHostsFiles("127.0.1.1")
 	} else {
 		if err := container.allocateNetwork(); err != nil {
 			return err
 		}
+		container.buildHostnameAndHostsFiles(container.NetworkSettings.IPAddress)
 	}
 
 	// Make sure the config is compatible with the current kernel
@@ -763,9 +788,23 @@ func (container *Container) Start() (err error) {
 
 	// Apply volumes from another container if requested
 	if container.Config.VolumesFrom != "" {
-		volumes := strings.Split(container.Config.VolumesFrom, ",")
-		for _, v := range volumes {
-			c := container.runtime.Get(v)
+		containerSpecs := strings.Split(container.Config.VolumesFrom, ",")
+		for _, containerSpec := range containerSpecs {
+			mountRW := true
+			specParts := strings.SplitN(containerSpec, ":", 2)
+			switch len(specParts) {
+			case 0:
+				return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom)
+			case 2:
+				switch specParts[1] {
+				case "ro":
+					mountRW = false
+				case "rw": // mountRW is already true
+				default:
+					return fmt.Errorf("Malformed volumes-from speficication: %s", containerSpec)
+				}
+			}
+			c := container.runtime.Get(specParts[0])
 			if c == nil {
 				return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID)
 			}
@@ -778,13 +817,14 @@ func (container *Container) Start() (err error) {
 				}
 				container.Volumes[volPath] = id
 				if isRW, exists := c.VolumesRW[volPath]; exists {
-					container.VolumesRW[volPath] = isRW
+					container.VolumesRW[volPath] = isRW && mountRW
 				}
 			}
 
 		}
 	}
 
+	volumesDriver := container.runtime.volumes.driver
 	// Create the requested volumes if they don't exist
 	for volPath := range container.Config.Volumes {
 		volPath = path.Clean(volPath)
@@ -804,13 +844,17 @@ func (container *Container) Start() (err error) {
 			}
 			// Otherwise create an directory in $ROOT/volumes/ and use that
 		} else {
-			c, err := container.runtime.volumes.Create(nil, container, "", "", nil)
+
+			// Do not pass a container as the parameter for the volume creation.
+			// The graph driver using the container's information ( Image ) to
+			// create the parent.
+			c, err := container.runtime.volumes.Create(nil, nil, "", "", nil)
 			if err != nil {
 				return err
 			}
-			srcPath, err = c.layer()
+			srcPath, err = volumesDriver.Get(c.ID)
 			if err != nil {
-				return err
+				return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err)
 			}
 			srcRW = true // RW by default
 		}
@@ -819,7 +863,7 @@ func (container *Container) Start() (err error) {
 		// Create the mountpoint
 		rootVolPath := path.Join(container.RootfsPath(), volPath)
 		if err := os.MkdirAll(rootVolPath, 0755); err != nil {
-			return nil
+			return err
 		}
 
 		// Do not copy or change permissions if we are mounting from the host
@@ -863,7 +907,13 @@ func (container *Container) Start() (err error) {
 		return err
 	}
 
+	var lxcStart string = "lxc-start"
+	if container.hostConfig.Privileged && container.runtime.capabilities.AppArmor {
+		lxcStart = path.Join(container.runtime.config.Root, "lxc-start-unconfined")
+	}
+
 	params := []string{
+		lxcStart,
 		"-n", container.ID,
 		"-f", container.lxcConfigPath(),
 		"--",
@@ -956,11 +1006,24 @@ func (container *Container) Start() (err error) {
 	params = append(params, "--", container.Path)
 	params = append(params, container.Args...)
 
-	var lxcStart string = "lxc-start"
-	if container.hostConfig.Privileged && container.runtime.capabilities.AppArmor {
-		lxcStart = path.Join(container.runtime.config.Root, "lxc-start-unconfined")
+	if RootIsShared() {
+		// lxc-start really needs / to be non-shared, or all kinds of stuff break
+		// when lxc-start unmount things and those unmounts propagate to the main
+		// mount namespace.
+		// What we really want is to clone into a new namespace and then
+		// mount / MS_REC|MS_SLAVE, but since we can't really clone or fork
+		// without exec in go we have to do this horrible shell hack...
+		shellString :=
+			"mount --make-rslave /; exec " +
+				utils.ShellQuoteArguments(params)
+
+		params = []string{
+			"unshare", "-m", "--", "/bin/sh", "-c", shellString,
+		}
 	}
-	container.cmd = exec.Command(lxcStart, params...)
+
+	container.cmd = exec.Command(params[0], params[1:]...)
+
 	// Setup logging of stdout and stderr to disk
 	if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil {
 		return err
@@ -1069,6 +1132,30 @@ func (container *Container) StderrPipe() (io.ReadCloser, error) {
 	return utils.NewBufReader(reader), nil
 }
 
+func (container *Container) buildHostnameAndHostsFiles(IP string) {
+	container.HostnamePath = path.Join(container.root, "hostname")
+	ioutil.WriteFile(container.HostnamePath, []byte(container.Config.Hostname+"\n"), 0644)
+
+	hostsContent := []byte(`
+127.0.0.1	localhost
+::1		localhost ip6-localhost ip6-loopback
+fe00::0		ip6-localnet
+ff00::0		ip6-mcastprefix
+ff02::1		ip6-allnodes
+ff02::2		ip6-allrouters
+`)
+
+	container.HostsPath = path.Join(container.root, "hosts")
+
+	if container.Config.Domainname != "" {
+		hostsContent = append([]byte(fmt.Sprintf("%s\t%s.%s %s\n", IP, container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...)
+	} else {
+		hostsContent = append([]byte(fmt.Sprintf("%s\t%s\n", IP, container.Config.Hostname)), hostsContent...)
+	}
+
+	ioutil.WriteFile(container.HostsPath, hostsContent, 0644)
+}
+
 func (container *Container) allocateNetwork() error {
 	if container.Config.NetworkDisabled {
 		return nil
@@ -1086,7 +1173,7 @@ func (container *Container) allocateNetwork() error {
 				Gateway: manager.bridgeNetwork.IP,
 				manager: manager,
 			}
-			if iface !=nil && iface.IPNet.IP != nil {
+			if iface != nil && iface.IPNet.IP != nil {
 				ipNum := ipToInt(iface.IPNet.IP)
 				manager.ipAllocator.inUse[ipNum] = struct{}{}
 			} else {
@@ -1217,7 +1304,7 @@ func (container *Container) monitor() {
 	container.State.setStopped(exitCode)
 
 	if container.runtime != nil && container.runtime.srv != nil {
-		container.runtime.srv.LogEvent("die", container.ShortID(), container.runtime.repositories.ImageName(container.Image))
+		container.runtime.srv.LogEvent("die", container.ID, container.runtime.repositories.ImageName(container.Image))
 	}
 
 	// Cleanup
@@ -1284,7 +1371,7 @@ func (container *Container) kill(sig int) error {
 	}
 
 	if output, err := exec.Command("lxc-kill", "-n", container.ID, strconv.Itoa(sig)).CombinedOutput(); err != nil {
-		log.Printf("error killing container %s (%s, %s)", container.ShortID(), output, err)
+		log.Printf("error killing container %s (%s, %s)", utils.TruncateID(container.ID), output, err)
 		return err
 	}
 
@@ -1304,9 +1391,9 @@ func (container *Container) Kill() error {
 	// 2. Wait for the process to die, in last resort, try to kill the process directly
 	if err := container.WaitTimeout(10 * time.Second); err != nil {
 		if container.cmd == nil {
-			return fmt.Errorf("lxc-kill failed, impossible to kill the container %s", container.ShortID())
+			return fmt.Errorf("lxc-kill failed, impossible to kill the container %s", utils.TruncateID(container.ID))
 		}
-		log.Printf("Container %s failed to exit within 10 seconds of lxc-kill %s - trying direct SIGKILL", "SIGKILL", container.ShortID())
+		log.Printf("Container %s failed to exit within 10 seconds of lxc-kill %s - trying direct SIGKILL", "SIGKILL", utils.TruncateID(container.ID))
 		if err := container.cmd.Process.Kill(); err != nil {
 			return err
 		}
@@ -1363,15 +1450,14 @@ func (container *Container) Resize(h, w int) error {
 }
 
 func (container *Container) ExportRw() (archive.Archive, error) {
-	return archive.Tar(container.rwPath(), archive.Uncompressed)
-}
-
-func (container *Container) RwChecksum() (string, error) {
-	rwData, err := archive.Tar(container.rwPath(), archive.Xz)
-	if err != nil {
-		return "", err
+	if err := container.EnsureMounted(); err != nil {
+		return nil, err
 	}
-	return utils.HashData(rwData)
+	if container.runtime == nil {
+		return nil, fmt.Errorf("Can't load storage driver for unregistered container %s", container.ID)
+	}
+
+	return container.runtime.Diff(container)
 }
 
 func (container *Container) Export() (archive.Archive, error) {
@@ -1397,28 +1483,17 @@ func (container *Container) WaitTimeout(timeout time.Duration) error {
 }
 
 func (container *Container) EnsureMounted() error {
-	if mounted, err := container.Mounted(); err != nil {
-		return err
-	} else if mounted {
-		return nil
-	}
+	// FIXME: EnsureMounted is deprecated because drivers are now responsible
+	// for re-entrant mounting in their Get() method.
 	return container.Mount()
 }
 
 func (container *Container) Mount() error {
-	image, err := container.GetImage()
-	if err != nil {
-		return err
-	}
-	return image.Mount(container.RootfsPath(), container.rwPath())
+	return container.runtime.Mount(container)
 }
 
-func (container *Container) Changes() ([]Change, error) {
-	image, err := container.GetImage()
-	if err != nil {
-		return nil, err
-	}
-	return image.Changes(container.rwPath())
+func (container *Container) Changes() ([]archive.Change, error) {
+	return container.runtime.Changes(container)
 }
 
 func (container *Container) GetImage() (*Image, error) {
@@ -1428,26 +1503,8 @@ func (container *Container) GetImage() (*Image, error) {
 	return container.runtime.graph.Get(container.Image)
 }
 
-func (container *Container) Mounted() (bool, error) {
-	return Mounted(container.RootfsPath())
-}
-
 func (container *Container) Unmount() error {
-	if _, err := os.Stat(container.RootfsPath()); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	return Unmount(container.RootfsPath())
-}
-
-// ShortID returns a shorthand version of the container's id for convenience.
-// A collision with other container shorthands is very unlikely, but possible.
-// In case of a collision a lookup with Runtime.Get() will fail, and the caller
-// will need to use a langer prefix, or the full-length container Id.
-func (container *Container) ShortID() string {
-	return utils.TruncateID(container.ID)
+	return container.runtime.Unmount(container)
 }
 
 func (container *Container) logPath(name string) string {
@@ -1476,11 +1533,7 @@ func (container *Container) lxcConfigPath() string {
 
 // This method must be exported to be used from the lxc template
 func (container *Container) RootfsPath() string {
-	return path.Join(container.root, "rootfs")
-}
-
-func (container *Container) rwPath() string {
-	return path.Join(container.root, "rw")
+	return container.rootfs
 }
 
 func validateID(id string) error {
@@ -1492,16 +1545,25 @@ func validateID(id string) error {
 
 // GetSize, return real size, virtual size
 func (container *Container) GetSize() (int64, int64) {
-	var sizeRw, sizeRootfs int64
+	var (
+		sizeRw, sizeRootfs int64
+		err                error
+		driver             = container.runtime.driver
+	)
 
-	filepath.Walk(container.rwPath(), func(path string, fileInfo os.FileInfo, err error) error {
-		if fileInfo != nil {
-			sizeRw += fileInfo.Size()
-		}
-		return nil
-	})
+	sizeRw, err = driver.Size(container.ID)
+	if err != nil {
+		utils.Errorf("Warning: driver %s couldn't return diff size of container %s: %s", driver, container.ID, err)
+		// FIXME: GetSize should return an error. Not changing it now in case
+		// there is a side-effect.
+		sizeRw = -1
+	}
 
-	_, err := os.Stat(container.RootfsPath())
+	if err := container.EnsureMounted(); err != nil {
+		utils.Errorf("Warning: failed to compute size of container rootfs %s: %s", container.ID, err)
+		return sizeRw, sizeRootfs
+	}
+	_, err = os.Stat(container.RootfsPath())
 	if err == nil {
 		filepath.Walk(container.RootfsPath(), func(path string, fileInfo os.FileInfo, err error) error {
 			if fileInfo != nil {
@@ -1531,7 +1593,11 @@ func (container *Container) Copy(resource string) (archive.Archive, error) {
 		filter = []string{path.Base(basePath)}
 		basePath = path.Dir(basePath)
 	}
-	return archive.TarFilter(basePath, archive.Uncompressed, filter)
+	return archive.TarFilter(basePath, &archive.TarOptions{
+		Compression: archive.Uncompressed,
+		Includes:    filter,
+		Recursive:   true,
+	})
 }
 
 // Returns true if the container exposes a certain port

container_test.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"bufio"
 	"fmt"
+	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -170,11 +171,11 @@ func TestDiff(t *testing.T) {
 	// Commit the container
 	rwTar, err := container1.ExportRw()
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 	img, err := runtime.graph.Create(rwTar, container1, "unit test commited image - diff", "", nil)
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 
 	// Create a new container from the commited image
@@ -1005,7 +1006,7 @@ func TestEnv(t *testing.T) {
 		"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 		"HOME=/",
 		"container=lxc",
-		"HOSTNAME=" + container.ShortID(),
+		"HOSTNAME=" + utils.TruncateID(container.ID),
 		"FALSE=true",
 		"TRUE=false",
 		"TRICKY=tri",
@@ -1338,6 +1339,67 @@ func TestBindMounts(t *testing.T) {
 	}
 }
 
+// Test that -volumes-from supports both read-only mounts
+func TestFromVolumesInReadonlyMode(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+	container, _, err := runtime.Create(
+		&Config{
+			Image:   GetTestImage(runtime).ID,
+			Cmd:     []string{"/bin/echo", "-n", "foobar"},
+			Volumes: map[string]struct{}{"/test": {}},
+		},
+		"",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container)
+	_, err = container.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !container.VolumesRW["/test"] {
+		t.Fail()
+	}
+
+	container2, _, err := runtime.Create(
+		&Config{
+			Image:       GetTestImage(runtime).ID,
+			Cmd:         []string{"/bin/echo", "-n", "foobar"},
+			VolumesFrom: container.ID + ":ro",
+		},
+		"",
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer runtime.Destroy(container2)
+
+	_, err = container2.Output()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if container.Volumes["/test"] != container2.Volumes["/test"] {
+		t.Logf("container volumes do not match: %s | %s ",
+			container.Volumes["/test"],
+			container2.Volumes["/test"])
+		t.Fail()
+	}
+
+	_, exists := container2.VolumesRW["/test"]
+	if !exists {
+		t.Logf("container2 is missing '/test' volume: %s", container2.VolumesRW)
+		t.Fail()
+	}
+
+	if container2.VolumesRW["/test"] != false {
+		t.Log("'/test' volume mounted in read-write mode, expected read-only")
+		t.Fail()
+	}
+}
+
 // Test that VolumesRW values are copied to the new container.  Regression test for #1201
 func TestVolumesFromReadonlyMount(t *testing.T) {
 	runtime := mkRuntime(t)
@@ -1680,3 +1742,67 @@ func TestRestartGhost(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestRemoveFile(t *testing.T) {
+	runtime := mkRuntime(t)
+	defer nuke(runtime)
+
+	container1, _ := mkContainer(runtime, []string{"_", "/bin/sh", "-c", "touch test.txt"}, t)
+	defer runtime.Destroy(container1)
+
+	if container1.State.Running {
+		t.Errorf("Container shouldn't be running")
+	}
+	if err := container1.Run(); err != nil {
+		t.Fatal(err)
+	}
+	if container1.State.Running {
+		t.Errorf("Container shouldn't be running")
+	}
+
+	commit := func(container *Container) (*Image, error) {
+		rwTar, err := container.ExportRw()
+		if err != nil {
+			return nil, err
+		}
+		img, err := runtime.graph.Create(rwTar, container, "unit test commited image", "", nil)
+		if err != nil {
+			return nil, err
+		}
+		return img, nil
+	}
+
+	img, err := commit(container1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container2, _ := mkContainer(runtime, []string{img.ID, "/bin/sh", "-c", "rm /test.txt"}, t)
+	defer runtime.Destroy(container2)
+
+	if err := container2.Run(); err != nil {
+		t.Fatal(err)
+	}
+
+	containerMount, err := runtime.driver.Get(container2.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(path.Join(containerMount, "test.txt")); err == nil {
+		t.Fatalf("test.txt should not exist")
+	}
+
+	img, err = commit(container2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	mountPoint, err := runtime.driver.Get(img.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	file := path.Join(mountPoint, "test.txt")
+	if _, err := os.Stat(file); err == nil {
+		t.Fatalf("The file %s should not exist\n", file)
+	}
+}

contrib/init/sysvinit/docker

@@ -29,7 +29,9 @@ if [ -f /etc/default/$BASE ]; then
 	. /etc/default/$BASE
 fi
 
-if [ "$1" = start ] && which initctl >/dev/null && initctl version | grep -q upstart; then
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | /bin/grep -q upstart; then
+	log_failure_msg "Docker is managed via upstart, try using service $BASE $1"
 	exit 1
 fi
 

contrib/init/upstart/docker.conf

@@ -6,5 +6,10 @@ stop on runlevel [!2345]
 respawn
 
 script
-	/usr/bin/docker -d
+	DOCKER=/usr/bin/$UPSTART_JOB
+	DOCKER_OPTS=
+	if [ -f /etc/default/$UPSTART_JOB ]; then
+		. /etc/default/$UPSTART_JOB
+	fi
+	"$DOCKER" -d $DOCKER_OPTS
 end script

contrib/vagrant-docker/README.md

@@ -1,3 +1,19 @@
-# Vagrant-docker
+# Vagrant integration
 
-This is a placeholder for the official vagrant-docker, a plugin for Vagrant (http://vagrantup.com) which exposes Docker as a provider.
+Currently there are at least 4 different projects that we are aware of that deals
+with integration with [Vagrant](http://vagrantup.com/) at different levels. One
+approach is to use Docker as a [provisioner](http://docs.vagrantup.com/v2/provisioning/index.html)
+which means you can create containers and pull base images on VMs using Docker's
+CLI and the other is to use Docker as a [provider](http://docs.vagrantup.com/v2/providers/index.html),
+meaning you can use Vagrant to control Docker containers.
+
+
+### Provisioners
+
+* [Vocker](https://github.com/fgrehm/vocker)
+* [Ventriloquist](https://github.com/fgrehm/ventriloquist)
+
+### Providers
+
+* [docker-provider](https://github.com/fgrehm/docker-provider)
+* [vagrant-shell](https://github.com/destructuring/vagrant-shell)

devmapper/deviceset.go

@@ -0,0 +1,803 @@
+package devmapper
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var (
+	DefaultDataLoopbackSize     int64  = 100 * 1024 * 1024 * 1024
+	DefaultMetaDataLoopbackSize int64  = 2 * 1024 * 1024 * 1024
+	DefaultBaseFsSize           uint64 = 10 * 1024 * 1024 * 1024
+)
+
+type DevInfo struct {
+	Hash          string     `json:"-"`
+	DeviceId      int        `json:"device_id"`
+	Size          uint64     `json:"size"`
+	TransactionId uint64     `json:"transaction_id"`
+	Initialized   bool       `json:"initialized"`
+	devices       *DeviceSet `json:"-"`
+}
+
+type MetaData struct {
+	Devices map[string]*DevInfo `json:devices`
+}
+
+type DeviceSet struct {
+	MetaData
+	sync.Mutex
+	root             string
+	devicePrefix     string
+	TransactionId    uint64
+	NewTransactionId uint64
+	nextFreeDevice   int
+	activeMounts     map[string]int
+}
+
+type DiskUsage struct {
+	Used  uint64
+	Total uint64
+}
+
+type Status struct {
+	PoolName         string
+	DataLoopback     string
+	MetadataLoopback string
+	Data             DiskUsage
+	Metadata         DiskUsage
+}
+
+func getDevName(name string) string {
+	return "/dev/mapper/" + name
+}
+
+func (info *DevInfo) Name() string {
+	hash := info.Hash
+	if hash == "" {
+		hash = "base"
+	}
+	return fmt.Sprintf("%s-%s", info.devices.devicePrefix, hash)
+}
+
+func (info *DevInfo) DevName() string {
+	return getDevName(info.Name())
+}
+
+func (devices *DeviceSet) loopbackDir() string {
+	return path.Join(devices.root, "devicemapper")
+}
+
+func (devices *DeviceSet) jsonFile() string {
+	return path.Join(devices.loopbackDir(), "json")
+}
+
+func (devices *DeviceSet) getPoolName() string {
+	return devices.devicePrefix + "-pool"
+}
+
+func (devices *DeviceSet) getPoolDevName() string {
+	return getDevName(devices.getPoolName())
+}
+
+func (devices *DeviceSet) hasImage(name string) bool {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	_, err := os.Stat(filename)
+	return err == nil
+}
+
+// ensureImage creates a sparse file of <size> bytes at the path
+// <root>/devicemapper/<name>.
+// If the file already exists, it does nothing.
+// Either way it returns the full path.
+func (devices *DeviceSet) ensureImage(name string, size int64) (string, error) {
+	dirname := devices.loopbackDir()
+	filename := path.Join(dirname, name)
+
+	if err := os.MkdirAll(dirname, 0700); err != nil && !os.IsExist(err) {
+		return "", err
+	}
+
+	if _, err := os.Stat(filename); err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+		utils.Debugf("Creating loopback file %s for device-manage use", filename)
+		file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			return "", err
+		}
+		defer file.Close()
+
+		if err = file.Truncate(size); err != nil {
+			return "", err
+		}
+	}
+	return filename, nil
+}
+
+func (devices *DeviceSet) allocateDeviceId() int {
+	// TODO: Add smarter reuse of deleted devices
+	id := devices.nextFreeDevice
+	devices.nextFreeDevice = devices.nextFreeDevice + 1
+	return id
+}
+
+func (devices *DeviceSet) allocateTransactionId() uint64 {
+	devices.NewTransactionId = devices.NewTransactionId + 1
+	return devices.NewTransactionId
+}
+
+func (devices *DeviceSet) saveMetadata() error {
+	jsonData, err := json.Marshal(devices.MetaData)
+	if err != nil {
+		return fmt.Errorf("Error encoding metaadata to json: %s", err)
+	}
+	tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json")
+	if err != nil {
+		return fmt.Errorf("Error creating metadata file: %s", err)
+	}
+
+	n, err := tmpFile.Write(jsonData)
+	if err != nil {
+		return fmt.Errorf("Error writing metadata to %s: %s", tmpFile.Name(), err)
+	}
+	if n < len(jsonData) {
+		return io.ErrShortWrite
+	}
+	if err := tmpFile.Sync(); err != nil {
+		return fmt.Errorf("Error syncing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return fmt.Errorf("Error closing metadata file %s: %s", tmpFile.Name(), err)
+	}
+	if err := os.Rename(tmpFile.Name(), devices.jsonFile()); err != nil {
+		return fmt.Errorf("Error committing metadata file", err)
+	}
+
+	if devices.NewTransactionId != devices.TransactionId {
+		if err = setTransactionId(devices.getPoolDevName(), devices.TransactionId, devices.NewTransactionId); err != nil {
+			return fmt.Errorf("Error setting devmapper transition ID: %s", err)
+		}
+		devices.TransactionId = devices.NewTransactionId
+	}
+	return nil
+}
+
+func (devices *DeviceSet) registerDevice(id int, hash string, size uint64) (*DevInfo, error) {
+	utils.Debugf("registerDevice(%v, %v)", id, hash)
+	info := &DevInfo{
+		Hash:          hash,
+		DeviceId:      id,
+		Size:          size,
+		TransactionId: devices.allocateTransactionId(),
+		Initialized:   false,
+		devices:       devices,
+	}
+
+	devices.Devices[hash] = info
+	if err := devices.saveMetadata(); err != nil {
+		// Try to remove unused device
+		delete(devices.Devices, hash)
+		return nil, err
+	}
+
+	return info, nil
+}
+
+func (devices *DeviceSet) activateDeviceIfNeeded(hash string) error {
+	utils.Debugf("activateDeviceIfNeeded(%v)", hash)
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	if devinfo, _ := getInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
+		return nil
+	}
+
+	return activateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
+}
+
+func (devices *DeviceSet) createFilesystem(info *DevInfo) error {
+	devname := info.DevName()
+
+	err := exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0,lazy_journal_init=0", devname).Run()
+	if err != nil {
+		err = exec.Command("mkfs.ext4", "-E", "discard,lazy_itable_init=0", devname).Run()
+	}
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) loadMetaData() error {
+	utils.Debugf("loadMetadata()")
+	defer utils.Debugf("loadMetadata END")
+	_, _, _, params, err := getStatus(devices.getPoolName())
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if _, err := fmt.Sscanf(params, "%d", &devices.TransactionId); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	devices.NewTransactionId = devices.TransactionId
+
+	jsonData, err := ioutil.ReadFile(devices.jsonFile())
+	if err != nil && !os.IsNotExist(err) {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	devices.MetaData.Devices = make(map[string]*DevInfo)
+	if jsonData != nil {
+		if err := json.Unmarshal(jsonData, &devices.MetaData); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	for hash, d := range devices.Devices {
+		d.Hash = hash
+		d.devices = devices
+
+		if d.DeviceId >= devices.nextFreeDevice {
+			devices.nextFreeDevice = d.DeviceId + 1
+		}
+
+		// If the transaction id is larger than the actual one we lost the device due to some crash
+		if d.TransactionId > devices.TransactionId {
+			utils.Debugf("Removing lost device %s with id %d", hash, d.TransactionId)
+			delete(devices.Devices, hash)
+		}
+	}
+	return nil
+}
+
+func (devices *DeviceSet) setupBaseImage() error {
+	oldInfo := devices.Devices[""]
+	if oldInfo != nil && oldInfo.Initialized {
+		return nil
+	}
+
+	if oldInfo != nil && !oldInfo.Initialized {
+		utils.Debugf("Removing uninitialized base image")
+		if err := devices.removeDevice(""); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	utils.Debugf("Initializing base device-manager snapshot")
+
+	id := devices.allocateDeviceId()
+
+	// Create initial device
+	if err := createDevice(devices.getPoolDevName(), id); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Registering base device (id %v) with FS size %v", id, DefaultBaseFsSize)
+	info, err := devices.registerDevice(id, "", DefaultBaseFsSize)
+	if err != nil {
+		_ = deleteDevice(devices.getPoolDevName(), id)
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	utils.Debugf("Creating filesystem on base device-manager snapshot")
+
+	if err = devices.activateDeviceIfNeeded(""); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	if err := devices.createFilesystem(info); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	info.Initialized = true
+	if err = devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func setCloseOnExec(name string) {
+	fileInfos, _ := ioutil.ReadDir("/proc/self/fd")
+	if fileInfos != nil {
+		for _, i := range fileInfos {
+			link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name()))
+			if link == name {
+				fd, err := strconv.Atoi(i.Name())
+				if err == nil {
+					syscall.CloseOnExec(fd)
+				}
+			}
+		}
+	}
+}
+
+func (devices *DeviceSet) log(level int, file string, line int, dmError int, message string) {
+	if level >= 7 {
+		return // Ignore _LOG_DEBUG
+	}
+
+	utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message)
+}
+
+func major(device uint64) uint64 {
+	return (device >> 8) & 0xfff
+}
+
+func minor(device uint64) uint64 {
+	return (device & 0xff) | ((device >> 12) & 0xfff00)
+}
+
+func (devices *DeviceSet) initDevmapper() error {
+	logInit(devices)
+
+	// Make sure the sparse images exist in <root>/devicemapper/data and
+	// <root>/devicemapper/metadata
+
+	createdLoopback := !devices.hasImage("data") || !devices.hasImage("metadata")
+	data, err := devices.ensureImage("data", DefaultDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (data): %s\n", err)
+		return err
+	}
+	metadata, err := devices.ensureImage("metadata", DefaultMetaDataLoopbackSize)
+	if err != nil {
+		utils.Debugf("Error device ensureImage (metadata): %s\n", err)
+		return err
+	}
+
+	// Set the device prefix from the device id and inode of the docker root dir
+
+	st, err := os.Stat(devices.root)
+	if err != nil {
+		return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
+	}
+	sysSt := st.Sys().(*syscall.Stat_t)
+	// "reg-" stands for "regular file".
+	// In the future we might use "dev-" for "device file", etc.
+	// docker-maj,min[-inode] stands for:
+	//	- Managed by docker
+	//	- The target of this device is at major <maj> and minor <min>
+	//	- If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
+	devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
+	utils.Debugf("Generated prefix: %s", devices.devicePrefix)
+
+	// Check for the existence of the device <prefix>-pool
+	utils.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
+	info, err := getInfo(devices.getPoolName())
+	if info == nil {
+		utils.Debugf("Error device getInfo: %s", err)
+		return err
+	}
+
+	// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
+	// that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
+	// so we add this badhack to make sure it closes itself
+	setCloseOnExec("/dev/mapper/control")
+
+	// If the pool doesn't exist, create it
+	if info.Exists == 0 {
+		utils.Debugf("Pool doesn't exist. Creating it.")
+
+		dataFile, err := AttachLoopDevice(data)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer dataFile.Close()
+
+		metadataFile, err := AttachLoopDevice(metadata)
+		if err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		defer metadataFile.Close()
+
+		if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// If we didn't just create the data or metadata image, we need to
+	// load the metadata from the existing file.
+	if !createdLoopback {
+		if err = devices.loadMetaData(); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+	}
+
+	// Setup the base image
+	if err := devices.setupBaseImage(); err != nil {
+		utils.Debugf("Error device setupBaseImage: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) AddDevice(hash, baseHash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if devices.Devices[hash] != nil {
+		return fmt.Errorf("hash %s already exists", hash)
+	}
+
+	baseInfo := devices.Devices[baseHash]
+	if baseInfo == nil {
+		return fmt.Errorf("Error adding device for '%s': can't find device for parent '%s'", hash, baseHash)
+	}
+
+	deviceId := devices.allocateDeviceId()
+
+	if err := devices.createSnapDevice(devices.getPoolDevName(), deviceId, baseInfo.Name(), baseInfo.DeviceId); err != nil {
+		utils.Debugf("Error creating snap device: %s\n", err)
+		return err
+	}
+
+	if _, err := devices.registerDevice(deviceId, hash, baseInfo.Size); err != nil {
+		deleteDevice(devices.getPoolDevName(), deviceId)
+		utils.Debugf("Error registering device: %s\n", err)
+		return err
+	}
+	return nil
+}
+
+func (devices *DeviceSet) removeDevice(hash string) error {
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("hash %s doesn't exists", hash)
+	}
+
+	devinfo, _ := getInfo(info.Name())
+	if devinfo != nil && devinfo.Exists != 0 {
+		if err := removeDevice(info.Name()); err != nil {
+			utils.Debugf("Error removing device: %s\n", err)
+			return err
+		}
+	}
+
+	if info.Initialized {
+		info.Initialized = false
+		if err := devices.saveMetadata(); err != nil {
+			utils.Debugf("Error saving meta data: %s\n", err)
+			return err
+		}
+	}
+
+	if err := deleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
+		utils.Debugf("Error deleting device: %s\n", err)
+		return err
+	}
+
+	devices.allocateTransactionId()
+	delete(devices.Devices, info.Hash)
+
+	if err := devices.saveMetadata(); err != nil {
+		devices.Devices[info.Hash] = info
+		utils.Debugf("Error saving meta data: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) RemoveDevice(hash string) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	return devices.removeDevice(hash)
+}
+
+func (devices *DeviceSet) deactivateDevice(hash string) error {
+	utils.Debugf("[devmapper] deactivateDevice(%s)", hash)
+	defer utils.Debugf("[devmapper] deactivateDevice END")
+	var devname string
+	// FIXME: shouldn't we just register the pool into devices?
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	devinfo, err := getInfo(devname)
+	if err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	if devinfo.Exists != 0 {
+		if err := removeDevice(devname); err != nil {
+			utils.Debugf("\n--->Err: %s\n", err)
+			return err
+		}
+		if err := devices.waitRemove(hash); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// waitRemove blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is removed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitRemove(hash string) error {
+	utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash)
+	defer utils.Debugf("[deviceset %s] waitRemove END", devices.devicePrefix, hash)
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			// If there is an error we assume the device doesn't exist.
+			// The error might actually be something else, but we can't differentiate.
+			return nil
+		}
+		utils.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
+		if devinfo.Exists == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
+	}
+	return nil
+}
+
+// waitClose blocks until either:
+// a) the device registered at <device_set_prefix>-<hash> is closed,
+// or b) the 1 second timeout expires.
+func (devices *DeviceSet) waitClose(hash string) error {
+	devname, err := devices.byHash(hash)
+	if err != nil {
+		return err
+	}
+	i := 0
+	for ; i < 1000; i += 1 {
+		devinfo, err := getInfo(devname)
+		if err != nil {
+			return err
+		}
+		utils.Debugf("Waiting for unmount of %s: opencount=%d", devname, devinfo.OpenCount)
+		if devinfo.OpenCount == 0 {
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	if i == 1000 {
+		return fmt.Errorf("Timeout while waiting for device %s to close", devname)
+	}
+	return nil
+}
+
+// byHash is a hack to allow looking up the deviceset's pool by the hash "pool".
+// FIXME: it seems probably cleaner to register the pool in devices.Devices,
+// but I am afraid of arcane implications deep in the devicemapper code,
+// so this will do.
+func (devices *DeviceSet) byHash(hash string) (devname string, err error) {
+	if hash == "pool" {
+		return devices.getPoolDevName(), nil
+	}
+	info := devices.Devices[hash]
+	if info == nil {
+		return "", fmt.Errorf("hash %s doesn't exists", hash)
+	}
+	return info.Name(), nil
+}
+
+func (devices *DeviceSet) Shutdown() error {
+	utils.Debugf("[deviceset %s] shutdown()", devices.devicePrefix)
+	defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix)
+	devices.Lock()
+	utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root)
+	defer devices.Unlock()
+
+	for path, count := range devices.activeMounts {
+		for i := count; i > 0; i-- {
+			if err := syscall.Unmount(path, 0); err != nil {
+				utils.Debugf("Shutdown unmounting %s, error: %s\n", path, err)
+			}
+		}
+		delete(devices.activeMounts, path)
+	}
+
+	for _, d := range devices.Devices {
+		if err := devices.waitClose(d.Hash); err != nil {
+			utils.Errorf("Warning: error waiting for device %s to unmount: %s\n", d.Hash, err)
+		}
+		if err := devices.deactivateDevice(d.Hash); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", d.Hash, err)
+		}
+	}
+
+	pool := devices.getPoolDevName()
+	if devinfo, err := getInfo(pool); err == nil && devinfo.Exists != 0 {
+		if err := devices.deactivateDevice("pool"); err != nil {
+			utils.Debugf("Shutdown deactivate %s , error: %s\n", pool, err)
+		}
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error {
+	devices.Lock()
+	defer devices.Unlock()
+
+	if err := devices.activateDeviceIfNeeded(hash); err != nil {
+		return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err)
+	}
+
+	info := devices.Devices[hash]
+
+	var flags uintptr = syscall.MS_MGC_VAL
+
+	if readOnly {
+		flags = flags | syscall.MS_RDONLY
+	}
+
+	err := syscall.Mount(info.DevName(), path, "ext4", flags, "discard")
+	if err != nil && err == syscall.EINVAL {
+		err = syscall.Mount(info.DevName(), path, "ext4", flags, "")
+	}
+	if err != nil {
+		return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err)
+	}
+
+	count := devices.activeMounts[path]
+	devices.activeMounts[path] = count + 1
+
+	return devices.setInitialized(hash)
+}
+
+func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error {
+	utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path)
+	defer utils.Debugf("[devmapper] UnmountDevice END")
+	devices.Lock()
+	defer devices.Unlock()
+
+	utils.Debugf("[devmapper] Unmount(%s)", path)
+	if err := syscall.Unmount(path, 0); err != nil {
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+	utils.Debugf("[devmapper] Unmount done")
+	// Wait for the unmount to be effective,
+	// by watching the value of Info.OpenCount for the device
+	if err := devices.waitClose(hash); err != nil {
+		return err
+	}
+
+	if count := devices.activeMounts[path]; count > 1 {
+		devices.activeMounts[path] = count - 1
+	} else {
+		delete(devices.activeMounts, path)
+	}
+
+	if deactivate {
+		devices.deactivateDevice(hash)
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) HasDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	return devices.Devices[hash] != nil
+}
+
+func (devices *DeviceSet) HasInitializedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	info := devices.Devices[hash]
+	return info != nil && info.Initialized
+}
+
+func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
+	devices.Lock()
+	defer devices.Unlock()
+
+	info := devices.Devices[hash]
+	if info == nil {
+		return false
+	}
+	devinfo, _ := getInfo(info.Name())
+	return devinfo != nil && devinfo.Exists != 0
+}
+
+func (devices *DeviceSet) setInitialized(hash string) error {
+	info := devices.Devices[hash]
+	if info == nil {
+		return fmt.Errorf("Unknown device %s", hash)
+	}
+
+	info.Initialized = true
+	if err := devices.saveMetadata(); err != nil {
+		info.Initialized = false
+		utils.Debugf("\n--->Err: %s\n", err)
+		return err
+	}
+
+	return nil
+}
+
+func (devices *DeviceSet) Status() *Status {
+	devices.Lock()
+	defer devices.Unlock()
+
+	status := &Status{}
+
+	status.PoolName = devices.getPoolName()
+	status.DataLoopback = path.Join(devices.loopbackDir(), "data")
+	status.MetadataLoopback = path.Join(devices.loopbackDir(), "metadata")
+
+	_, totalSizeInSectors, _, params, err := getStatus(devices.getPoolName())
+	if err == nil {
+		var transactionId, dataUsed, dataTotal, metadataUsed, metadataTotal uint64
+		if _, err := fmt.Sscanf(params, "%d %d/%d %d/%d", &transactionId, &metadataUsed, &metadataTotal, &dataUsed, &dataTotal); err == nil {
+			// Convert from blocks to bytes
+			blockSizeInSectors := totalSizeInSectors / dataTotal
+
+			status.Data.Used = dataUsed * blockSizeInSectors * 512
+			status.Data.Total = dataTotal * blockSizeInSectors * 512
+
+			// metadata blocks are always 4k
+			status.Metadata.Used = metadataUsed * 4096
+			status.Metadata.Total = metadataTotal * 4096
+		}
+	}
+
+	return status
+}
+
+func NewDeviceSet(root string) (*DeviceSet, error) {
+	SetDevDir("/dev")
+
+	devices := &DeviceSet{
+		root:         root,
+		MetaData:     MetaData{Devices: make(map[string]*DevInfo)},
+		activeMounts: make(map[string]int),
+	}
+
+	if err := devices.initDevmapper(); err != nil {
+		return nil, err
+	}
+
+	return devices, nil
+}

devmapper/devmapper.go

@@ -0,0 +1,505 @@
+package devmapper
+
+import (
+	"errors"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"os"
+	"runtime"
+)
+
+type DevmapperLogger interface {
+	log(level int, file string, line int, dmError int, message string)
+}
+
+const (
+	DeviceCreate TaskType = iota
+	DeviceReload
+	DeviceRemove
+	DeviceRemoveAll
+	DeviceSuspend
+	DeviceResume
+	DeviceInfo
+	DeviceDeps
+	DeviceRename
+	DeviceVersion
+	DeviceStatus
+	DeviceTable
+	DeviceWaitevent
+	DeviceList
+	DeviceClear
+	DeviceMknodes
+	DeviceListVersions
+	DeviceTargetMsg
+	DeviceSetGeometry
+)
+
+const (
+	AddNodeOnResume AddNodeType = iota
+	AddNodeOnCreate
+)
+
+var (
+	ErrTaskRun              = errors.New("dm_task_run failed")
+	ErrTaskSetName          = errors.New("dm_task_set_name failed")
+	ErrTaskSetMessage       = errors.New("dm_task_set_message failed")
+	ErrTaskSetAddNode       = errors.New("dm_task_set_add_node failed")
+	ErrTaskSetRo            = errors.New("dm_task_set_ro failed")
+	ErrTaskAddTarget        = errors.New("dm_task_add_target failed")
+	ErrTaskSetSector        = errors.New("dm_task_set_sector failed")
+	ErrTaskGetInfo          = errors.New("dm_task_get_info failed")
+	ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed")
+	ErrTaskSetCookie        = errors.New("dm_task_set_cookie failed")
+	ErrNilCookie            = errors.New("cookie ptr can't be nil")
+	ErrAttachLoopbackDevice = errors.New("loopback mounting failed")
+	ErrGetBlockSize         = errors.New("Can't get block size")
+	ErrUdevWait             = errors.New("wait on udev cookie failed")
+	ErrSetDevDir            = errors.New("dm_set_dev_dir failed")
+	ErrGetLibraryVersion    = errors.New("dm_get_library_version failed")
+	ErrCreateRemoveTask     = errors.New("Can't create task of type DeviceRemove")
+	ErrRunRemoveDevice      = errors.New("running removeDevice failed")
+	ErrInvalidAddNode       = errors.New("Invalide AddNoce type")
+)
+
+type (
+	Task struct {
+		unmanaged *CDmTask
+	}
+	Info struct {
+		Exists        int
+		Suspended     int
+		LiveTable     int
+		InactiveTable int
+		OpenCount     int32
+		EventNr       uint32
+		Major         uint32
+		Minor         uint32
+		ReadOnly      int
+		TargetCount   int32
+	}
+	TaskType    int
+	AddNodeType int
+)
+
+func (t *Task) destroy() {
+	if t != nil {
+		DmTaskDestory(t.unmanaged)
+		runtime.SetFinalizer(t, nil)
+	}
+}
+
+func TaskCreate(tasktype TaskType) *Task {
+	Ctask := DmTaskCreate(int(tasktype))
+	if Ctask == nil {
+		return nil
+	}
+	task := &Task{unmanaged: Ctask}
+	runtime.SetFinalizer(task, (*Task).destroy)
+	return task
+}
+
+func (t *Task) Run() error {
+	if res := DmTaskRun(t.unmanaged); res != 1 {
+		return ErrTaskRun
+	}
+	return nil
+}
+
+func (t *Task) SetName(name string) error {
+	if res := DmTaskSetName(t.unmanaged, name); res != 1 {
+		return ErrTaskSetName
+	}
+	return nil
+}
+
+func (t *Task) SetMessage(message string) error {
+	if res := DmTaskSetMessage(t.unmanaged, message); res != 1 {
+		return ErrTaskSetMessage
+	}
+	return nil
+}
+
+func (t *Task) SetSector(sector uint64) error {
+	if res := DmTaskSetSector(t.unmanaged, sector); res != 1 {
+		return ErrTaskSetSector
+	}
+	return nil
+}
+
+func (t *Task) SetCookie(cookie *uint, flags uint16) error {
+	if cookie == nil {
+		return ErrNilCookie
+	}
+	if res := DmTaskSetCookie(t.unmanaged, cookie, flags); res != 1 {
+		return ErrTaskSetCookie
+	}
+	return nil
+}
+
+func (t *Task) SetAddNode(addNode AddNodeType) error {
+	if addNode != AddNodeOnResume && addNode != AddNodeOnCreate {
+		return ErrInvalidAddNode
+	}
+	if res := DmTaskSetAddNode(t.unmanaged, addNode); res != 1 {
+		return ErrTaskSetAddNode
+	}
+	return nil
+}
+
+func (t *Task) SetRo() error {
+	if res := DmTaskSetRo(t.unmanaged); res != 1 {
+		return ErrTaskSetRo
+	}
+	return nil
+}
+
+func (t *Task) AddTarget(start, size uint64, ttype, params string) error {
+	if res := DmTaskAddTarget(t.unmanaged, start, size,
+		ttype, params); res != 1 {
+		return ErrTaskAddTarget
+	}
+	return nil
+}
+
+func (t *Task) GetInfo() (*Info, error) {
+	info := &Info{}
+	if res := DmTaskGetInfo(t.unmanaged, info); res != 1 {
+		return nil, ErrTaskGetInfo
+	}
+	return info, nil
+}
+
+func (t *Task) GetNextTarget(next uintptr) (nextPtr uintptr, start uint64,
+	length uint64, targetType string, params string) {
+
+	return DmGetNextTarget(t.unmanaged, next, &start, &length,
+			&targetType, &params),
+		start, length, targetType, params
+}
+
+func AttachLoopDevice(filename string) (*os.File, error) {
+	var fd int
+	res := DmAttachLoopDevice(filename, &fd)
+	if res == "" {
+		return nil, ErrAttachLoopbackDevice
+	}
+	return os.NewFile(uintptr(fd), res), nil
+}
+
+func UdevWait(cookie uint) error {
+	if res := DmUdevWait(cookie); res != 1 {
+		utils.Debugf("Failed to wait on udev cookie %d", cookie)
+		return ErrUdevWait
+	}
+	return nil
+}
+
+func LogInitVerbose(level int) {
+	DmLogInitVerbose(level)
+}
+
+var dmLogger DevmapperLogger = nil
+
+func logInit(logger DevmapperLogger) {
+	dmLogger = logger
+	LogWithErrnoInit()
+}
+
+func SetDevDir(dir string) error {
+	if res := DmSetDevDir(dir); res != 1 {
+		utils.Debugf("Error dm_set_dev_dir")
+		return ErrSetDevDir
+	}
+	return nil
+}
+
+func GetLibraryVersion() (string, error) {
+	var version string
+	if res := DmGetLibraryVersion(&version); res != 1 {
+		return "", ErrGetLibraryVersion
+	}
+	return version, nil
+}
+
+// Useful helper for cleanup
+func RemoveDevice(name string) error {
+	task := TaskCreate(DeviceRemove)
+	if task == nil {
+		return ErrCreateRemoveTask
+	}
+	if err := task.SetName(name); err != nil {
+		utils.Debugf("Can't set task name %s", name)
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return ErrRunRemoveDevice
+	}
+	return nil
+}
+
+func GetBlockDeviceSize(file *os.File) (uint64, error) {
+	size, errno := DmGetBlockSize(file.Fd())
+	if size == -1 || errno != 0 {
+		return 0, ErrGetBlockSize
+	}
+	return uint64(size), nil
+}
+
+// This is the programmatic example of "dmsetup create"
+func createPool(poolName string, dataFile *os.File, metadataFile *os.File) error {
+	task, err := createTask(DeviceCreate, poolName)
+	if task == nil {
+		return err
+	}
+
+	size, err := GetBlockDeviceSize(dataFile)
+	if err != nil {
+		return fmt.Errorf("Can't get data size")
+	}
+
+	params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768"
+	if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate (createPool)")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createTask(t TaskType, name string) (*Task, error) {
+	task := TaskCreate(t)
+	if task == nil {
+		return nil, fmt.Errorf("Can't create task of type %d", int(t))
+	}
+	if err := task.SetName(name); err != nil {
+		return nil, fmt.Errorf("Can't set task name %s", name)
+	}
+	return task, nil
+}
+
+func getInfo(name string) (*Info, error) {
+	task, err := createTask(DeviceInfo, name)
+	if task == nil {
+		return nil, err
+	}
+	if err := task.Run(); err != nil {
+		return nil, err
+	}
+	return task.GetInfo()
+}
+
+func getStatus(name string) (uint64, uint64, string, string, error) {
+	task, err := createTask(DeviceStatus, name)
+	if task == nil {
+		utils.Debugf("getStatus: Error createTask: %s", err)
+		return 0, 0, "", "", err
+	}
+	if err := task.Run(); err != nil {
+		utils.Debugf("getStatus: Error Run: %s", err)
+		return 0, 0, "", "", err
+	}
+
+	devinfo, err := task.GetInfo()
+	if err != nil {
+		utils.Debugf("getStatus: Error GetInfo: %s", err)
+		return 0, 0, "", "", err
+	}
+	if devinfo.Exists == 0 {
+		utils.Debugf("getStatus: Non existing device %s", name)
+		return 0, 0, "", "", fmt.Errorf("Non existing device %s", name)
+	}
+
+	_, start, length, target_type, params := task.GetNextTarget(0)
+	return start, length, target_type, params, nil
+}
+
+func setTransactionId(poolName string, oldId uint64, newId uint64) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("set_transaction_id %d %d", oldId, newId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running setTransactionId")
+	}
+	return nil
+}
+
+func suspendDevice(name string) error {
+	task, err := createTask(DeviceSuspend, name)
+	if task == nil {
+		return err
+	}
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+	return nil
+}
+
+func resumeDevice(name string) error {
+	task, err := createTask(DeviceResume, name)
+	if task == nil {
+		return err
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceSuspend")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func createDevice(poolName string, deviceId int) error {
+	utils.Debugf("[devmapper] createDevice(poolName=%v, deviceId=%v)", poolName, deviceId)
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_thin %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running createDevice")
+	}
+	return nil
+}
+
+func deleteDevice(poolName string, deviceId int) error {
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("delete %d", deviceId)); err != nil {
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running deleteDevice")
+	}
+	return nil
+}
+
+func removeDevice(name string) error {
+	utils.Debugf("[devmapper] removeDevice START")
+	defer utils.Debugf("[devmapper] removeDevice END")
+	task, err := createTask(DeviceRemove, name)
+	if task == nil {
+		return err
+	}
+	if err = task.Run(); err != nil {
+		return fmt.Errorf("Error running removeDevice")
+	}
+	return nil
+}
+
+func activateDevice(poolName string, name string, deviceId int, size uint64) error {
+	task, err := createTask(DeviceCreate, name)
+	if task == nil {
+		return err
+	}
+
+	params := fmt.Sprintf("%s %d", poolName, deviceId)
+	if err := task.AddTarget(0, size/512, "thin", params); err != nil {
+		return fmt.Errorf("Can't add target")
+	}
+	if err := task.SetAddNode(AddNodeOnCreate); err != nil {
+		return fmt.Errorf("Can't add node")
+	}
+
+	var cookie uint = 0
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		return fmt.Errorf("Can't set cookie")
+	}
+
+	if err := task.Run(); err != nil {
+		return fmt.Errorf("Error running DeviceCreate (activateDevice)")
+	}
+
+	UdevWait(cookie)
+
+	return nil
+}
+
+func (devices *DeviceSet) createSnapDevice(poolName string, deviceId int, baseName string, baseDeviceId int) error {
+	devinfo, _ := getInfo(baseName)
+	doSuspend := devinfo != nil && devinfo.Exists != 0
+
+	if doSuspend {
+		if err := suspendDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	task, err := createTask(DeviceTargetMsg, poolName)
+	if task == nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return err
+	}
+
+	if err := task.SetSector(0); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set sector")
+	}
+
+	if err := task.SetMessage(fmt.Sprintf("create_snap %d %d", deviceId, baseDeviceId)); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Can't set message")
+	}
+
+	if err := task.Run(); err != nil {
+		if doSuspend {
+			resumeDevice(baseName)
+		}
+		return fmt.Errorf("Error running DeviceCreate (createSnapDevice)")
+	}
+
+	if doSuspend {
+		if err := resumeDevice(baseName); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

devmapper/devmapper_log.go

@@ -0,0 +1,13 @@
+package devmapper
+
+import "C"
+
+// Due to the way cgo works this has to be in a separate file, as devmapper.go has
+// definitions in the cgo block, which is incompatible with using "//export"
+
+//export DevmapperLogCallback
+func DevmapperLogCallback(level C.int, file *C.char, line C.int, dm_errno_or_class C.int, message *C.char) {
+	if dmLogger != nil {
+		dmLogger.log(int(level), C.GoString(file), int(line), int(dm_errno_or_class), C.GoString(message))
+	}
+}

devmapper/devmapper_test.go

@@ -0,0 +1,285 @@
+package devmapper
+
+import (
+	"syscall"
+	"testing"
+)
+
+func TestTaskCreate(t *testing.T) {
+	// Test success
+	taskCreate(t, DeviceInfo)
+
+	// Test Failure
+	DmTaskCreate = dmTaskCreateFail
+	defer func() { DmTaskCreate = dmTaskCreateFct }()
+	if task := TaskCreate(-1); task != nil {
+		t.Fatalf("An error should have occured while creating an invalid task.")
+	}
+}
+
+func TestTaskRun(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	// Perform the RUN
+	if err := task.Run(); err != nil {
+		t.Fatal(err)
+	}
+	// Make sure we don't have error with GetInfo
+	if _, err := task.GetInfo(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskRun = dmTaskRunFail
+	defer func() { DmTaskRun = dmTaskRunFct }()
+
+	task = taskCreate(t, DeviceInfo)
+	// Perform the RUN
+	if err := task.Run(); err != ErrTaskRun {
+		t.Fatalf("An error should have occured while running task.")
+	}
+	// Make sure GetInfo also fails
+	if _, err := task.GetInfo(); err != ErrTaskGetInfo {
+		t.Fatalf("GetInfo should fail if task.Run() failed.")
+	}
+}
+
+func TestTaskSetName(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetName("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetName = dmTaskSetNameFail
+	defer func() { DmTaskSetName = dmTaskSetNameFct }()
+
+	if err := task.SetName("test"); err != ErrTaskSetName {
+		t.Fatalf("An error should have occured while runnign SetName.")
+	}
+}
+
+func TestTaskSetMessage(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetMessage("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetMessage = dmTaskSetMessageFail
+	defer func() { DmTaskSetMessage = dmTaskSetMessageFct }()
+
+	if err := task.SetMessage("test"); err != ErrTaskSetMessage {
+		t.Fatalf("An error should have occured while runnign SetMessage.")
+	}
+}
+
+func TestTaskSetSector(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetSector(128); err != nil {
+		t.Fatal(err)
+	}
+
+	DmTaskSetSector = dmTaskSetSectorFail
+	defer func() { DmTaskSetSector = dmTaskSetSectorFct }()
+
+	// Test failure
+	if err := task.SetSector(0); err != ErrTaskSetSector {
+		t.Fatalf("An error should have occured while running SetSector.")
+	}
+}
+
+func TestTaskSetCookie(t *testing.T) {
+	var (
+		cookie uint = 0
+		task        = taskCreate(t, DeviceInfo)
+	)
+
+	// Test success
+	if err := task.SetCookie(&cookie, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	if err := task.SetCookie(nil, 0); err != ErrNilCookie {
+		t.Fatalf("An error should have occured while running SetCookie with nil cookie.")
+	}
+
+	DmTaskSetCookie = dmTaskSetCookieFail
+	defer func() { DmTaskSetCookie = dmTaskSetCookieFct }()
+
+	if err := task.SetCookie(&cookie, 0); err != ErrTaskSetCookie {
+		t.Fatalf("An error should have occured while running SetCookie.")
+	}
+}
+
+func TestTaskSetAddNode(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetAddNode(0); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	if err := task.SetAddNode(-1); err != ErrInvalidAddNode {
+		t.Fatalf("An error should have occured running SetAddNode with wrong node.")
+	}
+
+	DmTaskSetAddNode = dmTaskSetAddNodeFail
+	defer func() { DmTaskSetAddNode = dmTaskSetAddNodeFct }()
+
+	if err := task.SetAddNode(0); err != ErrTaskSetAddNode {
+		t.Fatalf("An error should have occured running SetAddNode.")
+	}
+}
+
+func TestTaskSetRo(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.SetRo(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskSetRo = dmTaskSetRoFail
+	defer func() { DmTaskSetRo = dmTaskSetRoFct }()
+
+	if err := task.SetRo(); err != ErrTaskSetRo {
+		t.Fatalf("An error should have occured running SetRo.")
+	}
+}
+
+func TestTaskAddTarget(t *testing.T) {
+	task := taskCreate(t, DeviceInfo)
+
+	// Test success
+	if err := task.AddTarget(0, 128, "thinp", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test failure
+	DmTaskAddTarget = dmTaskAddTargetFail
+	defer func() { DmTaskAddTarget = dmTaskAddTargetFct }()
+
+	if err := task.AddTarget(0, 128, "thinp", ""); err != ErrTaskAddTarget {
+		t.Fatalf("An error should have occured running AddTarget.")
+	}
+}
+
+// func TestTaskGetInfo(t *testing.T) {
+// 	task := taskCreate(t, DeviceInfo)
+
+// 	// Test success
+// 	if _, err := task.GetInfo(); err != nil {
+// 		t.Fatal(err)
+// 	}
+
+// 	// Test failure
+// 	DmTaskGetInfo = dmTaskGetInfoFail
+// 	defer func() { DmTaskGetInfo = dmTaskGetInfoFct }()
+
+// 	if _, err := task.GetInfo(); err != ErrTaskGetInfo {
+// 		t.Fatalf("An error should have occured running GetInfo.")
+// 	}
+// }
+
+// func TestTaskGetNextTarget(t *testing.T) {
+// 	task := taskCreate(t, DeviceInfo)
+
+// 	if next, _, _, _, _ := task.GetNextTarget(0); next == 0 {
+// 		t.Fatalf("The next target should not be 0.")
+// 	}
+// }
+
+/// Utils
+func taskCreate(t *testing.T, taskType TaskType) *Task {
+	task := TaskCreate(taskType)
+	if task == nil {
+		t.Fatalf("Error creating task")
+	}
+	return task
+}
+
+/// Failure function replacement
+func dmTaskCreateFail(t int) *CDmTask {
+	return nil
+}
+
+func dmTaskRunFail(task *CDmTask) int {
+	return -1
+}
+
+func dmTaskSetNameFail(task *CDmTask, name string) int {
+	return -1
+}
+
+func dmTaskSetMessageFail(task *CDmTask, message string) int {
+	return -1
+}
+
+func dmTaskSetSectorFail(task *CDmTask, sector uint64) int {
+	return -1
+}
+
+func dmTaskSetCookieFail(task *CDmTask, cookie *uint, flags uint16) int {
+	return -1
+}
+
+func dmTaskSetAddNodeFail(task *CDmTask, addNode AddNodeType) int {
+	return -1
+}
+
+func dmTaskSetRoFail(task *CDmTask) int {
+	return -1
+}
+
+func dmTaskAddTargetFail(task *CDmTask,
+	start, size uint64, ttype, params string) int {
+	return -1
+}
+
+func dmTaskGetDriverVersionFail(task *CDmTask, version *string) int {
+	return -1
+}
+
+func dmTaskGetInfoFail(task *CDmTask, info *Info) int {
+	return -1
+}
+
+func dmGetNextTargetFail(task *CDmTask, next uintptr, start, length *uint64,
+	target, params *string) uintptr {
+	return 0
+}
+
+func dmAttachLoopDeviceFail(filename string, fd *int) string {
+	return ""
+}
+
+func sysGetBlockSizeFail(fd uintptr, size *uint64) syscall.Errno {
+	return 1
+}
+
+func dmGetBlockSizeFail(fd uintptr) int64 {
+	return -1
+}
+
+func dmUdevWaitFail(cookie uint) int {
+	return -1
+}
+
+func dmSetDevDirFail(dir string) int {
+	return -1
+}
+
+func dmGetLibraryVersionFail(version *string) int {
+	return -1
+}

devmapper/devmapper_wrapper.go

@@ -0,0 +1,330 @@
+package devmapper
+
+/*
+#cgo LDFLAGS: -L. -ldevmapper
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <libdevmapper.h>
+#include <linux/loop.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <linux/fs.h>
+#include <errno.h>
+
+#ifndef LOOP_CTL_GET_FREE
+#define LOOP_CTL_GET_FREE       0x4C82
+#endif
+
+// FIXME: this could easily be rewritten in go
+char*			attach_loop_device(const char *filename, int *loop_fd_out)
+{
+  struct loop_info64	loopinfo = {0};
+  struct stat		st;
+  char			buf[64];
+  int			i, loop_fd, fd, start_index;
+  char*			loopname;
+
+
+  *loop_fd_out = -1;
+
+  start_index = 0;
+  fd = open("/dev/loop-control", O_RDONLY);
+  if (fd >= 0) {
+    start_index = ioctl(fd, LOOP_CTL_GET_FREE);
+    close(fd);
+
+    if (start_index < 0)
+      start_index = 0;
+  }
+
+  fd = open(filename, O_RDWR);
+  if (fd < 0) {
+    perror("open");
+    return NULL;
+  }
+
+  loop_fd = -1;
+  for (i = start_index ; loop_fd < 0 ; i++ ) {
+    if (sprintf(buf, "/dev/loop%d", i) < 0) {
+	close(fd);
+	return NULL;
+    }
+
+    if (stat(buf, &st)) {
+      if (!S_ISBLK(st.st_mode)) {
+	 fprintf(stderr, "[error] Loopback device %s is not a block device.\n", buf);
+      } else if (errno == ENOENT) {
+	fprintf(stderr, "[error] There are no more loopback device available.\n");
+      } else {
+	fprintf(stderr, "[error] Unkown error trying to stat the loopback device %s (errno: %d).\n", buf, errno);
+      }
+      close(fd);
+      return NULL;
+    }
+
+    loop_fd = open(buf, O_RDWR);
+    if (loop_fd < 0 && errno == ENOENT) {
+      fprintf(stderr, "[error] The loopback device %s does not exists.\n", buf);
+      close(fd);
+      return NULL;
+    } else if (loop_fd < 0) {
+	fprintf(stderr, "[error] Unkown error openning the loopback device %s. (errno: %d)\n", buf, errno);
+	continue;
+    }
+
+    if (ioctl(loop_fd, LOOP_SET_FD, (void *)(size_t)fd) < 0) {
+      int errsv = errno;
+      close(loop_fd);
+      loop_fd = -1;
+      if (errsv != EBUSY) {
+        close(fd);
+        fprintf(stderr, "cannot set up loopback device %s: %s", buf, strerror(errsv));
+        return NULL;
+      }
+      continue;
+    }
+
+    close(fd);
+
+    strncpy((char*)loopinfo.lo_file_name, buf, LO_NAME_SIZE);
+    loopinfo.lo_offset = 0;
+    loopinfo.lo_flags = LO_FLAGS_AUTOCLEAR;
+
+    if (ioctl(loop_fd, LOOP_SET_STATUS64, &loopinfo) < 0) {
+      perror("ioctl LOOP_SET_STATUS64");
+      if (ioctl(loop_fd, LOOP_CLR_FD, 0) < 0) {
+        perror("ioctl LOOP_CLR_FD");
+      }
+      close(loop_fd);
+      fprintf (stderr, "cannot set up loopback device info");
+      return (NULL);
+    }
+
+    loopname = strdup(buf);
+    if (loopname == NULL) {
+      close(loop_fd);
+      return (NULL);
+    }
+
+    *loop_fd_out = loop_fd;
+    return (loopname);
+  }
+
+  return (NULL);
+}
+
+extern void DevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str);
+
+static void	log_cb(int level, const char *file, int line,
+		       int dm_errno_or_class, const char *f, ...)
+{
+  char buffer[256];
+  va_list ap;
+
+  va_start(ap, f);
+  vsnprintf(buffer, 256, f, ap);
+  va_end(ap);
+
+  DevmapperLogCallback(level, (char *)file, line, dm_errno_or_class, buffer);
+}
+
+static void	log_with_errno_init()
+{
+  dm_log_with_errno_init(log_cb);
+}
+
+*/
+import "C"
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+type (
+	CDmTask C.struct_dm_task
+)
+
+var (
+	DmTaskDestory       = dmTaskDestroyFct
+	DmTaskCreate        = dmTaskCreateFct
+	DmTaskRun           = dmTaskRunFct
+	DmTaskSetName       = dmTaskSetNameFct
+	DmTaskSetMessage    = dmTaskSetMessageFct
+	DmTaskSetSector     = dmTaskSetSectorFct
+	DmTaskSetCookie     = dmTaskSetCookieFct
+	DmTaskSetAddNode    = dmTaskSetAddNodeFct
+	DmTaskSetRo         = dmTaskSetRoFct
+	DmTaskAddTarget     = dmTaskAddTargetFct
+	DmTaskGetInfo       = dmTaskGetInfoFct
+	DmGetNextTarget     = dmGetNextTargetFct
+	DmGetBlockSize      = dmGetBlockSizeFct
+	DmAttachLoopDevice  = dmAttachLoopDeviceFct
+	DmUdevWait          = dmUdevWaitFct
+	DmLogInitVerbose    = dmLogInitVerboseFct
+	DmSetDevDir         = dmSetDevDirFct
+	DmGetLibraryVersion = dmGetLibraryVersionFct
+	LogWithErrnoInit    = logWithErrnoInitFct
+	GetBlockSize        = getBlockSizeFct
+)
+
+func free(p *C.char) {
+	C.free(unsafe.Pointer(p))
+}
+
+func dmTaskDestroyFct(task *CDmTask) {
+	C.dm_task_destroy((*C.struct_dm_task)(task))
+}
+
+func dmTaskCreateFct(taskType int) *CDmTask {
+	return (*CDmTask)(C.dm_task_create(C.int(taskType)))
+}
+
+func dmTaskRunFct(task *CDmTask) int {
+	return int(C.dm_task_run((*C.struct_dm_task)(task)))
+}
+
+func dmTaskSetNameFct(task *CDmTask, name string) int {
+	Cname := C.CString(name)
+	defer free(Cname)
+
+	return int(C.dm_task_set_name((*C.struct_dm_task)(task),
+		Cname))
+}
+
+func dmTaskSetMessageFct(task *CDmTask, message string) int {
+	Cmessage := C.CString(message)
+	defer free(Cmessage)
+
+	return int(C.dm_task_set_message((*C.struct_dm_task)(task),
+		Cmessage))
+}
+
+func dmTaskSetSectorFct(task *CDmTask, sector uint64) int {
+	return int(C.dm_task_set_sector((*C.struct_dm_task)(task),
+		C.uint64_t(sector)))
+}
+
+func dmTaskSetCookieFct(task *CDmTask, cookie *uint, flags uint16) int {
+	cCookie := C.uint32_t(*cookie)
+	defer func() {
+		*cookie = uint(cCookie)
+	}()
+	return int(C.dm_task_set_cookie((*C.struct_dm_task)(task), &cCookie,
+		C.uint16_t(flags)))
+}
+
+func dmTaskSetAddNodeFct(task *CDmTask, addNode AddNodeType) int {
+	return int(C.dm_task_set_add_node((*C.struct_dm_task)(task),
+		C.dm_add_node_t(addNode)))
+}
+
+func dmTaskSetRoFct(task *CDmTask) int {
+	return int(C.dm_task_set_ro((*C.struct_dm_task)(task)))
+}
+
+func dmTaskAddTargetFct(task *CDmTask,
+	start, size uint64, ttype, params string) int {
+
+	Cttype := C.CString(ttype)
+	defer free(Cttype)
+
+	Cparams := C.CString(params)
+	defer free(Cparams)
+
+	return int(C.dm_task_add_target((*C.struct_dm_task)(task),
+		C.uint64_t(start), C.uint64_t(size), Cttype, Cparams))
+}
+
+func dmGetBlockSizeFct(fd uintptr) (int64, syscall.Errno) {
+	var size int64
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, C.BLKGETSIZE64,
+		uintptr(unsafe.Pointer(&size)))
+	return size, err
+}
+
+func dmTaskGetInfoFct(task *CDmTask, info *Info) int {
+	Cinfo := C.struct_dm_info{}
+	defer func() {
+		info.Exists = int(Cinfo.exists)
+		info.Suspended = int(Cinfo.suspended)
+		info.LiveTable = int(Cinfo.live_table)
+		info.InactiveTable = int(Cinfo.inactive_table)
+		info.OpenCount = int32(Cinfo.open_count)
+		info.EventNr = uint32(Cinfo.event_nr)
+		info.Major = uint32(Cinfo.major)
+		info.Minor = uint32(Cinfo.minor)
+		info.ReadOnly = int(Cinfo.read_only)
+		info.TargetCount = int32(Cinfo.target_count)
+	}()
+	return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo))
+}
+
+func dmGetNextTargetFct(task *CDmTask, next uintptr, start, length *uint64,
+	target, params *string) uintptr {
+
+	var (
+		Cstart, Clength      C.uint64_t
+		CtargetType, Cparams *C.char
+	)
+	defer func() {
+		*start = uint64(Cstart)
+		*length = uint64(Clength)
+		*target = C.GoString(CtargetType)
+		*params = C.GoString(Cparams)
+	}()
+	nextp := C.dm_get_next_target((*C.struct_dm_task)(task),
+		unsafe.Pointer(next), &Cstart, &Clength, &CtargetType, &Cparams)
+	return uintptr(nextp)
+}
+
+func dmAttachLoopDeviceFct(filename string, fd *int) string {
+	cFilename := C.CString(filename)
+	defer free(cFilename)
+
+	var cFd C.int
+	defer func() {
+		*fd = int(cFd)
+	}()
+
+	ret := C.attach_loop_device(cFilename, &cFd)
+	defer free(ret)
+	return C.GoString(ret)
+}
+
+func getBlockSizeFct(fd uintptr, size *uint64) syscall.Errno {
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, C.BLKGETSIZE64,
+		uintptr(unsafe.Pointer(&size)))
+	return err
+}
+
+func dmUdevWaitFct(cookie uint) int {
+	return int(C.dm_udev_wait(C.uint32_t(cookie)))
+}
+
+func dmLogInitVerboseFct(level int) {
+	C.dm_log_init_verbose(C.int(level))
+}
+
+func logWithErrnoInitFct() {
+	C.log_with_errno_init()
+}
+
+func dmSetDevDirFct(dir string) int {
+	Cdir := C.CString(dir)
+	defer free(Cdir)
+
+	return int(C.dm_set_dev_dir(Cdir))
+}
+
+func dmGetLibraryVersionFct(version *string) int {
+	buffer := C.CString(string(make([]byte, 128)))
+	defer free(buffer)
+	defer func() {
+		*version = C.GoString(buffer)
+	}()
+	return int(C.dm_get_library_version(buffer, 128))
+}

devmapper/docker-device-tool/device_tool.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/devmapper"
+	"os"
+)
+
+func usage() {
+	fmt.Printf("Usage: %s [snap new-id base-id] | [remove id] | [mount id mountpoint]\n", os.Args[0])
+	os.Exit(1)
+}
+
+func main() {
+	devices := devmapper.NewDeviceSet("/var/lib/docker")
+
+	if len(os.Args) < 2 {
+		usage()
+	}
+
+	cmd := os.Args[1]
+	if cmd == "snap" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.AddDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "remove" {
+		if len(os.Args) < 3 {
+			usage()
+		}
+
+		err := devices.RemoveDevice(os.Args[2])
+		if err != nil {
+			fmt.Println("Can't remove device: ", err)
+			os.Exit(1)
+		}
+	} else if cmd == "mount" {
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		err := devices.MountDevice(os.Args[2], os.Args[3])
+		if err != nil {
+			fmt.Println("Can't create snap device: ", err)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Printf("Unknown command %s\n", cmd)
+		if len(os.Args) < 4 {
+			usage()
+		}
+
+		os.Exit(1)
+	}
+
+	return
+}

devmapper/driver.go

@@ -0,0 +1,77 @@
+package devmapper
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/graphdriver"
+	"os"
+	"path"
+)
+
+func init() {
+	graphdriver.Register("devicemapper", Init)
+}
+
+// Placeholder interfaces, to be replaced
+// at integration.
+
+// End of placeholder interfaces.
+
+type Driver struct {
+	*DeviceSet
+	home string
+}
+
+func Init(home string) (graphdriver.Driver, error) {
+	deviceSet, err := NewDeviceSet(home)
+	if err != nil {
+		return nil, err
+	}
+	d := &Driver{
+		DeviceSet: deviceSet,
+		home:      home,
+	}
+	return d, nil
+}
+
+func (d *Driver) String() string {
+	return "devicemapper"
+}
+
+func (d *Driver) Cleanup() error {
+	return d.DeviceSet.Shutdown()
+}
+
+func (d *Driver) Create(id string, parent string) error {
+	return d.DeviceSet.AddDevice(id, parent)
+}
+
+func (d *Driver) Remove(id string) error {
+	return d.DeviceSet.RemoveDevice(id)
+}
+
+func (d *Driver) Get(id string) (string, error) {
+	mp := path.Join(d.home, "mnt", id)
+	if err := d.mount(id, mp); err != nil {
+		return "", err
+	}
+	return mp, nil
+}
+
+func (d *Driver) Size(id string) (int64, error) {
+	return -1, fmt.Errorf("Not implemented")
+}
+
+func (d *Driver) mount(id, mountPoint string) error {
+	// Create the target directories if they don't exist
+	if err := os.MkdirAll(mountPoint, 0755); err != nil && !os.IsExist(err) {
+		return err
+	}
+	// If mountpoint is already mounted, do nothing
+	if mounted, err := Mounted(mountPoint); err != nil {
+		return fmt.Errorf("Error checking mountpoint: %s", err)
+	} else if mounted {
+		return nil
+	}
+	// Mount the device
+	return d.DeviceSet.MountDevice(id, mountPoint, false)
+}

devmapper/driver_test.go

@@ -0,0 +1,301 @@
+package devmapper
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+func init() {
+	// Reduce the size the the base fs and loopback for the tests
+	DefaultDataLoopbackSize = 300 * 1024 * 1024
+	DefaultMetaDataLoopbackSize = 200 * 1024 * 1024
+	DefaultBaseFsSize = 300 * 1024 * 1024
+
+}
+
+func mkTestDirectory(t *testing.T) string {
+	dir, err := ioutil.TempDir("", "docker-test-devmapper-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func newDriver(t *testing.T) *Driver {
+	home := mkTestDirectory(t)
+	d, err := Init(home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return d.(*Driver)
+}
+
+func cleanup(d *Driver) {
+	d.Cleanup()
+	os.RemoveAll(d.home)
+}
+
+func TestInit(t *testing.T) {
+	home := mkTestDirectory(t)
+	defer os.RemoveAll(home)
+	driver, err := Init(home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		if err := driver.Cleanup(); err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	id := "foo"
+	if err := driver.Create(id, ""); err != nil {
+		t.Fatal(err)
+	}
+	dir, err := driver.Get(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st, err := os.Stat(dir); err != nil {
+		t.Fatal(err)
+	} else if !st.IsDir() {
+		t.Fatalf("Get(%V) did not return a directory", id)
+	}
+}
+
+func TestDriverName(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if d.String() != "devicemapper" {
+		t.Fatalf("Expected driver name to be devicemapper got %s", d.String())
+	}
+}
+
+func TestDriverCreate(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestDriverRemove(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Remove("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCleanup(t *testing.T) {
+	d := newDriver(t)
+	defer os.RemoveAll(d.home)
+
+	mountPoints := make([]string, 2)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	// Mount the id
+	p, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountPoints[0] = p
+
+	if err := d.Create("2", "1"); err != nil {
+		t.Fatal(err)
+	}
+
+	p, err = d.Get("2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mountPoints[1] = p
+
+	// Ensure that all the mount points are currently mounted
+	for _, p := range mountPoints {
+		if mounted, err := Mounted(p); err != nil {
+			t.Fatal(err)
+		} else if !mounted {
+			t.Fatalf("Expected %s to be mounted", p)
+		}
+	}
+
+	// Ensure that devices are active
+	for _, p := range []string{"1", "2"} {
+		if !d.HasActivatedDevice(p) {
+			t.Fatalf("Expected %s to have an active device", p)
+		}
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that all the mount points are no longer mounted
+	for _, p := range mountPoints {
+		if mounted, err := Mounted(p); err != nil {
+			t.Fatal(err)
+		} else if mounted {
+			t.Fatalf("Expected %s to not be mounted", p)
+		}
+	}
+
+	// Ensure that devices are no longer activated
+	for _, p := range []string{"1", "2"} {
+		if d.HasActivatedDevice(p) {
+			t.Fatalf("Expected %s not be an active device", p)
+		}
+	}
+}
+
+func TestNotMounted(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	mounted, err := Mounted(path.Join(d.home, "mnt", "1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if mounted {
+		t.Fatal("Id 1 should not be mounted")
+	}
+}
+
+func TestMounted(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	mounted, err := Mounted(path.Join(d.home, "mnt", "1"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !mounted {
+		t.Fatal("Id 1 should be mounted")
+	}
+}
+
+func TestInitCleanedDriver(t *testing.T) {
+	d := newDriver(t)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := d.Cleanup(); err != nil {
+		t.Fatal(err)
+	}
+
+	driver, err := Init(d.home)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d = driver.(*Driver)
+	defer cleanup(d)
+
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMountMountedDriver(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	// Perform get on same id to ensure that it will
+	// not be mounted twice
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestGetReturnsValidDevice(t *testing.T) {
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	if !d.HasDevice("1") {
+		t.Fatalf("Expected id 1 to be in device set")
+	}
+
+	if _, err := d.Get("1"); err != nil {
+		t.Fatal(err)
+	}
+
+	if !d.HasActivatedDevice("1") {
+		t.Fatalf("Expected id 1 to be activated")
+	}
+
+	if !d.HasInitializedDevice("1") {
+		t.Fatalf("Expected id 1 to be initialized")
+	}
+}
+
+func TestDriverGetSize(t *testing.T) {
+	t.Skipf("Size is currently not implemented")
+
+	d := newDriver(t)
+	defer cleanup(d)
+
+	if err := d.Create("1", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	mountPoint, err := d.Get("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	size := int64(1024)
+
+	f, err := os.Create(path.Join(mountPoint, "test_file"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Truncate(size); err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+
+	diffSize, err := d.Size("1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diffSize != size {
+		t.Fatalf("Expected size %d got %d", size, diffSize)
+	}
+}

devmapper/mount.go

@@ -0,0 +1,27 @@
+package devmapper
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+// FIXME: this is copy-pasted from the aufs driver.
+// It should be moved into the core.
+
+func Mounted(mountpoint string) (bool, error) {
+	mntpoint, err := os.Stat(mountpoint)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	parent, err := os.Stat(filepath.Join(mountpoint, ".."))
+	if err != nil {
+		return false, err
+	}
+	mntpointSt := mntpoint.Sys().(*syscall.Stat_t)
+	parentSt := parent.Sys().(*syscall.Stat_t)
+	return mntpointSt.Dev != parentSt.Dev, nil
+}

docker/docker.go

@@ -4,9 +4,9 @@ import (
 	"flag"
 	"fmt"
 	"github.com/dotcloud/docker"
+	"github.com/dotcloud/docker/engine"
 	"github.com/dotcloud/docker/sysinit"
 	"github.com/dotcloud/docker/utils"
-	"github.com/dotcloud/docker/engine"
 	"log"
 	"os"
 	"strings"
@@ -71,7 +71,8 @@ func main() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		job := eng.Job("serveapi")
+		// Load plugin: httpapi
+		job := eng.Job("initapi")
 		job.Setenv("Pidfile", *pidfile)
 		job.Setenv("Root", *flRoot)
 		job.SetenvBool("AutoRestart", *flAutoRestart)
@@ -79,12 +80,17 @@ func main() {
 		job.Setenv("Dns", *flDns)
 		job.SetenvBool("EnableIptables", *flEnableIptables)
 		job.Setenv("BridgeIface", *bridgeName)
-		job.SetenvList("ProtoAddresses", flHosts)
 		job.Setenv("DefaultIp", *flDefaultIp)
 		job.SetenvBool("InterContainerCommunication", *flInterContainerComm)
 		if err := job.Run(); err != nil {
 			log.Fatal(err)
 		}
+		// Serve api
+		job = eng.Job("serveapi", flHosts...)
+		job.SetenvBool("Logging", true)
+		if err := job.Run(); err != nil {
+			log.Fatal(err)
+		}
 	} else {
 		if len(flHosts) > 1 {
 			log.Fatal("Please specify only one -H")

docs/sources/api/docker_remote_api.rst

@@ -26,14 +26,118 @@ Docker Remote API
 2. Versions
 ===========
 
-The current version of the API is 1.6
+The current version of the API is 1.7
 
 Calling /images/<name>/insert is the same as calling
-/v1.6/images/<name>/insert
+/v1.7/images/<name>/insert
 
 You can still call an old version of the api using
 /v1.0/images/<name>/insert
 
+v1.7
+****
+
+Full Documentation
+------------------
+
+:doc:`docker_remote_api_v1.7`
+
+What's new
+----------
+
+.. http:get:: /images/json
+
+   The format of the json returned from this uri changed.  Instead of an entry
+   for each repo/tag on an image, each image is only represented once, with a
+   nested attribute indicating the repo/tags that apply to that image.
+
+   Instead of:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Content-Type: application/json
+
+      [
+        {
+          "VirtualSize": 131506275,
+          "Size": 131506275,
+          "Created": 1365714795,
+          "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c",
+          "Tag": "12.04",
+          "Repository": "ubuntu"
+        },
+        {
+          "VirtualSize": 131506275,
+          "Size": 131506275,
+          "Created": 1365714795,
+          "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c",
+          "Tag": "latest",
+          "Repository": "ubuntu"
+        },
+        {
+          "VirtualSize": 131506275,
+          "Size": 131506275,
+          "Created": 1365714795,
+          "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c",
+          "Tag": "precise",
+          "Repository": "ubuntu"
+        },
+        {
+          "VirtualSize": 180116135,
+          "Size": 24653,
+          "Created": 1364102658,
+          "Id": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc",
+          "Tag": "12.10",
+          "Repository": "ubuntu"
+        },
+        {
+          "VirtualSize": 180116135,
+          "Size": 24653,
+          "Created": 1364102658,
+          "Id": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc",
+          "Tag": "quantal",
+          "Repository": "ubuntu"
+        }
+      ]
+
+   The returned json looks like this:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Content-Type: application/json
+      
+      [
+        {
+           "RepoTag": [
+             "ubuntu:12.04",
+             "ubuntu:precise",
+             "ubuntu:latest"
+           ],
+           "Id": "8dbd9e392a964056420e5d58ca5cc376ef18e2de93b5cc90e868a1bbc8318c1c",
+           "Created": 1365714795,
+           "Size": 131506275,
+           "VirtualSize": 131506275
+        },
+        {
+           "RepoTag": [
+             "ubuntu:12.10",
+             "ubuntu:quantal"
+           ],
+           "ParentId": "27cf784147099545",
+           "Id": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc",
+           "Created": 1364102658,
+           "Size": 24653,
+           "VirtualSize": 180116135
+        }
+      ]
+
+.. http:get:: /images/viz
+
+   This URI no longer exists.  The ``images -viz`` output is now generated in
+   the client, using the ``/images/json`` data.
+
 v1.6
 ****
 

docs/sources/api/docker_remote_api_v1.6.rst

@@ -121,8 +121,7 @@ Create a container
 		"AttachStdin":false,
 		"AttachStdout":true,
 		"AttachStderr":true,
-		"PortSpecs":null,
-		"Privileged": false,
+		"ExposedPorts":{},
 		"Tty":false,
 		"OpenStdin":false,
 		"StdinOnce":false,
@@ -135,7 +134,6 @@ Create a container
 		"Volumes":{},
 		"VolumesFrom":"",
 		"WorkingDir":""
-
 	   }
 	   
 	**Example response**:
@@ -157,6 +155,57 @@ Create a container
 	:statuscode 406: impossible to attach (container not running)
 	:statuscode 500: server error
 
+	**More Complex Example request, in 2 steps.**
+	**First, use create to expose a Private Port, which can be bound back to a Public Port at startup**:
+
+	.. sourcecode:: http
+
+	   POST /containers/create HTTP/1.1
+	   Content-Type: application/json
+
+	   {
+		"Cmd":[
+			"/usr/sbin/sshd","-D"
+		],
+		"Image":"image-with-sshd",
+		"ExposedPorts":{"22/tcp":{}}
+		}
+
+	**Example response**:
+
+	.. sourcecode:: http
+
+	   HTTP/1.1 201 OK
+	   Content-Type: application/json
+
+	   {
+		"Id":"e90e34656806"
+		"Warnings":[]
+	   }
+
+	**Second, start (using the ID returned above) the image we just created, mapping the ssh port 22 to something on the host**:
+
+	.. sourcecode:: http
+
+	   POST /containers/e90e34656806/start HTTP/1.1
+	   Content-Type: application/json
+
+	   {
+		"PortBindings": { "22/tcp": [{ "HostPort": "11022" }]} 
+		}
+
+	**Example response**:
+
+	.. sourcecode:: http
+
+		HTTP/1.1 204 No Content
+		Content-Type: text/plain; charset=utf-8
+		Content-Length: 0
+
+	**Now you can ssh into your new container on port 11022.**
+
+
+
 
 Inspect a container
 *******************
@@ -191,7 +240,7 @@ Inspect a container
 				"AttachStdin": false,
 				"AttachStdout": true,
 				"AttachStderr": true,
-				"PortSpecs": null,
+				"ExposedPorts": {},
 				"Tty": false,
 				"OpenStdin": false,
 				"StdinOnce": false,
@@ -362,7 +411,12 @@ Start a container
 
            {
                 "Binds":["/tmp:/tmp"],
-                "LxcConf":{"lxc.utsname":"docker"}
+                "LxcConf":{"lxc.utsname":"docker"},
+                "ContainerIDFile": "",
+                "Privileged": false,
+                "PortBindings": {"22/tcp": [{HostIp:"", HostPort:""}]},
+                "Links": [],
+                "PublishAllPorts": false
            }
 
         **Example response**:
@@ -795,7 +849,7 @@ Inspect an image
 				"AttachStdin":false,
 				"AttachStdout":false,
 				"AttachStderr":false,
-				"PortSpecs":null,
+				"ExposedPorts":{},
 				"Tty":true,
 				"OpenStdin":true,
 				"StdinOnce":false,
@@ -1141,7 +1195,7 @@ Create a new image from a container's changes
        
        {
            "Cmd": ["cat", "/world"],
-           "PortSpecs":["22"]
+           "ExposedPorts":{"22/tcp":{}}
        }
 
     **Example response**:

docs/sources/api/remote_api_client_libraries.rst

@@ -12,26 +12,28 @@ compatibility. Please file issues with the library owners.  If you
 find more library implementations, please list them in Docker doc bugs
 and we will add the libraries here.
 
-+----------------------+----------------+--------------------------------------------+
-| Language/Framework   | Name           | Repository                                 |
-+======================+================+============================================+
-| Python               | docker-py      | https://github.com/dotcloud/docker-py      |
-+----------------------+----------------+--------------------------------------------+
-| Ruby                 | docker-client  | https://github.com/geku/docker-client      |
-+----------------------+----------------+--------------------------------------------+
-| Ruby                 | docker-api     | https://github.com/swipely/docker-api      |
-+----------------------+----------------+--------------------------------------------+
-| Javascript (NodeJS)  | docker.io      | https://github.com/appersonlabs/docker.io  |
-|                      |                | Install via NPM: `npm install docker.io`   |
-+----------------------+----------------+--------------------------------------------+
-| Javascript           | docker-js      | https://github.com/dgoujard/docker-js      |
-+----------------------+----------------+--------------------------------------------+
-| Javascript (Angular) | dockerui       | https://github.com/crosbymichael/dockerui  |
-| **WebUI**            |                |                                            |
-+----------------------+----------------+--------------------------------------------+
-| Java                 | docker-java    | https://github.com/kpelykh/docker-java     |
-+----------------------+----------------+--------------------------------------------+
-| Erlang               | erldocker      | https://github.com/proger/erldocker        |
-+----------------------+----------------+--------------------------------------------+
-| Go                   | go-dockerclient| https://github.com/fsouza/go-dockerclient  |
-+----------------------+----------------+--------------------------------------------+
++----------------------+----------------+--------------------------------------------+----------+
+| Language/Framework   | Name           | Repository                                 | Status   |
++======================+================+============================================+==========+
+| Python               | docker-py      | https://github.com/dotcloud/docker-py      | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| Ruby                 | docker-client  | https://github.com/geku/docker-client      | Outdated |
++----------------------+----------------+--------------------------------------------+----------+
+| Ruby                 | docker-api     | https://github.com/swipely/docker-api      | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| Javascript (NodeJS)  | docker.io      | https://github.com/appersonlabs/docker.io  | Active   |
+|                      |                | Install via NPM: `npm install docker.io`   |          |
++----------------------+----------------+--------------------------------------------+----------+
+| Javascript           | docker-js      | https://github.com/dgoujard/docker-js      | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| Javascript (Angular) | dockerui       | https://github.com/crosbymichael/dockerui  | Active   |
+| **WebUI**            |                |                                            |          |
++----------------------+----------------+--------------------------------------------+----------+
+| Java                 | docker-java    | https://github.com/kpelykh/docker-java     | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| Erlang               | erldocker      | https://github.com/proger/erldocker        | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| Go                   | go-dockerclient| https://github.com/fsouza/go-dockerclient  | Active   |
++----------------------+----------------+--------------------------------------------+----------+
+| PHP                  | Alvine         | http://pear.alvine.io/ (alpha)             | Active   |
++----------------------+----------------+--------------------------------------------+----------+

docs/sources/commandline/cli.rst

@@ -245,6 +245,9 @@ Full -run example
     Usage: docker events
 
     Get real time events from the server
+    
+    -since="": Show previously created events and then stream.
+               (either seconds since epoch, or date string as below)
 
 .. _cli_events_example:
 
@@ -277,6 +280,23 @@ Shell 1: (Again .. now showing events)
     [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
     [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
 
+Show events in the past from a specified time
+.............................................
+
+.. code-block:: bash
+
+    $ sudo docker events -since 1378216169
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
+
+    $ sudo docker events -since '2013-09-03'
+    [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
+
+    $ sudo docker events -since '2013-09-03 15:49:29 +0200 CEST'
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die
+    [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop
 
 .. _cli_export:
 
@@ -315,8 +335,10 @@ Shell 1: (Again .. now showing events)
     List images
 
       -a=false: show all images
+      -notrunc=false: Don't truncate output
       -q=false: only show numeric IDs
-      -viz=false: output in graphviz format
+      -tree=false: output graph in tree format
+      -viz=false: output graph in graphviz format
 
 Displaying images visually
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -328,6 +350,36 @@ Displaying images visually
 .. image:: docker_images.gif
    :alt: Example inheritance graph of Docker images.
 
+
+Displaying image hierarchy
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+    sudo docker images -tree
+
+    |─8dbd9e392a96 Size: 131.5 MB (virtual 131.5 MB) Tags: ubuntu:12.04,ubuntu:latest,ubuntu:precise
+    └─27cf78414709 Size: 180.1 MB (virtual 180.1 MB)
+      └─b750fe79269d Size: 24.65 kB (virtual 180.1 MB) Tags: ubuntu:12.10,ubuntu:quantal
+        |─f98de3b610d5 Size: 12.29 kB (virtual 180.1 MB)
+        | └─7da80deb7dbf Size: 16.38 kB (virtual 180.1 MB)
+        |   └─65ed2fee0a34 Size: 20.66 kB (virtual 180.2 MB)
+        |     └─a2b9ea53dddc Size: 819.7 MB (virtual 999.8 MB)
+        |       └─a29b932eaba8 Size: 28.67 kB (virtual 999.9 MB)
+        |         └─e270a44f124d Size: 12.29 kB (virtual 999.9 MB) Tags: progrium/buildstep:latest
+        └─17e74ac162d8 Size: 53.93 kB (virtual 180.2 MB)
+          └─339a3f56b760 Size: 24.65 kB (virtual 180.2 MB)
+            └─904fcc40e34d Size: 96.7 MB (virtual 276.9 MB)
+              └─b1b0235328dd Size: 363.3 MB (virtual 640.2 MB)
+                └─7cb05d1acb3b Size: 20.48 kB (virtual 640.2 MB)
+                  └─47bf6f34832d Size: 20.48 kB (virtual 640.2 MB)
+                    └─f165104e82ed Size: 12.29 kB (virtual 640.2 MB)
+                      └─d9cf85a47b7e Size: 1.911 MB (virtual 642.2 MB)
+                        └─3ee562df86ca Size: 17.07 kB (virtual 642.2 MB)
+                          └─b05fc2d00e4a Size: 24.96 kB (virtual 642.2 MB)
+                            └─c96a99614930 Size: 12.29 kB (virtual 642.2 MB)
+                              └─a6a357a48c49 Size: 12.29 kB (virtual 642.2 MB) Tags: ndj/mongodb:latest
+
 .. _cli_import:
 
 ``import``
@@ -428,6 +480,12 @@ Insert file from github
 
 The main process inside the container will be sent SIGKILL.
 
+Known Issues (kill)
+~~~~~~~~~~~~~~~~~~~
+
+* :issue:`197` indicates that ``docker kill`` may leave directories
+  behind and make it difficult to remove the container.
+
 .. _cli_login:
 
 ``login``
@@ -536,6 +594,12 @@ The main process inside the container will be sent SIGKILL.
     Remove one or more containers
         -link="": Remove the link instead of the actual container
 
+Known Issues (rm)
+~~~~~~~~~~~~~~~~~~~
+
+* :issue:`197` indicates that ``docker kill`` may leave directories
+  behind and make it difficult to remove the container.
+
 
 Examples:
 ~~~~~~~~~
@@ -558,6 +622,15 @@ This will remove the container referenced under the link ``/redis``.
 This will remove the underlying link between ``/webapp`` and the ``/redis`` containers removing all
 network communication.
 
+.. code-block:: bash
+
+    $ docker rm `docker ps -a -q`
+
+
+This command will delete all stopped containers. The command ``docker ps -a -q`` will return all
+existing container IDs and pass them to the ``rm`` command which will delete them. Any running
+containers will not be deleted.
+
 .. _cli_rmi:
 
 ``rmi``
@@ -588,7 +661,7 @@ network communication.
       -h="": Container host name
       -i=false: Keep stdin open even if not attached
       -privileged=false: Give extended privileges to this container
-      -m=0: Memory limit (in bytes)
+      -m="": Memory limit (format: <number><optional unit>, where unit = b, k, m or g)
       -n=true: Enable networking for this container
       -p=[]: Map a network port to the container
       -rm=false: Automatically remove the container when it exits (incompatible with -d)
@@ -596,7 +669,7 @@ network communication.
       -u="": Username or UID
       -dns=[]: Set custom dns servers for the container
       -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume.
-      -volumes-from="": Mount all volumes from the given container
+      -volumes-from="": Mount all volumes from the given container(s)
       -entrypoint="": Overwrite the default entrypoint set by the image
       -w="": Working directory inside the container
       -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
@@ -688,6 +761,17 @@ can access the network and environment of the redis container via
 environment variables.  The ``-name`` flag will assign the name ``console``
 to the newly created container.
 
+.. code-block:: bash
+
+   docker run -volumes-from 777f7dc92da7,ba8c0c54f0f2:ro -i -t ubuntu pwd
+
+The ``-volumes-from`` flag mounts all the defined volumes from the
+refrence containers. Containers can be specified by a comma seperated
+list or by repetitions of the ``-volumes-from`` argument. The container
+id may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in
+read-only or read-write mode, respectively. By default, the volumes are mounted
+in the same mode (rw or ro) as the reference container.
+
 .. _cli_search:
 
 ``search``

docs/sources/conf.py

@@ -40,7 +40,11 @@ html_additional_pages = {
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinxcontrib.httpdomain']
+extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks']
+
+# Configure extlinks
+extlinks = { 'issue': ('https://github.com/dotcloud/docker/issues/%s',
+	               'Issue ') }
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/sources/contributing/contributing.rst

@@ -10,13 +10,16 @@ Want to hack on Docker? Awesome!
 The repository includes `all the instructions you need to get
 started <https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md>`_.
 
-The developer environment `Dockerfile <https://github.com/dotcloud/docker/blob/master/Dockerfile>`_
+The `developer environment Dockerfile
+<https://github.com/dotcloud/docker/blob/master/Dockerfile>`_
 specifies the tools and versions used to test and build Docker.
 
 If you're making changes to the documentation, see the
 `README.md <https://github.com/dotcloud/docker/blob/master/docs/README.md>`_.
 
-The documentation environment `Dockerfile <https://github.com/dotcloud/docker/blob/master/docs/Dockerfile>`_
+The `documentation environment Dockerfile
+<https://github.com/dotcloud/docker/blob/master/docs/Dockerfile>`_
 specifies the tools and versions used to build the Documentation.
 
-Further interesting details can be found in the `Packaging hints <https://github.com/dotcloud/docker/blob/master/hack/PACKAGERS.md>`_.
+Further interesting details can be found in the `Packaging hints
+<https://github.com/dotcloud/docker/blob/master/hack/PACKAGERS.md>`_.

docs/sources/examples/index.rst

@@ -24,4 +24,3 @@ to more substantial services like those which you might find in production.
    postgresql_service
    mongodb
    running_riak_service
-   linking_into_redis

docs/sources/examples/linking_into_redis.rst

@@ -1,137 +0,0 @@
-:title: Linking to an Redis container
-:description: Running redis linked into your web app
-:keywords: docker, example, networking, redis, link
-
-.. _linking_redis:
-
-Linking Redis
-=============
-
-.. include:: example_header.inc
-
-Building a Redis container to link as a child of our web application.
-
-Building the Redis container
-----------------------------
-
-Lets build a Redis image with the following Dockerfile.
-
-First checkout the Redis source code.
-
-.. code-block:: bash
-
-    git clone https://github.com/antirez/redis.git
-    cd redis
-    git checkout 2.6
-
-
-Now let's create a Dockerfile in the root of the Redis repository.
-
-.. code-block:: bash
-
-    # Build redis from source
-    # Make sure you have the redis source code checked out in
-    # the same directory as this Dockerfile
-    FROM ubuntu
-
-    RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list
-    RUN apt-get update
-    RUN apt-get upgrade -y
-
-    RUN apt-get install -y gcc make g++ build-essential libc6-dev tcl
-
-    ADD . /redis
-
-    RUN (cd /redis && make)
-
-    RUN mkdir -p /redis-data
-    VOLUME ["/redis-data"]
-    EXPOSE 6379
-
-    ENTRYPOINT ["/redis/src/redis-server"]
-    CMD ["--dir", "/redis-data"]
-
-    # docker build our new redis image from source
-    docker build -t redis-2.6 .
-
-
-We need to ``EXPOSE`` the default port of 6379 so that our link knows what ports 
-to connect to our Redis container on.  If you do not expose any ports for the
-image then docker will not be able to establish the link between containers.
-
-
-Run the Redis container
------------------------
-
-.. code-block:: bash
-
-    sudo docker run -d -e PASSWORD=docker -name redis redis-2.6 --requirepass docker
-
-This will run our Redis container with the password docker 
-to secure our service.  By specifying the ``-name`` flag on run 
-we will assign the name ``redis`` to this container.  If we do not specify a name for 
-our container via the ``-name`` flag docker will automatically generate a name for us.
-We can issue all the commands that you would expect; start, stop, attach, using the name for our container.
-The name also allows us to link other containers into this one.
-
-Linking Redis as a child
-------------------------
-
-Next we can start a new web application that has a dependency on Redis and apply a link 
-to connect both containers.  If you noticed when running our Redis server we did not use
-the ``-p`` flag to publish the Redis port to the host system.  Redis exposed port 6379 via the Dockerfile 
-and this is all we need to establish a link.
-
-Now let's start our web application with a link into Redis.
-
-.. code-block:: bash
-
-    sudo docker run -t -i -link redis:db -name webapp ubuntu bash
-
-    root@4c01db0b339c:/# env
-
-    HOSTNAME=4c01db0b339c
-    DB_NAME=/webapp/db
-    TERM=xterm
-    DB_PORT=tcp://172.17.0.8:6379
-    DB_PORT_6379_TCP=tcp://172.17.0.8:6379
-    DB_PORT_6379_TCP_PROTO=tcp
-    DB_PORT_6379_TCP_ADDR=172.17.0.8
-    DB_PORT_6379_TCP_PORT=6379
-    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-    PWD=/
-    DB_ENV_PASSWORD=docker
-    SHLVL=1
-    HOME=/
-    container=lxc
-    _=/usr/bin/env
-    root@4c01db0b339c:/#
-
-
-When we inspect the environment of the linked container we can see a few extra environment 
-variables have been added.  When you specified ``-link redis:db`` you are telling docker
-to link the container named ``redis`` into this new container with the alias ``db``.
-Environment variables are prefixed with the alias so that the parent container can access
-network and environment information from the containers that are linked into it.
-
-.. code-block:: bash
-
-    # The name of the child container
-    DB_NAME=/webapp/db
-
-    # The default protocol, ip, and port of the service running in the container
-    DB_PORT=tcp://172.17.0.8:6379
-
-    # A specific protocol, ip, and port of various services
-    DB_PORT_6379_TCP=tcp://172.17.0.8:6379
-    DB_PORT_6379_TCP_PROTO=tcp
-    DB_PORT_6379_TCP_ADDR=172.17.0.8
-    DB_PORT_6379_TCP_PORT=6379
-
-    # Get environment variables of the container 
-    DB_ENV_PASSWORD=docker
-
-
-Accessing the network information along with the environment of the child container allows
-us to easily connect to the Redis service on the specific IP and port and use the password
-specified in the environment.

docs/sources/examples/nodejs_web_app.rst

@@ -176,11 +176,11 @@ Run the image
 +++++++++++++
 
 Running your image with ``-d`` runs the container in detached mode, leaving the
-container running in the background. Run the image you previously built:
+container running in the background. The ``-p`` flag redirects a public port to a private port in the container. Run the image you previously built:
 
 .. code-block:: bash
 
-    sudo docker run -d <your username>/centos-node-hello
+    sudo docker run -p 49160:8080 -d <your username>/centos-node-hello
 
 Print the output of your app:
 

docs/sources/examples/python_web_app.rst

@@ -86,7 +86,7 @@ http://0.0.0.0:5000/`` in the log output.
 
 .. code-block:: bash
 
-    WEB_PORT=$(sudo docker port $WEB_WORKER 5000)
+    WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }')
 
 Look up the public-facing port which is NAT-ed. Find the private port
 used by the container and store it inside of the ``WEB_PORT`` variable.

docs/sources/examples/running_redis_service.rst

@@ -9,74 +9,93 @@ Redis Service
 
 .. include:: example_header.inc
 
-Very simple, no frills, Redis service.
+Very simple, no frills, Redis service attached to a web application using a link.
 
-Open a docker container
------------------------
+Create a docker container for Redis
+-----------------------------------
+
+Firstly, we create a ``Dockerfile`` for our new Redis image.
 
 .. code-block:: bash
 
-    sudo docker run -i -t ubuntu /bin/bash
+    FROM        ubuntu:12.10
+    RUN         apt-get update
+    RUN         apt-get -y install redis-server
+    EXPOSE      6379
+    ENTRYPOINT  ["/usr/bin/redis-server"]
 
-Building your image
--------------------
-
-Update your Docker container, install the Redis server.  Once
-installed, exit out of the Docker container.
+Next we build an image from our ``Dockerfile``. Replace ``<your username>`` 
+with your own user name.
 
 .. code-block:: bash
 
-    apt-get update
-    apt-get install redis-server
-    exit
-
-Snapshot the installation
--------------------------
-
-.. code-block:: bash
-
-    sudo docker ps -a  # grab the container id (this will be the first one in the list)
-    sudo docker commit <container_id> <your username>/redis
+    sudo docker build -t <your username>/redis
 
 Run the service
 ---------------
 
+Use the image we've just created and name your container ``redis``.
+
 Running the service with ``-d`` runs the container in detached mode, leaving the
-container running in the background. Use your snapshot.
+container running in the background.
+
+Importantly, we're not exposing any ports on our container. Instead we're going to 
+use a container link to provide access to our Redis database.
 
 .. code-block:: bash
 
-    sudo docker run -d -p 6379 <your username>/redis /usr/bin/redis-server
+    sudo docker run -name redis -d <your username>/redis
 
-Test 1
-++++++
+Create your web application container
+-------------------------------------
 
-Connect to the container with the ``redis-cli`` binary.
+Next we can create a container for our application. We're going to use the ``-link`` 
+flag to create a link to the ``redis`` container we've just created with an alias of 
+``db``. This will create a secure tunnel to the ``redis`` container and expose the 
+Redis instance running inside that container to only this container.
 
 .. code-block:: bash
 
-    sudo docker ps  # grab the new container id
-    sudo docker inspect <container_id>    # grab the ipaddress of the container
-    redis-cli -h <ipaddress> -p 6379
-    redis 10.0.3.32:6379> set docker awesome
+    sudo docker run -link redis:db -i -t ubuntu:12.10 /bin/bash
+
+Once inside our freshly created container we need to install Redis to get the 
+``redis-cli`` binary to test our connection.
+
+.. code-block:: bash
+
+    apt-get update
+    apt-get -y install redis-server
+    service redis-server stop
+
+Now we can test the connection. Firstly, let's look at the available environmental 
+variables in our web application container. We can use these to get the IP and port 
+of our ``redis`` container.
+
+.. code-block:: bash
+
+    env
+    . . .
+    DB_NAME=/violet_wolf/db
+    DB_PORT_6379_TCP_PORT=6379
+    DB_PORT=tcp://172.17.0.33:6379
+    DB_PORT_6379_TCP=tcp://172.17.0.33:6379
+    DB_PORT_6379_TCP_ADDR=172.17.0.33
+    DB_PORT_6379_TCP_PROTO=tcp
+
+We can see that we've got a small list of environmental varaibles prefixed with ``DB``. 
+The ``DB`` comes from the link alias specified when we launched the container. Let's use 
+the ``DB_PORT_6379_TCP_ADDR`` variable to connect to our Redis container.
+
+.. code-block:: bash
+
+    redis-cli -h $DB_PORT_6379_TCP_ADDR
+    redis 172.17.0.33:6379>
+    redis 172.17.0.33:6379> set docker awesome
     OK
-    redis 10.0.3.32:6379> get docker
+    redis 172.17.0.33:6379> get docker
     "awesome"
-    redis 10.0.3.32:6379> exit
+    redis 172.17.0.33:6379> exit
 
-Test 2
-++++++
+We could easily use this or other environment variables in our web application to make a 
+connection to our ``redis`` container.
 
-Connect to the host os with the ``redis-cli`` binary.
-
-.. code-block:: bash
-
-    sudo docker ps  # grab the new container id
-    sudo docker port <container_id> 6379  # grab the external port
-    ip addr show   # grab the host ip address
-    redis-cli -h <host ipaddress> -p <external port>
-    redis 192.168.0.1:49153> set docker awesome
-    OK
-    redis 192.168.0.1:49153> get docker
-    "awesome"
-    redis 192.168.0.1:49153> exit

docs/sources/installation/amazon.rst

@@ -102,26 +102,45 @@ Docker that way too. Vagrant 1.1 or higher is required.
    we need to set them there first. Make sure you have everything on
    amazon aws setup so you can (manually) deploy a new image to EC2.
 
+   Note that where possible these variables are the same as those honored by
+   the ec2 api tools.
    ::
 
-       export AWS_ACCESS_KEY_ID=xxx
-       export AWS_SECRET_ACCESS_KEY=xxx
+       export AWS_ACCESS_KEY=xxx
+       export AWS_SECRET_KEY=xxx
        export AWS_KEYPAIR_NAME=xxx
-       export AWS_SSH_PRIVKEY=xxx
+       export SSH_PRIVKEY_PATH=xxx
 
-   The environment variables are:
+       export BOX_NAME=xxx
+       export AWS_REGION=xxx
+       export AWS_AMI=xxx
+       export AWS_INSTANCE_TYPE=xxx
 
-   * ``AWS_ACCESS_KEY_ID`` - The API key used to make requests to AWS
-   * ``AWS_SECRET_ACCESS_KEY`` - The secret key to make AWS API requests
+   The required environment variables are:
+
+   * ``AWS_ACCESS_KEY`` - The API key used to make requests to AWS
+   * ``AWS_SECRET_KEY`` - The secret key to make AWS API requests
    * ``AWS_KEYPAIR_NAME`` - The name of the keypair used for this EC2 instance
-   * ``AWS_SSH_PRIVKEY`` - The path to the private key for the named
+   * ``SSH_PRIVKEY_PATH`` - The path to the private key for the named
      keypair, for example ``~/.ssh/docker.pem``
 
+   There are a number of optional environment variables:
+
+   * ``BOX_NAME`` - The name of the vagrant box to use.  Defaults to
+     ``ubuntu``.
+   * ``AWS_REGION`` - The aws region to spawn the vm in.  Defaults to
+     ``us-east-1``.
+   * ``AWS_AMI`` - The aws AMI to start with as a base.  This must be
+     be an ubuntu 12.04 precise image.  You must change this value if
+     ``AWS_REGION`` is set to a value other than ``us-east-1``.
+     This is because AMIs are region specific.  Defaults to ``ami-69f5a900``.
+   * ``AWS_INSTANCE_TYPE`` - The aws instance type.  Defaults to ``t1.micro``.
+
    You can check if they are set correctly by doing something like
 
    ::
 
-      echo $AWS_ACCESS_KEY_ID
+      echo $AWS_ACCESS_KEY
 
 6. Do the magic!
 

docs/sources/terms/container.rst

@@ -38,3 +38,10 @@ was when the container was stopped.
 You can promote a container to an :ref:`image_def` with ``docker
 commit``. Once a container is an image, you can use it as a parent for
 new containers.
+
+Container IDs
+.............
+All containers are identified by a 64 hexadecimal digit string (internally a 256bit 
+value). To simplify their use, a short ID of the first 12 characters can be used 
+on the commandline. There is a small possibility of short id collisions, so the 
+docker server will always return the long ID.

docs/sources/terms/image.rst

@@ -36,3 +36,11 @@ Base Image
 ..........
 
 An image that has no parent is a **base image**.
+
+Image IDs
+.........
+All images are identified by a 64 hexadecimal digit string (internally a 256bit 
+value). To simplify their use, a short ID of the first 12 characters can be used 
+on the command line. There is a small possibility of short id collisions, so the 
+docker server will always return the long ID.
+

docs/sources/use/basics.rst

@@ -22,22 +22,37 @@ specify the path to it and manually start it.
     # Run docker in daemon mode
     sudo <path to>/docker -d &
 
-
-Running an interactive shell
-----------------------------
+Download a pre-built image
+--------------------------
 
 .. code-block:: bash
 
   # Download an ubuntu image
   sudo docker pull ubuntu
 
+This will find the ``ubuntu`` image by name in the :ref:`Central Index 
+<searching_central_index>` and download it from the top-level Central 
+Repository to a local image cache.
+
+.. NOTE:: When the image has successfully downloaded, you will see a 12 
+character hash ``539c0211cd76: Download complete`` which is the short 
+form of the image ID. These short image IDs are the first 12 characters 
+of the full image ID - which can be found using ``docker inspect`` or 
+``docker images -notrunc=true``
+
+.. _dockergroup:
+
+Running an interactive shell
+----------------------------
+
+.. code-block:: bash
+
   # Run an interactive shell in the ubuntu image,
   # allocate a tty, attach stdin and stdout
   # To detach the tty without exiting the shell,
   # use the escape sequence Ctrl-p + Ctrl-q
   sudo docker run -i -t ubuntu /bin/bash
 
-.. _dockergroup:
 
 Why ``sudo``?
 -------------

docs/sources/use/builder.rst

@@ -116,6 +116,16 @@ core concepts of Docker where commits are cheap and containers can be
 created from any point in an image's history, much like source
 control.
 
+Known Issues (RUN)
+..................
+
+* :issue:`783` is about file permissions problems that can occur when
+  using the AUFS file system. You might notice it during an attempt to
+  ``rm`` a file, for example. The issue describes a workaround.
+* :issue:`2424` Locale will not be set automatically.
+
+
+
 3.4 CMD
 -------
 
@@ -211,8 +221,16 @@ destination container.
 All new files and directories are created with mode 0755, uid and gid
 0.
 
+.. note::
+   if you build using STDIN (``docker build - < somefile``), there is no build 
+   context, so the Dockerfile can only contain an URL based ADD statement.
+
 The copy obeys the following rules:
 
+* The ``<src>`` path must be inside the *context* of the build; you cannot 
+  ``ADD ../something /something``, because the first step of a 
+  ``docker build`` is to send the context directory (and subdirectories) to 
+  the docker daemon.
 * If ``<src>`` is a URL and ``<dest>`` does not end with a trailing slash,
   then a file is downloaded from the URL and copied to ``<dest>``.
 * If ``<src>`` is a URL and ``<dest>`` does end with a trailing slash,

docs/sources/use/index.rst

@@ -20,3 +20,4 @@ Contents:
    puppet
    host_integration
    working_with_volumes
+   working_with_links_names

docs/sources/use/working_with_links_names.rst

@@ -0,0 +1,104 @@
+:title: Working with Links and Names
+:description: How to create and use links and names
+:keywords: Examples, Usage, links, docker, documentation, examples, names, name, container naming
+
+.. _working_with_links_names:
+
+Working with Links and Names
+============================
+
+From version 0.6.5 you are now able to ``name`` a container and ``link`` it to another
+container by referring to its name. This will create a parent -> child relationship
+where the parent container can see selected information about its child.
+
+.. _run_name:
+
+Container Naming
+----------------
+
+.. versionadded:: v0.6.5
+
+You can now name your container by using the ``-name`` flag. If no name is provided, Docker
+will automatically generate a name. You can see this name using the ``docker ps`` command.
+
+.. code-block:: bash
+
+    # format is "sudo docker run -name <container_name> <image_name> <command>"
+    $ sudo docker run -name test ubuntu /bin/bash
+
+    # the flag "-a" Show all containers. Only running containers are shown by default.
+    $ sudo docker ps -a
+    CONTAINER ID        IMAGE                            COMMAND             CREATED             STATUS              PORTS               NAMES
+    2522602a0d99        ubuntu:12.04                     /bin/bash           14 seconds ago      Exit 0                                  test
+
+.. _run_link:
+
+Links: service discovery for docker
+-----------------------------------
+
+.. versionadded:: v0.6.5
+
+Links allow containers to discover and securely communicate with each other by using the
+flag ``-link name:alias``. Inter-container communication can be disabled with the daemon
+flag ``-icc=false``. With this flag set to false, Container A cannot access Container B
+unless explicitly allowed via a link. This is a huge win for securing your containers.
+When two containers are linked together Docker creates a parent child relationship
+between the containers. The parent container will be able to access information via
+environment variables of the child such as name, exposed ports, IP and other selected
+environment variables.
+
+When linking two containers Docker will use the exposed ports of the container to create
+a secure tunnel for the parent to access. If a database container only exposes port 8080
+then the linked container will only be allowed to access port 8080 and nothing else if
+inter-container communication is set to false.
+
+.. code-block:: bash
+
+    # Example: there is an image called redis-2.6 that exposes the port 6379 and starts redis-server.
+    # Let's name the container as "redis" based on that image and run it as daemon.
+    $ sudo docker run -d -name redis redis-2.6
+
+We can issue all the commands that you would expect using the name "redis"; start, stop,
+attach, using the name for our container. The name also allows us to link other containers
+into this one.
+
+Next, we can start a new web application that has a dependency on Redis and apply a link
+to connect both containers. If you noticed when running our Redis server we did not use
+the -p flag to publish the Redis port to the host system. Redis exposed port 6379 and
+this is all we need to establish a link.
+
+.. code-block:: bash
+
+    # Linking the redis container as a child
+    $ sudo docker run -t -i -link redis:db -name webapp ubuntu bash
+
+When you specified -link redis:db you are telling docker to link the container named redis
+into this new container with the alias db. Environment variables are prefixed with the alias
+so that the parent container can access network and environment information from the containers
+that are linked into it.
+
+If we inspect the environment variables of the second container, we would see all the information
+about the child container.
+
+.. code-block:: bash
+
+    $ root@4c01db0b339c:/# env
+
+    HOSTNAME=4c01db0b339c
+    DB_NAME=/webapp/db
+    TERM=xterm
+    DB_PORT=tcp://172.17.0.8:6379
+    DB_PORT_6379_TCP=tcp://172.17.0.8:6379
+    DB_PORT_6379_TCP_PROTO=tcp
+    DB_PORT_6379_TCP_ADDR=172.17.0.8
+    DB_PORT_6379_TCP_PORT=6379
+    PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+    PWD=/
+    SHLVL=1
+    HOME=/
+    container=lxc
+    _=/usr/bin/env
+    root@4c01db0b339c:/#
+
+Accessing the network information along with the environment of the child container allows
+us to easily connect to the Redis service on the specific IP and port in the environment.

docs/theme/docker/layout.html

@@ -129,7 +129,8 @@
                 <div class="row footer">
                     <div class="span12 tbox">
                         <div class="tbox">
-                            <p>Docker is an open source project, sponsored by  <a href="https://dotcloud.com">dotCloud</a>, under the <a href="https://github.com/dotcloud/docker/blob/master/LICENSE" title="Docker licence, hosted in the Github repository">apache 2.0 licence</a></p>
+                            <p>Docker is an open source project, sponsored by  <a href="https://www.docker.com">Docker Inc.</a>, under the <a href="https://github.com/dotcloud/docker/blob/master/LICENSE" title="Docker licence, hosted in the Github repository">apache 2.0 licence</a></p>
+                            <p>Documentation proudly hosted by <a href="http://www.readthedocs.org">Read the Docs</a></p>
                         </div>
 
                         <div class="social links">

engine/engine.go

@@ -2,20 +2,25 @@ package engine
 
 import (
 	"fmt"
-	"os"
-	"log"
-	"runtime"
 	"github.com/dotcloud/docker/utils"
+	"log"
+	"os"
+	"runtime"
+	"strings"
 )
 
-
 type Handler func(*Job) string
 
 var globalHandlers map[string]Handler
 
+func init() {
+	globalHandlers = make(map[string]Handler)
+}
+
 func Register(name string, handler Handler) error {
-	if globalHandlers == nil {
-		globalHandlers = make(map[string]Handler)
+	_, exists := globalHandlers[name]
+	if exists {
+		return fmt.Errorf("Can't overwrite global handler for command %s", name)
 	}
 	globalHandlers[name] = handler
 	return nil
@@ -25,8 +30,24 @@ func Register(name string, handler Handler) error {
 // It acts as a store for *containers*, and allows manipulation of these
 // containers by executing *jobs*.
 type Engine struct {
-	root		string
-	handlers	map[string]Handler
+	root     string
+	handlers map[string]Handler
+	hack     Hack // data for temporary hackery (see hack.go)
+	id       string
+}
+
+func (eng *Engine) Root() string {
+	return eng.root
+}
+
+func (eng *Engine) Register(name string, handler Handler) error {
+	eng.Logf("Register(%s) (handlers=%v)", name, eng.handlers)
+	_, exists := eng.handlers[name]
+	if exists {
+		return fmt.Errorf("Can't overwrite handler for command %s", name)
+	}
+	eng.handlers[name] = handler
+	return nil
 }
 
 // New initializes a new engine managing the directory specified at `root`.
@@ -56,22 +77,31 @@ func New(root string) (*Engine, error) {
 		return nil, err
 	}
 	eng := &Engine{
-		root:		root,
-		handlers:	globalHandlers,
+		root:     root,
+		handlers: make(map[string]Handler),
+		id:       utils.RandomString(),
+	}
+	// Copy existing global handlers
+	for k, v := range globalHandlers {
+		eng.handlers[k] = v
 	}
 	return eng, nil
 }
 
+func (eng *Engine) String() string {
+	return fmt.Sprintf("%s|%s", eng.Root(), eng.id[:8])
+}
+
 // Job creates a new job which can later be executed.
 // This function mimics `Command` from the standard os/exec package.
 func (eng *Engine) Job(name string, args ...string) *Job {
 	job := &Job{
-		eng:		eng,
-		Name:		name,
-		Args:		args,
-		Stdin:		os.Stdin,
-		Stdout:		os.Stdout,
-		Stderr:		os.Stderr,
+		Eng:    eng,
+		Name:   name,
+		Args:   args,
+		Stdin:  os.Stdin,
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
 	}
 	handler, exists := eng.handlers[name]
 	if exists {
@@ -80,3 +110,7 @@ func (eng *Engine) Job(name string, args ...string) *Job {
 	return job
 }
 
+func (eng *Engine) Logf(format string, args ...interface{}) (n int, err error) {
+	prefixedFormat := fmt.Sprintf("[%s] %s\n", eng, strings.TrimRight(format, "\n"))
+	return fmt.Fprintf(os.Stderr, prefixedFormat, args...)
+}

engine/hack.go

@@ -0,0 +1,21 @@
+package engine
+
+type Hack map[string]interface{}
+
+func (eng *Engine) Hack_GetGlobalVar(key string) interface{} {
+	if eng.hack == nil {
+		return nil
+	}
+	val, exists := eng.hack[key]
+	if !exists {
+		return nil
+	}
+	return val
+}
+
+func (eng *Engine) Hack_SetGlobalVar(key string, val interface{}) {
+	if eng.hack == nil {
+		eng.hack = make(Hack)
+	}
+	eng.hack[key] = val
+}

engine/job.go

@@ -1,11 +1,16 @@
 package engine
 
 import (
-	"io"
-	"strings"
-	"fmt"
+	"bufio"
+	"bytes"
 	"encoding/json"
-	"github.com/dotcloud/docker/utils"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
 )
 
 // A job is the fundamental unit of work in the docker engine.
@@ -20,26 +25,45 @@ import (
 // One slight variation is that jobs report their status as a string. The
 // string "0" indicates success, and any other strings indicates an error.
 // This allows for richer error reporting.
-// 
+//
 type Job struct {
-	eng	*Engine
-	Name	string
-	Args	[]string
-	env	[]string
-	Stdin	io.ReadCloser
-	Stdout	io.WriteCloser
-	Stderr	io.WriteCloser
-	handler	func(*Job) string
-	status	string
+	Eng     *Engine
+	Name    string
+	Args    []string
+	env     []string
+	Stdin   io.Reader
+	Stdout  io.Writer
+	Stderr  io.Writer
+	handler func(*Job) string
+	status  string
+	onExit  []func()
 }
 
 // Run executes the job and blocks until the job completes.
 // If the job returns a failure status, an error is returned
 // which includes the status.
 func (job *Job) Run() error {
-	randId := utils.RandomString()[:4]
-	fmt.Printf("Job #%s: %s\n", randId, job)
-	defer fmt.Printf("Job #%s: %s = '%s'", randId, job, job.status)
+	defer func() {
+		var wg sync.WaitGroup
+		for _, f := range job.onExit {
+			wg.Add(1)
+			go func(f func()) {
+				f()
+				wg.Done()
+			}(f)
+		}
+		wg.Wait()
+	}()
+	if job.Stdout != nil && job.Stdout != os.Stdout {
+		job.Stdout = io.MultiWriter(job.Stdout, os.Stdout)
+	}
+	if job.Stderr != nil && job.Stderr != os.Stderr {
+		job.Stderr = io.MultiWriter(job.Stderr, os.Stderr)
+	}
+	job.Eng.Logf("+job %s", job.CallString())
+	defer func() {
+		job.Eng.Logf("-job %s%s", job.CallString(), job.StatusString())
+	}()
 	if job.handler == nil {
 		job.status = "command not found"
 	} else {
@@ -51,27 +75,105 @@ func (job *Job) Run() error {
 	return nil
 }
 
+func (job *Job) StdoutParseLines(dst *[]string, limit int) {
+	job.parseLines(job.StdoutPipe(), dst, limit)
+}
+
+func (job *Job) StderrParseLines(dst *[]string, limit int) {
+	job.parseLines(job.StderrPipe(), dst, limit)
+}
+
+func (job *Job) parseLines(src io.Reader, dst *[]string, limit int) {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		scanner := bufio.NewScanner(src)
+		for scanner.Scan() {
+			// If the limit is reached, flush the rest of the source and return
+			if limit > 0 && len(*dst) >= limit {
+				io.Copy(ioutil.Discard, src)
+				return
+			}
+			line := scanner.Text()
+			// Append the line (with delimitor removed)
+			*dst = append(*dst, line)
+		}
+	}()
+	job.onExit = append(job.onExit, wg.Wait)
+}
+
+func (job *Job) StdoutParseString(dst *string) {
+	lines := make([]string, 0, 1)
+	job.StdoutParseLines(&lines, 1)
+	job.onExit = append(job.onExit, func() {
+		if len(lines) >= 1 {
+			*dst = lines[0]
+		}
+	})
+}
+
+func (job *Job) StderrParseString(dst *string) {
+	lines := make([]string, 0, 1)
+	job.StderrParseLines(&lines, 1)
+	job.onExit = append(job.onExit, func() { *dst = lines[0] })
+}
+
+func (job *Job) StdoutPipe() io.ReadCloser {
+	r, w := io.Pipe()
+	job.Stdout = w
+	job.onExit = append(job.onExit, func() { w.Close() })
+	return r
+}
+
+func (job *Job) StderrPipe() io.ReadCloser {
+	r, w := io.Pipe()
+	job.Stderr = w
+	job.onExit = append(job.onExit, func() { w.Close() })
+	return r
+}
+
+func (job *Job) CallString() string {
+	return fmt.Sprintf("%s(%s)", job.Name, strings.Join(job.Args, ", "))
+}
+
+func (job *Job) StatusString() string {
+	// FIXME: if a job returns the empty string, it will be printed
+	// as not having returned.
+	// (this only affects String which is a convenience function).
+	if job.status != "" {
+		var okerr string
+		if job.status == "0" {
+			okerr = "OK"
+		} else {
+			okerr = "ERR"
+		}
+		return fmt.Sprintf(" = %s (%s)", okerr, job.status)
+	}
+	return ""
+}
+
 // String returns a human-readable description of `job`
 func (job *Job) String() string {
-	return strings.Join(append([]string{job.Name}, job.Args...), " ")
+	return fmt.Sprintf("%s.%s%s", job.Eng, job.CallString(), job.StatusString())
 }
 
 func (job *Job) Getenv(key string) (value string) {
-        for _, kv := range job.env {
-                if strings.Index(kv, "=") == -1 {
-                        continue
-                }
-                parts := strings.SplitN(kv, "=", 2)
-                if parts[0] != key {
-                        continue
-                }
-                if len(parts) < 2 {
-                        value = ""
-                } else {
-                        value = parts[1]
-                }
-        }
-        return
+	for _, kv := range job.env {
+		if strings.Index(kv, "=") == -1 {
+			continue
+		}
+		parts := strings.SplitN(kv, "=", 2)
+		if parts[0] != key {
+			continue
+		}
+		if len(parts) < 2 {
+			value = ""
+		} else {
+			value = parts[1]
+		}
+	}
+	return
 }
 
 func (job *Job) GetenvBool(key string) (value bool) {
@@ -90,6 +192,19 @@ func (job *Job) SetenvBool(key string, value bool) {
 	}
 }
 
+func (job *Job) GetenvInt(key string) int64 {
+	s := strings.Trim(job.Getenv(key), " \t")
+	val, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return -1
+	}
+	return val
+}
+
+func (job *Job) SetenvInt(key string, value int64) {
+	job.Setenv(key, fmt.Sprintf("%d", value))
+}
+
 func (job *Job) GetenvList(key string) []string {
 	sval := job.Getenv(key)
 	l := make([]string, 0, 1)
@@ -109,5 +224,111 @@ func (job *Job) SetenvList(key string, value []string) error {
 }
 
 func (job *Job) Setenv(key, value string) {
-	job.env = append(job.env, key + "=" + value)
+	job.env = append(job.env, key+"="+value)
+}
+
+// DecodeEnv decodes `src` as a json dictionary, and adds
+// each decoded key-value pair to the environment.
+//
+// If `text` cannot be decoded as a json dictionary, an error
+// is returned.
+func (job *Job) DecodeEnv(src io.Reader) error {
+	m := make(map[string]interface{})
+	if err := json.NewDecoder(src).Decode(&m); err != nil {
+		return err
+	}
+	for k, v := range m {
+		// FIXME: we fix-convert float values to int, because
+		// encoding/json decodes integers to float64, but cannot encode them back.
+		// (See http://golang.org/src/pkg/encoding/json/decode.go#L46)
+		if fval, ok := v.(float64); ok {
+			job.SetenvInt(k, int64(fval))
+		} else if sval, ok := v.(string); ok {
+			job.Setenv(k, sval)
+		} else if val, err := json.Marshal(v); err == nil {
+			job.Setenv(k, string(val))
+		} else {
+			job.Setenv(k, fmt.Sprintf("%v", v))
+		}
+	}
+	return nil
+}
+
+func (job *Job) EncodeEnv(dst io.Writer) error {
+	m := make(map[string]interface{})
+	for k, v := range job.Environ() {
+		var val interface{}
+		if err := json.Unmarshal([]byte(v), &val); err == nil {
+			// FIXME: we fix-convert float values to int, because
+			// encoding/json decodes integers to float64, but cannot encode them back.
+			// (See http://golang.org/src/pkg/encoding/json/decode.go#L46)
+			if fval, isFloat := val.(float64); isFloat {
+				val = int(fval)
+			}
+			m[k] = val
+		} else {
+			m[k] = v
+		}
+	}
+	if err := json.NewEncoder(dst).Encode(&m); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (job *Job) ExportEnv(dst interface{}) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("ExportEnv %s", err)
+		}
+	}()
+	var buf bytes.Buffer
+	// step 1: encode/marshal the env to an intermediary json representation
+	if err := job.EncodeEnv(&buf); err != nil {
+		return err
+	}
+	// step 2: decode/unmarshal the intermediary json into the destination object
+	if err := json.NewDecoder(&buf).Decode(dst); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (job *Job) ImportEnv(src interface{}) (err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("ImportEnv: %s", err)
+		}
+	}()
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(src); err != nil {
+		return err
+	}
+	if err := job.DecodeEnv(&buf); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (job *Job) Environ() map[string]string {
+	m := make(map[string]string)
+	for _, kv := range job.env {
+		parts := strings.SplitN(kv, "=", 2)
+		m[parts[0]] = parts[1]
+	}
+	return m
+}
+
+func (job *Job) Logf(format string, args ...interface{}) (n int, err error) {
+	prefixedFormat := fmt.Sprintf("[%s] %s\n", job, strings.TrimRight(format, "\n"))
+	return fmt.Fprintf(job.Stderr, prefixedFormat, args...)
+}
+
+func (job *Job) Printf(format string, args ...interface{}) (n int, err error) {
+	return fmt.Fprintf(job.Stdout, format, args...)
+}
+
+func (job *Job) Errorf(format string, args ...interface{}) (n int, err error) {
+	return fmt.Fprintf(job.Stderr, format, args...)
+
 }

engine/utils.go

@@ -1,21 +1,21 @@
 package engine
 
 import (
-	"testing"
+	"fmt"
+	"github.com/dotcloud/docker/utils"
+	"io/ioutil"
 	"runtime"
 	"strings"
-	"fmt"
-	"io/ioutil"
-	"github.com/dotcloud/docker/utils"
+	"testing"
 )
 
 var globalTestID string
 
 func init() {
-	Register("dummy", func(job *Job) string { return ""; })
+	Register("dummy", func(job *Job) string { return "" })
 }
 
-func mkEngine(t *testing.T) *Engine {
+func newTestEngine(t *testing.T) *Engine {
 	// Use the caller function name as a prefix.
 	// This helps trace temp directories back to their test.
 	pc, _, _, _ := runtime.Caller(1)
@@ -38,5 +38,5 @@ func mkEngine(t *testing.T) *Engine {
 }
 
 func mkJob(t *testing.T, name string, args ...string) *Job {
-	return mkEngine(t).Job(name, args...)
+	return newTestEngine(t).Job(name, args...)
 }

gograph/gograph.go

@@ -48,7 +48,7 @@ type WalkFunc func(fullPath string, entity *Entity) error
 // Graph database for storing entities and their relationships
 type Database struct {
 	conn *sql.DB
-	mux  sync.Mutex
+	mux  sync.RWMutex
 }
 
 // Create a new graph database initialized with a root entity
@@ -138,7 +138,14 @@ func (db *Database) Set(fullPath, id string) (*Entity, error) {
 
 // Return true if a name already exists in the database
 func (db *Database) Exists(name string) bool {
-	return db.Get(name) != nil
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
+	e, err := db.get(name)
+	if err != nil {
+		return false
+	}
+	return e != nil
 }
 
 func (db *Database) setEdge(parentPath, name string, e *Entity) error {
@@ -165,6 +172,9 @@ func (db *Database) RootEntity() *Entity {
 
 // Return the entity for a given path
 func (db *Database) Get(name string) *Entity {
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
 	e, err := db.get(name)
 	if err != nil {
 		return nil
@@ -200,23 +210,36 @@ func (db *Database) get(name string) (*Entity, error) {
 // List all entities by from the name
 // The key will be the full path of the entity
 func (db *Database) List(name string, depth int) Entities {
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
 	out := Entities{}
 	e, err := db.get(name)
 	if err != nil {
 		return out
 	}
-	for c := range db.children(e, name, depth) {
+
+	children, err := db.children(e, name, depth, nil)
+	if err != nil {
+		return out
+	}
+
+	for _, c := range children {
 		out[c.FullPath] = c.Entity
 	}
 	return out
 }
 
+// Walk through the child graph of an entity, calling walkFunc for each child entity.
+// It is safe for walkFunc to call graph functions.
 func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error {
-	e, err := db.get(name)
+	children, err := db.Children(name, depth)
 	if err != nil {
 		return err
 	}
-	for c := range db.children(e, name, depth) {
+
+	// Note: the database lock must not be held while calling walkFunc
+	for _, c := range children {
 		if err := walkFunc(c.FullPath, c.Entity); err != nil {
 			return err
 		}
@@ -224,8 +247,24 @@ func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error {
 	return nil
 }
 
+// Return the children of the specified entity
+func (db *Database) Children(name string, depth int) ([]WalkMeta, error) {
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
+	e, err := db.get(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.children(e, name, depth, nil)
+}
+
 // Return the refrence count for a specified id
 func (db *Database) Refs(id string) int {
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
 	var count int
 	if err := db.conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil {
 		return 0
@@ -235,6 +274,9 @@ func (db *Database) Refs(id string) int {
 
 // Return all the id's path references
 func (db *Database) RefPaths(id string) Edges {
+	db.mux.RLock()
+	defer db.mux.RUnlock()
+
 	refs := Edges{}
 
 	rows, err := db.conn.Query("SELECT name, parent_id FROM edge WHERE entity_id = ?;", id)
@@ -356,56 +398,51 @@ type WalkMeta struct {
 	Edge     *Edge
 }
 
-func (db *Database) children(e *Entity, name string, depth int) <-chan WalkMeta {
-	out := make(chan WalkMeta)
+func (db *Database) children(e *Entity, name string, depth int, entities []WalkMeta) ([]WalkMeta, error) {
 	if e == nil {
-		close(out)
-		return out
+		return entities, nil
 	}
 
-	go func() {
-		rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id)
-		if err != nil {
-			close(out)
+	rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var entityId, entityName string
+		if err := rows.Scan(&entityId, &entityName); err != nil {
+			return nil, err
+		}
+		child := &Entity{entityId}
+		edge := &Edge{
+			ParentID: e.id,
+			Name:     entityName,
+			EntityID: child.id,
 		}
-		defer rows.Close()
 
-		for rows.Next() {
-			var entityId, entityName string
-			if err := rows.Scan(&entityId, &entityName); err != nil {
-				// Log error
-				continue
-			}
-			child := &Entity{entityId}
-			edge := &Edge{
-				ParentID: e.id,
-				Name:     entityName,
-				EntityID: child.id,
-			}
+		meta := WalkMeta{
+			Parent:   e,
+			Entity:   child,
+			FullPath: path.Join(name, edge.Name),
+			Edge:     edge,
+		}
 
-			meta := WalkMeta{
-				Parent:   e,
-				Entity:   child,
-				FullPath: path.Join(name, edge.Name),
-				Edge:     edge,
-			}
+		entities = append(entities, meta)
 
-			out <- meta
-			if depth == 0 {
-				continue
-			}
+		if depth != 0 {
 			nDepth := depth
 			if depth != -1 {
 				nDepth -= 1
 			}
-			sc := db.children(child, meta.FullPath, nDepth)
-			for c := range sc {
-				out <- c
+			entities, err = db.children(child, meta.FullPath, nDepth, entities)
+			if err != nil {
+				return nil, err
 			}
 		}
-		close(out)
-	}()
-	return out
+	}
+
+	return entities, nil
 }
 
 // Return the entity based on the parent path and name

graph.go

@@ -3,6 +3,7 @@ package docker
 import (
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -17,11 +18,12 @@ import (
 type Graph struct {
 	Root    string
 	idIndex *utils.TruncIndex
+	driver  graphdriver.Driver
 }
 
 // NewGraph instantiates a new graph at the given root path in the filesystem.
 // `root` will be created if it doesn't exist.
-func NewGraph(root string) (*Graph, error) {
+func NewGraph(root string, driver graphdriver.Driver) (*Graph, error) {
 	abspath, err := filepath.Abs(root)
 	if err != nil {
 		return nil, err
@@ -30,9 +32,11 @@ func NewGraph(root string) (*Graph, error) {
 	if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
+
 	graph := &Graph{
 		Root:    abspath,
 		idIndex: utils.NewTruncIndex(),
+		driver:  driver,
 	}
 	if err := graph.restore(); err != nil {
 		return nil, err
@@ -78,16 +82,22 @@ func (graph *Graph) Get(name string) (*Image, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Check that the filesystem layer exists
+	rootfs, err := graph.driver.Get(img.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+	}
 	if img.ID != id {
 		return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID)
 	}
 	img.graph = graph
 	if img.Size == 0 {
-		root, err := img.root()
+		size, err := utils.TreeSize(rootfs)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("Error computing size of rootfs %s: %s", img.ID, err)
 		}
-		if err := StoreSize(img, root); err != nil {
+		img.Size = size
+		if err := img.SaveSize(graph.imageRoot(id)); err != nil {
 			return nil, err
 		}
 	}
@@ -131,14 +141,24 @@ func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Im
 	if err != nil {
 		return fmt.Errorf("Mktemp failed: %s", err)
 	}
-	if err := StoreImage(img, jsonData, layerData, tmp); err != nil {
+
+	// Create root filesystem in the driver
+	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
+		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	// Mount the root filesystem so we can apply the diff/layer
+	rootfs, err := graph.driver.Get(img.ID)
+	if err != nil {
+		return fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err)
+	}
+	img.graph = graph
+	if err := StoreImage(img, jsonData, layerData, tmp, rootfs); err != nil {
 		return err
 	}
 	// Commit
 	if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
 		return err
 	}
-	img.graph = graph
 	graph.idIndex.Add(img.ID)
 	return nil
 }
@@ -152,7 +172,7 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression,
 	if err != nil {
 		return nil, err
 	}
-	tmp, err := graph.tmp()
+	tmp, err := graph.Mktemp("")
 	if err != nil {
 		return nil, err
 	}
@@ -160,42 +180,25 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression,
 	if err != nil {
 		return nil, err
 	}
-	return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root)
+	return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp)
 }
 
 // Mktemp creates a temporary sub-directory inside the graph's filesystem.
 func (graph *Graph) Mktemp(id string) (string, error) {
-	if id == "" {
-		id = GenerateID()
+	dir := path.Join(graph.Root, "_tmp", GenerateID())
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return "", err
 	}
-	tmp, err := graph.tmp()
-	if err != nil {
-		return "", fmt.Errorf("Couldn't create temp: %s", err)
-	}
-	if tmp.Exists(id) {
-		return "", fmt.Errorf("Image %s already exists", id)
-	}
-	return tmp.imageRoot(id), nil
+	return dir, nil
 }
 
-// getDockerInitLayer returns the path of a layer containing a mountpoint suitable
+// setupInitLayer populates a directory with mountpoints suitable
 // for bind-mounting dockerinit into the container. The mountpoint is simply an
 // empty file at /.dockerinit
 //
 // This extra layer is used by all containers as the top-most ro layer. It protects
 // the container from unwanted side-effects on the rw layer.
-func (graph *Graph) getDockerInitLayer() (string, error) {
-	tmp, err := graph.tmp()
-	if err != nil {
-		return "", err
-	}
-	initLayer := tmp.imageRoot("_dockerinit")
-	if err := os.Mkdir(initLayer, 0755); err != nil && !os.IsExist(err) {
-		// If directory already existed, keep going.
-		// For all other errors, abort.
-		return "", err
-	}
-
+func setupInitLayer(initLayer string) error {
 	for pth, typ := range map[string]string{
 		"/dev/pts":         "dir",
 		"/dev/shm":         "dir",
@@ -214,32 +217,27 @@ func (graph *Graph) getDockerInitLayer() (string, error) {
 				switch typ {
 				case "dir":
 					if err := os.MkdirAll(path.Join(initLayer, pth), 0755); err != nil {
-						return "", err
+						return err
 					}
 				case "file":
 					if err := os.MkdirAll(path.Join(initLayer, path.Dir(pth)), 0755); err != nil {
-						return "", err
+						return err
 					}
 
 					if f, err := os.OpenFile(path.Join(initLayer, pth), os.O_CREATE, 0755); err != nil {
-						return "", err
+						return err
 					} else {
 						f.Close()
 					}
 				}
 			} else {
-				return "", err
+				return err
 			}
 		}
 	}
 
 	// Layer is ready to use, if it wasn't before.
-	return initLayer, nil
-}
-
-func (graph *Graph) tmp() (*Graph, error) {
-	// Changed to _tmp from :tmp:, because it messed with ":" separators in aufs branch syntax...
-	return NewGraph(path.Join(graph.Root, "_tmp"))
+	return nil
 }
 
 // Check if given error is "not empty".
@@ -271,6 +269,9 @@ func (graph *Graph) Delete(name string) error {
 	if err != nil {
 		return err
 	}
+	// Remove rootfs data from the driver
+	graph.driver.Remove(id)
+	// Remove the trashed image directory
 	return os.RemoveAll(tmp)
 }
 

graph_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"errors"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -145,12 +146,12 @@ func TestMount(t *testing.T) {
 	if err := os.MkdirAll(rw, 0700); err != nil {
 		t.Fatal(err)
 	}
-	if err := image.Mount(rootfs, rw); err != nil {
+	if _, err := graph.driver.Get(image.ID); err != nil {
 		t.Fatal(err)
 	}
 	// FIXME: test for mount contents
 	defer func() {
-		if err := Unmount(rootfs); err != nil {
+		if err := graph.driver.Cleanup(); err != nil {
 			t.Error(err)
 		}
 	}()
@@ -295,7 +296,11 @@ func tempGraph(t *testing.T) *Graph {
 	if err != nil {
 		t.Fatal(err)
 	}
-	graph, err := NewGraph(tmp)
+	backend, err := graphdriver.New(tmp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph, err := NewGraph(tmp, backend)
 	if err != nil {
 		t.Fatal(err)
 	}

graphdriver/driver.go

@@ -0,0 +1,86 @@
+package graphdriver
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/utils"
+	"os"
+	"path"
+)
+
+type InitFunc func(root string) (Driver, error)
+
+type Driver interface {
+	Create(id, parent string) error
+	Remove(id string) error
+
+	Get(id string) (dir string, err error)
+	Size(id string) (bytes int64, err error)
+
+	Cleanup() error
+}
+
+type Differ interface {
+	Diff(id string) (archive.Archive, error)
+	Changes(id string) ([]archive.Change, error)
+	ApplyDiff(id string, diff archive.Archive) error
+}
+
+var (
+	// All registred drivers
+	drivers map[string]InitFunc
+	// Slice of drivers that should be used in an order
+	priority = []string{
+		"aufs",
+		"devicemapper",
+		"dummy",
+	}
+)
+
+func init() {
+	drivers = make(map[string]InitFunc)
+}
+
+func Register(name string, initFunc InitFunc) error {
+	if _, exists := drivers[name]; exists {
+		return fmt.Errorf("Name already registered %s", name)
+	}
+	drivers[name] = initFunc
+
+	return nil
+}
+
+func getDriver(name, home string) (Driver, error) {
+	if initFunc, exists := drivers[name]; exists {
+		return initFunc(path.Join(home, name))
+	}
+	return nil, fmt.Errorf("No such driver: %s", name)
+}
+
+func New(root string) (Driver, error) {
+	var driver Driver
+	var lastError error
+	// Use environment variable DOCKER_DRIVER to force a choice of driver
+	if name := os.Getenv("DOCKER_DRIVER"); name != "" {
+		return getDriver(name, root)
+	}
+	// Check for priority drivers first
+	for _, name := range priority {
+		driver, lastError = getDriver(name, root)
+		if lastError != nil {
+			utils.Debugf("Error loading driver %s: %s", name, lastError)
+			continue
+		}
+		return driver, nil
+	}
+
+	// Check all registered drivers if no priority driver is found
+	for _, initFunc := range drivers {
+		driver, lastError = initFunc(root)
+		if lastError != nil {
+			continue
+		}
+		return driver, nil
+	}
+	return nil, lastError
+}

graphdriver/dummy/driver.go

@@ -0,0 +1,78 @@
+package dummy
+
+import (
+	"fmt"
+	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
+	"os"
+	"path"
+)
+
+func init() {
+	graphdriver.Register("dummy", Init)
+}
+
+func Init(home string) (graphdriver.Driver, error) {
+	d := &Driver{
+		home: home,
+	}
+	return d, nil
+}
+
+type Driver struct {
+	home string
+}
+
+func (d *Driver) String() string {
+	return "dummy"
+}
+
+func (d *Driver) Cleanup() error {
+	return nil
+}
+
+func (d *Driver) Create(id string, parent string) error {
+	dir := d.dir(id)
+	if err := os.MkdirAll(path.Dir(dir), 0700); err != nil {
+		return err
+	}
+	if err := os.Mkdir(dir, 0700); err != nil {
+		return err
+	}
+	if parent == "" {
+		return nil
+	}
+	parentDir, err := d.Get(parent)
+	if err != nil {
+		return fmt.Errorf("%s: %s", parent, err)
+	}
+	if err := archive.CopyWithTar(parentDir, dir); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *Driver) dir(id string) string {
+	return path.Join(d.home, "dir", path.Base(id))
+}
+
+func (d *Driver) Remove(id string) error {
+	if _, err := os.Stat(d.dir(id)); err != nil {
+		return err
+	}
+	return os.RemoveAll(d.dir(id))
+}
+
+func (d *Driver) Get(id string) (string, error) {
+	dir := d.dir(id)
+	if st, err := os.Stat(dir); err != nil {
+		return "", err
+	} else if !st.IsDir() {
+		return "", fmt.Errorf("%s: not a directory", dir)
+	}
+	return dir, nil
+}
+
+func (d *Driver) Size(id string) (int64, error) {
+	return -1, fmt.Errorf("Not implemented")
+}

hack/ROADMAP.md

@@ -5,7 +5,7 @@ It is a curated selection of planned improvements which are either important, di
 
 For a more complete view of planned and requested improvements, see [the Github issues](https://github.com/dotcloud/docker/issues).
 
-Tu suggest changes to the roadmap, including additions, please write the change as if it were already in effect, and make a pull request.
+To suggest changes to the roadmap, including additions, please write the change as if it were already in effect, and make a pull request.
 
 
 ## Container wiring and service discovery

hack/infrastructure/docker-ci/Dockerfile

@@ -1,14 +1,16 @@
-# VERSION:        0.22
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
-# DESCRIPTION:    Deploy docker-ci on Amazon EC2
+# VERSION:        0.25
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
+# DESCRIPTION:    Deploy docker-ci on Digital Ocean
 # COMMENTS:
 #     CONFIG_JSON is an environment variable json string loaded as:
 #
 # export CONFIG_JSON='
-#     { "AWS_TAG":             "EC2_instance_name",
-#       "AWS_ACCESS_KEY":      "EC2_access_key",
-#       "AWS_SECRET_KEY":      "EC2_secret_key",
+#     { "DROPLET_NAME":        "docker-ci",
+#       "DO_CLIENT_ID":        "Digital_Ocean_client_id",
+#       "DO_API_KEY":          "Digital_Ocean_api_key",
+#       "DOCKER_KEY_ID":       "Digital_Ocean_ssh_key_id",
+#       "DOCKER_CI_KEY_PATH":  "docker-ci_private_key_path",
 #       "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
 #       "DOCKER_CI_KEY":       "$(cat docker-ci_ssh_private_key.key)",
 #       "BUILDBOT_PWD":        "Buildbot_server_password",
@@ -33,9 +35,11 @@
 
 from ubuntu:12.04
 
-run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
-run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim
-run pip install boto fabric
+run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \
+    > /etc/apt/sources.list
+run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \
+    python-pip ssh rsync less vim
+run pip install requests fabric
 
 # Add deployment code and set default container command
 add . /docker-ci

hack/infrastructure/docker-ci/VERSION

@@ -0,0 +1 @@
+0.4.5

hack/infrastructure/docker-ci/buildbot/master.cfg

@@ -43,7 +43,7 @@ c['slavePortnum'] = PORT_MASTER
 
 # Schedulers
 c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker',
-    'index','registry','coverage','nightlyrelease'])]
+    'index','registry','docker-coverage','registry-coverage','nightlyrelease'])]
 c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None,
     change_filter=filter.ChangeFilter(branch='master',
     repository='https://github.com/dotcloud/docker'), builderNames=['docker'])]
@@ -51,7 +51,7 @@ c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
     change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
     builderNames=['pullrequest'])]
 c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease',
-    'coverage'], hour=7, minute=00)]
+    'docker-coverage','registry-coverage'], hour=7, minute=00)]
 c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
     hour=range(0,24,4), minute=15)]
 
@@ -76,17 +76,25 @@ c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
 
 # Docker coverage test
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='Coverage', logEnviron=False,
+factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False,
     usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
     DOCKER_CI_PATH)))
-c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
+c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'],
+    factory=factory)]
+
+# Docker registry coverage test
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False,
+    usePTY=True, command='docker run registry_coverage'.format(
+    DOCKER_CI_PATH)))
+c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'],
     factory=factory)]
 
 # Registry functional test
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='registry', logEnviron=False,
     command='. {0}/master/credentials.cfg; '
-    '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH),
+    '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
     usePTY=True))
 c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
     factory=factory)]
@@ -95,16 +103,17 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='index', logEnviron=False,
     command='. {0}/master/credentials.cfg; '
-    '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH),
+    '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
     usePTY=True))
 c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
     factory=factory)]
 
 # Docker nightly release
+nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET='
+    'test.docker.io dockerbuilder hack/dind dockerbuild.sh')
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False,
-    usePTY=True, command='docker run -privileged'
-    ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder'))
+factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,
+    usePTY=True, command=nightlyrelease_cmd))
 c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
     factory=factory)]
 

hack/infrastructure/docker-ci/deployment.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 
-import os, sys, re, json, base64
-from boto.ec2.connection import EC2Connection
+import os, sys, re, json, requests, base64
 from subprocess import call
 from fabric import api
 from fabric.api import cd, run, put, sudo
 from os import environ as env
+from datetime import datetime
 from time import sleep
 
 # Remove SSH private key as it needs more processing
@@ -20,42 +20,41 @@ for key in CONFIG:
 env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
     env['CONFIG_JSON'],flags=re.DOTALL)
 
-
-AWS_TAG = env.get('AWS_TAG','docker-ci')
-AWS_KEY_NAME = 'dotcloud-dev'       # Same as CONFIG_JSON['DOCKER_CI_PUB']
-AWS_AMI = 'ami-d582d6bc'            # Ubuntu 13.04
-AWS_REGION = 'us-east-1'
-AWS_TYPE = 'm1.small'
-AWS_SEC_GROUPS = 'gateway'
-AWS_IMAGE_USER = 'ubuntu'
+DROPLET_NAME = env.get('DROPLET_NAME','docker-ci')
+TIMEOUT = 120            # Seconds before timeout droplet creation
+IMAGE_ID = 1004145       # Docker on Ubuntu 13.04
+REGION_ID = 4            # New York 2
+SIZE_ID = 62             # memory 2GB
+DO_IMAGE_USER = 'root'   # Image user on Digital Ocean
+API_URL = 'https://api.digitalocean.com/'
 DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
 DOCKER_CI_PATH = '/docker-ci'
 CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
 
 
-class AWS_EC2:
-    '''Amazon EC2'''
-    def __init__(self, access_key, secret_key):
+class DigitalOcean():
+
+    def __init__(self, key, client):
         '''Set default API parameters'''
-        self.handler = EC2Connection(access_key, secret_key)
-    def create_instance(self, tag, instance_type):
-        reservation = self.handler.run_instances(**instance_type)
-        instance = reservation.instances[0]
-        sleep(10)
-        while instance.state != 'running':
-            sleep(5)
-            instance.update()
-            print "Instance state: %s" % (instance.state)
-        instance.add_tag("Name",tag)
-        print "instance %s done!" % (instance.id)
-        return instance.ip_address
-    def get_instances(self):
-        return self.handler.get_all_instances()
-    def get_tags(self):
-        return dict([(i.instances[0].id, i.instances[0].tags['Name'])
-            for i in self.handler.get_all_instances() if i.instances[0].tags])
-    def del_instance(self, instance_id):
-        self.handler.terminate_instances(instance_ids=[instance_id])
+        self.key = key
+        self.client = client
+        self.api_url = API_URL
+
+    def api(self, cmd_path, api_arg={}):
+        '''Make api call'''
+        api_arg.update({'api_key':self.key, 'client_id':self.client})
+        resp = requests.get(self.api_url + cmd_path, params=api_arg).text
+        resp = json.loads(resp)
+        if resp['status'] != 'OK':
+            raise Exception(resp['error_message'])
+        return resp
+
+    def droplet_data(self, name):
+        '''Get droplet data'''
+        data = self.api('droplets')
+        data = [droplet for droplet in data['droplets']
+            if droplet['name'] == name]
+        return data[0] if data else {}
 
 
 def json_fmt(data):
@@ -63,20 +62,36 @@ def json_fmt(data):
     return json.dumps(data, sort_keys = True, indent = 2)
 
 
-# Create EC2 API handler
-ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
+do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID'])
 
-# Stop processing if AWS_TAG exists on EC2
-if AWS_TAG in ec2.get_tags().values():
-    print ('Instance: {} already deployed. Not further processing.'
-        .format(AWS_TAG))
+# Get DROPLET_NAME data
+data = do.droplet_data(DROPLET_NAME)
+
+# Stop processing if DROPLET_NAME exists on Digital Ocean
+if data:
+    print ('Droplet: {} already deployed. Not further processing.'
+        .format(DROPLET_NAME))
     exit(1)
 
-ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
-    'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
+# Create droplet
+do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID,
+    'image_id':IMAGE_ID, 'size_id':SIZE_ID,
+    'ssh_key_ids':[env['DOCKER_KEY_ID']]})
 
-# Wait 30 seconds for the machine to boot
-sleep(30)
+# Wait for droplet to be created.
+start_time = datetime.now()
+while (data.get('status','') != 'active' and (
+ datetime.now()-start_time).seconds < TIMEOUT):
+    data = do.droplet_data(DROPLET_NAME)
+    print data['status']
+    sleep(3)
+
+# Wait for the machine to boot
+sleep(15)
+
+# Get droplet IP
+ip = str(data['ip_address'])
+print 'droplet: {}    ip: {}'.format(DROPLET_NAME, ip)
 
 # Create docker-ci ssh private key so docker-ci docker container can communicate
 # with its EC2 instance
@@ -86,7 +101,7 @@ os.chmod('/root/.ssh/id_rsa',0600)
 open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
 
 api.env.host_string = ip
-api.env.user = AWS_IMAGE_USER
+api.env.user = DO_IMAGE_USER
 api.env.key_filename = '/root/.ssh/id_rsa'
 
 # Correct timezone
@@ -100,20 +115,17 @@ sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB']))
 credentials = {
     'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'],
     'AWS_SECRET_KEY': env['PKG_SECRET_KEY'],
-    'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'],
-    'INDEX_AUTH': env['INDEX_AUTH']}
+    'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']}
 open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
     base64.b64encode(json.dumps(credentials)))
 
 # Transfer docker
 sudo('mkdir -p ' + DOCKER_CI_PATH)
-sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
-call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
+sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH))
+call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip,
     os.path.dirname(DOCKER_CI_PATH)), shell=True)
 
 # Install Docker and Buildbot dependencies
-sudo('addgroup docker')
-sudo('usermod -a -G docker ubuntu')
 sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
 sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
 sudo('echo deb https://get.docker.io/ubuntu docker main >'
@@ -123,7 +135,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
     ' > /etc/apt/sources.list; apt-get update')
 sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
     ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
-    ' aufs-tools make libfontconfig libevent-dev')
+    ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev')
 sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
     'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
 sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
@@ -135,13 +147,13 @@ sudo('curl -s https://phantomjs.googlecode.com/files/'
     'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin'
     ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs')
 
-# Preventively reboot docker-ci daily
-sudo('ln -s /sbin/reboot /etc/cron.daily')
-
 # Build docker-ci containers
 sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
+sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH))
 sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
     DOCKER_CI_PATH))
+sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format(
+    DOCKER_CI_PATH))
 
 # Download docker-ci testing container
 sudo('docker pull mzdaniel/test_docker')
@@ -154,3 +166,6 @@ sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'
     env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'],
     env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'],
     env['REGISTRY_SECRET_KEY']))
+
+# Preventively reboot docker-ci daily
+sudo('ln -s /sbin/reboot /etc/cron.daily')

hack/infrastructure/docker-ci/docker-test/Dockerfile

@@ -1,6 +1,6 @@
-# VERSION:        0.3
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# VERSION:        0.4
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
 # DESCRIPTION:    Testing docker PRs and commits on top of master using
 # REFERENCES:     This code reuses the excellent implementation of
 #                 Docker in Docker made by Jerome Petazzoni.
@@ -15,15 +15,10 @@
 # TO_RUN:         docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch]
 
 from docker
-maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+maintainer Daniel Mizyrycki <daniel@docker.com>
 
-# Setup go environment. Extracted from /Dockerfile
-env     CGO_ENABLED 0
-env     GOROOT  /goroot
-env     PATH    $PATH:/goroot/bin
-env     GOPATH  /go:/go/src/github.com/dotcloud/docker/vendor
-volume  /var/lib/docker
-workdir /go/src/github.com/dotcloud/docker
+# Setup go in PATH. Extracted from /Dockerfile
+env PATH /usr/local/go/bin:$PATH
 
 # Add test_docker.sh
 add test_docker.sh /usr/bin/test_docker.sh

hack/infrastructure/docker-ci/docker-test/test_docker.sh

@@ -8,31 +8,26 @@ BRANCH=${3-master}
 # Compute test paths
 DOCKER_PATH=/go/src/github.com/dotcloud/docker
 
+# Timestamp
+echo
+date; echo
+
 # Fetch latest master
+cd /
 rm -rf /go
-mkdir -p $DOCKER_PATH
+git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH
 cd $DOCKER_PATH
-git init .
-git fetch -q http://github.com/dotcloud/docker master
-git reset --hard FETCH_HEAD
 
 # Merge commit
-#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS
-git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1
-
-# Merge commit in top of master
 git fetch -q "$REPO" "$BRANCH"
-git merge --no-edit $COMMIT || exit 1
+git merge --no-edit $COMMIT || exit 255
 
 # Test commit
-go test -v; exit_status=$?
+./hack/make.sh test; exit_status=$?
 
 # Display load if test fails
-if [ $exit_status -eq 1 ] ; then
+if [ $exit_status -ne 0 ] ; then
     uptime; echo; free
 fi
 
-# Cleanup testing directory
-rm -rf $BASE_PATH
-
 exit $exit_status

hack/infrastructure/docker-ci/functionaltests/test_registry.sh

@@ -8,10 +8,12 @@ rm -rf docker-registry
 # Setup the environment
 export SETTINGS_FLAVOR=test
 export DOCKER_REGISTRY_CONFIG=config_test.yml
+export PYTHONPATH=$(pwd)/docker-registry/test
 
 # Get latest docker registry
 git clone -q https://github.com/dotcloud/docker-registry.git
 cd docker-registry
+sed -Ei "s#(boto_bucket: ).+#\1_env:S3_BUCKET#" config_test.yml
 
 # Get dependencies
 pip install -q -r requirements.txt
@@ -20,7 +22,6 @@ pip install -q tox
 
 # Run registry tests
 tox || exit 1
-export PYTHONPATH=$(pwd)/docker-registry
 python -m unittest discover -p s3.py -s test || exit 1
 python -m unittest discover -p workflow.py -s test
 

hack/infrastructure/docker-ci/nightlyrelease/Dockerfile

@@ -1,20 +1,19 @@
-# VERSION:        1.2
-# DOCKER-VERSION  0.6.3
-# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# VERSION:        1.6
+# DOCKER-VERSION  0.6.6
+# AUTHOR:         Daniel Mizyrycki <daniel@docker.com>
 # DESCRIPTION:    Build docker nightly release using Docker in Docker.
 # REFERENCES:     This code reuses the excellent implementation of docker in docker
 #                 made by Jerome Petazzoni.  https://github.com/jpetazzo/dind
 # COMMENTS:
 #   release_credentials.json is a base64 json encoded file containing:
 #       { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id",
-#         "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key'
-#         "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature'
-#         "INDEX_AUTH='Encripted_index_authentication' }
+#         "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key",
+#         "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" }
 # TO_BUILD:       docker build -t dockerbuilder .
-# TO_RELEASE:     docker run -i -t -privileged  -e AWS_S3_BUCKET="test.docker.io" dockerbuilder
+# TO_RELEASE:     docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh
 
 from docker
-maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+maintainer Daniel Mizyrycki <daniel@docker.com>
 
 # Add docker dependencies and downloading packages
 run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
@@ -24,11 +23,8 @@ run apt-get update; apt-get install -y -q wget python2.7
 run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker
 
 # Add proto docker builder
-add ./dockerbuild /usr/bin/dockerbuild
-run chmod +x /usr/bin/dockerbuild
+add ./dockerbuild.sh /usr/bin/dockerbuild.sh
+run chmod +x /usr/bin/dockerbuild.sh
 
 # Add release credentials
 add ./release_credentials.json /root/release_credentials.json
-
-# Launch build process in a container
-cmd dockerbuild

hack/infrastructure/docker-ci/nightlyrelease/dockerbuild

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH
-# are decoded from /root/release_credentials.json
-# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
-
-# Enable debugging
-set -x
-
-# Fetch docker master branch
-rm -rf  /go/src/github.com/dotcloud/docker
-cd /
-git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
-cd /go/src/github.com/dotcloud/docker
-
-# Launch docker daemon using dind inside the container
-./hack/dind /usr/bin/docker -d &
-sleep 5
-
-# Add an uncommitted change to generate a timestamped release
-date > timestamp
-
-# Build the docker package using /Dockerfile
-docker build -t docker .
-
-# Run Docker unittests binary and Ubuntu package
-docker run -privileged docker hack/make.sh
-exit_status=$?
-
-# Display load if test fails
-if [ $exit_status -eq 1 ] ; then
-    uptime; echo; free
-    exit 1
-fi
-
-# Commit binary and ubuntu bundles for release
-docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release
-
-# Turn debug off to load credentials from the environment
-set +x
-eval $(cat /root/release_credentials.json  | python -c '
-import sys,json,base64;
-d=json.loads(base64.b64decode(sys.stdin.read()));
-exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
-set -x
-
-# Push docker nightly
-echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release  hack/release.sh
-set +x
-docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE  release  hack/release.sh

hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded
+# from /root/release_credentials.json
+# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
+
+# Turn debug off to load credentials from the environment
+set +x
+eval $(cat /root/release_credentials.json  | python -c '
+import sys,json,base64;
+d=json.loads(base64.b64decode(sys.stdin.read()));
+exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
+
+# Fetch docker master branch
+set -x
+cd /
+rm -rf /go
+git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
+cd /go/src/github.com/dotcloud/docker
+
+# Launch docker daemon using dind inside the container
+/usr/bin/docker version
+/usr/bin/docker -d &
+sleep 5
+
+# Build Docker release container
+docker build -t docker .
+
+# Test docker and if everything works well, release
+echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh
+set +x
+docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh
+exit_status=$?
+
+# Display load if test fails
+set -x
+if [ $exit_status -ne 0 ] ; then
+    uptime; echo; free
+    exit 1
+fi

hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json

@@ -1 +0,0 @@
-eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0=

hack/infrastructure/docker-ci/registry-coverage/Dockerfile

@@ -0,0 +1,18 @@
+# VERSION:        0.1
+# DOCKER-VERSION  0.6.4
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# DESCRIPTION:    Docker registry coverage
+# COMMENTS:       Add registry coverage into the docker-ci image
+# TO_BUILD:       docker build -t registry_coverage .
+# TO_RUN:         docker run registry_coverage
+
+from docker-ci
+maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+
+# Add registry_coverager.sh and dependencies
+run pip install coverage flask pyyaml requests simplejson python-glanceclient \
+    blinker redis boto gevent rsa mock
+add registry_coverage.sh /usr/bin/registry_coverage.sh
+run chmod +x /usr/bin/registry_coverage.sh
+
+cmd "/usr/bin/registry_coverage.sh"

hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -x
+
+# Setup the environment
+REGISTRY_PATH=/data/docker-registry
+export SETTINGS_FLAVOR=test
+export DOCKER_REGISTRY_CONFIG=config_test.yml
+export PYTHONPATH=$REGISTRY_PATH/test
+
+# Fetch latest docker-registry master
+rm -rf $REGISTRY_PATH
+git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH
+cd $REGISTRY_PATH
+
+# Generate coverage
+coverage run -m unittest discover test || exit 1
+coverage report --include='./*' --omit='./test/*'

hack/infrastructure/docker-ci/report/deployment.py

@@ -34,7 +34,7 @@ env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read()
 
 DROPLET_NAME = env.get('DROPLET_NAME','report')
 TIMEOUT = 120            # Seconds before timeout droplet creation
-IMAGE_ID = 894856        # Docker on Ubuntu 13.04
+IMAGE_ID = 1004145       # Docker on Ubuntu 13.04
 REGION_ID = 4            # New York 2
 SIZE_ID = 66             # memory 512MB
 DO_IMAGE_USER = 'root'   # Image user on Digital Ocean

hack/make/dyntest

@@ -22,7 +22,12 @@ bundle_test() {
 		for test_dir in $(find_test_dirs); do (
 			set -x
 			cd $test_dir
+			
+			# Install packages that are dependencies of the tests.
+			#   Note: Does not run the tests.
 			go test -i -ldflags "$LDFLAGS" $BUILDFLAGS
+			
+			# Run the tests with the optional $TESTFLAGS.
 			export TEST_DOCKERINIT_PATH=$DEST/../dynbinary/dockerinit-$VERSION
 			go test -v -ldflags "$LDFLAGS -X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" $BUILDFLAGS $TESTFLAGS
 		)  done

hack/make/test

@@ -16,7 +16,12 @@ bundle_test() {
 		for test_dir in $(find_test_dirs); do (
 			set -x
 			cd $test_dir
+			
+			# Install packages that are dependencies of the tests.
+			#   Note: Does not run the tests.
 			go test -i -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS
+			
+			# Run the tests with the optional $TESTFLAGS.
 			go test -v -ldflags "$LDFLAGS $LDFLAGS_STATIC" $BUILDFLAGS $TESTFLAGS
 		)  done
 	} 2>&1 | tee $DEST/test.log

hack/make/ubuntu

@@ -10,7 +10,7 @@ fi
 PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)"
 PACKAGE_URL="http://www.docker.io/"
 PACKAGE_MAINTAINER="docker@dotcloud.com"
-PACKAGE_DESCRIPTION="lxc-docker is a Linux container runtime
+PACKAGE_DESCRIPTION="Linux container runtime
 Docker complements LXC with a high-level API which operates at the process
 level. It runs unix processes with strong guarantees of isolation and
 repeatability across servers.
@@ -37,27 +37,51 @@ bundle_ubuntu() {
 	# This will fail if the binary bundle hasn't been built
 	cp $DEST/../binary/docker-$VERSION $DIR/usr/bin/docker
 
-	# Generate postinst/prerm scripts
-	cat >/tmp/postinst <<'EOF'
+	# Generate postinst/prerm/postrm scripts
+	cat > /tmp/postinst <<'EOF'
 #!/bin/sh
-service docker stop || true
-grep -q '^docker:' /etc/group || groupadd --system docker || true
-service docker start
-EOF
-	cat >/tmp/prerm <<'EOF'
-#!/bin/sh
-service docker stop || true
+set -e
+set -u
 
-case "$1" in
-	purge|remove|abort-install)
-		groupdel docker || true
-		;;
-		
-	upgrade|failed-upgrade|abort-upgrade)
-		# don't touch docker group
-		;;
-esac
+getent group docker > /dev/null || groupadd --system docker || true
+
+update-rc.d docker defaults > /dev/null || true
+if [ -n "$2" ]; then
+	_dh_action=restart
+else
+	_dh_action=start
+fi
+service docker $_dh_action 2>/dev/null || true
+
+#DEBHELPER#
 EOF
+	cat > /tmp/prerm <<'EOF'
+#!/bin/sh
+set -e
+set -u
+
+service docker stop 2>/dev/null || true
+
+#DEBHELPER#
+EOF
+	cat > /tmp/postrm <<'EOF'
+#!/bin/sh
+set -e
+set -u
+
+if [ "$1" = "purge" ] ; then
+	update-rc.d docker remove > /dev/null || true
+fi
+
+# In case this system is running systemd, we make systemd reload the unit files
+# to pick up changes.
+if [ -d /run/systemd/system ] ; then
+	systemctl --system daemon-reload > /dev/null || true
+fi
+
+#DEBHELPER#
+EOF
+	# TODO swaths of these were borrowed from debhelper's auto-inserted stuff, because we're still using fpm - we need to use debhelper instead, and somehow reconcile Ubuntu that way
 	chmod +x /tmp/postinst /tmp/prerm
 
 	(
@@ -66,6 +90,7 @@ EOF
 		    --name lxc-docker-$VERSION --version $PKGVERSION \
 		    --after-install /tmp/postinst \
 		    --before-remove /tmp/prerm \
+		    --after-remove /tmp/postrm \
 		    --architecture "$PACKAGE_ARCHITECTURE" \
 		    --prefix / \
 		    --depends lxc \
@@ -82,6 +107,8 @@ EOF
 		    --vendor "$PACKAGE_VENDOR" \
 		    --config-files /etc/init/docker.conf \
 		    --config-files /etc/init.d/docker \
+		    --config-files /etc/default/docker \
+		    --deb-compression xz \
 		    -t deb .
 		mkdir empty
 		fpm -s dir -C empty \
@@ -92,7 +119,12 @@ EOF
 		    --maintainer "$PACKAGE_MAINTAINER" \
 		    --url "$PACKAGE_URL" \
 		    --vendor "$PACKAGE_VENDOR" \
+		    --config-files /etc/init/docker.conf \
+		    --config-files /etc/init.d/docker \
+		    --config-files /etc/default/docker \
+		    --deb-compression xz \
 		    -t deb .
+		# note: the --config-files lines have to be duplicated to stop overwrite on package upgrade (since we have to use this funky virtual package)
 	)
 }
 

hack/release.sh

@@ -97,7 +97,7 @@ write_to_s3() {
 	DEST=$1
 	F=`mktemp`
 	cat > $F
-	s3cmd --acl-public put $F $DEST
+	s3cmd --acl-public --mime-type='text/plain' put $F $DEST
 	rm -f $F
 }
 
@@ -107,14 +107,14 @@ s3_url() {
 			echo "https://$BUCKET"
 			;;
 		*)
-			echo "http://$BUCKET.s3.amazonaws.com"
+			s3cmd ws-info s3://$BUCKET | awk -v 'FS=: +' '/http:\/\/'$BUCKET'/ { gsub(/\/+$/, "", $2); print $2 }'
 			;;
 	esac
 }
 
 # Upload the 'ubuntu' bundle to S3:
 # 1. A full APT repository is published at $BUCKET/ubuntu/
-# 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/info
+# 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/index
 release_ubuntu() {
 	[ -e bundles/$VERSION/ubuntu ] || {
 		echo >&2 './hack/make.sh must be run before release_ubuntu'
@@ -168,7 +168,7 @@ EOF
 
 	# Upload repo
 	s3cmd --acl-public sync $APTDIR/ s3://$BUCKET/ubuntu/
-	cat <<EOF | write_to_s3 s3://$BUCKET/ubuntu/info
+	cat <<EOF | write_to_s3 s3://$BUCKET/ubuntu/index
 # Add the repository to your APT sources
 echo deb $(s3_url)/ubuntu docker main > /etc/apt/sources.list.d/docker.list
 # Then import the repository key
@@ -180,7 +180,12 @@ apt-get update ; apt-get install -y lxc-docker
 # Alternatively, just use the curl-able install.sh script provided at $(s3_url)
 #
 EOF
-	echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu/info"
+
+	# Add redirect at /ubuntu/info for URL-backwards-compatibility
+	rm -rf /tmp/emptyfile && touch /tmp/emptyfile
+	s3cmd --acl-public --add-header='x-amz-website-redirect-location:/ubuntu/' --mime-type='text/plain' put /tmp/emptyfile s3://$BUCKET/ubuntu/info
+
+	echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu"
 }
 
 # Upload a static binary to S3
@@ -189,14 +194,20 @@ release_binary() {
 		echo >&2 './hack/make.sh must be run before release_binary'
 		exit 1
 	}
+
 	S3DIR=s3://$BUCKET/builds/Linux/x86_64
 	s3cmd --acl-public put bundles/$VERSION/binary/docker-$VERSION $S3DIR/docker-$VERSION
-	cat <<EOF | write_to_s3 s3://$BUCKET/builds/info
+	cat <<EOF | write_to_s3 s3://$BUCKET/builds/index
 # To install, run the following command as root:
 curl -O $(s3_url)/builds/Linux/x86_64/docker-$VERSION && chmod +x docker-$VERSION && sudo mv docker-$VERSION /usr/local/bin/docker
 # Then start docker in daemon mode:
 sudo /usr/local/bin/docker -d
 EOF
+
+	# Add redirect at /builds/info for URL-backwards-compatibility
+	rm -rf /tmp/emptyfile && touch /tmp/emptyfile
+	s3cmd --acl-public --add-header='x-amz-website-redirect-location:/builds/' --mime-type='text/plain' put /tmp/emptyfile s3://$BUCKET/builds/info
+
 	if [ -z "$NOLATEST" ]; then
 		echo "Copying docker-$VERSION to docker-latest"
 		s3cmd --acl-public cp $S3DIR/docker-$VERSION $S3DIR/docker-latest

image.go

@@ -6,14 +6,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/graphdriver"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
-	"os/exec"
 	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -61,39 +59,30 @@ func LoadImage(root string) (*Image, error) {
 		}
 	}
 
-	// Check that the filesystem layer exists
-	if stat, err := os.Stat(layerPath(root)); err != nil {
-		if os.IsNotExist(err) {
-			return nil, fmt.Errorf("Couldn't load image %s: no filesystem layer", img.ID)
-		}
-		return nil, err
-	} else if !stat.IsDir() {
-		return nil, fmt.Errorf("Couldn't load image %s: %s is not a directory", img.ID, layerPath(root))
-	}
 	return img, nil
 }
 
-func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root string) error {
-	// Check that root doesn't already exist
-	if _, err := os.Stat(root); err == nil {
-		return fmt.Errorf("Image %s already exists", img.ID)
-	} else if !os.IsNotExist(err) {
-		return err
-	}
+func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root, rootfs string) error {
 	// Store the layer
-	layer := layerPath(root)
+	layer := rootfs
 	if err := os.MkdirAll(layer, 0755); err != nil {
 		return err
 	}
 
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		start := time.Now()
-		utils.Debugf("Start untar layer")
-		if err := archive.Untar(layerData, layer); err != nil {
-			return err
+		if differ, ok := img.graph.driver.(graphdriver.Differ); ok {
+			if err := differ.ApplyDiff(img.ID, layerData); err != nil {
+				return err
+			}
+		} else {
+			start := time.Now()
+			utils.Debugf("Start untar layer")
+			if err := archive.ApplyLayer(layer, layerData); err != nil {
+				return err
+			}
+			utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 		}
-		utils.Debugf("Untar time: %vs", time.Now().Sub(start).Seconds())
 	}
 
 	// If raw json is provided, then use it
@@ -108,104 +97,43 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root str
 			return err
 		}
 	}
-
-	return StoreSize(img, root)
-}
-
-func StoreSize(img *Image, root string) error {
-	layer := layerPath(root)
-
-	var totalSize int64 = 0
-	filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error {
-		totalSize += fileInfo.Size()
-		return nil
-	})
-	img.Size = totalSize
-
-	if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil {
-		return nil
+	// Compute and save the size of the rootfs
+	size, err := utils.TreeSize(rootfs)
+	if err != nil {
+		return fmt.Errorf("Error computing size of rootfs %s: %s", img.ID, err)
+	}
+	img.Size = size
+	if err := img.SaveSize(root); err != nil {
+		return err
 	}
 
 	return nil
 }
 
-func layerPath(root string) string {
-	return path.Join(root, "layer")
+// SaveSize stores the current `size` value of `img` in the directory `root`.
+func (img *Image) SaveSize(root string) error {
+	if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(img.Size))), 0600); err != nil {
+		return fmt.Errorf("Error storing image size in %s/layersize: %s", root, err)
+	}
+	return nil
 }
 
 func jsonPath(root string) string {
 	return path.Join(root, "json")
 }
 
-func MountAUFS(ro []string, rw string, target string) error {
-	// FIXME: Now mount the layers
-	rwBranch := fmt.Sprintf("%v=rw", rw)
-	roBranches := ""
-	for _, layer := range ro {
-		roBranches += fmt.Sprintf("%v=ro+wh:", layer)
-	}
-	branches := fmt.Sprintf("br:%v:%v", rwBranch, roBranches)
-
-	branches += ",xino=/dev/shm/aufs.xino"
-
-	//if error, try to load aufs kernel module
-	if err := mount("none", target, "aufs", 0, branches); err != nil {
-		log.Printf("Kernel does not support AUFS, trying to load the AUFS module with modprobe...")
-		if err := exec.Command("modprobe", "aufs").Run(); err != nil {
-			return fmt.Errorf("Unable to load the AUFS module")
-		}
-		log.Printf("...module loaded.")
-		if err := mount("none", target, "aufs", 0, branches); err != nil {
-			return fmt.Errorf("Unable to mount using aufs")
-		}
-	}
-	return nil
-}
-
 // TarLayer returns a tar archive of the image's filesystem layer.
 func (image *Image) TarLayer(compression archive.Compression) (archive.Archive, error) {
-	layerPath, err := image.layer()
+	if image.graph == nil {
+		return nil, fmt.Errorf("Can't load storage driver for unregistered image %s", image.ID)
+	}
+	layerPath, err := image.graph.driver.Get(image.ID)
 	if err != nil {
 		return nil, err
 	}
 	return archive.Tar(layerPath, compression)
 }
 
-func (image *Image) Mount(root, rw string) error {
-	if mounted, err := Mounted(root); err != nil {
-		return err
-	} else if mounted {
-		return fmt.Errorf("%s is already mounted", root)
-	}
-	layers, err := image.layers()
-	if err != nil {
-		return err
-	}
-	// Create the target directories if they don't exist
-	if err := os.Mkdir(root, 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-	if err := os.Mkdir(rw, 0755); err != nil && !os.IsExist(err) {
-		return err
-	}
-	if err := MountAUFS(layers, rw, root); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (image *Image) Changes(rw string) ([]Change, error) {
-	layers, err := image.layers()
-	if err != nil {
-		return nil, err
-	}
-	return Changes(layers, rw)
-}
-
-func (image *Image) ShortID() string {
-	return utils.TruncateID(image.ID)
-}
-
 func ValidateID(id string) error {
 	if id == "" {
 		return fmt.Errorf("Image id can't be empty")
@@ -241,39 +169,6 @@ func (img *Image) History() ([]*Image, error) {
 	return parents, nil
 }
 
-// layers returns all the filesystem layers needed to mount an image
-// FIXME: @shykes refactor this function with the new error handling
-//        (I'll do it if I have time tonight, I focus on the rest)
-func (img *Image) layers() ([]string, error) {
-	var list []string
-	var e error
-	if err := img.WalkHistory(
-		func(img *Image) (err error) {
-			if layer, err := img.layer(); err != nil {
-				e = err
-			} else if layer != "" {
-				list = append(list, layer)
-			}
-			return err
-		},
-	); err != nil {
-		return nil, err
-	} else if e != nil { // Did an error occur inside the handler?
-		return nil, e
-	}
-	if len(list) == 0 {
-		return nil, fmt.Errorf("No layer found for image %s\n", img.ID)
-	}
-
-	// Inject the dockerinit layer (empty place-holder for mount-binding dockerinit)
-	if dockerinitLayer, err := img.getDockerInitLayer(); err != nil {
-		return nil, err
-	} else {
-		list = append([]string{dockerinitLayer}, list...)
-	}
-	return list, nil
-}
-
 func (img *Image) WalkHistory(handler func(*Image) error) (err error) {
 	currentImg := img
 	for currentImg != nil {
@@ -300,13 +195,6 @@ func (img *Image) GetParent() (*Image, error) {
 	return img.graph.Get(img.Parent)
 }
 
-func (img *Image) getDockerInitLayer() (string, error) {
-	if img.graph == nil {
-		return "", fmt.Errorf("Can't lookup dockerinit layer of unregistered image")
-	}
-	return img.graph.getDockerInitLayer()
-}
-
 func (img *Image) root() (string, error) {
 	if img.graph == nil {
 		return "", fmt.Errorf("Can't lookup root of unregistered image")
@@ -314,15 +202,6 @@ func (img *Image) root() (string, error) {
 	return img.graph.imageRoot(img.ID), nil
 }
 
-// Return the path of an image's layer
-func (img *Image) layer() (string, error) {
-	root, err := img.root()
-	if err != nil {
-		return "", err
-	}
-	return layerPath(root), nil
-}
-
 func (img *Image) getParentsSize(size int64) int64 {
 	parentImage, err := img.GetParent()
 	if err != nil || parentImage == nil {

iptables/iptables.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -27,8 +28,10 @@ type Chain struct {
 }
 
 func NewChain(name, bridge string) (*Chain, error) {
-	if err := Raw("-t", "nat", "-N", name); err != nil {
+	if output, err := Raw("-t", "nat", "-N", name); err != nil {
 		return nil, err
+	} else if len(output) != 0 {
+		return nil, fmt.Errorf("Error creating new iptables chain: %s", output)
 	}
 	chain := &Chain{
 		Name:   name,
@@ -52,13 +55,25 @@ func RemoveExistingChain(name string) error {
 }
 
 func (c *Chain) Forward(action Action, ip net.IP, port int, proto, dest_addr string, dest_port int) error {
-	return Raw("-t", "nat", fmt.Sprint(action), c.Name,
+	daddr := ip.String()
+	if ip.IsUnspecified() {
+		// iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we
+		// want "0.0.0.0/0". "0/0" is correctly interpreted as "any
+		// value" by both iptables and ip6tables.
+		daddr = "0/0"
+	}
+	if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name,
 		"-p", proto,
-		"-d", ip.String(),
+		"-d", daddr,
 		"--dport", strconv.Itoa(port),
 		"!", "-i", c.Bridge,
 		"-j", "DNAT",
-		"--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port)))
+		"--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return fmt.Errorf("Error iptables forward: %s", output)
+	}
+	return nil
 }
 
 func (c *Chain) Prerouting(action Action, args ...string) error {
@@ -66,7 +81,12 @@ func (c *Chain) Prerouting(action Action, args ...string) error {
 	if len(args) > 0 {
 		a = append(a, args...)
 	}
-	return Raw(append(a, "-j", c.Name)...)
+	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return fmt.Errorf("Error iptables prerouting: %s", output)
+	}
+	return nil
 }
 
 func (c *Chain) Output(action Action, args ...string) error {
@@ -74,7 +94,12 @@ func (c *Chain) Output(action Action, args ...string) error {
 	if len(args) > 0 {
 		a = append(a, args...)
 	}
-	return Raw(append(a, "-j", c.Name)...)
+	if output, err := Raw(append(a, "-j", c.Name)...); err != nil {
+		return err
+	} else if len(output) != 0 {
+		return fmt.Errorf("Error iptables output: %s", output)
+	}
+	return nil
 }
 
 func (c *Chain) Remove() error {
@@ -94,17 +119,23 @@ func (c *Chain) Remove() error {
 
 // Check if an existing rule exists
 func Exists(args ...string) bool {
-	return Raw(append([]string{"-C"}, args...)...) == nil
+	if _, err := Raw(append([]string{"-C"}, args...)...); err != nil {
+		return false
+	}
+	return true
 }
 
-func Raw(args ...string) error {
+func Raw(args ...string) ([]byte, error) {
 	path, err := exec.LookPath("iptables")
 	if err != nil {
-		return ErrIptablesNotFound
+		return nil, ErrIptablesNotFound
 	}
-	if err := exec.Command(path, args...).Run(); err != nil {
-		return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " "))
+	if os.Getenv("DEBUG") != "" {
+		fmt.Printf("[DEBUG] [iptables]: %s, %v\n", path, args)
 	}
-	return nil
-
+	output, err := exec.Command(path, args...).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err)
+	}
+	return output, err
 }

iptables/iptables_test.go

@@ -6,13 +6,13 @@ import (
 )
 
 func TestIptables(t *testing.T) {
-	if err := Raw("-L"); err != nil {
+	if _, err := Raw("-L"); err != nil {
 		t.Fatal(err)
 	}
 	path := os.Getenv("PATH")
 	os.Setenv("PATH", "")
 	defer os.Setenv("PATH", path)
-	if err := Raw("-L"); err == nil {
+	if _, err := Raw("-L"); err == nil {
 		t.Fatal("Not finding iptables in the PATH should cause an error")
 	}
 }

links.go

@@ -120,7 +120,7 @@ func (l *Link) Disable() {
 
 func (l *Link) toggle(action string, ignoreErrors bool) error {
 	for _, p := range l.Ports {
-		if err := iptables.Raw(action, "FORWARD",
+		if output, err := iptables.Raw(action, "FORWARD",
 			"-i", l.BridgeInterface, "-o", l.BridgeInterface,
 			"-p", p.Proto(),
 			"-s", l.ParentIP,
@@ -128,9 +128,11 @@ func (l *Link) toggle(action string, ignoreErrors bool) error {
 			"-d", l.ChildIP,
 			"-j", "ACCEPT"); !ignoreErrors && err != nil {
 			return err
+		} else if len(output) != 0 {
+			return fmt.Errorf("Error toggle iptables forward: %s", output)
 		}
 
-		if err := iptables.Raw(action, "FORWARD",
+		if output, err := iptables.Raw(action, "FORWARD",
 			"-i", l.BridgeInterface, "-o", l.BridgeInterface,
 			"-p", p.Proto(),
 			"-s", l.ChildIP,
@@ -138,6 +140,8 @@ func (l *Link) toggle(action string, ignoreErrors bool) error {
 			"-d", l.ParentIP,
 			"-j", "ACCEPT"); !ignoreErrors && err != nil {
 			return err
+		} else if len(output) != 0 {
+			return fmt.Errorf("Error toggle iptables forward: %s", output)
 		}
 	}
 	return nil

namesgenerator/names-generator.go

@@ -12,7 +12,7 @@ type NameChecker interface {
 
 var (
 	colors  = [...]string{"white", "silver", "gray", "black", "blue", "green", "cyan", "yellow", "gold", "orange", "brown", "red", "violet", "pink", "magenta", "purple", "maroon", "crimson", "plum", "fuchsia", "lavender", "slate", "navy", "azure", "aqua", "olive", "teal", "lime", "beige", "tan", "sienna"}
-  animals = [...]string{"ant", "bear", "bird", "cat", "chicken", "cow", "deer", "dog", "donkey", "duck", "fish", "fox", "frog", "horse", "kangaroo", "koala", "lemur", "lion", "lizard", "monkey", "octopus", "pig", "shark", "sheep", "sloth", "spider", "squirrel", "tiger", "toad", "weasel", "whale", "wolf"}
+	animals = [...]string{"ant", "bear", "bird", "cat", "chicken", "cow", "deer", "dog", "donkey", "duck", "fish", "fox", "frog", "horse", "kangaroo", "koala", "lemur", "lion", "lizard", "monkey", "octopus", "pig", "shark", "sheep", "sloth", "spider", "squirrel", "tiger", "toad", "weasel", "whale", "wolf"}
 )
 
 func GenerateRandomName(checker NameChecker) (string, error) {

netlink/netlink_darwin.go

@@ -9,7 +9,6 @@ func NetworkGetRoutes() ([]*net.IPNet, error) {
 	return nil, fmt.Errorf("Not implemented")
 }
 
-
 func NetworkLinkAdd(name string, linkType string) error {
 	return fmt.Errorf("Not implemented")
 }
@@ -18,7 +17,6 @@ func NetworkLinkUp(iface *net.Interface) error {
 	return fmt.Errorf("Not implemented")
 }
 
-
 func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error {
 	return fmt.Errorf("Not implemented")
 }

network.go

@@ -168,10 +168,28 @@ func CreateBridgeIface(config *DaemonConfig) error {
 	}
 
 	if config.EnableIptables {
-		if err := iptables.Raw("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr,
+		// Enable NAT
+		if output, err := iptables.Raw("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr,
 			"!", "-d", ifaceAddr, "-j", "MASQUERADE"); err != nil {
 			return fmt.Errorf("Unable to enable network bridge NAT: %s", err)
+		} else if len(output) != 0 {
+			return fmt.Errorf("Error iptables postrouting: %s", output)
 		}
+
+		// Accept incoming packets for existing connections
+		if output, err := iptables.Raw("-I", "FORWARD", "-o", config.BridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil {
+			return fmt.Errorf("Unable to allow incoming packets: %s", err)
+		} else if len(output) != 0 {
+			return fmt.Errorf("Error iptables allow incoming: %s", output)
+		}
+
+		// Accept all non-intercontainer outgoing packets
+		if output, err := iptables.Raw("-I", "FORWARD", "-i", config.BridgeIface, "!", "-o", config.BridgeIface, "-j", "ACCEPT"); err != nil {
+			return fmt.Errorf("Unable to allow outgoing packets: %s", err)
+		} else if len(output) != 0 {
+			return fmt.Errorf("Error iptables allow outgoing: %s", output)
+		}
+
 	}
 	return nil
 }
@@ -678,18 +696,30 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) {
 
 	// Configure iptables for link support
 	if config.EnableIptables {
-		args := []string{"FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j", "DROP"}
+		args := []string{"FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j"}
+		acceptArgs := append(args, "ACCEPT")
+		dropArgs := append(args, "DROP")
 
 		if !config.InterContainerCommunication {
-			if !iptables.Exists(args...) {
+			iptables.Raw(append([]string{"-D"}, acceptArgs...)...)
+			if !iptables.Exists(dropArgs...) {
 				utils.Debugf("Disable inter-container communication")
-				if err := iptables.Raw(append([]string{"-A"}, args...)...); err != nil {
+				if output, err := iptables.Raw(append([]string{"-I"}, dropArgs...)...); err != nil {
 					return nil, fmt.Errorf("Unable to prevent intercontainer communication: %s", err)
+				} else if len(output) != 0 {
+					return nil, fmt.Errorf("Error disabling intercontainer communication: %s", output)
 				}
 			}
 		} else {
-			utils.Debugf("Enable inter-container communication")
-			iptables.Raw(append([]string{"-D"}, args...)...)
+			iptables.Raw(append([]string{"-D"}, dropArgs...)...)
+			if !iptables.Exists(acceptArgs...) {
+				utils.Debugf("Enable inter-container communication")
+				if output, err := iptables.Raw(append([]string{"-I"}, acceptArgs...)...); err != nil {
+					return nil, fmt.Errorf("Unable to allow intercontainer communication: %s", err)
+				} else if len(output) != 0 {
+					return nil, fmt.Errorf("Error enabling intercontainer communication: %s", output)
+				}
+			}
 		}
 	}
 

runtime.go

@@ -5,7 +5,12 @@ import (
 	"container/list"
 	"database/sql"
 	"fmt"
+	"github.com/dotcloud/docker/archive"
+	_ "github.com/dotcloud/docker/aufs"
+	_ "github.com/dotcloud/docker/devmapper"
 	"github.com/dotcloud/docker/gograph"
+	"github.com/dotcloud/docker/graphdriver"
+	_ "github.com/dotcloud/docker/graphdriver/dummy"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
@@ -13,6 +18,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -39,6 +45,7 @@ type Runtime struct {
 	srv            *Server
 	config         *DaemonConfig
 	containerGraph *gograph.Database
+	driver         graphdriver.Driver
 }
 
 // List returns an array of all containers registered in the runtime.
@@ -117,6 +124,13 @@ func (runtime *Runtime) Register(container *Container) error {
 		return err
 	}
 
+	// Get the root filesystem from the driver
+	rootfs, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return fmt.Errorf("Error getting container filesystem %s from driver %s: %s", container.ID, runtime.driver, err)
+	}
+	container.rootfs = rootfs
+
 	// init the wait lock
 	container.waitLock = make(chan struct{})
 
@@ -172,6 +186,7 @@ func (runtime *Runtime) Register(container *Container) error {
 	if !container.State.Running {
 		close(container.waitLock)
 	} else if !nomonitor {
+		container.allocateNetwork()
 		go container.monitor()
 	}
 	return nil
@@ -181,7 +196,7 @@ func (runtime *Runtime) ensureName(container *Container) error {
 	if container.Name == "" {
 		name, err := generateRandomName(runtime)
 		if err != nil {
-			name = container.ShortID()
+			name = utils.TruncateID(container.ID)
 		}
 		container.Name = name
 
@@ -221,12 +236,8 @@ func (runtime *Runtime) Destroy(container *Container) error {
 		return err
 	}
 
-	if mounted, err := container.Mounted(); err != nil {
-		return err
-	} else if mounted {
-		if err := container.Unmount(); err != nil {
-			return fmt.Errorf("Unable to unmount container %v: %v", container.ID, err)
-		}
+	if err := runtime.driver.Remove(container.ID); err != nil {
+		return fmt.Errorf("Driver %s failed to remove root filesystem %s: %s", runtime.driver, container.ID, err)
 	}
 
 	if _, err := runtime.containerGraph.Purge(container.ID); err != nil {
@@ -286,13 +297,12 @@ func (runtime *Runtime) restore() error {
 	// Any containers that are left over do not exist in the graph
 	for _, container := range containers {
 		// Try to set the default name for a container if it exists prior to links
-		name, err := generateRandomName(runtime)
+		container.Name, err = generateRandomName(runtime)
 		if err != nil {
-			container.Name = container.ShortID()
+			container.Name = utils.TruncateID(container.ID)
 		}
-		container.Name = name
 
-		if _, err := runtime.containerGraph.Set(name, container.ID); err != nil {
+		if _, err := runtime.containerGraph.Set(container.Name, container.ID); err != nil {
 			utils.Debugf("Setting default id - %s", err)
 		}
 		register(container)
@@ -444,6 +454,21 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin
 		return nil, nil, err
 	}
 
+	initID := fmt.Sprintf("%s-init", container.ID)
+	if err := runtime.driver.Create(initID, img.ID); err != nil {
+		return nil, nil, err
+	}
+	initPath, err := runtime.driver.Get(initID)
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := setupInitLayer(initPath); err != nil {
+		return nil, nil, err
+	}
+
+	if err := runtime.driver.Create(container.ID, initID); err != nil {
+		return nil, nil, err
+	}
 	resolvConf, err := utils.GetResolvConf()
 	if err != nil {
 		return nil, nil, err
@@ -482,32 +507,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin
 		return nil, nil, err
 	}
 
-	// Step 3: if hostname, build hostname and hosts files
-	container.HostnamePath = path.Join(container.root, "hostname")
-	ioutil.WriteFile(container.HostnamePath, []byte(container.Config.Hostname+"\n"), 0644)
-
-	hostsContent := []byte(`
-127.0.0.1	localhost
-::1		localhost ip6-localhost ip6-loopback
-fe00::0		ip6-localnet
-ff00::0		ip6-mcastprefix
-ff02::1		ip6-allnodes
-ff02::2		ip6-allrouters
-`)
-
-	container.HostsPath = path.Join(container.root, "hosts")
-
-	if container.Config.Domainname != "" {
-		hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...)
-		hostsContent = append([]byte(fmt.Sprintf("127.0.0.1\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...)
-	} else {
-		hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s\n", container.Config.Hostname)), hostsContent...)
-		hostsContent = append([]byte(fmt.Sprintf("127.0.0.1\t%s\n", container.Config.Hostname)), hostsContent...)
-	}
-
-	ioutil.WriteFile(container.HostsPath, hostsContent, 0644)
-
-	// Step 4: register the container
+	// Step 3: register the container
 	if err := runtime.Register(container); err != nil {
 		return nil, nil, err
 	}
@@ -609,6 +609,13 @@ func NewRuntime(config *DaemonConfig) (*Runtime, error) {
 }
 
 func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
+	// Load storage driver
+	driver, err := graphdriver.New(config.Root)
+	if err != nil {
+		return nil, err
+	}
+	utils.Debugf("Using graph driver %s", driver)
+
 	runtimeRepo := path.Join(config.Root, "containers")
 
 	if err := os.MkdirAll(runtimeRepo, 0700); err != nil && !os.IsExist(err) {
@@ -618,11 +625,11 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 	if err := linkLxcStart(config.Root); err != nil {
 		return nil, err
 	}
-	g, err := NewGraph(path.Join(config.Root, "graph"))
+	g, err := NewGraph(path.Join(config.Root, "graph"), driver)
 	if err != nil {
 		return nil, err
 	}
-	volumes, err := NewGraph(path.Join(config.Root, "volumes"))
+	volumes, err := NewGraph(path.Join(config.Root, "volumes"), driver)
 	if err != nil {
 		return nil, err
 	}
@@ -667,6 +674,7 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 		volumes:        volumes,
 		config:         config,
 		containerGraph: graph,
+		driver:         driver,
 	}
 
 	if err := runtime.restore(); err != nil {
@@ -677,9 +685,81 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) {
 
 func (runtime *Runtime) Close() error {
 	runtime.networkManager.Close()
+	runtime.driver.Cleanup()
 	return runtime.containerGraph.Close()
 }
 
+func (runtime *Runtime) Mount(container *Container) error {
+	dir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, runtime.driver, err)
+	}
+	if container.rootfs == "" {
+		container.rootfs = dir
+	} else if container.rootfs != dir {
+		return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
+			runtime.driver, container.ID, container.rootfs, dir)
+	}
+	return nil
+}
+
+func (runtime *Runtime) Unmount(container *Container) error {
+	// FIXME: Unmount is deprecated because drivers are responsible for mounting
+	// and unmounting when necessary. Use driver.Remove() instead.
+	return nil
+}
+
+func (runtime *Runtime) Changes(container *Container) ([]archive.Change, error) {
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Changes(container.ID)
+	}
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	initDir, err := runtime.driver.Get(container.ID + "-init")
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container init rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+	return archive.ChangesDirs(cDir, initDir)
+}
+
+func (runtime *Runtime) Diff(container *Container) (archive.Archive, error) {
+	if differ, ok := runtime.driver.(graphdriver.Differ); ok {
+		return differ.Diff(container.ID)
+	}
+
+	changes, err := runtime.Changes(container)
+	if err != nil {
+		return nil, err
+	}
+
+	cDir, err := runtime.driver.Get(container.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Error getting container rootfs %s from driver %s: %s", container.ID, container.runtime.driver, err)
+	}
+
+	files := make([]string, 0)
+	deletions := make([]string, 0)
+	for _, change := range changes {
+		if change.Kind == archive.ChangeModify || change.Kind == archive.ChangeAdd {
+			files = append(files, change.Path)
+		}
+		if change.Kind == archive.ChangeDelete {
+			base := filepath.Base(change.Path)
+			dir := filepath.Dir(change.Path)
+			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
+		}
+	}
+	// FIXME: Why do we create whiteout files inside Tar code ?
+	return archive.TarFilter(cDir, &archive.TarOptions{
+		Compression: archive.Uncompressed,
+		Includes:    files,
+		Recursive:   false,
+		CreateFiles: deletions,
+	})
+}
+
 func linkLxcStart(root string) error {
 	sourcePath, err := exec.LookPath("lxc-start")
 	if err != nil {

runtime_test.go

@@ -3,11 +3,13 @@ package docker
 import (
 	"bytes"
 	"fmt"
+	"github.com/dotcloud/docker/engine"
 	"github.com/dotcloud/docker/sysinit"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"log"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -22,6 +24,7 @@ import (
 const (
 	unitTestImageName     = "docker-test-image"
 	unitTestImageID       = "83599e29c455eb719f77d799bc7c51521b9551972f5a850d7ad265bc1b5292f6" // 1.0
+	unitTestImageIDShort  = "83599e29c455"
 	unitTestNetworkBridge = "testdockbr0"
 	unitTestStoreBase     = "/var/lib/docker/unit-tests"
 	testDaemonAddr        = "127.0.0.1:4270"
@@ -35,6 +38,9 @@ var (
 )
 
 func nuke(runtime *Runtime) error {
+	if nonuke := os.Getenv("NONUKE"); nonuke != "" {
+		return nil
+	}
 	var wg sync.WaitGroup
 	for _, container := range runtime.List() {
 		wg.Add(1)
@@ -118,28 +124,25 @@ func init() {
 }
 
 func setupBaseImage() {
-	config := &DaemonConfig{
-		Root:        unitTestStoreBase,
-		AutoRestart: false,
-		BridgeIface: unitTestNetworkBridge,
-	}
-	runtime, err := NewRuntimeFromDirectory(config)
+	eng, err := engine.New(unitTestStoreBase)
 	if err != nil {
+		log.Fatalf("Can't initialize engine at %s: %s", unitTestStoreBase, err)
+	}
+	job := eng.Job("initapi")
+	job.Setenv("Root", unitTestStoreBase)
+	job.SetenvBool("Autorestart", false)
+	job.Setenv("BridgeIface", unitTestNetworkBridge)
+	if err := job.Run(); err != nil {
 		log.Fatalf("Unable to create a runtime for tests:", err)
 	}
-
-	// Create the "Server"
-	srv := &Server{
-		runtime:     runtime,
-		pullingPool: make(map[string]struct{}),
-		pushingPool: make(map[string]struct{}),
-	}
+	srv := mkServerFromEngine(eng, log.New(os.Stderr, "", 0))
+	runtime := srv.runtime
 
 	// If the unit test is not found, try to download it.
 	if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID {
 		// Retrieve the Image
 		if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, nil, true); err != nil {
-			log.Fatalf("Unable to pull the test image:", err)
+			log.Fatalf("Unable to pull the test image: %s", err)
 		}
 	}
 }
@@ -149,18 +152,22 @@ func spawnGlobalDaemon() {
 		utils.Debugf("Global runtime already exists. Skipping.")
 		return
 	}
-	globalRuntime = mkRuntime(log.New(os.Stderr, "", 0))
-	srv := &Server{
-		runtime:     globalRuntime,
-		pullingPool: make(map[string]struct{}),
-		pushingPool: make(map[string]struct{}),
-	}
+	t := log.New(os.Stderr, "", 0)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	globalRuntime = srv.runtime
 
 	// Spawn a Daemon
 	go func() {
 		utils.Debugf("Spawning global daemon for integration tests")
-		if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil {
-			log.Fatalf("Unable to spawn the test daemon:", err)
+		listenURL := &url.URL{
+			Scheme: testDaemonProto,
+			Host:   testDaemonAddr,
+		}
+		job := eng.Job("serveapi", listenURL.String())
+		job.SetenvBool("Logging", os.Getenv("DEBUG") != "")
+		if err := job.Run(); err != nil {
+			log.Fatalf("Unable to spawn the test daemon: %s", err)
 		}
 	}()
 	// Give some time to ListenAndServer to actually start
@@ -180,7 +187,7 @@ func GetTestImage(runtime *Runtime) *Image {
 			return image
 		}
 	}
-	log.Fatalf("Test image %v not found", unitTestImageID)
+	log.Fatalf("Test image %v not found in %s: %s", unitTestImageID, runtime.graph.Root, imgs)
 	return nil
 }
 
@@ -642,20 +649,17 @@ func TestReloadContainerLinks(t *testing.T) {
 }
 
 func TestDefaultContainerName(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
-	srv := &Server{runtime: runtime}
 
 	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	shortId, _, err := srv.ContainerCreate(config, "some_name")
-	if err != nil {
-		t.Fatal(err)
-	}
-	container := runtime.Get(shortId)
+	container := runtime.Get(createNamedTestContainer(eng, config, t, "some_name"))
 	containerID := container.ID
 
 	if container.Name != "/some_name" {
@@ -679,20 +683,17 @@ func TestDefaultContainerName(t *testing.T) {
 }
 
 func TestRandomContainerName(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
-	srv := &Server{runtime: runtime}
 
 	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	shortId, _, err := srv.ContainerCreate(config, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	container := runtime.Get(shortId)
+	container := runtime.Get(createTestContainer(eng, config, t))
 	containerID := container.ID
 
 	if container.Name == "" {
@@ -716,20 +717,17 @@ func TestRandomContainerName(t *testing.T) {
 }
 
 func TestLinkChildContainer(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
-	srv := &Server{runtime: runtime}
 
 	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	shortId, _, err := srv.ContainerCreate(config, "/webapp")
-	if err != nil {
-		t.Fatal(err)
-	}
-	container := runtime.Get(shortId)
+	container := runtime.Get(createNamedTestContainer(eng, config, t, "/webapp"))
 
 	webapp, err := runtime.GetByName("/webapp")
 	if err != nil {
@@ -745,12 +743,7 @@ func TestLinkChildContainer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	shortId, _, err = srv.ContainerCreate(config, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	childContainer := runtime.Get(shortId)
+	childContainer := runtime.Get(createTestContainer(eng, config, t))
 
 	if err := runtime.RegisterLink(webapp, childContainer, "db"); err != nil {
 		t.Fatal(err)
@@ -767,20 +760,17 @@ func TestLinkChildContainer(t *testing.T) {
 }
 
 func TestGetAllChildren(t *testing.T) {
-	runtime := mkRuntime(t)
+	eng := NewTestEngine(t)
+	srv := mkServerFromEngine(eng, t)
+	runtime := srv.runtime
 	defer nuke(runtime)
-	srv := &Server{runtime: runtime}
 
 	config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	shortId, _, err := srv.ContainerCreate(config, "/webapp")
-	if err != nil {
-		t.Fatal(err)
-	}
-	container := runtime.Get(shortId)
+	container := runtime.Get(createNamedTestContainer(eng, config, t, "/webapp"))
 
 	webapp, err := runtime.GetByName("/webapp")
 	if err != nil {
@@ -796,12 +786,7 @@ func TestGetAllChildren(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	shortId, _, err = srv.ContainerCreate(config, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	childContainer := runtime.Get(shortId)
+	childContainer := runtime.Get(createTestContainer(eng, config, t))
 
 	if err := runtime.RegisterLink(webapp, childContainer, "db"); err != nil {
 		t.Fatal(err)