Dockerfile: add kvmtool and linux-container for clr execdriver

This patch adds the needed pkgs to docker/docker development Dockerfile
in order to test the clr execdriver in the development container.

Signed-off-by: Antonio Murdaca <runcom@redhat.com>

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## 1.8.3 (2015-10-12)
+
+### Distribution
+
+- Fix layer IDs lead to local graph poisoning (CVE-2014-8178)
+- Fix manifest validation and parsing logic errors allow pull-by-digest validation bypass (CVE-2014-8179)
++ Add `--disable-legacy-registry` to prevent a daemon from using a v1 registry
+
+## 1.8.2 (2015-09-10)
+
+### Distribution
+
+- Fixes rare edge case of handling GNU LongLink and LongName entries.
+- Fix ^C on docker pull.
+- Fix docker pull issues on client disconnection.
+- Fix issue that caused the daemon to panic when loggers weren't configured properly.
+- Fix goroutine leak pulling images from registry V2.
+
+### Runtime
+
+- Fix a bug mounting cgroups for docker daemons running inside docker containers.
+- Initialize log configuration properly.
+
+### Client:
+
+- Handle `-q` flag in `docker ps` properly when there is a default format.
+
+### Networking
+
+- Fix several corner cases with netlink.
+
+### Contrib
+
+- Fix several issues with bash completion.
+
+## 1.8.1 (2015-08-12)
+
+### Distribution
+
+- Fix a bug where pushing multiple tags would result in invalid images
+
 ## 1.8.0 (2015-08-11)
 
 ### Distribution
@@ -235,7 +276,7 @@
 #### Notable Features since 1.3.0
 + Set key=value labels to the daemon (displayed in `docker info`), applied with
   new `-label` daemon flag
-+ Add support for `ENV` in Dockerfile of the form: 
++ Add support for `ENV` in Dockerfile of the form:
   `ENV name=value name2=value2...`
 + New Overlayfs Storage Driver
 + `docker info` now returns an `ID` and `Name` field
@@ -697,7 +738,7 @@
 - Fix broken images API for version less than 1.7
 - Use the right encoding for all API endpoints which return JSON
 - Move remote api client to api/
-- Queue calls to the API using generic socket wait 
+- Queue calls to the API using generic socket wait
 
 #### Runtime
 
@@ -777,7 +818,7 @@ With the ongoing changes to the networking and execution subsystems of docker te
 - Do not add hostname when networking is disabled
 * Return most recent image from the cache by date
 - Return all errors from docker wait
-* Add Content-Type Header "application/json" to GET /version and /info responses 
+* Add Content-Type Header "application/json" to GET /version and /info responses
 
 #### Other
 
@@ -805,7 +846,7 @@ With the ongoing changes to the networking and execution subsystems of docker te
 #### Runtime
 
 - Only get the image's rootfs when we need to calculate the image size
-- Correctly handle unmapping UDP ports 
+- Correctly handle unmapping UDP ports
 * Make CopyFileWithTar use a pipe instead of a buffer to save memory on docker build
 - Fix login message to say pull instead of push
 - Fix "docker load" help by removing "SOURCE" prompt and mentioning STDIN

Dockerfile

@@ -58,6 +58,12 @@ RUN apt-get update && apt-get install -y \
 	libzfs-dev \
 	--no-install-recommends
 
+# clr
+RUN	echo deb http://download.opensuse.org/repositories/home:/clearlinux:/preview/xUbuntu_15.04/ / > /etc/apt/sources.list.d/clear-containers-docker.list
+RUN apt-get --allow-unauthenticated update && apt-get install -y --force-yes \
+	kvmtool\
+	linux-container
+
 # Get lvm2 source for compiling statically
 RUN git clone -b v2_02_103 https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2
 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags
@@ -127,7 +133,7 @@ RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint
 RUN gem install --no-rdoc --no-ri fpm --version 1.3.2
 
 # Install registry
-ENV REGISTRY_COMMIT 2317f721a3d8428215a2b65da4ae85212ed473b4
+ENV REGISTRY_COMMIT ec87e9b6971d831f0eff752ddb54fb64693e51cd
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/docker/distribution.git "$GOPATH/src/github.com/docker/distribution" \

VERSION

@@ -1 +1 @@
-1.8.0
+1.8.3-clear-containers

api/client/ps.go

@@ -95,7 +95,7 @@ func (cli *DockerCli) CmdPs(args ...string) error {
 
 	f := *format
 	if len(f) == 0 {
-		if len(cli.PsFormat()) > 0 {
+		if len(cli.PsFormat()) > 0 && !*quiet {
 			f = cli.PsFormat()
 		} else {
 			f = "table"

contrib/builder/deb/ubuntu-precise/Dockerfile

@@ -2,7 +2,7 @@
 # THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
 #
 
-FROM ubuntu-debootstrap:precise
+FROM ubuntu:precise
 
 RUN apt-get update && apt-get install -y bash-completion  build-essential curl ca-certificates debhelper  git libapparmor-dev  libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 

contrib/builder/deb/ubuntu-trusty/Dockerfile

@@ -2,7 +2,7 @@
 # THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
 #
 
-FROM ubuntu-debootstrap:wily
+FROM ubuntu:trusty
 
 RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 

contrib/builder/deb/ubuntu-vivid/Dockerfile

@@ -2,7 +2,7 @@
 # THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
 #
 
-FROM ubuntu-debootstrap:vivid
+FROM ubuntu:vivid
 
 RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 

contrib/builder/deb/ubuntu-wily/Dockerfile

@@ -2,7 +2,7 @@
 # THIS FILE IS AUTOGENERATED; SEE "contrib/builder/deb/generate.sh"!
 #
 
-FROM ubuntu-debootstrap:trusty
+FROM ubuntu:wily
 
 RUN apt-get update && apt-get install -y bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-systemd git libapparmor-dev libdevmapper-dev libsqlite3-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 

contrib/completion/bash/docker

@@ -139,7 +139,7 @@ __docker_value_of_option() {
 	local counter=$((command_pos + 1))
 	while [ $counter -lt $cword ]; do
 		case ${words[$counter]} in
-			$option_glob )
+			@($option_glob) )
 				echo ${words[$counter + 1]}
 				break
 				;;
@@ -229,11 +229,12 @@ __docker_log_driver_options() {
 	# see docs/reference/logging/index.md
 	local fluentd_options="fluentd-address fluentd-tag"
 	local gelf_options="gelf-address gelf-tag"
+	local json_file_options="max-file max-size"
 	local syslog_options="syslog-address syslog-facility syslog-tag"
 
 	case $(__docker_value_of_option --log-driver) in
 		'')
-			COMPREPLY=( $( compgen -W "$fluentd_options $gelf_options $syslog_options" -S = -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "$fluentd_options $gelf_options $json_file_options $syslog_options" -S = -- "$cur" ) )
 			;;
 		fluentd)
 			COMPREPLY=( $( compgen -W "$fluentd_options" -S = -- "$cur" ) )
@@ -241,6 +242,9 @@ __docker_log_driver_options() {
 		gelf)
 			COMPREPLY=( $( compgen -W "$gelf_options" -S = -- "$cur" ) )
 			;;
+		json-file)
+			COMPREPLY=( $( compgen -W "$json_file_options" -S = -- "$cur" ) )
+			;;
 		syslog)
 			COMPREPLY=( $( compgen -W "$syslog_options" -S = -- "$cur" ) )
 			;;
@@ -320,7 +324,7 @@ __docker_signals() {
 _docker_docker() {
 	local boolean_options="
 		$global_boolean_options
-		--help -h
+		--help
 		--version -v
 	"
 
@@ -338,8 +342,6 @@ _docker_docker() {
 			;;
 	esac
 
-	__docker_complete_log_driver_options && return
-
 	case "$cur" in
 		-*)
 			COMPREPLY=( $( compgen -W "$boolean_options $global_options_with_args" -- "$cur" ) )
@@ -460,7 +462,8 @@ _docker_create() {
 _docker_daemon() {
 	local boolean_options="
 		$global_boolean_options
-		--help -h
+		--disable-legacy-registry
+		--help
 		--icc=false
 		--ip-forward=false
 		--ip-masq=false
@@ -512,7 +515,39 @@ _docker_daemon() {
 			return
 			;;
 		--storage-driver|-s)
-			COMPREPLY=( $( compgen -W "aufs devicemapper btrfs overlay" -- "$(echo $cur | tr '[:upper:]' '[:lower:]')" ) )
+			COMPREPLY=( $( compgen -W "aufs btrfs devicemapper overlay vfs zfs" -- "$(echo $cur | tr '[:upper:]' '[:lower:]')" ) )
+			return
+			;;
+		--storage-opt)
+			local devicemapper_options="
+				dm.basesize
+				dm.blkdiscard
+				dm.blocksize
+				dm.fs
+				dm.loopdatasize
+				dm.loopmetadatasize
+				dm.mkfsarg
+				dm.mountopt
+				dm.override_udev_sync_check
+				dm.thinpooldev
+			"
+			local zfs_options="zfs.fsname"
+
+			case $(__docker_value_of_option '--storage-driver|-s') in
+				'')
+					COMPREPLY=( $( compgen -W "$devicemapper_options $zfs_options" -S = -- "$cur" ) )
+					;;
+				devicemapper)
+					COMPREPLY=( $( compgen -W "$devicemapper_options" -S = -- "$cur" ) )
+					;;
+				zfs)
+					COMPREPLY=( $( compgen -W "$zfs_options" -S = -- "$cur" ) )
+					;;
+				*)
+					return
+					;;
+			esac
+			compopt -o nospace
 			return
 			;;
 		--log-level|-l)
@@ -528,6 +563,27 @@ _docker_daemon() {
 			;;
 	esac
 
+	__docker_complete_log_driver_options && return
+
+	case "${words[$cword-2]}$prev=" in
+		*dm.blkdiscard=*)
+			COMPREPLY=( $( compgen -W "false true" -- "${cur#=}" ) )
+			return
+			;;
+		*dm.fs=*)
+			COMPREPLY=( $( compgen -W "ext4 xfs" -- "${cur#=}" ) )
+			return
+			;;
+		*dm.override_udev_sync_check=*)
+			COMPREPLY=( $( compgen -W "false true" -- "${cur#=}" ) )
+			return
+			;;
+		*dm.thinpooldev=*)
+			_filedir
+			return
+			;;
+	esac
+
 	case "$cur" in
 		-*)
 			COMPREPLY=( $( compgen -W "$boolean_options $options_with_args" -- "$cur" ) )
@@ -869,7 +925,7 @@ _docker_ps() {
 			compopt -o nospace
 			return
 			;;
-		-n)
+		--format|-n)
 			return
 			;;
 	esac
@@ -893,7 +949,7 @@ _docker_ps() {
 
 	case "$cur" in
 		-*)
-			COMPREPLY=( $( compgen -W "--all -a --before --filter -f --help --latest -l -n --no-trunc --quiet -q --size -s --since" -- "$cur" ) )
+			COMPREPLY=( $( compgen -W "--all -a --before --filter -f --format --help --latest -l -n --no-trunc --quiet -q --size -s --since" -- "$cur" ) )
 			;;
 	esac
 }
@@ -998,15 +1054,16 @@ _docker_rmi() {
 _docker_run() {
 	local options_with_args="
 		--add-host
-		--blkio-weight
 		--attach -a
+		--blkio-weight
 		--cap-add
 		--cap-drop
 		--cgroup-parent
 		--cidfile
-		--cpuset
 		--cpu-period
 		--cpu-quota
+		--cpuset-cpus
+		--cpuset-mems
 		--cpu-shares -c
 		--device
 		--dns
@@ -1018,8 +1075,8 @@ _docker_run() {
 		--group-add
 		--hostname -h
 		--ipc
-		--label -l
 		--label-file
+		--label -l
 		--link
 		--log-driver
 		--log-opt
@@ -1027,14 +1084,15 @@ _docker_run() {
 		--mac-address
 		--memory -m
 		--memory-swap
+		--memory-swappiness
 		--name
 		--net
 		--pid
 		--publish -p
 		--restart
 		--security-opt
-		--user -u
 		--ulimit
+		--user -u
 		--uts
 		--volumes-from
 		--volume -v
@@ -1042,8 +1100,10 @@ _docker_run() {
 	"
 
 	local all_options="$options_with_args
+		--disable-content-trust=false
 		--help
 		--interactive -i
+		--oom-kill-disable
 		--privileged
 		--publish-all -P
 		--read-only
@@ -1053,7 +1113,7 @@ _docker_run() {
 	[ "$command" = "run" ] && all_options="$all_options
 		--detach -d
 		--rm
-		--sig-proxy
+		--sig-proxy=false
 	"
 
 	local options_with_args_glob=$(__docker_to_extglob "$options_with_args")

contrib/completion/zsh/_docker

@@ -597,6 +597,7 @@ _docker() {
         "($help -d --daeamon)"{-d,--daemon}"[Enable daemon mode]" \
         "($help)--default-gateway[Container default gateway IPv4 address]:IPv4 address: " \
         "($help)--default-gateway-v6[Container default gateway IPv6 address]:IPv6 address: " \
+        "($help)--disable-legacy-registry[Do not contact legacy registries]" \
         "($help)*--dns=-[DNS server to use]:DNS: " \
         "($help)*--dns-search=-[DNS search domains to use]" \
         "($help)*--default-ulimit=-[Set default ulimit settings for containers]:ulimit: " \

daemon/config_linux.go

@@ -11,7 +11,7 @@ import (
 var (
 	defaultPidFile = "/var/run/docker.pid"
 	defaultGraph   = "/var/lib/docker"
-	defaultExec    = "native"
+	defaultExec    = "clr"
 )
 
 // Config defines the configuration of a docker daemon.

daemon/daemon.go

@@ -35,7 +35,6 @@ import (
 	"github.com/docker/docker/pkg/truncindex"
 	"github.com/docker/docker/registry"
 	"github.com/docker/docker/runconfig"
-	"github.com/docker/docker/trust"
 	"github.com/docker/libnetwork"
 	"github.com/opencontainers/runc/libcontainer/netlink"
 )
@@ -670,10 +669,6 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 	if err := system.MkdirAll(trustDir, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	trustService, err := trust.NewTrustStore(trustDir)
-	if err != nil {
-		return nil, fmt.Errorf("could not create trust store: %s", err)
-	}
 
 	eventsService := events.New()
 	logrus.Debug("Creating repository list")
@@ -682,7 +677,6 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 		Key:      trustKey,
 		Registry: registryService,
 		Events:   eventsService,
-		Trust:    trustService,
 	}
 	repositories, err := graph.NewTagStore(filepath.Join(config.Root, "repositories-"+d.driver.String()), tagCfg)
 	if err != nil {
@@ -979,7 +973,7 @@ func getDefaultRouteMtu() (int, error) {
 		return 0, err
 	}
 	for _, r := range routes {
-		if r.Default {
+		if r.Default && r.Iface != nil {
 			return r.Iface.MTU, nil
 		}
 	}

daemon/execdriver/clr/driver.go

@@ -0,0 +1,721 @@
+// +build linux
+
+package clr
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"net/http"
+	"os"
+	"os/exec"
+	"net"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/daemon/execdriver"
+	"github.com/docker/docker/pkg/mount"
+	sysinfo "github.com/docker/docker/pkg/system"
+	"github.com/docker/docker/pkg/term"
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/kr/pty"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+const (
+	// Clear Linux for Intel(R) Architecture
+	driverName = "clr"
+
+	envVarPrefix = "CLR_"
+
+	// Command used for lkvm control
+	lkvmName = "lkvm"
+
+	// local "latest" information
+	clrFile = "latest"
+
+	// upstream base URL
+	clrURL = "https://download.clearlinux.org"
+
+	// upstream latest release file
+	latestFile = "https://download.clearlinux.org/latest"
+
+	// clr kernel (not bzimage)
+	clrKernel = "/usr/lib/kernel/vmlinux.container"
+)
+
+type driver struct {
+	root             string // root path for the driver to use
+	libPath          string
+	initPath         string
+	version          string
+	apparmor         bool
+	sharedRoot       bool
+	activeContainers map[string]*activeContainer
+	machineMemory    int64
+	containerPid     int
+	sync.Mutex
+}
+
+type activeContainer struct {
+	container *configs.Config
+	cmd       *exec.Cmd
+}
+
+func getTapIf(c *execdriver.Command) string {
+	return fmt.Sprintf("tb-%s", c.ID[:12])
+}
+
+func getClrVersion(libPath string) string {
+	txt, err := ioutil.ReadFile(path.Join(libPath, clrFile))
+	if err != nil {
+		return ""
+	}
+	return strings.Split(string(txt), "\n")[0]
+}
+
+func fetchLatest(libPath string) error {
+	out, err := os.Create(path.Join(libPath, clrFile))
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(latestFile)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	_, err = io.Copy(out, resp.Body)
+
+	return err
+}
+
+func fetchImage(version, libPath string) error {
+	// TODO: Add checksum validation
+	outfile := fmt.Sprintf("clear-%s-containers.img.xz", version)
+	url := fmt.Sprintf("%s/releases/%s/clear/%s", clrURL, version, outfile)
+	outpath := path.Join(libPath, outfile)
+	var output []byte
+
+	logrus.Debugf("Fetching clr version: %s, %s", version, outpath)
+	out, err := os.Create(outpath)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	// Consider progress feedback ?
+	_, err = io.Copy(out, resp.Body)
+
+	if err != nil {
+		return err
+	}
+
+	// decompress the file
+	cmd := exec.Command("unxz", outpath)
+	cmd.Dir = libPath
+
+	if output, err = cmd.CombinedOutput(); err != nil {
+		logrus.Debugf("Unable to extract image %s: %s", version, output)
+		return err
+	}
+
+	return nil
+}
+
+// NewDriver creates a new clear linux execution driver.
+func NewDriver(root, libPath, initPath string, apparmor bool) (*driver, error) {
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
+	meminfo, err := sysinfo.ReadMemInfo()
+	if err != nil {
+		return nil, err
+	}
+	version, err := prepareClr(libPath)
+	if err != nil {
+		return nil, err
+	}
+	return &driver{
+		apparmor:         apparmor,
+		root:             root,
+		libPath:          libPath,
+		initPath:         initPath,
+		version:          version,
+		sharedRoot:       false,
+		activeContainers: make(map[string]*activeContainer),
+		// FIXME:
+		machineMemory: meminfo.MemTotal,
+	}, nil
+}
+
+func prepareClr(libPath string) (string, error) {
+	var version = getClrVersion(libPath)
+	var nversion string
+	logrus.Debugf("%s preparing environment", driverName)
+
+	err := fetchLatest(libPath)
+	if err != nil {
+		return "", err
+	}
+
+	nversion = getClrVersion(libPath)
+	if nversion != version && version != "" {
+		logrus.Debugf("Updating to clr version: %s", nversion)
+		err = fetchImage(nversion, libPath)
+	} else if version == "" {
+		logrus.Debugf("Installing clr version: %s", nversion)
+		err = fetchImage(nversion, libPath)
+	} else {
+		logrus.Debugf("Using clr version: %s", nversion)
+	}
+
+	return nversion, nil
+}
+
+func (d *driver) Name() string {
+	return fmt.Sprintf("%s-%s", driverName, d.version)
+}
+
+func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (execdriver.ExitStatus, error) {
+	var (
+		term execdriver.Terminal
+		err  error
+	)
+
+	container, err := d.createContainer(c)
+	if err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+
+	memoryMiB := c.HostConfig.Memory
+	if memoryMiB == 0 {
+		memoryMiB = 1024
+	} else {
+		// docker passes the value as bytes
+		memoryMiB = memoryMiB / int64(math.Pow(2, 20))
+	}
+
+	workingDirVar := fmt.Sprintf("%s%s=%q", envVarPrefix, "WORKINGDIR", c.WorkingDir)
+	c.ProcessConfig.Cmd.Env = append(c.ProcessConfig.Cmd.Env, workingDirVar)
+
+	userVar := fmt.Sprintf("%s%s=%q", envVarPrefix, "USER", c.ProcessConfig.User)
+	c.ProcessConfig.Cmd.Env = append(c.ProcessConfig.Cmd.Env, userVar)
+
+	if err := d.setupNetwork(c); err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+
+	if c.ProcessConfig.Tty {
+		term, err = NewTtyConsole(&c.ProcessConfig, pipes)
+	} else {
+		term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes)
+	}
+	if err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+	c.ProcessConfig.Terminal = term
+
+	d.Lock()
+	d.activeContainers[c.ID] = &activeContainer{
+		container: container,
+		cmd:       &c.ProcessConfig.Cmd,
+	}
+	d.Unlock()
+
+	if err := d.generateEnvConfig(c); err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+
+	if err := d.generateDockerInit(c); err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+
+	for _, m := range c.Mounts {
+		dest := path.Join(c.Rootfs, m.Destination)
+
+		if m.Destination == "/etc/hostname" {
+			continue
+		}
+
+		if !pathExists(m.Source) {
+			continue
+		}
+
+		opts := "bind"
+
+		if m.Private {
+			opts = opts + ",rprivate"
+		}
+		if m.Slave {
+			opts = opts + ",rslave"
+		}
+
+		// This may look racy, but it isn't since the VM isn't
+		// running yet.
+		//
+		// The check is necessary to handle bind mounting of
+		// regular files correctly since without it we may be
+		// attempting to create a directory where there already
+		// exists a normal file.
+		if !pathExists(dest) {
+			if err := os.MkdirAll(dest, 0750); err != nil {
+				return execdriver.ExitStatus{ExitCode: -1}, err
+			}
+		}
+
+		if err := mount.Mount(m.Source, dest, "", opts); err != nil {
+			return execdriver.ExitStatus{ExitCode: -1}, err
+		}
+
+		if !m.Writable {
+			if err := mount.Mount("", dest, "", "bind,remount,ro"); err != nil {
+				return execdriver.ExitStatus{ExitCode: -1}, err
+			}
+		}
+		defer mount.Unmount(dest)
+	}
+
+	var args []string
+	// various things for lkvm
+	ifname := getTapIf(c)
+	// FIXME: Should be real hostname from like process/container struct
+	hostname := c.ID[0:12]
+	img := fmt.Sprintf("%s/clear-%s-containers.img", d.libPath, d.version)
+	memory := fmt.Sprintf("%d", memoryMiB)
+	// FIXME: Locked cores to 6 ?
+	cores := fmt.Sprintf("%d", 6)
+	ipaddr := c.NetworkSettings.IPAddress
+	gateway := c.NetworkSettings.Gateway
+	macaddr := c.NetworkSettings.MacAddress
+
+	args = append(args, c.ProcessConfig.Entrypoint)
+	args = append(args, c.ProcessConfig.Arguments...)
+
+	rootParams := fmt.Sprintf("root=/dev/plkvm0p1 rootfstype=ext4 rootflags=dax,data=ordered "+
+		"init=/usr/lib/systemd/systemd systemd.unit=container.target rw tsc=reliable "+
+		"systemd.show_status=false "+
+		"no_timer_check rcupdate.rcu_expedited=1 console=hvc0 quiet ip=%s::%s::%s::off",
+		ipaddr, gateway, hostname)
+
+	params := []string{
+		lkvmName, "run", "-c", cores, "-m", memory,
+		"--name", c.ID, "--console", "virtio",
+		"--kernel", clrKernel,
+		"--params", rootParams,
+		"--shmem", fmt.Sprintf("0x200000000:0:file=%s:private", img),
+		"--network", fmt.Sprintf("mode=tap,script=none,tapif=%s,guest_mac=%s", ifname, macaddr),
+		"--9p", fmt.Sprintf("%s,rootfs", c.Rootfs),
+	}
+
+	logrus.Debugf("%s params %s", driverName, params)
+	var (
+		name = params[0]
+		arg  = params[1:]
+	)
+	aname, err := exec.LookPath(name)
+	if err != nil {
+		aname = name
+	}
+	c.ProcessConfig.Path = aname
+	c.ProcessConfig.Args = append([]string{name}, arg...)
+	c.ProcessConfig.Env = []string{fmt.Sprintf("HOME=%s", d.root)}
+
+	// Start the container. Since it runs synchronously, we don't Wait()
+	// for it since we need to check the status to determine if it did
+	// actually start successfully.
+	if err := c.ProcessConfig.Start(); err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+
+	var (
+		waitErr  error
+		waitLock = make(chan struct{})
+	)
+	go func() {
+		if err := c.ProcessConfig.Wait(); err != nil {
+			if _, ok := err.(*exec.ExitError); !ok { // Do not propagate the error if it's simply a status code != 0
+				waitErr = err
+			}
+		}
+		close(waitLock)
+	}()
+
+	// FIXME: need to create state.json for Stats() to work.
+	c.ContainerPid = c.ProcessConfig.Process.Pid
+	d.containerPid = c.ProcessConfig.Process.Pid
+
+	if startCallback != nil {
+		logrus.Debugf("Invoking startCallback")
+		startCallback(&c.ProcessConfig, c.ProcessConfig.Process.Pid)
+	}
+
+	// FIXME:
+	oomKill := false
+
+	// Wait for the VM to shutdown
+	<-waitLock
+	exitCode := getExitCode(c)
+
+	cExitStatus, cerr := d.cleanupVM(c)
+
+	if cerr != nil {
+		waitErr = cerr
+		exitCode = cExitStatus
+	}
+
+	// check oom error
+	if oomKill {
+		exitCode = 137
+	}
+
+	return execdriver.ExitStatus{ExitCode: exitCode, OOMKilled: false}, waitErr
+}
+
+func pathExists(path string) bool {
+	if _, err := os.Stat(path); err == nil {
+		return true
+	}
+
+	return false
+}
+
+func pathExecutable(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+
+	mode := s.Mode()
+	if mode&0111 != 0 {
+		return true
+	}
+	return false
+}
+
+func (d *driver) cleanupVM(c *execdriver.Command) (exitStatus int, err error) {
+	cmd := exec.Command("ip", "tuntap", "del", "dev", getTapIf(c), "mode", "tap")
+	var output []byte
+
+	if output, err = cmd.CombinedOutput(); err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			waitStatus := exitError.Sys().(syscall.WaitStatus)
+			exitStatus = waitStatus.ExitStatus()
+		}
+		logrus.Debugf("teardown failed for vm %s: %s (%s)", c.ID, string(output), err.Error())
+	}
+
+	// doesn't matter if this fails
+	// lkvm could have removed it, and stale sockets are not fatal
+	_ = os.Remove(fmt.Sprintf("%s/.lkvm/%s.sock", d.root, c.ID))
+
+	return exitStatus, err
+}
+
+// createContainer populates and configures the container type with the
+// data provided by the execdriver.Command
+func (d *driver) createContainer(c *execdriver.Command) (*configs.Config, error) {
+	return execdriver.InitContainer(c), nil
+}
+
+/// Return the exit code of the process
+// if the process has not exited -1 will be returned
+func getExitCode(c *execdriver.Command) int {
+	if c.ProcessConfig.ProcessState == nil {
+		return -1
+	}
+	return c.ProcessConfig.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
+}
+
+func (d *driver) lkvmCommand(c *execdriver.Command, arg string) ([]byte, error) {
+	args := append([]string{lkvmName}, arg)
+	if c != nil {
+		args = append(args, "--name", c.ID)
+	}
+	cmd := exec.Command(lkvmName, args...)
+	cmd.Env = []string{fmt.Sprintf("HOME=%s", d.root)}
+	return cmd.Output()
+}
+
+// Kill sends a signal to workload
+func (d *driver) Kill(c *execdriver.Command, sig int) error {
+	// Not supported
+	return nil
+}
+
+func (d *driver) Pause(c *execdriver.Command) error {
+	_, err := d.lkvmCommand(c, "pause")
+	return err
+}
+
+func (d *driver) Unpause(c *execdriver.Command) error {
+	_, err := d.lkvmCommand(c, "resume")
+	return err
+}
+
+// Terminate forcibly stops a container
+func (d *driver) Terminate(c *execdriver.Command) error {
+	_, err := d.lkvmCommand(c, "stop")
+	return err
+}
+
+func (d *driver) containerDir(containerID string) string {
+	return path.Join(d.libPath, "containers", containerID)
+}
+
+// isDigit returns true if s can be represented as an integer
+func isDigit(s string) bool {
+	if _, err := strconv.Atoi(s); err == nil {
+		return true
+	}
+
+	return false
+}
+
+func (d *driver) getInfo(id string) ([]byte, error) {
+	output, err := d.lkvmCommand(nil, "list")
+	if err != nil {
+		return nil, err
+	}
+
+	for i, line := range strings.Split(string(output), "\n") {
+		if i < 2 {
+			continue
+		}
+
+		fields := strings.Fields(strings.TrimSpace(line))
+
+		if len(fields) != 3 {
+			continue
+		}
+
+		if !isDigit(fields[0]) {
+			continue
+		}
+
+		if fields[1] != id {
+			continue
+		}
+
+		return []byte(line), nil
+	}
+
+	return []byte(fmt.Sprintf("-1 %s stopped", id)), nil
+}
+
+type info struct {
+	ID     string
+	driver *driver
+}
+
+func (i *info) IsRunning() bool {
+	output, err := i.driver.getInfo(i.ID)
+	if err != nil {
+		logrus.Errorf("Error getting info for %s container %s: %s (%s)",
+			driverName, i.ID, err, output)
+		return false
+	}
+
+	clrInfo, err := parseClrInfo(i.ID, string(output))
+
+	if err != nil {
+		return false
+	}
+
+	return clrInfo.Running
+}
+
+func (d *driver) Info(id string) execdriver.Info {
+	return &info{
+		ID:     id,
+		driver: d,
+	}
+}
+
+func (d *driver) GetPidsForContainer(id string) ([]int, error) {
+	// The VM doesn't expose the worload pid(s), so the only meaningful
+	// pid is that of the VM
+	return []int{d.containerPid}, nil
+}
+
+// TtyConsole is a type to represent a pseud-oterminal (see pty(7))
+type TtyConsole struct {
+	MasterPty *os.File
+	SlavePty  *os.File
+}
+
+// NewTtyConsole returns a new TtyConsole object.
+func NewTtyConsole(processConfig *execdriver.ProcessConfig, pipes *execdriver.Pipes) (*TtyConsole, error) {
+	// lxc is special in that we cannot create the master outside of the container without
+	// opening the slave because we have nothing to provide to the cmd.  We have to open both then do
+	// the crazy setup on command right now instead of passing the console path to lxc and telling it
+	// to open up that console.  we save a couple of openfiles in the native driver because we can do
+	// this.
+	ptyMaster, ptySlave, err := pty.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	tty := &TtyConsole{
+		MasterPty: ptyMaster,
+		SlavePty:  ptySlave,
+	}
+
+	if err := tty.AttachPipes(&processConfig.Cmd, pipes); err != nil {
+		tty.Close()
+		return nil, err
+	}
+
+	processConfig.Console = tty.SlavePty.Name()
+
+	return tty, nil
+}
+
+// Master returns the master end of the pty
+func (t *TtyConsole) Master() *os.File {
+	return t.MasterPty
+}
+
+// Resize modifies the size of the pty terminal being used.
+func (t *TtyConsole) Resize(h, w int) error {
+	return term.SetWinsize(t.MasterPty.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)})
+}
+
+// AttachPipes associates the specified pipes with the pty master.
+func (t *TtyConsole) AttachPipes(command *exec.Cmd, pipes *execdriver.Pipes) error {
+	command.Stdout = t.SlavePty
+	command.Stderr = t.SlavePty
+
+	go func() {
+		if wb, ok := pipes.Stdout.(interface {
+			CloseWriters() error
+		}); ok {
+			defer wb.CloseWriters()
+		}
+
+		io.Copy(pipes.Stdout, t.MasterPty)
+	}()
+
+	if pipes.Stdin != nil {
+		command.Stdin = t.SlavePty
+		command.SysProcAttr.Setctty = true
+
+		go func() {
+			io.Copy(t.MasterPty, pipes.Stdin)
+
+			pipes.Stdin.Close()
+		}()
+	}
+	return nil
+}
+
+// Close closes both ends of the pty.
+func (t *TtyConsole) Close() error {
+	t.SlavePty.Close()
+	return t.MasterPty.Close()
+}
+
+func (d *driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessConfig, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) {
+	return -1, fmt.Errorf("Unsupported: Exec is not supported by the %q driver", driverName)
+
+}
+
+// Clean up after an Exec
+func (d *driver) Clean(id string) error {
+	return nil
+}
+
+func (d *driver) generateEnvConfig(c *execdriver.Command) error {
+	data := []byte(strings.Join(c.ProcessConfig.Env, "\n"))
+
+	p := path.Join(d.libPath, "containers", c.ID, "config.env")
+	c.Mounts = append(c.Mounts, execdriver.Mount{
+		Source:      p,
+		Destination: "/.dockerenv",
+		Writable:    false,
+		Private:     true,
+	})
+
+	return ioutil.WriteFile(p, data, 0600)
+}
+
+func (d *driver) generateDockerInit(c *execdriver.Command) error {
+	p := fmt.Sprintf("%s/.containerexec", c.Rootfs)
+	var args []string
+
+	if pathExecutable(p) {
+		return nil
+	}
+
+	args = append(args, c.ProcessConfig.Entrypoint)
+	args = append(args, c.ProcessConfig.Arguments...)
+
+	data := []byte(fmt.Sprintf("#!/bin/sh\n%s\n", strings.Join(args, " ")))
+
+	return ioutil.WriteFile(p, data, 0755)
+}
+
+func (d *driver) setupNetwork(c *execdriver.Command) error {
+	ifname := getTapIf(c)
+
+	var bridgeName string
+	var bridgeLinkName string
+	var output []byte
+	var err error
+
+	for _, info := range c.EndpointInfo {
+		if mac, ok := info[netlabel.MacAddress].(net.HardwareAddr); ok {
+			if mac.String() == c.NetworkSettings.MacAddress {
+				bridgeName = info[netlabel.BridgeName].(string)
+				bridgeLinkName = info[netlabel.BridgeLinkName].(string)
+			}
+		}
+	}
+
+	// Strip existing veth
+	cmd := exec.Command("ip", "link", "del", bridgeLinkName)
+	if output, err = cmd.CombinedOutput(); err != nil {
+		logrus.Debugf("%s setupNetwork error: %v, %s", driverName, cmd.Args, output)
+		return err
+	}
+
+	cmd = exec.Command("ip", "tuntap", "add", "dev", ifname, "mode", "tap", "vnet_hdr")
+	if output, err = cmd.CombinedOutput(); err != nil {
+		logrus.Debugf("%s setupNetwork error: %v, %s", driverName, cmd.Args, output)
+		return err
+	}
+	cmd = exec.Command("ip", "link", "set", "dev", ifname, "master", bridgeName)
+	if output, err = cmd.CombinedOutput(); err != nil {
+		logrus.Debugf("%s setupNetwork error: %v, %s", driverName, cmd.Args, output)
+		return err
+	}
+
+	cmd = exec.Command("ip", "link", "set", "dev", ifname, "up")
+	if output, err = cmd.CombinedOutput(); err != nil {
+		logrus.Debugf("%s setupNetwork error: %v, %s", driverName, cmd.Args, output)
+		return err
+	}
+
+	return err
+}
+
+func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
+	if _, ok := d.activeContainers[id]; !ok {
+		return nil, fmt.Errorf("%s is not a key in active containers", id)
+	}
+	// FIXME:
+	return execdriver.Stats(d.containerDir(id), d.activeContainers[id].container.Cgroups.Memory, d.machineMemory)
+}

daemon/execdriver/clr/info.go

@@ -0,0 +1,51 @@
+// +build linux
+
+package clr
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrCannotParse = errors.New("cannot parse raw input")
+)
+
+type clrInfo struct {
+	Running bool
+	Pid     int
+}
+
+func parseClrInfo(name, raw string) (*clrInfo, error) {
+	if raw == "" {
+		return nil, ErrCannotParse
+	}
+	var (
+		err  error
+		info = &clrInfo{}
+	)
+
+	fields := strings.Fields(strings.TrimSpace(raw))
+
+	// The format is expected to be:
+	//
+	// <pid> <name> <state>
+	//
+	if len(fields) != 3 {
+		return nil, ErrCannotParse
+	}
+
+	info.Pid, err = strconv.Atoi(fields[0])
+	if err != nil {
+		return nil, ErrCannotParse
+	}
+
+	if fields[1] != name {
+		return nil, ErrCannotParse
+	}
+
+	info.Running = fields[2] == "running"
+
+	return info, nil
+}

daemon/execdriver/clr/init.go

@@ -0,0 +1,97 @@
+// +build linux
+
+package clr
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"syscall"
+)
+
+// Args provided to the init function for a driver
+type InitArgs struct {
+	User       string
+	Gateway    string
+	Ip         string
+	WorkDir    string
+	Privileged bool
+	Env        []string
+	Args       []string
+	Mtu        int
+	Console    string
+	Pipe       int
+	Root       string
+	CapAdd     string
+	CapDrop    string
+}
+
+func getArgs() *InitArgs {
+	var (
+		// Get cmdline arguments
+		user       = flag.String("u", "", "username or uid")
+		gateway    = flag.String("g", "", "gateway address")
+		ip         = flag.String("i", "", "ip address")
+		workDir    = flag.String("w", "", "workdir")
+		privileged = flag.Bool("privileged", false, "privileged mode")
+		mtu        = flag.Int("mtu", 1500, "interface mtu")
+		capAdd     = flag.String("cap-add", "", "capabilities to add")
+		capDrop    = flag.String("cap-drop", "", "capabilities to drop")
+	)
+
+	flag.Parse()
+
+	return &InitArgs{
+		User:       *user,
+		Gateway:    *gateway,
+		Ip:         *ip,
+		WorkDir:    *workDir,
+		Privileged: *privileged,
+		Args:       flag.Args(),
+		Mtu:        *mtu,
+		CapAdd:     *capAdd,
+		CapDrop:    *capDrop,
+	}
+}
+
+// Clear environment pollution introduced by lxc-start
+func setupEnv(args *InitArgs) error {
+	// Get env
+	var env []string
+	dockerenv, err := os.Open(".dockerenv")
+	if err != nil {
+		return fmt.Errorf("Unable to load environment variables: %v", err)
+	}
+	defer dockerenv.Close()
+	if err := json.NewDecoder(dockerenv).Decode(&env); err != nil {
+		return fmt.Errorf("Unable to decode environment variables: %v", err)
+	}
+	// Propagate the plugin-specific container env variable
+	env = append(env, "container="+os.Getenv("container"))
+
+	args.Env = env
+
+	os.Clearenv()
+	for _, kv := range args.Env {
+		parts := strings.SplitN(kv, "=", 2)
+		if len(parts) == 1 {
+			parts = append(parts, "")
+		}
+		os.Setenv(parts[0], parts[1])
+	}
+
+	return nil
+}
+
+// Setup working directory
+func setupWorkingDirectory(args *InitArgs) error {
+	if args.WorkDir == "" {
+		return nil
+	}
+	if err := syscall.Chdir(args.WorkDir); err != nil {
+		return fmt.Errorf("Unable to change dir to %v: %v", args.WorkDir, err)
+	}
+	return nil
+}

daemon/execdriver/driver.go

@@ -7,7 +7,9 @@ import (
 	"time"
 
 	// TODO Windows: Factor out ulimit
+	"github.com/docker/docker/daemon/network"
 	"github.com/docker/docker/pkg/ulimit"
+	"github.com/docker/docker/runconfig"
 	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/runc/libcontainer/configs"
 )
@@ -150,31 +152,34 @@ type ProcessConfig struct {
 //
 // Process wrapps an os/exec.Cmd to add more metadata
 type Command struct {
-	ID                 string            `json:"id"`
+	ID                 string                   `json:"id"`
-	Rootfs             string            `json:"rootfs"` // root fs of the container
+	Rootfs             string                   `json:"rootfs"` // root fs of the container
-	ReadonlyRootfs     bool              `json:"readonly_rootfs"`
+	ReadonlyRootfs     bool                     `json:"readonly_rootfs"`
-	InitPath           string            `json:"initpath"` // dockerinit
+	InitPath           string                   `json:"initpath"` // dockerinit
-	WorkingDir         string            `json:"working_dir"`
+	WorkingDir         string                   `json:"working_dir"`
-	ConfigPath         string            `json:"config_path"` // this should be able to be removed when the lxc template is moved into the driver
+	ConfigPath         string                   `json:"config_path"` // this should be able to be removed when the lxc template is moved into the driver
-	Network            *Network          `json:"network"`
+	Network            *Network                 `json:"network"`
-	Ipc                *Ipc              `json:"ipc"`
+	Ipc                *Ipc                     `json:"ipc"`
-	Pid                *Pid              `json:"pid"`
+	Pid                *Pid                     `json:"pid"`
-	UTS                *UTS              `json:"uts"`
+	UTS                *UTS                     `json:"uts"`
-	Resources          *Resources        `json:"resources"`
+	Resources          *Resources               `json:"resources"`
-	Mounts             []Mount           `json:"mounts"`
+	Mounts             []Mount                  `json:"mounts"`
-	AllowedDevices     []*configs.Device `json:"allowed_devices"`
+	AllowedDevices     []*configs.Device        `json:"allowed_devices"`
-	AutoCreatedDevices []*configs.Device `json:"autocreated_devices"`
+	AutoCreatedDevices []*configs.Device        `json:"autocreated_devices"`
-	CapAdd             []string          `json:"cap_add"`
+	CapAdd             []string                 `json:"cap_add"`
-	CapDrop            []string          `json:"cap_drop"`
+	CapDrop            []string                 `json:"cap_drop"`
-	GroupAdd           []string          `json:"group_add"`
+	GroupAdd           []string                 `json:"group_add"`
-	ContainerPid       int               `json:"container_pid"`  // the pid for the process inside a container
+	ContainerPid       int                      `json:"container_pid"`  // the pid for the process inside a container
-	ProcessConfig      ProcessConfig     `json:"process_config"` // Describes the init process of the container.
+	ProcessConfig      ProcessConfig            `json:"process_config"` // Describes the init process of the container.
-	ProcessLabel       string            `json:"process_label"`
+	ProcessLabel       string                   `json:"process_label"`
-	MountLabel         string            `json:"mount_label"`
+	MountLabel         string                   `json:"mount_label"`
-	LxcConfig          []string          `json:"lxc_config"`
+	LxcConfig          []string                 `json:"lxc_config"`
-	AppArmorProfile    string            `json:"apparmor_profile"`
+	AppArmorProfile    string                   `json:"apparmor_profile"`
-	CgroupParent       string            `json:"cgroup_parent"` // The parent cgroup for this command.
+	CgroupParent       string                   `json:"cgroup_parent"` // The parent cgroup for this command.
-	FirstStart         bool              `json:"first_start"`
+	FirstStart         bool                     `json:"first_start"`
-	LayerPaths         []string          `json:"layer_paths"` // Windows needs to know the layer paths and folder for a command
+	LayerPaths         []string                 `json:"layer_paths"` // Windows needs to know the layer paths and folder for a command
-	LayerFolder        string            `json:"layer_folder"`
+	LayerFolder        string                   `json:"layer_folder"`
+	NetworkSettings    *network.Settings        `json:"network_settings"`
+	EndpointInfo       []map[string]interface{} `json:"endpoint_info"`
+	HostConfig         *runconfig.HostConfig
 }

daemon/execdriver/execdrivers/execdrivers_linux.go

@@ -8,13 +8,17 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
+	"github.com/docker/docker/daemon/execdriver/clr"
 	"github.com/docker/docker/daemon/execdriver/lxc"
 	"github.com/docker/docker/daemon/execdriver/native"
 	"github.com/docker/docker/pkg/sysinfo"
 )
 
 func NewDriver(name string, options []string, root, libPath, initPath string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
+	rootPath := path.Join(root, "execdriver", name)
 	switch name {
+	case "clr":
+		return clr.NewDriver(rootPath, libPath, initPath, sysInfo.AppArmor)
 	case "lxc":
 		// we want to give the lxc driver the full docker root because it needs
 		// to access and write config and template files in /var/lib/docker/containers/*
@@ -22,7 +26,7 @@ func NewDriver(name string, options []string, root, libPath, initPath string, sy
 		logrus.Warn("LXC built-in support is deprecated.")
 		return lxc.NewDriver(root, libPath, initPath, sysInfo.AppArmor)
 	case "native":
-		return native.NewDriver(path.Join(root, "execdriver", "native"), initPath, options)
+		return native.NewDriver(rootPath, initPath, options)
 	}
 	return nil, fmt.Errorf("unknown exec driver %s", name)
 }

daemon/graphdriver/devmapper/deviceset.go

@@ -1482,12 +1482,16 @@ func (devices *DeviceSet) deactivatePool() error {
 	if err != nil {
 		return err
 	}
-	if d, err := devicemapper.GetDeps(devname); err == nil {
+
-		// Access to more Debug output
+	if devinfo.Exists == 0 {
-		logrus.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d)
+		return nil
 	}
-	if devinfo.Exists != 0 {
+	if err := devicemapper.RemoveDevice(devname); err != nil {
-		return devicemapper.RemoveDevice(devname)
+		return err
+	}
+
+	if d, err := devicemapper.GetDeps(devname); err == nil {
+		logrus.Warnf("[devmapper] device %s still has %d active dependents", devname, d.Count)
 	}
 
 	return nil

daemon/monitor.go

@@ -138,6 +138,30 @@ func (m *containerMonitor) Start() error {
 
 		m.lastStartTime = time.Now()
 
+		// Make the network settings available to the execution
+		// driver to allow for integration with libnetwork networking.
+		m.container.command.NetworkSettings = m.container.NetworkSettings
+
+		// Allow the execution driver to query memory limits
+		m.container.command.HostConfig = m.container.hostConfig
+
+		// Make the network endpoint details available to the execution
+		// driver as well.
+		n, _err := m.container.daemon.netController.NetworkByID(m.container.NetworkSettings.NetworkID)
+		if _err == nil {
+			var eps []map[string]interface{}
+
+			for _, ep := range n.Endpoints() {
+				info, err := ep.DriverInfo()
+				if err != nil {
+					continue
+				}
+				eps = append(eps, info)
+			}
+
+			m.container.command.EndpointInfo = eps
+		}
+
 		if exitStatus, err = m.container.daemon.Run(m.container, pipes, m.callback); err != nil {
 			// if we receive an internal error from the initial start of a container then lets
 			// return it instead of entering the restart loop

docker/daemon.go

@@ -76,6 +76,7 @@ func NewDaemonCli() *DaemonCli {
 
 	// TODO(tiborvass): remove InstallFlags?
 	daemonConfig := new(daemon.Config)
+	daemonConfig.LogConfig.Config = make(map[string]string)
 	daemonConfig.InstallFlags(daemonFlags, presentInHelp)
 	daemonConfig.InstallFlags(flag.CommandLine, absentFromHelp)
 	registryOptions := new(registry.Options)
@@ -208,10 +209,6 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error {
 		}()
 	}
 
-	if cli.LogConfig.Config == nil {
-		cli.LogConfig.Config = make(map[string]string)
-	}
-
 	serverConfig := &apiserver.ServerConfig{
 		Logging:     true,
 		EnableCors:  cli.EnableCors,

docker/docker.go

@@ -28,7 +28,7 @@ func main() {
 	flag.Merge(flag.CommandLine, clientFlags.FlagSet, commonFlags.FlagSet)
 
 	flag.Usage = func() {
-		fmt.Fprint(os.Stdout, "Usage: docker [OPTIONS] COMMAND [arg...]\n"+daemonUsage+"       docker [ -h | --help | -v | --version ]\n\n")
+		fmt.Fprint(os.Stdout, "Usage: docker [OPTIONS] COMMAND [arg...]\n"+daemonUsage+"       docker [ --help | -v | --version ]\n\n")
 		fmt.Fprint(os.Stdout, "A self-sufficient runtime for containers.\n\nOptions:\n")
 
 		flag.CommandLine.SetOutput(os.Stdout)

docs/misc/deprecated.md

@@ -18,7 +18,7 @@ The following list of features are deprecated.
 **Target For Removal In Release: v1.10**
 
 The built-in LXC execution driver is deprecated for an external implementation.
-The lxc-conf flag and API fields will also be removed. 
+The lxc-conf flag and API fields will also be removed.
 
 ### Old Command Line Options
 **Deprecated In Release: [v1.8.0](/release-notes/#docker-engine-1-8-0)**
@@ -29,7 +29,7 @@ The flags `-d` and `--daemon` are deprecated in favor of the `daemon` subcommand
 
     docker daemon -H ...
 
-The following single-dash (`-opt`) variant of certain command line options 
+The following single-dash (`-opt`) variant of certain command line options
 are deprecated and replaced with double-dash options (`--opt`):
 
     docker attach -nostdin
@@ -68,3 +68,7 @@ The following double-dash options are deprecated and have no replacement:
     docker ps --since-id
     docker ps --before-id
     docker search --trusted
+
+### Interacting with V1 registries
+
+Version 1.8.3 adds a flag (`--disable-legacy-registry=false`) which prevents the docker daemon from `pull`, `push`, and `login` operations against v1 registries.  Though disabled by default, this signals the intent to deprecate the v1 protocol.

docs/reference/commandline/cli.md

@@ -16,7 +16,7 @@ or execute `docker help`:
     $ docker
       Usage: docker [OPTIONS] COMMAND [arg...]
              docker daemon [ --help | ... ]
-             docker [ -h | --help | -v | --version ]
+             docker [ --help | -v | --version ]
 
         -H, --host=[]: The socket(s) to bind to in daemon mode, specified using one or more tcp://host:port, unix:///path/to/socket, fd://* or fd://socketfd.
 

docs/reference/commandline/daemon.md

@@ -32,7 +32,7 @@ parent = "smn_cli"
       -G, --group="docker"                   Group for the unix socket
       -g, --graph="/var/lib/docker"          Root of the Docker runtime
       -H, --host=[]                          Daemon socket(s) to connect to
-      -h, --help=false                       Print usage
+      --help=false                           Print usage
       --icc=true                             Enable inter-container communication
       --insecure-registry=[]                 Enable insecure registry communication
       --ip=0.0.0.0                           Default IP when binding container ports
@@ -45,6 +45,7 @@ parent = "smn_cli"
       --log-driver="json-file"               Default driver for container logs
       --log-opt=[]                           Log driver specific options
       --mtu=0                                Set the containers network MTU
+      --disable-legacy-registry=false        Do not contact legacy registries
       -p, --pidfile="/var/run/docker.pid"    Path to use for daemon PID file
       --registry-mirror=[]                   Preferred Docker registry mirror
       -s, --storage-driver=""                Storage driver to use
@@ -182,7 +183,7 @@ options for `zfs` start with `zfs`.
 
      If using a block device for device mapper storage, it is best to use `lvm`
      to create and manage the thin-pool volume. This volume is then handed to Docker
-     to exclusively create snapshot volumes needed for images and containers.  
+     to exclusively create snapshot volumes needed for images and containers.
 
      Managing the thin-pool outside of Docker makes for the most feature-rich
      method of having Docker utilize device mapper thin provisioning as the
@@ -218,7 +219,7 @@ options for `zfs` start with `zfs`.
  *  `dm.loopdatasize`
 
     >**Note**: This option configures devicemapper loopback, which should not be used in production.
-		
+
     Specifies the size to use when creating the loopback file for the
     "data" device which is used for the thin pool. The default size is
     100G. The file is sparse, so it will not initially take up this
@@ -444,6 +445,13 @@ Local registries, whose IP address falls in the 127.0.0.0/8 range, are
 automatically marked as insecure as of Docker 1.3.2. It is not recommended to
 rely on this, as it may change in the future.
 
+## Legacy Registries
+
+Enabling `--disable-legacy-registry` forces a docker daemon to only interact with
+registries which support the V2 protocol.  Specifically, the daemon will not
+attempt `push`, `pull` and `login` to v1 registries.  The exception to this
+is `search` which can still be performed on v1 registries.
+
 ## Running a Docker daemon behind a HTTPS_PROXY
 
 When running inside a LAN that uses a `HTTPS` proxy, the Docker Hub
@@ -466,7 +474,7 @@ use the proxy
 `--default-ulimit` allows you to set the default `ulimit` options to use for
 all containers. It takes the same options as `--ulimit` for `docker run`. If
 these defaults are not set, `ulimit` settings will be inherited, if not set on
-`docker run`, from the Docker daemon. Any `--ulimit` options passed to 
+`docker run`, from the Docker daemon. Any `--ulimit` options passed to
 `docker run` will overwrite these defaults.
 
 Be careful setting `nproc` with the `ulimit` flag as `nproc` is designed by Linux to

docs/userguide/dockervolumes.md

@@ -74,16 +74,21 @@ The output will provide details on the container configurations including the
 volumes. The output should look something similar to the following:
 
     ...
-    "Volumes": {
+    Mounts": [
-        "/webapp": "/var/lib/docker/volumes/fac362...80535"
+        {
-    },
+            "Name": "fac362...80535",
-    "VolumesRW": {
+            "Source": "/var/lib/docker/volumes/fac362...80535/_data",
-        "/webapp": true
+            "Destination": "/webapp",
-    }
+            "Driver": "local",
+            "Mode": "",
+            "RW": true
+        }
+    ]
     ...
 
-You will notice in the above 'Volumes' is specifying the location on the host and 
+You will notice in the above 'Source' is specifying the location on the host and 
-'VolumesRW' is specifying that the volume is read/write.
+'Destination' is specifying the volume location inside the container. `RW` shows
+if the volume is read/write.
 
 ### Mount a host directory as a data volume
 

graph/export.go

@@ -114,6 +114,8 @@ func (s *TagStore) ImageExport(imageExportConfig *ImageExportConfig) error {
 // FIXME: this should be a top-level function, not a class method
 func (s *TagStore) exportImage(name, tempdir string) error {
 	for n := name; n != ""; {
+		img, err := s.LookupImage(n)
+
 		// temporary directory
 		tmpImageDir := filepath.Join(tempdir, n)
 		if err := os.Mkdir(tmpImageDir, os.FileMode(0755)); err != nil {
@@ -130,15 +132,17 @@ func (s *TagStore) exportImage(name, tempdir string) error {
 			return err
 		}
 
+		imageInspectRaw, err := json.Marshal(img)
+		if err != nil {
+			return err
+		}
+
 		// serialize json
 		json, err := os.Create(filepath.Join(tmpImageDir, "json"))
 		if err != nil {
 			return err
 		}
-		imageInspectRaw, err := s.LookupRaw(n)
+
-		if err != nil {
-			return err
-		}
 		written, err := json.Write(imageInspectRaw)
 		if err != nil {
 			return err
@@ -156,11 +160,6 @@ func (s *TagStore) exportImage(name, tempdir string) error {
 			return err
 		}
 
-		// find parent
-		img, err := s.LookupImage(n)
-		if err != nil {
-			return err
-		}
 		n = img.Parent
 	}
 	return nil

graph/fixtures/validate_manifest/bad_manifest

@@ -0,0 +1,38 @@
+{
+   "schemaVersion": 2,
+   "name": "library/hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
+      }
+   ],
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
+               "kty": "EC",
+               "x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
+               "y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
+            },
+            "alg": "ES256"
+         },
+         "signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
+      }
+   ]
+}

graph/fixtures/validate_manifest/extra_data_manifest

@@ -0,0 +1,46 @@
+{
+   "schemaVersion": 1,
+   "name": "library/hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
+      }
+   ],
+   "fsLayers": [
+      {
+         "blobSum": "sha256:ffff95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:ffff658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
+      }
+   ],
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
+               "kty": "EC",
+               "x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
+               "y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
+            },
+            "alg": "ES256"
+         },
+         "signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
+      }
+   ]
+}

graph/fixtures/validate_manifest/good_manifest

@@ -0,0 +1,38 @@
+{
+   "schemaVersion": 1,
+   "name": "library/hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
+      }
+   ],
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "OIH7:HQFS:44FK:45VB:3B53:OIAG:TPL4:ATF5:6PNE:MGHN:NHQX:2GE4",
+               "kty": "EC",
+               "x": "Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ",
+               "y": "zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8"
+            },
+            "alg": "ES256"
+         },
+         "signature": "Y6xaFz9Sy-OtcnKQS1Ilq3Dh8cu4h3nBTJCpOTF1XF7vKtcxxA_xMP8-SgDo869SJ3VsvgPL9-Xn-OoYG2rb1A",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjMxOTcsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wOS0xMVQwNDoxMzo0OFoifQ"
+      }
+   ]
+}

graph/fixtures/validate_manifest/no_signature_manifest

@@ -0,0 +1,22 @@
+{
+   "schemaVersion": 1,
+   "name": "library/hello-world",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:03f4658f8b782e12230c1783426bd3bacce651ce582a4ffb6fbbfa2079428ecb"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"id\":\"af340544ed62de0680f441c71fa1a80cb084678fed42bae393e543faea3a572c\",\"parent\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.608577814Z\",\"container\":\"c2b715156f640c7ac7d98472ea24335aba5432a1323a3bb722697e6d37ef794f\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [\\\"/hello\\\"]\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/hello\"],\"Image\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"535020c3e8add9d6bb06e5ac15a261e73d9b213d62fb2c14d752b8e189b2b912\",\"created\":\"2015-08-06T23:53:22.241352727Z\",\"container\":\"9aeb0006ffa72a8287564caaea87625896853701459261d3b569e320c0c9d5dc\",\"container_config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) COPY file:4abd3bff60458ca3b079d7b131ce26b2719055a030dfa96ff827da2b7c7038a7 in /\"],\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"docker_version\":\"1.7.1\",\"config\":{\"Hostname\":\"9aeb0006ffa7\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":null,\"Cmd\":null,\"Image\":\"\",\"Volumes\":null,\"VolumeDriver\":\"\",\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":960}\n"
+      }
+   ]
+}

graph/graph.go

@@ -32,6 +32,26 @@ import (
 	"github.com/vbatts/tar-split/tar/storage"
 )
 
+// v1ImageDescriptor is a non-content-addressable image descriptor
+type v1ImageDescriptor struct {
+	img *image.Image
+}
+
+// ID returns the image ID specified in the image structure.
+func (img v1ImageDescriptor) ID() string {
+	return img.img.ID
+}
+
+// Parent returns the parent ID specified in the image structure.
+func (img v1ImageDescriptor) Parent() string {
+	return img.img.Parent
+}
+
+// MarshalConfig renders the image structure into JSON.
+func (img v1ImageDescriptor) MarshalConfig() ([]byte, error) {
+	return json.Marshal(img.img)
+}
+
 // The type is used to protect pulling or building related image
 // layers from deleteing when filtered by dangling=true
 // The key of layers is the images ID which is pulling or building
@@ -86,10 +106,12 @@ type Graph struct {
 
 // file names for ./graph/<ID>/
 const (
-	jsonFileName      = "json"
+	jsonFileName            = "json"
-	layersizeFileName = "layersize"
+	layersizeFileName       = "layersize"
-	digestFileName    = "checksum"
+	digestFileName          = "checksum"
-	tarDataFileName   = "tar-data.json.gz"
+	tarDataFileName         = "tar-data.json.gz"
+	v1CompatibilityFileName = "v1Compatibility"
+	parentFileName          = "parent"
 )
 
 var (
@@ -214,23 +236,36 @@ func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, contain
 		img.ContainerConfig = *containerConfig
 	}
 
-	if err := graph.Register(img, layerData); err != nil {
+	if err := graph.Register(v1ImageDescriptor{img}, layerData); err != nil {
 		return nil, err
 	}
 	return img, nil
 }
 
 // Register imports a pre-existing image into the graph.
-func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) {
+// Returns nil if the image is already registered.
+func (graph *Graph) Register(im image.ImageDescriptor, layerData archive.ArchiveReader) (err error) {
+	imgID := im.ID()
 
-	if err := image.ValidateID(img.ID); err != nil {
+	if err := image.ValidateID(imgID); err != nil {
 		return err
 	}
 
 	// We need this entire operation to be atomic within the engine. Note that
 	// this doesn't mean Register is fully safe yet.
-	graph.imageMutex.Lock(img.ID)
+	graph.imageMutex.Lock(imgID)
-	defer graph.imageMutex.Unlock(img.ID)
+	defer graph.imageMutex.Unlock(imgID)
+
+	return graph.register(im, layerData)
+}
+
+func (graph *Graph) register(im image.ImageDescriptor, layerData archive.ArchiveReader) (err error) {
+	imgID := im.ID()
+
+	// Skip register if image is already registered
+	if graph.Exists(imgID) {
+		return nil
+	}
 
 	// The returned `error` must be named in this function's signature so that
 	// `err` is not shadowed in this deferred cleanup.
@@ -238,19 +273,14 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
 		// If any error occurs, remove the new dir from the driver.
 		// Don't check for errors since the dir might not have been created.
 		if err != nil {
-			graph.driver.Remove(img.ID)
+			graph.driver.Remove(imgID)
 		}
 	}()
 
-	// (This is a convenience to save time. Race conditions are taken care of by os.Rename)
-	if graph.Exists(img.ID) {
-		return fmt.Errorf("Image %s already exists", img.ID)
-	}
-
 	// Ensure that the image root does not exist on the filesystem
 	// when it is not registered in the graph.
 	// This is common when you switch from one graph driver to another
-	if err := os.RemoveAll(graph.imageRoot(img.ID)); err != nil && !os.IsNotExist(err) {
+	if err := os.RemoveAll(graph.imageRoot(imgID)); err != nil && !os.IsNotExist(err) {
 		return err
 	}
 
@@ -258,7 +288,7 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
 	// (the graph is the source of truth).
 	// Ignore errors, since we don't know if the driver correctly returns ErrNotExist.
 	// (FIXME: make that mandatory for drivers).
-	graph.driver.Remove(img.ID)
+	graph.driver.Remove(imgID)
 
 	tmp, err := graph.mktemp("")
 	defer os.RemoveAll(tmp)
@@ -266,20 +296,33 @@ func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader)
 		return fmt.Errorf("mktemp failed: %s", err)
 	}
 
+	parent := im.Parent()
+
 	// Create root filesystem in the driver
-	if err := createRootFilesystemInDriver(graph, img, layerData); err != nil {
+	if err := createRootFilesystemInDriver(graph, imgID, parent, layerData); err != nil {
 		return err
 	}
 
 	// Apply the diff/layer
-	if err := graph.storeImage(img, layerData, tmp); err != nil {
+	config, err := im.MarshalConfig()
+	if err != nil {
+		return err
+	}
+	if err := graph.storeImage(imgID, parent, config, layerData, tmp); err != nil {
 		return err
 	}
 	// Commit
-	if err := os.Rename(tmp, graph.imageRoot(img.ID)); err != nil {
+	if err := os.Rename(tmp, graph.imageRoot(imgID)); err != nil {
 		return err
 	}
-	graph.idIndex.Add(img.ID)
+	graph.idIndex.Add(imgID)
+	return nil
+}
+
+func createRootFilesystemInDriver(graph *Graph, id, parent string, layerData archive.ArchiveReader) error {
+	if err := graph.driver.Create(id, parent); err != nil {
+		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, id, err)
+	}
 	return nil
 }
 
@@ -462,6 +505,21 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) {
 	if err := dec.Decode(img); err != nil {
 		return nil, err
 	}
+
+	if img.ID == "" {
+		img.ID = id
+	}
+
+	if img.Parent == "" && img.ParentID != "" && img.ParentID.Validate() == nil {
+		img.Parent = img.ParentID.Hex()
+	}
+
+	// compatibilityID for parent
+	parent, err := ioutil.ReadFile(filepath.Join(root, parentFileName))
+	if err == nil && len(parent) > 0 {
+		img.Parent = string(parent)
+	}
+
 	if err := image.ValidateID(img.ID); err != nil {
 		return nil, err
 	}
@@ -496,7 +554,13 @@ func (graph *Graph) saveSize(root string, size int) error {
 }
 
 // SetDigest sets the digest for the image layer to the provided value.
-func (graph *Graph) SetDigest(id string, dgst digest.Digest) error {
+func (graph *Graph) SetLayerDigest(id string, dgst digest.Digest) error {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.setLayerDigest(id, dgst)
+}
+func (graph *Graph) setLayerDigest(id string, dgst digest.Digest) error {
 	root := graph.imageRoot(id)
 	if err := ioutil.WriteFile(filepath.Join(root, digestFileName), []byte(dgst.String()), 0600); err != nil {
 		return fmt.Errorf("Error storing digest in %s/%s: %s", root, digestFileName, err)
@@ -505,7 +569,14 @@ func (graph *Graph) SetDigest(id string, dgst digest.Digest) error {
 }
 
 // GetDigest gets the digest for the provide image layer id.
-func (graph *Graph) GetDigest(id string) (digest.Digest, error) {
+func (graph *Graph) GetLayerDigest(id string) (digest.Digest, error) {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.getLayerDigest(id)
+}
+
+func (graph *Graph) getLayerDigest(id string) (digest.Digest, error) {
 	root := graph.imageRoot(id)
 	cs, err := ioutil.ReadFile(filepath.Join(root, digestFileName))
 	if err != nil {
@@ -517,6 +588,76 @@ func (graph *Graph) GetDigest(id string) (digest.Digest, error) {
 	return digest.ParseDigest(string(cs))
 }
 
+// SetV1CompatibilityConfig stores the v1Compatibility JSON data associated
+// with the image in the manifest to the disk
+func (graph *Graph) SetV1CompatibilityConfig(id string, data []byte) error {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.setV1CompatibilityConfig(id, data)
+}
+func (graph *Graph) setV1CompatibilityConfig(id string, data []byte) error {
+	root := graph.imageRoot(id)
+	return ioutil.WriteFile(filepath.Join(root, v1CompatibilityFileName), data, 0600)
+}
+
+// GetV1CompatibilityConfig reads the v1Compatibility JSON data for the image
+// from the disk
+func (graph *Graph) GetV1CompatibilityConfig(id string) ([]byte, error) {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	return graph.getV1CompatibilityConfig(id)
+}
+
+func (graph *Graph) getV1CompatibilityConfig(id string) ([]byte, error) {
+	root := graph.imageRoot(id)
+	return ioutil.ReadFile(filepath.Join(root, v1CompatibilityFileName))
+}
+
+// GenerateV1CompatibilityChain makes sure v1Compatibility JSON data exists
+// for the image. If it doesn't it generates and stores it for the image and
+// all of it's parents based on the image config JSON.
+func (graph *Graph) GenerateV1CompatibilityChain(id string) ([]byte, error) {
+	graph.imageMutex.Lock(id)
+	defer graph.imageMutex.Unlock(id)
+
+	if v1config, err := graph.getV1CompatibilityConfig(id); err == nil {
+		return v1config, nil
+	}
+
+	// generate new, store it to disk
+	img, err := graph.Get(id)
+	if err != nil {
+		return nil, err
+	}
+
+	digestPrefix := string(digest.Canonical) + ":"
+	img.ID = strings.TrimPrefix(img.ID, digestPrefix)
+
+	if img.Parent != "" {
+		parentConfig, err := graph.GenerateV1CompatibilityChain(img.Parent)
+		if err != nil {
+			return nil, err
+		}
+		var parent struct{ ID string }
+		err = json.Unmarshal(parentConfig, &parent)
+		if err != nil {
+			return nil, err
+		}
+		img.Parent = parent.ID
+	}
+
+	json, err := json.Marshal(img)
+	if err != nil {
+		return nil, err
+	}
+	if err := graph.setV1CompatibilityConfig(id, json); err != nil {
+		return nil, err
+	}
+	return json, nil
+}
+
 // RawJSON returns the JSON representation for an image as a byte array.
 func (graph *Graph) RawJSON(id string) ([]byte, error) {
 	root := graph.imageRoot(id)
@@ -533,11 +674,11 @@ func jsonPath(root string) string {
 	return filepath.Join(root, jsonFileName)
 }
 
-func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData archive.ArchiveReader, root string) error {
+func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData archive.ArchiveReader, root string) (size int64, err error) {
 	// this is saving the tar-split metadata
 	mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
 	if err != nil {
-		return err
+		return 0, err
 	}
 	mfz := gzip.NewWriter(mf)
 	metaPacker := storage.NewJSONPacker(mfz)
@@ -546,21 +687,21 @@ func (graph *Graph) disassembleAndApplyTarLayer(img *image.Image, layerData arch
 
 	inflatedLayerData, err := archive.DecompressStream(layerData)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
 	// we're passing nil here for the file putter, because the ApplyDiff will
 	// handle the extraction of the archive
 	rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
 	if err != nil {
-		return err
+		return 0, err
 	}
 
-	if img.Size, err = graph.driver.ApplyDiff(img.ID, img.Parent, archive.ArchiveReader(rdr)); err != nil {
+	if size, err = graph.driver.ApplyDiff(id, parent, archive.ArchiveReader(rdr)); err != nil {
-		return err
+		return 0, err
 	}
 
-	return nil
+	return
 }
 
 func (graph *Graph) assembleTarLayer(img *image.Image) (archive.Archive, error) {

graph/graph_test.go

@@ -73,7 +73,7 @@ func TestInterruptedRegister(t *testing.T) {
 		Created: time.Now(),
 	}
 	w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
-	graph.Register(image, badArchive)
+	graph.Register(v1ImageDescriptor{image}, badArchive)
 	if _, err := graph.Get(image.ID); err == nil {
 		t.Fatal("Image should not exist after Register is interrupted")
 	}
@@ -82,7 +82,7 @@ func TestInterruptedRegister(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := graph.Register(image, goodArchive); err != nil {
+	if err := graph.Register(v1ImageDescriptor{image}, goodArchive); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -130,7 +130,7 @@ func TestRegister(t *testing.T) {
 		Comment: "testing",
 		Created: time.Now(),
 	}
-	err = graph.Register(image, archive)
+	err = graph.Register(v1ImageDescriptor{image}, archive)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,7 +212,7 @@ func TestDelete(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Test delete twice (pull -> rm -> pull -> rm)
-	if err := graph.Register(img1, archive); err != nil {
+	if err := graph.Register(v1ImageDescriptor{img1}, archive); err != nil {
 		t.Fatal(err)
 	}
 	if err := graph.Delete(img1.ID); err != nil {
@@ -246,9 +246,19 @@ func TestByParent(t *testing.T) {
 		Created: time.Now(),
 		Parent:  parentImage.ID,
 	}
-	_ = graph.Register(parentImage, archive1)
+
-	_ = graph.Register(childImage1, archive2)
+	err := graph.Register(v1ImageDescriptor{parentImage}, archive1)
-	_ = graph.Register(childImage2, archive3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = graph.Register(v1ImageDescriptor{childImage1}, archive2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = graph.Register(v1ImageDescriptor{childImage2}, archive3)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	byParent := graph.ByParent()
 	numChildren := len(byParent[parentImage.ID])

graph/graph_unix.go

@@ -3,8 +3,7 @@
 package graph
 
 import (
-	"encoding/json"
+	"io/ioutil"
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -74,13 +73,6 @@ func SetupInitLayer(initLayer string) error {
 	return nil
 }
 
-func createRootFilesystemInDriver(graph *Graph, img *image.Image, layerData archive.ArchiveReader) error {
-	if err := graph.driver.Create(img.ID, img.Parent); err != nil {
-		return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
-	}
-	return nil
-}
-
 func (graph *Graph) restoreBaseImages() ([]string, error) {
 	return nil, nil
 }
@@ -88,26 +80,35 @@ func (graph *Graph) restoreBaseImages() ([]string, error) {
 // storeImage stores file system layer data for the given image to the
 // graph's storage driver. Image metadata is stored in a file
 // at the specified root directory.
-func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) {
+func (graph *Graph) storeImage(id, parent string, config []byte, layerData archive.ArchiveReader, root string) (err error) {
+	var size int64
 	// Store the layer. If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil {
+		if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
 			return err
 		}
 	}
 
-	if err := graph.saveSize(root, int(img.Size)); err != nil {
+	if err := graph.saveSize(root, int(size)); err != nil {
 		return err
 	}
 
-	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
+		return err
+	}
+
+	// If image is pointing to a parent via CompatibilityID write the reference to disk
+	img, err := image.NewImgJSON(config)
 	if err != nil {
 		return err
 	}
 
-	defer f.Close()
+	if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() {
-
+		if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
-	return json.NewEncoder(f).Encode(img)
+			return err
+		}
+	}
+	return nil
 }
 
 // TarLayer returns a tar archive of the image's filesystem layer.

graph/graph_windows.go

@@ -3,9 +3,8 @@
 package graph
 
 import (
-	"encoding/json"
+	"io/ioutil"
-	"fmt"
+	"path/filepath"
-	"os"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/graphdriver/windows"
@@ -19,41 +18,6 @@ func SetupInitLayer(initLayer string) error {
 	return nil
 }
 
-func createRootFilesystemInDriver(graph *Graph, img *image.Image, layerData archive.ArchiveReader) error {
-	if wd, ok := graph.driver.(*windows.WindowsGraphDriver); ok {
-		if img.Container != "" && layerData == nil {
-			logrus.Debugf("Copying from container %s.", img.Container)
-
-			var ids []string
-			if img.Parent != "" {
-				parentImg, err := graph.Get(img.Parent)
-				if err != nil {
-					return err
-				}
-
-				ids, err = graph.ParentLayerIds(parentImg)
-				if err != nil {
-					return err
-				}
-			}
-
-			if err := wd.CopyDiff(img.Container, img.ID, wd.LayerIdsToPaths(ids)); err != nil {
-				return fmt.Errorf("Driver %s failed to copy image rootfs %s: %s", graph.driver, img.Container, err)
-			}
-		} else if img.Parent == "" {
-			if err := graph.driver.Create(img.ID, img.Parent); err != nil {
-				return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
-			}
-		}
-	} else {
-		// This fallback allows the use of VFS during daemon development.
-		if err := graph.driver.Create(img.ID, img.Parent); err != nil {
-			return fmt.Errorf("Driver %s failed to create image rootfs %s: %s", graph.driver, img.ID, err)
-		}
-	}
-	return nil
-}
-
 func (graph *Graph) restoreBaseImages() ([]string, error) {
 	// TODO Windows. This needs implementing (@swernli)
 	return nil, nil
@@ -71,42 +35,28 @@ func (graph *Graph) ParentLayerIds(img *image.Image) (ids []string, err error) {
 // storeImage stores file system layer data for the given image to the
 // graph's storage driver. Image metadata is stored in a file
 // at the specified root directory.
-func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader, root string) (err error) {
+func (graph *Graph) storeImage(id, parent string, config []byte, layerData archive.ArchiveReader, root string) (err error) {
-
+	var size int64
 	if wd, ok := graph.driver.(*windows.WindowsGraphDriver); ok {
 		// Store the layer. If layerData is not nil and this isn't a base image,
 		// unpack it into the new layer
-		if layerData != nil && img.Parent != "" {
+		if layerData != nil && parent != "" {
 			var ids []string
-			if img.Parent != "" {
+			parentImg, err := graph.Get(parent)
-				parentImg, err := graph.Get(img.Parent)
+			if err != nil {
-				if err != nil {
+				return err
-					return err
-				}
-
-				ids, err = graph.ParentLayerIds(parentImg)
-				if err != nil {
-					return err
-				}
 			}
 
-			if img.Size, err = wd.Import(img.ID, layerData, wd.LayerIdsToPaths(ids)); err != nil {
+			ids, err = graph.ParentLayerIds(parentImg)
+			if err != nil {
+				return err
+			}
+
+			if size, err = wd.Import(id, layerData, wd.LayerIdsToPaths(ids)); err != nil {
 				return err
 			}
 		}
 
-		if err := graph.saveSize(root, int(img.Size)); err != nil {
-			return err
-		}
-
-		f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
-		if err != nil {
-			return err
-		}
-
-		defer f.Close()
-
-		return json.NewEncoder(f).Encode(img)
 	} else {
 		// We keep this functionality here so that we can still work with the
 		// VFS driver during development. This will not be used for actual running
@@ -115,23 +65,32 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader
 
 		// Store the layer. If layerData is not nil, unpack it into the new layer
 		if layerData != nil {
-			if err := graph.disassembleAndApplyTarLayer(img, layerData, root); err != nil {
+			if size, err = graph.disassembleAndApplyTarLayer(id, parent, layerData, root); err != nil {
 				return err
 			}
 		}
 
-		if err := graph.saveSize(root, int(img.Size)); err != nil {
+		if err := graph.saveSize(root, size); err != nil {
 			return err
 		}
 
-		f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+		if err := ioutil.WriteFile(jsonPath(root), config, 0600); err != nil {
+			return err
+		}
+
+		// If image is pointing to a parent via CompatibilityID write the reference to disk
+		img, err := image.NewImgJSON(config)
 		if err != nil {
 			return err
 		}
 
-		defer f.Close()
+		if img.ParentID.Validate() == nil && parent != img.ParentID.Hex() {
+			if err := ioutil.WriteFile(filepath.Join(root, parentFileName), []byte(parent), 0600); err != nil {
+				return err
+			}
+		}
 
-		return json.NewEncoder(f).Encode(img)
+		return nil
 	}
 }
 

graph/load.go

@@ -125,7 +125,7 @@ func (s *TagStore) recursiveLoad(address, tmpImageDir string) error {
 				}
 			}
 		}
-		if err := s.graph.Register(img, layer); err != nil {
+		if err := s.graph.Register(v1ImageDescriptor{img}, layer); err != nil {
 			return err
 		}
 	}

graph/pull.go

@@ -86,12 +86,6 @@ func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConf
 	for _, endpoint := range endpoints {
 		logrus.Debugf("Trying to pull %s from %s %s", repoInfo.LocalName, endpoint.URL, endpoint.Version)
 
-		if !endpoint.Mirror && (endpoint.Official || endpoint.Version == registry.APIVersion2) {
-			if repoInfo.Official {
-				s.trustService.UpdateBase()
-			}
-		}
-
 		puller, err := NewPuller(s, endpoint, repoInfo, imagePullConfig, sf)
 		if err != nil {
 			lastErr = err

graph/pull_v1.go

@@ -59,6 +59,9 @@ func (p *v1Puller) Pull(tag string) (fallback bool, err error) {
 		// TODO(dmcgowan): Check if should fallback
 		return false, err
 	}
+	out := p.config.OutStream
+	out.Write(p.sf.FormatStatus("", "%s: this image was pulled from a legacy registry.  Important: This registry version will not be supported in future versions of docker.", p.repoInfo.CanonicalName))
+
 	return false, nil
 }
 
@@ -123,7 +126,7 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 	defer func() {
 		p.graph.Release(sessionID, imgIDs...)
 	}()
-	for _, image := range repoData.ImgList {
+	for _, imgData := range repoData.ImgList {
 		downloadImage := func(img *registry.ImgData) {
 			if askedTag != "" && img.Tag != askedTag {
 				errors <- nil
@@ -136,6 +139,11 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 				return
 			}
 
+			if err := image.ValidateID(img.ID); err != nil {
+				errors <- err
+				return
+			}
+
 			// ensure no two downloads of the same image happen at the same time
 			if c, err := p.poolAdd("pull", "img:"+img.ID); err != nil {
 				if c != nil {
@@ -196,7 +204,7 @@ func (p *v1Puller) pullRepository(askedTag string) error {
 			errors <- nil
 		}
 
-		go downloadImage(image)
+		go downloadImage(imgData)
 	}
 
 	var lastError error
@@ -304,7 +312,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro
 				layersDownloaded = true
 				defer layer.Close()
 
-				err = p.graph.Register(img,
+				err = p.graph.Register(v1ImageDescriptor{img},
 					progressreader.New(progressreader.Config{
 						In:        layer,
 						Out:       out,

graph/pull_v2.go

@@ -1,11 +1,13 @@
 package graph
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"sync"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution"
@@ -16,9 +18,7 @@ import (
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/registry"
-	"github.com/docker/docker/trust"
 	"github.com/docker/docker/utils"
-	"github.com/docker/libtrust"
 	"golang.org/x/net/context"
 )
 
@@ -73,17 +73,18 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) {
 
 	}
 
-	c, err := p.poolAdd("pull", taggedName)
+	poolKey := "v2:" + taggedName
+	c, err := p.poolAdd("pull", poolKey)
 	if err != nil {
 		if c != nil {
 			// Another pull of the same repository is already taking place; just wait for it to finish
-			p.sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", p.repoInfo.CanonicalName)
+			p.config.OutStream.Write(p.sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", p.repoInfo.CanonicalName))
 			<-c
 			return nil
 		}
 		return err
 	}
-	defer p.poolRemove("pull", taggedName)
+	defer p.poolRemove("pull", poolKey)
 
 	var layersDownloaded bool
 	for _, tag := range tags {
@@ -103,7 +104,7 @@ func (p *v2Puller) pullV2Repository(tag string) (err error) {
 
 // downloadInfo is used to pass information from download to extractor
 type downloadInfo struct {
-	img     *image.Image
+	img     contentAddressableDescriptor
 	tmpFile *os.File
 	digest  digest.Digest
 	layer   distribution.ReadSeekCloser
@@ -112,37 +113,92 @@ type downloadInfo struct {
 	out     io.Writer // Download progress is written here.
 }
 
+// contentAddressableDescriptor is used to pass image data from a manifest to the
+// graph.
+type contentAddressableDescriptor struct {
+	id              string
+	parent          string
+	strongID        digest.Digest
+	compatibilityID string
+	config          []byte
+	v1Compatibility []byte
+}
+
+func newContentAddressableImage(v1Compatibility []byte, blobSum digest.Digest, parent digest.Digest) (contentAddressableDescriptor, error) {
+	img := contentAddressableDescriptor{
+		v1Compatibility: v1Compatibility,
+	}
+
+	var err error
+	img.config, err = image.MakeImageConfig(v1Compatibility, blobSum, parent)
+	if err != nil {
+		return img, err
+	}
+	img.strongID, err = image.StrongID(img.config)
+	if err != nil {
+		return img, err
+	}
+
+	unmarshalledConfig, err := image.NewImgJSON(v1Compatibility)
+	if err != nil {
+		return img, err
+	}
+
+	img.compatibilityID = unmarshalledConfig.ID
+	img.id = img.strongID.Hex()
+
+	return img, nil
+}
+
+// ID returns the actual ID to be used for the downloaded image. This may be
+// a computed ID.
+func (img contentAddressableDescriptor) ID() string {
+	return img.id
+}
+
+// Parent returns the parent ID to be used for the image. This may be a
+// computed ID.
+func (img contentAddressableDescriptor) Parent() string {
+	return img.parent
+}
+
+// MarshalConfig renders the image structure into JSON.
+func (img contentAddressableDescriptor) MarshalConfig() ([]byte, error) {
+	return img.config, nil
+}
+
 type errVerification struct{}
 
 func (errVerification) Error() string { return "verification failed" }
 
 func (p *v2Puller) download(di *downloadInfo) {
-	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID)
+	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.id)
 
 	out := di.out
 
-	if c, err := p.poolAdd("pull", "img:"+di.img.ID); err != nil {
+	poolKey := "v2img:" + di.img.id
+	if c, err := p.poolAdd("pull", poolKey); err != nil {
 		if c != nil {
-			out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Layer already being pulled by another client. Waiting.", nil))
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Layer already being pulled by another client. Waiting.", nil))
 			<-c
-			out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil))
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
 		} else {
-			logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.ID, err)
+			logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.id, err)
 		}
 		di.err <- nil
 		return
 	}
 
-	defer p.poolRemove("pull", "img:"+di.img.ID)
+	defer p.poolRemove("pull", poolKey)
 	tmpFile, err := ioutil.TempFile("", "GetImageBlob")
 	if err != nil {
 		di.err <- err
 		return
 	}
 
-	blobs := p.repo.Blobs(nil)
+	blobs := p.repo.Blobs(context.Background())
 
-	desc, err := blobs.Stat(nil, di.digest)
+	desc, err := blobs.Stat(context.Background(), di.digest)
 	if err != nil {
 		logrus.Debugf("Error statting layer: %v", err)
 		di.err <- err
@@ -150,7 +206,7 @@ func (p *v2Puller) download(di *downloadInfo) {
 	}
 	di.size = desc.Size
 
-	layerDownload, err := blobs.Open(nil, di.digest)
+	layerDownload, err := blobs.Open(context.Background(), di.digest)
 	if err != nil {
 		logrus.Debugf("Error fetching layer: %v", err)
 		di.err <- err
@@ -170,12 +226,12 @@ func (p *v2Puller) download(di *downloadInfo) {
 		Formatter: p.sf,
 		Size:      int(di.size),
 		NewLines:  false,
-		ID:        stringid.TruncateID(di.img.ID),
+		ID:        stringid.TruncateID(di.img.id),
 		Action:    "Downloading",
 	})
 	io.Copy(tmpFile, reader)
 
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil))
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Verifying Checksum", nil))
 
 	if !verifier.Verified() {
 		err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
@@ -184,16 +240,16 @@ func (p *v2Puller) download(di *downloadInfo) {
 		return
 	}
 
-	out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil))
+	out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.id), "Download complete", nil))
 
-	logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, tmpFile.Name())
+	logrus.Debugf("Downloaded %s to tempfile %s", di.img.id, tmpFile.Name())
 	di.tmpFile = tmpFile
 	di.layer = layerDownload
 
 	di.err <- nil
 }
 
-func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) {
+func (p *v2Puller) pullV2Tag(tag, taggedName string) (tagUpdated bool, err error) {
 	logrus.Debugf("Pulling tag from V2 registry: %q", tag)
 	out := p.config.OutStream
 
@@ -202,16 +258,28 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 		return false, err
 	}
 
-	manifest, err := manSvc.GetByTag(tag)
+	unverifiedManifest, err := manSvc.GetByTag(tag)
 	if err != nil {
 		return false, err
 	}
-	verified, err = p.validateManifest(manifest, tag)
+	if unverifiedManifest == nil {
+		return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
+	}
+	var verifiedManifest *manifest.Manifest
+	verifiedManifest, err = verifyManifest(unverifiedManifest, tag)
 	if err != nil {
 		return false, err
 	}
-	if verified {
+
-		logrus.Printf("Image manifest for %s has been verified", taggedName)
+	// remove duplicate layers and check parent chain validity
+	err = fixManifestLayers(verifiedManifest)
+	if err != nil {
+		return false, err
+	}
+
+	imgs, err := p.getImageInfos(*verifiedManifest)
+	if err != nil {
+		return false, err
 	}
 
 	// By using a pipeWriter for each of the downloads to write their progress
@@ -223,6 +291,9 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 	go func() {
 		if _, err := io.Copy(out, pipeReader); err != nil {
 			logrus.Errorf("error copying from layer download progress reader: %s", err)
+			if err := pipeReader.CloseWithError(err); err != nil {
+				logrus.Errorf("error closing the progress reader: %s", err)
+			}
 		}
 	}()
 	defer func() {
@@ -231,45 +302,52 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 			// until all current readers/writers are done using the pipe then
 			// set the error. All successive reads/writes will return with this
 			// error.
-			pipeWriter.CloseWithError(errors.New("download canceled"))
+			pipeWriter.CloseWithError(fmt.Errorf("download canceled %+v", err))
+		} else {
+			// If no error then just close the pipe.
+			pipeWriter.Close()
 		}
 	}()
 
 	out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
 
-	downloads := make([]downloadInfo, len(manifest.FSLayers))
+	downloads := make([]downloadInfo, len(verifiedManifest.FSLayers))
 
 	layerIDs := []string{}
 	defer func() {
 		p.graph.Release(p.sessionID, layerIDs...)
 	}()
 
-	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
+	for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- {
-		img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
-		if err != nil {
-			logrus.Debugf("error getting image v1 json: %v", err)
-			return false, err
-		}
-		downloads[i].img = img
-		downloads[i].digest = manifest.FSLayers[i].BlobSum
 
-		p.graph.Retain(p.sessionID, img.ID)
+		img := imgs[i]
-		layerIDs = append(layerIDs, img.ID)
+		downloads[i].img = img
+		downloads[i].digest = verifiedManifest.FSLayers[i].BlobSum
+
+		p.graph.Retain(p.sessionID, img.id)
+		layerIDs = append(layerIDs, img.id)
+
+		p.graph.imageMutex.Lock(img.id)
 
 		// Check if exists
-		if p.graph.Exists(img.ID) {
+		if p.graph.Exists(img.id) {
-			logrus.Debugf("Image already exists: %s", img.ID)
+			if err := p.validateImageInGraph(img.id, imgs, i); err != nil {
+				p.graph.imageMutex.Unlock(img.id)
+				return false, fmt.Errorf("image validation failed: %v", err)
+			}
+			logrus.Debugf("Image already exists: %s", img.id)
+			p.graph.imageMutex.Unlock(img.id)
 			continue
 		}
+		p.graph.imageMutex.Unlock(img.id)
 
-		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
+		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.id), "Pulling fs layer", nil))
 
-		downloads[i].err = make(chan error)
+		downloads[i].err = make(chan error, 1)
 		downloads[i].out = pipeWriter
 		go p.download(&downloads[i])
 	}
 
-	var tagUpdated bool
 	for i := len(downloads) - 1; i >= 0; i-- {
 		d := &downloads[i]
 		if d.err != nil {
@@ -283,36 +361,58 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 			defer d.tmpFile.Close()
 			d.tmpFile.Seek(0, 0)
 			if d.tmpFile != nil {
+				err := func() error {
+					reader := progressreader.New(progressreader.Config{
+						In:        d.tmpFile,
+						Out:       out,
+						Formatter: p.sf,
+						Size:      int(d.size),
+						NewLines:  false,
+						ID:        stringid.TruncateID(d.img.id),
+						Action:    "Extracting",
+					})
 
-				reader := progressreader.New(progressreader.Config{
+					p.graph.imageMutex.Lock(d.img.id)
-					In:        d.tmpFile,
+					defer p.graph.imageMutex.Unlock(d.img.id)
-					Out:       out,
-					Formatter: p.sf,
-					Size:      int(d.size),
-					NewLines:  false,
-					ID:        stringid.TruncateID(d.img.ID),
-					Action:    "Extracting",
-				})
 
-				err = p.graph.Register(d.img, reader)
+					// Must recheck the data on disk if any exists.
+					// This protects against races where something
+					// else is written to the graph under this ID
+					// after attemptIDReuse.
+					if p.graph.Exists(d.img.id) {
+						if err := p.validateImageInGraph(d.img.id, imgs, i); err != nil {
+							return fmt.Errorf("image validation failed: %v", err)
+						}
+					}
+
+					if err := p.graph.register(d.img, reader); err != nil {
+						return err
+					}
+
+					if err := p.graph.setLayerDigest(d.img.id, d.digest); err != nil {
+						return err
+					}
+
+					if err := p.graph.setV1CompatibilityConfig(d.img.id, d.img.v1Compatibility); err != nil {
+						return err
+					}
+
+					return nil
+				}()
 				if err != nil {
 					return false, err
 				}
 
-				if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
-					return false, err
-				}
-
 				// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
 			}
-			out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Pull complete", nil))
 			tagUpdated = true
 		} else {
-			out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil))
+			out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.id), "Already exists", nil))
 		}
 	}
 
-	manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName)
+	manifestDigest, _, err := digestFromManifest(unverifiedManifest, p.repoInfo.LocalName)
 	if err != nil {
 		return false, err
 	}
@@ -332,21 +432,17 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 		}
 	}
 
-	if verified && tagUpdated {
-		out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
-	}
-
 	if utils.DigestReference(tag) {
 		// TODO(stevvooe): Ideally, we should always set the digest so we can
 		// use the digest whether we pull by it or not. Unfortunately, the tag
 		// store treats the digest as a separate tag, meaning there may be an
 		// untagged digest image that would seem to be dangling by a user.
-		if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.ID); err != nil {
+		if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.id); err != nil {
 			return false, err
 		}
 	} else {
 		// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
-		if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil {
+		if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.id, true); err != nil {
 			return false, err
 		}
 	}
@@ -358,82 +454,262 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error)
 	return tagUpdated, nil
 }
 
-// verifyTrustedKeys checks the keys provided against the trust store,
+func verifyManifest(signedManifest *manifest.SignedManifest, tag string) (m *manifest.Manifest, err error) {
-// ensuring that the provided keys are trusted for the namespace. The keys
-// provided from this method must come from the signatures provided as part of
-// the manifest JWS package, obtained from unpackSignedManifest or libtrust.
-func (p *v2Puller) verifyTrustedKeys(namespace string, keys []libtrust.PublicKey) (verified bool, err error) {
-	if namespace[0] != '/' {
-		namespace = "/" + namespace
-	}
-
-	for _, key := range keys {
-		b, err := key.MarshalJSON()
-		if err != nil {
-			return false, fmt.Errorf("error marshalling public key: %s", err)
-		}
-		// Check key has read/write permission (0x03)
-		v, err := p.trustService.CheckKey(namespace, b, 0x03)
-		if err != nil {
-			vErr, ok := err.(trust.NotVerifiedError)
-			if !ok {
-				return false, fmt.Errorf("error running key check: %s", err)
-			}
-			logrus.Debugf("Key check result: %v", vErr)
-		}
-		verified = v
-	}
-
-	if verified {
-		logrus.Debug("Key check result: verified")
-	}
-
-	return
-}
-
-func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (verified bool, err error) {
 	// If pull by digest, then verify the manifest digest. NOTE: It is
 	// important to do this first, before any other content validation. If the
 	// digest cannot be verified, don't even bother with those other things.
 	if manifestDigest, err := digest.ParseDigest(tag); err == nil {
 		verifier, err := digest.NewDigestVerifier(manifestDigest)
 		if err != nil {
-			return false, err
+			return nil, err
 		}
-		payload, err := m.Payload()
+		payload, err := signedManifest.Payload()
 		if err != nil {
-			return false, err
+			// If this failed, the signatures section was corrupted
+			// or missing. Treat the entire manifest as the payload.
+			payload = signedManifest.Raw
 		}
 		if _, err := verifier.Write(payload); err != nil {
-			return false, err
+			return nil, err
 		}
 		if !verifier.Verified() {
 			err := fmt.Errorf("image verification failed for digest %s", manifestDigest)
 			logrus.Error(err)
-			return false, err
+			return nil, err
+		}
+
+		var verifiedManifest manifest.Manifest
+		if err = json.Unmarshal(payload, &verifiedManifest); err != nil {
+			return nil, err
+		}
+		m = &verifiedManifest
+	} else {
+		m = &signedManifest.Manifest
+	}
+
+	if m.SchemaVersion != 1 {
+		return nil, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
+	}
+	if len(m.FSLayers) != len(m.History) {
+		return nil, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
+	}
+	if len(m.FSLayers) == 0 {
+		return nil, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
+	}
+	return m, nil
+}
+
+// fixManifestLayers removes repeated layers from the manifest and checks the
+// correctness of the parent chain.
+func fixManifestLayers(m *manifest.Manifest) error {
+	images := make([]*image.Image, len(m.FSLayers))
+	for i := range m.FSLayers {
+		img, err := image.NewImgJSON([]byte(m.History[i].V1Compatibility))
+		if err != nil {
+			return err
+		}
+		images[i] = img
+		if err := image.ValidateID(img.ID); err != nil {
+			return err
 		}
 	}
 
-	// TODO(tiborvass): what's the usecase for having manifest == nil and err == nil ? Shouldn't be the error be "DoesNotExist" ?
+	if images[len(images)-1].Parent != "" {
-	if m == nil {
+		return errors.New("Invalid parent ID in the base layer of the image.")
-		return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
 	}
-	if m.SchemaVersion != 1 {
+
-		return false, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
+	// check general duplicates to error instead of a deadlock
+	idmap := make(map[string]struct{})
+
+	var lastID string
+	for _, img := range images {
+		// skip IDs that appear after each other, we handle those later
+		if _, exists := idmap[img.ID]; img.ID != lastID && exists {
+			return fmt.Errorf("ID %+v appears multiple times in manifest.", img.ID)
+		}
+		lastID = img.ID
+		idmap[lastID] = struct{}{}
 	}
-	if len(m.FSLayers) != len(m.History) {
+
-		return false, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
+	// backwards loop so that we keep the remaining indexes after removing items
+	for i := len(images) - 2; i >= 0; i-- {
+		if images[i].ID == images[i+1].ID { // repeated ID. remove and continue
+			m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...)
+			m.History = append(m.History[:i], m.History[i+1:]...)
+		} else if images[i].Parent != images[i+1].ID {
+			return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", images[i+1].ID, images[i].Parent)
+		}
 	}
-	if len(m.FSLayers) == 0 {
+
-		return false, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
+	return nil
-	}
+}
-	keys, err := manifest.Verify(m)
+
-	if err != nil {
+// getImageInfos returns an imageinfo struct for every image in the manifest.
-		return false, fmt.Errorf("error verifying manifest for tag %q: %v", tag, err)
+// These objects contain both calculated strongIDs and compatibilityIDs found
-	}
+// in v1Compatibility object.
-	verified, err = p.verifyTrustedKeys(m.Name, keys)
+func (p *v2Puller) getImageInfos(m manifest.Manifest) ([]contentAddressableDescriptor, error) {
-	if err != nil {
+	imgs := make([]contentAddressableDescriptor, len(m.FSLayers))
-		return false, fmt.Errorf("error verifying manifest keys: %v", err)
+
-	}
+	var parent digest.Digest
-	return verified, nil
+	for i := len(imgs) - 1; i >= 0; i-- {
+		var err error
+		imgs[i], err = newContentAddressableImage([]byte(m.History[i].V1Compatibility), m.FSLayers[i].BlobSum, parent)
+		if err != nil {
+			return nil, err
+		}
+		parent = imgs[i].strongID
+	}
+
+	p.attemptIDReuse(imgs)
+
+	return imgs, nil
+}
+
+var idReuseLock sync.Mutex
+
+// attemptIDReuse does a best attempt to match verified compatibilityIDs
+// already in the graph with the computed strongIDs so we can keep using them.
+// This process will never fail but may just return the strongIDs if none of
+// the compatibilityIDs exists or can be verified. If the strongIDs themselves
+// fail verification, we deterministically generate alternate IDs to use until
+// we find one that's available or already exists with the correct data.
+func (p *v2Puller) attemptIDReuse(imgs []contentAddressableDescriptor) {
+	// This function needs to be protected with a global lock, because it
+	// locks multiple IDs at once, and there's no good way to make sure
+	// the locking happens a deterministic order.
+	idReuseLock.Lock()
+	defer idReuseLock.Unlock()
+
+	idMap := make(map[string]struct{})
+	for _, img := range imgs {
+		idMap[img.id] = struct{}{}
+		idMap[img.compatibilityID] = struct{}{}
+
+		if p.graph.Exists(img.compatibilityID) {
+			if _, err := p.graph.GenerateV1CompatibilityChain(img.compatibilityID); err != nil {
+				logrus.Debugf("Migration v1Compatibility generation error: %v", err)
+				return
+			}
+		}
+	}
+	for id := range idMap {
+		p.graph.imageMutex.Lock(id)
+		defer p.graph.imageMutex.Unlock(id)
+	}
+
+	// continueReuse controls whether the function will try to find
+	// existing layers on disk under the old v1 IDs, to avoid repulling
+	// them. The hashes are checked to ensure these layers are okay to
+	// use. continueReuse starts out as true, but is set to false if
+	// the code encounters something that doesn't match the expected hash.
+	continueReuse := true
+
+	for i := len(imgs) - 1; i >= 0; i-- {
+		if p.graph.Exists(imgs[i].id) {
+			// Found an image in the graph under the strongID. Validate the
+			// image before using it.
+			if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
+				continueReuse = false
+				logrus.Debugf("not using existing strongID: %v", err)
+
+				// The strong ID existed in the graph but didn't
+				// validate successfully. We can't use the strong ID
+				// because it didn't validate successfully. Treat the
+				// graph like a hash table with probing... compute
+				// SHA256(id) until we find an ID that either doesn't
+				// already exist in the graph, or has existing content
+				// that validates successfully.
+				for {
+					if err := p.tryNextID(imgs, i, idMap); err != nil {
+						logrus.Debug(err.Error())
+					} else {
+						break
+					}
+				}
+			}
+			continue
+		}
+
+		if continueReuse {
+			compatibilityID := imgs[i].compatibilityID
+			if err := p.validateImageInGraph(compatibilityID, imgs, i); err != nil {
+				logrus.Debugf("stopping ID reuse: %v", err)
+				continueReuse = false
+			} else {
+				// The compatibility ID exists in the graph and was
+				// validated. Use it.
+				imgs[i].id = compatibilityID
+			}
+		}
+	}
+
+	// fix up the parents of the images
+	for i := 0; i < len(imgs); i++ {
+		if i == len(imgs)-1 { // Base layer
+			imgs[i].parent = ""
+		} else {
+			imgs[i].parent = imgs[i+1].id
+		}
+	}
+}
+
+// validateImageInGraph checks that an image in the graph has the expected
+// strongID. id is the entry in the graph to check, imgs is the slice of
+// images being processed (for access to the parent), and i is the index
+// into this slice which the graph entry should be checked against.
+func (p *v2Puller) validateImageInGraph(id string, imgs []contentAddressableDescriptor, i int) error {
+	img, err := p.graph.Get(id)
+	if err != nil {
+		return fmt.Errorf("missing: %v", err)
+	}
+	layerID, err := p.graph.getLayerDigest(id)
+	if err != nil {
+		return fmt.Errorf("digest: %v", err)
+	}
+	var parentID digest.Digest
+	if i != len(imgs)-1 {
+		if img.Parent != imgs[i+1].id { // comparing that graph points to validated ID
+			return fmt.Errorf("parent: %v %v", img.Parent, imgs[i+1].id)
+		} else {
+			parentID = imgs[i+1].strongID
+		}
+	} else if img.Parent != "" {
+		return fmt.Errorf("unexpected parent: %v", img.Parent)
+	}
+
+	v1Config, err := p.graph.getV1CompatibilityConfig(img.ID)
+	if err != nil {
+		return fmt.Errorf("v1Compatibility: %v %v", img.ID, err)
+	}
+
+	json, err := image.MakeImageConfig(v1Config, layerID, parentID)
+	if err != nil {
+		return fmt.Errorf("make config: %v", err)
+	}
+
+	if dgst, err := image.StrongID(json); err == nil && dgst == imgs[i].strongID {
+		logrus.Debugf("Validated %v as %v", dgst, id)
+	} else {
+		return fmt.Errorf("digest mismatch: %v %v, error: %v", dgst, imgs[i].strongID, err)
+	}
+
+	// All clear
+	return nil
+}
+
+func (p *v2Puller) tryNextID(imgs []contentAddressableDescriptor, i int, idMap map[string]struct{}) error {
+	nextID, _ := digest.FromBytes([]byte(imgs[i].id))
+	imgs[i].id = nextID.Hex()
+
+	if _, exists := idMap[imgs[i].id]; !exists {
+		p.graph.imageMutex.Lock(imgs[i].id)
+		defer p.graph.imageMutex.Unlock(imgs[i].id)
+	}
+
+	if p.graph.Exists(imgs[i].id) {
+		if err := p.validateImageInGraph(imgs[i].id, imgs, i); err != nil {
+			return fmt.Errorf("not using existing strongID permutation %s: %v", imgs[i].id, err)
+		}
+	}
+	return nil
 }

graph/pull_v2_test.go

@@ -0,0 +1,194 @@
+package graph
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+)
+
+// TestValidateManifest verifies the validateManifest function
+func TestValidateManifest(t *testing.T) {
+	expectedDigest := "sha256:02fee8c3220ba806531f606525eceb83f4feb654f62b207191b1c9209188dedd"
+	expectedFSLayer0 := digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")
+
+	// Good manifest
+
+	goodManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/good_manifest")
+	if err != nil {
+		t.Fatal("error reading fixture:", err)
+	}
+
+	var goodSignedManifest manifest.SignedManifest
+	err = json.Unmarshal(goodManifestBytes, &goodSignedManifest)
+	if err != nil {
+		t.Fatal("error unmarshaling manifest:", err)
+	}
+
+	verifiedManifest, err := verifyManifest(&goodSignedManifest, expectedDigest)
+	if err != nil {
+		t.Fatal("validateManifest failed:", err)
+	}
+
+	if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
+		t.Fatal("unexpected FSLayer in good manifest")
+	}
+
+	// "Extra data" manifest
+
+	extraDataManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/extra_data_manifest")
+	if err != nil {
+		t.Fatal("error reading fixture:", err)
+	}
+
+	var extraDataSignedManifest manifest.SignedManifest
+	err = json.Unmarshal(extraDataManifestBytes, &extraDataSignedManifest)
+	if err != nil {
+		t.Fatal("error unmarshaling manifest:", err)
+	}
+
+	verifiedManifest, err = verifyManifest(&extraDataSignedManifest, expectedDigest)
+	if err != nil {
+		t.Fatal("validateManifest failed:", err)
+	}
+
+	if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
+		t.Fatal("unexpected FSLayer in extra data manifest")
+	}
+
+	// Bad manifest
+
+	badManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/bad_manifest")
+	if err != nil {
+		t.Fatal("error reading fixture:", err)
+	}
+
+	var badSignedManifest manifest.SignedManifest
+	err = json.Unmarshal(badManifestBytes, &badSignedManifest)
+	if err != nil {
+		t.Fatal("error unmarshaling manifest:", err)
+	}
+
+	verifiedManifest, err = verifyManifest(&badSignedManifest, expectedDigest)
+	if err == nil || !strings.HasPrefix(err.Error(), "image verification failed for digest") {
+		t.Fatal("expected validateManifest to fail with digest error")
+	}
+
+	// Manifest with no signature
+
+	expectedWholeFileDigest := "7ec3615a120efcdfc270e9c7ea4183330775a3e52a09e2efb194b9a7c18e5ff7"
+
+	noSignatureManifestBytes, err := ioutil.ReadFile("fixtures/validate_manifest/no_signature_manifest")
+	if err != nil {
+		t.Fatal("error reading fixture:", err)
+	}
+
+	var noSignatureSignedManifest manifest.SignedManifest
+	err = json.Unmarshal(noSignatureManifestBytes, &noSignatureSignedManifest)
+	if err != nil {
+		t.Fatal("error unmarshaling manifest:", err)
+	}
+
+	verifiedManifest, err = verifyManifest(&noSignatureSignedManifest, expectedWholeFileDigest)
+	if err != nil {
+		t.Fatal("validateManifest failed:", err)
+	}
+
+	if verifiedManifest.FSLayers[0].BlobSum != expectedFSLayer0 {
+		t.Fatal("unexpected FSLayer in no-signature manifest")
+	}
+}
+
+// TestFixManifestLayers checks that fixManifestLayers removes a duplicate
+// layer, and that it makes no changes to the manifest when called a second
+// time, after the duplicate is removed.
+func TestFixManifestLayers(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	duplicateLayerManifestExpectedOutput := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
+		t.Fatalf("unexpected error from fixManifestLayers: %v", err)
+	}
+
+	if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
+		t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest")
+	}
+
+	// Run fixManifestLayers again and confirm that it doesn't change the
+	// manifest (which no longer has duplicate layers).
+	if err := fixManifestLayers(&duplicateLayerManifest); err != nil {
+		t.Fatalf("unexpected error from fixManifestLayers: %v", err)
+	}
+
+	if !reflect.DeepEqual(duplicateLayerManifest, duplicateLayerManifestExpectedOutput) {
+		t.Fatal("incorrect output from fixManifestLayers on duplicate layer manifest (second pass)")
+	}
+}
+
+// TestFixManifestLayersBaseLayerParent makes sure that fixManifestLayers fails
+// if the base layer configuration specifies a parent.
+func TestFixManifestLayersBaseLayerParent(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"parent\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID in the base layer of the image.") {
+		t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
+	}
+}
+
+// TestFixManifestLayersBadParent makes sure that fixManifestLayers fails
+// if an image configuration specifies a parent that doesn't directly follow
+// that (deduplicated) image in the image history.
+func TestFixManifestLayersBadParent(t *testing.T) {
+	duplicateLayerManifest := manifest.Manifest{
+		FSLayers: []manifest.FSLayer{
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4")},
+			{BlobSum: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa")},
+		},
+		History: []manifest.History{
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9\",\"parent\":\"ac3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:11.368300679Z\",\"container\":\"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) ENTRYPOINT [\\\"/go/bin/dnsdock\\\"]\"],\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":null,\"Image\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":[\"/go/bin/dnsdock\"],\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\n"},
+			{V1Compatibility: "{\"id\":\"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02\",\"created\":\"2015-08-19T16:49:07.568027497Z\",\"container\":\"fe9e5a5264a843c9292d17b736c92dd19bdb49986a8782d7389964ddaff887cc\",\"container_config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cd /go/src/github.com/tonistiigi/dnsdock \\u0026\\u0026     go get -v github.com/tools/godep \\u0026\\u0026     godep restore \\u0026\\u0026     go install -ldflags \\\"-X main.version `git describe --tags HEAD``if [[ -n $(command git status --porcelain --untracked-files=no 2\\u003e/dev/null) ]]; then echo \\\"-dirty\\\"; fi`\\\" ./...\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"docker_version\":\"1.6.2\",\"config\":{\"Hostname\":\"03797203757d\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\"GOLANG_VERSION=1.4.1\",\"GOPATH=/go\"],\"Cmd\":[\"/bin/bash\"],\"Image\":\"e3b0ff09e647595dafee15c54cd632c900df9e82b1d4d313b1e20639a1461779\",\"Volumes\":null,\"WorkingDir\":\"/go\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"Labels\":{}},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":118430532}\n"},
+		},
+	}
+
+	if err := fixManifestLayers(&duplicateLayerManifest); err == nil || !strings.Contains(err.Error(), "Invalid parent ID.") {
+		t.Fatalf("expected an invalid parent ID error from fixManifestLayers")
+	}
+}

graph/push.go

@@ -29,13 +29,12 @@ func (s *TagStore) NewPusher(endpoint registry.APIEndpoint, localRepo Repository
 	switch endpoint.Version {
 	case registry.APIVersion2:
 		return &v2Pusher{
-			TagStore:   s,
+			TagStore:  s,
-			endpoint:   endpoint,
+			endpoint:  endpoint,
-			localRepo:  localRepo,
+			localRepo: localRepo,
-			repoInfo:   repoInfo,
+			repoInfo:  repoInfo,
-			config:     imagePushConfig,
+			config:    imagePushConfig,
-			sf:         sf,
+			sf:        sf,
-			layersSeen: make(map[string]bool),
 		}, nil
 	case registry.APIVersion1:
 		return &v1Pusher{

graph/push_v1.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/registry/client/transport"
+	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/streamformatter"
@@ -127,7 +128,7 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [
 			continue
 		}
 		// If the image does not have a tag it still needs to be sent to the
-		// registry with an empty tag so that it is accociated with the repository
+		// registry with an empty tag so that it is associated with the repository
 		imageIndex = append(imageIndex, &registry.ImgData{
 			ID:  id,
 			Tag: "",
@@ -137,9 +138,10 @@ func (s *TagStore) createImageIndex(images []string, tags map[string][]string) [
 }
 
 type imagePushData struct {
-	id       string
+	id              string
-	endpoint string
+	compatibilityID string
-	tokens   []string
+	endpoint        string
+	tokens          []string
 }
 
 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists
@@ -147,7 +149,7 @@ type imagePushData struct {
 func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) {
 	defer wg.Done()
 	for image := range images {
-		if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil {
+		if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil {
 			logrus.Errorf("Error in LookupRemoteImage: %s", err)
 			imagesToPush <- image.id
 			continue
@@ -181,10 +183,15 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags
 		pushes <- shouldPush
 	}()
 	for _, id := range imageIDs {
+		compatibilityID, err := p.getV1ID(id)
+		if err != nil {
+			return err
+		}
 		imageData <- imagePushData{
-			id:       id,
+			id:              id,
-			endpoint: endpoint,
+			compatibilityID: compatibilityID,
-			tokens:   repo.Tokens,
+			endpoint:        endpoint,
+			tokens:          repo.Tokens,
 		}
 	}
 	// close the channel to notify the workers that there will be no more images to check.
@@ -204,7 +211,11 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags
 		}
 		for _, tag := range tags[id] {
 			p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag))
-			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil {
+			compatibilityID, err := p.getV1ID(id)
+			if err != nil {
+				return err
+			}
+			if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil {
 				return err
 			}
 		}
@@ -214,7 +225,6 @@ func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags
 
 // pushRepository pushes layers that do not already exist on the registry.
 func (p *v1Pusher) pushRepository(tag string) error {
-
 	logrus.Debugf("Local repo: %s", p.localRepo)
 	p.out = ioutils.NewWriteFlusher(p.config.OutStream)
 	imgList, tags, err := p.getImageList(tag)
@@ -227,6 +237,12 @@ func (p *v1Pusher) pushRepository(tag string) error {
 	logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo)
 	for _, data := range imageIndex {
 		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
+
+		// convert IDs to compatibilityIDs, imageIndex only used in registry calls
+		data.ID, err = p.getV1ID(data.ID)
+		if err != nil {
+			return err
+		}
 	}
 
 	if _, err := p.poolAdd("push", p.repoInfo.LocalName); err != nil {
@@ -256,20 +272,27 @@ func (p *v1Pusher) pushRepository(tag string) error {
 }
 
 func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, err error) {
-	jsonRaw, err := p.graph.RawJSON(imgID)
+	jsonRaw, err := p.getV1Config(imgID)
 	if err != nil {
 		return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
 	}
 	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil))
 
+	compatibilityID, err := p.getV1ID(imgID)
+	if err != nil {
+		return "", err
+	}
+
+	// General rule is to use ID for graph accesses and compatibilityID for
+	// calls to session.registry()
 	imgData := &registry.ImgData{
-		ID: imgID,
+		ID: compatibilityID,
 	}
 
 	// Send the json
 	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
 		if err == registry.ErrAlreadyExists {
-			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil))
+			p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil))
 			return "", nil
 		}
 		return "", err
@@ -282,7 +305,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string,
 	defer os.RemoveAll(layerData.Name())
 
 	// Send the layer
-	logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size)
+	logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size)
 
 	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID,
 		progressreader.New(progressreader.Config{
@@ -291,7 +314,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string,
 			Formatter: p.sf,
 			Size:      int(layerData.Size),
 			NewLines:  false,
-			ID:        stringid.TruncateID(imgData.ID),
+			ID:        stringid.TruncateID(imgID),
 			Action:    "Pushing",
 		}), ep, jsonRaw)
 	if err != nil {
@@ -304,6 +327,30 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string,
 		return "", err
 	}
 
-	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil))
+	p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil))
 	return imgData.Checksum, nil
 }
+
+// getV1ID returns the compatibilityID for the ID in the graph. compatibilityID
+// is read from from the v1Compatibility config file in the disk.
+func (p *v1Pusher) getV1ID(id string) (string, error) {
+	jsonData, err := p.getV1Config(id)
+	if err != nil {
+		return "", err
+	}
+	img, err := image.NewImgJSON(jsonData)
+	if err != nil {
+		return "", err
+	}
+	return img.ID, nil
+}
+
+// getV1Config returns v1Compatibility config for the image in the graph. If
+// there is no v1Compatibility file on disk for the image
+func (p *v1Pusher) getV1Config(id string) ([]byte, error) {
+	jsonData, err := p.graph.GenerateV1CompatibilityChain(id)
+	if err != nil {
+		return nil, err
+	}
+	return jsonData, nil
+}

graph/push_v2.go

@@ -27,11 +27,6 @@ type v2Pusher struct {
 	config    *ImagePushConfig
 	sf        *streamformatter.StreamFormatter
 	repo      distribution.Repository
-
-	// layersSeen is the set of layers known to exist on the remote side.
-	// This avoids redundant queries when pushing multiple tags that
-	// involve the same layers.
-	layersSeen map[string]bool
 }
 
 func (p *v2Pusher) Push() (fallback bool, err error) {
@@ -92,6 +87,8 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 		return fmt.Errorf("tag does not exist: %s", tag)
 	}
 
+	layersSeen := make(map[string]bool)
+
 	layer, err := p.graph.Get(layerId)
 	if err != nil {
 		return err
@@ -120,7 +117,7 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 			return err
 		}
 
-		if p.layersSeen[layer.ID] {
+		if layersSeen[layer.ID] {
 			break
 		}
 
@@ -132,16 +129,11 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 			}
 		}
 
-		jsonData, err := p.graph.RawJSON(layer.ID)
-		if err != nil {
-			return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err)
-		}
-
 		var exists bool
-		dgst, err := p.graph.GetDigest(layer.ID)
+		dgst, err := p.graph.GetLayerDigest(layer.ID)
 		switch err {
 		case nil:
-			_, err := p.repo.Blobs(nil).Stat(nil, dgst)
+			_, err := p.repo.Blobs(context.Background()).Stat(context.Background(), dgst)
 			switch err {
 			case nil:
 				exists = true
@@ -161,21 +153,27 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
 		// if digest was empty or not saved, or if blob does not exist on the remote repository,
 		// then fetch it.
 		if !exists {
-			if pushDigest, err := p.pushV2Image(p.repo.Blobs(nil), layer); err != nil {
+			if pushDigest, err := p.pushV2Image(p.repo.Blobs(context.Background()), layer); err != nil {
 				return err
 			} else if pushDigest != dgst {
 				// Cache new checksum
-				if err := p.graph.SetDigest(layer.ID, pushDigest); err != nil {
+				if err := p.graph.SetLayerDigest(layer.ID, pushDigest); err != nil {
 					return err
 				}
 				dgst = pushDigest
 			}
 		}
 
+		// read v1Compatibility config, generate new if needed
+		jsonData, err := p.graph.GenerateV1CompatibilityChain(layer.ID)
+		if err != nil {
+			return err
+		}
+
 		m.FSLayers = append(m.FSLayers, manifest.FSLayer{BlobSum: dgst})
 		m.History = append(m.History, manifest.History{V1Compatibility: string(jsonData)})
 
-		p.layersSeen[layer.ID] = true
+		layersSeen[layer.ID] = true
 	}
 
 	logrus.Infof("Signed manifest for %s:%s using daemon's key: %s", p.repo.Name(), tag, p.trustKey.KeyID())
@@ -229,7 +227,7 @@ func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (d
 
 	// Send the layer
 	logrus.Debugf("rendered layer for %s of [%d] size", img.ID, size)
-	layerUpload, err := bs.Create(nil)
+	layerUpload, err := bs.Create(context.Background())
 	if err != nil {
 		return "", err
 	}
@@ -253,7 +251,7 @@ func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (d
 	}
 
 	desc := distribution.Descriptor{Digest: dgst}
-	if _, err := layerUpload.Commit(nil, desc); err != nil {
+	if _, err := layerUpload.Commit(context.Background(), desc); err != nil {
 		return "", err
 	}
 

graph/registry.go

@@ -100,8 +100,9 @@ func NewV2Repository(repoInfo *registry.RepositoryInfo, endpoint registry.APIEnd
 func digestFromManifest(m *manifest.SignedManifest, localName string) (digest.Digest, int, error) {
 	payload, err := m.Payload()
 	if err != nil {
-		logrus.Debugf("could not retrieve manifest payload: %v", err)
+		// If this failed, the signatures section was corrupted
-		return "", 0, err
+		// or missing. Treat the entire manifest as the payload.
+		payload = m.Raw
 	}
 	manifestDigest, err := digest.FromBytes(payload)
 	if err != nil {

graph/service.go

@@ -10,20 +10,6 @@ import (
 	"github.com/docker/docker/api/types"
 )
 
-func (s *TagStore) LookupRaw(name string) ([]byte, error) {
-	image, err := s.LookupImage(name)
-	if err != nil || image == nil {
-		return nil, fmt.Errorf("No such image %s", name)
-	}
-
-	imageInspectRaw, err := s.graph.RawJSON(image.ID)
-	if err != nil {
-		return nil, err
-	}
-
-	return imageInspectRaw, nil
-}
-
 // Lookup return an image encoded in JSON
 func (s *TagStore) Lookup(name string) (*types.ImageInspect, error) {
 	image, err := s.LookupImage(name)

graph/tags.go

@@ -19,7 +19,6 @@ import (
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/registry"
-	"github.com/docker/docker/trust"
 	"github.com/docker/docker/utils"
 	"github.com/docker/libtrust"
 )
@@ -38,7 +37,6 @@ type TagStore struct {
 	pushingPool     map[string]chan struct{}
 	registryService *registry.Service
 	eventsService   *events.Events
-	trustService    *trust.TrustStore
 }
 
 type Repository map[string]string
@@ -66,7 +64,6 @@ type TagStoreConfig struct {
 	Key      libtrust.PrivateKey
 	Registry *registry.Service
 	Events   *events.Events
-	Trust    *trust.TrustStore
 }
 
 func NewTagStore(path string, cfg *TagStoreConfig) (*TagStore, error) {
@@ -84,7 +81,6 @@ func NewTagStore(path string, cfg *TagStoreConfig) (*TagStore, error) {
 		pushingPool:     make(map[string]chan struct{}),
 		registryService: cfg.Registry,
 		eventsService:   cfg.Events,
-		trustService:    cfg.Trust,
 	}
 	// Load the json file if it exists, otherwise create it.
 	if err := store.reload(); os.IsNotExist(err) {

graph/tags_unit_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	_ "github.com/docker/docker/daemon/graphdriver/vfs" // import the vfs driver so it is used in the tests
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/trust"
 	"github.com/docker/docker/utils"
 )
 
@@ -62,15 +61,9 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 
-	trust, err := trust.NewTrustStore(root + "/trust")
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	tagCfg := &TagStoreConfig{
 		Graph:  graph,
 		Events: events.New(),
-		Trust:  trust,
 	}
 	store, err := NewTagStore(path.Join(root, "tags"), tagCfg)
 	if err != nil {
@@ -81,7 +74,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 	img := &image.Image{ID: testOfficialImageID}
-	if err := graph.Register(img, officialArchive); err != nil {
+	if err := graph.Register(v1ImageDescriptor{img}, officialArchive); err != nil {
 		t.Fatal(err)
 	}
 	if err := store.Tag(testOfficialImageName, "", testOfficialImageID, false); err != nil {
@@ -92,7 +85,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 	img = &image.Image{ID: testPrivateImageID}
-	if err := graph.Register(img, privateArchive); err != nil {
+	if err := graph.Register(v1ImageDescriptor{img}, privateArchive); err != nil {
 		t.Fatal(err)
 	}
 	if err := store.Tag(testPrivateImageName, "", testPrivateImageID, false); err != nil {

hack/make/release-deb

@@ -20,21 +20,70 @@ APTDIR=$DOCKER_RELEASE_DIR/apt/repo
 # setup the apt repo (if it does not exist)
 mkdir -p "$APTDIR/conf" "$APTDIR/db"
 
+# supported arches/sections
+arches=( amd64 i386 )
+components=( main testing experimental )
+
 # create/update distributions file
-if [[ ! -f "$APTDIR/conf/distributions" ]]; then
+if [ ! -f "$APTDIR/conf/distributions" ]; then
 	for suite in $(exec contrib/reprepro/suites.sh); do
 		cat <<-EOF
 		Origin: Docker
 		Suite: $suite
 		Codename: $suite
-		Architectures: amd64 i386
+		Architectures: ${arches[*]}
-		Components: main testing experimental
+		Components: ${components[*]}
 		Description: Docker APT Repository
 
 		EOF
 	done > "$APTDIR/conf/distributions"
 fi
 
+# create/update distributions file
+if [ ! -f "$APTDIR/conf/apt-ftparchive.conf" ]; then
+	cat <<-EOF > "$APTDIR/conf/apt-ftparchive.conf"
+	Dir {
+		ArchiveDir "${APTDIR}";
+		CacheDir "${APTDIR}/db";
+	};
+
+	Default {
+		Packages::Compress ". gzip bzip2";
+		Sources::Compress ". gzip bzip2";
+		Contents::Compress ". gzip bzip2";
+	};
+
+	TreeDefault {
+		BinCacheDB "packages-\$(SECTION)-\$(ARCH).db";
+		Directory "pool/\$(SECTION)";
+		Packages "\$(DIST)/\$(SECTION)/binary-\$(ARCH)/Packages";
+		SrcDirectory "pool/\$(SECTION)";
+		Sources "\$(DIST)/\$(SECTION)/source/Sources";
+		Contents "\$(DIST)/\$(SECTION)/Contents-\$(ARCH)";
+		FileList "$APTDIR/\$(DIST)/\$(SECTION)/filelist";
+	};
+	EOF
+
+	for suite in $(exec contrib/reprepro/suites.sh); do
+		cat <<-EOF
+		Tree "dists/${suite}" {
+			Sections "main testing experimental";
+			Architectures "${arches[*]}";
+		}
+
+		EOF
+	done >> "$APTDIR/conf/apt-ftparchive.conf"
+fi
+
+if [ ! -f "$APTDIR/conf/docker-engine-release.conf" ]; then
+	cat <<-EOF > "$APTDIR/conf/docker-engine-release.conf"
+	APT::FTPArchive::Release::Origin "Docker";
+	APT::FTPArchive::Release::Components "${components[*]}";
+	APT::FTPArchive::Release::Label "Docker APT Repository";
+	APT::FTPArchive::Release::Architectures "${arches[*]}";
+	EOF
+fi
+
 # set the component and priority for the version being released
 component="main"
 priority=700
@@ -67,4 +116,35 @@ for dir in contrib/builder/deb/*/; do
 	reprepro -v --keepunreferencedfiles \
 		-S docker-engine -P "$priority" -C "$component" \
 		-b "$APTDIR" includedeb "$codename" "${DEBFILE[@]}"
+
+	# update the filelist for this codename/component
+	find "$APTDIR/pool/$component" \
+		-name *~${codename#*-}*.deb > "$APTDIR/dists/$codename/$component/filelist"
+done
+
+
+# run the apt-ftparchive commands so we can have pinning
+apt-ftparchive generate "$APTDIR/conf/apt-ftparchive.conf"
+
+for dir in contrib/builder/deb/*/; do
+	version="$(basename "$dir")"
+	codename="${version//debootstrap-}"	
+
+	apt-ftparchive \
+		-o "APT::FTPArchive::Release::Codename=$codename" \
+		-o "APT::FTPArchive::Release::Suite=$codename" \
+		-c "$APTDIR/conf/docker-engine-release.conf" \
+		release \
+		"$APTDIR/dists/$codename" > "$APTDIR/dists/$codename/Release"
+
+	for arch in "${arches[@]}"; do
+		apt-ftparchive \
+			-o "APT::FTPArchive::Release::Codename=$codename" \
+			-o "APT::FTPArchive::Release::Suite=$codename" \
+			-o "APT::FTPArchive::Release::Component=$component" \
+			-o "APT::FTPArchive::Release::Architecture=$arch" \
+			-c "$APTDIR/conf/docker-engine-release.conf" \
+			release \
+			"$APTDIR/dists/$codename/$component/binary-$arch" > "$APTDIR/dists/$codename/$component/binary-$arch/Release"
+	done
 done

hack/vendor.sh

@@ -21,29 +21,29 @@ clone git golang.org/x/net 3cffabab72adf04f8e3b01c5baf775361837b5fe https://gith
 clone hg code.google.com/p/gosqlite 74691fb6f837
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork bd3eecc96f3c05a4acef1bedcf74397bc6850d22
+clone git github.com/docker/libnetwork bc565c2d295067c1a43674a23a473ec6336d7fd4
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
 clone git github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b
 clone git github.com/hashicorp/memberlist 9a1e242e454d2443df330bdd51a436d5a9058fc4
 clone git github.com/hashicorp/serf 7151adcef72687bf95f451a2e0ba15cb19412bf2
 clone git github.com/docker/libkv 60c7c881345b3c67defc7f93a8297debf041d43c
 clone git github.com/vishvananda/netns 493029407eeb434d0c2d44e02ea072ff2488d322
-clone git github.com/vishvananda/netlink 20397a138846e4d6590e01783ed023ed7e1c38a6
+clone git github.com/vishvananda/netlink 4b5dce31de6d42af5bb9811c6d265472199e0fec
 clone git github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060
 clone git github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374
 clone git github.com/coreos/go-etcd v2.0.0
 clone git github.com/hashicorp/consul v0.5.2
 
 # get graph and distribution packages
-clone git github.com/docker/distribution 7dc8d4a26b689bd4892f2f2322dbce0b7119d686
+clone git github.com/docker/distribution ec87e9b6971d831f0eff752ddb54fb64693e51cd # docker/1.8 branch
-clone git github.com/vbatts/tar-split v0.9.4
+clone git github.com/vbatts/tar-split v0.9.6
 
 clone git github.com/docker/notary 8e8122eb5528f621afcd4e2854c47302f17392f7
 clone git github.com/endophage/gotuf a592b03b28b02bb29bb5878308fb1abed63383b5
 clone git github.com/tent/canonical-json-go 96e4ba3a7613a1216cbd1badca4efe382adea337
 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c
 
-clone git github.com/opencontainers/runc v0.0.2 # libcontainer
+clone git github.com/opencontainers/runc v0.0.2.1 # libcontainer
 # libcontainer deps (see src/github.com/docker/libcontainer/update-vendor.sh)
 clone git github.com/coreos/go-systemd v2
 clone git github.com/godbus/dbus v2

image/fixtures/post1.9/expected_computed_id

@@ -0,0 +1 @@
+sha256:f2722a8ec6926e02fa9f2674072cbc2a25cf0f449f27350f613cd843b02c9105

image/fixtures/post1.9/expected_config

@@ -0,0 +1 @@
+{"architecture":"amd64","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2015-09-08T21:30:30.807853054Z","docker_version":"1.9.0-dev","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

image/fixtures/post1.9/layer_id

@@ -0,0 +1 @@
+sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/post1.9/parent_id

@@ -0,0 +1 @@
+sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/post1.9/v1compatibility

@@ -0,0 +1 @@
+{"id":"8dfb96b5d09e6cf6f376d81f1e2770ee5ede309f9bd9e079688c9782649ab326","parent":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","created":"2015-09-08T21:30:30.807853054Z","container":"fb1f7270da9519308361b99dc8e0d30f12c24dfd28537c2337ece995ac853a16","container_config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":["/bin/sh","-c","#(nop) ADD file:11998b2a4d664a75cd0c3f4e4cb1837434e0f997ba157a0ac1d3c68a07aa2f4f in /"],"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"docker_version":"1.9.0-dev","config":{"Hostname":"fb1f7270da95","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["foo=bar"],"Cmd":null,"Image":"361a94d06b2b781b2f1ee6c72e1cbbfbbd032a103e26a3db75b431743829ae4f","Volumes":null,"VolumeDriver":"","WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"architecture":"amd64","os":"linux"}

image/fixtures/pre1.9/expected_computed_id

@@ -0,0 +1 @@
+sha256:fd6ebfedda8ea140a9380767e15bd32c6e899303cfe34bc4580c931f2f816f89

image/fixtures/pre1.9/expected_config

@@ -0,0 +1 @@
+{"architecture":"amd64","config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"OnBuild":[],"Labels":{}},"created":"2015-08-19T16:49:11.368300679Z","docker_version":"1.6.2","layer_id":"sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a","os":"linux","parent_id":"sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02"}

image/fixtures/pre1.9/layer_id

@@ -0,0 +1 @@
+sha256:31176893850e05d308cdbfef88877e460d50c8063883fb13eb5753097da6422a

image/fixtures/pre1.9/parent_id

@@ -0,0 +1 @@
+sha256:ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02

image/fixtures/pre1.9/v1compatibility

@@ -0,0 +1 @@
+{"id":"3b38edc92eb7c074812e217b41a6ade66888531009d6286a6f5f36a06f9841b9","parent":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","created":"2015-08-19T16:49:11.368300679Z","container":"d91be3479d5b1e84b0c00d18eea9dc777ca0ad166d51174b24283e2e6f104253","container_config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":["/bin/sh","-c","#(nop) ENTRYPOINT [\"/go/bin/dnsdock\"]"],"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"docker_version":"1.6.2","config":{"Hostname":"03797203757d","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"Cpuset":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/go/bin:/usr/src/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin","GOLANG_VERSION=1.4.1","GOPATH=/go"],"Cmd":null,"Image":"ec3025ca8cc9bcab039e193e20ec647c2da3c53a74020f2ba611601f9b2c6c02","Volumes":null,"WorkingDir":"/go","Entrypoint":["/go/bin/dnsdock"],"NetworkDisabled":false,"MacAddress":"","OnBuild":[],"Labels":{}},"architecture":"amd64","os":"linux","Size":0}

image/image.go

@@ -6,24 +6,59 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/docker/pkg/version"
 	"github.com/docker/docker/runconfig"
 )
 
 var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
 
+// noFallbackMinVersion is the minimum version for which v1compatibility
+// information will not be marshaled through the Image struct to remove
+// blank fields.
+var noFallbackMinVersion = version.Version("1.8.3")
+
+// ImageDescriptor provides the information necessary to register an image in
+// the graph.
+type ImageDescriptor interface {
+	ID() string
+	Parent() string
+	MarshalConfig() ([]byte, error)
+}
+
+// Image stores the image configuration.
+// All fields in this struct must be marked `omitempty` to keep getting
+// predictable hashes from the old `v1Compatibility` configuration.
 type Image struct {
-	ID              string            `json:"id"`
+	// ID a unique 64 character identifier of the image
-	Parent          string            `json:"parent,omitempty"`
+	ID string `json:"id,omitempty"`
-	Comment         string            `json:"comment,omitempty"`
+	// Parent id of the image
-	Created         time.Time         `json:"created"`
+	Parent string `json:"parent,omitempty"`
-	Container       string            `json:"container,omitempty"`
+	// Comment user added comment
-	ContainerConfig runconfig.Config  `json:"container_config,omitempty"`
+	Comment string `json:"comment,omitempty"`
-	DockerVersion   string            `json:"docker_version,omitempty"`
+	// Created timestamp when image was created
-	Author          string            `json:"author,omitempty"`
+	Created time.Time `json:"created"`
-	Config          *runconfig.Config `json:"config,omitempty"`
+	// Container is the id of the container used to commit
-	Architecture    string            `json:"architecture,omitempty"`
+	Container string `json:"container,omitempty"`
-	OS              string            `json:"os,omitempty"`
+	// ContainerConfig  is the configuration of the container that is committed into the image
-	Size            int64
+	ContainerConfig runconfig.Config `json:"container_config,omitempty"`
+	// DockerVersion specifies version on which image is built
+	DockerVersion string `json:"docker_version,omitempty"`
+	// Author of the image
+	Author string `json:"author,omitempty"`
+	// Config is the configuration of the container received from the client
+	Config *runconfig.Config `json:"config,omitempty"`
+	// Architecture is the hardware that the image is build and runs on
+	Architecture string `json:"architecture,omitempty"`
+	// OS is the operating system used to build and run the image
+	OS string `json:"os,omitempty"`
+	// Size is the total size of the image including all layers it is composed of
+	Size int64 `json:",omitempty"` // capitalized for backwards compatibility
+	// ParentID specifies the strong, content address of the parent configuration.
+	ParentID digest.Digest `json:"parent_id,omitempty"`
+	// LayerID provides the content address of the associated layer.
+	LayerID digest.Digest `json:"layer_id,omitempty"`
 }
 
 // Build an Image object from raw json data
@@ -44,3 +79,70 @@ func ValidateID(id string) error {
 	}
 	return nil
 }
+
+// MakeImageConfig returns immutable configuration JSON for image based on the
+// v1Compatibility object, layer digest and parent StrongID. SHA256() of this
+// config is the new image ID (strongID).
+func MakeImageConfig(v1Compatibility []byte, layerID, parentID digest.Digest) ([]byte, error) {
+
+	// Detect images created after 1.8.3
+	img, err := NewImgJSON(v1Compatibility)
+	if err != nil {
+		return nil, err
+	}
+	useFallback := version.Version(img.DockerVersion).LessThan(noFallbackMinVersion)
+
+	if useFallback {
+		// Fallback for pre-1.8.3. Calculate base config based on Image struct
+		// so that fields with default values added by Docker will use same ID
+		logrus.Debugf("Using fallback hash for %v", layerID)
+
+		v1Compatibility, err = json.Marshal(img)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var c map[string]*json.RawMessage
+	if err := json.Unmarshal(v1Compatibility, &c); err != nil {
+		return nil, err
+	}
+
+	if err := layerID.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid layerID: %v", err)
+	}
+
+	c["layer_id"] = rawJSON(layerID)
+
+	if parentID != "" {
+		if err := parentID.Validate(); err != nil {
+			return nil, fmt.Errorf("invalid parentID %v", err)
+		}
+		c["parent_id"] = rawJSON(parentID)
+	}
+
+	delete(c, "id")
+	delete(c, "parent")
+	delete(c, "Size") // Size is calculated from data on disk and is inconsitent
+
+	return json.Marshal(c)
+}
+
+// StrongID returns image ID for the config JSON.
+func StrongID(configJSON []byte) (digest.Digest, error) {
+	digester := digest.Canonical.New()
+	if _, err := digester.Hash().Write(configJSON); err != nil {
+		return "", err
+	}
+	dgst := digester.Digest()
+	logrus.Debugf("H(%v) = %v", string(configJSON), dgst)
+	return dgst, nil
+}
+
+func rawJSON(value interface{}) *json.RawMessage {
+	jsonval, err := json.Marshal(value)
+	if err != nil {
+		return nil
+	}
+	return (*json.RawMessage)(&jsonval)
+}

image/image_test.go

@@ -0,0 +1,55 @@
+package image
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+var fixtures = []string{
+	"fixtures/pre1.9",
+	"fixtures/post1.9",
+}
+
+func loadFixtureFile(t *testing.T, path string) []byte {
+	fileData, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Fatalf("error opening %s: %v", path, err)
+	}
+
+	return bytes.TrimSpace(fileData)
+}
+
+// TestMakeImageConfig makes sure that MakeImageConfig returns the expected
+// canonical JSON for a reference Image.
+func TestMakeImageConfig(t *testing.T) {
+	for _, fixture := range fixtures {
+		v1Compatibility := loadFixtureFile(t, fixture+"/v1compatibility")
+		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
+		layerID := digest.Digest(loadFixtureFile(t, fixture+"/layer_id"))
+		parentID := digest.Digest(loadFixtureFile(t, fixture+"/parent_id"))
+
+		json, err := MakeImageConfig(v1Compatibility, layerID, parentID)
+		if err != nil {
+			t.Fatalf("MakeImageConfig on %s returned error: %v", fixture, err)
+		}
+		if !bytes.Equal(json, expectedConfig) {
+			t.Fatalf("did not get expected JSON for %s\nexpected: %s\ngot: %s", fixture, expectedConfig, json)
+		}
+	}
+}
+
+// TestGetStrongID makes sure that GetConfigJSON returns the expected
+// hash for a reference Image.
+func TestGetStrongID(t *testing.T) {
+	for _, fixture := range fixtures {
+		expectedConfig := loadFixtureFile(t, fixture+"/expected_config")
+		expectedComputedID := digest.Digest(loadFixtureFile(t, fixture+"/expected_computed_id"))
+
+		if id, err := StrongID(expectedConfig); err != nil || id != expectedComputedID {
+			t.Fatalf("did not get expected ID for %s\nexpected: %s\ngot: %s\nerror: %v", fixture, expectedComputedID, id, err)
+		}
+	}
+}

integration-cli/check_test.go

@@ -31,15 +31,24 @@ func init() {
 type DockerRegistrySuite struct {
 	ds  *DockerSuite
 	reg *testRegistryV2
+	d   *Daemon
 }
 
 func (s *DockerRegistrySuite) SetUpTest(c *check.C) {
 	s.reg = setupRegistry(c)
+	s.d = NewDaemon(c)
 }
 
 func (s *DockerRegistrySuite) TearDownTest(c *check.C) {
 	s.reg.Close()
 	s.ds.TearDownTest(c)
+	if s.reg != nil {
+		s.reg.Close()
+	}
+	if s.ds != nil {
+		s.ds.TearDownTest(c)
+	}
+	s.d.Stop()
 }
 
 func init() {

integration-cli/docker_cli_ps_test.go

@@ -2,7 +2,10 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -554,3 +557,23 @@ func (s *DockerSuite) TestPsFormatHeaders(c *check.C) {
 		c.Fatalf(`Expected 'NAMES\ntest\n', got %v`, out)
 	}
 }
+
+func (s *DockerSuite) TestPsDefaultFormatAndQuiet(c *check.C) {
+	config := `{
+		"psFormat": "{{ .ID }} default"
+}`
+	d, err := ioutil.TempDir("", "integration-cli-")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(d)
+
+	err = ioutil.WriteFile(filepath.Join(d, "config.json"), []byte(config), 0644)
+	c.Assert(err, check.IsNil)
+
+	out, _ := dockerCmd(c, "run", "--name=test", "-d", "busybox", "top")
+	id := strings.TrimSpace(out)
+
+	out, _ = dockerCmd(c, "--config", d, "ps", "-q")
+	if !strings.HasPrefix(id, strings.TrimSpace(out)) {
+		c.Fatalf("Expected to print only the container id, got %v\n", out)
+	}
+}

integration-cli/docker_cli_pull_test.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"io/ioutil"
+	"github.com/docker/distribution/digest"
-
 	"github.com/go-check/check"
 )
 
@@ -369,3 +372,291 @@ func (s *DockerTrustSuite) TestTrustedPullWithExpiredSnapshot(c *check.C) {
 		}
 	})
 }
+
+// Test that pull continues after client has disconnected. #15589
+func (s *DockerTrustSuite) TestPullClientDisconnect(c *check.C) {
+	testRequires(c, Network)
+
+	repoName := "hello-world:latest"
+
+	dockerCmdWithError(c, "rmi", repoName) // clean just in case
+
+	pullCmd := exec.Command(dockerBinary, "pull", repoName)
+
+	stdout, err := pullCmd.StdoutPipe()
+	c.Assert(err, check.IsNil)
+
+	err = pullCmd.Start()
+	c.Assert(err, check.IsNil)
+
+	// cancel as soon as we get some output
+	buf := make([]byte, 10)
+	_, err = stdout.Read(buf)
+	c.Assert(err, check.IsNil)
+
+	err = pullCmd.Process.Kill()
+	c.Assert(err, check.IsNil)
+
+	maxAttempts := 20
+	for i := 0; ; i++ {
+		if _, _, err := dockerCmdWithError(c, "inspect", repoName); err == nil {
+			break
+		}
+		if i >= maxAttempts {
+			c.Fatal("Timeout reached. Image was not pulled after client disconnected.")
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+}
+
+type idAndParent struct {
+	ID     string
+	Parent string
+}
+
+func inspectImage(c *check.C, imageRef string) idAndParent {
+	out, _ := dockerCmd(c, "inspect", imageRef)
+	var inspectOutput []idAndParent
+	err := json.Unmarshal([]byte(out), &inspectOutput)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	return inspectOutput[0]
+}
+
+func imageID(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).ID
+}
+
+func imageParent(c *check.C, imageRef string) string {
+	return inspectImage(c, imageRef).Parent
+}
+
+// TestPullMigration verifies that pulling an image based on layers
+// that already exists locally will reuse those existing layers.
+func (s *DockerRegistrySuite) TestPullMigration(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/migration"
+
+	baseImage := repoName + ":base"
+	_, err := buildImage(baseImage, fmt.Sprintf(`
+	    FROM scratch
+	    ENV IMAGE base
+	    CMD echo %s
+	`, baseImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	baseParentBeforePush := imageParent(c, baseImage)
+
+	derivedImage := repoName + ":derived"
+	_, err = buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that the parent of this pulled image is the original base
+	// image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, derivedImage)
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Remove everything, repull, and make sure everything uses computed IDs
+	dockerCmd(c, "rmi", baseImage, derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull3 := imageID(c, derivedImage)
+	derivedParentAfterPull3 := imageParent(c, derivedImage)
+	derivedGrandparentAfterPull3 := imageParent(c, derivedParentAfterPull3)
+
+	if derivedIDAfterPull3 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a second repull")
+	}
+
+	if derivedParentAfterPull3 == baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) should not match base image's original ID (%s)", derivedParentAfterPull3, derivedIDBeforePush)
+	}
+
+	if derivedGrandparentAfterPull3 == baseParentBeforePush {
+		c.Fatal("base image's parent ID should have been rewritten on pull")
+	}
+}
+
+// TestPullMigrationRun verifies that pulling an image based on layers
+// that already exists locally will result in an image that runs properly.
+func (s *DockerRegistrySuite) TestPullMigrationRun(c *check.C) {
+	type idAndParent struct {
+		ID     string
+		Parent string
+	}
+
+	derivedImage := privateRegistryURL + "/dockercli/migration-run"
+	baseImage := "busybox"
+
+	_, err := buildImage(derivedImage, fmt.Sprintf(`
+	    FROM %s
+	    RUN dd if=/dev/zero of=/file bs=1024 count=1024
+	    CMD echo %s
+	`, baseImage, derivedImage), true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	baseIDBeforePush := imageID(c, baseImage)
+	derivedIDBeforePush := imageID(c, derivedImage)
+
+	dockerCmd(c, "push", derivedImage)
+
+	// Remove derived image from the local store
+	dockerCmd(c, "rmi", derivedImage)
+
+	// Repull
+	dockerCmd(c, "pull", derivedImage)
+
+	// Check that this pulled image is based on the original base image
+	derivedIDAfterPull1 := imageID(c, derivedImage)
+	derivedParentAfterPull1 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull1 == derivedIDBeforePush {
+		c.Fatal("image's ID should have changed on after deleting and pulling")
+	}
+
+	if derivedParentAfterPull1 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull1, baseIDBeforePush)
+	}
+
+	// Make sure the image runs correctly
+	out, _ := dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+
+	// Confirm that repushing and repulling does not change the computed ID
+	dockerCmd(c, "push", derivedImage)
+	dockerCmd(c, "rmi", derivedImage)
+	dockerCmd(c, "pull", derivedImage)
+
+	derivedIDAfterPull2 := imageID(c, derivedImage)
+	derivedParentAfterPull2 := imageParent(c, imageParent(c, derivedImage))
+
+	if derivedIDAfterPull2 != derivedIDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repush/repull")
+	}
+
+	if derivedParentAfterPull2 != baseIDBeforePush {
+		c.Fatalf("pulled image's parent ID (%s) does not match base image's ID (%s)", derivedParentAfterPull2, baseIDBeforePush)
+	}
+
+	// Make sure the image still runs
+	out, _ = dockerCmd(c, "run", "--rm", derivedImage)
+	if strings.TrimSpace(out) != derivedImage {
+		c.Fatalf("expected %s; got %s", derivedImage, out)
+	}
+}
+
+// TestPullConflict provides coverage of the situation where a computed
+// strongID conflicts with some unverifiable data in the graph.
+func (s *DockerRegistrySuite) TestPullConflict(c *check.C) {
+	repoName := privateRegistryURL + "/dockercli/conflict"
+
+	_, err := buildImage(repoName, `
+	    FROM scratch
+	    ENV IMAGE conflict
+	    CMD echo conflict
+	`, true)
+	if err != nil {
+		c.Fatal(err)
+	}
+
+	dockerCmd(c, "push", repoName)
+
+	// Pull to make it content-addressable
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "pull", repoName)
+
+	IDBeforeLoad := imageID(c, repoName)
+
+	// Load/save to turn this into an unverified image with the same ID
+	tmpDir, err := ioutil.TempDir("", "conflict-save-output")
+	if err != nil {
+		c.Errorf("failed to create temporary directory: %s", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tarFile := filepath.Join(tmpDir, "repo.tar")
+
+	dockerCmd(c, "save", "-o", tarFile, repoName)
+	dockerCmd(c, "rmi", repoName)
+	dockerCmd(c, "load", "-i", tarFile)
+
+	// Check that the the ID is the same after save/load.
+	IDAfterLoad := imageID(c, repoName)
+
+	if IDAfterLoad != IDBeforeLoad {
+		c.Fatal("image's ID should be the same after save/load")
+	}
+
+	// Repull
+	dockerCmd(c, "pull", repoName)
+
+	// Check that the ID is now different because of the conflict.
+	IDAfterPull1 := imageID(c, repoName)
+
+	// Expect the new ID to be SHA256(oldID)
+	expectedIDDigest, err := digest.FromBytes([]byte(IDBeforeLoad))
+	if err != nil {
+		c.Fatalf("digest error: %v", err)
+	}
+	expectedID := expectedIDDigest.Hex()
+	if IDAfterPull1 != expectedID {
+		c.Fatalf("image's ID should have changed on pull to %s (got %s)", expectedID, IDAfterPull1)
+	}
+
+	// A second pull should use the new ID again.
+	dockerCmd(c, "pull", repoName)
+
+	IDAfterPull2 := imageID(c, repoName)
+
+	if IDAfterPull2 != IDAfterPull1 {
+		c.Fatal("image's ID unexpectedly changed after a repull")
+	}
+}

integration-cli/docker_cli_push_test.go

@@ -60,31 +60,40 @@ func (s *DockerRegistrySuite) TestPushMultipleTags(c *check.C) {
 
 	dockerCmd(c, "tag", "busybox", repoTag2)
 
-	out, _ := dockerCmd(c, "push", repoName)
+	dockerCmd(c, "push", repoName)
 
-	// There should be no duplicate hashes in the output
+	// Ensure layer list is equivalent for repoTag1 and repoTag2
-	imageSuccessfullyPushed := ": Image successfully pushed"
+	out1, _ := dockerCmd(c, "pull", repoTag1)
+	if strings.Contains(out1, "Tag t1 not found") {
+		c.Fatalf("Unable to pull pushed image: %s", out1)
+	}
 	imageAlreadyExists := ": Image already exists"
-	imagePushHashes := make(map[string]struct{})
+	var out1Lines []string
-	outputLines := strings.Split(out, "\n")
+	for _, outputLine := range strings.Split(out1, "\n") {
-	for _, outputLine := range outputLines {
+		if strings.Contains(outputLine, imageAlreadyExists) {
-		if strings.Contains(outputLine, imageSuccessfullyPushed) {
+			out1Lines = append(out1Lines, outputLine)
-			hash := strings.TrimSuffix(outputLine, imageSuccessfullyPushed)
-			if _, present := imagePushHashes[hash]; present {
-				c.Fatalf("Duplicate image push: %s", outputLine)
-			}
-			imagePushHashes[hash] = struct{}{}
-		} else if strings.Contains(outputLine, imageAlreadyExists) {
-			hash := strings.TrimSuffix(outputLine, imageAlreadyExists)
-			if _, present := imagePushHashes[hash]; present {
-				c.Fatalf("Duplicate image push: %s", outputLine)
-			}
-			imagePushHashes[hash] = struct{}{}
 		}
 	}
 
-	if len(imagePushHashes) == 0 {
+	out2, _ := dockerCmd(c, "pull", repoTag2)
-		c.Fatal(`Expected at least one line containing "Image successfully pushed"`)
+	if strings.Contains(out2, "Tag t2 not found") {
+		c.Fatalf("Unable to pull pushed image: %s", out1)
+	}
+	var out2Lines []string
+	for _, outputLine := range strings.Split(out2, "\n") {
+		if strings.Contains(outputLine, imageAlreadyExists) {
+			out1Lines = append(out1Lines, outputLine)
+		}
+	}
+
+	if len(out1Lines) != len(out2Lines) {
+		c.Fatalf("Mismatched output length:\n%s\n%s", out1, out2)
+	}
+
+	for i := range out1Lines {
+		if out1Lines[i] != out2Lines[i] {
+			c.Fatalf("Mismatched output line:\n%s\n%s", out1Lines[i], out2Lines[i])
+		}
 	}
 }
 

integration-cli/docker_cli_v2_only.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/go-check/check"
+)
+
+func makefile(contents string) (string, func(), error) {
+	cleanup := func() {
+
+	}
+
+	f, err := ioutil.TempFile(".", "tmp")
+	if err != nil {
+		return "", cleanup, err
+	}
+	err = ioutil.WriteFile(f.Name(), []byte(contents), os.ModePerm)
+	if err != nil {
+		return "", cleanup, err
+	}
+
+	cleanup = func() {
+		err := os.Remove(f.Name())
+		if err != nil {
+			fmt.Println("Error removing tmpfile")
+		}
+	}
+	return f.Name(), cleanup, nil
+
+}
+
+// TestV2Only ensures that a daemon in v2-only mode does not
+// attempt to contact any v1 registry endpoints.
+func (s *DockerRegistrySuite) TestV2Only(c *check.C) {
+	reg, err := newTestRegistry(c)
+	if err != nil {
+		c.Fatal(err.Error())
+	}
+
+	reg.registerHandler("/v2/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(404)
+	})
+
+	reg.registerHandler("/v1/.*", func(w http.ResponseWriter, r *http.Request) {
+		c.Fatal("V1 registry contacted")
+	})
+
+	repoName := fmt.Sprintf("%s/busybox", reg.hostport)
+
+	err = s.d.Start("--insecure-registry", reg.hostport, "--disable-legacy-registry=true")
+	if err != nil {
+		c.Fatalf("Error starting daemon: %s", err.Error())
+	}
+
+	dockerfileName, cleanup, err := makefile(fmt.Sprintf("FROM %s/busybox", reg.hostport))
+	if err != nil {
+		c.Fatalf("Unable to create test dockerfile")
+	}
+	defer cleanup()
+
+	s.d.Cmd("build", "--file", dockerfileName, ".")
+
+	s.d.Cmd("run", repoName)
+	s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport)
+	s.d.Cmd("tag", "busybox", repoName)
+	s.d.Cmd("push", repoName)
+	s.d.Cmd("pull", repoName)
+}
+
+// TestV1 starts a daemon in 'normal' mode
+// and ensure v1 endpoints are hit for the following operations:
+// login, push, pull, build & run
+func (s *DockerRegistrySuite) TestV1(c *check.C) {
+	reg, err := newTestRegistry(c)
+	if err != nil {
+		c.Fatal(err.Error())
+	}
+
+	v2Pings := 0
+	reg.registerHandler("/v2/", func(w http.ResponseWriter, r *http.Request) {
+		v2Pings++
+		// V2 ping 404 causes fallback to v1
+		w.WriteHeader(404)
+	})
+
+	v1Pings := 0
+	reg.registerHandler("/v1/_ping", func(w http.ResponseWriter, r *http.Request) {
+		v1Pings++
+	})
+
+	v1Logins := 0
+	reg.registerHandler("/v1/users/", func(w http.ResponseWriter, r *http.Request) {
+		v1Logins++
+	})
+
+	v1Repo := 0
+	reg.registerHandler("/v1/repositories/busybox/", func(w http.ResponseWriter, r *http.Request) {
+		v1Repo++
+	})
+
+	reg.registerHandler("/v1/repositories/busybox/images", func(w http.ResponseWriter, r *http.Request) {
+		v1Repo++
+	})
+
+	err = s.d.Start("--insecure-registry", reg.hostport, "--disable-legacy-registry=false")
+	if err != nil {
+		c.Fatalf("Error starting daemon: %s", err.Error())
+	}
+
+	dockerfileName, cleanup, err := makefile(fmt.Sprintf("FROM %s/busybox", reg.hostport))
+	if err != nil {
+		c.Fatalf("Unable to create test dockerfile")
+	}
+	defer cleanup()
+
+	s.d.Cmd("build", "--file", dockerfileName, ".")
+	if v1Repo == 0 {
+		c.Errorf("Expected v1 repository access after build")
+	}
+
+	repoName := fmt.Sprintf("%s/busybox", reg.hostport)
+	s.d.Cmd("run", repoName)
+	if v1Repo == 1 {
+		c.Errorf("Expected v1 repository access after run")
+	}
+
+	s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", reg.hostport)
+	if v1Logins == 0 {
+		c.Errorf("Expected v1 login attempt")
+	}
+
+	s.d.Cmd("tag", "busybox", repoName)
+	s.d.Cmd("push", repoName)
+
+	if v1Repo != 2 || v1Pings != 1 {
+		c.Error("Not all endpoints contacted after push", v1Repo, v1Pings)
+	}
+
+	s.d.Cmd("pull", repoName)
+	if v1Repo != 3 {
+		c.Errorf("Expected v1 repository access after pull")
+	}
+
+}

integration-cli/registry_mock.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"regexp"
+	"strings"
+	"sync"
+
+	"github.com/go-check/check"
+)
+
+type handlerFunc func(w http.ResponseWriter, r *http.Request)
+
+type testRegistry struct {
+	server   *httptest.Server
+	hostport string
+	handlers map[string]handlerFunc
+	mu       sync.Mutex
+}
+
+func (tr *testRegistry) registerHandler(path string, h handlerFunc) {
+	tr.mu.Lock()
+	defer tr.mu.Unlock()
+	tr.handlers[path] = h
+}
+
+func newTestRegistry(c *check.C) (*testRegistry, error) {
+	testReg := &testRegistry{handlers: make(map[string]handlerFunc)}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		url := r.URL.String()
+
+		var matched bool
+		var err error
+		for re, function := range testReg.handlers {
+			matched, err = regexp.MatchString(re, url)
+			if err != nil {
+				c.Fatalf("Error with handler regexp")
+				return
+			}
+			if matched {
+				function(w, r)
+				break
+			}
+		}
+
+		if !matched {
+			c.Fatal("Unable to match", url, "with regexp")
+		}
+	}))
+
+	testReg.server = ts
+	testReg.hostport = strings.Replace(ts.URL, "http://", "", 1)
+	return testReg, nil
+}

man/docker.1.md

@@ -23,7 +23,7 @@ its own man page which explain usage and arguments.
 To see the man page for a command run **man docker <command>**.
 
 # OPTIONS
-**-h**, **--help**
+**--help**
   Print usage statement
 
 **--api-cors-header**=""
@@ -53,6 +53,9 @@ To see the man page for a command run **man docker <command>**.
 **--default-ulimit**=[]
   Set default ulimits for containers.
 
+**--disable-legacy-registry=**true|false
+  Do not contact legacy registries
+
 **--dns**=""
   Force Docker to use specific DNS servers
 
@@ -368,7 +371,7 @@ the size of images and containers. The default value is 100G. Note,
 thin devices are inherently "sparse", so a 100G device which is mostly
 empty doesn't use 100 GB of space on the pool. However, the filesystem
 will use more space for base images the larger the device
-is. 
+is.
 
 This value affects the system-wide "base" empty filesystem that may already
 be initialized and inherited by pulled images. Typically, a change to this
@@ -519,12 +522,12 @@ daemon with a supported environment.
 
 Use the **--exec-opt** flags to specify options to the exec-driver. The only
 driver that accepts this flag is the *native* (libcontainer) driver. As a
-result, you must also specify **-s=**native for this option to have effect. The 
+result, you must also specify **-s=**native for this option to have effect. The
 following is the only *native* option:
 
 #### native.cgroupdriver
-Specifies the management of the container's `cgroups`. You can specify 
+Specifies the management of the container's `cgroups`. You can specify
-`cgroupfs` or `systemd`. If you specify `systemd` and it is not available, the 
+`cgroupfs` or `systemd`. If you specify `systemd` and it is not available, the
 system uses `cgroupfs`.
 
 #### Client

pkg/devicemapper/devmapper_wrapper.go

@@ -38,7 +38,10 @@ static void	log_with_errno_init()
 */
 import "C"
 
-import "unsafe"
+import (
+	"reflect"
+	"unsafe"
+)
 
 type (
 	CDmTask C.struct_dm_task
@@ -184,12 +187,21 @@ func dmTaskGetDepsFct(task *CDmTask) *Deps {
 	if Cdeps == nil {
 		return nil
 	}
+
+	// golang issue: https://github.com/golang/go/issues/11925
+	hdr := reflect.SliceHeader{
+		Data: uintptr(unsafe.Pointer(uintptr(unsafe.Pointer(Cdeps)) + unsafe.Sizeof(*Cdeps))),
+		Len:  int(Cdeps.count),
+		Cap:  int(Cdeps.count),
+	}
+	devices := *(*[]C.uint64_t)(unsafe.Pointer(&hdr))
+
 	deps := &Deps{
 		Count:  uint32(Cdeps.count),
 		Filler: uint32(Cdeps.filler),
 	}
-	for _, device := range Cdeps.device {
+	for _, device := range devices {
-		deps.Device = append(deps.Device, (uint64)(device))
+		deps.Device = append(deps.Device, uint64(device))
 	}
 	return deps
 }

pkg/parsers/operatingsystem/operatingsystem_linux.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"io/ioutil"
+	"os"
 )
 
 var (
@@ -12,12 +13,22 @@ var (
 
 	// file to check to determine Operating System
 	etcOsRelease = "/etc/os-release"
+
+	// used by stateless systems like Clear Linux
+	altEtcOSRelease = "/usr/lib/os-release"
 )
 
 func GetOperatingSystem() (string, error) {
 	b, err := ioutil.ReadFile(etcOsRelease)
 	if err != nil {
-		return "", err
+		if _, err2 := os.Stat(altEtcOSRelease); err2 == nil {
+			b, err2 = ioutil.ReadFile(altEtcOSRelease)
+			if err2 != nil {
+				return "", err2
+			}
+		} else {
+			return "", err
+		}
 	}
 	if i := bytes.Index(b, []byte("PRETTY_NAME")); i >= 0 {
 		b = b[i+13:]

registry/config.go

@@ -48,6 +48,10 @@ var (
 	ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")")
 
 	emptyServiceConfig = NewServiceConfig(nil)
+
+	// V2Only controls access to legacy registries.  If it is set to true via the
+	// command line flag the daemon will not attempt to contact v1 legacy registries
+	V2Only = false
 )
 
 // InstallFlags adds command-line options to the top-level flag parser for
@@ -57,6 +61,7 @@ func (options *Options) InstallFlags(cmd *flag.FlagSet, usageFn func(string) str
 	cmd.Var(&options.Mirrors, []string{"-registry-mirror"}, usageFn("Preferred Docker registry mirror"))
 	options.InsecureRegistries = opts.NewListOpts(ValidateIndexName)
 	cmd.Var(&options.InsecureRegistries, []string{"-insecure-registry"}, usageFn("Enable insecure registry communication"))
+	cmd.BoolVar(&V2Only, []string{"-disable-legacy-registry"}, false, "Do not contact legacy registries")
 }
 
 type netIPNet net.IPNet

registry/endpoint.go

@@ -42,8 +42,9 @@ func scanForAPIVersion(address string) (string, APIVersion) {
 	return address, APIVersionUnknown
 }
 
-// NewEndpoint parses the given address to return a registry endpoint.
+// NewEndpoint parses the given address to return a registry endpoint.  v can be used to
-func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) {
+// specify a specific endpoint version
+func NewEndpoint(index *IndexInfo, metaHeaders http.Header, v APIVersion) (*Endpoint, error) {
 	tlsConfig, err := newTLSConfig(index.Name, index.Secure)
 	if err != nil {
 		return nil, err
@@ -52,6 +53,9 @@ func NewEndpoint(index *IndexInfo, metaHeaders http.Header) (*Endpoint, error) {
 	if err != nil {
 		return nil, err
 	}
+	if v != APIVersionUnknown {
+		endpoint.Version = v
+	}
 	if err := validateEndpoint(endpoint); err != nil {
 		return nil, err
 	}
@@ -111,11 +115,6 @@ func newEndpoint(address string, tlsConfig *tls.Config, metaHeaders http.Header)
 	return endpoint, nil
 }
 
-// GetEndpoint returns a new endpoint with the specified headers
-func (repoInfo *RepositoryInfo) GetEndpoint(metaHeaders http.Header) (*Endpoint, error) {
-	return NewEndpoint(repoInfo.Index, metaHeaders)
-}
-
 // Endpoint stores basic information about a registry endpoint.
 type Endpoint struct {
 	client         *http.Client

registry/registry.go

@@ -48,6 +48,10 @@ func init() {
 	httpVersion = append(httpVersion, useragent.VersionInfo{"arch", runtime.GOARCH})
 
 	dockerUserAgent = useragent.AppendVersions("", httpVersion...)
+
+	if runtime.GOOS != "linux" {
+		V2Only = true
+	}
 }
 
 func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {

registry/registry_test.go

@@ -23,7 +23,7 @@ const (
 
 func spawnTestRegistrySession(t *testing.T) *Session {
 	authConfig := &cliconfig.AuthConfig{}
-	endpoint, err := NewEndpoint(makeIndex("/v1/"), nil)
+	endpoint, err := NewEndpoint(makeIndex("/v1/"), nil, APIVersionUnknown)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -50,7 +50,7 @@ func spawnTestRegistrySession(t *testing.T) *Session {
 
 func TestPingRegistryEndpoint(t *testing.T) {
 	testPing := func(index *IndexInfo, expectedStandalone bool, assertMessage string) {
-		ep, err := NewEndpoint(index, nil)
+		ep, err := NewEndpoint(index, nil, APIVersionUnknown)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -70,7 +70,7 @@ func TestPingRegistryEndpoint(t *testing.T) {
 func TestEndpoint(t *testing.T) {
 	// Simple wrapper to fail test if err != nil
 	expandEndpoint := func(index *IndexInfo) *Endpoint {
-		endpoint, err := NewEndpoint(index, nil)
+		endpoint, err := NewEndpoint(index, nil, APIVersionUnknown)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -79,7 +79,7 @@ func TestEndpoint(t *testing.T) {
 
 	assertInsecureIndex := func(index *IndexInfo) {
 		index.Secure = true
-		_, err := NewEndpoint(index, nil)
+		_, err := NewEndpoint(index, nil, APIVersionUnknown)
 		assertNotEqual(t, err, nil, index.Name+": Expected error for insecure index")
 		assertEqual(t, strings.Contains(err.Error(), "insecure-registry"), true, index.Name+": Expected insecure-registry  error for insecure index")
 		index.Secure = false
@@ -87,7 +87,7 @@ func TestEndpoint(t *testing.T) {
 
 	assertSecureIndex := func(index *IndexInfo) {
 		index.Secure = true
-		_, err := NewEndpoint(index, nil)
+		_, err := NewEndpoint(index, nil, APIVersionUnknown)
 		assertNotEqual(t, err, nil, index.Name+": Expected cert error for secure index")
 		assertEqual(t, strings.Contains(err.Error(), "certificate signed by unknown authority"), true, index.Name+": Expected cert error for secure index")
 		index.Secure = false
@@ -153,7 +153,7 @@ func TestEndpoint(t *testing.T) {
 	}
 	for _, address := range badEndpoints {
 		index.Name = address
-		_, err := NewEndpoint(index, nil)
+		_, err := NewEndpoint(index, nil, APIVersionUnknown)
 		checkNotEqual(t, err, nil, "Expected error while expanding bad endpoint")
 	}
 }

registry/service.go

@@ -2,14 +2,11 @@ package registry
 
 import (
 	"crypto/tls"
-	"fmt"
 	"net/http"
 	"net/url"
-	"strings"
 
 	"github.com/docker/distribution/registry/client/auth"
 	"github.com/docker/docker/cliconfig"
-	"github.com/docker/docker/pkg/tlsconfig"
 )
 
 // Service is a registry service. It tracks configuration data such as a list
@@ -39,7 +36,14 @@ func (s *Service) Auth(authConfig *cliconfig.AuthConfig) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	endpoint, err := NewEndpoint(index, nil)
+
+	endpointVersion := APIVersion(APIVersionUnknown)
+	if V2Only {
+		// Override the endpoint to only attempt a v2 ping
+		endpointVersion = APIVersion2
+	}
+
+	endpoint, err := NewEndpoint(index, nil, endpointVersion)
 	if err != nil {
 		return "", err
 	}
@@ -56,10 +60,11 @@ func (s *Service) Search(term string, authConfig *cliconfig.AuthConfig, headers
 	}
 
 	// *TODO: Search multiple indexes.
-	endpoint, err := repoInfo.GetEndpoint(http.Header(headers))
+	endpoint, err := NewEndpoint(repoInfo.Index, http.Header(headers), APIVersionUnknown)
 	if err != nil {
 		return nil, err
 	}
+
 	r, err := NewSession(endpoint.client, authConfig, endpoint)
 	if err != nil {
 		return nil, err
@@ -112,108 +117,32 @@ func (s *Service) tlsConfigForMirror(mirror string) (*tls.Config, error) {
 // It gives preference to v2 endpoints over v1, mirrors over the actual
 // registry, and HTTPS over plain HTTP.
 func (s *Service) LookupPullEndpoints(repoName string) (endpoints []APIEndpoint, err error) {
-	return s.lookupEndpoints(repoName, false)
+	return s.lookupEndpoints(repoName)
 }
 
 // LookupPushEndpoints creates an list of endpoints to try to push to, in order of preference.
 // It gives preference to v2 endpoints over v1, and HTTPS over plain HTTP.
 // Mirrors are not included.
 func (s *Service) LookupPushEndpoints(repoName string) (endpoints []APIEndpoint, err error) {
-	return s.lookupEndpoints(repoName, true)
+	return s.lookupEndpoints(repoName)
 }
 
-func (s *Service) lookupEndpoints(repoName string, isPush bool) (endpoints []APIEndpoint, err error) {
+func (s *Service) lookupEndpoints(repoName string) (endpoints []APIEndpoint, err error) {
-	var cfg = tlsconfig.ServerDefault
+	endpoints, err = s.lookupV2Endpoints(repoName)
-	tlsConfig := &cfg
-	if strings.HasPrefix(repoName, DefaultNamespace+"/") {
-		if !isPush {
-			// v2 mirrors for pull only
-			for _, mirror := range s.Config.Mirrors {
-				mirrorTLSConfig, err := s.tlsConfigForMirror(mirror)
-				if err != nil {
-					return nil, err
-				}
-				endpoints = append(endpoints, APIEndpoint{
-					URL: mirror,
-					// guess mirrors are v2
-					Version:      APIVersion2,
-					Mirror:       true,
-					TrimHostname: true,
-					TLSConfig:    mirrorTLSConfig,
-				})
-			}
-		}
-		// v2 registry
-		endpoints = append(endpoints, APIEndpoint{
-			URL:          DefaultV2Registry,
-			Version:      APIVersion2,
-			Official:     true,
-			TrimHostname: true,
-			TLSConfig:    tlsConfig,
-		})
-		// v1 registry
-		endpoints = append(endpoints, APIEndpoint{
-			URL:          DefaultV1Registry,
-			Version:      APIVersion1,
-			Official:     true,
-			TrimHostname: true,
-			TLSConfig:    tlsConfig,
-		})
-		return endpoints, nil
-	}
 
-	slashIndex := strings.IndexRune(repoName, '/')
-	if slashIndex <= 0 {
-		return nil, fmt.Errorf("invalid repo name: missing '/':  %s", repoName)
-	}
-	hostname := repoName[:slashIndex]
-
-	tlsConfig, err = s.TLSConfig(hostname)
 	if err != nil {
 		return nil, err
 	}
-	isSecure := !tlsConfig.InsecureSkipVerify
 
-	v2Versions := []auth.APIVersion{
+	if V2Only {
-		{
+		return endpoints, nil
-			Type:    "registry",
-			Version: "2.0",
-		},
-	}
-	endpoints = []APIEndpoint{
-		{
-			URL:           "https://" + hostname,
-			Version:       APIVersion2,
-			TrimHostname:  true,
-			TLSConfig:     tlsConfig,
-			VersionHeader: DefaultRegistryVersionHeader,
-			Versions:      v2Versions,
-		},
-		{
-			URL:          "https://" + hostname,
-			Version:      APIVersion1,
-			TrimHostname: true,
-			TLSConfig:    tlsConfig,
-		},
 	}
 
-	if !isSecure {
+	legacyEndpoints, err := s.lookupV1Endpoints(repoName)
-		endpoints = append(endpoints, APIEndpoint{
+	if err != nil {
-			URL:          "http://" + hostname,
+		return nil, err
-			Version:      APIVersion2,
-			TrimHostname: true,
-			// used to check if supposed to be secure via InsecureSkipVerify
-			TLSConfig:     tlsConfig,
-			VersionHeader: DefaultRegistryVersionHeader,
-			Versions:      v2Versions,
-		}, APIEndpoint{
-			URL:          "http://" + hostname,
-			Version:      APIVersion1,
-			TrimHostname: true,
-			// used to check if supposed to be secure via InsecureSkipVerify
-			TLSConfig: tlsConfig,
-		})
 	}
+	endpoints = append(endpoints, legacyEndpoints...)
 
 	return endpoints, nil
 }

registry/service_v1.go

@@ -0,0 +1,54 @@
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/docker/docker/pkg/tlsconfig"
+)
+
+func (s *Service) lookupV1Endpoints(repoName string) (endpoints []APIEndpoint, err error) {
+	var cfg = tlsconfig.ServerDefault
+	tlsConfig := &cfg
+	if strings.HasPrefix(repoName, DefaultNamespace+"/") {
+		endpoints = append(endpoints, APIEndpoint{
+			URL:          DefaultV1Registry,
+			Version:      APIVersion1,
+			Official:     true,
+			TrimHostname: true,
+			TLSConfig:    tlsConfig,
+		})
+		return endpoints, nil
+	}
+
+	slashIndex := strings.IndexRune(repoName, '/')
+	if slashIndex <= 0 {
+		return nil, fmt.Errorf("invalid repo name: missing '/':  %s", repoName)
+	}
+	hostname := repoName[:slashIndex]
+
+	tlsConfig, err = s.TLSConfig(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	endpoints = []APIEndpoint{
+		{
+			URL:          "https://" + hostname,
+			Version:      APIVersion1,
+			TrimHostname: true,
+			TLSConfig:    tlsConfig,
+		},
+	}
+
+	if tlsConfig.InsecureSkipVerify {
+		endpoints = append(endpoints, APIEndpoint{ // or this
+			URL:          "http://" + hostname,
+			Version:      APIVersion1,
+			TrimHostname: true,
+			// used to check if supposed to be secure via InsecureSkipVerify
+			TLSConfig: tlsConfig,
+		})
+	}
+	return endpoints, nil
+}

registry/service_v2.go

@@ -0,0 +1,83 @@
+package registry
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/docker/distribution/registry/client/auth"
+	"github.com/docker/docker/pkg/tlsconfig"
+)
+
+func (s *Service) lookupV2Endpoints(repoName string) (endpoints []APIEndpoint, err error) {
+	var cfg = tlsconfig.ServerDefault
+	tlsConfig := &cfg
+	if strings.HasPrefix(repoName, DefaultNamespace+"/") {
+		// v2 mirrors
+		for _, mirror := range s.Config.Mirrors {
+			mirrorTLSConfig, err := s.tlsConfigForMirror(mirror)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, APIEndpoint{
+				URL: mirror,
+				// guess mirrors are v2
+				Version:      APIVersion2,
+				Mirror:       true,
+				TrimHostname: true,
+				TLSConfig:    mirrorTLSConfig,
+			})
+		}
+		// v2 registry
+		endpoints = append(endpoints, APIEndpoint{
+			URL:          DefaultV2Registry,
+			Version:      APIVersion2,
+			Official:     true,
+			TrimHostname: true,
+			TLSConfig:    tlsConfig,
+		})
+
+		return endpoints, nil
+	}
+
+	slashIndex := strings.IndexRune(repoName, '/')
+	if slashIndex <= 0 {
+		return nil, fmt.Errorf("invalid repo name: missing '/':  %s", repoName)
+	}
+	hostname := repoName[:slashIndex]
+
+	tlsConfig, err = s.TLSConfig(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	v2Versions := []auth.APIVersion{
+		{
+			Type:    "registry",
+			Version: "2.0",
+		},
+	}
+	endpoints = []APIEndpoint{
+		{
+			URL:           "https://" + hostname,
+			Version:       APIVersion2,
+			TrimHostname:  true,
+			TLSConfig:     tlsConfig,
+			VersionHeader: DefaultRegistryVersionHeader,
+			Versions:      v2Versions,
+		},
+	}
+
+	if tlsConfig.InsecureSkipVerify {
+		endpoints = append(endpoints, APIEndpoint{
+			URL:          "http://" + hostname,
+			Version:      APIVersion2,
+			TrimHostname: true,
+			// used to check if supposed to be secure via InsecureSkipVerify
+			TLSConfig:     tlsConfig,
+			VersionHeader: DefaultRegistryVersionHeader,
+			Versions:      v2Versions,
+		})
+	}
+
+	return endpoints, nil
+}

runconfig/config.go

@@ -114,29 +114,31 @@ func NewCommand(parts ...string) *Command {
 // Note: the Config structure should hold only portable information about the container.
 // Here, "portable" means "independent from the host we are running on".
 // Non-portable information *should* appear in HostConfig.
+// All fields added to this struct must be marked `omitempty` to keep getting
+// predictable hashes from the old `v1Compatibility` configuration.
 type Config struct {
-	Hostname        string
+	Hostname        string                // Hostname
-	Domainname      string
+	Domainname      string                // Domainname
-	User            string
+	User            string                // User that will run the command(s) inside the container
-	AttachStdin     bool
+	AttachStdin     bool                  // Attach the standard input, makes possible user interaction
-	AttachStdout    bool
+	AttachStdout    bool                  // Attach the standard output
-	AttachStderr    bool
+	AttachStderr    bool                  // Attach the standard error
-	ExposedPorts    map[nat.Port]struct{}
+	ExposedPorts    map[nat.Port]struct{} `json:",omitempty"` // List of exposed ports
-	PublishService  string
+	PublishService  string                `json:",omitempty"` // Name of the network service exposed by the container
-	Tty             bool // Attach standard streams to a tty, including stdin if it is not closed.
+	Tty             bool                  // Attach standard streams to a tty, including stdin if it is not closed.
-	OpenStdin       bool // Open stdin
+	OpenStdin       bool                  // Open stdin
-	StdinOnce       bool // If true, close stdin after the 1 attached client disconnects.
+	StdinOnce       bool                  // If true, close stdin after the 1 attached client disconnects.
-	Env             []string
+	Env             []string              // List of environment variable to set in the container
-	Cmd             *Command
+	Cmd             *Command              // Command to run when starting the container
-	Image           string // Name of the image as it was passed by the operator (eg. could be symbolic)
+	Image           string                // Name of the image as it was passed by the operator (eg. could be symbolic)
-	Volumes         map[string]struct{}
+	Volumes         map[string]struct{}   // List of volumes (mounts) used for the container
-	VolumeDriver    string
+	VolumeDriver    string                `json:",omitempty"` // Name of the volume driver used to mount volumes
-	WorkingDir      string
+	WorkingDir      string                // Current directory (PWD) in the command will be launched
-	Entrypoint      *Entrypoint
+	Entrypoint      *Entrypoint           // Entrypoint to run when starting the container
-	NetworkDisabled bool
+	NetworkDisabled bool                  `json:",omitempty"` // Is network disabled
-	MacAddress      string
+	MacAddress      string                `json:",omitempty"` // Mac Address of the container
-	OnBuild         []string
+	OnBuild         []string              // ONBUILD metadata that were defined on the image Dockerfile
-	Labels          map[string]string
+	Labels          map[string]string     // List of labels set to this container
 }
 
 type ContainerConfigWrapper struct {

trust/service.go

@@ -1,53 +0,0 @@
-package trust
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/libtrust"
-)
-
-type NotVerifiedError string
-
-func (e NotVerifiedError) Error() string {
-	return string(e)
-}
-
-func (t *TrustStore) CheckKey(ns string, key []byte, perm uint16) (bool, error) {
-	if len(key) == 0 {
-		return false, fmt.Errorf("Missing PublicKey")
-	}
-	pk, err := libtrust.UnmarshalPublicKeyJWK(key)
-	if err != nil {
-		return false, fmt.Errorf("Error unmarshalling public key: %v", err)
-	}
-
-	if perm == 0 {
-		perm = 0x03
-	}
-
-	t.RLock()
-	defer t.RUnlock()
-	if t.graph == nil {
-		return false, NotVerifiedError("no graph")
-	}
-
-	// Check if any expired grants
-	verified, err := t.graph.Verify(pk, ns, perm)
-	if err != nil {
-		return false, fmt.Errorf("Error verifying key to namespace: %s", ns)
-	}
-	if !verified {
-		logrus.Debugf("Verification failed for %s using key %s", ns, pk.KeyID())
-		return false, NotVerifiedError("not verified")
-	}
-	if t.expiration.Before(time.Now()) {
-		return false, NotVerifiedError("expired")
-	}
-	return true, nil
-}
-
-func (t *TrustStore) UpdateBase() {
-	t.fetch()
-}

trust/trusts.go

@@ -1,195 +0,0 @@
-package trust
-
-import (
-	"crypto/x509"
-	"errors"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/libtrust/trustgraph"
-)
-
-type TrustStore struct {
-	path          string
-	caPool        *x509.CertPool
-	graph         trustgraph.TrustGraph
-	expiration    time.Time
-	fetcher       *time.Timer
-	fetchTime     time.Duration
-	autofetch     bool
-	httpClient    *http.Client
-	baseEndpoints map[string]*url.URL
-
-	sync.RWMutex
-}
-
-// defaultFetchtime represents the starting duration to wait between
-// fetching sections of the graph.  Unsuccessful fetches should
-// increase time between fetching.
-const defaultFetchtime = 45 * time.Second
-
-var baseEndpoints = map[string]string{"official": "https://dvjy3tqbc323p.cloudfront.net/trust/official.json"}
-
-func NewTrustStore(path string) (*TrustStore, error) {
-	abspath, err := filepath.Abs(path)
-	if err != nil {
-		return nil, err
-	}
-
-	// Create base graph url map
-	endpoints := map[string]*url.URL{}
-	for name, endpoint := range baseEndpoints {
-		u, err := url.Parse(endpoint)
-		if err != nil {
-			return nil, err
-		}
-		endpoints[name] = u
-	}
-
-	// Load grant files
-	t := &TrustStore{
-		path:          abspath,
-		caPool:        nil,
-		httpClient:    &http.Client{},
-		fetchTime:     time.Millisecond,
-		baseEndpoints: endpoints,
-	}
-
-	if err := t.reload(); err != nil {
-		return nil, err
-	}
-
-	return t, nil
-}
-
-func (t *TrustStore) reload() error {
-	t.Lock()
-	defer t.Unlock()
-
-	matches, err := filepath.Glob(filepath.Join(t.path, "*.json"))
-	if err != nil {
-		return err
-	}
-	statements := make([]*trustgraph.Statement, len(matches))
-	for i, match := range matches {
-		f, err := os.Open(match)
-		if err != nil {
-			return err
-		}
-		statements[i], err = trustgraph.LoadStatement(f, nil)
-		if err != nil {
-			f.Close()
-			return err
-		}
-		f.Close()
-	}
-	if len(statements) == 0 {
-		if t.autofetch {
-			logrus.Debugf("No grants, fetching")
-			t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
-		}
-		return nil
-	}
-
-	grants, expiration, err := trustgraph.CollapseStatements(statements, true)
-	if err != nil {
-		return err
-	}
-
-	t.expiration = expiration
-	t.graph = trustgraph.NewMemoryGraph(grants)
-	logrus.Debugf("Reloaded graph with %d grants expiring at %s", len(grants), expiration)
-
-	if t.autofetch {
-		nextFetch := expiration.Sub(time.Now())
-		if nextFetch < 0 {
-			nextFetch = defaultFetchtime
-		} else {
-			nextFetch = time.Duration(0.8 * (float64)(nextFetch))
-		}
-		t.fetcher = time.AfterFunc(nextFetch, t.fetch)
-	}
-
-	return nil
-}
-
-func (t *TrustStore) fetchBaseGraph(u *url.URL) (*trustgraph.Statement, error) {
-	req := &http.Request{
-		Method:     "GET",
-		URL:        u,
-		Proto:      "HTTP/1.1",
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     make(http.Header),
-		Body:       nil,
-		Host:       u.Host,
-	}
-
-	resp, err := t.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode == 404 {
-		return nil, errors.New("base graph does not exist")
-	}
-
-	defer resp.Body.Close()
-
-	return trustgraph.LoadStatement(resp.Body, t.caPool)
-}
-
-// fetch retrieves updated base graphs.  This function cannot error, it
-// should only log errors
-func (t *TrustStore) fetch() {
-	t.Lock()
-	defer t.Unlock()
-
-	if t.autofetch && t.fetcher == nil {
-		// Do nothing ??
-		return
-	}
-
-	fetchCount := 0
-	for bg, ep := range t.baseEndpoints {
-		statement, err := t.fetchBaseGraph(ep)
-		if err != nil {
-			logrus.Infof("Trust graph fetch failed: %s", err)
-			continue
-		}
-		b, err := statement.Bytes()
-		if err != nil {
-			logrus.Infof("Bad trust graph statement: %s", err)
-			continue
-		}
-		// TODO check if value differs
-		if err := ioutil.WriteFile(path.Join(t.path, bg+".json"), b, 0600); err != nil {
-			logrus.Infof("Error writing trust graph statement: %s", err)
-		}
-		fetchCount++
-	}
-	logrus.Debugf("Fetched %d base graphs at %s", fetchCount, time.Now())
-
-	if fetchCount > 0 {
-		go func() {
-			if err := t.reload(); err != nil {
-				logrus.Infof("Reload of trust graph failed: %s", err)
-			}
-		}()
-		t.fetchTime = defaultFetchtime
-		t.fetcher = nil
-	} else if t.autofetch {
-		maxTime := 10 * defaultFetchtime
-		t.fetchTime = time.Duration(1.5 * (float64)(t.fetchTime+time.Second))
-		if t.fetchTime > maxTime {
-			t.fetchTime = maxTime
-		}
-		t.fetcher = time.AfterFunc(t.fetchTime, t.fetch)
-	}
-}

vendor/src/github.com/docker/distribution/registry/client/repository.go

@@ -359,25 +359,18 @@ type blobs struct {
 	distribution.BlobDeleter
 }
 
-func sanitizeLocation(location, source string) (string, error) {
+func sanitizeLocation(location, base string) (string, error) {
+	baseURL, err := url.Parse(base)
+	if err != nil {
+		return "", err
+	}
+
 	locationURL, err := url.Parse(location)
 	if err != nil {
 		return "", err
 	}
 
-	if locationURL.Scheme == "" {
+	return baseURL.ResolveReference(locationURL).String(), nil
-		sourceURL, err := url.Parse(source)
-		if err != nil {
-			return "", err
-		}
-		locationURL = &url.URL{
-			Scheme: sourceURL.Scheme,
-			Host:   sourceURL.Host,
-			Path:   location,
-		}
-		location = locationURL.String()
-	}
-	return location, nil
 }
 
 func (bs *blobs) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {

vendor/src/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -19,6 +19,7 @@ import (
 	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/options"
 	"github.com/docker/libnetwork/portmapper"
+	"github.com/docker/libnetwork/sandbox"
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 )
@@ -89,6 +90,7 @@ type bridgeNetwork struct {
 	config     *networkConfiguration
 	endpoints  map[types.UUID]*bridgeEndpoint // key: endpoint id
 	portMapper *portmapper.PortMapper
+	veth       *netlink.Veth
 	sync.Mutex
 }
 
@@ -544,6 +546,8 @@ func (d *driver) getNetworks() []*bridgeNetwork {
 func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) error {
 	var err error
 
+	defer sandbox.InitOSContext()()
+
 	// Sanity checks
 	d.Lock()
 	if _, ok := d.networks[id]; ok {
@@ -695,6 +699,8 @@ func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) err
 func (d *driver) DeleteNetwork(nid types.UUID) error {
 	var err error
 
+	defer sandbox.InitOSContext()()
+
 	// Get network handler and remove it from driver
 	d.Lock()
 	n, ok := d.networks[nid]
@@ -822,6 +828,8 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn
 		err      error
 	)
 
+	defer sandbox.InitOSContext()()
+
 	if epInfo == nil {
 		return errors.New("invalid endpoint info passed")
 	}
@@ -894,11 +902,13 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn
 		return err
 	}
 
+	logrus.Warnf("network %v", n)
+	logrus.Warnf("veth %v", n.veth)
 	// Generate and add the interface pipe host <-> sandbox
-	veth := &netlink.Veth{
+	n.veth = &netlink.Veth{
 		LinkAttrs: netlink.LinkAttrs{Name: hostIfName, TxQLen: 0},
 		PeerName:  containerIfName}
-	if err = netlink.LinkAdd(veth); err != nil {
+	if err = netlink.LinkAdd(n.veth); err != nil {
 		return err
 	}
 
@@ -1029,6 +1039,8 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn
 func (d *driver) DeleteEndpoint(nid, eid types.UUID) error {
 	var err error
 
+	defer sandbox.InitOSContext()()
+
 	// Get the network handler and make sure it exists
 	d.Lock()
 	n, ok := d.networks[nid]
@@ -1163,11 +1175,18 @@ func (d *driver) EndpointOperInfo(nid, eid types.UUID) (map[string]interface{},
 		m[netlabel.MacAddress] = ep.macAddress
 	}
 
+	// Add details of the bridge
+	m[netlabel.BridgeName] = n.config.BridgeName
+	m[netlabel.BridgePeername] = n.veth.PeerName
+	m[netlabel.BridgeLinkName] = n.veth.LinkAttrs.Name
+
 	return m, nil
 }
 
 // Join method is invoked when a Sandbox is attached to an endpoint.
 func (d *driver) Join(nid, eid types.UUID, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
+	defer sandbox.InitOSContext()()
+
 	network, err := d.getNetwork(nid)
 	if err != nil {
 		return err
@@ -1211,6 +1230,8 @@ func (d *driver) Join(nid, eid types.UUID, sboxKey string, jinfo driverapi.JoinI
 
 // Leave method is invoked when a Sandbox detaches from an endpoint.
 func (d *driver) Leave(nid, eid types.UUID) error {
+	defer sandbox.InitOSContext()()
+
 	network, err := d.getNetwork(nid)
 	if err != nil {
 		return err

vendor/src/github.com/docker/libnetwork/drivers/overlay/ov_utils.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/docker/libnetwork/netutils"
+	"github.com/docker/libnetwork/sandbox"
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 )
@@ -21,6 +22,8 @@ func validateID(nid, eid types.UUID) error {
 }
 
 func createVethPair() (string, string, error) {
+	defer sandbox.InitOSContext()()
+
 	// Generate a name for what will be the host side pipe interface
 	name1, err := netutils.GenerateIfaceName(vethPrefix, vethLen)
 	if err != nil {
@@ -45,6 +48,8 @@ func createVethPair() (string, string, error) {
 }
 
 func createVxlan(vni uint32) (string, error) {
+	defer sandbox.InitOSContext()()
+
 	name, err := netutils.GenerateIfaceName("vxlan", 7)
 	if err != nil {
 		return "", fmt.Errorf("error generating vxlan name: %v", err)
@@ -68,6 +73,8 @@ func createVxlan(vni uint32) (string, error) {
 }
 
 func deleteVxlan(name string) error {
+	defer sandbox.InitOSContext()()
+
 	link, err := netlink.LinkByName(name)
 	if err != nil {
 		return fmt.Errorf("failed to find vxlan interface with name %s: %v", name, err)

vendor/src/github.com/docker/libnetwork/netlabel/labels.go

@@ -24,6 +24,17 @@ const (
 	//EnableIPv6 constant represents enabling IPV6 at network level
 	EnableIPv6 = Prefix + ".enable_ipv6"
 
+	// BridgeName constant represents the name of the network bridge
+	BridgeName = "io.docker.network.bridge.name"
+
+	// BridgePeername constant represents the interface name provided to
+	// the container.
+	BridgePeername = "io.docker.network.bridge.peername"
+
+	// BridgeLinkName constant represents the interface name created on
+	// the host side.
+	BridgeLinkName = "io.docker.network.bridge.linkname"
+
 	// KVProvider constant represents the KV provider backend
 	KVProvider = DriverPrefix + ".kv_provider"
 

vendor/src/github.com/docker/libnetwork/netutils/test_utils.go

@@ -1,41 +0,0 @@
-package netutils
-
-import (
-	"flag"
-	"runtime"
-	"syscall"
-	"testing"
-)
-
-var runningInContainer = flag.Bool("incontainer", false, "Indicates if the test is running in a container")
-
-// IsRunningInContainer returns whether the test is running inside a container.
-func IsRunningInContainer() bool {
-	return (*runningInContainer)
-}
-
-// SetupTestNetNS joins a new network namespace, and returns its associated
-// teardown function.
-//
-// Example usage:
-//
-//     defer SetupTestNetNS(t)()
-//
-func SetupTestNetNS(t *testing.T) func() {
-	runtime.LockOSThread()
-	if err := syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
-		t.Fatalf("Failed to enter netns: %v", err)
-	}
-
-	fd, err := syscall.Open("/proc/self/ns/net", syscall.O_RDONLY, 0)
-	if err != nil {
-		t.Fatal("Failed to open netns file")
-	}
-
-	return func() {
-		if err := syscall.Close(fd); err != nil {
-			t.Logf("Warning: netns closing failed (%v)", err)
-		}
-		runtime.UnlockOSThread()
-	}
-}

vendor/src/github.com/docker/libnetwork/sandbox/namespace_linux.go

@@ -26,6 +26,8 @@ var (
 	gpmWg            sync.WaitGroup
 	gpmCleanupPeriod = 60 * time.Second
 	gpmChan          = make(chan chan struct{})
+	nsOnce           sync.Once
+	initNs           netns.NsHandle
 )
 
 // The networkNamespace type is the linux implementation of the Sandbox
@@ -242,15 +244,37 @@ func (n *networkNamespace) InvokeFunc(f func()) error {
 	})
 }
 
-func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error {
+func getLink() (string, error) {
-	runtime.LockOSThread()
+	return os.Readlink(fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid()))
-	defer runtime.UnlockOSThread()
+}
 
-	origns, err := netns.Get()
+func nsInit() {
-	if err != nil {
+	var err error
-		return err
+
+	if initNs, err = netns.Get(); err != nil {
+		log.Errorf("could not get initial namespace: %v", err)
 	}
-	defer origns.Close()
+}
+
+// InitOSContext initializes OS context while configuring network resources
+func InitOSContext() func() {
+	runtime.LockOSThread()
+	nsOnce.Do(nsInit)
+	if err := netns.Set(initNs); err != nil {
+		linkInfo, linkErr := getLink()
+		if linkErr != nil {
+			linkInfo = linkErr.Error()
+		}
+
+		log.Errorf("failed to set to initial namespace, %v, initns fd %d: %v",
+			linkInfo, initNs, err)
+	}
+
+	return runtime.UnlockOSThread
+}
+
+func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error {
+	defer InitOSContext()()
 
 	f, err := os.OpenFile(path, os.O_RDONLY, 0)
 	if err != nil {
@@ -269,10 +293,10 @@ func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD
 	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
 		return err
 	}
-	defer netns.Set(origns)
+	defer netns.Set(initNs)
 
 	// Invoked after the namespace switch.
-	return postfunc(int(origns))
+	return postfunc(int(initNs))
 }
 
 func (n *networkNamespace) nsPath() string {

vendor/src/github.com/docker/libnetwork/sandbox/namespace_windows.go

@@ -21,3 +21,8 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) {
 // and waits for it.
 func GC() {
 }
+
+// InitOSContext initializes OS context while configuring network resources
+func InitOSContext() func() {
+	return func() {}
+}

vendor/src/github.com/docker/libnetwork/sandbox/sandbox_freebsd.go

@@ -21,3 +21,8 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) {
 // and waits for it.
 func GC() {
 }
+
+// InitOSContext initializes OS context while configuring network resources
+func InitOSContext() func() {
+	return func() {}
+}

vendor/src/github.com/docker/libtrust/trustgraph/graph.go

@@ -1,50 +0,0 @@
-package trustgraph
-
-import "github.com/docker/libtrust"
-
-// TrustGraph represents a graph of authorization mapping
-// public keys to nodes and grants between nodes.
-type TrustGraph interface {
-	// Verifies that the given public key is allowed to perform
-	// the given action on the given node according to the trust
-	// graph.
-	Verify(libtrust.PublicKey, string, uint16) (bool, error)
-
-	// GetGrants returns an array of all grant chains which are used to
-	// allow the requested permission.
-	GetGrants(libtrust.PublicKey, string, uint16) ([][]*Grant, error)
-}
-
-// Grant represents a transfer of permission from one part of the
-// trust graph to another. This is the only way to delegate
-// permission between two different sub trees in the graph.
-type Grant struct {
-	// Subject is the namespace being granted
-	Subject string
-
-	// Permissions is a bit map of permissions
-	Permission uint16
-
-	// Grantee represents the node being granted
-	// a permission scope.  The grantee can be
-	// either a namespace item or a key id where namespace
-	// items will always start with a '/'.
-	Grantee string
-
-	// statement represents the statement used to create
-	// this object.
-	statement *Statement
-}
-
-// Permissions
-//  Read node 0x01 (can read node, no sub nodes)
-//  Write node 0x02 (can write to node object, cannot create subnodes)
-//  Read subtree 0x04 (delegates read to each sub node)
-//  Write subtree 0x08 (delegates write to each sub node, included create on the subject)
-//
-// Permission shortcuts
-// ReadItem = 0x01
-// WriteItem = 0x03
-// ReadAccess = 0x07
-// WriteAccess = 0x0F
-// Delegate = 0x0F

vendor/src/github.com/docker/libtrust/trustgraph/memory_graph.go

@@ -1,133 +0,0 @@
-package trustgraph
-
-import (
-	"strings"
-
-	"github.com/docker/libtrust"
-)
-
-type grantNode struct {
-	grants   []*Grant
-	children map[string]*grantNode
-}
-
-type memoryGraph struct {
-	roots map[string]*grantNode
-}
-
-func newGrantNode() *grantNode {
-	return &grantNode{
-		grants:   []*Grant{},
-		children: map[string]*grantNode{},
-	}
-}
-
-// NewMemoryGraph returns a new in memory trust graph created from
-// a static list of grants.  This graph is immutable after creation
-// and any alterations should create a new instance.
-func NewMemoryGraph(grants []*Grant) TrustGraph {
-	roots := map[string]*grantNode{}
-	for _, grant := range grants {
-		parts := strings.Split(grant.Grantee, "/")
-		nodes := roots
-		var node *grantNode
-		var nodeOk bool
-		for _, part := range parts {
-			node, nodeOk = nodes[part]
-			if !nodeOk {
-				node = newGrantNode()
-				nodes[part] = node
-			}
-			if part != "" {
-				node.grants = append(node.grants, grant)
-			}
-			nodes = node.children
-		}
-	}
-	return &memoryGraph{roots}
-}
-
-func (g *memoryGraph) getGrants(name string) []*Grant {
-	nameParts := strings.Split(name, "/")
-	nodes := g.roots
-	var node *grantNode
-	var nodeOk bool
-	for _, part := range nameParts {
-		node, nodeOk = nodes[part]
-		if !nodeOk {
-			return nil
-		}
-		nodes = node.children
-	}
-	return node.grants
-}
-
-func isSubName(name, sub string) bool {
-	if strings.HasPrefix(name, sub) {
-		if len(name) == len(sub) || name[len(sub)] == '/' {
-			return true
-		}
-	}
-	return false
-}
-
-type walkFunc func(*Grant, []*Grant) bool
-
-func foundWalkFunc(*Grant, []*Grant) bool {
-	return true
-}
-
-func (g *memoryGraph) walkGrants(start, target string, permission uint16, f walkFunc, chain []*Grant, visited map[*Grant]bool, collect bool) bool {
-	if visited == nil {
-		visited = map[*Grant]bool{}
-	}
-	grants := g.getGrants(start)
-	subGrants := make([]*Grant, 0, len(grants))
-	for _, grant := range grants {
-		if visited[grant] {
-			continue
-		}
-		visited[grant] = true
-		if grant.Permission&permission == permission {
-			if isSubName(target, grant.Subject) {
-				if f(grant, chain) {
-					return true
-				}
-			} else {
-				subGrants = append(subGrants, grant)
-			}
-		}
-	}
-	for _, grant := range subGrants {
-		var chainCopy []*Grant
-		if collect {
-			chainCopy = make([]*Grant, len(chain)+1)
-			copy(chainCopy, chain)
-			chainCopy[len(chainCopy)-1] = grant
-		} else {
-			chainCopy = nil
-		}
-
-		if g.walkGrants(grant.Subject, target, permission, f, chainCopy, visited, collect) {
-			return true
-		}
-	}
-	return false
-}
-
-func (g *memoryGraph) Verify(key libtrust.PublicKey, node string, permission uint16) (bool, error) {
-	return g.walkGrants(key.KeyID(), node, permission, foundWalkFunc, nil, nil, false), nil
-}
-
-func (g *memoryGraph) GetGrants(key libtrust.PublicKey, node string, permission uint16) ([][]*Grant, error) {
-	grants := [][]*Grant{}
-	collect := func(grant *Grant, chain []*Grant) bool {
-		grantChain := make([]*Grant, len(chain)+1)
-		copy(grantChain, chain)
-		grantChain[len(grantChain)-1] = grant
-		grants = append(grants, grantChain)
-		return false
-	}
-	g.walkGrants(key.KeyID(), node, permission, collect, nil, nil, true)
-	return grants, nil
-}

vendor/src/github.com/docker/libtrust/trustgraph/statement.go

@@ -1,227 +0,0 @@
-package trustgraph
-
-import (
-	"crypto/x509"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/docker/libtrust"
-)
-
-type jsonGrant struct {
-	Subject    string `json:"subject"`
-	Permission uint16 `json:"permission"`
-	Grantee    string `json:"grantee"`
-}
-
-type jsonRevocation struct {
-	Subject    string `json:"subject"`
-	Revocation uint16 `json:"revocation"`
-	Grantee    string `json:"grantee"`
-}
-
-type jsonStatement struct {
-	Revocations []*jsonRevocation `json:"revocations"`
-	Grants      []*jsonGrant      `json:"grants"`
-	Expiration  time.Time         `json:"expiration"`
-	IssuedAt    time.Time         `json:"issuedAt"`
-}
-
-func (g *jsonGrant) Grant(statement *Statement) *Grant {
-	return &Grant{
-		Subject:    g.Subject,
-		Permission: g.Permission,
-		Grantee:    g.Grantee,
-		statement:  statement,
-	}
-}
-
-// Statement represents a set of grants made from a verifiable
-// authority.  A statement has an expiration associated with it
-// set by the authority.
-type Statement struct {
-	jsonStatement
-
-	signature *libtrust.JSONSignature
-}
-
-// IsExpired returns whether the statement has expired
-func (s *Statement) IsExpired() bool {
-	return s.Expiration.Before(time.Now().Add(-10 * time.Second))
-}
-
-// Bytes returns an indented json representation of the statement
-// in a byte array.  This value can be written to a file or stream
-// without alteration.
-func (s *Statement) Bytes() ([]byte, error) {
-	return s.signature.PrettySignature("signatures")
-}
-
-// LoadStatement loads and verifies a statement from an input stream.
-func LoadStatement(r io.Reader, authority *x509.CertPool) (*Statement, error) {
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-	js, err := libtrust.ParsePrettySignature(b, "signatures")
-	if err != nil {
-		return nil, err
-	}
-	payload, err := js.Payload()
-	if err != nil {
-		return nil, err
-	}
-	var statement Statement
-	err = json.Unmarshal(payload, &statement.jsonStatement)
-	if err != nil {
-		return nil, err
-	}
-
-	if authority == nil {
-		_, err = js.Verify()
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		_, err = js.VerifyChains(authority)
-		if err != nil {
-			return nil, err
-		}
-	}
-	statement.signature = js
-
-	return &statement, nil
-}
-
-// CreateStatements creates and signs a statement from a stream of grants
-// and revocations in a JSON array.
-func CreateStatement(grants, revocations io.Reader, expiration time.Duration, key libtrust.PrivateKey, chain []*x509.Certificate) (*Statement, error) {
-	var statement Statement
-	err := json.NewDecoder(grants).Decode(&statement.jsonStatement.Grants)
-	if err != nil {
-		return nil, err
-	}
-	err = json.NewDecoder(revocations).Decode(&statement.jsonStatement.Revocations)
-	if err != nil {
-		return nil, err
-	}
-	statement.jsonStatement.Expiration = time.Now().UTC().Add(expiration)
-	statement.jsonStatement.IssuedAt = time.Now().UTC()
-
-	b, err := json.MarshalIndent(&statement.jsonStatement, "", "   ")
-	if err != nil {
-		return nil, err
-	}
-
-	statement.signature, err = libtrust.NewJSONSignature(b)
-	if err != nil {
-		return nil, err
-	}
-	err = statement.signature.SignWithChain(key, chain)
-	if err != nil {
-		return nil, err
-	}
-
-	return &statement, nil
-}
-
-type statementList []*Statement
-
-func (s statementList) Len() int {
-	return len(s)
-}
-
-func (s statementList) Less(i, j int) bool {
-	return s[i].IssuedAt.Before(s[j].IssuedAt)
-}
-
-func (s statementList) Swap(i, j int) {
-	s[i], s[j] = s[j], s[i]
-}
-
-// CollapseStatements returns a single list of the valid statements as well as the
-// time when the next grant will expire.
-func CollapseStatements(statements []*Statement, useExpired bool) ([]*Grant, time.Time, error) {
-	sorted := make(statementList, 0, len(statements))
-	for _, statement := range statements {
-		if useExpired || !statement.IsExpired() {
-			sorted = append(sorted, statement)
-		}
-	}
-	sort.Sort(sorted)
-
-	var minExpired time.Time
-	var grantCount int
-	roots := map[string]*grantNode{}
-	for i, statement := range sorted {
-		if statement.Expiration.Before(minExpired) || i == 0 {
-			minExpired = statement.Expiration
-		}
-		for _, grant := range statement.Grants {
-			parts := strings.Split(grant.Grantee, "/")
-			nodes := roots
-			g := grant.Grant(statement)
-			grantCount = grantCount + 1
-
-			for _, part := range parts {
-				node, nodeOk := nodes[part]
-				if !nodeOk {
-					node = newGrantNode()
-					nodes[part] = node
-				}
-				node.grants = append(node.grants, g)
-				nodes = node.children
-			}
-		}
-
-		for _, revocation := range statement.Revocations {
-			parts := strings.Split(revocation.Grantee, "/")
-			nodes := roots
-
-			var node *grantNode
-			var nodeOk bool
-			for _, part := range parts {
-				node, nodeOk = nodes[part]
-				if !nodeOk {
-					break
-				}
-				nodes = node.children
-			}
-			if node != nil {
-				for _, grant := range node.grants {
-					if isSubName(grant.Subject, revocation.Subject) {
-						grant.Permission = grant.Permission &^ revocation.Revocation
-					}
-				}
-			}
-		}
-	}
-
-	retGrants := make([]*Grant, 0, grantCount)
-	for _, rootNodes := range roots {
-		retGrants = append(retGrants, rootNodes.grants...)
-	}
-
-	return retGrants, minExpired, nil
-}
-
-// FilterStatements filters the statements to statements including the given grants.
-func FilterStatements(grants []*Grant) ([]*Statement, error) {
-	statements := map[*Statement]bool{}
-	for _, grant := range grants {
-		if grant.statement != nil {
-			statements[grant.statement] = true
-		}
-	}
-	retStatements := make([]*Statement, len(statements))
-	var i int
-	for statement := range statements {
-		retStatements[i] = statement
-		i++
-	}
-	return retStatements, nil
-}

vendor/src/github.com/fluent/fluent-logger-golang/fluent/fluent.go

@@ -3,6 +3,7 @@ package fluent
 import (
 	"errors"
 	"fmt"
+	"io"
 	"math"
 	"net"
 	"reflect"
@@ -33,7 +34,7 @@ type Config struct {
 
 type Fluent struct {
 	Config
-	conn         net.Conn
+	conn         io.WriteCloser
 	pending      []byte
 	reconnecting bool
 	mu           sync.Mutex

vendor/src/github.com/fluent/fluent-logger-golang/fluent/version.go

@@ -1,3 +1,3 @@
 package fluent
 
-const Version = "0.5.1"
+const Version = "1.0.0"

vendor/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go

@@ -236,7 +236,7 @@ func getCgroupData(c *configs.Cgroup, pid int) (*data, error) {
 }
 
 func (raw *data) parent(subsystem, mountpoint, src string) (string, error) {
-	initPath, err := cgroups.GetInitCgroupDir(subsystem)
+	initPath, err := cgroups.GetThisCgroupDir(subsystem)
 	if err != nil {
 		return "", err
 	}

vendor/src/github.com/vbatts/tar-split/archive/tar/reader.go

@@ -159,17 +159,19 @@ func (tr *Reader) Next() (*Header, error) {
 		if err != nil {
 			return nil, err
 		}
-		var b []byte
+		var buf []byte
 		if tr.RawAccounting {
 			if _, err = tr.rawBytes.Write(realname); err != nil {
 				return nil, err
 			}
-			b = tr.RawBytes()
+			buf = make([]byte, tr.rawBytes.Len())
+			copy(buf[:], tr.RawBytes())
 		}
 		hdr, err := tr.Next()
 		// since the above call to Next() resets the buffer, we need to throw the bytes over
 		if tr.RawAccounting {
-			if _, err = tr.rawBytes.Write(b); err != nil {
+			buf = append(buf, tr.RawBytes()...)
+			if _, err = tr.rawBytes.Write(buf); err != nil {
 				return nil, err
 			}
 		}
@@ -181,17 +183,19 @@ func (tr *Reader) Next() (*Header, error) {
 		if err != nil {
 			return nil, err
 		}
-		var b []byte
+		var buf []byte
 		if tr.RawAccounting {
 			if _, err = tr.rawBytes.Write(realname); err != nil {
 				return nil, err
 			}
-			b = tr.RawBytes()
+			buf = make([]byte, tr.rawBytes.Len())
+			copy(buf[:], tr.RawBytes())
 		}
 		hdr, err := tr.Next()
 		// since the above call to Next() resets the buffer, we need to throw the bytes over
 		if tr.RawAccounting {
-			if _, err = tr.rawBytes.Write(b); err != nil {
+			buf = append(buf, tr.RawBytes()...)
+			if _, err = tr.rawBytes.Write(buf); err != nil {
 				return nil, err
 			}
 		}

vendor/src/github.com/vbatts/tar-split/tar/asm/assemble.go

@@ -9,7 +9,7 @@ import (
 	"github.com/vbatts/tar-split/tar/storage"
 )
 
-// NewOutputTarStream returns an io.ReadCloser that is an assemble tar archive
+// NewOutputTarStream returns an io.ReadCloser that is an assembled tar archive
 // stream.
 //
 // It takes a storage.FileGetter, for mapping the file payloads that are to be read in,
@@ -62,7 +62,6 @@ func NewOutputTarStream(fg storage.FileGetter, up storage.Unpacker) io.ReadClose
 				fh.Close()
 			}
 		}
-		pw.Close()
 	}()
 	return pr
 }

vendor/src/github.com/vbatts/tar-split/tar/asm/disassemble.go

@@ -22,8 +22,8 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
 	// What to do here... folks will want their own access to the Reader that is
 	// their tar archive stream, but we'll need that same stream to use our
 	// forked 'archive/tar'.
-	// Perhaps do an io.TeeReader that hand back an io.Reader for them to read
+	// Perhaps do an io.TeeReader that hands back an io.Reader for them to read
-	// from, and we'll mitm the stream to store metadata.
+	// from, and we'll MITM the stream to store metadata.
 	// We'll need a storage.FilePutter too ...
 
 	// Another concern, whether to do any storage.FilePutter operations, such that we
@@ -32,7 +32,7 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
 	// Perhaps we have a DiscardFilePutter that is a bit bucket.
 
 	// we'll return the pipe reader, since TeeReader does not buffer and will
-	// only read what the outputRdr Read's. Since Tar archive's have padding on
+	// only read what the outputRdr Read's. Since Tar archives have padding on
 	// the end, we want to be the one reading the padding, even if the user's
 	// `archive/tar` doesn't care.
 	pR, pW := io.Pipe()
@@ -55,13 +55,15 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
 				}
 				// even when an EOF is reached, there is often 1024 null bytes on
 				// the end of an archive. Collect them too.
-				_, err := p.AddEntry(storage.Entry{
+				if b := tr.RawBytes(); len(b) > 0 {
-					Type:    storage.SegmentType,
+					_, err := p.AddEntry(storage.Entry{
-					Payload: tr.RawBytes(),
+						Type:    storage.SegmentType,
-				})
+						Payload: b,
-				if err != nil {
+					})
-					pW.CloseWithError(err)
+					if err != nil {
-					return
+						pW.CloseWithError(err)
+						return
+					}
 				}
 				break // not return. We need the end of the reader.
 			}
@@ -69,12 +71,15 @@ func NewInputTarStream(r io.Reader, p storage.Packer, fp storage.FilePutter) (io
 				break // not return. We need the end of the reader.
 			}
 
-			if _, err := p.AddEntry(storage.Entry{
+			if b := tr.RawBytes(); len(b) > 0 {
-				Type:    storage.SegmentType,
+				_, err := p.AddEntry(storage.Entry{
-				Payload: tr.RawBytes(),
+					Type:    storage.SegmentType,
-			}); err != nil {
+					Payload: b,
-				pW.CloseWithError(err)
+				})
-				return
+				if err != nil {
+					pW.CloseWithError(err)
+					return
+				}
 			}
 
 			var csum []byte

vendor/src/github.com/vbatts/tar-split/tar/storage/doc.go

@@ -5,7 +5,7 @@ Packing and unpacking the Entries of the stream. The types of streams are
 either segments of raw bytes (for the raw headers and various padding) and for
 an entry marking a file payload.
 
-The raw bytes are stored precisely in the packed (marshalled) Entry. Where as
+The raw bytes are stored precisely in the packed (marshalled) Entry, whereas
 the file payload marker include the name of the file, size, and crc64 checksum
 (for basic file integrity).
 */

vendor/src/github.com/vbatts/tar-split/tar/storage/entry.go

@@ -19,11 +19,11 @@ const (
 	// SegmentType represents a raw bytes segment from the archive stream. These raw
 	// byte segments consist of the raw headers and various padding.
 	//
-	// It's payload is to be marshalled base64 encoded.
+	// Its payload is to be marshalled base64 encoded.
 	SegmentType
 )
 
-// Entry is a the structure for packing and unpacking the information read from
+// Entry is the structure for packing and unpacking the information read from
 // the Tar archive.
 //
 // FileType Payload checksum is using `hash/crc64` for basic file integrity,
@@ -32,8 +32,8 @@ const (
 // collisions in a sample of 18.2 million, CRC64 had none.
 type Entry struct {
 	Type     Type   `json:"type"`
-	Name     string `json:"name",omitempty`
+	Name     string `json:"name,omitempty"`
-	Size     int64  `json:"size",omitempty`
+	Size     int64  `json:"size,omitempty"`
-	Payload  []byte `json:"payload"` // SegmentType store payload here; FileType store crc64 checksum here;
+	Payload  []byte `json:"payload"` // SegmentType stores payload here; FileType stores crc64 checksum here;
 	Position int    `json:"position"`
 }

vendor/src/github.com/vbatts/tar-split/tar/storage/getter.go

@@ -5,14 +5,13 @@ import (
 	"errors"
 	"hash/crc64"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 )
 
-// FileGetter is the interface for getting a stream of a file payload, address
+// FileGetter is the interface for getting a stream of a file payload,
-// by name/filename. Presumably, the names will be scoped to relative file
+// addressed by name/filename. Presumably, the names will be scoped to relative
-// paths.
+// file paths.
 type FileGetter interface {
 	// Get returns a stream for the provided file path
 	Get(filename string) (output io.ReadCloser, err error)
@@ -60,15 +59,15 @@ func (bfgp bufferFileGetPutter) Get(name string) (io.ReadCloser, error) {
 }
 
 func (bfgp *bufferFileGetPutter) Put(name string, r io.Reader) (int64, []byte, error) {
-	c := crc64.New(CRCTable)
+	crc := crc64.New(CRCTable)
-	tRdr := io.TeeReader(r, c)
+	buf := bytes.NewBuffer(nil)
-	b := bytes.NewBuffer([]byte{})
+	cw := io.MultiWriter(crc, buf)
-	i, err := io.Copy(b, tRdr)
+	i, err := io.Copy(cw, r)
 	if err != nil {
 		return 0, nil, err
 	}
-	bfgp.files[name] = b.Bytes()
+	bfgp.files[name] = buf.Bytes()
-	return i, c.Sum(nil), nil
+	return i, crc.Sum(nil), nil
 }
 
 type readCloserWrapper struct {
@@ -77,7 +76,7 @@ type readCloserWrapper struct {
 
 func (w *readCloserWrapper) Close() error { return nil }
 
-// NewBufferFileGetPutter is simple in memory FileGetPutter
+// NewBufferFileGetPutter is a simple in-memory FileGetPutter
 //
 // Implication is this is memory intensive...
 // Probably best for testing or light weight cases.
@@ -97,8 +96,7 @@ type bitBucketFilePutter struct {
 
 func (bbfp *bitBucketFilePutter) Put(name string, r io.Reader) (int64, []byte, error) {
 	c := crc64.New(CRCTable)
-	tRdr := io.TeeReader(r, c)
+	i, err := io.Copy(c, r)
-	i, err := io.Copy(ioutil.Discard, tRdr)
 	return i, c.Sum(nil), err
 }
 

vendor/src/github.com/vbatts/tar-split/tar/storage/packer.go

@@ -8,8 +8,8 @@ import (
 	"path/filepath"
 )
 
-// ErrDuplicatePath is occured when a tar archive has more than one entry for
+// ErrDuplicatePath occurs when a tar archive has more than one entry for the
-// the same file path
+// same file path
 var ErrDuplicatePath = errors.New("duplicates of file paths not supported")
 
 // Packer describes the methods to pack Entries to a storage destination
@@ -65,7 +65,7 @@ func (jup *jsonUnpacker) Next() (*Entry, error) {
 		if _, ok := jup.seen[cName]; ok {
 			return nil, ErrDuplicatePath
 		}
-		jup.seen[cName] = emptyByte
+		jup.seen[cName] = struct{}{}
 	}
 
 	return &e, err
@@ -90,11 +90,7 @@ type jsonPacker struct {
 	seen seenNames
 }
 
-type seenNames map[string]byte
+type seenNames map[string]struct{}
-
-// used in the seenNames map. byte is a uint8, and we'll re-use the same one
-// for minimalism.
-const emptyByte byte = 0
 
 func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
 	// check early for dup name
@@ -103,7 +99,7 @@ func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
 		if _, ok := jp.seen[cName]; ok {
 			return -1, ErrDuplicatePath
 		}
-		jp.seen[cName] = emptyByte
+		jp.seen[cName] = struct{}{}
 	}
 
 	e.Position = jp.pos
@@ -117,7 +113,7 @@ func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
 	return e.Position, nil
 }
 
-// NewJSONPacker provides an Packer that writes each Entry (SegmentType and
+// NewJSONPacker provides a Packer that writes each Entry (SegmentType and
 // FileType) as a json document.
 //
 // The Entries are delimited by new line.