From 9419eade34a80ebb096082e7eb8fb804ceba39d2 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 20 Sep 2015 17:29:41 +0200 Subject: [PATCH 001/134] daemon: execdriver: lxc: fix set memory swap On LXC memory swap was only set to memory_limit*2 even if a value for memory swap was provided. This patch fix this behavior to be the same as the native driver and set correct memory swap in the template. Also add a test specifically for LXC but w/o adding a new test requirement. Signed-off-by: Antonio Murdaca --- daemon/execdriver/lxc/lxc_template.go | 14 ++----------- .../execdriver/lxc/lxc_template_unit_test.go | 8 +++++--- integration-cli/docker_cli_run_unix_test.go | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/daemon/execdriver/lxc/lxc_template.go b/daemon/execdriver/lxc/lxc_template.go index 975c5f19e..6b7996f04 100644 --- a/daemon/execdriver/lxc/lxc_template.go +++ b/daemon/execdriver/lxc/lxc_template.go @@ -91,9 +91,9 @@ lxc.mount.entry = {{$value.Source}} {{escapeFstabSpaces $ROOTFS}}/{{escapeFstabS {{if .Resources}} {{if .Resources.Memory}} lxc.cgroup.memory.limit_in_bytes = {{.Resources.Memory}} -{{with $memSwap := getMemorySwap .Resources}} -lxc.cgroup.memory.memsw.limit_in_bytes = {{$memSwap}} {{end}} +{{if gt .Resources.MemorySwap 0}} +lxc.cgroup.memory.memsw.limit_in_bytes = {{.Resources.MemorySwap}} {{end}} {{if gt .Resources.MemoryReservation 0}} lxc.cgroup.memory.soft_limit_in_bytes = {{.Resources.MemoryReservation}} @@ -209,15 +209,6 @@ func isDirectory(source string) string { return "file" } -func getMemorySwap(v *execdriver.Resources) int64 { - // By default, MemorySwap is set to twice the size of RAM. - // If you want to omit MemorySwap, set it to `-1'. - if v.MemorySwap < 0 { - return 0 - } - return v.Memory * 2 -} - func getLabel(c map[string][]string, name string) string { label := c["label"] for _, l := range label { @@ -242,7 +233,6 @@ func getHostname(env []string) string { func init() { var err error funcMap := template.FuncMap{ - "getMemorySwap": getMemorySwap, "escapeFstabSpaces": escapeFstabSpaces, "formatMountLabel": label.FormatMountLabel, "isDirectory": isDirectory, diff --git a/daemon/execdriver/lxc/lxc_template_unit_test.go b/daemon/execdriver/lxc/lxc_template_unit_test.go index afb5b1eb6..01bc3eaef 100644 --- a/daemon/execdriver/lxc/lxc_template_unit_test.go +++ b/daemon/execdriver/lxc/lxc_template_unit_test.go @@ -34,6 +34,7 @@ func TestLXCConfig(t *testing.T) { memMin = 33554432 memMax = 536870912 mem = memMin + r.Intn(memMax-memMin) + swap = memMax cpuMin = 100 cpuMax = 10000 cpu = cpuMin + r.Intn(cpuMax-cpuMin) @@ -46,8 +47,9 @@ func TestLXCConfig(t *testing.T) { command := &execdriver.Command{ ID: "1", Resources: &execdriver.Resources{ - Memory: int64(mem), - CPUShares: int64(cpu), + Memory: int64(mem), + MemorySwap: int64(swap), + CPUShares: int64(cpu), }, Network: &execdriver.Network{ Mtu: 1500, @@ -63,7 +65,7 @@ func TestLXCConfig(t *testing.T) { fmt.Sprintf("lxc.cgroup.memory.limit_in_bytes = %d", mem)) grepFile(t, p, - fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", mem*2)) + fmt.Sprintf("lxc.cgroup.memory.memsw.limit_in_bytes = %d", swap)) } func TestCustomLxcConfig(t *testing.T) { diff --git a/integration-cli/docker_cli_run_unix_test.go b/integration-cli/docker_cli_run_unix_test.go index 785aad410..e062a0830 100644 --- a/integration-cli/docker_cli_run_unix_test.go +++ b/integration-cli/docker_cli_run_unix_test.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/sysinfo" + "github.com/docker/docker/pkg/units" "github.com/go-check/check" "github.com/kr/pty" ) @@ -435,3 +436,22 @@ func (s *DockerSuite) TestRunInvalidCPUShares(c *check.C) { expected = "The maximum allowed cpu-shares is" c.Assert(out, checker.Contains, expected) } + +func (s *DockerSuite) TestRunWithCorrectMemorySwapOnLXC(c *check.C) { + testRequires(c, memoryLimitSupport) + testRequires(c, swapMemorySupport) + testRequires(c, SameHostDaemon) + + out, _ := dockerCmd(c, "run", "-d", "-m", "16m", "--memory-swap", "64m", "busybox", "top") + if _, err := os.Stat("/sys/fs/cgroup/memory/lxc"); err != nil { + c.Skip("Excecution driver must be LXC for this test") + } + id := strings.TrimSpace(out) + memorySwap, err := ioutil.ReadFile(fmt.Sprintf("/sys/fs/cgroup/memory/lxc/%s/memory.memsw.limit_in_bytes", id)) + c.Assert(err, check.IsNil) + cgSwap, err := strconv.ParseInt(strings.TrimSpace(string(memorySwap)), 10, 64) + c.Assert(err, check.IsNil) + swap, err := units.RAMInBytes("64m") + c.Assert(err, check.IsNil) + c.Assert(cgSwap, check.Equals, swap) +} From 9b313adb51c902d5369d5826df9a977b23b0bab8 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Thu, 24 Sep 2015 12:13:19 +0200 Subject: [PATCH 002/134] daemon: execdriver: lxc: fix cgroup paths When running LXC dind (outer docker is started with native driver) cgroup paths point to `/docker/CID` inside `/proc/self/mountinfo` but these paths aren't mounted (root is wrong). This fix just discard the cgroup dir from mountinfo and set it to root `/`. This patch fixes/skip OOM LXC tests that were failing. Fix #16520 Signed-off-by: Antonio Murdaca Signed-off-by: Antonio Murdaca --- daemon/execdriver/lxc/driver.go | 36 +++++++++++++------ daemon/execdriver/native/driver.go | 1 - .../docker_cli_events_unix_test.go | 2 ++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 30b36e775..12793bd01 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -324,24 +324,20 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd c.ContainerPid = pid - oomKill := false - oomKillNotification, err := notifyOnOOM(cgroupPaths) - if hooks.Start != nil { logrus.Debugf("Invoking startCallback") - hooks.Start(&c.ProcessConfig, pid, oomKillNotification) - + chOOM := make(chan struct{}) + close(chOOM) + hooks.Start(&c.ProcessConfig, pid, chOOM) } + oomKillNotification := notifyChannelOOM(cgroupPaths) + <-waitLock exitCode := getExitCode(c) - if err == nil { - _, oomKill = <-oomKillNotification - logrus.Debugf("oomKill error: %v, waitErr: %v", oomKill, waitErr) - } else { - logrus.Warnf("Your kernel does not support OOM notifications: %s", err) - } + _, oomKill := <-oomKillNotification + logrus.Debugf("oomKill error: %v, waitErr: %v", oomKill, waitErr) // check oom error if oomKill { @@ -351,6 +347,17 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd return execdriver.ExitStatus{ExitCode: exitCode, OOMKilled: oomKill}, waitErr } +func notifyChannelOOM(paths map[string]string) <-chan struct{} { + oom, err := notifyOnOOM(paths) + if err != nil { + logrus.Warnf("Your kernel does not support OOM notifications: %s", err) + c := make(chan struct{}) + close(c) + return c + } + return oom +} + // copy from libcontainer func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { dir := paths["memory"] @@ -386,11 +393,13 @@ func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) { buf := make([]byte, 8) for { if _, err := eventfd.Read(buf); err != nil { + logrus.Warn(err) return } // When a cgroup is destroyed, an event is sent to eventfd. // So if the control path is gone, return instead of notifying. if _, err := os.Lstat(eventControlPath); os.IsNotExist(err) { + logrus.Warn(err) return } ch <- struct{}{} @@ -424,6 +433,11 @@ func cgroupPaths(containerID string) (map[string]string, error) { //unsupported subystem continue } + // if we are running dind + dockerPathIdx := strings.LastIndex(cgroupDir, "docker") + if dockerPathIdx != -1 { + cgroupDir = cgroupDir[:dockerPathIdx-1] + } path := filepath.Join(cgroupRoot, cgroupDir, "lxc", containerID) paths[subsystem] = path } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 09f84a37b..94f200a31 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -167,7 +167,6 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd oom := notifyOnOOM(cont) if hooks.Start != nil { - pid, err := p.Pid() if err != nil { p.Signal(os.Kill) diff --git a/integration-cli/docker_cli_events_unix_test.go b/integration-cli/docker_cli_events_unix_test.go index ca7c09221..9115809b8 100644 --- a/integration-cli/docker_cli_events_unix_test.go +++ b/integration-cli/docker_cli_events_unix_test.go @@ -56,6 +56,7 @@ func (s *DockerSuite) TestEventsRedirectStdout(c *check.C) { func (s *DockerSuite) TestEventsOOMDisableFalse(c *check.C) { testRequires(c, DaemonIsLinux) + testRequires(c, NativeExecDriver) testRequires(c, oomControl) errChan := make(chan error) @@ -103,6 +104,7 @@ func (s *DockerSuite) TestEventsOOMDisableFalse(c *check.C) { func (s *DockerSuite) TestEventsOOMDisableTrue(c *check.C) { testRequires(c, DaemonIsLinux) + testRequires(c, NativeExecDriver) testRequires(c, oomControl) errChan := make(chan error) From fbe20aa39328bff9dc129e99f2368002ea6a5abd Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Tue, 13 Oct 2015 14:18:05 -0700 Subject: [PATCH 003/134] update tests Signed-off-by: Jessica Frazelle --- integration-cli/check_test.go | 1 + integration-cli/docker_cli_diff_test.go | 1 + integration-cli/docker_cli_run_test.go | 40 +++------------------ integration-cli/docker_cli_run_unix_test.go | 2 +- integration-cli/docker_utils.go | 36 +++++++++++++++++++ 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index 575f8ea71..030b07f0e 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -31,6 +31,7 @@ func (s *DockerSuite) TearDownTest(c *check.C) { deleteAllContainers() deleteAllImages() deleteAllVolumes() + deleteAllNetworks() } func init() { diff --git a/integration-cli/docker_cli_diff_test.go b/integration-cli/docker_cli_diff_test.go index 60eff132c..42f1d89fb 100644 --- a/integration-cli/docker_cli_diff_test.go +++ b/integration-cli/docker_cli_diff_test.go @@ -61,6 +61,7 @@ func (s *DockerSuite) TestDiffEnsureOnlyKmsgAndPtmx(c *check.C) { "C /dev": true, "A /dev/full": true, // busybox "C /dev/ptmx": true, // libcontainer + "A /dev/mqueue": true, // lxc "A /dev/kmsg": true, // lxc "A /dev/fd": true, "A /dev/fuse": true, diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index 0ceb768e9..53361dfdf 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -1197,7 +1197,7 @@ func (s *DockerSuite) TestRunNonRootUserResolvName(c *check.C) { // uses the host's /etc/resolv.conf and does not have any dns options provided. func (s *DockerSuite) TestRunResolvconfUpdate(c *check.C) { // Not applicable on Windows as testing unix specific functionality - testRequires(c, SameHostDaemon, DaemonIsLinux) + testRequires(c, SameHostDaemon, DaemonIsLinux, NativeExecDriver) tmpResolvConf := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\n") tmpLocalhostResolvConf := []byte("nameserver 127.0.0.1") @@ -3425,13 +3425,10 @@ func (s *DockerSuite) TestContainersInUserDefinedNetwork(c *check.C) { dockerCmd(c, "run", "-d", "--net=testnetwork", "--name=first", "busybox", "top") c.Assert(waitRun("first"), check.IsNil) dockerCmd(c, "run", "-t", "--net=testnetwork", "--name=second", "busybox", "ping", "-c", "1", "first") - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork") } func (s *DockerSuite) TestContainersInMultipleNetworks(c *check.C) { - testRequires(c, DaemonIsLinux, NotUserNamespace) + testRequires(c, DaemonIsLinux, NotUserNamespace, NativeExecDriver) // Create 2 networks using bridge driver dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork1") dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork2") @@ -3447,14 +3444,10 @@ func (s *DockerSuite) TestContainersInMultipleNetworks(c *check.C) { dockerCmd(c, "network", "connect", "testnetwork2", "second") // Check connectivity between containers dockerCmd(c, "exec", "second", "ping", "-c", "1", "first.testnetwork2") - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork1") - dockerCmd(c, "network", "rm", "testnetwork2") } func (s *DockerSuite) TestContainersNetworkIsolation(c *check.C) { - testRequires(c, DaemonIsLinux, NotUserNamespace) + testRequires(c, DaemonIsLinux, NotUserNamespace, NativeExecDriver) // Create 2 networks using bridge driver dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork1") dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork2") @@ -3478,11 +3471,6 @@ func (s *DockerSuite) TestContainersNetworkIsolation(c *check.C) { // ping must fail again _, _, err = dockerCmdWithError("exec", "first", "ping", "-c", "1", "second") c.Assert(err, check.NotNil) - - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork1") - dockerCmd(c, "network", "rm", "testnetwork2") } func (s *DockerSuite) TestNetworkRmWithActiveContainers(c *check.C) { @@ -3501,17 +3489,14 @@ func (s *DockerSuite) TestNetworkRmWithActiveContainers(c *check.C) { dockerCmd(c, "stop", "first") _, _, err = dockerCmdWithError("network", "rm", "testnetwork1") c.Assert(err, check.NotNil) - - dockerCmd(c, "stop", "second") - // Network delete must succeed after all the connected containers are inactive - dockerCmd(c, "network", "rm", "testnetwork1") } func (s *DockerSuite) TestContainerRestartInMultipleNetworks(c *check.C) { - testRequires(c, DaemonIsLinux, NotUserNamespace) + testRequires(c, DaemonIsLinux, NotUserNamespace, NativeExecDriver) // Create 2 networks using bridge driver dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork1") dockerCmd(c, "network", "create", "-d", "bridge", "testnetwork2") + // Run and connect containers to testnetwork1 dockerCmd(c, "run", "-d", "--net=testnetwork1", "--name=first", "busybox", "top") c.Assert(waitRun("first"), check.IsNil) @@ -3536,11 +3521,6 @@ func (s *DockerSuite) TestContainerRestartInMultipleNetworks(c *check.C) { dockerCmd(c, "start", "second") dockerCmd(c, "exec", "first", "ping", "-c", "1", "second.testnetwork1") dockerCmd(c, "exec", "second", "ping", "-c", "1", "first.testnetwork2") - - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork1") - dockerCmd(c, "network", "rm", "testnetwork2") } func (s *DockerSuite) TestContainerWithConflictingHostNetworks(c *check.C) { @@ -3555,8 +3535,6 @@ func (s *DockerSuite) TestContainerWithConflictingHostNetworks(c *check.C) { // Connecting to the user defined network must fail _, _, err := dockerCmdWithError("network", "connect", "testnetwork1", "first") c.Assert(err, check.NotNil) - dockerCmd(c, "stop", "first") - dockerCmd(c, "network", "rm", "testnetwork1") } func (s *DockerSuite) TestContainerWithConflictingSharedNetwork(c *check.C) { @@ -3574,10 +3552,6 @@ func (s *DockerSuite) TestContainerWithConflictingSharedNetwork(c *check.C) { out, _, err := dockerCmdWithError("network", "connect", "testnetwork1", "second") c.Assert(err, check.NotNil) c.Assert(out, checker.Contains, runconfig.ErrConflictSharedNetwork.Error()) - - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork1") } func (s *DockerSuite) TestContainerWithConflictingNoneNetwork(c *check.C) { @@ -3600,10 +3574,6 @@ func (s *DockerSuite) TestContainerWithConflictingNoneNetwork(c *check.C) { // Connect second container to none network. it must fail as well _, _, err = dockerCmdWithError("network", "connect", "none", "second") c.Assert(err, check.NotNil) - - dockerCmd(c, "stop", "first") - dockerCmd(c, "stop", "second") - dockerCmd(c, "network", "rm", "testnetwork1") } // #11957 - stdin with no tty does not exit if stdin is not closed even though container exited diff --git a/integration-cli/docker_cli_run_unix_test.go b/integration-cli/docker_cli_run_unix_test.go index e062a0830..694890271 100644 --- a/integration-cli/docker_cli_run_unix_test.go +++ b/integration-cli/docker_cli_run_unix_test.go @@ -420,7 +420,7 @@ func (s *DockerSuite) TestRunInvalidCpusetMemsFlagValue(c *check.C) { } func (s *DockerSuite) TestRunInvalidCPUShares(c *check.C) { - testRequires(c, cpuShare) + testRequires(c, cpuShare, NativeExecDriver) out, _, err := dockerCmdWithError("run", "--cpu-shares", "1", "busybox", "echo", "test") c.Assert(err, check.NotNil, check.Commentf(out)) expected := "The minimum allowed cpu-shares is 2" diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index 7ade4706c..caba24ab4 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -506,6 +506,42 @@ func deleteAllContainers() error { return nil } +func deleteAllNetworks() error { + networks, err := getAllNetworks() + if err != nil { + return err + } + var errors []string + for _, n := range networks { + if n.Name != "bridge" { + status, b, err := sockRequest("DELETE", "/networks/"+n.Name, nil) + if err != nil { + errors = append(errors, err.Error()) + continue + } + if status != http.StatusNoContent { + errors = append(errors, fmt.Sprintf("error deleting network %s: %s", n.Name, string(b))) + } + } + } + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, "\n")) + } + return nil +} + +func getAllNetworks() ([]types.NetworkResource, error) { + var networks []types.NetworkResource + _, b, err := sockRequest("GET", "/networks", nil) + if err != nil { + return nil, err + } + if err := json.Unmarshal(b, &networks); err != nil { + return nil, err + } + return networks, nil +} + func deleteAllVolumes() error { volumes, err := getAllVolumes() if err != nil { From 38cd4eb1b2fda92b9043c1f144f14bc3677337c6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 14 Oct 2015 17:34:56 +0200 Subject: [PATCH 004/134] Add bash completion for `docker inspect --size` Signed-off-by: Harald Albers --- contrib/completion/bash/docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 0375e6f95..739d97b2d 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -906,7 +906,7 @@ _docker_inspect() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--format -f --type --help" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--format -f --help --size -s --type" -- "$cur" ) ) ;; *) case $(__docker_value_of_option --type) in From d1e3a75934a34ca17dbdca0e679e3baf97e8f6b5 Mon Sep 17 00:00:00 2001 From: Jian Zhang Date: Mon, 28 Sep 2015 09:45:10 +0800 Subject: [PATCH 005/134] Improve the way we deliver Examples in command line. (Add descriptive titles) Signed-off-by: Jian Zhang --- docs/reference/commandline/build.md | 56 ++++++++++------ docs/reference/commandline/run.md | 99 ++++++++++++++++++----------- 2 files changed, 98 insertions(+), 57 deletions(-) diff --git a/docs/reference/commandline/build.md b/docs/reference/commandline/build.md index 94d2cc764..a0c600759 100644 --- a/docs/reference/commandline/build.md +++ b/docs/reference/commandline/build.md @@ -128,6 +128,8 @@ See also: ## Examples +### Build with PATH + $ docker build . Uploading context 10240 bytes Step 1 : FROM busybox @@ -168,6 +170,31 @@ The transfer of context from the local machine to the Docker daemon is what the If you wish to keep the intermediate containers after the build is complete, you must use `--rm=false`. This does not affect the build cache. +### Build with URL + + $ docker build github.com/creack/docker-firefox + +This will clone the GitHub repository and use the cloned repository as context. +The Dockerfile at the root of the repository is used as Dockerfile. Note that +you can specify an arbitrary Git repository by using the `git://` or `git@` +schema. + +### Build with - + + $ docker build - < Dockerfile + +This will read a Dockerfile from `STDIN` without context. Due to the lack of a +context, no contents of any local directory will be sent to the Docker daemon. +Since there is no context, a Dockerfile `ADD` only works if it refers to a +remote URL. + + $ docker build - < context.tar.gz + +This will build an image for a compressed context read from `STDIN`. Supported +formats are: bzip2, gzip and xz. + +### Usage of .dockerignore + $ docker build . Uploading context 18.829 MB Uploading context @@ -193,29 +220,14 @@ directory from the context. Its effect can be seen in the changed size of the uploaded context. The builder reference contains detailed information on [creating a .dockerignore file](../builder.md#dockerignore-file) +### Tag image (-t) + $ docker build -t vieux/apache:2.0 . This will build like the previous example, but it will then tag the resulting image. The repository name will be `vieux/apache` and the tag will be `2.0` - $ docker build - < Dockerfile - -This will read a Dockerfile from `STDIN` without context. Due to the lack of a -context, no contents of any local directory will be sent to the Docker daemon. -Since there is no context, a Dockerfile `ADD` only works if it refers to a -remote URL. - - $ docker build - < context.tar.gz - -This will build an image for a compressed context read from `STDIN`. Supported -formats are: bzip2, gzip and xz. - - $ docker build github.com/creack/docker-firefox - -This will clone the GitHub repository and use the cloned repository as context. -The Dockerfile at the root of the repository is used as Dockerfile. Note that -you can specify an arbitrary Git repository by using the `git://` or `git@` -schema. +### Specify Dockerfile (-f) $ docker build -f Dockerfile.debug . @@ -248,14 +260,20 @@ the command line. > repeatable builds on remote Docker hosts. This is also the reason why > `ADD ../file` will not work. +### Optional parent cgroup (--cgroup-parent) + When `docker build` is run with the `--cgroup-parent` option the containers used in the build will be run with the [corresponding `docker run` flag](../run.md#specifying-custom-cgroups). +### Set ulimits in container (--ulimit) + Using the `--ulimit` option with `docker build` will cause each build step's container to be started using those [`--ulimit` flag values](../run.md#setting-ulimits-in-a-container). +### Set build-time variables (--build-arg) + You can use `ENV` instructions in a Dockerfile to define variable values. These values persist in the built image. However, often persistence is not what you want. Users want to specify variables differently @@ -263,7 +281,7 @@ depending on which host they build an image on. A good example is `http_proxy` or source versions for pulling intermediate files. The `ARG` instruction lets Dockerfile authors define values that users -can set at build-time using the `---build-arg` flag: +can set at build-time using the `--build-arg` flag: $ docker build --build-arg HTTP_PROXY=http://10.20.30.2:1234 . diff --git a/docs/reference/commandline/run.md b/docs/reference/commandline/run.md index 9db6229a4..1229d1222 100644 --- a/docs/reference/commandline/run.md +++ b/docs/reference/commandline/run.md @@ -92,6 +92,8 @@ and linking containers. ## Examples +### Assign name and allocate psuedo-TTY (--name, -it) + $ docker run --name test -it debian root@d6c0fe130dba:/# exit 13 $ echo $? @@ -106,6 +108,8 @@ In the example, the `bash` shell is quit by entering `exit 13`. This exit code is passed on to the caller of `docker run`, and is recorded in the `test` container's metadata. +### Capture container ID (--cidfile) + $ docker run --cidfile /tmp/docker_test.cid ubuntu echo "test" This will create a container and print `test` to the console. The `cidfile` @@ -113,6 +117,8 @@ flag makes Docker attempt to create a new file and write the container ID to it. If the file exists already, Docker will return an error. Docker will close this file when `docker run` exits. +### Full container capabilities (--privileged) + $ docker run -t -i --rm ubuntu bash root@bc338942ef20:/# mount -t tmpfs none /mnt mount: permission denied @@ -132,11 +138,15 @@ lifts all the limitations enforced by the `device` cgroup controller. In other words, the container can then do almost everything that the host can do. This flag exists to allow special use-cases, like running Docker within Docker. +### Set working directory (-w) + $ docker run -w /path/to/dir/ -i -t ubuntu pwd The `-w` lets the command being executed inside directory given, here `/path/to/dir/`. If the path does not exists it is created inside the container. +### Mount volume (-v, --read-only) + $ docker run -v `pwd`:`pwd` -w `pwd` -i -t ubuntu pwd The `-v` flag mounts the current working directory into the container. The `-w` @@ -166,6 +176,8 @@ binary (such as that provided by [https://get.docker.com]( https://get.docker.com)), you give the container the full access to create and manipulate the host's Docker daemon. +### Publish or expose port (-p, --expose) + $ docker run -p 127.0.0.1:80:8080 ubuntu bash This binds port `8080` of the container to port `80` on `127.0.0.1` of @@ -179,6 +191,8 @@ publishing the port to the host system's interfaces. The [Docker User Guide](../../userguide/dockerlinks.md) explains in detail how to manipulate ports in Docker. +### Set environment variables (-e, --env, --env-file) + $ docker run -e MYVAR1 --env MYVAR2=foo --env-file ./env.list ubuntu bash This sets environmental variables in the container. For illustration all three @@ -247,7 +261,9 @@ An example of a file passed with `--env-file` 123qwe=bar org.spring.config=something -A label is a a `key=value` pair that applies metadata to a container. To label a container with two labels: +### Set metadata on container (-l, --label, --label-file) + +A label is a `key=value` pair that applies metadata to a container. To label a container with two labels: $ docker run -l my-label --label com.example.foo=bar ubuntu bash @@ -281,6 +297,8 @@ For additional information on working with labels, see [*Labels - custom metadata in Docker*](../../userguide/labels-custom-metadata.md) in the Docker User Guide. +### Add link to another container (--link) + $ docker run --link /redis:redis --name console ubuntu bash The `--link` flag will link the container named `/redis` into the newly @@ -295,6 +313,8 @@ example as: The `--name` flag will assign the name `console` to the newly created container. +### Mount volumes from container (--volumes-from) + $ docker run --volumes-from 777f7dc92da7 --volumes-from ba8c0c54f0f2:ro -i -t ubuntu pwd The `--volumes-from` flag mounts all the defined volumes from the referenced @@ -317,6 +337,8 @@ content label. Shared volume labels allow all containers to read/write content. The `Z` option tells Docker to label the content with a private unshared label. Only the current container can use a private volume. +### Attach to STDIN/STDOUT/STDERR (-a) + The `-a` flag tells `docker run` to bind to the container's `STDIN`, `STDOUT` or `STDERR`. This makes it possible to manipulate the output and input as needed. @@ -340,6 +362,8 @@ logs could be retrieved using `docker logs`. This is useful if you need to pipe a file or something else into a container and retrieve the container's ID once the container has finished running. +### Add host device to container (--device) + $ docker run --device=/dev/sdc:/dev/xvdc --device=/dev/sdd --device=/dev/zero:/dev/nulo -i -t ubuntu ls -l /dev/{xvdc,sdd,nulo} brw-rw---- 1 root disk 8, 2 Feb 9 16:05 /dev/xvdc brw-rw---- 1 root disk 8, 3 Feb 9 16:05 /dev/sdd @@ -375,38 +399,7 @@ flag: > that may be removed should not be added to untrusted containers with > `--device`. -**A complete example:** - - $ docker run -d --name static static-web-files sh - $ docker run -d --expose=8098 --name riak riakserver - $ docker run -d -m 100m -e DEVELOPMENT=1 -e BRANCH=example-code -v $(pwd):/app/bin:ro --name app appserver - $ docker run -d -p 1443:443 --dns=10.0.0.1 --dns-search=dev.org -v /var/log/httpd --volumes-from static --link riak --link app -h www.sven.dev.org --name web webserver - $ docker run -t -i --rm --volumes-from web -w /var/log/httpd busybox tail -f access.log - -This example shows five containers that might be set up to test a web -application change: - -1. Start a pre-prepared volume image `static-web-files` (in the background) - that has CSS, image and static HTML in it, (with a `VOLUME` instruction in - the Dockerfile to allow the web server to use those files); -2. Start a pre-prepared `riakserver` image, give the container name `riak` and - expose port `8098` to any containers that link to it; -3. Start the `appserver` image, restricting its memory usage to 100MB, setting - two environment variables `DEVELOPMENT` and `BRANCH` and bind-mounting the - current directory (`$(pwd)`) in the container in read-only mode as `/app/bin`; -4. Start the `webserver`, mapping port `443` in the container to port `1443` on - the Docker server, setting the DNS server to `10.0.0.1` and DNS search - domain to `dev.org`, creating a volume to put the log files into (so we can - access it from another container), then importing the files from the volume - exposed by the `static` container, and linking to all exposed ports from - `riak` and `app`. Lastly, we set the hostname to `web.sven.dev.org` so its - consistent with the pre-generated SSL certificate; -5. Finally, we create a container that runs `tail -f access.log` using the logs - volume from the `web` container, setting the workdir to `/var/log/httpd`. The - `--rm` option means that when the container exits, the container's layer is - removed. - -## Restart policies +### Restart policies (--restart) Use Docker's `--restart` to specify a container's *restart policy*. A restart policy controls whether the Docker daemon restarts a container after exit. @@ -468,7 +461,7 @@ More detailed information on restart policies can be found in the [Restart Policies (--restart)](../run.md#restart-policies-restart) section of the Docker run reference page. -## Adding entries to a container hosts file +### Add entries to container hosts file (--add-host) You can add other hosts into a container's `/etc/hosts` file by using one or more `--add-host` flags. This example adds a static address for a host named @@ -499,7 +492,7 @@ For IPv6 use the `-6` flag instead of the `-4` flag. For other network devices, replace `eth0` with the correct device name (for example `docker0` for the bridge device). -### Setting ulimits in a container +### Set ulimits in container (--ulimit) Since setting `ulimit` settings in a container requires extra privileges not available in the default container, you can set these using the `--ulimit` flag. @@ -519,13 +512,12 @@ available in the default container, you can set these using the `--ulimit` flag. The values are sent to the appropriate `syscall` as they are set. Docker doesn't perform any byte conversion. Take this into account when setting the values. -#### For `nproc` usage: +#### For `nproc` usage Be careful setting `nproc` with the `ulimit` flag as `nproc` is designed by Linux to set the maximum number of processes available to a user, not to a container. For example, start four containers with `daemon` user: - docker run -d -u daemon --ulimit nproc=3 busybox top docker run -d -u daemon --ulimit nproc=3 busybox top docker run -d -u daemon --ulimit nproc=3 busybox top @@ -535,8 +527,39 @@ The 4th container fails and reports "[8] System error: resource temporarily unav This fails because the caller set `nproc=3` resulting in the first three containers using up the three processes quota set for the `daemon` user. -### Stopping a container with a specific signal +### Stop container with signal (--stop-signal) The `--stop-signal` flag sets the system call signal that will be sent to the container to exit. This signal can be a valid unsigned number that matches a position in the kernel's syscall table, for instance 9, or a signal name in the format SIGNAME, for instance SIGKILL. + +### A complete example + + $ docker run -d --name static static-web-files sh + $ docker run -d --expose=8098 --name riak riakserver + $ docker run -d -m 100m -e DEVELOPMENT=1 -e BRANCH=example-code -v $(pwd):/app/bin:ro --name app appserver + $ docker run -d -p 1443:443 --dns=10.0.0.1 --dns-search=dev.org -v /var/log/httpd --volumes-from static --link riak --link app -h www.sven.dev.org --name web webserver + $ docker run -t -i --rm --volumes-from web -w /var/log/httpd busybox tail -f access.log + +This example shows five containers that might be set up to test a web +application change: + +1. Start a pre-prepared volume image `static-web-files` (in the background) + that has CSS, image and static HTML in it, (with a `VOLUME` instruction in + the Dockerfile to allow the web server to use those files); +2. Start a pre-prepared `riakserver` image, give the container name `riak` and + expose port `8098` to any containers that link to it; +3. Start the `appserver` image, restricting its memory usage to 100MB, setting + two environment variables `DEVELOPMENT` and `BRANCH` and bind-mounting the + current directory (`$(pwd)`) in the container in read-only mode as `/app/bin`; +4. Start the `webserver`, mapping port `443` in the container to port `1443` on + the Docker server, setting the DNS server to `10.0.0.1` and DNS search + domain to `dev.org`, creating a volume to put the log files into (so we can + access it from another container), then importing the files from the volume + exposed by the `static` container, and linking to all exposed ports from + `riak` and `app`. Lastly, we set the hostname to `web.sven.dev.org` so its + consistent with the pre-generated SSL certificate; +5. Finally, we create a container that runs `tail -f access.log` using the logs + volume from the `web` container, setting the workdir to `/var/log/httpd`. The + `--rm` option means that when the container exits, the container's layer is + removed. \ No newline at end of file From 17def9a2c6c71c1ac9701332d7beb0d6553a4aba Mon Sep 17 00:00:00 2001 From: Sally O'Malley Date: Wed, 14 Oct 2015 10:45:12 -0400 Subject: [PATCH 006/134] add clarity to -p option Signed-off-by: Sally O'Malley --- man/docker-run.1.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/man/docker-run.1.md b/man/docker-run.1.md index 443933457..e59e414f3 100644 --- a/man/docker-run.1.md +++ b/man/docker-run.1.md @@ -353,10 +353,14 @@ ports and the exposed ports, use `docker port`. **-p**, **--publish**=[] Publish a container's port, or range of ports, to the host. - format: ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort | containerPort - Both hostPort and containerPort can be specified as a range of ports. - When specifying ranges for both, the number of container ports in the range must match the number of host ports in the range. (e.g., `-p 1234-1236:1234-1236/tcp`) - (use 'docker port' to see the actual mapping) + + Format: `ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort | containerPort` +Both hostPort and containerPort can be specified as a range of ports. +When specifying ranges for both, the number of container ports in the range must match the number of host ports in the range. +(e.g., `docker run -p 1234-1236:1222-1224 --name thisWorks -t busybox` +but not `docker run -p 1230-1236:1230-1240 --name RangeContainerPortsBiggerThanRangeHostPorts -t busybox`) +With ip: `docker run -p 127.0.0.1:$HOSTPORT:$CONTAINERPORT --name CONTAINER -t someimage` +Use `docker port` to see the actual mapping: `docker port CONTAINER $CONTAINERPORT` **--pid**=host Set the PID mode for the container From fb525a33e1e6e0a1cdc690ab8514553be120058b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 13 Oct 2015 17:02:55 -0700 Subject: [PATCH 007/134] only display 'Engine Version' when it's not empty Signed-off-by: Victor Vieux --- api/client/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client/info.go b/api/client/info.go index 22b7ebb80..5cef4b0d7 100644 --- a/api/client/info.go +++ b/api/client/info.go @@ -35,7 +35,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Containers: %d\n", info.Containers) fmt.Fprintf(cli.out, "Images: %d\n", info.Images) - fmt.Fprintf(cli.out, "Engine Version: %s\n", info.ServerVersion) + ioutils.FprintfIfNotEmpty(cli.out, "Engine Version: %s\n", info.ServerVersion) ioutils.FprintfIfNotEmpty(cli.out, "Storage Driver: %s\n", info.Driver) if info.DriverStatus != nil { for _, pair := range info.DriverStatus { From ac4349bd30b5d1c041205eb3eff1245570f89cab Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 14 Oct 2015 17:24:13 -0700 Subject: [PATCH 008/134] use Server Version Signed-off-by: Victor Vieux --- api/client/info.go | 2 +- docs/reference/commandline/info.md | 2 +- docs/userguide/labels-custom-metadata.md | 2 +- man/docker-info.1.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/client/info.go b/api/client/info.go index 5cef4b0d7..06171826a 100644 --- a/api/client/info.go +++ b/api/client/info.go @@ -35,7 +35,7 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Containers: %d\n", info.Containers) fmt.Fprintf(cli.out, "Images: %d\n", info.Images) - ioutils.FprintfIfNotEmpty(cli.out, "Engine Version: %s\n", info.ServerVersion) + ioutils.FprintfIfNotEmpty(cli.out, "Server Version: %s\n", info.ServerVersion) ioutils.FprintfIfNotEmpty(cli.out, "Storage Driver: %s\n", info.Driver) if info.DriverStatus != nil { for _, pair := range info.DriverStatus { diff --git a/docs/reference/commandline/info.md b/docs/reference/commandline/info.md index 446b68697..1794df40b 100644 --- a/docs/reference/commandline/info.md +++ b/docs/reference/commandline/info.md @@ -22,7 +22,7 @@ For example: $ docker -D info Containers: 14 Images: 52 - Engine Version: 1.9.0 + Server Version: 1.9.0 Storage Driver: aufs Root Dir: /var/lib/docker/aufs Backing Filesystem: extfs diff --git a/docs/userguide/labels-custom-metadata.md b/docs/userguide/labels-custom-metadata.md index 3bab14835..e4ac7c4cd 100644 --- a/docs/userguide/labels-custom-metadata.md +++ b/docs/userguide/labels-custom-metadata.md @@ -188,7 +188,7 @@ These labels appear as part of the `docker info` output for the daemon: $ docker -D info Containers: 12 Images: 672 - Engine Version: 1.9.0 + Server Version: 1.9.0 Storage Driver: aufs Root Dir: /var/lib/docker/aufs Backing Filesystem: extfs diff --git a/man/docker-info.1.md b/man/docker-info.1.md index 1aca0b5b2..f67a4fb00 100644 --- a/man/docker-info.1.md +++ b/man/docker-info.1.md @@ -33,7 +33,7 @@ Here is a sample output: # docker info Containers: 14 Images: 52 - Engine Version: 1.9.0 + Server Version: 1.9.0 Storage Driver: aufs Root Dir: /var/lib/docker/aufs Dirs: 80 From 14897a0b47d46b74dc36b8e4e710929c1f45abee Mon Sep 17 00:00:00 2001 From: Madhu Venugopal Date: Thu, 15 Oct 2015 03:10:39 -0700 Subject: [PATCH 009/134] Added `network` to docker --help and help cleanup Fixes https://github.com/docker/docker/issues/16909 Signed-off-by: Madhu Venugopal --- cli/common.go | 1 + contrib/completion/bash/docker | 2 +- docs/reference/commandline/network_ls.md | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/common.go b/cli/common.go index d3aa391be..c03d9a90e 100644 --- a/cli/common.go +++ b/cli/common.go @@ -45,6 +45,7 @@ var dockerCommands = []Command{ {"login", "Register or log in to a Docker registry"}, {"logout", "Log out from a Docker registry"}, {"logs", "Fetch the logs of a container"}, + {"network", "Manage Docker networks"}, {"pause", "Pause all processes within a container"}, {"port", "List port mappings or a specific mapping for the CONTAINER"}, {"ps", "List containers"}, diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 739d97b2d..c1720e458 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -1060,7 +1060,7 @@ _docker_network_ls() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --latest -l -n --no-trunc --quiet -q" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --no-trunc --quiet -q" -- "$cur" ) ) ;; esac } diff --git a/docs/reference/commandline/network_ls.md b/docs/reference/commandline/network_ls.md index 0d2294e6f..09f290f06 100644 --- a/docs/reference/commandline/network_ls.md +++ b/docs/reference/commandline/network_ls.md @@ -14,8 +14,6 @@ parent = "smn_cli" Lists all the networks created by the user --help=false Print usage - -l, --latest=false Show the latest network created - -n=-1 Show n last created networks --no-trunc=false Do not truncate the output -q, --quiet=false Only display numeric IDs From bbd690e2e2f42a1130342d6074c5a4ba10c0f7d7 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 15 Oct 2015 09:54:48 +0200 Subject: [PATCH 010/134] Add zsh completion for '--ipam-driver --subnet --ip-range --gateway --aux-address' for 'docker network create' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index cd1d66789..b33fd9884 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -253,6 +253,11 @@ __docker_network_subcommand() { _arguments -A '-*' \ $opts_help \ "($help -d --driver)"{-d,--driver=}"[Driver to manage the Network]:driver:(null host bridge overlay)" \ + "($help)--ipam-driver=-[IP Address Management Driver]:driver:(default)" \ + "($help)*--subnet=-[Subnet in CIDR format that represents a network segment]:IP/mask: " \ + "($help)*--ip-range=-[Allocate container ip from a sub-range]:IP/mask: " \ + "($help)*--gateway=-[ipv4 or ipv6 Gateway for the master subnet]:IP: " \ + "($help)*--aux-address[Auxiliary ipv4 or ipv6 addresses used by network driver]:key=IP: " \ "($help -)1:Network Name: " && ret=0 ;; (inspect|rm) From e51ffc7dd3456655ea9c7b7ca2cceaf959989cf5 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 15 Oct 2015 21:12:31 +0200 Subject: [PATCH 011/134] Remove '-n -l --latest' options from 'docker network ls' in zsh completion Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index b33fd9884..813878a89 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -268,8 +268,6 @@ __docker_network_subcommand() { (ls) _arguments \ $opts_help \ - "($help -l --latest)"{-l,--latest}"[Show the latest network created]" \ - "($help)-n=-[Show n last created networks]:Number of networks: " \ "($help)--no-trunc[Do not truncate the output]" \ "($help -q --quiet)"{-q,--quiet}"[Only display numeric IDs]" && ret=0 ;; From 90d352e76513c23bc9d2a6f4237e5d26ddbc5af6 Mon Sep 17 00:00:00 2001 From: Derek Ch Date: Sat, 10 Oct 2015 03:24:21 +0730 Subject: [PATCH 012/134] fix a race crash when building with "ADD some-broken.tar.xz ..." MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The race is between pools.Put which calls buf.Reset and exec.Cmd doing io.Copy from the buffer; it caused a runtime crash, as described in #16924: ``` docker-daemon cat the-tarball.xz | xz -d -c -q | docker-untar /path/to/... (aufs ) ``` When docker-untar side fails (like try to set xattr on aufs, or a broken tar), invokeUnpack will be responsible to exhaust all input, otherwise `xz` will be write pending for ever. this change add a receive only channel to cmdStream, and will close it to notify it's now safe to close the input stream; in CmdStream the change to use Stdin / Stdout / Stderr keeps the code simple, os/exec.Cmd will spawn goroutines and call io.Copy automatically. the CmdStream is actually called in the same file only, change it lowercase to mark as private. [...] INFO[0000] Docker daemon commit=0a8c2e3 execdriver=native-0.2 graphdriver=aufs version=1.8.2 DEBU[0006] Calling POST /build INFO[0006] POST /v1.20/build?cgroupparent=&cpuperiod=0&cpuquota=0&cpusetcpus=&cpusetmems=&cpushares=0&dockerfile=Dockerfile&memory=0&memswap=0&rm=1&t=gentoo-x32&ulimits=null DEBU[0008] [BUILDER] Cache miss DEBU[0009] Couldn't untar /home/lib-docker-v1.8.2-tmp/tmp/docker-build316710953/stage3-x32-20151004.tar.xz to /home/lib-docker-v1.8.2-tmp/aufs/mnt/d909abb87150463939c13e8a349b889a72d9b14f0cfcab42a8711979be285537: Untar re-exec error: exit status 1: output: operation not supported DEBU[0009] CopyFileWithTar(/home/lib-docker-v1.8.2-tmp/tmp/docker-build316710953/stage3-x32-20151004.tar.xz, /home/lib-docker-v1.8.2-tmp/aufs/mnt/d909abb87150463939c13e8a349b889a72d9b14f0cfcab42a8711979be285537/) panic: runtime error: slice bounds out of range goroutine 42 [running]: bufio.(*Reader).fill(0xc208187800) /usr/local/go/src/bufio/bufio.go:86 +0x2db bufio.(*Reader).WriteTo(0xc208187800, 0x7ff39602d150, 0xc2083f11a0, 0x508000, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:449 +0x27e io.Copy(0x7ff39602d150, 0xc2083f11a0, 0x7ff3960261f8, 0xc208187800, 0x0, 0x0, 0x0) /usr/local/go/src/io/io.go:354 +0xb2 github.com/docker/docker/pkg/archive.funcĀ·006() /go/src/github.com/docker/docker/pkg/archive/archive.go:817 +0x71 created by github.com/docker/docker/pkg/archive.CmdStream /go/src/github.com/docker/docker/pkg/archive/archive.go:819 +0x1ec goroutine 1 [chan receive]: main.(*DaemonCli).CmdDaemon(0xc20809da30, 0xc20800a020, 0xd, 0xd, 0x0, 0x0) /go/src/github.com/docker/docker/docker/daemon.go:289 +0x1781 reflect.callMethod(0xc208140090, 0xc20828fce0) /usr/local/go/src/reflect/value.go:605 +0x179 reflect.methodValueCall(0xc20800a020, 0xd, 0xd, 0x1, 0xc208140090, 0x0, 0x0, 0xc208140090, 0x0, 0x45343f, ...) /usr/local/go/src/reflect/asm_amd64.s:29 +0x36 github.com/docker/docker/cli.(*Cli).Run(0xc208129fb0, 0xc20800a010, 0xe, 0xe, 0x0, 0x0) /go/src/github.com/docker/docker/cli/cli.go:89 +0x38e main.main() /go/src/github.com/docker/docker/docker/docker.go:69 +0x428 goroutine 5 [syscall]: os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:21 +0x1f created by os/signal.initĀ·1 /usr/local/go/src/os/signal/signal_unix.go:27 +0x35 Signed-off-by: Derek Ch --- pkg/archive/archive.go | 70 +++++++++++-------------------- pkg/archive/archive_test.go | 6 +-- pkg/chrootarchive/archive_unix.go | 6 +++ 3 files changed, 34 insertions(+), 48 deletions(-) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 69b7beebf..50fbbba0b 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -20,6 +20,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/promise" "github.com/docker/docker/pkg/system" @@ -116,10 +117,10 @@ func DetectCompression(source []byte) Compression { return Uncompressed } -func xzDecompress(archive io.Reader) (io.ReadCloser, error) { +func xzDecompress(archive io.Reader) (io.ReadCloser, <-chan struct{}, error) { args := []string{"xz", "-d", "-c", "-q"} - return CmdStream(exec.Command(args[0], args[1:]...), archive) + return cmdStream(exec.Command(args[0], args[1:]...), archive) } // DecompressStream decompress the archive and returns a ReaderCloser with the decompressed archive. @@ -148,12 +149,15 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) { readBufWrapper := p.NewReadCloserWrapper(buf, bz2Reader) return readBufWrapper, nil case Xz: - xzReader, err := xzDecompress(buf) + xzReader, chdone, err := xzDecompress(buf) if err != nil { return nil, err } readBufWrapper := p.NewReadCloserWrapper(buf, xzReader) - return readBufWrapper, nil + return ioutils.NewReadCloserWrapper(readBufWrapper, func() error { + <-chdone + return readBufWrapper.Close() + }), nil default: return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) } @@ -925,57 +929,33 @@ func CopyFileWithTar(src, dst string) (err error) { return defaultArchiver.CopyFileWithTar(src, dst) } -// CmdStream executes a command, and returns its stdout as a stream. +// cmdStream executes a command, and returns its stdout as a stream. // If the command fails to run or doesn't complete successfully, an error // will be returned, including anything written on stderr. -func CmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { - if input != nil { - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - // Write stdin if any - go func() { - io.Copy(stdin, input) - stdin.Close() - }() - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, err - } +func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, <-chan struct{}, error) { + chdone := make(chan struct{}) + cmd.Stdin = input pipeR, pipeW := io.Pipe() - errChan := make(chan []byte) - // Collect stderr, we will use it in case of an error - go func() { - errText, e := ioutil.ReadAll(stderr) - if e != nil { - errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")") - } - errChan <- errText - }() + cmd.Stdout = pipeW + var errBuf bytes.Buffer + cmd.Stderr = &errBuf + + // Run the command and return the pipe + if err := cmd.Start(); err != nil { + return nil, nil, err + } + // Copy stdout to the returned pipe go func() { - _, err := io.Copy(pipeW, stdout) - if err != nil { - pipeW.CloseWithError(err) - } - errText := <-errChan if err := cmd.Wait(); err != nil { - pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errText)) + pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) } else { pipeW.Close() } + close(chdone) }() - // Run the command and return the pipe - if err := cmd.Start(); err != nil { - return nil, err - } - return pipeR, nil + + return pipeR, chdone, nil } // NewTempArchive reads the content of src into a temporary file, and returns the contents diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index b9bfc2390..6c54c02d1 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -160,7 +160,7 @@ func TestExtensionXz(t *testing.T) { func TestCmdStreamLargeStderr(t *testing.T) { cmd := exec.Command("/bin/sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello") - out, err := CmdStream(cmd, nil) + out, _, err := cmdStream(cmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) } @@ -181,7 +181,7 @@ func TestCmdStreamLargeStderr(t *testing.T) { func TestCmdStreamBad(t *testing.T) { badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") - out, err := CmdStream(badCmd, nil) + out, _, err := cmdStream(badCmd, nil) if err != nil { t.Fatalf("Failed to start command: %s", err) } @@ -196,7 +196,7 @@ func TestCmdStreamBad(t *testing.T) { func TestCmdStreamGood(t *testing.T) { cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0") - out, err := CmdStream(cmd, nil) + out, _, err := cmdStream(cmd, nil) if err != nil { t.Fatal(err) } diff --git a/pkg/chrootarchive/archive_unix.go b/pkg/chrootarchive/archive_unix.go index 83331425f..51a43f67d 100644 --- a/pkg/chrootarchive/archive_unix.go +++ b/pkg/chrootarchive/archive_unix.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "os" "runtime" "syscall" @@ -79,6 +80,11 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T w.Close() if err := cmd.Wait(); err != nil { + // when `xz -d -c -q | docker-untar ...` failed on docker-untar side, + // we need to exhaust `xz`'s output, otherwise the `xz` side will be + // pending on write pipe forever + io.Copy(ioutil.Discard, decompressedArchive) + return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output) } return nil From 374849407350ba79ed7029e08815277ad2823feb Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Fri, 16 Oct 2015 06:07:52 -0700 Subject: [PATCH 013/134] Removing extra tic Signed-off-by: Mary Anthony --- docs/reference/logging/fluentd.md | 2 +- docs/reference/logging/overview.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/logging/fluentd.md b/docs/reference/logging/fluentd.md index 5e9aaad4c..c0fed799e 100644 --- a/docs/reference/logging/fluentd.md +++ b/docs/reference/logging/fluentd.md @@ -14,7 +14,7 @@ weight=2 The `fluentd` logging driver sends container logs to the [Fluentd](http://www.fluentd.org/) collector as structured log data. Then, users can use any of the [various output plugins of -Fluentd](http://www.fluentd.org/plugins) to write these logs to various +Fluentd](http://dwww.fluentd.org/plugins) to write these logs to various destinations. In addition to the log message itself, the `fluentd` log diff --git a/docs/reference/logging/overview.md b/docs/reference/logging/overview.md index ad5847418..86042084c 100644 --- a/docs/reference/logging/overview.md +++ b/docs/reference/logging/overview.md @@ -39,7 +39,7 @@ Then, run a container and specify values for the `labels` or `env`. For example ``` docker run --label foo=bar -e fizz=buzz -d -P training/webapp python app.py -```` +``` This adds additional fields to the log depending on the driver, e.g. for `json-file` that looks like: From 78afc5876a79467cca706de66609e12dbce57916 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Fri, 16 Oct 2015 06:09:03 -0700 Subject: [PATCH 014/134] bad d Signed-off-by: Mary Anthony --- docs/reference/logging/fluentd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/logging/fluentd.md b/docs/reference/logging/fluentd.md index c0fed799e..5e9aaad4c 100644 --- a/docs/reference/logging/fluentd.md +++ b/docs/reference/logging/fluentd.md @@ -14,7 +14,7 @@ weight=2 The `fluentd` logging driver sends container logs to the [Fluentd](http://www.fluentd.org/) collector as structured log data. Then, users can use any of the [various output plugins of -Fluentd](http://dwww.fluentd.org/plugins) to write these logs to various +Fluentd](http://www.fluentd.org/plugins) to write these logs to various destinations. In addition to the log message itself, the `fluentd` log From 447c28b165636483076eb186ec5fac7ce2c9fd2b Mon Sep 17 00:00:00 2001 From: liaoqingwei Date: Wed, 14 Oct 2015 22:33:13 +0800 Subject: [PATCH 015/134] Use of checkers on docker_cli_network_unix_test.go. Signed-off-by: liaoqingwei --- .../docker_cli_network_unix_test.go | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/integration-cli/docker_cli_network_unix_test.go b/integration-cli/docker_cli_network_unix_test.go index c25bd8840..592104728 100644 --- a/integration-cli/docker_cli_network_unix_test.go +++ b/integration-cli/docker_cli_network_unix_test.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/docker/docker/api/types" + "github.com/docker/docker/pkg/integration/checker" "github.com/docker/libnetwork/driverapi" "github.com/go-check/check" ) @@ -43,9 +44,7 @@ func (s *DockerNetworkSuite) TearDownTest(c *check.C) { func (s *DockerNetworkSuite) SetUpSuite(c *check.C) { mux := http.NewServeMux() s.server = httptest.NewServer(mux) - if s.server == nil { - c.Fatal("Failed to start a HTTP Server") - } + c.Assert(s.server, check.NotNil, check.Commentf("Failed to start a HTTP Server")) mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") @@ -67,14 +66,12 @@ func (s *DockerNetworkSuite) SetUpSuite(c *check.C) { fmt.Fprintf(w, "null") }) - if err := os.MkdirAll("/etc/docker/plugins", 0755); err != nil { - c.Fatal(err) - } + err := os.MkdirAll("/etc/docker/plugins", 0755) + c.Assert(err, checker.IsNil) fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", dummyNetworkDriver) - if err := ioutil.WriteFile(fileName, []byte(s.server.URL), 0644); err != nil { - c.Fatal(err) - } + err = ioutil.WriteFile(fileName, []byte(s.server.URL), 0644) + c.Assert(err, checker.IsNil) } func (s *DockerNetworkSuite) TearDownSuite(c *check.C) { @@ -84,9 +81,8 @@ func (s *DockerNetworkSuite) TearDownSuite(c *check.C) { s.server.Close() - if err := os.RemoveAll("/etc/docker/plugins"); err != nil { - c.Fatal(err) - } + err := os.RemoveAll("/etc/docker/plugins") + c.Assert(err, checker.IsNil) } func assertNwIsAvailable(c *check.C, name string) { @@ -140,8 +136,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkConnectDisconnect(c *check.C) { assertNwIsAvailable(c, "test") nr := getNwResource(c, "test") - c.Assert(nr.Name, check.Equals, "test") - c.Assert(len(nr.Containers), check.Equals, 0) + c.Assert(nr.Name, checker.Equals, "test") + c.Assert(len(nr.Containers), checker.Equals, 0) // run a container out, _ := dockerCmd(c, "run", "-d", "--name", "test", "busybox", "top") @@ -153,20 +149,20 @@ func (s *DockerNetworkSuite) TestDockerNetworkConnectDisconnect(c *check.C) { // inspect the network to make sure container is connected nr = getNetworkResource(c, nr.ID) - c.Assert(len(nr.Containers), check.Equals, 1) + c.Assert(len(nr.Containers), checker.Equals, 1) c.Assert(nr.Containers[containerID], check.NotNil) // check if container IP matches network inspect ip, _, err := net.ParseCIDR(nr.Containers[containerID].IPv4Address) c.Assert(err, check.IsNil) containerIP := findContainerIP(c, "test") - c.Assert(ip.String(), check.Equals, containerIP) + c.Assert(ip.String(), checker.Equals, containerIP) // disconnect container from the network dockerCmd(c, "network", "disconnect", "test", containerID) nr = getNwResource(c, "test") - c.Assert(nr.Name, check.Equals, "test") - c.Assert(len(nr.Containers), check.Equals, 0) + c.Assert(nr.Name, checker.Equals, "test") + c.Assert(len(nr.Containers), checker.Equals, 0) // check if network connect fails for inactive containers dockerCmd(c, "stop", containerID) @@ -223,13 +219,13 @@ func (s *DockerNetworkSuite) TestDockerNetworkInspect(c *check.C) { assertNwIsAvailable(c, "br0") nr := getNetworkResource(c, "br0") - c.Assert(nr.Driver, check.Equals, "bridge") - c.Assert(nr.Scope, check.Equals, "local") - c.Assert(nr.IPAM.Driver, check.Equals, "default") - c.Assert(len(nr.IPAM.Config), check.Equals, 1) - c.Assert(nr.IPAM.Config[0].Subnet, check.Equals, "172.28.0.0/16") - c.Assert(nr.IPAM.Config[0].IPRange, check.Equals, "172.28.5.0/24") - c.Assert(nr.IPAM.Config[0].Gateway, check.Equals, "172.28.5.254") + c.Assert(nr.Driver, checker.Equals, "bridge") + c.Assert(nr.Scope, checker.Equals, "local") + c.Assert(nr.IPAM.Driver, checker.Equals, "default") + c.Assert(len(nr.IPAM.Config), checker.Equals, 1) + c.Assert(nr.IPAM.Config[0].Subnet, checker.Equals, "172.28.0.0/16") + c.Assert(nr.IPAM.Config[0].IPRange, checker.Equals, "172.28.5.0/24") + c.Assert(nr.IPAM.Config[0].Gateway, checker.Equals, "172.28.5.254") dockerCmd(c, "network", "rm", "br0") } From 02a8baf069d2eef63dae18956d6a9e9a94521e9c Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 16 Oct 2015 08:48:12 +0200 Subject: [PATCH 016/134] Add zsh completion for 'docker build --build-arg' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 813878a89..ef1b580ef 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -459,6 +459,7 @@ __docker_subcommand() { _arguments \ $opts_help \ $opts_cpumemlimit \ + "($help)*--build-arg[Set build-time variables]:=: " \ "($help -f --file)"{-f,--file=-}"[Name of the Dockerfile]:Dockerfile:_files" \ "($help)--force-rm[Always remove intermediate containers]" \ "($help)--no-cache[Do not use cache when building the image]" \ From b4207be09d9acc2d9857a90044792b886c2c47f3 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 16 Oct 2015 08:30:56 +0200 Subject: [PATCH 017/134] Add zsh completion for 'unless-stopped' restart policy Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index ef1b580ef..8af42b475 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -438,7 +438,7 @@ __docker_subcommand() { "($help)--pid=-[PID namespace to use]:PID: " "($help)--privileged[Give extended privileges to this container]" "($help)--read-only[Mount the container's root filesystem as read only]" - "($help)--restart=-[Restart policy]:restart policy:(no on-failure always)" + "($help)--restart=-[Restart policy]:restart policy:(no on-failure always unless-stopped)" "($help)*--security-opt=-[Security options]:security option: " "($help -t --tty)"{-t,--tty}"[Allocate a pseudo-tty]" "($help -u --user)"{-u,--user=-}"[Username or UID]:user:_users" From daa20c724af3ac626e4e70b0954634f72c29e348 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 16 Oct 2015 08:37:16 +0200 Subject: [PATCH 018/134] Deprecate 'docker run -c' option in zsh completion Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 8af42b475..a0f96c12f 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -394,7 +394,7 @@ __docker_subcommand() { opts_help=("(: -)--help[Print usage]") opts_cpumemlimit=( - "($help -c --cpu-shares)"{-c,--cpu-shares=-}"[CPU shares (relative weight)]:CPU shares:(0 10 100 200 500 800 1000)" + "($help)--cpu-shares=-[CPU shares (relative weight)]:CPU shares:(0 10 100 200 500 800 1000)" "($help)--cgroup-parent=-[Parent cgroup for the container]:cgroup: " "($help)--cpu-period=-[Limit the CPU CFS (Completely Fair Scheduler) period]:CPU period: " "($help)--cpu-quota=-[Limit the CPU CFS (Completely Fair Scheduler) quota]:CPU quota: " From 71f5c74a048fe1e6725bea05049f159b96653587 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 15 Oct 2015 15:50:13 -0400 Subject: [PATCH 019/134] Correct API docs for /images/create Signed-off-by: Daniel Nephin --- docs/reference/api/docker_remote_api_v1.21.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/reference/api/docker_remote_api_v1.21.md b/docs/reference/api/docker_remote_api_v1.21.md index e2afd3b6a..3b6f1da01 100644 --- a/docs/reference/api/docker_remote_api_v1.21.md +++ b/docs/reference/api/docker_remote_api_v1.21.md @@ -1465,12 +1465,15 @@ a base64-encoded AuthConfig object. Query Parameters: -- **fromImage** – Name of the image to pull. +- **fromImage** – Name of the image to pull. The name may include a tag or + digest. This parameter may only be used when pulling an image. - **fromSrc** – Source to import. The value may be a URL from which the image can be retrieved or `-` to read the image from the request body. -- **repo** – Repository name. -- **tag** – Tag. -- **registry** – The registry to pull from. + This parameter may only be used when importing an image. +- **repo** – Repository name given to an image when it is imported. + The repo may include a tag. This parameter may only be used when importing + an image. +- **tag** – Tag or digest. Request Headers: From 417386caa6f44b5d11118a3c6df0219a59c60196 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Thu, 15 Oct 2015 15:43:24 -0700 Subject: [PATCH 020/134] updating notary and gotuf with latest bugfixes Signed-off-by: David Lawrence --- hack/vendor.sh | 4 +- .../docker/notary/client/changelist/change.go | 22 ++++++ .../github.com/docker/notary/client/client.go | 73 +++++++++++++++++-- .../docker/notary/client/helpers.go | 46 ++++++++++-- .../endophage/gotuf/client/client.go | 61 +++++++--------- .../endophage/gotuf/client/errors.go | 8 ++ .../github.com/endophage/gotuf/data/keys.go | 2 +- .../github.com/endophage/gotuf/data/roles.go | 3 +- .../endophage/gotuf/store/httpstore.go | 1 + vendor/src/github.com/endophage/gotuf/tuf.go | 36 +++++++-- 10 files changed, 199 insertions(+), 57 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index f78e44497..71b7807bf 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -41,8 +41,8 @@ clone git github.com/boltdb/bolt v1.0 clone git github.com/docker/distribution 20c4b7a1805a52753dfd593ee1cc35558722a0ce # docker/1.9 branch clone git github.com/vbatts/tar-split v0.9.10 -clone git github.com/docker/notary ac05822d7d71ef077df3fc24f506672282a1feea -clone git github.com/endophage/gotuf 9bcdad0308e34a49f38448b8ad436ad8860825ce +clone git github.com/docker/notary 089d8450d8928aa1c58fd03f09cabbde9bcb4590 +clone git github.com/endophage/gotuf 876c31a61bc4aa0dae09bb8ef3946dc26dd04924 clone git github.com/jfrazelle/go 6e461eb70cb4187b41a84e9a567d7137bdbe0f16 clone git github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c diff --git a/vendor/src/github.com/docker/notary/client/changelist/change.go b/vendor/src/github.com/docker/notary/client/changelist/change.go index 867c23051..dfdaed5c3 100644 --- a/vendor/src/github.com/docker/notary/client/changelist/change.go +++ b/vendor/src/github.com/docker/notary/client/changelist/change.go @@ -1,5 +1,9 @@ package changelist +import ( + "github.com/endophage/gotuf/data" +) + // Scopes for TufChanges are simply the TUF roles. // Unfortunately because of targets delegations, we can only // cover the base roles. @@ -10,6 +14,17 @@ const ( ScopeTimestamp = "timestamp" ) +// Types for TufChanges are namespaced by the Role they +// are relevant for. The Root and Targets roles are the +// only ones for which user action can cause a change, as +// all changes in Snapshot and Timestamp are programatically +// generated base on Root and Targets changes. +const ( + TypeRootRole = "role" + TypeTargetsTarget = "target" + TypeTargetsDelegation = "delegation" +) + // TufChange represents a change to a TUF repo type TufChange struct { // Abbreviated because Go doesn't permit a field and method of the same name @@ -20,6 +35,13 @@ type TufChange struct { Data []byte `json:"data"` } +// TufRootData represents a modification of the keys associated +// with a role that appears in the root.json +type TufRootData struct { + Keys []data.TUFKey `json:"keys"` + RoleName string `json:"role"` +} + // NewTufChange initializes a tufChange object func NewTufChange(action string, role, changeType, changePath string, content []byte) *TufChange { return &TufChange{ diff --git a/vendor/src/github.com/docker/notary/client/client.go b/vendor/src/github.com/docker/notary/client/client.go index ee376053c..ca57a89f1 100644 --- a/vendor/src/github.com/docker/notary/client/client.go +++ b/vendor/src/github.com/docker/notary/client/client.go @@ -245,6 +245,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error { if err != nil { return err } + defer cl.Close() logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length) meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes} @@ -258,7 +259,7 @@ func (r *NotaryRepository) AddTarget(target *Target) error { if err != nil { return err } - return cl.Close() + return nil } // RemoveTarget creates a new changelist entry to remove a target from the repository @@ -326,6 +327,17 @@ func (r *NotaryRepository) GetTargetByName(name string) (*Target, error) { return &Target{Name: name, Hashes: meta.Hashes, Length: meta.Length}, nil } +// GetChangelist returns the list of the repository's unpublished changes +func (r *NotaryRepository) GetChangelist() (changelist.Changelist, error) { + changelistDir := filepath.Join(r.tufRepoPath, "changelist") + cl, err := changelist.NewFileChangelist(changelistDir) + if err != nil { + logrus.Debug("Error initializing changelist") + return nil, err + } + return cl, nil +} + // Publish pushes the local changes in signed material to the remote notary-server // Conceptually it performs an operation similar to a `git rebase` func (r *NotaryRepository) Publish() error { @@ -371,11 +383,8 @@ func (r *NotaryRepository) Publish() error { return err } } - // load the changelist for this repo - changelistDir := filepath.Join(r.tufRepoPath, "changelist") - cl, err := changelist.NewFileChangelist(changelistDir) + cl, err := r.GetChangelist() if err != nil { - logrus.Debug("Error initializing changelist") return err } // apply the changelist to the repo @@ -445,7 +454,7 @@ func (r *NotaryRepository) Publish() error { // This is not a critical problem when only a single host is pushing // but will cause weird behaviour if changelist cleanup is failing // and there are multiple hosts writing to the repo. - logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", changelistDir) + logrus.Warn("Unable to clear changelist. You may want to manually delete the folder ", filepath.Join(r.tufRepoPath, "changelist")) } return nil } @@ -596,3 +605,55 @@ func (r *NotaryRepository) bootstrapClient() (*tufclient.Client, error) { r.fileStore, ), nil } + +// RotateKeys removes all existing keys associated with role and adds +// the keys specified by keyIDs to the role. These changes are staged +// in a changelist until publish is called. +func (r *NotaryRepository) RotateKeys() error { + for _, role := range []string{"targets", "snapshot"} { + key, err := r.cryptoService.Create(role, data.ECDSAKey) + if err != nil { + return err + } + err = r.rootFileKeyChange(role, changelist.ActionCreate, key) + if err != nil { + return err + } + } + return nil +} + +func (r *NotaryRepository) rootFileKeyChange(role, action string, key data.PublicKey) error { + cl, err := changelist.NewFileChangelist(filepath.Join(r.tufRepoPath, "changelist")) + if err != nil { + return err + } + defer cl.Close() + + k, ok := key.(*data.TUFKey) + if !ok { + return errors.New("Invalid key type found during rotation.") + } + + meta := changelist.TufRootData{ + RoleName: role, + Keys: []data.TUFKey{*k}, + } + metaJSON, err := json.Marshal(meta) + if err != nil { + return err + } + + c := changelist.NewTufChange( + action, + changelist.ScopeRoot, + changelist.TypeRootRole, + role, + metaJSON, + ) + err = cl.Add(c) + if err != nil { + return err + } + return nil +} diff --git a/vendor/src/github.com/docker/notary/client/helpers.go b/vendor/src/github.com/docker/notary/client/helpers.go index 476ef08b7..50be86c6c 100644 --- a/vendor/src/github.com/docker/notary/client/helpers.go +++ b/vendor/src/github.com/docker/notary/client/helpers.go @@ -7,7 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/notary/client/changelist" - "github.com/endophage/gotuf" + tuf "github.com/endophage/gotuf" "github.com/endophage/gotuf/data" "github.com/endophage/gotuf/keys" "github.com/endophage/gotuf/store" @@ -38,14 +38,16 @@ func applyChangelist(repo *tuf.TufRepo, cl changelist.Changelist) error { } switch c.Scope() { case changelist.ScopeTargets: - err := applyTargetsChange(repo, c) - if err != nil { - return err - } + err = applyTargetsChange(repo, c) + case changelist.ScopeRoot: + err = applyRootChange(repo, c) default: logrus.Debug("scope not supported: ", c.Scope()) } index++ + if err != nil { + return err + } } logrus.Debugf("applied %d change(s)", index) return nil @@ -75,6 +77,40 @@ func applyTargetsChange(repo *tuf.TufRepo, c changelist.Change) error { return nil } +func applyRootChange(repo *tuf.TufRepo, c changelist.Change) error { + var err error + switch c.Type() { + case changelist.TypeRootRole: + err = applyRootRoleChange(repo, c) + default: + logrus.Debug("type of root change not yet supported: ", c.Type()) + } + return err // might be nil +} + +func applyRootRoleChange(repo *tuf.TufRepo, c changelist.Change) error { + switch c.Action() { + case changelist.ActionCreate: + // replaces all keys for a role + d := &changelist.TufRootData{} + err := json.Unmarshal(c.Content(), d) + if err != nil { + return err + } + k := []data.PublicKey{} + for _, key := range d.Keys { + k = append(k, data.NewPublicKey(key.Algorithm(), key.Public())) + } + err = repo.ReplaceBaseKeys(d.RoleName, k...) + if err != nil { + return err + } + default: + logrus.Debug("action not yet supported for root: ", c.Action()) + } + return nil +} + func nearExpiry(r *data.SignedRoot) bool { plus6mo := time.Now().AddDate(0, 6, 0) return r.Signed.Expires.Before(plus6mo) diff --git a/vendor/src/github.com/endophage/gotuf/client/client.go b/vendor/src/github.com/endophage/gotuf/client/client.go index 2bcda4b74..532e47478 100644 --- a/vendor/src/github.com/endophage/gotuf/client/client.go +++ b/vendor/src/github.com/endophage/gotuf/client/client.go @@ -175,19 +175,7 @@ func (c *Client) downloadRoot() error { var s *data.Signed var raw []byte if download { - logrus.Debug("downloading new root") - raw, err = c.remote.GetMeta(role, size) - if err != nil { - return err - } - hash := sha256.Sum256(raw) - if expectedSha256 != nil && !bytes.Equal(hash[:], expectedSha256) { - // if we don't have an expected sha256, we're going to trust the root - // based purely on signature and expiry time validation - return fmt.Errorf("Remote root sha256 did not match snapshot root sha256: %#x vs. %#x", hash, []byte(expectedSha256)) - } - s = &data.Signed{} - err = json.Unmarshal(raw, s) + raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return err } @@ -247,6 +235,8 @@ func (c Client) verifyRoot(role string, s *data.Signed, minVersion int) error { } // downloadTimestamp is responsible for downloading the timestamp.json +// Timestamps are special in that we ALWAYS attempt to download and only +// use cache if the download fails (and the cache is still valid). func (c *Client) downloadTimestamp() error { logrus.Debug("downloadTimestamp") role := data.RoleName("timestamp") @@ -271,7 +261,6 @@ func (c *Client) downloadTimestamp() error { } // unlike root, targets and snapshot, always try and download timestamps // from remote, only using the cache one if we couldn't reach remote. - logrus.Debug("Downloading timestamp") raw, err := c.remote.GetMeta(role, maxSize) var s *data.Signed if err != nil || len(raw) == 0 { @@ -286,6 +275,7 @@ func (c *Client) downloadTimestamp() error { } return err } + logrus.Debug("using cached timestamp") s = old } else { download = true @@ -351,17 +341,7 @@ func (c *Client) downloadSnapshot() error { } var s *data.Signed if download { - logrus.Debug("downloading new snapshot") - raw, err = c.remote.GetMeta(role, size) - if err != nil { - return err - } - genHash := sha256.Sum256(raw) - if !bytes.Equal(genHash[:], expectedSha256) { - return fmt.Errorf("Retrieved snapshot did not verify against hash in timestamp.") - } - s = &data.Signed{} - err = json.Unmarshal(raw, s) + raw, s, err = c.downloadSigned(role, size, expectedSha256) if err != nil { return err } @@ -390,8 +370,7 @@ func (c *Client) downloadSnapshot() error { } // downloadTargets is responsible for downloading any targets file -// including delegates roles. It will download the whole tree of -// delegated roles below the given one +// including delegates roles. func (c *Client) downloadTargets(role string) error { role = data.RoleName(role) // this will really only do something for base targets role snap := c.local.Snapshot.Signed @@ -418,6 +397,24 @@ func (c *Client) downloadTargets(role string) error { return nil } +func (c *Client) downloadSigned(role string, size int64, expectedSha256 []byte) ([]byte, *data.Signed, error) { + logrus.Debugf("downloading new %s", role) + raw, err := c.remote.GetMeta(role, size) + if err != nil { + return nil, nil, err + } + genHash := sha256.Sum256(raw) + if !bytes.Equal(genHash[:], expectedSha256) { + return nil, nil, ErrChecksumMismatch{role: role} + } + s := &data.Signed{} + err = json.Unmarshal(raw, s) + if err != nil { + return nil, nil, err + } + return raw, s, nil +} + func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.Files, consistent bool, threshold int) (*data.Signed, error) { // require role exists in snapshots roleMeta, ok := snapshotMeta[role] @@ -454,25 +451,19 @@ func (c Client) GetTargetsFile(role string, keyIDs []string, snapshotMeta data.F } else { download = true } - } + size := snapshotMeta[role].Length var s *data.Signed if download { rolePath, err := c.RoleTargetsPath(role, hex.EncodeToString(expectedSha256), consistent) if err != nil { return nil, err } - raw, err = c.remote.GetMeta(rolePath, snapshotMeta[role].Length) + raw, s, err = c.downloadSigned(rolePath, size, expectedSha256) if err != nil { return nil, err } - s = &data.Signed{} - err = json.Unmarshal(raw, s) - if err != nil { - logrus.Error("Error unmarshalling targets file:", err) - return nil, err - } } else { logrus.Debug("using cached ", role) s = old diff --git a/vendor/src/github.com/endophage/gotuf/client/errors.go b/vendor/src/github.com/endophage/gotuf/client/errors.go index 776e6a69e..311e74a8d 100644 --- a/vendor/src/github.com/endophage/gotuf/client/errors.go +++ b/vendor/src/github.com/endophage/gotuf/client/errors.go @@ -10,6 +10,14 @@ var ( ErrInsufficientKeys = errors.New("tuf: insufficient keys to meet threshold") ) +type ErrChecksumMismatch struct { + role string +} + +func (e ErrChecksumMismatch) Error() string { + return fmt.Sprintf("tuf: checksum for %s did not match", e.role) +} + type ErrMissingRemoteMetadata struct { Name string } diff --git a/vendor/src/github.com/endophage/gotuf/data/keys.go b/vendor/src/github.com/endophage/gotuf/data/keys.go index 3df1ce05c..eccccc420 100644 --- a/vendor/src/github.com/endophage/gotuf/data/keys.go +++ b/vendor/src/github.com/endophage/gotuf/data/keys.go @@ -71,7 +71,7 @@ func (k TUFKey) Public() []byte { return k.Value.Public } -func (k *TUFKey) Private() []byte { +func (k TUFKey) Private() []byte { return k.Value.Private } diff --git a/vendor/src/github.com/endophage/gotuf/data/roles.go b/vendor/src/github.com/endophage/gotuf/data/roles.go index d3047d784..1034393e1 100644 --- a/vendor/src/github.com/endophage/gotuf/data/roles.go +++ b/vendor/src/github.com/endophage/gotuf/data/roles.go @@ -24,7 +24,7 @@ var ValidRoles = map[string]string{ func SetValidRoles(rs map[string]string) { // iterate ValidRoles - for k, _ := range ValidRoles { + for k := range ValidRoles { if v, ok := rs[k]; ok { ValidRoles[k] = v } @@ -88,6 +88,7 @@ type Role struct { Name string `json:"name"` Paths []string `json:"paths,omitempty"` PathHashPrefixes []string `json:"path_hash_prefixes,omitempty"` + Email string `json:"email,omitempty"` } func NewRole(name string, threshold int, keyIDs, paths, pathHashPrefixes []string) (*Role, error) { diff --git a/vendor/src/github.com/endophage/gotuf/store/httpstore.go b/vendor/src/github.com/endophage/gotuf/store/httpstore.go index 1a82b094c..730438267 100644 --- a/vendor/src/github.com/endophage/gotuf/store/httpstore.go +++ b/vendor/src/github.com/endophage/gotuf/store/httpstore.go @@ -90,6 +90,7 @@ func (s HTTPStore) GetMeta(name string, size int64) ([]byte, error) { if resp.StatusCode == http.StatusNotFound { return nil, ErrMetaNotFound{} } else if resp.StatusCode != http.StatusOK { + logrus.Debugf("received HTTP status %d when requesting %s.", resp.StatusCode, name) return nil, ErrServerUnavailable{code: resp.StatusCode} } if resp.ContentLength > size { diff --git a/vendor/src/github.com/endophage/gotuf/tuf.go b/vendor/src/github.com/endophage/gotuf/tuf.go index 4d226aceb..39af54018 100644 --- a/vendor/src/github.com/endophage/gotuf/tuf.go +++ b/vendor/src/github.com/endophage/gotuf/tuf.go @@ -71,24 +71,46 @@ func NewTufRepo(keysDB *keys.KeyDB, cryptoService signed.CryptoService) *TufRepo } // AddBaseKeys is used to add keys to the role in root.json -func (tr *TufRepo) AddBaseKeys(role string, keys ...*data.TUFKey) error { +func (tr *TufRepo) AddBaseKeys(role string, keys ...data.PublicKey) error { if tr.Root == nil { return ErrNotLoaded{role: "root"} } + ids := []string{} for _, k := range keys { // Store only the public portion - pubKey := *k - pubKey.Value.Private = nil - tr.Root.Signed.Keys[pubKey.ID()] = &pubKey - tr.keysDB.AddKey(&pubKey) + pubKey := data.NewPrivateKey(k.Algorithm(), k.Public(), nil) + tr.Root.Signed.Keys[pubKey.ID()] = pubKey + tr.keysDB.AddKey(k) tr.Root.Signed.Roles[role].KeyIDs = append(tr.Root.Signed.Roles[role].KeyIDs, pubKey.ID()) + ids = append(ids, pubKey.ID()) } + r, err := data.NewRole( + role, + tr.Root.Signed.Roles[role].Threshold, + ids, + nil, + nil, + ) + if err != nil { + return err + } + tr.keysDB.AddRole(r) tr.Root.Dirty = true return nil } -// RemoveKeys is used to remove keys from the roles in root.json +// ReplaceBaseKeys is used to replace all keys for the given role with the new keys +func (tr *TufRepo) ReplaceBaseKeys(role string, keys ...data.PublicKey) error { + r := tr.keysDB.GetRole(role) + err := tr.RemoveBaseKeys(role, r.KeyIDs...) + if err != nil { + return err + } + return tr.AddBaseKeys(role, keys...) +} + +// RemoveBaseKeys is used to remove keys from the roles in root.json func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error { if tr.Root == nil { return ErrNotLoaded{role: "root"} @@ -119,7 +141,7 @@ func (tr *TufRepo) RemoveBaseKeys(role string, keyIDs ...string) error { } // remove keys no longer in use by any roles - for k, _ := range toDelete { + for k := range toDelete { delete(tr.Root.Signed.Keys, k) } tr.Root.Dirty = true From 45cfeed39dd331bf9e50fbe198f502ccf6b16a8c Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 16 Oct 2015 08:25:28 +0200 Subject: [PATCH 021/134] Add zsh completion for 'docker import -m --message' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index a0f96c12f..31ec3c669 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -642,6 +642,7 @@ __docker_subcommand() { _arguments \ $opts_help \ "($help -c --change)*"{-c,--change=-}"[Apply Dockerfile instruction to the created image]:Dockerfile:_files" \ + "($help -m --message)"{-m,--message=-}"[Set commit message for imported image]:message: " \ "($help -):URL:(- http:// file://)" \ "($help -): :__docker_repositories_with_tags" && ret=0 ;; From 10f946d312c7a2b60f1e92ecba2c4de3830a4d4b Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 16 Oct 2015 21:18:27 +0200 Subject: [PATCH 022/134] Zsh completion : all --