From 7d95d300abd9d9cc811713e53fd41adec66f2e00 Mon Sep 17 00:00:00 2001 From: Zilin Du Date: Tue, 8 Oct 2013 14:29:22 -0700 Subject: [PATCH 01/79] replace 127.0.0.1 by the assigned IP address in the container's /etc/hosts file. --- container.go | 28 ++++++++++++++++++++++++++++ runtime.go | 27 +-------------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/container.go b/container.go index 316e0d259..c6317c087 100644 --- a/container.go +++ b/container.go @@ -579,10 +579,12 @@ func (container *Container) Start(hostConfig *HostConfig) error { } if container.runtime.networkManager.disabled { container.Config.NetworkDisabled = true + container.buildHostnameAndHostsFiles("127.0.0.1") } else { if err := container.allocateNetwork(); err != nil { return err } + container.buildHostnameAndHostsFiles(container.NetworkSettings.IPAddress) } // Make sure the config is compatible with the current kernel @@ -868,6 +870,32 @@ func (container *Container) StderrPipe() (io.ReadCloser, error) { return utils.NewBufReader(reader), nil } +func (container *Container) buildHostnameAndHostsFiles(IP string) { + container.HostnamePath = path.Join(container.root, "hostname") + ioutil.WriteFile(container.HostnamePath, []byte(container.Config.Hostname+"\n"), 0644) + + hostsContent := []byte(` +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +`) + + container.HostsPath = path.Join(container.root, "hosts") + + if container.Config.Domainname != "" { + hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) + hostsContent = append([]byte(fmt.Sprintf("%s\t%s.%s %s\n", IP, container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) + } else { + hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s\n", container.Config.Hostname)), hostsContent...) + hostsContent = append([]byte(fmt.Sprintf("%s\t%s\n", IP, container.Config.Hostname)), hostsContent...) + } + + ioutil.WriteFile(container.HostsPath, hostsContent, 0644) +} + func (container *Container) allocateNetwork() error { if container.Config.NetworkDisabled { return nil diff --git a/runtime.go b/runtime.go index d77ecca72..e4bca0457 100644 --- a/runtime.go +++ b/runtime.go @@ -368,32 +368,7 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { return nil, err } - // Step 3: if hostname, build hostname and hosts files - container.HostnamePath = path.Join(container.root, "hostname") - ioutil.WriteFile(container.HostnamePath, []byte(container.Config.Hostname+"\n"), 0644) - - hostsContent := []byte(` -127.0.0.1 localhost -::1 localhost ip6-localhost ip6-loopback -fe00::0 ip6-localnet -ff00::0 ip6-mcastprefix -ff02::1 ip6-allnodes -ff02::2 ip6-allrouters -`) - - container.HostsPath = path.Join(container.root, "hosts") - - if container.Config.Domainname != "" { - hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) - hostsContent = append([]byte(fmt.Sprintf("127.0.0.1\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) - } else { - hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s\n", container.Config.Hostname)), hostsContent...) - hostsContent = append([]byte(fmt.Sprintf("127.0.0.1\t%s\n", container.Config.Hostname)), hostsContent...) - } - - ioutil.WriteFile(container.HostsPath, hostsContent, 0644) - - // Step 4: register the container + // Step 3: register the container if err := runtime.Register(container); err != nil { return nil, err } From c2912c82aa7398a10d39fe114eb6328abd636d82 Mon Sep 17 00:00:00 2001 From: Zilin Du Date: Thu, 10 Oct 2013 09:44:48 -0700 Subject: [PATCH 02/79] change 127.0.0.1 -> 127.0.1.1 & remove ::1 -> hostname mapping --- container.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/container.go b/container.go index c6317c087..555f4a524 100644 --- a/container.go +++ b/container.go @@ -579,7 +579,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { } if container.runtime.networkManager.disabled { container.Config.NetworkDisabled = true - container.buildHostnameAndHostsFiles("127.0.0.1") + container.buildHostnameAndHostsFiles("127.0.1.1") } else { if err := container.allocateNetwork(); err != nil { return err @@ -886,10 +886,8 @@ ff02::2 ip6-allrouters container.HostsPath = path.Join(container.root, "hosts") if container.Config.Domainname != "" { - hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s.%s %s\n", container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) hostsContent = append([]byte(fmt.Sprintf("%s\t%s.%s %s\n", IP, container.Config.Hostname, container.Config.Domainname, container.Config.Hostname)), hostsContent...) } else { - hostsContent = append([]byte(fmt.Sprintf("::1\t\t%s\n", container.Config.Hostname)), hostsContent...) hostsContent = append([]byte(fmt.Sprintf("%s\t%s\n", IP, container.Config.Hostname)), hostsContent...) } From 9ee9d2f9959390d1cda56accbbd975df18d157ad Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 3 Oct 2013 18:23:29 +0000 Subject: [PATCH 03/79] Container memory limit can be specified in kilobytes, megabytes or gigabytes -m 10 # 10 bytes -m 10b # 10 bytes -m 10k # 10240 bytes (10 * 1024) -m 10m # 10485760 bytes (10 * 1024 * 1024) -m 10g # 10737418240 bytes (10 * 1024 * 1024 * 1024) Units are case-insensitive, and 'kb', 'mb' and 'gb' are equivalent to 'k', 'm' and 'g'. --- container.go | 22 ++++++++++++++++----- docs/sources/commandline/cli.rst | 2 +- utils/utils.go | 32 +++++++++++++++++++++++++++++++ utils/utils_test.go | 33 ++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 6 deletions(-) diff --git a/container.go b/container.go index e67c8f5b0..30c010a66 100644 --- a/container.go +++ b/container.go @@ -162,7 +162,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached") flTty := cmd.Bool("t", false, "Allocate a pseudo-tty") - flMemory := cmd.Int64("m", 0, "Memory limit (in bytes)") + flMemoryString := cmd.String("m", "", "Memory limit (format: , where unit = b, k, m or g)") flContainerIDFile := cmd.String("cidfile", "", "Write the container ID to the file") flNetwork := cmd.Bool("n", true, "Enable networking for this container") flPrivileged := cmd.Bool("privileged", false, "Give extended privileges to this container") @@ -170,9 +170,9 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, cmd.Bool("sig-proxy", true, "Proxify all received signal to the process (even in non-tty mode)") cmd.String("name", "", "Assign a name to the container") - if capabilities != nil && *flMemory > 0 && !capabilities.MemoryLimit { + if capabilities != nil && *flMemoryString != "" && !capabilities.MemoryLimit { //fmt.Fprintf(stdout, "WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.\n") - *flMemory = 0 + *flMemoryString = "" } flCpuShares := cmd.Int64("c", 0, "CPU shares (relative weight)") @@ -239,6 +239,18 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, } } + var flMemory int64 + + if *flMemoryString != "" { + parsedMemory, err := utils.RAMInBytes(*flMemoryString) + + if err != nil { + return nil, nil, cmd, err + } + + flMemory = parsedMemory + } + var binds []string // add any bind targets to the list of container volumes @@ -306,7 +318,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, Tty: *flTty, NetworkDisabled: !*flNetwork, OpenStdin: *flStdin, - Memory: *flMemory, + Memory: flMemory, CpuShares: *flCpuShares, AttachStdin: flAttach.Get("stdin"), AttachStdout: flAttach.Get("stdout"), @@ -330,7 +342,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, Links: flLinks, } - if capabilities != nil && *flMemory > 0 && !capabilities.SwapLimit { + if capabilities != nil && flMemory > 0 && !capabilities.SwapLimit { //fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") config.MemorySwap = -1 } diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index c6579da35..a3c2b9eed 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -568,7 +568,7 @@ network communication. -h="": Container host name -i=false: Keep stdin open even if not attached -privileged=false: Give extended privileges to this container - -m=0: Memory limit (in bytes) + -m="": Memory limit (format: , where unit = b, k, m or g) -n=true: Enable networking for this container -p=[]: Map a network port to the container -rm=false: Automatically remove the container when it exits (incompatible with -d) diff --git a/utils/utils.go b/utils/utils.go index d53094397..c37b3cc6a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -20,6 +20,7 @@ import ( "strings" "sync" "time" + "regexp" ) var ( @@ -176,6 +177,37 @@ func HumanSize(size int64) string { return fmt.Sprintf("%.4g %s", sizef, units[i]) } +// Parses a human-readable string representing an amount of RAM +// in bytes, kibibytes, mebibytes or gibibytes, and returns the +// number of bytes, or -1 if the string is unparseable. +// Units are case-insensitive, and the 'b' suffix is optional. +func RAMInBytes(size string) (bytes int64, err error) { + re, error := regexp.Compile("^(\\d+)([kKmMgG])?[bB]?$") + if error != nil { return -1, error } + + matches := re.FindStringSubmatch(size) + + if len(matches) != 3 { + return -1, fmt.Errorf("Invalid size: '%s'", size) + } + + memLimit, error := strconv.ParseInt(matches[1], 10, 0) + if error != nil { return -1, error } + + unit := strings.ToLower(matches[2]) + + if unit == "k" { + memLimit *= 1024 + } else if unit == "m" { + memLimit *= 1024*1024 + } else if unit == "g" { + memLimit *= 1024*1024*1024 + } + + return memLimit, nil +} + + func Trunc(s string, maxlen int) string { if len(s) <= maxlen { return s diff --git a/utils/utils_test.go b/utils/utils_test.go index 49f19bf75..8775bd089 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -265,6 +265,39 @@ func TestHumanSize(t *testing.T) { } } +func TestRAMInBytes(t *testing.T) { + assertRAMInBytes(t, "32", false, 32) + assertRAMInBytes(t, "32b", false, 32) + assertRAMInBytes(t, "32B", false, 32) + assertRAMInBytes(t, "32k", false, 32*1024) + assertRAMInBytes(t, "32K", false, 32*1024) + assertRAMInBytes(t, "32kb", false, 32*1024) + assertRAMInBytes(t, "32Kb", false, 32*1024) + assertRAMInBytes(t, "32Mb", false, 32*1024*1024) + assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024) + + assertRAMInBytes(t, "", true, -1) + assertRAMInBytes(t, "hello", true, -1) + assertRAMInBytes(t, "-32", true, -1) + assertRAMInBytes(t, " 32 ", true, -1) + assertRAMInBytes(t, "32 mb", true, -1) + assertRAMInBytes(t, "32m b", true, -1) + assertRAMInBytes(t, "32bm", true, -1) +} + +func assertRAMInBytes(t *testing.T, size string, expectError bool, expectedBytes int64) { + actualBytes, err := RAMInBytes(size) + if (err != nil) && !expectError { + t.Errorf("Unexpected error parsing '%s': %s", size, err) + } + if (err == nil) && expectError { + t.Errorf("Expected to get an error parsing '%s', but got none (bytes=%d)", size, actualBytes) + } + if actualBytes != expectedBytes { + t.Errorf("Expected '%s' to parse as %d bytes, got %d", size, expectedBytes, actualBytes) + } +} + func TestParseHost(t *testing.T) { if addr, err := ParseHost("127.0.0.1", 4243, "0.0.0.0"); err != nil || addr != "tcp://0.0.0.0:4243" { t.Errorf("0.0.0.0 -> expected tcp://0.0.0.0:4243, got %s", addr) From 4194617bfe1e29bae2c2515aefbed5f1f2a1ebc1 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Wed, 23 Oct 2013 19:04:03 -0700 Subject: [PATCH 04/79] Add known issues to sections via new "issues" extension. --- docs/sources/conf.py | 6 +++++- docs/sources/use/builder.rst | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/sources/conf.py b/docs/sources/conf.py index 7aa27c7b8..0ccd4a4ed 100644 --- a/docs/sources/conf.py +++ b/docs/sources/conf.py @@ -40,7 +40,11 @@ html_additional_pages = { # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinxcontrib.httpdomain'] +extensions = ['sphinxcontrib.httpdomain', 'sphinx.ext.extlinks'] + +# Configure extlinks +extlinks = { 'issue': ('https://github.com/dotcloud/docker/issues/%s', + 'Issue ') } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index d1747c3fb..a1fd85df9 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -116,6 +116,16 @@ core concepts of Docker where commits are cheap and containers can be created from any point in an image's history, much like source control. +Known Issues +............ + +* :issue:`783` is about file permissions problems that can occur when + using the AUFS file system. You might notice it during an attempt to + ``rm`` a file, for example. The issue describes a workaround. + + + + 3.4 CMD ------- From 5a9adfe9fb5bc988185ef41ca637fbeee73e947c Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Fri, 1 Nov 2013 16:06:46 -0700 Subject: [PATCH 05/79] Add known issues. Fix build warnings. --- docs/sources/commandline/cli.rst | 12 ++++++++++++ docs/sources/contributing/contributing.rst | 9 ++++++--- docs/sources/use/builder.rst | 6 +++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 8f778c32d..ab622bff5 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -408,6 +408,12 @@ Insert file from github The main process inside the container will be sent SIGKILL. +Known Issues (kill) +~~~~~~~~~~~~~~~~~~~ + +* :issue:`197` indicates that ``docker kill`` may leave directories + behind and make it difficult to remove the container. + .. _cli_login: ``login`` @@ -516,6 +522,12 @@ The main process inside the container will be sent SIGKILL. Remove one or more containers -link="": Remove the link instead of the actual container +Known Issues (rm) +~~~~~~~~~~~~~~~~~~~ + +* :issue:`197` indicates that ``docker kill`` may leave directories + behind and make it difficult to remove the container. + Examples: ~~~~~~~~~ diff --git a/docs/sources/contributing/contributing.rst b/docs/sources/contributing/contributing.rst index 3cdb0b6f1..3b3b3f8f8 100644 --- a/docs/sources/contributing/contributing.rst +++ b/docs/sources/contributing/contributing.rst @@ -10,13 +10,16 @@ Want to hack on Docker? Awesome! The repository includes `all the instructions you need to get started `_. -The developer environment `Dockerfile `_ +The `developer environment Dockerfile +`_ specifies the tools and versions used to test and build Docker. If you're making changes to the documentation, see the `README.md `_. -The documentation environment `Dockerfile `_ +The `documentation environment Dockerfile +`_ specifies the tools and versions used to build the Documentation. -Further interesting details can be found in the `Packaging hints `_. +Further interesting details can be found in the `Packaging hints +`_. diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index a1fd85df9..e48f3f2a4 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -116,13 +116,13 @@ core concepts of Docker where commits are cheap and containers can be created from any point in an image's history, much like source control. -Known Issues -............ +Known Issues (RUN) +.................. * :issue:`783` is about file permissions problems that can occur when using the AUFS file system. You might notice it during an attempt to ``rm`` a file, for example. The issue describes a workaround. - +* :issue:`2424` Locale will not be set automatically. From 433c8e9c7da7cd3cd952c3dce3763db70fc450e5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 26 Oct 2013 17:19:35 -0700 Subject: [PATCH 06/79] Separate a) initialization of the http api and b) actually serving the api into 2 distinct jobs --- config.go | 2 -- docker/docker.go | 8 ++++++-- server.go | 31 +++++++++++++++---------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/config.go b/config.go index 40c47e692..d5f852c26 100644 --- a/config.go +++ b/config.go @@ -9,7 +9,6 @@ import ( type DaemonConfig struct { Pidfile string Root string - ProtoAddresses []string AutoRestart bool EnableCors bool Dns []string @@ -36,7 +35,6 @@ func ConfigFromJob(job *engine.Job) *DaemonConfig { } else { config.BridgeIface = DefaultNetworkBridge } - config.ProtoAddresses = job.GetenvList("ProtoAddresses") config.DefaultIp = net.ParseIP(job.Getenv("DefaultIp")) config.InterContainerCommunication = job.GetenvBool("InterContainerCommunication") return &config diff --git a/docker/docker.go b/docker/docker.go index c500633a7..2fc864adf 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -71,7 +71,8 @@ func main() { if err != nil { log.Fatal(err) } - job := eng.Job("serveapi") + // Load plugin: httpapi + job := eng.Job("initapi") job.Setenv("Pidfile", *pidfile) job.Setenv("Root", *flRoot) job.SetenvBool("AutoRestart", *flAutoRestart) @@ -79,12 +80,15 @@ func main() { job.Setenv("Dns", *flDns) job.SetenvBool("EnableIptables", *flEnableIptables) job.Setenv("BridgeIface", *bridgeName) - job.SetenvList("ProtoAddresses", flHosts) job.Setenv("DefaultIp", *flDefaultIp) job.SetenvBool("InterContainerCommunication", *flInterContainerComm) if err := job.Run(); err != nil { log.Fatal(err) } + // Serve api + if err := eng.Job("serveapi", flHosts...).Run(); err != nil { + log.Fatal(err) + } } else { if len(flHosts) > 1 { log.Fatal("Please specify only one -H") diff --git a/server.go b/server.go index 314df0256..d8b70b88a 100644 --- a/server.go +++ b/server.go @@ -33,30 +33,20 @@ func (srv *Server) Close() error { } func init() { - engine.Register("serveapi", JobServeApi) + engine.Register("initapi", jobInitApi) } -func JobServeApi(job *engine.Job) string { +// jobInitApi runs the remote api server `srv` as a daemon, +// Only one api server can run at the same time - this is enforced by a pidfile. +// The signals SIGINT, SIGKILL and SIGTERM are intercepted for cleanup. +func jobInitApi(job *engine.Job) string { srv, err := NewServer(ConfigFromJob(job)) if err != nil { return err.Error() } - defer srv.Close() - if err := srv.Daemon(); err != nil { - return err.Error() - } - return "0" -} - -// Daemon runs the remote api server `srv` as a daemon, -// Only one api server can run at the same time - this is enforced by a pidfile. -// The signals SIGINT, SIGKILL and SIGTERM are intercepted for cleanup. -func (srv *Server) Daemon() error { if err := utils.CreatePidFile(srv.runtime.config.Pidfile); err != nil { log.Fatal(err) } - defer utils.RemovePidFile(srv.runtime.config.Pidfile) - c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) go func() { @@ -66,8 +56,17 @@ func (srv *Server) Daemon() error { srv.Close() os.Exit(0) }() + err = engine.Register("serveapi", func(job *engine.Job) string { + return srv.ListenAndServe(job.Args...).Error() + }) + if err != nil { + return err.Error() + } + return "0" +} - protoAddrs := srv.runtime.config.ProtoAddresses + +func (srv *Server) ListenAndServe(protoAddrs ...string) error { chErrors := make(chan error, len(protoAddrs)) for _, protoAddr := range protoAddrs { protoAddrParts := strings.SplitN(protoAddr, "://", 2) From f9cb6ae46a3478c19e85a2a159c4ac31223ec499 Mon Sep 17 00:00:00 2001 From: Daniel Garcia Date: Sun, 13 Oct 2013 15:58:54 -0500 Subject: [PATCH 07/79] Add ability to mount volumes in readonly mode using -volumes-from --- container.go | 24 ++++++++++--- container_test.go | 62 ++++++++++++++++++++++++++++++++ docs/sources/commandline/cli.rst | 13 ++++++- 3 files changed, 93 insertions(+), 6 deletions(-) diff --git a/container.go b/container.go index a22484c2d..db444e758 100644 --- a/container.go +++ b/container.go @@ -199,7 +199,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, cmd.Var(flVolumes, "v", "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") var flVolumesFrom utils.ListOpts - cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container") + cmd.Var(&flVolumesFrom, "volumes-from", "Mount volumes from the specified container(s)") flEntrypoint := cmd.String("entrypoint", "", "Overwrite the default entrypoint of the image") @@ -749,9 +749,23 @@ func (container *Container) Start() (err error) { // Apply volumes from another container if requested if container.Config.VolumesFrom != "" { - volumes := strings.Split(container.Config.VolumesFrom, ",") - for _, v := range volumes { - c := container.runtime.Get(v) + containerSpecs := strings.Split(container.Config.VolumesFrom, ",") + for _, containerSpec := range containerSpecs { + mountRW := true + specParts := strings.SplitN(containerSpec, ":", 2) + switch len(specParts) { + case 0: + return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom) + case 2: + switch specParts[1] { + case "ro": + mountRW = false + case "rw": // mountRW is already true + default: + return fmt.Errorf("Malformed volumes-from speficication: %s", containerSpec) + } + } + c := container.runtime.Get(specParts[0]) if c == nil { return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID) } @@ -764,7 +778,7 @@ func (container *Container) Start() (err error) { } container.Volumes[volPath] = id if isRW, exists := c.VolumesRW[volPath]; exists { - container.VolumesRW[volPath] = isRW + container.VolumesRW[volPath] = isRW && mountRW } } diff --git a/container_test.go b/container_test.go index cbabffc36..5540bb9c2 100644 --- a/container_test.go +++ b/container_test.go @@ -1338,6 +1338,68 @@ func TestBindMounts(t *testing.T) { } } +// Test that -volumes-from supports both read-only mounts +func TestFromVolumesInReadonlyMode(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + container, _, err := runtime.Create( + &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"/bin/echo", "-n", "foobar"}, + Volumes: map[string]struct{}{"/test": {}}, + }, + "", + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + _, err = container.Output() + if err != nil { + t.Fatal(err) + } + if !container.VolumesRW["/test"] { + t.Fail() + } + + container2, _, err := runtime.Create( + &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"/bin/echo", "-n", "foobar"}, + VolumesFrom: container.ID + ":ro", + }, + "", + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container2) + + _, err = container2.Output() + if err != nil { + t.Fatal(err) + } + + if container.Volumes["/test"] != container2.Volumes["/test"] { + t.Logf("container volumes do not match: %s | %s ", + container.Volumes["/test"], + container2.Volumes["/test"]) + t.Fail() + } + + _, exists := container2.VolumesRW["/test"] + if !exists { + t.Logf("container2 is missing '/test' volume: %s", container2.VolumesRW) + t.Fail() + } + + if container2.VolumesRW["/test"] != false { + t.Log("'/test' volume mounted in read-write mode, expected read-only") + t.Fail() + } +} + + // Test that VolumesRW values are copied to the new container. Regression test for #1201 func TestVolumesFromReadonlyMount(t *testing.T) { runtime := mkRuntime(t) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index da0c262c4..786fca6a6 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -576,7 +576,7 @@ network communication. -u="": Username or UID -dns=[]: Set custom dns servers for the container -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume. - -volumes-from="": Mount all volumes from the given container + -volumes-from="": Mount all volumes from the given container(s) -entrypoint="": Overwrite the default entrypoint set by the image -w="": Working directory inside the container -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" @@ -668,6 +668,17 @@ can access the network and environment of the redis container via environment variables. The ``-name`` flag will assign the name ``console`` to the newly created container. +.. code-block:: bash + + docker run -volumes-from 777f7dc92da7,ba8c0c54f0f2:ro -i -t ubuntu pwd + +The ``-volumes-from`` flag mounts all the defined volumes from the +refrence containers. Containers can be specified by a comma seperated +list or by repetitions of the ``-volumes-from`` argument. The container +id may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in +read-only or read-write mode, respectively. By default, the volumes are mounted +in the same mode (rw or ro) as the reference container. + .. _cli_search: ``search`` From 958b4a8757e83c3fada757b10dd1be4ab7bff5ee Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 26 Oct 2013 19:24:01 -0700 Subject: [PATCH 08/79] Engine: 'start' starts the specified container --- api.go | 24 +++++-------- api_test.go | 6 ++-- engine/engine.go | 10 +++--- engine/hack.go | 23 ++++++++++++ engine/job.go | 60 ++++++++++++++++++++++++++++++- engine/{init_test.go => utils.go} | 4 +-- server.go | 54 ++++++++++++++++++---------- server_test.go | 52 ++++++++++++++++++--------- utils_test.go | 13 +++++++ 9 files changed, 186 insertions(+), 60 deletions(-) create mode 100644 engine/hack.go rename engine/{init_test.go => utils.go} (90%) diff --git a/api.go b/api.go index 0c06b2e6d..1a3d92de3 100644 --- a/api.go +++ b/api.go @@ -639,26 +639,20 @@ func deleteImages(srv *Server, version float64, w http.ResponseWriter, r *http.R } func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - var hostConfig *HostConfig - // allow a nil body for backwards compatibility - if r.Body != nil { - if matchesContentType(r.Header.Get("Content-Type"), "application/json") { - hostConfig = &HostConfig{} - if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil { - return err - } - } - } - if vars == nil { return fmt.Errorf("Missing parameter") } name := vars["name"] - // Register any links from the host config before starting the container - if err := srv.RegisterLinks(name, hostConfig); err != nil { - return err + job := srv.Eng.Job("start", name) + // allow a nil body for backwards compatibility + if r.Body != nil { + if matchesContentType(r.Header.Get("Content-Type"), "application/json") { + if err := job.DecodeEnv(r.Body); err != nil { + return err + } + } } - if err := srv.ContainerStart(name, hostConfig); err != nil { + if err := job.Run(); err != nil { return err } w.WriteHeader(http.StatusNoContent) diff --git a/api_test.go b/api_test.go index fcca8ce83..50a86e3f2 100644 --- a/api_test.go +++ b/api_test.go @@ -781,11 +781,11 @@ func TestPostContainersRestart(t *testing.T) { } func TestPostContainersStart(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} - container, _, err := runtime.Create( &Config{ Image: GetTestImage(runtime).ID, diff --git a/engine/engine.go b/engine/engine.go index 8d67242ca..a0d5a3c4a 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -13,10 +13,11 @@ type Handler func(*Job) string var globalHandlers map[string]Handler +func init() { + globalHandlers = make(map[string]Handler) +} + func Register(name string, handler Handler) error { - if globalHandlers == nil { - globalHandlers = make(map[string]Handler) - } globalHandlers[name] = handler return nil } @@ -27,6 +28,7 @@ func Register(name string, handler Handler) error { type Engine struct { root string handlers map[string]Handler + hack Hack // data for temporary hackery (see hack.go) } // New initializes a new engine managing the directory specified at `root`. @@ -66,7 +68,7 @@ func New(root string) (*Engine, error) { // This function mimics `Command` from the standard os/exec package. func (eng *Engine) Job(name string, args ...string) *Job { job := &Job{ - eng: eng, + Eng: eng, Name: name, Args: args, Stdin: os.Stdin, diff --git a/engine/hack.go b/engine/hack.go new file mode 100644 index 000000000..7f6e79c0e --- /dev/null +++ b/engine/hack.go @@ -0,0 +1,23 @@ +package engine + + +type Hack map[string]interface{} + + +func (eng *Engine) Hack_GetGlobalVar(key string) interface{} { + if eng.hack == nil { + return nil + } + val, exists := eng.hack[key] + if !exists { + return nil + } + return val +} + +func (eng *Engine) Hack_SetGlobalVar(key string, val interface{}) { + if eng.hack == nil { + eng.hack = make(Hack) + } + eng.hack[key] = val +} diff --git a/engine/job.go b/engine/job.go index 0bde2a0be..5c02fe15d 100644 --- a/engine/job.go +++ b/engine/job.go @@ -1,6 +1,7 @@ package engine import ( + "bytes" "io" "strings" "fmt" @@ -22,7 +23,7 @@ import ( // This allows for richer error reporting. // type Job struct { - eng *Engine + Eng *Engine Name string Args []string env []string @@ -111,3 +112,60 @@ func (job *Job) SetenvList(key string, value []string) error { func (job *Job) Setenv(key, value string) { job.env = append(job.env, key + "=" + value) } + +// DecodeEnv decodes `src` as a json dictionary, and adds +// each decoded key-value pair to the environment. +// +// If `text` cannot be decoded as a json dictionary, an error +// is returned. +func (job *Job) DecodeEnv(src io.Reader) error { + m := make(map[string]interface{}) + if err := json.NewDecoder(src).Decode(&m); err != nil { + return err + } + for k, v := range m { + if sval, ok := v.(string); ok { + job.Setenv(k, sval) + } else if val, err := json.Marshal(v); err == nil { + job.Setenv(k, string(val)) + } else { + job.Setenv(k, fmt.Sprintf("%v", v)) + } + } + return nil +} + +func (job *Job) EncodeEnv(dst io.Writer) error { + return json.NewEncoder(dst).Encode(job.Environ()) +} + +func (job *Job) ExportEnv(dst interface{}) error { + var buf bytes.Buffer + if err := job.EncodeEnv(&buf); err != nil { + return err + } + if err := json.NewDecoder(&buf).Decode(dst); err != nil { + return err + } + return nil +} + +func (job *Job) ImportEnv(src interface{}) error { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(src); err != nil { + return err + } + if err := job.DecodeEnv(&buf); err != nil { + return err + } + return nil +} + +func (job *Job) Environ() map[string]string { + m := make(map[string]string) + for _, kv := range job.env { + parts := strings.SplitN(kv, "=", 2) + m[parts[0]] = parts[1] + } + return m +} diff --git a/engine/init_test.go b/engine/utils.go similarity index 90% rename from engine/init_test.go rename to engine/utils.go index 5c03ded87..58e5f9aae 100644 --- a/engine/init_test.go +++ b/engine/utils.go @@ -15,7 +15,7 @@ func init() { Register("dummy", func(job *Job) string { return ""; }) } -func mkEngine(t *testing.T) *Engine { +func NewTestEngine(t *testing.T) *Engine { // Use the caller function name as a prefix. // This helps trace temp directories back to their test. pc, _, _, _ := runtime.Caller(1) @@ -38,5 +38,5 @@ func mkEngine(t *testing.T) *Engine { } func mkJob(t *testing.T, name string, args ...string) *Job { - return mkEngine(t).Job(name, args...) + return NewTestEngine(t).Job(name, args...) } diff --git a/server.go b/server.go index d8b70b88a..efc34d14e 100644 --- a/server.go +++ b/server.go @@ -40,7 +40,7 @@ func init() { // Only one api server can run at the same time - this is enforced by a pidfile. // The signals SIGINT, SIGKILL and SIGTERM are intercepted for cleanup. func jobInitApi(job *engine.Job) string { - srv, err := NewServer(ConfigFromJob(job)) + srv, err := NewServer(job.Eng, ConfigFromJob(job)) if err != nil { return err.Error() } @@ -56,17 +56,19 @@ func jobInitApi(job *engine.Job) string { srv.Close() os.Exit(0) }() - err = engine.Register("serveapi", func(job *engine.Job) string { - return srv.ListenAndServe(job.Args...).Error() - }) - if err != nil { + job.Eng.Hack_SetGlobalVar("httpapi.server", srv) + if err := engine.Register("start", srv.ContainerStart); err != nil { + return err.Error() + } + if err := engine.Register("serveapi", srv.ListenAndServe); err != nil { return err.Error() } return "0" } -func (srv *Server) ListenAndServe(protoAddrs ...string) error { +func (srv *Server) ListenAndServe(job *engine.Job) string { + protoAddrs := job.Args chErrors := make(chan error, len(protoAddrs)) for _, protoAddr := range protoAddrs { protoAddrParts := strings.SplitN(protoAddr, "://", 2) @@ -80,7 +82,7 @@ func (srv *Server) ListenAndServe(protoAddrs ...string) error { log.Println("/!\\ DON'T BIND ON ANOTHER IP ADDRESS THAN 127.0.0.1 IF YOU DON'T KNOW WHAT YOU'RE DOING /!\\") } default: - return fmt.Errorf("Invalid protocol format.") + return "Invalid protocol format." } go func() { chErrors <- ListenAndServe(protoAddrParts[0], protoAddrParts[1], srv, true) @@ -89,10 +91,10 @@ func (srv *Server) ListenAndServe(protoAddrs ...string) error { for i := 0; i < len(protoAddrs); i += 1 { err := <-chErrors if err != nil { - return err + return err.Error() } } - return nil + return "0" } func (srv *Server) DockerVersion() APIVersion { @@ -1282,8 +1284,7 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { return fmt.Errorf("No such container: %s", name) } - // Register links - if hostConfig != nil && hostConfig.Links != nil { + if hostConfig.Links != nil { for _, l := range hostConfig.Links { parts, err := parseLink(l) if err != nil { @@ -1296,7 +1297,6 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { if child == nil { return fmt.Errorf("Could not get container for %s", parts["name"]) } - if err := runtime.RegisterLink(container, child, parts["alias"]); err != nil { return err } @@ -1312,22 +1312,36 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { return nil } -func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error { +func (srv *Server) ContainerStart(job *engine.Job) string { + if len(job.Args) < 1 { + return fmt.Sprintf("Usage: %s container_id", job.Name) + } + name := job.Args[0] runtime := srv.runtime container := runtime.Get(name) if container == nil { - return fmt.Errorf("No such container: %s", name) + return fmt.Sprintf("No such container: %s", name) } - if hostConfig != nil { - container.hostConfig = hostConfig + // If no environment was set, then no hostconfig was passed. + if len(job.Environ()) > 0 { + var hostConfig HostConfig + if err := job.ExportEnv(&hostConfig); err != nil { + return err.Error() + } + // Register any links from the host config before starting the container + // FIXME: we could just pass the container here, no need to lookup by name again. + if err := srv.RegisterLinks(name, &hostConfig); err != nil { + return err.Error() + } + container.hostConfig = &hostConfig container.ToDisk() } if err := container.Start(); err != nil { - return fmt.Errorf("Cannot start container %s: %s", name, err) + return fmt.Sprintf("Cannot start container %s: %s", name, err) } srv.LogEvent("start", container.ShortID(), runtime.repositories.ImageName(container.Image)) - return nil + return "0" } func (srv *Server) ContainerStop(name string, t int) error { @@ -1478,12 +1492,13 @@ func (srv *Server) ContainerCopy(name string, resource string, out io.Writer) er } -func NewServer(config *DaemonConfig) (*Server, error) { +func NewServer(eng *engine.Engine, config *DaemonConfig) (*Server, error) { runtime, err := NewRuntime(config) if err != nil { return nil, err } srv := &Server{ + Eng: eng, runtime: runtime, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), @@ -1527,4 +1542,5 @@ type Server struct { events []utils.JSONMessage listeners map[string]chan utils.JSONMessage reqFactory *utils.HTTPRequestFactory + Eng *engine.Engine } diff --git a/server_test.go b/server_test.go index 7f9cbaadf..20894f8fe 100644 --- a/server_test.go +++ b/server_test.go @@ -1,6 +1,7 @@ package docker import ( + "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "strings" "testing" @@ -109,10 +110,11 @@ func TestCreateRm(t *testing.T) { } func TestCreateRmVolumes(t *testing.T) { - runtime := mkRuntime(t) - defer nuke(runtime) + eng := engine.NewTestEngine(t) - srv := &Server{runtime: runtime} + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime + defer nuke(runtime) config, hostConfig, _, err := ParseRun([]string{"-v", "/srv", GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { @@ -128,8 +130,11 @@ func TestCreateRmVolumes(t *testing.T) { t.Errorf("Expected 1 container, %v found", len(runtime.List())) } - err = srv.ContainerStart(id, hostConfig) - if err != nil { + job := eng.Job("start", id) + if err := job.ImportEnv(hostConfig); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { t.Fatal(err) } @@ -169,11 +174,11 @@ func TestCommit(t *testing.T) { } func TestCreateStartRestartStopStartKillRm(t *testing.T) { - runtime := mkRuntime(t) + eng := engine.NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} - config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) @@ -188,7 +193,11 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Errorf("Expected 1 container, %v found", len(runtime.List())) } - if err := srv.ContainerStart(id, hostConfig); err != nil { + job := eng.Job("start", id) + if err := job.ImportEnv(hostConfig); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { t.Fatal(err) } @@ -200,7 +209,11 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - if err := srv.ContainerStart(id, hostConfig); err != nil { + job = eng.Job("start", id) + if err := job.ImportEnv(hostConfig); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { t.Fatal(err) } @@ -384,9 +397,10 @@ func TestLogEvent(t *testing.T) { } func TestRmi(t *testing.T) { - runtime := mkRuntime(t) + eng := engine.NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} initialImages, err := srv.Images(false, "") if err != nil { @@ -404,8 +418,11 @@ func TestRmi(t *testing.T) { } //To remove - err = srv.ContainerStart(containerID, hostConfig) - if err != nil { + job := eng.Job("start", containerID) + if err := job.ImportEnv(hostConfig); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { t.Fatal(err) } @@ -425,8 +442,11 @@ func TestRmi(t *testing.T) { } //To remove - err = srv.ContainerStart(containerID, hostConfig) - if err != nil { + job = eng.Job("start", containerID) + if err := job.ImportEnv(hostConfig); err != nil { + t.Fatal(err) + } + if err := job.Run(); err != nil { t.Fatal(err) } diff --git a/utils_test.go b/utils_test.go index 14cbd7c6b..282954285 100644 --- a/utils_test.go +++ b/utils_test.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -40,6 +41,18 @@ func mkRuntime(f Fataler) *Runtime { return runtime } +func mkServerFromEngine(eng *engine.Engine, t Fataler) *Server { + iSrv := eng.Hack_GetGlobalVar("httpapi.server") + if iSrv == nil { + t.Fatal("Legacy server field not set in engine") + } + srv, ok := iSrv.(*Server) + if !ok { + t.Fatal("Legacy server field in engine does not cast to *Server") + } + return srv +} + // A common interface to access the Fatal method of // both testing.B and testing.T. type Fataler interface { From 7b17d555992c65c6b10da21c8aa48062a1aba0d9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 06:51:43 +0000 Subject: [PATCH 09/79] httpapi: don't create a pidfile if it isn't set in the configuration --- server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index efc34d14e..de79dd966 100644 --- a/server.go +++ b/server.go @@ -44,8 +44,11 @@ func jobInitApi(job *engine.Job) string { if err != nil { return err.Error() } - if err := utils.CreatePidFile(srv.runtime.config.Pidfile); err != nil { - log.Fatal(err) + if srv.runtime.config.Pidfile != "" { + job.Logf("Creating pidfile") + if err := utils.CreatePidFile(srv.runtime.config.Pidfile); err != nil { + log.Fatal(err) + } } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) From 847411a1ee6e5ee5d051fc4729425215dc0c8561 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 06:54:51 +0000 Subject: [PATCH 10/79] Engine: fix a bug which caused handlers to be shared between multiple engine instances --- engine/engine.go | 22 +++++++++++++++++++++- server.go | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index a0d5a3c4a..956847ade 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -18,6 +18,10 @@ func init() { } func Register(name string, handler Handler) error { + _, exists := globalHandlers[name] + if exists { + return fmt.Errorf("Can't overwrite global handler for command %s", name) + } globalHandlers[name] = handler return nil } @@ -31,6 +35,17 @@ type Engine struct { hack Hack // data for temporary hackery (see hack.go) } +func (eng *Engine) Register(name string, handler Handler) error { + eng.Logf("Register(%s) (handlers=%v)", name, eng.handlers) + _, exists := eng.handlers[name] + if exists { + return fmt.Errorf("Can't overwrite handler for command %s", name) + } + eng.handlers[name] = handler + return nil +} + + // New initializes a new engine managing the directory specified at `root`. // `root` is used to store containers and any other state private to the engine. // Changing the contents of the root without executing a job will cause unspecified @@ -59,7 +74,12 @@ func New(root string) (*Engine, error) { } eng := &Engine{ root: root, - handlers: globalHandlers, + handlers: make(map[string]Handler), + id: utils.RandomString(), + } + // Copy existing global handlers + for k, v := range globalHandlers { + eng.handlers[k] = v } return eng, nil } diff --git a/server.go b/server.go index de79dd966..72ad39cdc 100644 --- a/server.go +++ b/server.go @@ -60,10 +60,10 @@ func jobInitApi(job *engine.Job) string { os.Exit(0) }() job.Eng.Hack_SetGlobalVar("httpapi.server", srv) - if err := engine.Register("start", srv.ContainerStart); err != nil { + if err := job.Eng.Register("start", srv.ContainerStart); err != nil { return err.Error() } - if err := engine.Register("serveapi", srv.ListenAndServe); err != nil { + if err := job.Eng.Register("serveapi", srv.ListenAndServe); err != nil { return err.Error() } return "0" From ca6f0aa107117d2125a63eb5e78d74095bf08a4c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 06:57:29 +0000 Subject: [PATCH 11/79] Engine: don't export private testing utilities --- engine/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/utils.go b/engine/utils.go index 58e5f9aae..14ee4742f 100644 --- a/engine/utils.go +++ b/engine/utils.go @@ -15,7 +15,7 @@ func init() { Register("dummy", func(job *Job) string { return ""; }) } -func NewTestEngine(t *testing.T) *Engine { +func newTestEngine(t *testing.T) *Engine { // Use the caller function name as a prefix. // This helps trace temp directories back to their test. pc, _, _, _ := runtime.Caller(1) @@ -38,5 +38,5 @@ func NewTestEngine(t *testing.T) *Engine { } func mkJob(t *testing.T, name string, args ...string) *Job { - return NewTestEngine(t).Job(name, args...) + return newTestEngine(t).Job(name, args...) } From 4e7cb37dcc18975010df630d8c9580a3a65e0e69 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 07:01:15 +0000 Subject: [PATCH 12/79] Engine: improved logging and identification of jobs --- engine/engine.go | 15 +++++++++++++++ engine/job.go | 28 +++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index 956847ade..565c6ed4f 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -5,6 +5,7 @@ import ( "os" "log" "runtime" + "strings" "github.com/dotcloud/docker/utils" ) @@ -33,6 +34,11 @@ type Engine struct { root string handlers map[string]Handler hack Hack // data for temporary hackery (see hack.go) + id string +} + +func (eng *Engine) Root() string { + return eng.root } func (eng *Engine) Register(name string, handler Handler) error { @@ -84,6 +90,10 @@ func New(root string) (*Engine, error) { return eng, nil } +func (eng *Engine) String() string { + return fmt.Sprintf("%s|%s", eng.Root(), eng.id[:8]) +} + // Job creates a new job which can later be executed. // This function mimics `Command` from the standard os/exec package. func (eng *Engine) Job(name string, args ...string) *Job { @@ -102,3 +112,8 @@ func (eng *Engine) Job(name string, args ...string) *Job { return job } + +func (eng *Engine) Logf(format string, args ...interface{}) (n int, err error) { + prefixedFormat := fmt.Sprintf("[%s] %s\n", eng, strings.TrimRight(format, "\n")) + return fmt.Printf(prefixedFormat, args...) +} diff --git a/engine/job.go b/engine/job.go index 5c02fe15d..7a261409e 100644 --- a/engine/job.go +++ b/engine/job.go @@ -6,7 +6,6 @@ import ( "strings" "fmt" "encoding/json" - "github.com/dotcloud/docker/utils" ) // A job is the fundamental unit of work in the docker engine. @@ -38,9 +37,10 @@ type Job struct { // If the job returns a failure status, an error is returned // which includes the status. func (job *Job) Run() error { - randId := utils.RandomString()[:4] - fmt.Printf("Job #%s: %s\n", randId, job) - defer fmt.Printf("Job #%s: %s = '%s'", randId, job, job.status) + job.Logf("{") + defer func() { + job.Logf("}") + }() if job.handler == nil { job.status = "command not found" } else { @@ -54,7 +54,20 @@ func (job *Job) Run() error { // String returns a human-readable description of `job` func (job *Job) String() string { - return strings.Join(append([]string{job.Name}, job.Args...), " ") + s := fmt.Sprintf("%s.%s(%s)", job.Eng, job.Name, strings.Join(job.Args, ", ")) + // FIXME: if a job returns the empty string, it will be printed + // as not having returned. + // (this only affects String which is a convenience function). + if job.status != "" { + var okerr string + if job.status == "0" { + okerr = "OK" + } else { + okerr = "ERR" + } + s = fmt.Sprintf("%s = %s (%s)", s, okerr, job.status) + } + return s } func (job *Job) Getenv(key string) (value string) { @@ -169,3 +182,8 @@ func (job *Job) Environ() map[string]string { } return m } + +func (job *Job) Logf(format string, args ...interface{}) (n int, err error) { + prefixedFormat := fmt.Sprintf("[%s] %s\n", job, strings.TrimRight(format, "\n")) + return fmt.Fprintf(job.Stdout, prefixedFormat, args...) +} From 02ddaad5d985186eed94dea4105a57fa21ba24db Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 07:06:43 +0000 Subject: [PATCH 13/79] Engine: optional environment variable 'Logging' in 'serveapi' --- docker/docker.go | 4 +++- server.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 2fc864adf..e58bd4001 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -86,7 +86,9 @@ func main() { log.Fatal(err) } // Serve api - if err := eng.Job("serveapi", flHosts...).Run(); err != nil { + job := eng.Job("serveapi", flHosts...) + job.Setenv("Logging", true) + if err := job.Run(); err != nil { log.Fatal(err) } } else { diff --git a/server.go b/server.go index 72ad39cdc..c041491ea 100644 --- a/server.go +++ b/server.go @@ -88,7 +88,8 @@ func (srv *Server) ListenAndServe(job *engine.Job) string { return "Invalid protocol format." } go func() { - chErrors <- ListenAndServe(protoAddrParts[0], protoAddrParts[1], srv, true) + // FIXME: merge Server.ListenAndServe with ListenAndServe + chErrors <- ListenAndServe(protoAddrParts[0], protoAddrParts[1], srv, job.GetenvBool("Logging")) }() } for i := 0; i < len(protoAddrs); i += 1 { From 5a85456d481a5f88fc0efc02c41b3bff987c0ed1 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 07:13:45 +0000 Subject: [PATCH 14/79] Hack: simplify the creation of test directories --- runtime_test.go | 45 ++++++++++++----------- server_test.go | 7 ++-- utils/utils.go | 6 ++++ utils_test.go | 96 ++++++++++++++++++++++++++----------------------- 4 files changed, 85 insertions(+), 69 deletions(-) diff --git a/runtime_test.go b/runtime_test.go index 4e46b7b6d..4aac2f344 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -3,6 +3,7 @@ package docker import ( "bytes" "fmt" + "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/sysinit" "github.com/dotcloud/docker/utils" "io" @@ -17,6 +18,7 @@ import ( "syscall" "testing" "time" + "net/url" ) const ( @@ -118,22 +120,19 @@ func init() { } func setupBaseImage() { - config := &DaemonConfig{ - Root: unitTestStoreBase, - AutoRestart: false, - BridgeIface: unitTestNetworkBridge, - } - runtime, err := NewRuntimeFromDirectory(config) + eng, err := engine.New(unitTestStoreBase) if err != nil { + log.Fatalf("Can't initialize engine at %s: %s", unitTestStoreBase, err) + } + job := eng.Job("initapi") + job.Setenv("Root", unitTestStoreBase) + job.SetenvBool("Autorestart", false) + job.Setenv("BridgeIface", unitTestNetworkBridge) + if err := job.Run(); err != nil { log.Fatalf("Unable to create a runtime for tests:", err) } - - // Create the "Server" - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), - } + srv := mkServerFromEngine(eng, log.New(os.Stderr, "", 0)) + runtime := srv.runtime // If the unit test is not found, try to download it. if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { @@ -149,18 +148,22 @@ func spawnGlobalDaemon() { utils.Debugf("Global runtime already exists. Skipping.") return } - globalRuntime = mkRuntime(log.New(os.Stderr, "", 0)) - srv := &Server{ - runtime: globalRuntime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), - } + t := log.New(os.Stderr, "", 0) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + globalRuntime = srv.runtime // Spawn a Daemon go func() { utils.Debugf("Spawning global daemon for integration tests") - if err := ListenAndServe(testDaemonProto, testDaemonAddr, srv, os.Getenv("DEBUG") != ""); err != nil { - log.Fatalf("Unable to spawn the test daemon:", err) + listenURL := &url.URL{ + Scheme: testDaemonProto, + Host: testDaemonAddr, + } + job := eng.Job("serveapi", listenURL.String()) + job.SetenvBool("Logging", os.Getenv("DEBUG") != "") + if err := job.Run(); err != nil { + log.Fatalf("Unable to spawn the test daemon: %s", err) } }() // Give some time to ListenAndServer to actually start diff --git a/server_test.go b/server_test.go index 20894f8fe..a4bfb0155 100644 --- a/server_test.go +++ b/server_test.go @@ -1,7 +1,6 @@ package docker import ( - "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "strings" "testing" @@ -110,7 +109,7 @@ func TestCreateRm(t *testing.T) { } func TestCreateRmVolumes(t *testing.T) { - eng := engine.NewTestEngine(t) + eng := NewTestEngine(t) srv := mkServerFromEngine(eng, t) runtime := srv.runtime @@ -174,7 +173,7 @@ func TestCommit(t *testing.T) { } func TestCreateStartRestartStopStartKillRm(t *testing.T) { - eng := engine.NewTestEngine(t) + eng := NewTestEngine(t) srv := mkServerFromEngine(eng, t) runtime := srv.runtime defer nuke(runtime) @@ -397,7 +396,7 @@ func TestLogEvent(t *testing.T) { } func TestRmi(t *testing.T) { - eng := engine.NewTestEngine(t) + eng := NewTestEngine(t) srv := mkServerFromEngine(eng, t) runtime := srv.runtime defer nuke(runtime) diff --git a/utils/utils.go b/utils/utils.go index cadd09503..5fd5e4e5b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -27,6 +27,12 @@ var ( INITSHA1 string // sha1sum of separate static dockerinit, if Docker itself was compiled dynamically via ./hack/make.sh dynbinary ) +// A common interface to access the Fatal method of +// both testing.B and testing.T. +type Fataler interface { + Fatal(args ...interface{}) +} + // ListOpts type type ListOpts []string diff --git a/utils_test.go b/utils_test.go index 282954285..95e12b639 100644 --- a/utils_test.go +++ b/utils_test.go @@ -21,27 +21,24 @@ var globalTestID string // Create a temporary runtime suitable for unit testing. // Call t.Fatal() at the first error. -func mkRuntime(f Fataler) *Runtime { - // Use the caller function name as a prefix. - // This helps trace temp directories back to their test. - pc, _, _, _ := runtime.Caller(1) - callerLongName := runtime.FuncForPC(pc).Name() - parts := strings.Split(callerLongName, ".") - callerShortName := parts[len(parts)-1] - if globalTestID == "" { - globalTestID = GenerateID()[:4] - } - prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, callerShortName) - utils.Debugf("prefix = '%s'", prefix) - - runtime, err := newTestRuntime(prefix) +func mkRuntime(f utils.Fataler) *Runtime { + root, err := newTestDirectory(unitTestStoreBase) if err != nil { f.Fatal(err) } - return runtime + config := &DaemonConfig{ + Root: root, + AutoRestart: false, + } + r, err := NewRuntimeFromDirectory(config) + if err != nil { + f.Fatal(err) + } + r.UpdateCapabilities(true) + return r } -func mkServerFromEngine(eng *engine.Engine, t Fataler) *Server { +func mkServerFromEngine(eng *engine.Engine, t utils.Fataler) *Server { iSrv := eng.Hack_GetGlobalVar("httpapi.server") if iSrv == nil { t.Fatal("Legacy server field not set in engine") @@ -53,42 +50,53 @@ func mkServerFromEngine(eng *engine.Engine, t Fataler) *Server { return srv } -// A common interface to access the Fatal method of -// both testing.B and testing.T. -type Fataler interface { - Fatal(args ...interface{}) + +func NewTestEngine(t utils.Fataler) *engine.Engine { + root, err := newTestDirectory(unitTestStoreBase) + if err != nil { + t.Fatal(err) + } + eng, err := engine.New(root) + if err != nil { + t.Fatal(err) + } + // Load default plugins + // (This is manually copied and modified from main() until we have a more generic plugin system) + job := eng.Job("initapi") + job.Setenv("Root", root) + job.SetenvBool("AutoRestart", false) + if err := job.Run(); err != nil { + t.Fatal(err) + } + return eng } -func newTestRuntime(prefix string) (runtime *Runtime, err error) { +func newTestDirectory(templateDir string) (dir string, err error) { + if globalTestID == "" { + globalTestID = GenerateID()[:4] + } + prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, getCallerName(2)) if prefix == "" { prefix = "docker-test-" } - utils.Debugf("prefix = %s", prefix) - utils.Debugf("newTestRuntime start") - root, err := ioutil.TempDir("", prefix) - defer func() { - utils.Debugf("newTestRuntime: %s", root) - }() - if err != nil { - return nil, err + dir, err = ioutil.TempDir("", prefix) + if err = os.Remove(dir); err != nil { + return } - if err := os.Remove(root); err != nil { - return nil, err - } - if err := utils.CopyDirectory(unitTestStoreBase, root); err != nil { - return nil, err + if err = utils.CopyDirectory(templateDir, dir); err != nil { + return } + return +} - config := &DaemonConfig{ - Root: root, - AutoRestart: false, - } - runtime, err = NewRuntimeFromDirectory(config) - if err != nil { - return nil, err - } - runtime.UpdateCapabilities(true) - return runtime, nil +func getCallerName(depth int) string { + // Use the caller function name as a prefix. + // This helps trace temp directories back to their test. + pc, _, _, _ := runtime.Caller(depth + 1) + callerLongName := runtime.FuncForPC(pc).Name() + parts := strings.Split(callerLongName, ".") + callerShortName := parts[len(parts)-1] + return callerShortName } // Write `content` to the file at path `dst`, creating it if necessary, From 434f06d03dc2825cb4f348a88ddc6d1aa17ea19c Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 07:15:22 +0000 Subject: [PATCH 15/79] Engine: fix a bug when encoding a job environment to json --- api.go | 3 +++ engine/job.go | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 1a3d92de3..18251eba2 100644 --- a/api.go +++ b/api.go @@ -644,6 +644,9 @@ func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r } name := vars["name"] job := srv.Eng.Job("start", name) + if err := job.ImportEnv(HostConfig{}); err != nil { + return fmt.Errorf("Couldn't initialize host configuration") + } // allow a nil body for backwards compatibility if r.Body != nil { if matchesContentType(r.Header.Get("Content-Type"), "application/json") { diff --git a/engine/job.go b/engine/job.go index 7a261409e..b6296eb91 100644 --- a/engine/job.go +++ b/engine/job.go @@ -149,10 +149,22 @@ func (job *Job) DecodeEnv(src io.Reader) error { } func (job *Job) EncodeEnv(dst io.Writer) error { - return json.NewEncoder(dst).Encode(job.Environ()) + m := make(map[string]interface{}) + for k, v := range job.Environ() { + var val interface{} + if err := json.Unmarshal([]byte(v), &val); err == nil { + m[k] = val + } else { + m[k] = v + } + } + if err := json.NewEncoder(dst).Encode(&m); err != nil { + return err + } + return nil } -func (job *Job) ExportEnv(dst interface{}) error { +func (job *Job) ExportEnv(dst interface{}) (err error) { var buf bytes.Buffer if err := job.EncodeEnv(&buf); err != nil { return err From d3f074494a9594bc268bf4c639a7aea0934ec7c0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 07:16:32 +0000 Subject: [PATCH 16/79] Better error reporting in engine logs and unit tests --- runtime_test.go | 2 +- server.go | 7 ++++++- utils_test.go | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/runtime_test.go b/runtime_test.go index 4aac2f344..946a8ebf6 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -183,7 +183,7 @@ func GetTestImage(runtime *Runtime) *Image { return image } } - log.Fatalf("Test image %v not found", unitTestImageID) + log.Fatalf("Test image %v not found in %s: %s", unitTestImageID, runtime.graph.Root, imgs) return nil } diff --git a/server.go b/server.go index c041491ea..0a1270bed 100644 --- a/server.go +++ b/server.go @@ -40,6 +40,7 @@ func init() { // Only one api server can run at the same time - this is enforced by a pidfile. // The signals SIGINT, SIGKILL and SIGTERM are intercepted for cleanup. func jobInitApi(job *engine.Job) string { + job.Logf("Creating server") srv, err := NewServer(job.Eng, ConfigFromJob(job)) if err != nil { return err.Error() @@ -50,6 +51,7 @@ func jobInitApi(job *engine.Job) string { log.Fatal(err) } } + job.Logf("Setting up signal traps") c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill, os.Signal(syscall.SIGTERM)) go func() { @@ -1288,7 +1290,7 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { return fmt.Errorf("No such container: %s", name) } - if hostConfig.Links != nil { + if hostConfig != nil && hostConfig.Links != nil { for _, l := range hostConfig.Links { parts, err := parseLink(l) if err != nil { @@ -1317,11 +1319,14 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { } func (srv *Server) ContainerStart(job *engine.Job) string { + job.Logf("srv engine = %s", srv.Eng.Root()) + job.Logf("job engine = %s", job.Eng.Root()) if len(job.Args) < 1 { return fmt.Sprintf("Usage: %s container_id", job.Name) } name := job.Args[0] runtime := srv.runtime + job.Logf("loading containers from %s", runtime.repository) container := runtime.Get(name) if container == nil { return fmt.Sprintf("No such container: %s", name) diff --git a/utils_test.go b/utils_test.go index 95e12b639..529af127e 100644 --- a/utils_test.go +++ b/utils_test.go @@ -41,11 +41,11 @@ func mkRuntime(f utils.Fataler) *Runtime { func mkServerFromEngine(eng *engine.Engine, t utils.Fataler) *Server { iSrv := eng.Hack_GetGlobalVar("httpapi.server") if iSrv == nil { - t.Fatal("Legacy server field not set in engine") + panic("Legacy server field not set in engine") } srv, ok := iSrv.(*Server) if !ok { - t.Fatal("Legacy server field in engine does not cast to *Server") + panic("Legacy server field in engine does not cast to *Server") } return srv } From 5c42b2b5122c1db08d229c258da26869b4d4d9cc Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Nov 2013 19:57:40 +0000 Subject: [PATCH 17/79] Fix main() --- docker/docker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index e58bd4001..213ec73fd 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -86,8 +86,8 @@ func main() { log.Fatal(err) } // Serve api - job := eng.Job("serveapi", flHosts...) - job.Setenv("Logging", true) + job = eng.Job("serveapi", flHosts...) + job.SetenvBool("Logging", true) if err := job.Run(); err != nil { log.Fatal(err) } From e5f8ab6160401fb541121da5b5cbc3af4fce28b7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 27 Oct 2013 19:20:00 -0700 Subject: [PATCH 18/79] Engine: 'create' creates a container and prints its ID on stdout --- api.go | 31 ++++----- api_test.go | 28 ++++---- engine/engine.go | 2 +- engine/job.go | 165 +++++++++++++++++++++++++++++++++++++++++++---- runtime_test.go | 58 ++++++----------- server.go | 35 ++++++---- server_test.go | 64 +++++++----------- utils_test.go | 16 +++++ 8 files changed, 260 insertions(+), 139 deletions(-) diff --git a/api.go b/api.go index 18251eba2..1fc74d764 100644 --- a/api.go +++ b/api.go @@ -526,43 +526,36 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r if err := parseForm(r); err != nil { return nil } - config := &Config{} out := &APIRun{} - name := r.Form.Get("name") - - if err := json.NewDecoder(r.Body).Decode(config); err != nil { + job := srv.Eng.Job("create", r.Form.Get("name")) + if err := job.DecodeEnv(r.Body); err != nil { return err } - resolvConf, err := utils.GetResolvConf() if err != nil { return err } - - if !config.NetworkDisabled && len(config.Dns) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) { + if !job.GetenvBool("NetworkDisabled") && len(job.Getenv("Dns")) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) { out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns)) - config.Dns = defaultDns + job.SetenvList("Dns", defaultDns) } - - id, warnings, err := srv.ContainerCreate(config, name) - if err != nil { + // Read container ID from the first line of stdout + job.StdoutParseString(&out.ID) + // Read warnings from stderr + job.StderrParseLines(&out.Warnings, 0) + if err := job.Run(); err != nil { return err } - out.ID = id - for _, warning := range warnings { - out.Warnings = append(out.Warnings, warning) - } - - if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit { + if job.GetenvInt("Memory") > 0 && !srv.runtime.capabilities.MemoryLimit { log.Println("WARNING: Your kernel does not support memory limit capabilities. Limitation discarded.") out.Warnings = append(out.Warnings, "Your kernel does not support memory limit capabilities. Limitation discarded.") } - if config.Memory > 0 && !srv.runtime.capabilities.SwapLimit { + if job.GetenvInt("Memory") > 0 && !srv.runtime.capabilities.SwapLimit { log.Println("WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.") out.Warnings = append(out.Warnings, "Your kernel does not support memory swap capabilities. Limitation discarded.") } - if !config.NetworkDisabled && srv.runtime.capabilities.IPv4ForwardingDisabled { + if !job.GetenvBool("NetworkDisabled") && srv.runtime.capabilities.IPv4ForwardingDisabled { log.Println("Warning: IPv4 forwarding is disabled.") out.Warnings = append(out.Warnings, "IPv4 forwarding is disabled.") } diff --git a/api_test.go b/api_test.go index 50a86e3f2..7b0dfaa07 100644 --- a/api_test.go +++ b/api_test.go @@ -634,11 +634,11 @@ func TestPostCommit(t *testing.T) { } func TestPostContainersCreate(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} - configJSON, err := json.Marshal(&Config{ Image: GetTestImage(runtime).ID, Memory: 33554432, @@ -786,22 +786,18 @@ func TestPostContainersStart(t *testing.T) { runtime := srv.runtime defer nuke(runtime) - container, _, err := runtime.Create( + id := createTestContainer( + eng, &Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/cat"}, OpenStdin: true, }, - "", - ) - if err != nil { - t.Fatal(err) - } - defer runtime.Destroy(container) + t) hostConfigJSON, err := json.Marshal(&HostConfig{}) - req, err := http.NewRequest("POST", "/containers/"+container.ID+"/start", bytes.NewReader(hostConfigJSON)) + req, err := http.NewRequest("POST", "/containers/"+id+"/start", bytes.NewReader(hostConfigJSON)) if err != nil { t.Fatal(err) } @@ -809,22 +805,26 @@ func TestPostContainersStart(t *testing.T) { req.Header.Set("Content-Type", "application/json") r := httptest.NewRecorder() - if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": id}); err != nil { t.Fatal(err) } if r.Code != http.StatusNoContent { t.Fatalf("%d NO CONTENT expected, received %d\n", http.StatusNoContent, r.Code) } + container := runtime.Get(id) + if container == nil { + t.Fatalf("Container %s was not created", id) + } // Give some time to the process to start + // FIXME: use Wait once it's available as a job container.WaitTimeout(500 * time.Millisecond) - if !container.State.Running { t.Errorf("Container should be running") } r = httptest.NewRecorder() - if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err == nil { + if err = postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": id}); err == nil { t.Fatalf("A running container should be able to be started") } diff --git a/engine/engine.go b/engine/engine.go index 565c6ed4f..428ef706c 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -115,5 +115,5 @@ func (eng *Engine) Job(name string, args ...string) *Job { func (eng *Engine) Logf(format string, args ...interface{}) (n int, err error) { prefixedFormat := fmt.Sprintf("[%s] %s\n", eng, strings.TrimRight(format, "\n")) - return fmt.Printf(prefixedFormat, args...) + return fmt.Fprintf(os.Stderr, prefixedFormat, args...) } diff --git a/engine/job.go b/engine/job.go index b6296eb91..ece8d8f8d 100644 --- a/engine/job.go +++ b/engine/job.go @@ -1,11 +1,16 @@ package engine import ( + "bufio" "bytes" "io" + "io/ioutil" + "strconv" "strings" "fmt" + "sync" "encoding/json" + "os" ) // A job is the fundamental unit of work in the docker engine. @@ -26,20 +31,38 @@ type Job struct { Name string Args []string env []string - Stdin io.ReadCloser - Stdout io.WriteCloser - Stderr io.WriteCloser + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer handler func(*Job) string status string + onExit []func() } // Run executes the job and blocks until the job completes. // If the job returns a failure status, an error is returned // which includes the status. func (job *Job) Run() error { - job.Logf("{") defer func() { - job.Logf("}") + var wg sync.WaitGroup + for _, f := range job.onExit { + wg.Add(1) + go func(f func()) { + f() + wg.Done() + }(f) + } + wg.Wait() + }() + if job.Stdout != nil && job.Stdout != os.Stdout { + job.Stdout = io.MultiWriter(job.Stdout, os.Stdout) + } + if job.Stderr != nil && job.Stderr != os.Stderr { + job.Stderr = io.MultiWriter(job.Stderr, os.Stderr) + } + job.Eng.Logf("+job %s", job.CallString()) + defer func() { + job.Eng.Logf("-job %s%s", job.CallString(), job.StatusString()) }() if job.handler == nil { job.status = "command not found" @@ -52,9 +75,66 @@ func (job *Job) Run() error { return nil } -// String returns a human-readable description of `job` -func (job *Job) String() string { - s := fmt.Sprintf("%s.%s(%s)", job.Eng, job.Name, strings.Join(job.Args, ", ")) +func (job *Job) StdoutParseLines(dst *[]string, limit int) { + job.parseLines(job.StdoutPipe(), dst, limit) +} + +func (job *Job) StderrParseLines(dst *[]string, limit int) { + job.parseLines(job.StderrPipe(), dst, limit) +} + +func (job *Job) parseLines(src io.Reader, dst *[]string, limit int) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + scanner := bufio.NewScanner(src) + for scanner.Scan() { + // If the limit is reached, flush the rest of the source and return + if limit > 0 && len(*dst) >= limit { + io.Copy(ioutil.Discard, src) + return + } + line := scanner.Text() + // Append the line (with delimitor removed) + *dst = append(*dst, line) + } + }() + job.onExit = append(job.onExit, wg.Wait) +} + +func (job *Job) StdoutParseString(dst *string) { + lines := make([]string, 0, 1) + job.StdoutParseLines(&lines, 1) + job.onExit = append(job.onExit, func() { if len(lines) >= 1 { *dst = lines[0] }}) +} + +func (job *Job) StderrParseString(dst *string) { + lines := make([]string, 0, 1) + job.StderrParseLines(&lines, 1) + job.onExit = append(job.onExit, func() { *dst = lines[0]; }) +} + +func (job *Job) StdoutPipe() io.ReadCloser { + r, w := io.Pipe() + job.Stdout = w + job.onExit = append(job.onExit, func(){ w.Close() }) + return r +} + +func (job *Job) StderrPipe() io.ReadCloser { + r, w := io.Pipe() + job.Stderr = w + job.onExit = append(job.onExit, func(){ w.Close() }) + return r +} + + +func (job *Job) CallString() string { + return fmt.Sprintf("%s(%s)", job.Name, strings.Join(job.Args, ", ")) +} + +func (job *Job) StatusString() string { // FIXME: if a job returns the empty string, it will be printed // as not having returned. // (this only affects String which is a convenience function). @@ -65,9 +145,14 @@ func (job *Job) String() string { } else { okerr = "ERR" } - s = fmt.Sprintf("%s = %s (%s)", s, okerr, job.status) + return fmt.Sprintf(" = %s (%s)", okerr, job.status) } - return s + return "" +} + +// String returns a human-readable description of `job` +func (job *Job) String() string { + return fmt.Sprintf("%s.%s%s", job.Eng, job.CallString(), job.StatusString()) } func (job *Job) Getenv(key string) (value string) { @@ -104,6 +189,19 @@ func (job *Job) SetenvBool(key string, value bool) { } } +func (job *Job) GetenvInt(key string) int64 { + s := strings.Trim(job.Getenv(key), " \t") + val, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return -1 + } + return val +} + +func (job *Job) SetenvInt(key string, value int64) { + job.Setenv(key, fmt.Sprintf("%d", value)) +} + func (job *Job) GetenvList(key string) []string { sval := job.Getenv(key) l := make([]string, 0, 1) @@ -137,13 +235,21 @@ func (job *Job) DecodeEnv(src io.Reader) error { return err } for k, v := range m { - if sval, ok := v.(string); ok { + // FIXME: we fix-convert float values to int, because + // encoding/json decodes integers to float64, but cannot encode them back. + // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) + if fval, ok := v.(float64); ok { + job.Logf("Converted to float: %v->%v", v, fval) + job.SetenvInt(k, int64(fval)) + } else if sval, ok := v.(string); ok { + job.Logf("Converted to string: %v->%v", v, sval) job.Setenv(k, sval) } else if val, err := json.Marshal(v); err == nil { job.Setenv(k, string(val)) } else { job.Setenv(k, fmt.Sprintf("%v", v)) } + job.Logf("Decoded %s=%#v to %s=%#v", k, v, k, job.Getenv(k)) } return nil } @@ -153,10 +259,17 @@ func (job *Job) EncodeEnv(dst io.Writer) error { for k, v := range job.Environ() { var val interface{} if err := json.Unmarshal([]byte(v), &val); err == nil { + // FIXME: we fix-convert float values to int, because + // encoding/json decodes integers to float64, but cannot encode them back. + // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) + if fval, isFloat := val.(float64); isFloat { + val = int(fval) + } m[k] = val } else { m[k] = v } + job.Logf("Encoded %s=%#v to %s=%#v", k, v, k, m[k]) } if err := json.NewEncoder(dst).Encode(&m); err != nil { return err @@ -165,21 +278,38 @@ func (job *Job) EncodeEnv(dst io.Writer) error { } func (job *Job) ExportEnv(dst interface{}) (err error) { + fmt.Fprintf(os.Stderr, "ExportEnv()\n") + defer func() { + if err != nil { + err = fmt.Errorf("ExportEnv %s", err) + } + }() var buf bytes.Buffer + job.Logf("ExportEnv: step 1: encode/marshal the env to an intermediary json representation") + fmt.Fprintf(os.Stderr, "Printed ExportEnv step 1\n") if err := job.EncodeEnv(&buf); err != nil { return err } + job.Logf("ExportEnv: step 1 complete: json=|%s|", buf) + job.Logf("ExportEnv: step 2: decode/unmarshal the intermediary json into the destination object") if err := json.NewDecoder(&buf).Decode(dst); err != nil { return err } + job.Logf("ExportEnv: step 2 complete") return nil } -func (job *Job) ImportEnv(src interface{}) error { +func (job *Job) ImportEnv(src interface{}) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("ImportEnv: %s", err) + } + }() var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(src); err != nil { return err } + job.Logf("ImportEnv: json=|%s|", buf) if err := job.DecodeEnv(&buf); err != nil { return err } @@ -197,5 +327,14 @@ func (job *Job) Environ() map[string]string { func (job *Job) Logf(format string, args ...interface{}) (n int, err error) { prefixedFormat := fmt.Sprintf("[%s] %s\n", job, strings.TrimRight(format, "\n")) - return fmt.Fprintf(job.Stdout, prefixedFormat, args...) + return fmt.Fprintf(job.Stderr, prefixedFormat, args...) +} + +func (job *Job) Printf(format string, args ...interface{}) (n int, err error) { + return fmt.Fprintf(job.Stdout, format, args...) +} + +func (job *Job) Errorf(format string, args ...interface{}) (n int, err error) { + return fmt.Fprintf(job.Stderr, format, args...) + } diff --git a/runtime_test.go b/runtime_test.go index 946a8ebf6..ce6946596 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -645,20 +645,17 @@ func TestReloadContainerLinks(t *testing.T) { } func TestDefaultContainerName(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } - shortId, _, err := srv.ContainerCreate(config, "some_name") - if err != nil { - t.Fatal(err) - } - container := runtime.Get(shortId) + container := runtime.Get(createNamedTestContainer(eng, config, t, "some_name")) containerID := container.ID if container.Name != "/some_name" { @@ -682,20 +679,17 @@ func TestDefaultContainerName(t *testing.T) { } func TestRandomContainerName(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } - shortId, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } - container := runtime.Get(shortId) + container := runtime.Get(createTestContainer(eng, config, t)) containerID := container.ID if container.Name == "" { @@ -719,20 +713,17 @@ func TestRandomContainerName(t *testing.T) { } func TestLinkChildContainer(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } - shortId, _, err := srv.ContainerCreate(config, "/webapp") - if err != nil { - t.Fatal(err) - } - container := runtime.Get(shortId) + container := runtime.Get(createNamedTestContainer(eng, config, t, "/webapp")) webapp, err := runtime.GetByName("/webapp") if err != nil { @@ -748,12 +739,7 @@ func TestLinkChildContainer(t *testing.T) { t.Fatal(err) } - shortId, _, err = srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } - - childContainer := runtime.Get(shortId) + childContainer := runtime.Get(createTestContainer(eng, config, t)) if err := runtime.RegisterLink(webapp, childContainer, "db"); err != nil { t.Fatal(err) @@ -770,20 +756,17 @@ func TestLinkChildContainer(t *testing.T) { } func TestGetAllChildren(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } - shortId, _, err := srv.ContainerCreate(config, "/webapp") - if err != nil { - t.Fatal(err) - } - container := runtime.Get(shortId) + container := runtime.Get(createNamedTestContainer(eng, config, t, "/webapp")) webapp, err := runtime.GetByName("/webapp") if err != nil { @@ -799,12 +782,7 @@ func TestGetAllChildren(t *testing.T) { t.Fatal(err) } - shortId, _, err = srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } - - childContainer := runtime.Get(shortId) + childContainer := runtime.Get(createTestContainer(eng, config, t)) if err := runtime.RegisterLink(webapp, childContainer, "db"); err != nil { t.Fatal(err) diff --git a/server.go b/server.go index 0a1270bed..6196df898 100644 --- a/server.go +++ b/server.go @@ -62,6 +62,9 @@ func jobInitApi(job *engine.Job) string { os.Exit(0) }() job.Eng.Hack_SetGlobalVar("httpapi.server", srv) + if err := job.Eng.Register("create", srv.ContainerCreate); err != nil { + return err.Error() + } if err := job.Eng.Register("start", srv.ContainerStart); err != nil { return err.Error() } @@ -1009,33 +1012,43 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return nil } -func (srv *Server) ContainerCreate(config *Config, name string) (string, []string, error) { - if config.Memory != 0 && config.Memory < 524288 { - return "", nil, fmt.Errorf("Memory limit must be given in bytes (minimum 524288 bytes)") +func (srv *Server) ContainerCreate(job *engine.Job) string { + var name string + if len(job.Args) == 1 { + name = job.Args[0] + } else if len(job.Args) > 1 { + return fmt.Sprintf("Usage: %s ", job.Name) + } + var config Config + if err := job.ExportEnv(&config); err != nil { + return err.Error() + } + if config.Memory != 0 && config.Memory < 524288 { + return "Memory limit must be given in bytes (minimum 524288 bytes)" } - if config.Memory > 0 && !srv.runtime.capabilities.MemoryLimit { config.Memory = 0 } - if config.Memory > 0 && !srv.runtime.capabilities.SwapLimit { config.MemorySwap = -1 } - container, buildWarnings, err := srv.runtime.Create(config, name) + container, buildWarnings, err := srv.runtime.Create(&config, name) if err != nil { if srv.runtime.graph.IsNotExist(err) { - _, tag := utils.ParseRepositoryTag(config.Image) if tag == "" { tag = DEFAULTTAG } - - return "", nil, fmt.Errorf("No such image: %s (tag: %s)", config.Image, tag) + return fmt.Sprintf("No such image: %s (tag: %s)", config.Image, tag) } - return "", nil, err + return err.Error() } srv.LogEvent("create", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) - return container.ShortID(), buildWarnings, nil + job.Printf("%s\n", container.ShortID()) + for _, warning := range buildWarnings { + job.Errorf("%s\n", warning) + } + return "0" } func (srv *Server) ContainerRestart(name string, t int) error { diff --git a/server_test.go b/server_test.go index a4bfb0155..111043eca 100644 --- a/server_test.go +++ b/server_test.go @@ -79,20 +79,17 @@ func TestContainerTagImageDelete(t *testing.T) { } func TestCreateRm(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} - config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } - id, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + id := createTestContainer(eng, config, t) if len(runtime.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(runtime.List())) @@ -120,10 +117,7 @@ func TestCreateRmVolumes(t *testing.T) { t.Fatal(err) } - id, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + id := createTestContainer(eng, config, t) if len(runtime.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(runtime.List())) @@ -152,20 +146,17 @@ func TestCreateRmVolumes(t *testing.T) { } func TestCommit(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) - srv := &Server{runtime: runtime} - config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) } - id, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + id := createTestContainer(eng, config, t) if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { t.Fatal(err) @@ -183,10 +174,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { t.Fatal(err) } - id, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + id := createTestContainer(eng, config, t) if len(runtime.List()) != 1 { t.Errorf("Expected 1 container, %v found", len(runtime.List())) @@ -232,22 +220,22 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { } func TestRunWithTooLowMemoryLimit(t *testing.T) { - runtime := mkRuntime(t) + eng := NewTestEngine(t) + srv := mkServerFromEngine(eng, t) + runtime := srv.runtime defer nuke(runtime) // Try to create a container with a memory limit of 1 byte less than the minimum allowed limit. - if _, _, err := (*Server).ContainerCreate(&Server{runtime: runtime}, - &Config{ - Image: GetTestImage(runtime).ID, - Memory: 524287, - CpuShares: 1000, - Cmd: []string{"/bin/cat"}, - }, - "", - ); err == nil { + job := eng.Job("create") + job.Setenv("Image", GetTestImage(runtime).ID) + job.Setenv("Memory", "524287") + job.Setenv("CpuShares", "1000") + job.SetenvList("Cmd", []string{"/bin/cat"}) + var id string + job.StdoutParseString(&id) + if err := job.Run(); err == nil { t.Errorf("Memory limit is smaller than the allowed limit. Container creation should've failed!") } - } func TestContainerTop(t *testing.T) { @@ -411,10 +399,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - containerID, _, err := srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + containerID := createTestContainer(eng, config, t) //To remove job := eng.Job("start", containerID) @@ -435,10 +420,7 @@ func TestRmi(t *testing.T) { t.Fatal(err) } - containerID, _, err = srv.ContainerCreate(config, "") - if err != nil { - t.Fatal(err) - } + containerID = createTestContainer(eng, config, t) //To remove job = eng.Job("start", containerID) diff --git a/utils_test.go b/utils_test.go index 529af127e..2e8c0ceb1 100644 --- a/utils_test.go +++ b/utils_test.go @@ -38,6 +38,22 @@ func mkRuntime(f utils.Fataler) *Runtime { return r } +func createNamedTestContainer(eng *engine.Engine, config *Config, f utils.Fataler, name string) (shortId string) { + job := eng.Job("create", name) + if err := job.ImportEnv(config); err != nil { + f.Fatal(err) + } + job.StdoutParseString(&shortId) + if err := job.Run(); err != nil { + f.Fatal(err) + } + return +} + +func createTestContainer(eng *engine.Engine, config *Config, f utils.Fataler) (shortId string) { + return createNamedTestContainer(eng, config, f, "") +} + func mkServerFromEngine(eng *engine.Engine, t utils.Fataler) *Server { iSrv := eng.Hack_GetGlobalVar("httpapi.server") if iSrv == nil { From 8d6df3a7e2080d4fad9743beb159f12caa0ff6f7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Nov 2013 22:22:37 +0000 Subject: [PATCH 19/79] Remove debug messages --- engine/job.go | 13 ++----------- server.go | 3 --- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/engine/job.go b/engine/job.go index ece8d8f8d..419ac7346 100644 --- a/engine/job.go +++ b/engine/job.go @@ -239,17 +239,14 @@ func (job *Job) DecodeEnv(src io.Reader) error { // encoding/json decodes integers to float64, but cannot encode them back. // (See http://golang.org/src/pkg/encoding/json/decode.go#L46) if fval, ok := v.(float64); ok { - job.Logf("Converted to float: %v->%v", v, fval) job.SetenvInt(k, int64(fval)) } else if sval, ok := v.(string); ok { - job.Logf("Converted to string: %v->%v", v, sval) job.Setenv(k, sval) } else if val, err := json.Marshal(v); err == nil { job.Setenv(k, string(val)) } else { job.Setenv(k, fmt.Sprintf("%v", v)) } - job.Logf("Decoded %s=%#v to %s=%#v", k, v, k, job.Getenv(k)) } return nil } @@ -269,7 +266,6 @@ func (job *Job) EncodeEnv(dst io.Writer) error { } else { m[k] = v } - job.Logf("Encoded %s=%#v to %s=%#v", k, v, k, m[k]) } if err := json.NewEncoder(dst).Encode(&m); err != nil { return err @@ -278,24 +274,20 @@ func (job *Job) EncodeEnv(dst io.Writer) error { } func (job *Job) ExportEnv(dst interface{}) (err error) { - fmt.Fprintf(os.Stderr, "ExportEnv()\n") defer func() { if err != nil { err = fmt.Errorf("ExportEnv %s", err) } }() var buf bytes.Buffer - job.Logf("ExportEnv: step 1: encode/marshal the env to an intermediary json representation") - fmt.Fprintf(os.Stderr, "Printed ExportEnv step 1\n") + // step 1: encode/marshal the env to an intermediary json representation if err := job.EncodeEnv(&buf); err != nil { return err } - job.Logf("ExportEnv: step 1 complete: json=|%s|", buf) - job.Logf("ExportEnv: step 2: decode/unmarshal the intermediary json into the destination object") + // step 2: decode/unmarshal the intermediary json into the destination object if err := json.NewDecoder(&buf).Decode(dst); err != nil { return err } - job.Logf("ExportEnv: step 2 complete") return nil } @@ -309,7 +301,6 @@ func (job *Job) ImportEnv(src interface{}) (err error) { if err := json.NewEncoder(&buf).Encode(src); err != nil { return err } - job.Logf("ImportEnv: json=|%s|", buf) if err := job.DecodeEnv(&buf); err != nil { return err } diff --git a/server.go b/server.go index 6196df898..abcb001bb 100644 --- a/server.go +++ b/server.go @@ -1332,14 +1332,11 @@ func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { } func (srv *Server) ContainerStart(job *engine.Job) string { - job.Logf("srv engine = %s", srv.Eng.Root()) - job.Logf("job engine = %s", job.Eng.Root()) if len(job.Args) < 1 { return fmt.Sprintf("Usage: %s container_id", job.Name) } name := job.Args[0] runtime := srv.runtime - job.Logf("loading containers from %s", runtime.repository) container := runtime.Get(name) if container == nil { return fmt.Sprintf("No such container: %s", name) From 1dc34e2b965253a62d52f84a0f548334d4d6aa9d Mon Sep 17 00:00:00 2001 From: Daniel Norberg Date: Tue, 5 Nov 2013 18:26:07 -0500 Subject: [PATCH 20/79] lock around read operations in graph Writes and reads will fail with ErrBusy if there's concurrent reads or writes, respectively. It is not sufficient to only lock around writes. --- gograph/gograph.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/gograph/gograph.go b/gograph/gograph.go index 32b3a491b..2b2546434 100644 --- a/gograph/gograph.go +++ b/gograph/gograph.go @@ -138,7 +138,14 @@ func (db *Database) Set(fullPath, id string) (*Entity, error) { // Return true if a name already exists in the database func (db *Database) Exists(name string) bool { - return db.Get(name) != nil + db.mux.Lock() + defer db.mux.Unlock() + + e, err := db.get(name) + if err != nil { + return false + } + return e != nil } func (db *Database) setEdge(parentPath, name string, e *Entity) error { @@ -165,6 +172,9 @@ func (db *Database) RootEntity() *Entity { // Return the entity for a given path func (db *Database) Get(name string) *Entity { + db.mux.Lock() + defer db.mux.Unlock() + e, err := db.get(name) if err != nil { return nil @@ -200,6 +210,9 @@ func (db *Database) get(name string) (*Entity, error) { // List all entities by from the name // The key will be the full path of the entity func (db *Database) List(name string, depth int) Entities { + db.mux.Lock() + defer db.mux.Unlock() + out := Entities{} e, err := db.get(name) if err != nil { @@ -212,6 +225,9 @@ func (db *Database) List(name string, depth int) Entities { } func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { + db.mux.Lock() + defer db.mux.Unlock() + e, err := db.get(name) if err != nil { return err @@ -226,6 +242,9 @@ func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { // Return the refrence count for a specified id func (db *Database) Refs(id string) int { + db.mux.Lock() + defer db.mux.Unlock() + var count int if err := db.conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil { return 0 @@ -235,6 +254,9 @@ func (db *Database) Refs(id string) int { // Return all the id's path references func (db *Database) RefPaths(id string) Edges { + db.mux.Lock() + defer db.mux.Unlock() + refs := Edges{} rows, err := db.conn.Query("SELECT name, parent_id FROM edge WHERE entity_id = ?;", id) From 04aca7c9e3edddc57a1fbc11e57e6c62e6847126 Mon Sep 17 00:00:00 2001 From: Daniel Norberg Date: Tue, 5 Nov 2013 22:07:14 -0500 Subject: [PATCH 21/79] gograph: Use RWMutex to allow concurrent readers --- gograph/gograph.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gograph/gograph.go b/gograph/gograph.go index 2b2546434..626bf53ed 100644 --- a/gograph/gograph.go +++ b/gograph/gograph.go @@ -48,7 +48,7 @@ type WalkFunc func(fullPath string, entity *Entity) error // Graph database for storing entities and their relationships type Database struct { conn *sql.DB - mux sync.Mutex + mux sync.RWMutex } // Create a new graph database initialized with a root entity @@ -138,8 +138,8 @@ func (db *Database) Set(fullPath, id string) (*Entity, error) { // Return true if a name already exists in the database func (db *Database) Exists(name string) bool { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() e, err := db.get(name) if err != nil { @@ -172,8 +172,8 @@ func (db *Database) RootEntity() *Entity { // Return the entity for a given path func (db *Database) Get(name string) *Entity { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() e, err := db.get(name) if err != nil { @@ -210,8 +210,8 @@ func (db *Database) get(name string) (*Entity, error) { // List all entities by from the name // The key will be the full path of the entity func (db *Database) List(name string, depth int) Entities { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() out := Entities{} e, err := db.get(name) @@ -225,8 +225,8 @@ func (db *Database) List(name string, depth int) Entities { } func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() e, err := db.get(name) if err != nil { @@ -242,8 +242,8 @@ func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { // Return the refrence count for a specified id func (db *Database) Refs(id string) int { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() var count int if err := db.conn.QueryRow("SELECT COUNT(*) FROM edge WHERE entity_id = ?;", id).Scan(&count); err != nil { @@ -254,8 +254,8 @@ func (db *Database) Refs(id string) int { // Return all the id's path references func (db *Database) RefPaths(id string) Edges { - db.mux.Lock() - defer db.mux.Unlock() + db.mux.RLock() + defer db.mux.RUnlock() refs := Edges{} From 20881f1f787e093a6f3e9360624d8b436b9f7379 Mon Sep 17 00:00:00 2001 From: Daniel Norberg Date: Tue, 5 Nov 2013 22:15:41 -0500 Subject: [PATCH 22/79] gograph: allow Walk() reentrance Hold the read lock while reading the child graph, then walk over the children without any lock, in order to avoid deadlock. --- gograph/gograph.go | 101 ++++++++++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 43 deletions(-) diff --git a/gograph/gograph.go b/gograph/gograph.go index 626bf53ed..aa6a4126a 100644 --- a/gograph/gograph.go +++ b/gograph/gograph.go @@ -218,21 +218,28 @@ func (db *Database) List(name string, depth int) Entities { if err != nil { return out } - for c := range db.children(e, name, depth) { + + children, err := db.children(e, name, depth, nil) + if err != nil { + return out + } + + for _, c := range children { out[c.FullPath] = c.Entity } return out } +// Walk through the child graph of an entity, calling walkFunc for each child entity. +// It is safe for walkFunc to call graph functions. func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { - db.mux.RLock() - defer db.mux.RUnlock() - - e, err := db.get(name) + children, err := db.Children(name, depth) if err != nil { return err } - for c := range db.children(e, name, depth) { + + // Note: the database lock must not be held while calling walkFunc + for _, c := range children { if err := walkFunc(c.FullPath, c.Entity); err != nil { return err } @@ -240,6 +247,19 @@ func (db *Database) Walk(name string, walkFunc WalkFunc, depth int) error { return nil } +// Return the children of the specified entity +func (db *Database) Children(name string, depth int) ([]WalkMeta, error) { + db.mux.RLock() + defer db.mux.RUnlock() + + e, err := db.get(name) + if err != nil { + return nil, err + } + + return db.children(e, name, depth, nil) +} + // Return the refrence count for a specified id func (db *Database) Refs(id string) int { db.mux.RLock() @@ -378,56 +398,51 @@ type WalkMeta struct { Edge *Edge } -func (db *Database) children(e *Entity, name string, depth int) <-chan WalkMeta { - out := make(chan WalkMeta) +func (db *Database) children(e *Entity, name string, depth int, entities []WalkMeta) ([]WalkMeta, error) { if e == nil { - close(out) - return out + return entities, nil } - go func() { - rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id) - if err != nil { - close(out) + rows, err := db.conn.Query("SELECT entity_id, name FROM edge where parent_id = ?;", e.id) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var entityId, entityName string + if err := rows.Scan(&entityId, &entityName); err != nil { + return nil, err + } + child := &Entity{entityId} + edge := &Edge{ + ParentID: e.id, + Name: entityName, + EntityID: child.id, } - defer rows.Close() - for rows.Next() { - var entityId, entityName string - if err := rows.Scan(&entityId, &entityName); err != nil { - // Log error - continue - } - child := &Entity{entityId} - edge := &Edge{ - ParentID: e.id, - Name: entityName, - EntityID: child.id, - } + meta := WalkMeta{ + Parent: e, + Entity: child, + FullPath: path.Join(name, edge.Name), + Edge: edge, + } - meta := WalkMeta{ - Parent: e, - Entity: child, - FullPath: path.Join(name, edge.Name), - Edge: edge, - } + entities = append(entities, meta) - out <- meta - if depth == 0 { - continue - } + if depth != 0 { nDepth := depth if depth != -1 { nDepth -= 1 } - sc := db.children(child, meta.FullPath, nDepth) - for c := range sc { - out <- c + entities, err = db.children(child, meta.FullPath, nDepth, entities) + if err != nil { + return nil, err } } - close(out) - }() - return out + } + + return entities, nil } // Return the entity based on the parent path and name From 70f44d5531a5c6dcb96b7e608a96beb52e93b506 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 6 Nov 2013 10:00:24 -0800 Subject: [PATCH 23/79] Update documentation to reflect changes in Config and HostConfig --- docs/sources/api/docker_remote_api_v1.6.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/sources/api/docker_remote_api_v1.6.rst b/docs/sources/api/docker_remote_api_v1.6.rst index defe145d0..09da63d8d 100644 --- a/docs/sources/api/docker_remote_api_v1.6.rst +++ b/docs/sources/api/docker_remote_api_v1.6.rst @@ -121,8 +121,7 @@ Create a container "AttachStdin":false, "AttachStdout":true, "AttachStderr":true, - "PortSpecs":null, - "Privileged": false, + "ExposedPorts":{}, "Tty":false, "OpenStdin":false, "StdinOnce":false, @@ -135,7 +134,6 @@ Create a container "Volumes":{}, "VolumesFrom":"", "WorkingDir":"" - } **Example response**: @@ -191,7 +189,7 @@ Inspect a container "AttachStdin": false, "AttachStdout": true, "AttachStderr": true, - "PortSpecs": null, + "ExposedPorts": {}, "Tty": false, "OpenStdin": false, "StdinOnce": false, @@ -362,7 +360,12 @@ Start a container { "Binds":["/tmp:/tmp"], - "LxcConf":{"lxc.utsname":"docker"} + "LxcConf":{"lxc.utsname":"docker"}, + "ContainerIDFile": "", + "Privileged": false, + "PortBindings": {"22/tcp": [{HostIp:"", HostPort:""}]}, + "Links": [], + "PublishAllPorts": false } **Example response**: @@ -795,7 +798,7 @@ Inspect an image "AttachStdin":false, "AttachStdout":false, "AttachStderr":false, - "PortSpecs":null, + "ExposedPorts":{}, "Tty":true, "OpenStdin":true, "StdinOnce":false, @@ -1141,7 +1144,7 @@ Create a new image from a container's changes { "Cmd": ["cat", "/world"], - "PortSpecs":["22"] + "ExposedPorts":{"22/tcp":{}} } **Example response**: From c5bc7d515836cffee90ba5bb342272bba64d9f37 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 12 Sep 2013 15:17:39 +0200 Subject: [PATCH 24/79] Utils: Add ShellQuoteArguments --- utils/utils.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 4941eae42..623f00296 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1139,6 +1139,41 @@ func (e *StatusError) Error() string { return fmt.Sprintf("Status: %d", e.Status) } +func quote(word string, buf *bytes.Buffer) { + // Bail out early for "simple" strings + if word != "" && !strings.ContainsAny(word, "\\'\"`${[|&;<>()~*?! \t\n") { + buf.WriteString(word) + return + } + + buf.WriteString("'") + + for i := 0; i < len(word); i++ { + b := word[i] + if b == '\'' { + // Replace literal ' with a close ', a \', and a open ' + buf.WriteString("'\\''") + } else { + buf.WriteByte(b) + } + } + + buf.WriteString("'") +} + +// Take a list of strings and escape them so they will be handled right +// when passed as arguments to an program via a shell +func ShellQuoteArguments(args []string) string { + var buf bytes.Buffer + for i, arg := range args { + if i != 0 { + buf.WriteByte(' ') + } + quote(arg, &buf) + } + return buf.String() +} + func IsClosedError(err error) bool { /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. * See: From 157d99a72786c454dfaad8b0800914cc80879aa8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Oct 2013 16:16:51 +0100 Subject: [PATCH 25/79] lxc: Work around lxc-start need for private mounts lxc-start requires / to be mounted private, otherwise the changes it does inside the container (both mounts and unmounts) will propagate out to the host. We work around this by starting up lxc-start in its own namespace where we set / to rshared. Unfortunately go can't really execute any code between clone and exec, so we can't do this in a nice way. Instead we have a horrible hack that use the unshare command, the shell and the mount command... --- container.go | 27 +++++++++++++++++++++++---- utils.go | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/container.go b/container.go index 50bf2ec67..ac21119d1 100644 --- a/container.go +++ b/container.go @@ -863,7 +863,13 @@ func (container *Container) Start() (err error) { return err } + var lxcStart string = "lxc-start" + if container.hostConfig.Privileged && container.runtime.capabilities.AppArmor { + lxcStart = path.Join(container.runtime.config.Root, "lxc-start-unconfined") + } + params := []string{ + lxcStart, "-n", container.ID, "-f", container.lxcConfigPath(), "--", @@ -956,11 +962,24 @@ func (container *Container) Start() (err error) { params = append(params, "--", container.Path) params = append(params, container.Args...) - var lxcStart string = "lxc-start" - if container.hostConfig.Privileged && container.runtime.capabilities.AppArmor { - lxcStart = path.Join(container.runtime.config.Root, "lxc-start-unconfined") + if RootIsShared() { + // lxc-start really needs / to be non-shared, or all kinds of stuff break + // when lxc-start unmount things and those unmounts propagate to the main + // mount namespace. + // What we really want is to clone into a new namespace and then + // mount / MS_REC|MS_SLAVE, but since we can't really clone or fork + // without exec in go we have to do this horrible shell hack... + shellString := + "mount --make-rslave /; exec " + + utils.ShellQuoteArguments(params) + + params = []string{ + "unshare", "-m", "--", "/bin/sh", "-c", shellString, + } } - container.cmd = exec.Command(lxcStart, params...) + + container.cmd = exec.Command(params[0], params[1:]...) + // Setup logging of stdout and stderr to disk if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil { return err diff --git a/utils.go b/utils.go index 81715881a..22d83d6be 100644 --- a/utils.go +++ b/utils.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/dotcloud/docker/namesgenerator" "github.com/dotcloud/docker/utils" + "io/ioutil" "strconv" "strings" ) @@ -301,6 +302,20 @@ func parseLink(rawLink string) (map[string]string, error) { return utils.PartParser("name:alias", rawLink) } +func RootIsShared() bool { + if data, err := ioutil.ReadFile("/proc/self/mountinfo"); err == nil { + for _, line := range strings.Split(string(data), "\n") { + cols := strings.Split(line, " ") + if len(cols) >= 6 && cols[4] == "/" { + return strings.HasPrefix(cols[6], "shared") + } + } + } + + // No idea, probably safe to assume so + return true +} + type checker struct { runtime *Runtime } From 50dd9791f75efef77a85bbe6556c85de0d845c0a Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 18 Oct 2013 09:27:34 -0700 Subject: [PATCH 26/79] testing infrastructure, issue #1800: Refactor docker testing using Docker in Docker --- hack/infrastructure/docker-ci/deployment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/infrastructure/docker-ci/deployment.py b/hack/infrastructure/docker-ci/deployment.py index 655bdd43b..453fad759 100755 --- a/hack/infrastructure/docker-ci/deployment.py +++ b/hack/infrastructure/docker-ci/deployment.py @@ -138,6 +138,9 @@ sudo('curl -s https://phantomjs.googlecode.com/files/' # Preventively reboot docker-ci daily sudo('ln -s /sbin/reboot /etc/cron.daily') +# Preventively reboot docker-ci daily +sudo('ln -s /sbin/reboot /etc/cron.daily') + # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( From 17172276366fc3114ae52eb6c83aafbec004d5f3 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Sun, 27 Oct 2013 12:13:03 -0700 Subject: [PATCH 27/79] docker-ci 0.35. Push docker coverage and testing into docker-ci production. Update docker nightlyrelease --- hack/infrastructure/docker-ci/Dockerfile | 6 ++- .../docker-ci/buildbot/master.cfg | 27 ++++++---- hack/infrastructure/docker-ci/deployment.py | 6 ++- .../docker-ci/docker-test/test_docker.sh | 15 ++---- .../functionaltests/test_registry.sh | 1 + .../docker-ci/nightlyrelease/Dockerfile | 18 +++---- .../docker-ci/nightlyrelease/dockerbuild | 50 ------------------- .../docker-ci/nightlyrelease/dockerbuild.sh | 40 +++++++++++++++ .../nightlyrelease/release_credentials.json | 1 - .../docker-ci/registry-coverage/Dockerfile | 18 +++++++ .../registry-coverage/registry_coverage.sh | 18 +++++++ 11 files changed, 113 insertions(+), 87 deletions(-) delete mode 100644 hack/infrastructure/docker-ci/nightlyrelease/dockerbuild create mode 100644 hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh delete mode 100644 hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json create mode 100644 hack/infrastructure/docker-ci/registry-coverage/Dockerfile create mode 100755 hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh diff --git a/hack/infrastructure/docker-ci/Dockerfile b/hack/infrastructure/docker-ci/Dockerfile index 3ac8d90d2..bb49944d1 100644 --- a/hack/infrastructure/docker-ci/Dockerfile +++ b/hack/infrastructure/docker-ci/Dockerfile @@ -33,8 +33,10 @@ from ubuntu:12.04 -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \ + > /etc/apt/sources.list +run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \ + python-pip ssh rsync less vim run pip install boto fabric # Add deployment code and set default container command diff --git a/hack/infrastructure/docker-ci/buildbot/master.cfg b/hack/infrastructure/docker-ci/buildbot/master.cfg index 52bf495df..9ca5fc035 100644 --- a/hack/infrastructure/docker-ci/buildbot/master.cfg +++ b/hack/infrastructure/docker-ci/buildbot/master.cfg @@ -43,7 +43,7 @@ c['slavePortnum'] = PORT_MASTER # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker', - 'index','registry','coverage','nightlyrelease'])] + 'index','registry','docker-coverage','registry-coverage','nightlyrelease'])] c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None, change_filter=filter.ChangeFilter(branch='master', repository='https://github.com/dotcloud/docker'), builderNames=['docker'])] @@ -51,7 +51,7 @@ c['schedulers'] += [SingleBranchScheduler(name='pullrequest', change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, builderNames=['pullrequest'])] c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease', - 'coverage'], hour=7, minute=00)] + 'docker-coverage','registry-coverage'], hour=7, minute=00)] c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'], hour=range(0,24,4), minute=15)] @@ -76,17 +76,25 @@ c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], # Docker coverage test factory = BuildFactory() -factory.addStep(ShellCommand(description='Coverage', logEnviron=False, +factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False, usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format( DOCKER_CI_PATH))) -c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], +c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'], + factory=factory)] + +# Docker registry coverage test +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False, + usePTY=True, command='docker run registry_coverage'.format( + DOCKER_CI_PATH))) +c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'], factory=factory)] # Registry functional test factory = BuildFactory() factory.addStep(ShellCommand(description='registry', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory=factory)] @@ -95,16 +103,17 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory = BuildFactory() factory.addStep(ShellCommand(description='index', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'], factory=factory)] # Docker nightly release +nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET=' + 'test.docker.io dockerbuilder hack/dind dockerbuild.sh') factory = BuildFactory() -factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False, - usePTY=True, command='docker run -privileged' - ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')) +factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False, + usePTY=True, command=nightlyrelease_cmd)) c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'], factory=factory)] diff --git a/hack/infrastructure/docker-ci/deployment.py b/hack/infrastructure/docker-ci/deployment.py index 453fad759..1a389d277 100755 --- a/hack/infrastructure/docker-ci/deployment.py +++ b/hack/infrastructure/docker-ci/deployment.py @@ -100,8 +100,7 @@ sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) credentials = { 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], - 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'], - 'INDEX_AUTH': env['INDEX_AUTH']} + 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']} open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( base64.b64encode(json.dumps(credentials))) @@ -143,8 +142,11 @@ sudo('ln -s /sbin/reboot /etc/cron.daily') # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) +sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( DOCKER_CI_PATH)) +sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format( + DOCKER_CI_PATH)) # Download docker-ci testing container sudo('docker pull mzdaniel/test_docker') diff --git a/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/hack/infrastructure/docker-ci/docker-test/test_docker.sh index 895e4d964..c8cfe147e 100755 --- a/hack/infrastructure/docker-ci/docker-test/test_docker.sh +++ b/hack/infrastructure/docker-ci/docker-test/test_docker.sh @@ -9,30 +9,21 @@ BRANCH=${3-master} DOCKER_PATH=/go/src/github.com/dotcloud/docker # Fetch latest master +cd / rm -rf /go -mkdir -p $DOCKER_PATH +git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH cd $DOCKER_PATH -git init . -git fetch -q http://github.com/dotcloud/docker master -git reset --hard FETCH_HEAD # Merge commit -#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS -git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1 - -# Merge commit in top of master git fetch -q "$REPO" "$BRANCH" git merge --no-edit $COMMIT || exit 1 # Test commit -go test -v; exit_status=$? +./hack/make.sh test; exit_status=$? # Display load if test fails if [ $exit_status -eq 1 ] ; then uptime; echo; free fi -# Cleanup testing directory -rm -rf $BASE_PATH - exit $exit_status diff --git a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh index 8bcd355c7..d175f66d1 100755 --- a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh +++ b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh @@ -12,6 +12,7 @@ export DOCKER_REGISTRY_CONFIG=config_test.yml # Get latest docker registry git clone -q https://github.com/dotcloud/docker-registry.git cd docker-registry +sed -Ei "s#(boto_bucket: ).+#\1_env:S3_BUCKET#" config_test.yml # Get dependencies pip install -q -r requirements.txt diff --git a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile index 541f3a958..953d7c11c 100644 --- a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -1,5 +1,5 @@ -# VERSION: 1.2 -# DOCKER-VERSION 0.6.3 +# VERSION: 1.5 +# DOCKER-VERSION 0.6.4 # AUTHOR: Daniel Mizyrycki # DESCRIPTION: Build docker nightly release using Docker in Docker. # REFERENCES: This code reuses the excellent implementation of docker in docker @@ -7,11 +7,10 @@ # COMMENTS: # release_credentials.json is a base64 json encoded file containing: # { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id", -# "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key' -# "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature' -# "INDEX_AUTH='Encripted_index_authentication' } +# "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key", +# "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" } # TO_BUILD: docker build -t dockerbuilder . -# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder +# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh from docker maintainer Daniel Mizyrycki @@ -24,11 +23,8 @@ run apt-get update; apt-get install -y -q wget python2.7 run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker # Add proto docker builder -add ./dockerbuild /usr/bin/dockerbuild -run chmod +x /usr/bin/dockerbuild +add ./dockerbuild.sh /usr/bin/dockerbuild.sh +run chmod +x /usr/bin/dockerbuild.sh # Add release credentials add ./release_credentials.json /root/release_credentials.json - -# Launch build process in a container -cmd dockerbuild diff --git a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild deleted file mode 100644 index 83a7157a3..000000000 --- a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH -# are decoded from /root/release_credentials.json -# Variable AWS_S3_BUCKET is passed to the environment from docker run -e - -# Enable debugging -set -x - -# Fetch docker master branch -rm -rf /go/src/github.com/dotcloud/docker -cd / -git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker -cd /go/src/github.com/dotcloud/docker - -# Launch docker daemon using dind inside the container -./hack/dind /usr/bin/docker -d & -sleep 5 - -# Add an uncommitted change to generate a timestamped release -date > timestamp - -# Build the docker package using /Dockerfile -docker build -t docker . - -# Run Docker unittests binary and Ubuntu package -docker run -privileged docker hack/make.sh -exit_status=$? - -# Display load if test fails -if [ $exit_status -eq 1 ] ; then - uptime; echo; free - exit 1 -fi - -# Commit binary and ubuntu bundles for release -docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release - -# Turn debug off to load credentials from the environment -set +x -eval $(cat /root/release_credentials.json | python -c ' -import sys,json,base64; -d=json.loads(base64.b64decode(sys.stdin.read())); -exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') -set -x - -# Push docker nightly -echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh -set +x -docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE release hack/release.sh diff --git a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh new file mode 100644 index 000000000..457db3f88 --- /dev/null +++ b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded +# from /root/release_credentials.json +# Variable AWS_S3_BUCKET is passed to the environment from docker run -e + +# Turn debug off to load credentials from the environment +set +x +eval $(cat /root/release_credentials.json | python -c ' +import sys,json,base64; +d=json.loads(base64.b64decode(sys.stdin.read())); +exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') + +# Fetch docker master branch +set -x +cd / +rm -rf /go +git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker +cd /go/src/github.com/dotcloud/docker + +# Launch docker daemon using dind inside the container +/usr/bin/docker version +/usr/bin/docker -d & +sleep 5 + +# Build Docker release container +docker build -t docker . + +# Test docker and if everything works well, release +echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh +set +x +docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh +exit_status=$? + +# Display load if test fails +set -x +if [ $exit_status -eq 1 ] ; then + uptime; echo; free + exit 1 +fi diff --git a/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json b/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json deleted file mode 100644 index ed6d53ecd..000000000 --- a/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json +++ /dev/null @@ -1 +0,0 @@ -eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0= diff --git a/hack/infrastructure/docker-ci/registry-coverage/Dockerfile b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile new file mode 100644 index 000000000..59c914fb2 --- /dev/null +++ b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile @@ -0,0 +1,18 @@ +# VERSION: 0.1 +# DOCKER-VERSION 0.6.4 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Docker registry coverage +# COMMENTS: Add registry coverage into the docker-ci image +# TO_BUILD: docker build -t registry_coverage . +# TO_RUN: docker run registry_coverage + +from docker-ci +maintainer Daniel Mizyrycki + +# Add registry_coverager.sh and dependencies +run pip install coverage flask pyyaml requests simplejson python-glanceclient \ + blinker redis gevent +add registry_coverage.sh /usr/bin/registry_coverage.sh +run chmod +x /usr/bin/registry_coverage.sh + +cmd "/usr/bin/registry_coverage.sh" diff --git a/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh new file mode 100755 index 000000000..e9f017265 --- /dev/null +++ b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -x + +# Compute test paths +REGISTRY_PATH=/data/docker-registry + +# Fetch latest docker-registry master +rm -rf $REGISTRY_PATH +git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH +cd $REGISTRY_PATH + +# Generate coverage +export SETTINGS_FLAVOR=test +export DOCKER_REGISTRY_CONFIG=config_test.yml + +coverage run -m unittest discover test || exit 1 +coverage report --include='./*' --omit='./test/*' From 0cbeda73910d6d1d3336da93c5f94df0bdd5fa44 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 28 Oct 2013 16:51:31 -0700 Subject: [PATCH 28/79] docker-ci 0.36. Patch hack/dind with latest code for nightly release to work. --- hack/infrastructure/docker-ci/nightlyrelease/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile index 953d7c11c..6762cb468 100644 --- a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -22,6 +22,11 @@ run apt-get update; apt-get install -y -q wget python2.7 # Add production docker binary run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker +#### FIXME. Temporarily install docker and dind with proper apparmor handling +run wget -q -O /usr/bin/docker http://test.docker.io/test/docker; chmod +x /usr/bin/docker +run wget -q -O /go/src/github.com/dotcloud/docker/hack/dind http://raw.github.com/jpetazzo/docker/escape-apparmor-confinement/hack/dind +run chmod +x /go/src/github.com/dotcloud/docker/hack/dind + # Add proto docker builder add ./dockerbuild.sh /usr/bin/dockerbuild.sh run chmod +x /usr/bin/dockerbuild.sh From efb4c800a7c8167cf2202466a720c0fad7001a5d Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Tue, 29 Oct 2013 19:37:56 -0700 Subject: [PATCH 29/79] docker-ci 0.37. Patch hack/dind with latest code for docker-test. --- .../infrastructure/docker-ci/docker-test/Dockerfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/hack/infrastructure/docker-ci/docker-test/Dockerfile b/hack/infrastructure/docker-ci/docker-test/Dockerfile index 66cb9762e..229d696b8 100644 --- a/hack/infrastructure/docker-ci/docker-test/Dockerfile +++ b/hack/infrastructure/docker-ci/docker-test/Dockerfile @@ -17,13 +17,12 @@ from docker maintainer Daniel Mizyrycki -# Setup go environment. Extracted from /Dockerfile -env CGO_ENABLED 0 -env GOROOT /goroot -env PATH $PATH:/goroot/bin -env GOPATH /go:/go/src/github.com/dotcloud/docker/vendor -volume /var/lib/docker -workdir /go/src/github.com/dotcloud/docker +#### FIXME. Temporarily install docker and dind with proper apparmor handling +run wget -q -O /go/src/github.com/dotcloud/docker/hack/dind http://raw.github.com/jpetazzo/docker/escape-apparmor-confinement/hack/dind +run chmod +x /go/src/github.com/dotcloud/docker/hack/dind + +# Setup go to the PATH. Extracted from /Dockerfile +env PATH /usr/local/go/bin:$PATH # Add test_docker.sh add test_docker.sh /usr/bin/test_docker.sh From e7df38dbd0c7bf2d27632809fc49cacb0cc7f869 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Tue, 29 Oct 2013 22:14:28 -0700 Subject: [PATCH 30/79] docker-ci 0.40. Migrate docker-ci to Digital Ocean. --- hack/infrastructure/docker-ci/Dockerfile | 12 ++- hack/infrastructure/docker-ci/deployment.py | 112 +++++++++++--------- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/hack/infrastructure/docker-ci/Dockerfile b/hack/infrastructure/docker-ci/Dockerfile index bb49944d1..3f6c34441 100644 --- a/hack/infrastructure/docker-ci/Dockerfile +++ b/hack/infrastructure/docker-ci/Dockerfile @@ -1,14 +1,16 @@ # VERSION: 0.22 # DOCKER-VERSION 0.6.3 # AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Deploy docker-ci on Amazon EC2 +# DESCRIPTION: Deploy docker-ci on Digital Ocean # COMMENTS: # CONFIG_JSON is an environment variable json string loaded as: # # export CONFIG_JSON=' -# { "AWS_TAG": "EC2_instance_name", -# "AWS_ACCESS_KEY": "EC2_access_key", -# "AWS_SECRET_KEY": "EC2_secret_key", +# { "DROPLET_NAME": "docker-ci", +# "DO_CLIENT_ID": "Digital_Ocean_client_id", +# "DO_API_KEY": "Digital_Ocean_api_key", +# "DOCKER_KEY_ID": "Digital_Ocean_ssh_key_id", +# "DOCKER_CI_KEY_PATH": "docker-ci_private_key_path", # "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", # "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)", # "BUILDBOT_PWD": "Buildbot_server_password", @@ -37,7 +39,7 @@ run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \ > /etc/apt/sources.list run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \ python-pip ssh rsync less vim -run pip install boto fabric +run pip install requests fabric # Add deployment code and set default container command add . /docker-ci diff --git a/hack/infrastructure/docker-ci/deployment.py b/hack/infrastructure/docker-ci/deployment.py index 1a389d277..ee000eb7b 100755 --- a/hack/infrastructure/docker-ci/deployment.py +++ b/hack/infrastructure/docker-ci/deployment.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -import os, sys, re, json, base64 -from boto.ec2.connection import EC2Connection +import os, sys, re, json, requests, base64 from subprocess import call from fabric import api from fabric.api import cd, run, put, sudo from os import environ as env +from datetime import datetime from time import sleep # Remove SSH private key as it needs more processing @@ -20,42 +20,41 @@ for key in CONFIG: env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', env['CONFIG_JSON'],flags=re.DOTALL) - -AWS_TAG = env.get('AWS_TAG','docker-ci') -AWS_KEY_NAME = 'dotcloud-dev' # Same as CONFIG_JSON['DOCKER_CI_PUB'] -AWS_AMI = 'ami-d582d6bc' # Ubuntu 13.04 -AWS_REGION = 'us-east-1' -AWS_TYPE = 'm1.small' -AWS_SEC_GROUPS = 'gateway' -AWS_IMAGE_USER = 'ubuntu' +DROPLET_NAME = env.get('DROPLET_NAME','docker-ci') +TIMEOUT = 120 # Seconds before timeout droplet creation +IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 +REGION_ID = 4 # New York 2 +SIZE_ID = 62 # memory 2GB +DO_IMAGE_USER = 'root' # Image user on Digital Ocean +API_URL = 'https://api.digitalocean.com/' DOCKER_PATH = '/go/src/github.com/dotcloud/docker' DOCKER_CI_PATH = '/docker-ci' CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) -class AWS_EC2: - '''Amazon EC2''' - def __init__(self, access_key, secret_key): +class digital_ocean(): + + def __init__(self, key, client): '''Set default API parameters''' - self.handler = EC2Connection(access_key, secret_key) - def create_instance(self, tag, instance_type): - reservation = self.handler.run_instances(**instance_type) - instance = reservation.instances[0] - sleep(10) - while instance.state != 'running': - sleep(5) - instance.update() - print "Instance state: %s" % (instance.state) - instance.add_tag("Name",tag) - print "instance %s done!" % (instance.id) - return instance.ip_address - def get_instances(self): - return self.handler.get_all_instances() - def get_tags(self): - return dict([(i.instances[0].id, i.instances[0].tags['Name']) - for i in self.handler.get_all_instances() if i.instances[0].tags]) - def del_instance(self, instance_id): - self.handler.terminate_instances(instance_ids=[instance_id]) + self.key = key + self.client = client + self.api_url = API_URL + + def api(self, cmd_path, api_arg={}): + '''Make api call''' + api_arg.update({'api_key':self.key, 'client_id':self.client}) + resp = requests.get(self.api_url + cmd_path, params=api_arg).text + resp = json.loads(resp) + if resp['status'] != 'OK': + raise Exception(resp['error_message']) + return resp + + def droplet_data(self, name): + '''Get droplet data''' + data = self.api('droplets') + data = [droplet for droplet in data['droplets'] + if droplet['name'] == name] + return data[0] if data else {} def json_fmt(data): @@ -63,20 +62,36 @@ def json_fmt(data): return json.dumps(data, sort_keys = True, indent = 2) -# Create EC2 API handler -ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY']) +do = digital_ocean(env['DO_API_KEY'], env['DO_CLIENT_ID']) -# Stop processing if AWS_TAG exists on EC2 -if AWS_TAG in ec2.get_tags().values(): - print ('Instance: {} already deployed. Not further processing.' - .format(AWS_TAG)) +# Get DROPLET_NAME data +data = do.droplet_data(DROPLET_NAME) + +# Stop processing if DROPLET_NAME exists on Digital Ocean +if data: + print ('Droplet: {} already deployed. Not further processing.' + .format(DROPLET_NAME)) exit(1) -ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE, - 'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME}) +# Create droplet +do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID, + 'image_id':IMAGE_ID, 'size_id':SIZE_ID, + 'ssh_key_ids':[env['DOCKER_KEY_ID']]}) -# Wait 30 seconds for the machine to boot -sleep(30) +# Wait for droplet to be created. +start_time = datetime.now() +while (data.get('status','') != 'active' and ( + datetime.now()-start_time).seconds < TIMEOUT): + data = do.droplet_data(DROPLET_NAME) + print data['status'] + sleep(3) + +# Wait for the machine to boot +sleep(15) + +# Get droplet IP +ip = str(data['ip_address']) +print 'droplet: {} ip: {}'.format(DROPLET_NAME, ip) # Create docker-ci ssh private key so docker-ci docker container can communicate # with its EC2 instance @@ -86,7 +101,7 @@ os.chmod('/root/.ssh/id_rsa',0600) open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') api.env.host_string = ip -api.env.user = AWS_IMAGE_USER +api.env.user = DO_IMAGE_USER api.env.key_filename = '/root/.ssh/id_rsa' # Correct timezone @@ -106,13 +121,11 @@ open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( # Transfer docker sudo('mkdir -p ' + DOCKER_CI_PATH) -sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH)) -call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip, +sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH)) +call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip, os.path.dirname(DOCKER_CI_PATH)), shell=True) # Install Docker and Buildbot dependencies -sudo('addgroup docker') -sudo('usermod -a -G docker ubuntu') sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker') sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -') sudo('echo deb https://get.docker.io/ubuntu docker main >' @@ -122,7 +135,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' ' > /etc/apt/sources.list; apt-get update') sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' - ' aufs-tools make libfontconfig libevent-dev') + ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev') sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | ' 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') sudo('GOPATH=/go go get -d github.com/dotcloud/docker') @@ -137,9 +150,6 @@ sudo('curl -s https://phantomjs.googlecode.com/files/' # Preventively reboot docker-ci daily sudo('ln -s /sbin/reboot /etc/cron.daily') -# Preventively reboot docker-ci daily -sudo('ln -s /sbin/reboot /etc/cron.daily') - # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) From 249f76bebdb3c24ae0a59e3be7ae73fd1d88c065 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Wed, 6 Nov 2013 20:05:26 -0800 Subject: [PATCH 31/79] docker-ci 0.4.5: Sync tests with progress in docker and docker-registry. Use revamped shiny DinD. --- .gitignore | 1 + hack/infrastructure/docker-ci/Dockerfile | 6 +++--- hack/infrastructure/docker-ci/VERSION | 1 + hack/infrastructure/docker-ci/deployment.py | 10 +++++----- .../docker-ci/docker-test/Dockerfile | 14 +++++--------- .../docker-ci/docker-test/test_docker.sh | 8 ++++++-- .../docker-ci/functionaltests/test_registry.sh | 2 +- .../docker-ci/nightlyrelease/Dockerfile | 13 ++++--------- .../docker-ci/nightlyrelease/dockerbuild.sh | 2 +- .../docker-ci/registry-coverage/Dockerfile | 2 +- .../registry-coverage/registry_coverage.sh | 8 ++++---- hack/infrastructure/docker-ci/report/deployment.py | 2 +- 12 files changed, 33 insertions(+), 36 deletions(-) create mode 100644 hack/infrastructure/docker-ci/VERSION diff --git a/.gitignore b/.gitignore index 8cf66168e..00d66de3e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ bundles/ .hg/ .git/ vendor/pkg/ +pyenv diff --git a/hack/infrastructure/docker-ci/Dockerfile b/hack/infrastructure/docker-ci/Dockerfile index 3f6c34441..d894330ff 100644 --- a/hack/infrastructure/docker-ci/Dockerfile +++ b/hack/infrastructure/docker-ci/Dockerfile @@ -1,6 +1,6 @@ -# VERSION: 0.22 -# DOCKER-VERSION 0.6.3 -# AUTHOR: Daniel Mizyrycki +# VERSION: 0.25 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki # DESCRIPTION: Deploy docker-ci on Digital Ocean # COMMENTS: # CONFIG_JSON is an environment variable json string loaded as: diff --git a/hack/infrastructure/docker-ci/VERSION b/hack/infrastructure/docker-ci/VERSION new file mode 100644 index 000000000..0bfccb080 --- /dev/null +++ b/hack/infrastructure/docker-ci/VERSION @@ -0,0 +1 @@ +0.4.5 diff --git a/hack/infrastructure/docker-ci/deployment.py b/hack/infrastructure/docker-ci/deployment.py index ee000eb7b..c04219d52 100755 --- a/hack/infrastructure/docker-ci/deployment.py +++ b/hack/infrastructure/docker-ci/deployment.py @@ -32,7 +32,7 @@ DOCKER_CI_PATH = '/docker-ci' CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) -class digital_ocean(): +class DigitalOcean(): def __init__(self, key, client): '''Set default API parameters''' @@ -62,7 +62,7 @@ def json_fmt(data): return json.dumps(data, sort_keys = True, indent = 2) -do = digital_ocean(env['DO_API_KEY'], env['DO_CLIENT_ID']) +do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID']) # Get DROPLET_NAME data data = do.droplet_data(DROPLET_NAME) @@ -147,9 +147,6 @@ sudo('curl -s https://phantomjs.googlecode.com/files/' 'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin' ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs') -# Preventively reboot docker-ci daily -sudo('ln -s /sbin/reboot /etc/cron.daily') - # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) @@ -169,3 +166,6 @@ sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}' env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'], env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'], env['REGISTRY_SECRET_KEY'])) + +# Preventively reboot docker-ci daily +sudo('ln -s /sbin/reboot /etc/cron.daily') diff --git a/hack/infrastructure/docker-ci/docker-test/Dockerfile b/hack/infrastructure/docker-ci/docker-test/Dockerfile index 229d696b8..0f3a63f5f 100644 --- a/hack/infrastructure/docker-ci/docker-test/Dockerfile +++ b/hack/infrastructure/docker-ci/docker-test/Dockerfile @@ -1,6 +1,6 @@ -# VERSION: 0.3 -# DOCKER-VERSION 0.6.3 -# AUTHOR: Daniel Mizyrycki +# VERSION: 0.4 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki # DESCRIPTION: Testing docker PRs and commits on top of master using # REFERENCES: This code reuses the excellent implementation of # Docker in Docker made by Jerome Petazzoni. @@ -15,13 +15,9 @@ # TO_RUN: docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch] from docker -maintainer Daniel Mizyrycki +maintainer Daniel Mizyrycki -#### FIXME. Temporarily install docker and dind with proper apparmor handling -run wget -q -O /go/src/github.com/dotcloud/docker/hack/dind http://raw.github.com/jpetazzo/docker/escape-apparmor-confinement/hack/dind -run chmod +x /go/src/github.com/dotcloud/docker/hack/dind - -# Setup go to the PATH. Extracted from /Dockerfile +# Setup go in PATH. Extracted from /Dockerfile env PATH /usr/local/go/bin:$PATH # Add test_docker.sh diff --git a/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/hack/infrastructure/docker-ci/docker-test/test_docker.sh index c8cfe147e..cf8fdb90b 100755 --- a/hack/infrastructure/docker-ci/docker-test/test_docker.sh +++ b/hack/infrastructure/docker-ci/docker-test/test_docker.sh @@ -8,6 +8,10 @@ BRANCH=${3-master} # Compute test paths DOCKER_PATH=/go/src/github.com/dotcloud/docker +# Timestamp +echo +date; echo + # Fetch latest master cd / rm -rf /go @@ -16,13 +20,13 @@ cd $DOCKER_PATH # Merge commit git fetch -q "$REPO" "$BRANCH" -git merge --no-edit $COMMIT || exit 1 +git merge --no-edit $COMMIT || exit 255 # Test commit ./hack/make.sh test; exit_status=$? # Display load if test fails -if [ $exit_status -eq 1 ] ; then +if [ $exit_status -ne 0 ] ; then uptime; echo; free fi diff --git a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh index d175f66d1..58642529c 100755 --- a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh +++ b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh @@ -8,6 +8,7 @@ rm -rf docker-registry # Setup the environment export SETTINGS_FLAVOR=test export DOCKER_REGISTRY_CONFIG=config_test.yml +export PYTHONPATH=$(pwd)/docker-registry/test # Get latest docker registry git clone -q https://github.com/dotcloud/docker-registry.git @@ -21,7 +22,6 @@ pip install -q tox # Run registry tests tox || exit 1 -export PYTHONPATH=$(pwd)/docker-registry python -m unittest discover -p s3.py -s test || exit 1 python -m unittest discover -p workflow.py -s test diff --git a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile index 6762cb468..2100a9e8e 100644 --- a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -1,6 +1,6 @@ -# VERSION: 1.5 -# DOCKER-VERSION 0.6.4 -# AUTHOR: Daniel Mizyrycki +# VERSION: 1.6 +# DOCKER-VERSION 0.6.6 +# AUTHOR: Daniel Mizyrycki # DESCRIPTION: Build docker nightly release using Docker in Docker. # REFERENCES: This code reuses the excellent implementation of docker in docker # made by Jerome Petazzoni. https://github.com/jpetazzo/dind @@ -13,7 +13,7 @@ # TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh from docker -maintainer Daniel Mizyrycki +maintainer Daniel Mizyrycki # Add docker dependencies and downloading packages run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list @@ -22,11 +22,6 @@ run apt-get update; apt-get install -y -q wget python2.7 # Add production docker binary run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker -#### FIXME. Temporarily install docker and dind with proper apparmor handling -run wget -q -O /usr/bin/docker http://test.docker.io/test/docker; chmod +x /usr/bin/docker -run wget -q -O /go/src/github.com/dotcloud/docker/hack/dind http://raw.github.com/jpetazzo/docker/escape-apparmor-confinement/hack/dind -run chmod +x /go/src/github.com/dotcloud/docker/hack/dind - # Add proto docker builder add ./dockerbuild.sh /usr/bin/dockerbuild.sh run chmod +x /usr/bin/dockerbuild.sh diff --git a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh index 457db3f88..80caaec25 100644 --- a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh +++ b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh @@ -34,7 +34,7 @@ exit_status=$? # Display load if test fails set -x -if [ $exit_status -eq 1 ] ; then +if [ $exit_status -ne 0 ] ; then uptime; echo; free exit 1 fi diff --git a/hack/infrastructure/docker-ci/registry-coverage/Dockerfile b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile index 59c914fb2..e544645b6 100644 --- a/hack/infrastructure/docker-ci/registry-coverage/Dockerfile +++ b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile @@ -11,7 +11,7 @@ maintainer Daniel Mizyrycki # Add registry_coverager.sh and dependencies run pip install coverage flask pyyaml requests simplejson python-glanceclient \ - blinker redis gevent + blinker redis boto gevent rsa mock add registry_coverage.sh /usr/bin/registry_coverage.sh run chmod +x /usr/bin/registry_coverage.sh diff --git a/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh index e9f017265..e16cea8e3 100755 --- a/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh +++ b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh @@ -2,8 +2,11 @@ set -x -# Compute test paths +# Setup the environment REGISTRY_PATH=/data/docker-registry +export SETTINGS_FLAVOR=test +export DOCKER_REGISTRY_CONFIG=config_test.yml +export PYTHONPATH=$REGISTRY_PATH/test # Fetch latest docker-registry master rm -rf $REGISTRY_PATH @@ -11,8 +14,5 @@ git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH cd $REGISTRY_PATH # Generate coverage -export SETTINGS_FLAVOR=test -export DOCKER_REGISTRY_CONFIG=config_test.yml - coverage run -m unittest discover test || exit 1 coverage report --include='./*' --omit='./test/*' diff --git a/hack/infrastructure/docker-ci/report/deployment.py b/hack/infrastructure/docker-ci/report/deployment.py index d5efb4a96..5b2eaf3ca 100755 --- a/hack/infrastructure/docker-ci/report/deployment.py +++ b/hack/infrastructure/docker-ci/report/deployment.py @@ -34,7 +34,7 @@ env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read() DROPLET_NAME = env.get('DROPLET_NAME','report') TIMEOUT = 120 # Seconds before timeout droplet creation -IMAGE_ID = 894856 # Docker on Ubuntu 13.04 +IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 REGION_ID = 4 # New York 2 SIZE_ID = 66 # memory 512MB DO_IMAGE_USER = 'root' # Image user on Digital Ocean From 97c3de7e6b92b13efa30c5c840ce2fab07b8794e Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Thu, 7 Nov 2013 12:20:23 -0800 Subject: [PATCH 32/79] Fix 2585 and clean up warning in contributing.rst --- docs/sources/api/remote_api_client_libraries.rst | 2 ++ docs/sources/contributing/contributing.rst | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sources/api/remote_api_client_libraries.rst b/docs/sources/api/remote_api_client_libraries.rst index bd8610eaf..8243d02c8 100644 --- a/docs/sources/api/remote_api_client_libraries.rst +++ b/docs/sources/api/remote_api_client_libraries.rst @@ -35,3 +35,5 @@ and we will add the libraries here. +----------------------+----------------+--------------------------------------------+ | Go | go-dockerclient| https://github.com/fsouza/go-dockerclient | +----------------------+----------------+--------------------------------------------+ +| PHP | Alvine | http://pear.alvine.io/ (alpha) | ++----------------------+----------------+--------------------------------------------+ diff --git a/docs/sources/contributing/contributing.rst b/docs/sources/contributing/contributing.rst index 3cdb0b6f1..f4a6b472f 100644 --- a/docs/sources/contributing/contributing.rst +++ b/docs/sources/contributing/contributing.rst @@ -10,13 +10,13 @@ Want to hack on Docker? Awesome! The repository includes `all the instructions you need to get started `_. -The developer environment `Dockerfile `_ +The `developer environment Dockerfile `_ specifies the tools and versions used to test and build Docker. If you're making changes to the documentation, see the `README.md `_. -The documentation environment `Dockerfile `_ +The `documentation environment Dockerfile `_ specifies the tools and versions used to build the Documentation. Further interesting details can be found in the `Packaging hints `_. From ec4657b28a3e97447921357d454df974e0979ac6 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 1 Nov 2013 18:29:25 -0500 Subject: [PATCH 33/79] network: add iptables rules to explicitly allow forwarding Explicitly enable container networking for Fedora and other distros that have a REJECT all rule at the end of their FORWARD table. --- AUTHORS | 1 + network.go | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index 64f2ce21a..d002104e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -94,6 +94,7 @@ Jonathan Rudenberg Joost Cassee Jordan Arentsen Joseph Anthony Pasquale Holsten +Josh Poimboeuf Julien Barbier Jérôme Petazzoni Karan Lyons diff --git a/network.go b/network.go index 638fd94de..3864cca8c 100644 --- a/network.go +++ b/network.go @@ -168,12 +168,28 @@ func CreateBridgeIface(config *DaemonConfig) error { } if config.EnableIptables { + // Enable NAT if output, err := iptables.Raw("-t", "nat", "-A", "POSTROUTING", "-s", ifaceAddr, "!", "-d", ifaceAddr, "-j", "MASQUERADE"); err != nil { return fmt.Errorf("Unable to enable network bridge NAT: %s", err) } else if len(output) != 0 { return fmt.Errorf("Error iptables postrouting: %s", output) } + + // Accept incoming packets for existing connections + if output, err := iptables.Raw("-I", "FORWARD", "-o", config.BridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil { + return fmt.Errorf("Unable to allow incoming packets: %s", err) + } else if len(output) != 0 { + return fmt.Errorf("Error iptables allow incoming: %s", output) + } + + // Accept all non-intercontainer outgoing packets + if output, err := iptables.Raw("-I", "FORWARD", "-i", config.BridgeIface, "!", "-o", config.BridgeIface, "-j", "ACCEPT"); err != nil { + return fmt.Errorf("Unable to allow outgoing packets: %s", err) + } else if len(output) != 0 { + return fmt.Errorf("Error iptables allow outgoing: %s", output) + } + } return nil } @@ -680,20 +696,30 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { // Configure iptables for link support if config.EnableIptables { - args := []string{"FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j", "DROP"} + args := []string{"FORWARD", "-i", config.BridgeIface, "-o", config.BridgeIface, "-j"} + acceptArgs := append(args, "ACCEPT") + dropArgs := append(args, "DROP") if !config.InterContainerCommunication { - if !iptables.Exists(args...) { + iptables.Raw(append([]string{"-D"}, acceptArgs...)...) + if !iptables.Exists(dropArgs...) { utils.Debugf("Disable inter-container communication") - if output, err := iptables.Raw(append([]string{"-A"}, args...)...); err != nil { + if output, err := iptables.Raw(append([]string{"-I"}, dropArgs...)...); err != nil { return nil, fmt.Errorf("Unable to prevent intercontainer communication: %s", err) } else if len(output) != 0 { - return nil, fmt.Errorf("Error enabling iptables: %s", output) + return nil, fmt.Errorf("Error disabling intercontainer communication: %s", output) } } } else { - utils.Debugf("Enable inter-container communication") - iptables.Raw(append([]string{"-D"}, args...)...) + iptables.Raw(append([]string{"-D"}, dropArgs...)...) + if !iptables.Exists(acceptArgs...) { + utils.Debugf("Enable inter-container communication") + if output, err := iptables.Raw(append([]string{"-I"}, acceptArgs...)...); err != nil { + return nil, fmt.Errorf("Unable to allow intercontainer communication: %s", err) + } else if len(output) != 0 { + return nil, fmt.Errorf("Error enabling intercontainer communication: %s", output) + } + } } } From 49c4231f077cb0b4804768231396c596d070a22f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 7 Nov 2013 14:31:25 -0800 Subject: [PATCH 34/79] fix mergeConfig with new ports --- utils.go | 9 +++++++++ utils_test.go | 28 ++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/utils.go b/utils.go index 81715881a..1393a5c0a 100644 --- a/utils.go +++ b/utils.go @@ -89,6 +89,15 @@ func MergeConfig(userConf, imageConf *Config) error { } if userConf.ExposedPorts == nil || len(userConf.ExposedPorts) == 0 { userConf.ExposedPorts = imageConf.ExposedPorts + } else if imageConf.ExposedPorts != nil { + if userConf.ExposedPorts == nil { + userConf.ExposedPorts = make(map[Port]struct{}) + } + for port := range imageConf.ExposedPorts { + if _, exists := userConf.ExposedPorts[port]; !exists { + userConf.ExposedPorts[port] = struct{}{} + } + } } if userConf.PortSpecs != nil && len(userConf.PortSpecs) > 0 { diff --git a/utils_test.go b/utils_test.go index 87426f38e..589cd405e 100644 --- a/utils_test.go +++ b/utils_test.go @@ -247,7 +247,9 @@ func TestMergeConfig(t *testing.T) { Volumes: volumesUser, } - MergeConfig(configUser, configImage) + if err := MergeConfig(configUser, configImage); err != nil { + t.Error(err) + } if len(configUser.Dns) != 3 { t.Fatalf("Expected 3 dns, 1.1.1.1, 2.2.2.2 and 3.3.3.3, found %d", len(configUser.Dns)) @@ -259,7 +261,7 @@ func TestMergeConfig(t *testing.T) { } if len(configUser.ExposedPorts) != 3 { - t.Fatalf("Expected 3 portSpecs, 1111, 2222 and 3333, found %d", len(configUser.PortSpecs)) + t.Fatalf("Expected 3 ExposedPorts, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts)) } for portSpecs := range configUser.ExposedPorts { if portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" { @@ -287,6 +289,28 @@ func TestMergeConfig(t *testing.T) { if configUser.VolumesFrom != "1111" { t.Fatalf("Expected VolumesFrom to be 1111, found %s", configUser.VolumesFrom) } + + ports, _, err := parsePortSpecs([]string{"0000"}) + if err != nil { + t.Error(err) + } + configImage2 := &Config{ + ExposedPorts: ports, + } + + if err := MergeConfig(configUser, configImage2); err != nil { + t.Error(err) + } + + if len(configUser.ExposedPorts) != 4 { + t.Fatalf("Expected 4 ExposedPorts, 0000, 1111, 2222 and 3333, found %d", len(configUser.ExposedPorts)) + } + for portSpecs := range configUser.ExposedPorts { + if portSpecs.Port() != "0000" && portSpecs.Port() != "1111" && portSpecs.Port() != "2222" && portSpecs.Port() != "3333" { + t.Fatalf("Expected 0000 or 1111 or 2222 or 3333, found %s", portSpecs) + } + } + } func TestParseLxcConfOpt(t *testing.T) { From 01fea3cf116b768720b542ab65cbd1c2695848d0 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 22 Oct 2013 21:48:10 +1000 Subject: [PATCH 35/79] Closes #2328 - allow the user to specify a string timestamp (not just a unix epoch) in the string format that the docker cli shows to the user --- commands.go | 14 ++++++++++++-- docs/sources/commandline/cli.rst | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/commands.go b/commands.go index d41f0f86b..00be9a60e 100644 --- a/commands.go +++ b/commands.go @@ -1387,7 +1387,7 @@ func (cli *DockerCli) CmdCommit(args ...string) error { func (cli *DockerCli) CmdEvents(args ...string) error { cmd := Subcmd("events", "[OPTIONS]", "Get real time events from the server") - since := cmd.String("since", "", "Show events previously created (used for polling).") + since := cmd.String("since", "", "Show previously created events and then stream.") if err := cmd.Parse(args); err != nil { return nil } @@ -1399,7 +1399,17 @@ func (cli *DockerCli) CmdEvents(args ...string) error { v := url.Values{} if *since != "" { - v.Set("since", *since) + loc := time.FixedZone(time.Now().Zone()) + format := "2006-01-02 15:04:05 -0700 MST" + if len(*since) < len(format) { + format = format[:len(*since)] + } + + if t, err := time.ParseInLocation(format, *since, loc); err == nil { + v.Set("since", strconv.FormatInt(t.Unix(), 10)) + } else { + v.Set("since", *since) + } } if err := cli.stream("GET", "/events?"+v.Encode(), nil, cli.out, nil); err != nil { diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 6d56bccc3..d6e47d178 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -245,6 +245,9 @@ Full -run example Usage: docker events Get real time events from the server + + -since="": Show previously created events and then stream. + (either seconds since epoch, or date string as below) .. _cli_events_example: @@ -277,6 +280,23 @@ Shell 1: (Again .. now showing events) [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop +Show events in the past from a specified time +............................................. + +.. code-block:: bash + + $ sudo docker events -since 1378216169 + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop + + $ sudo docker events -since '2013-09-03' + [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop + + $ sudo docker events -since '2013-09-03 15:49:29 +0200 CEST' + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop .. _cli_export: From ef57752bce5c333d0e2b8352a84d071f21cce132 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 7 Nov 2013 17:30:51 -0800 Subject: [PATCH 36/79] Deprecate old tagging format --- commands.go | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/commands.go b/commands.go index 00be9a60e..12cc71365 100644 --- a/commands.go +++ b/commands.go @@ -913,8 +913,16 @@ func (cli *DockerCli) CmdImport(args ...string) error { cmd.Usage() return nil } - src := cmd.Arg(0) - repository, tag := utils.ParseRepositoryTag(cmd.Arg(1)) + + var src, repository, tag string + + if cmd.NArg() == 3 { + fmt.Fprintf(cli.err, "[DEPRECATED] The format 'URL|- [REPOSITORY [TAG]]' as been deprecated. Please use URL|- [REPOSITORY[:TAG]]\n") + src, repository, tag = cmd.Arg(0), cmd.Arg(1), cmd.Arg(2) + } else { + src = cmd.Arg(0) + repository, tag = utils.ParseRepositoryTag(cmd.Arg(1)) + } v := url.Values{} v.Set("repo", repository) v.Set("tag", tag) @@ -1349,8 +1357,16 @@ func (cli *DockerCli) CmdCommit(args ...string) error { if err := cmd.Parse(args); err != nil { return nil } - name := cmd.Arg(0) - repository, tag := utils.ParseRepositoryTag(cmd.Arg(1)) + + var name, repository, tag string + + if cmd.NArg() == 3 { + fmt.Fprintf(cli.err, "[DEPRECATED] The format 'CONTAINER [REPOSITORY [TAG]]' as been deprecated. Please use CONTAINER [REPOSITORY[:TAG]]\n") + name, repository, tag = cmd.Arg(0), cmd.Arg(1), cmd.Arg(2) + } else { + name = cmd.Arg(0) + repository, tag = utils.ParseRepositoryTag(cmd.Arg(1)) + } if name == "" { cmd.Usage() @@ -1666,9 +1682,16 @@ func (cli *DockerCli) CmdTag(args ...string) error { return nil } - v := url.Values{} - repository, tag := utils.ParseRepositoryTag(cmd.Arg(1)) + var repository, tag string + if cmd.NArg() == 3 { + fmt.Fprintf(cli.err, "[DEPRECATED] The format 'IMAGE [REPOSITORY [TAG]]' as been deprecated. Please use IMAGE [REPOSITORY[:TAG]]\n") + repository, tag = cmd.Arg(1), cmd.Arg(2) + } else { + repository, tag = utils.ParseRepositoryTag(cmd.Arg(1)) + } + + v := url.Values{} v.Set("repo", repository) v.Set("tag", tag) From 49d7b87cfc4385470a5ecf181f92c13b8391c002 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 7 Nov 2013 18:54:00 -0800 Subject: [PATCH 37/79] prevent panic if you use API in a wrong way --- container.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index 6cf35ec70..419ba84d2 100644 --- a/container.go +++ b/container.go @@ -133,7 +133,11 @@ type PortBinding struct { type Port string func (p Port) Proto() string { - return strings.Split(string(p), "/")[1] + parts := strings.Split(string(p), "/") + if len(parts) == 1 { + return "tcp" + } + return parts[1] } func (p Port) Port() string { From 5957dd909134fc3c3dc3e165b83559feb89d9f5b Mon Sep 17 00:00:00 2001 From: David Sissitka Date: Fri, 20 Sep 2013 04:55:17 -0400 Subject: [PATCH 38/79] Make "docker insert" errors obvious Closes #1130 See also #1942 --- api.go | 9 +++------ commands.go | 5 +---- server.go | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/api.go b/api.go index 61252ab9a..8f31ab028 100644 --- a/api.go +++ b/api.go @@ -479,15 +479,12 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht w.Header().Set("Content-Type", "application/json") } sf := utils.NewStreamFormatter(version > 1.0) - imgID, err := srv.ImageInsert(name, url, path, w, sf) + err := srv.ImageInsert(name, url, path, w, sf) if err != nil { - if sf.Used() { - w.Write(sf.FormatError(err)) - return nil - } + w.Write(sf.FormatError(err)) } - return writeJSON(w, http.StatusOK, &APIID{ID: imgID}) + return nil } func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/commands.go b/commands.go index 00be9a60e..1dc82e3bd 100644 --- a/commands.go +++ b/commands.go @@ -130,10 +130,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error { v.Set("url", cmd.Arg(1)) v.Set("path", cmd.Arg(2)) - if err := cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out, nil); err != nil { - return err - } - return nil + return cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out, nil) } // mkBuildContext returns an archive of an empty context with the contents diff --git a/server.go b/server.go index 344102902..eb4e74e0b 100644 --- a/server.go +++ b/server.go @@ -198,39 +198,39 @@ func (srv *Server) ImagesSearch(term string) ([]registry.SearchResult, error) { return results.Results, nil } -func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.StreamFormatter) (string, error) { +func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) img, err := srv.runtime.repositories.LookupImage(name) if err != nil { - return "", err + return err } file, err := utils.Download(url, out) if err != nil { - return "", err + return err } defer file.Body.Close() config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.capabilities) if err != nil { - return "", err + return err } c, _, err := srv.runtime.Create(config, "") if err != nil { - return "", err + return err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf, true), path); err != nil { - return "", err + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf, false), path); err != nil { + return err } // FIXME: Handle custom repo, tag comment, author img, err = srv.runtime.Commit(c, "", "", img.Comment, img.Author, nil) if err != nil { - return "", err + return err } - out.Write(sf.FormatStatus("", img.ID)) - return img.ShortID(), nil + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Image created")) + return nil } func (srv *Server) ImagesViz(out io.Writer) error { From bf8e0277bbd1c2df2310bc20ecc4003d1ed7a657 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Thu, 7 Nov 2013 23:34:54 -0600 Subject: [PATCH 39/79] Add ImageInsert tests --- server_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server_test.go b/server_test.go index 4072344f3..3376eebd6 100644 --- a/server_test.go +++ b/server_test.go @@ -3,6 +3,7 @@ package docker import ( "github.com/dotcloud/docker/utils" "strings" + "io/ioutil" "testing" "time" ) @@ -521,3 +522,25 @@ func TestImagesFilter(t *testing.T) { t.Fatal("incorrect number of matches returned") } } + +func TestImageInsert(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + srv := &Server{runtime: runtime} + sf := utils.NewStreamFormatter(true) + + // bad image name fails + if err := srv.ImageInsert("foo", "https://www.docker.io/static/img/docker-top-logo.png", "/foo", ioutil.Discard, sf); err == nil { + t.Fatal("expected an error and got none") + } + + // bad url fails + if err := srv.ImageInsert(GetTestImage(runtime).ID, "http://bad_host_name_that_will_totally_fail.com/", "/foo", ioutil.Discard, sf); err == nil { + t.Fatal("expected an error and got none") + } + + // success returns nil + if err := srv.ImageInsert(GetTestImage(runtime).ID, "https://www.docker.io/static/img/docker-top-logo.png", "/foo", ioutil.Discard, sf); err != nil { + t.Fatalf("expected no error, but got %v", err) + } +} From e7fdcc15c5eb3812c71dd61f22a8d77d3ae72e36 Mon Sep 17 00:00:00 2001 From: Michael Stapelberg Date: Fri, 8 Nov 2013 22:52:10 +0100 Subject: [PATCH 40/79] =?UTF-8?q?Return=20=E2=80=9Cerr=E2=80=9D=20instead?= =?UTF-8?q?=20of=20=E2=80=9Cnil=E2=80=9D=20when=20MkdirAll()=20fails=20whe?= =?UTF-8?q?n=20binding=20a=20volume?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container.go b/container.go index 419ba84d2..f9247eccc 100644 --- a/container.go +++ b/container.go @@ -837,7 +837,7 @@ func (container *Container) Start() (err error) { // Create the mountpoint rootVolPath := path.Join(container.RootfsPath(), volPath) if err := os.MkdirAll(rootVolPath, 0755); err != nil { - return nil + return err } // Do not copy or change permissions if we are mounting from the host From ca174ae84d37d89af726fa41e25a11bdaeca2067 Mon Sep 17 00:00:00 2001 From: Sean Cronin Date: Fri, 8 Nov 2013 17:17:39 -0500 Subject: [PATCH 41/79] Removes duplicate changelog entry --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c961746..8c08cefaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,6 @@ + Prevent DNS server conflicts in CreateBridgeIface + Validate bind mounts on the server side + Use parent image config in docker build -* Fix regression in /etc/hosts #### Client From 2448058ee23292e2a167bbd6a2e138ad99e8b3b8 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 8 Nov 2013 16:28:41 -0600 Subject: [PATCH 42/79] setup network when reconnecting to ghost container Re-adding the line to setup the network when reconnecting to a ghost container. It was inadvertently removed by commit 31638ab2ad2a5380d447780f05f7aa078c9421f5. --- runtime.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime.go b/runtime.go index 671694fce..581f6a8d8 100644 --- a/runtime.go +++ b/runtime.go @@ -172,6 +172,7 @@ func (runtime *Runtime) Register(container *Container) error { if !container.State.Running { close(container.waitLock) } else if !nomonitor { + container.allocateNetwork() go container.monitor() } return nil From b8e7ec1b74fdeaeab56a8ffab801dcd87b7361ca Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 8 Nov 2013 15:44:52 -0700 Subject: [PATCH 43/79] Update release script with proper support for non-*.docker.io bucket URLs --- hack/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/release.sh b/hack/release.sh index 56538ea70..23718dd63 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -107,7 +107,7 @@ s3_url() { echo "https://$BUCKET" ;; *) - echo "http://$BUCKET.s3.amazonaws.com" + s3cmd ws-info s3://$BUCKET | awk -v 'FS=: +' '/http:\/\/'$BUCKET'/ { gsub(/\/+$/, "", $2); print $2 }' ;; esac } From f56945d71bcce6e7d18b3471bf4744ddd70b1783 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 8 Nov 2013 15:45:18 -0700 Subject: [PATCH 44/79] Update release script to move https://get.docker.io/ubuntu/info to https://get.docker.io/ubuntu/ and provide a backwards-compatibility redirect (same for /builds/info) --- hack/release.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/hack/release.sh b/hack/release.sh index 23718dd63..06b76a1c7 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -114,7 +114,7 @@ s3_url() { # Upload the 'ubuntu' bundle to S3: # 1. A full APT repository is published at $BUCKET/ubuntu/ -# 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/info +# 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/index release_ubuntu() { [ -e bundles/$VERSION/ubuntu ] || { echo >&2 './hack/make.sh must be run before release_ubuntu' @@ -168,7 +168,7 @@ EOF # Upload repo s3cmd --acl-public sync $APTDIR/ s3://$BUCKET/ubuntu/ - cat < /etc/apt/sources.list.d/docker.list # Then import the repository key @@ -180,7 +180,12 @@ apt-get update ; apt-get install -y lxc-docker # Alternatively, just use the curl-able install.sh script provided at $(s3_url) # EOF - echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu/info" + + # Add redirect at /ubuntu/info for URL-backwards-compatibility + rm -rf /tmp/emptyfile && touch /tmp/emptyfile + s3cmd --acl-public --add-header='x-amz-website-redirect-location:/ubuntu/' --mime-type='text/plain' put /tmp/emptyfile s3://$BUCKET/ubuntu/info + + echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu" } # Upload a static binary to S3 @@ -189,14 +194,20 @@ release_binary() { echo >&2 './hack/make.sh must be run before release_binary' exit 1 } + S3DIR=s3://$BUCKET/builds/Linux/x86_64 s3cmd --acl-public put bundles/$VERSION/binary/docker-$VERSION $S3DIR/docker-$VERSION - cat < Date: Fri, 8 Nov 2013 15:01:01 -0800 Subject: [PATCH 45/79] prevent deletion if image is used by a running container --- server.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server.go b/server.go index 344102902..ceb3401f1 100644 --- a/server.go +++ b/server.go @@ -1257,6 +1257,26 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { } return nil, nil } + + // Prevent deletion if image is used by a running container + for _, container := range srv.runtime.List() { + if container.State.Running { + parent, err := srv.runtime.repositories.LookupImage(container.Image) + if err != nil { + return nil, err + } + + if err := parent.WalkHistory(func(p *Image) error { + if img.ID == p.ID { + return fmt.Errorf("Conflict, cannot delete %s because the running container %s is using it", name, container.ID) + } + return nil + }); err != nil { + return nil, err + } + } + } + if strings.Contains(img.ID, name) { //delete via ID return srv.deleteImage(img, "", "") From 498b6031b12da56c442d7c2b501f500766652b2d Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 8 Nov 2013 15:41:45 -0700 Subject: [PATCH 46/79] Update ubuntu packaging script, especially to stop docker group deletion --- hack/make/ubuntu | 70 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/hack/make/ubuntu b/hack/make/ubuntu index 5834172fd..d4b9fd0b3 100644 --- a/hack/make/ubuntu +++ b/hack/make/ubuntu @@ -10,7 +10,7 @@ fi PACKAGE_ARCHITECTURE="$(dpkg-architecture -qDEB_HOST_ARCH)" PACKAGE_URL="http://www.docker.io/" PACKAGE_MAINTAINER="docker@dotcloud.com" -PACKAGE_DESCRIPTION="lxc-docker is a Linux container runtime +PACKAGE_DESCRIPTION="Linux container runtime Docker complements LXC with a high-level API which operates at the process level. It runs unix processes with strong guarantees of isolation and repeatability across servers. @@ -37,27 +37,51 @@ bundle_ubuntu() { # This will fail if the binary bundle hasn't been built cp $DEST/../binary/docker-$VERSION $DIR/usr/bin/docker - # Generate postinst/prerm scripts - cat >/tmp/postinst <<'EOF' + # Generate postinst/prerm/postrm scripts + cat > /tmp/postinst <<'EOF' #!/bin/sh -service docker stop || true -grep -q '^docker:' /etc/group || groupadd --system docker || true -service docker start -EOF - cat >/tmp/prerm <<'EOF' -#!/bin/sh -service docker stop || true +set -e +set -u -case "$1" in - purge|remove|abort-install) - groupdel docker || true - ;; - - upgrade|failed-upgrade|abort-upgrade) - # don't touch docker group - ;; -esac +getent group docker > /dev/null || groupadd --system docker || true + +update-rc.d docker defaults > /dev/null || true +if [ -n "$2" ]; then + _dh_action=restart +else + _dh_action=start +fi +service docker $_dh_action 2>/dev/null || true + +#DEBHELPER# EOF + cat > /tmp/prerm <<'EOF' +#!/bin/sh +set -e +set -u + +service docker stop 2>/dev/null || true + +#DEBHELPER# +EOF + cat > /tmp/postrm <<'EOF' +#!/bin/sh +set -e +set -u + +if [ "$1" = "purge" ] ; then + update-rc.d docker remove > /dev/null || true +fi + +# In case this system is running systemd, we make systemd reload the unit files +# to pick up changes. +if [ -d /run/systemd/system ] ; then + systemctl --system daemon-reload > /dev/null || true +fi + +#DEBHELPER# +EOF + # TODO swaths of these were borrowed from debhelper's auto-inserted stuff, because we're still using fpm - we need to use debhelper instead, and somehow reconcile Ubuntu that way chmod +x /tmp/postinst /tmp/prerm ( @@ -66,6 +90,7 @@ EOF --name lxc-docker-$VERSION --version $PKGVERSION \ --after-install /tmp/postinst \ --before-remove /tmp/prerm \ + --after-remove /tmp/postrm \ --architecture "$PACKAGE_ARCHITECTURE" \ --prefix / \ --depends lxc \ @@ -82,6 +107,8 @@ EOF --vendor "$PACKAGE_VENDOR" \ --config-files /etc/init/docker.conf \ --config-files /etc/init.d/docker \ + --config-files /etc/default/docker \ + --deb-compression xz \ -t deb . mkdir empty fpm -s dir -C empty \ @@ -92,7 +119,12 @@ EOF --maintainer "$PACKAGE_MAINTAINER" \ --url "$PACKAGE_URL" \ --vendor "$PACKAGE_VENDOR" \ + --config-files /etc/init/docker.conf \ + --config-files /etc/init.d/docker \ + --config-files /etc/default/docker \ + --deb-compression xz \ -t deb . + # note: the --config-files lines have to be duplicated to stop overwrite on package upgrade (since we have to use this funky virtual package) ) } From 403f9fc357d64ccbdf82e1c4cbad2946eb1d1080 Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Fri, 8 Nov 2013 16:47:42 -0800 Subject: [PATCH 47/79] - Added delete all the containers example --- docs/sources/commandline/cli.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 4e3d369e1..28fac978f 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -622,6 +622,15 @@ This will remove the container referenced under the link ``/redis``. This will remove the underlying link between ``/webapp`` and the ``/redis`` containers removing all network communication. +.. code-block:: bash + + $ docker rm `docker ps -a -q` + + +This command will delete all the stopped containers. The command ``docker ps -a -q`` will return all +the existing container's id and the ``rm`` command takes those id's and delete them. The running containers +will not be deleted, even though they will appear on that id's list. + .. _cli_rmi: ``rmi`` From b3974abe4f01d408850e245c9b52c77f3571e0b2 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 25 Oct 2013 11:59:59 +1000 Subject: [PATCH 48/79] make all image ID and container ID API responses use the Long ID (Closes #2098) --- AUTHORS | 1 + commands_test.go | 128 +++++++++++++++++++++++++++++-- container.go | 16 +--- container_test.go | 3 +- docs/sources/terms/container.rst | 7 ++ docs/sources/terms/image.rst | 8 ++ docs/sources/use/basics.rst | 23 +++++- image.go | 4 - runtime.go | 4 +- server.go | 34 ++++---- 10 files changed, 182 insertions(+), 46 deletions(-) diff --git a/AUTHORS b/AUTHORS index 64f2ce21a..711c7718f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -165,6 +165,7 @@ Sridatta Thatipamala Sridhar Ratnakumar Steeve Morin Stefan Praszalowicz +Sven Dowideit Thatcher Peskens Thermionix Thijs Terlouw diff --git a/commands_test.go b/commands_test.go index 6c6a8e975..1778f1b89 100644 --- a/commands_test.go +++ b/commands_test.go @@ -6,6 +6,8 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" + "os" + "path" "regexp" "strings" "testing" @@ -381,8 +383,8 @@ func TestRunAttachStdin(t *testing.T) { if err != nil { t.Fatal(err) } - if cmdOutput != container.ShortID()+"\n" { - t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ShortID()+"\n", cmdOutput) + if cmdOutput != container.ID+"\n" { + t.Fatalf("Wrong output: should be '%s', not '%s'\n", container.ID+"\n", cmdOutput) } }) @@ -459,7 +461,7 @@ func TestRunDetach(t *testing.T) { }) } -// TestAttachDetach checks that attach in tty mode can be detached +// TestAttachDetach checks that attach in tty mode can be detached using the long container ID func TestAttachDetach(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() @@ -486,8 +488,8 @@ func TestAttachDetach(t *testing.T) { container = globalRuntime.List()[0] - if strings.Trim(string(buf[:n]), " \r\n") != container.ShortID() { - t.Fatalf("Wrong ID received. Expect %s, received %s", container.ShortID(), buf[:n]) + if strings.Trim(string(buf[:n]), " \r\n") != container.ID { + t.Fatalf("Wrong ID received. Expect %s, received %s", container.ID, buf[:n]) } }) setTimeout(t, "Starting container timed out", 10*time.Second, func() { @@ -501,7 +503,69 @@ func TestAttachDetach(t *testing.T) { ch = make(chan struct{}) go func() { defer close(ch) - if err := cli.CmdAttach(container.ShortID()); err != nil { + if err := cli.CmdAttach(container.ID); err != nil { + if err != io.ErrClosedPipe { + t.Fatal(err) + } + } + }() + + setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err != io.ErrClosedPipe { + t.Fatal(err) + } + } + }) + + setTimeout(t, "Escape sequence timeout", 5*time.Second, func() { + stdinPipe.Write([]byte{16, 17}) + if err := stdinPipe.Close(); err != nil { + t.Fatal(err) + } + }) + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) + + // wait for CmdRun to return + setTimeout(t, "Waiting for CmdAttach timed out", 15*time.Second, func() { + <-ch + }) + + time.Sleep(500 * time.Millisecond) + if !container.State.Running { + t.Fatal("The detached container should be still running") + } + + setTimeout(t, "Waiting for container to die timedout", 5*time.Second, func() { + container.Kill() + }) +} + +// TestAttachDetachTruncatedID checks that attach in tty mode can be detached +func TestAttachDetachTruncatedID(t *testing.T) { + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + go stdout.Read(make([]byte, 1024)) + setTimeout(t, "Starting container timed out", 2*time.Second, func() { + if err := cli.CmdRun("-i", "-t", "-d", unitTestImageID, "cat"); err != nil { + t.Fatal(err) + } + }) + + container := globalRuntime.List()[0] + + stdin, stdinPipe = io.Pipe() + stdout, stdoutPipe = io.Pipe() + cli = NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + + ch := make(chan struct{}) + go func() { + defer close(ch) + if err := cli.CmdAttach(utils.TruncateID(container.ID)); err != nil { if err != io.ErrClosedPipe { t.Fatal(err) } @@ -825,3 +889,55 @@ run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] return image } + +// #2098 - Docker cidFiles only contain short version of the containerId +//sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" +// TestRunCidFile tests that run -cidfile returns the longid +func TestRunCidFile(t *testing.T) { + stdout, stdoutPipe := io.Pipe() + + tmpDir, err := ioutil.TempDir("", "TestRunCidFile") + if err != nil { + t.Fatal(err) + } + tmpCidFile := path.Join(tmpDir, "cid") + + cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c := make(chan struct{}) + go func() { + defer close(c) + if err := cli.CmdRun("-cidfile", tmpCidFile, unitTestImageID, "ls"); err != nil { + t.Fatal(err) + } + }() + + defer os.RemoveAll(tmpDir) + setTimeout(t, "Reading command output time out", 2*time.Second, func() { + cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') + if err != nil { + t.Fatal(err) + } + if len(cmdOutput) < 1 { + t.Fatalf("'ls' should return something , not '%s'", cmdOutput) + } + //read the tmpCidFile + buffer, err := ioutil.ReadFile(tmpCidFile) + if err != nil { + t.Fatal(err) + } + id := string(buffer) + + if len(id) != len("2bf44ea18873287bd9ace8a4cb536a7cbe134bed67e805fdf2f58a57f69b320c") { + t.Fatalf("-cidfile should be a long id, not '%s'", id) + } + //test that its a valid cid? (though the container is gone..) + //remove the file and dir. + }) + + setTimeout(t, "CmdRun timed out", 5*time.Second, func() { + <-c + }) + +} diff --git a/container.go b/container.go index 419ba84d2..330ddb0d2 100644 --- a/container.go +++ b/container.go @@ -1235,7 +1235,7 @@ func (container *Container) monitor() { container.State.setStopped(exitCode) if container.runtime != nil && container.runtime.srv != nil { - container.runtime.srv.LogEvent("die", container.ShortID(), container.runtime.repositories.ImageName(container.Image)) + container.runtime.srv.LogEvent("die", container.ID, container.runtime.repositories.ImageName(container.Image)) } // Cleanup @@ -1302,7 +1302,7 @@ func (container *Container) kill(sig int) error { } if output, err := exec.Command("lxc-kill", "-n", container.ID, strconv.Itoa(sig)).CombinedOutput(); err != nil { - log.Printf("error killing container %s (%s, %s)", container.ShortID(), output, err) + log.Printf("error killing container %s (%s, %s)", utils.TruncateID(container.ID), output, err) return err } @@ -1322,9 +1322,9 @@ func (container *Container) Kill() error { // 2. Wait for the process to die, in last resort, try to kill the process directly if err := container.WaitTimeout(10 * time.Second); err != nil { if container.cmd == nil { - return fmt.Errorf("lxc-kill failed, impossible to kill the container %s", container.ShortID()) + return fmt.Errorf("lxc-kill failed, impossible to kill the container %s", utils.TruncateID(container.ID)) } - log.Printf("Container %s failed to exit within 10 seconds of lxc-kill %s - trying direct SIGKILL", "SIGKILL", container.ShortID()) + log.Printf("Container %s failed to exit within 10 seconds of lxc-kill %s - trying direct SIGKILL", "SIGKILL", utils.TruncateID(container.ID)) if err := container.cmd.Process.Kill(); err != nil { return err } @@ -1460,14 +1460,6 @@ func (container *Container) Unmount() error { return Unmount(container.RootfsPath()) } -// ShortID returns a shorthand version of the container's id for convenience. -// A collision with other container shorthands is very unlikely, but possible. -// In case of a collision a lookup with Runtime.Get() will fail, and the caller -// will need to use a langer prefix, or the full-length container Id. -func (container *Container) ShortID() string { - return utils.TruncateID(container.ID) -} - func (container *Container) logPath(name string) string { return path.Join(container.root, fmt.Sprintf("%s-%s.log", container.ID, name)) } diff --git a/container_test.go b/container_test.go index d51946ece..26007a732 100644 --- a/container_test.go +++ b/container_test.go @@ -3,6 +3,7 @@ package docker import ( "bufio" "fmt" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "math/rand" @@ -1005,7 +1006,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", - "HOSTNAME=" + container.ShortID(), + "HOSTNAME=" + utils.TruncateID(container.ID), "FALSE=true", "TRUE=false", "TRICKY=tri", diff --git a/docs/sources/terms/container.rst b/docs/sources/terms/container.rst index aeb7b1c3a..206664bd8 100644 --- a/docs/sources/terms/container.rst +++ b/docs/sources/terms/container.rst @@ -38,3 +38,10 @@ was when the container was stopped. You can promote a container to an :ref:`image_def` with ``docker commit``. Once a container is an image, you can use it as a parent for new containers. + +Container IDs +............. +All containers are identified by a 64 hexadecimal digit string (internally a 256bit +value). To simplify their use, a short ID of the first 12 characters can be used +on the commandline. There is a small possibility of short id collisions, so the +docker server will always return the long ID. diff --git a/docs/sources/terms/image.rst b/docs/sources/terms/image.rst index dafda1f3f..6d5c8b2e7 100644 --- a/docs/sources/terms/image.rst +++ b/docs/sources/terms/image.rst @@ -36,3 +36,11 @@ Base Image .......... An image that has no parent is a **base image**. + +Image IDs +......... +All images are identified by a 64 hexadecimal digit string (internally a 256bit +value). To simplify their use, a short ID of the first 12 characters can be used +on the command line. There is a small possibility of short id collisions, so the +docker server will always return the long ID. + diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 0097b6836..d1ad081f9 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -22,22 +22,37 @@ specify the path to it and manually start it. # Run docker in daemon mode sudo /docker -d & - -Running an interactive shell ----------------------------- +Download a pre-built image +-------------------------- .. code-block:: bash # Download an ubuntu image sudo docker pull ubuntu +This will find the ``ubuntu`` image by name in the :ref:`Central Index +` and download it from the top-level Central +Repository to a local image cache. + +.. NOTE:: When the image has successfully downloaded, you will see a 12 +character hash ``539c0211cd76: Download complete`` which is the short +form of the image ID. These short image IDs are the first 12 characters +of the full image ID - which can be found using ``docker inspect`` or +``docker images -notrunc=true`` + +.. _dockergroup: + +Running an interactive shell +---------------------------- + +.. code-block:: bash + # Run an interactive shell in the ubuntu image, # allocate a tty, attach stdin and stdout # To detach the tty without exiting the shell, # use the escape sequence Ctrl-p + Ctrl-q sudo docker run -i -t ubuntu /bin/bash -.. _dockergroup: Why ``sudo``? ------------- diff --git a/image.go b/image.go index 94cccaac6..c600273c1 100644 --- a/image.go +++ b/image.go @@ -202,10 +202,6 @@ func (image *Image) Changes(rw string) ([]Change, error) { return Changes(layers, rw) } -func (image *Image) ShortID() string { - return utils.TruncateID(image.ID) -} - func ValidateID(id string) error { if id == "" { return fmt.Errorf("Image id can't be empty") diff --git a/runtime.go b/runtime.go index 671694fce..6a3b76a59 100644 --- a/runtime.go +++ b/runtime.go @@ -181,7 +181,7 @@ func (runtime *Runtime) ensureName(container *Container) error { if container.Name == "" { name, err := generateRandomName(runtime) if err != nil { - name = container.ShortID() + name = utils.TruncateID(container.ID) } container.Name = name @@ -288,7 +288,7 @@ func (runtime *Runtime) restore() error { // Try to set the default name for a container if it exists prior to links container.Name, err = generateRandomName(runtime) if err != nil { - container.Name = container.ShortID() + container.Name = utils.TruncateID(container.ID) } if _, err := runtime.containerGraph.Set(container.Name, container.ID); err != nil { diff --git a/server.go b/server.go index 344102902..2e2361262 100644 --- a/server.go +++ b/server.go @@ -154,7 +154,7 @@ func (srv *Server) ContainerKill(name string, sig int) error { if err := container.Kill(); err != nil { return fmt.Errorf("Cannot kill container %s: %s", name, err) } - srv.LogEvent("kill", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) + srv.LogEvent("kill", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { // Otherwise, just send the requested signal if err := container.kill(sig); err != nil { @@ -180,7 +180,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { if _, err := io.Copy(out, data); err != nil { return err } - srv.LogEvent("export", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) + srv.LogEvent("export", container.ID, srv.runtime.repositories.ImageName(container.Image)) return nil } return fmt.Errorf("No such container: %s", name) @@ -230,7 +230,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } out.Write(sf.FormatStatus("", img.ID)) - return img.ShortID(), nil + return img.ID, nil } func (srv *Server) ImagesViz(out io.Writer) error { @@ -250,9 +250,9 @@ func (srv *Server) ImagesViz(out io.Writer) error { return fmt.Errorf("Error while getting parent image: %v", err) } if parentImage != nil { - out.Write([]byte(" \"" + parentImage.ShortID() + "\" -> \"" + image.ShortID() + "\"\n")) + out.Write([]byte(" \"" + parentImage.ID + "\" -> \"" + image.ID + "\"\n")) } else { - out.Write([]byte(" base -> \"" + image.ShortID() + "\" [style=invis]\n")) + out.Write([]byte(" base -> \"" + image.ID + "\" [style=invis]\n")) } } @@ -465,7 +465,7 @@ func (srv *Server) Containers(all, size bool, n int, since, before string) []API continue } if before != "" { - if container.ShortID() == before { + if container.ID == before || utils.TruncateID(container.ID) == before { foundBefore = true continue } @@ -476,7 +476,7 @@ func (srv *Server) Containers(all, size bool, n int, since, before string) []API if displayed == n { break } - if container.ShortID() == since { + if container.ID == since || utils.TruncateID(container.ID) == since { break } displayed++ @@ -518,7 +518,7 @@ func (srv *Server) ContainerCommit(name, repo, tag, author, comment string, conf if err != nil { return "", err } - return img.ShortID(), err + return img.ID, err } func (srv *Server) ContainerTag(name, repo, tag string, force bool) error { @@ -1017,7 +1017,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return err } } - out.Write(sf.FormatStatus("", img.ShortID())) + out.Write(sf.FormatStatus("", img.ID)) return nil } @@ -1046,8 +1046,8 @@ func (srv *Server) ContainerCreate(config *Config, name string) (string, []strin } return "", nil, err } - srv.LogEvent("create", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) - return container.ShortID(), buildWarnings, nil + srv.LogEvent("create", container.ID, srv.runtime.repositories.ImageName(container.Image)) + return container.ID, buildWarnings, nil } func (srv *Server) ContainerRestart(name string, t int) error { @@ -1055,7 +1055,7 @@ func (srv *Server) ContainerRestart(name string, t int) error { if err := container.Restart(t); err != nil { return fmt.Errorf("Cannot restart container %s: %s", name, err) } - srv.LogEvent("restart", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) + srv.LogEvent("restart", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { return fmt.Errorf("No such container: %s", name) } @@ -1111,7 +1111,7 @@ func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool) if err := srv.runtime.Destroy(container); err != nil { return fmt.Errorf("Cannot destroy container %s: %s", name, err) } - srv.LogEvent("destroy", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) + srv.LogEvent("destroy", container.ID, srv.runtime.repositories.ImageName(container.Image)) if removeVolume { // Retrieve all volumes from all remaining containers @@ -1228,8 +1228,8 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro return nil, err } if tagDeleted { - imgs = append(imgs, APIRmi{Untagged: img.ShortID()}) - srv.LogEvent("untag", img.ShortID(), "") + imgs = append(imgs, APIRmi{Untagged: img.ID}) + srv.LogEvent("untag", img.ID, "") } } if len(srv.runtime.repositories.ByID()[img.ID]) == 0 { @@ -1364,7 +1364,7 @@ func (srv *Server) ContainerStart(name string, hostConfig *HostConfig) error { if err := container.Start(); err != nil { return fmt.Errorf("Cannot start container %s: %s", name, err) } - srv.LogEvent("start", container.ShortID(), runtime.repositories.ImageName(container.Image)) + srv.LogEvent("start", container.ID, runtime.repositories.ImageName(container.Image)) return nil } @@ -1374,7 +1374,7 @@ func (srv *Server) ContainerStop(name string, t int) error { if err := container.Stop(t); err != nil { return fmt.Errorf("Cannot stop container %s: %s", name, err) } - srv.LogEvent("stop", container.ShortID(), srv.runtime.repositories.ImageName(container.Image)) + srv.LogEvent("stop", container.ID, srv.runtime.repositories.ImageName(container.Image)) } else { return fmt.Errorf("No such container: %s", name) } From 7f1b179c67476efa7dbafda55541b515fbe0f346 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sat, 9 Nov 2013 02:28:04 -0700 Subject: [PATCH 49/79] Fix the display of get.docker.io in Firefox by making our index files text/plain --- hack/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/release.sh b/hack/release.sh index 06b76a1c7..931ab6f9a 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -97,7 +97,7 @@ write_to_s3() { DEST=$1 F=`mktemp` cat > $F - s3cmd --acl-public put $F $DEST + s3cmd --acl-public --mime-type='text/plain' put $F $DEST rm -f $F } From 4ec0b515786ce266234d350fee872764974d2218 Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Sat, 9 Nov 2013 11:08:43 -0800 Subject: [PATCH 50/79] - Updated description --- docs/sources/commandline/cli.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index 28fac978f..026785214 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -627,9 +627,9 @@ network communication. $ docker rm `docker ps -a -q` -This command will delete all the stopped containers. The command ``docker ps -a -q`` will return all -the existing container's id and the ``rm`` command takes those id's and delete them. The running containers -will not be deleted, even though they will appear on that id's list. +This command will delete all stopped containers. The command ``docker ps -a -q`` will return all +existing container IDs and pass them to the ``rm`` command which will delete them. Any running +containers will not be deleted. .. _cli_rmi: From 8ba8783bcc0ec3a0c5391445d52e2b2a9d0a3f8a Mon Sep 17 00:00:00 2001 From: David Anderson Date: Sat, 9 Nov 2013 19:31:08 -0800 Subject: [PATCH 51/79] Correctly express "any address" to iptables. Iptables interprets "-d 0.0.0.0" as "-d 0.0.0.0/32", not /0. This results in the DNAT rule never matching any traffic if not bound to a specific host IP. Fixes #2598 --- iptables/iptables.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/iptables/iptables.go b/iptables/iptables.go index 82ecf8bb5..0438bcbd8 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -55,9 +55,16 @@ func RemoveExistingChain(name string) error { } func (c *Chain) Forward(action Action, ip net.IP, port int, proto, dest_addr string, dest_port int) error { + daddr := ip.String() + if ip.IsUnspecified() { + // iptables interprets "0.0.0.0" as "0.0.0.0/32", whereas we + // want "0.0.0.0/0". "0/0" is correctly interpreted as "any + // value" by both iptables and ip6tables. + daddr = "0/0" + } if output, err := Raw("-t", "nat", fmt.Sprint(action), c.Name, "-p", proto, - "-d", ip.String(), + "-d", daddr, "--dport", strconv.Itoa(port), "!", "-i", c.Bridge, "-j", "DNAT", From 8cc19765b48d1a429b840b731ed5fd5b81fbda3c Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Sun, 10 Nov 2013 00:06:55 -0600 Subject: [PATCH 52/79] Edits after code review Return long image ID Return streamformatted error or "raw" error --- api.go | 6 +++++- server.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 8f31ab028..ce46d5012 100644 --- a/api.go +++ b/api.go @@ -481,7 +481,11 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht sf := utils.NewStreamFormatter(version > 1.0) err := srv.ImageInsert(name, url, path, w, sf) if err != nil { - w.Write(sf.FormatError(err)) + if sf.Used() { + w.Write(sf.FormatError(err)) + return nil + } + return err } return nil diff --git a/server.go b/server.go index eb4e74e0b..93ca61365 100644 --- a/server.go +++ b/server.go @@ -229,7 +229,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. if err != nil { return err } - out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Image created")) + out.Write(sf.FormatStatus(img.ID, "Image created")) return nil } From 5a1bfd9aa9a0983b5141dd8e393cc67a68fc6590 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sat, 9 Nov 2013 14:10:00 -0500 Subject: [PATCH 53/79] Added status column to API client table --- .../api/remote_api_client_libraries.rst | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/sources/api/remote_api_client_libraries.rst b/docs/sources/api/remote_api_client_libraries.rst index 8243d02c8..f00ab1c2b 100644 --- a/docs/sources/api/remote_api_client_libraries.rst +++ b/docs/sources/api/remote_api_client_libraries.rst @@ -12,28 +12,28 @@ compatibility. Please file issues with the library owners. If you find more library implementations, please list them in Docker doc bugs and we will add the libraries here. -+----------------------+----------------+--------------------------------------------+ -| Language/Framework | Name | Repository | -+======================+================+============================================+ -| Python | docker-py | https://github.com/dotcloud/docker-py | -+----------------------+----------------+--------------------------------------------+ -| Ruby | docker-client | https://github.com/geku/docker-client | -+----------------------+----------------+--------------------------------------------+ -| Ruby | docker-api | https://github.com/swipely/docker-api | -+----------------------+----------------+--------------------------------------------+ -| Javascript (NodeJS) | docker.io | https://github.com/appersonlabs/docker.io | -| | | Install via NPM: `npm install docker.io` | -+----------------------+----------------+--------------------------------------------+ -| Javascript | docker-js | https://github.com/dgoujard/docker-js | -+----------------------+----------------+--------------------------------------------+ -| Javascript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | -| **WebUI** | | | -+----------------------+----------------+--------------------------------------------+ -| Java | docker-java | https://github.com/kpelykh/docker-java | -+----------------------+----------------+--------------------------------------------+ -| Erlang | erldocker | https://github.com/proger/erldocker | -+----------------------+----------------+--------------------------------------------+ -| Go | go-dockerclient| https://github.com/fsouza/go-dockerclient | -+----------------------+----------------+--------------------------------------------+ -| PHP | Alvine | http://pear.alvine.io/ (alpha) | -+----------------------+----------------+--------------------------------------------+ ++----------------------+----------------+--------------------------------------------+----------+ +| Language/Framework | Name | Repository | Status | ++======================+================+============================================+==========+ +| Python | docker-py | https://github.com/dotcloud/docker-py | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| Ruby | docker-client | https://github.com/geku/docker-client | Outdated | ++----------------------+----------------+--------------------------------------------+----------+ +| Ruby | docker-api | https://github.com/swipely/docker-api | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| Javascript (NodeJS) | docker.io | https://github.com/appersonlabs/docker.io | Active | +| | | Install via NPM: `npm install docker.io` | | ++----------------------+----------------+--------------------------------------------+----------+ +| Javascript | docker-js | https://github.com/dgoujard/docker-js | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| Javascript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | Active | +| **WebUI** | | | | ++----------------------+----------------+--------------------------------------------+----------+ +| Java | docker-java | https://github.com/kpelykh/docker-java | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| Erlang | erldocker | https://github.com/proger/erldocker | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| Go | go-dockerclient| https://github.com/fsouza/go-dockerclient | Active | ++----------------------+----------------+--------------------------------------------+----------+ +| PHP | Alvine | http://pear.alvine.io/ (alpha) | Active | ++----------------------+----------------+--------------------------------------------+----------+ From ccbb5d34927dc1905984bead3ebb576c0ea20960 Mon Sep 17 00:00:00 2001 From: Galen Sampson Date: Wed, 6 Nov 2013 19:16:56 -0800 Subject: [PATCH 54/79] Vagrantfile updates. - Remove the overrides config.vm.box and config.vm.box_url and use the same values for all providers. - Use the same private key path for all providers. It is still possible to set a different private key path through the environment variable SSH_PRIVKEY_PATH if desired (your AWS key may be different from your Virtualbox key). - Allow the environment variable AWS_INSTANCE_TYPE to specify the instance type of instead of hard coding the AWS instance type as 't1.micro'. 't1.micro' is still the default if unspecified. - Use the same environment variables for keys as the Amazon provided EC2 API tools. This allows people who already have the EC2 tools set up correctly to use 'vagrant up' with less environment configuration than before. - Rewrite the provisioning code. The goal is to be idempotent and to correctly install docker for all providers instead of just virtualbox. It will conditionally install the virtualbox guest additions if virtualbox is the provider. - Update the AWS install documentation to reflect the changes. --- Vagrantfile | 163 +++++++++++++++++++-------- docs/sources/installation/amazon.rst | 35 ++++-- 2 files changed, 146 insertions(+), 52 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 93a2219fa..a0bb38ca4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,65 +4,135 @@ BOX_NAME = ENV['BOX_NAME'] || "ubuntu" BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box" VF_BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64_vmware_fusion.box" +AWS_BOX_URI = ENV['BOX_URI'] || "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box" AWS_REGION = ENV['AWS_REGION'] || "us-east-1" -AWS_AMI = ENV['AWS_AMI'] || "ami-d0f89fb9" +AWS_AMI = ENV['AWS_AMI'] || "ami-69f5a900" +AWS_INSTANCE_TYPE = ENV['AWS_INSTANCE_TYPE'] || 't1.micro' + FORWARD_DOCKER_PORTS = ENV['FORWARD_DOCKER_PORTS'] +SSH_PRIVKEY_PATH = ENV["SSH_PRIVKEY_PATH"] + +# A script to upgrade from the 12.04 kernel to the raring backport kernel (3.8) +# and install docker. +$script = <