From b611198d286d2f4ebd7526c623dff8e523691698 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Tue, 3 Jun 2014 15:46:01 +0800 Subject: [PATCH 001/517] mount of /.dockerinit is not needed for native driver, so move it into lxc driver Docker-DCO-1.1-Signed-off-by: Albert Zhang (github: zhgwenming) --- daemon/execdriver/execdrivers/execdrivers.go | 2 +- daemon/execdriver/lxc/driver.go | 5 ++++- daemon/volumes.go | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/daemon/execdriver/execdrivers/execdrivers.go b/daemon/execdriver/execdrivers/execdrivers.go index 2e18454a0..f43514408 100644 --- a/daemon/execdriver/execdrivers/execdrivers.go +++ b/daemon/execdriver/execdrivers/execdrivers.go @@ -15,7 +15,7 @@ func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdrive // we want to give the lxc driver the full docker root because it needs // to access and write config and template files in /var/lib/docker/containers/* // to be backwards compatible - return lxc.NewDriver(root, sysInfo.AppArmor) + return lxc.NewDriver(root, initPath, sysInfo.AppArmor) case "native": return native.NewDriver(path.Join(root, "execdriver", "native"), initPath) } diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 59daf1afe..d4a00dcca 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -54,11 +54,12 @@ func init() { type driver struct { root string // root path for the driver to use + initPath string apparmor bool sharedRoot bool } -func NewDriver(root string, apparmor bool) (*driver, error) { +func NewDriver(root, initPath string, apparmor bool) (*driver, error) { // setup unconfined symlink if err := linkLxcStart(root); err != nil { return nil, err @@ -66,6 +67,7 @@ func NewDriver(root string, apparmor bool) (*driver, error) { return &driver{ apparmor: apparmor, root: root, + initPath: initPath, sharedRoot: rootIsShared(), }, nil } @@ -79,6 +81,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba if err := execdriver.SetTerminal(c, pipes); err != nil { return -1, err } + c.Mounts = append(c.Mounts, execdriver.Mount{d.initPath, c.InitPath, false, true}) if err := d.generateEnvConfig(c); err != nil { return -1, err } diff --git a/daemon/volumes.go b/daemon/volumes.go index f4b3921c9..1de8ff98a 100644 --- a/daemon/volumes.go +++ b/daemon/volumes.go @@ -36,7 +36,6 @@ func prepareVolumesForContainer(container *Container) error { func setupMountsForContainer(container *Container) error { mounts := []execdriver.Mount{ - {container.daemon.sysInitPath, "/.dockerinit", false, true}, {container.ResolvConfPath, "/etc/resolv.conf", false, true}, } From a084f4bc61a81eb9076246d85d6cc5168b32a949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 11 Jun 2014 17:09:19 -0700 Subject: [PATCH 002/517] Select masquerade by outgoing interface rather than by destination subnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker-DCO-1.1-Signed-off-by: Jérôme Petazzoni (github: jpetazzo) --- daemon/networkdriver/bridge/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/networkdriver/bridge/driver.go b/daemon/networkdriver/bridge/driver.go index 8c5db9f84..1cde7c9be 100644 --- a/daemon/networkdriver/bridge/driver.go +++ b/daemon/networkdriver/bridge/driver.go @@ -175,7 +175,7 @@ func InitDriver(job *engine.Job) engine.Status { func setupIPTables(addr net.Addr, icc bool) error { // Enable NAT - natArgs := []string{"POSTROUTING", "-t", "nat", "-s", addr.String(), "!", "-d", addr.String(), "-j", "MASQUERADE"} + natArgs := []string{"POSTROUTING", "-t", "nat", "-s", addr.String(), "!", "-o", bridgeIface, "-j", "MASQUERADE"} if !iptables.Exists(natArgs...) { if output, err := iptables.Raw(append([]string{"-I"}, natArgs...)...); err != nil { From 05243104fc0a0ef9537766cf5bd920824665eb78 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 4 Dec 2013 15:03:51 +0100 Subject: [PATCH 003/517] Add support for client certificates for registries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets you specify custom client TLS certificates and CA root for a specific registry hostname. Docker will then verify the registry against the CA and present the client cert when talking to that registry. This allows the registry to verify that the client has a proper key, indicating that the client is allowed to access the images. A custom cert is configured by creating a directory in /etc/docker/certs.d with the same name as the registry hostname. Inside this directory all *.crt files are added as CA Roots (if none exists, the system default is used) and pair of files .key and .cert indicate a custom certificate to present to the registry. If there are multiple certificates each one will be tried in alphabetical order, proceeding to the next if we get a 403 of 5xx response. So, an example setup would be: /etc/docker/certs.d/ └── localhost ├── client.cert ├── client.key └── localhost.crt A simple way to test this setup is to use an apache server to host a registry. Just copy a registry tree into the apache root, here is an example one containing the busybox image: http://people.gnome.org/~alexl/v1.tar.gz Then add this conf file as /etc/httpd/conf.d/registry.conf: # This must be in the root context, otherwise it causes a re-negotiation # which is not supported by the tls implementation in go SSLVerifyClient optional_no_ca Action cert-protected /cgi-bin/cert.cgi SetHandler cert-protected Header set x-docker-registry-version "0.6.2" SetEnvIf Host (.*) custom_host=$1 Header set X-Docker-Endpoints "%{custom_host}e" And this as /var/www/cgi-bin/cert.cgi #!/bin/bash if [ "$HTTPS" != "on" ]; then echo "Status: 403 Not using SSL" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then echo "Status: 403 Client certificate invalid" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" echo "x-docker-registry-version: 0.6.2" echo "X-Docker-Endpoints: $SERVER_NAME" echo "X-Docker-Size: 0" echo cat $PATH_TRANSLATED This will return 403 for all accessed to /v1 unless *any* client cert is presented. Obviously a real implementation would verify more details about the certificate. Example client certs can be generated with: openssl genrsa -out client.key 1024 openssl req -new -x509 -text -key client.key -out client.cert Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) --- docs/mkdocs.yml | 1 + docs/sources/articles/certificates.md | 83 ++++++++++ docs/sources/use.md | 14 ++ registry/registry.go | 227 ++++++++++++++++++++------ 4 files changed, 272 insertions(+), 53 deletions(-) create mode 100644 docs/sources/articles/certificates.md create mode 100644 docs/sources/use.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 5c3147a28..00810e2cb 100755 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -83,6 +83,7 @@ pages: - ['articles/security.md', 'Articles', 'Security'] - ['articles/https.md', 'Articles', 'Running Docker with HTTPS'] - ['articles/host_integration.md', 'Articles', 'Automatically starting Containers'] +- ['articles/certificates.md', 'Articles', 'Using certificates for repository client verification'] - ['articles/using_supervisord.md', 'Articles', 'Using Supervisor'] - ['articles/cfengine_process_management.md', 'Articles', 'Process management with CFEngine'] - ['articles/puppet.md', 'Articles', 'Using Puppet'] diff --git a/docs/sources/articles/certificates.md b/docs/sources/articles/certificates.md new file mode 100644 index 000000000..3772a7bc5 --- /dev/null +++ b/docs/sources/articles/certificates.md @@ -0,0 +1,83 @@ +page_title: Using certificates for repository client verification +page_description: How to set up per-repository client certificates +page_keywords: Usage, repository, certificate, root, docker, documentation, examples + +# Using certificates for repository client verification + +This lets you specify custom client TLS certificates and CA root for a +specific registry hostname. Docker will then verify the registry +against the CA and present the client cert when talking to that +registry. This allows the registry to verify that the client has a +proper key, indicating that the client is allowed to access the +images. + +A custom cert is configured by creating a directory in +`/etc/docker/certs.d` with the same name as the registry hostname. Inside +this directory all .crt files are added as CA Roots (if none exists, +the system default is used) and pair of files `$filename.key` and +`$filename.cert` indicate a custom certificate to present to the +registry. + +If there are multiple certificates each one will be tried in +alphabetical order, proceeding to the next if we get a 403 of 5xx +response. + +So, an example setup would be:: + + /etc/docker/certs.d/ + └── localhost + ├── client.cert + ├── client.key + └── localhost.crt + +A simple way to test this setup is to use an apache server to host a +registry. Just copy a registry tree into the apache root, +[here](http://people.gnome.org/~alexl/v1.tar.gz) is an example one +containing the busybox image. + +Then add this conf file as `/etc/httpd/conf.d/registry.conf`: + + # This must be in the root context, otherwise it causes a re-negotiation + # which is not supported by the tls implementation in go + SSLVerifyClient optional_no_ca + + + Action cert-protected /cgi-bin/cert.cgi + SetHandler cert-protected + + Header set x-docker-registry-version "0.6.2" + SetEnvIf Host (.*) custom_host=$1 + Header set X-Docker-Endpoints "%{custom_host}e" + + +And this as `/var/www/cgi-bin/cert.cgi`: + + #!/bin/bash + if [ "$HTTPS" != "on" ]; then + echo "Status: 403 Not using SSL" + echo "x-docker-registry-version: 0.6.2" + echo + exit 0 + fi + if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then + echo "Status: 403 Client certificate invalid" + echo "x-docker-registry-version: 0.6.2" + echo + exit 0 + fi + echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" + echo "x-docker-registry-version: 0.6.2" + echo "X-Docker-Endpoints: $SERVER_NAME" + echo "X-Docker-Size: 0" + echo + + cat $PATH_TRANSLATED + +This will return 403 for all accessed to `/v1` unless any client cert is +presented. Obviously a real implementation would verify more details +about the certificate. + +Example client certs can be generated with:: + + openssl genrsa -out client.key 1024 + openssl req -new -x509 -text -key client.key -out client.cert diff --git a/docs/sources/use.md b/docs/sources/use.md new file mode 100644 index 000000000..20c5eb2c5 --- /dev/null +++ b/docs/sources/use.md @@ -0,0 +1,14 @@ +# Use + +## Contents: + + - [First steps with Docker](basics/) + - [Share Images via Repositories](workingwithrepository/) + - [Redirect Ports](port_redirection/) + - [Configure Networking](networking/) + - [Automatically Start Containers](host_integration/) + - [Share Directories via Volumes](working_with_volumes/) + - [Link Containers](working_with_links_names/) + - [Link via an Ambassador Container](ambassador_pattern_linking/) + - [Using Puppet](puppet/) + - [Using certificates for repository client verification](certificates/) diff --git a/registry/registry.go b/registry/registry.go index 24c55125c..748636dca 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/sha256" _ "crypto/sha512" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -13,6 +15,8 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "os" + "path" "regexp" "runtime" "strconv" @@ -29,31 +33,155 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) +type TimeoutType uint32 + +const ( + NoTimeout TimeoutType = iota + ReceiveTimeout + ConnectTimeout +) + +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { + tlsConfig := tls.Config{RootCAs: roots} + + if cert != nil { + tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + } + + httpTransport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + } + + switch timeout { + case ConnectTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, 5*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, nil + } + case ReceiveTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, 1*time.Minute) + return conn, nil + } + } + + return &http.Client{ + Transport: httpTransport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + Jar: jar, + } +} + +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err + } + + var ( + pool *x509.CertPool + certs []*tls.Certificate + ) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } else { + pool.AppendCertsFromPEM(data) + } + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } else { + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + if len(certs) == 0 { + client := newClient(jar, pool, nil, timeout) + res, err := client.Do(req) + if err != nil { + return nil, nil, err + } + return res, client, nil + } else { + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + if i == len(certs)-1 { + // If this is the last cert, always return the result + return res, client, err + } else { + // Otherwise, continue to next cert if 403 or 5xx + if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { + return res, client, err + } + } + } + } + + return nil, nil, nil +} + func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - httpDial := func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - httpTransport := &http.Transport{ - Dial: httpDial, - Proxy: http.ProxyFromEnvironment, - } - client := &http.Client{Transport: httpTransport} - resp, err := client.Get(endpoint + "_ping") + + req, err := http.NewRequest("GET", endpoint+"_ping", nil) if err != nil { return RegistryInfo{Standalone: false}, err } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) @@ -171,6 +299,10 @@ func setTokenAuth(req *http.Request, token []string) { } } +func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -179,7 +311,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -214,7 +346,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo return false } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { utils.Errorf("Error in LookupRemoteImage %s", err) return false @@ -231,7 +363,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -260,6 +392,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i var ( retries = 5 headRes *http.Response + client *http.Client hasResume bool = false imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) @@ -267,9 +400,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + setTokenAuth(headReq, token) for i := 1; i <= retries; i++ { - headRes, err = r.client.Do(headReq) + headRes, client, err = r.doRequest(headReq) if err != nil && i == retries { return nil, fmt.Errorf("Eror while making head request: %s\n", err) } else if err != nil { @@ -290,10 +424,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i setTokenAuth(req, token) if hasResume { utils.Debugf("server supports resume") - return utils.ResumableRequestReader(r.client, req, 5, imgSize), nil + return utils.ResumableRequestReader(client, req, 5, imgSize), nil } utils.Debugf("server doesn't support resume") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -319,7 +453,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -380,7 +514,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -448,13 +582,13 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) + r.jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -484,7 +618,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -525,7 +659,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return "", "", fmt.Errorf("Failed to upload layer: %s", err) } @@ -562,7 +696,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return err } @@ -610,7 +744,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.Header["X-Docker-Endpoints"] = regs } - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -629,7 +763,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat if validate { req.Header["X-Docker-Endpoints"] = regs } - res, err = r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -688,7 +822,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -750,10 +884,11 @@ type RegistryInfo struct { } type Registry struct { - client *http.Client authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType } func trustedLocation(req *http.Request) bool { @@ -791,30 +926,16 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - httpTransport := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - } - if timeout { - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = utils.NewTimeoutConn(conn, 1*time.Minute) - return conn, nil - } - } r = &Registry{ - authConfig: authConfig, - client: &http.Client{ - Transport: httpTransport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - }, + authConfig: authConfig, indexEndpoint: indexEndpoint, } - r.client.Jar, err = cookiejar.New(nil) + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) if err != nil { return nil, err } From 06cd125e64c30886f66e6490a1cd4e7a7ba23655 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 18 Jun 2014 10:51:44 -0400 Subject: [PATCH 004/517] Increase size of buffer for signals The chan struct used to forward signals to containers was one element only, which caused some signals to be dropped when many were being received. Increasing the size of the chan buffer makes this much less likely to happen. Docker-DCO-1.1-Signed-off-by: Matt Heon (github: mheon) --- api/client/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/client/commands.go b/api/client/commands.go index a6a2e3553..0a30bc3d5 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -535,7 +535,7 @@ func (cli *DockerCli) CmdRestart(args ...string) error { } func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal { - sigc := make(chan os.Signal, 1) + sigc := make(chan os.Signal, 128) signal.CatchAll(sigc) go func() { for s := range sigc { From ff1ea0064b49bd1d72cafe5d02be9eab4d54b683 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 20 Jun 2014 09:40:29 -0400 Subject: [PATCH 005/517] Add cpuguy83 as volumes maintainer Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) --- daemon/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 daemon/MAINTAINERS diff --git a/daemon/MAINTAINERS b/daemon/MAINTAINERS new file mode 100644 index 000000000..7ab013cd8 --- /dev/null +++ b/daemon/MAINTAINERS @@ -0,0 +1 @@ +volumes.go: Brian Goff (@cpuguy83) From f55fa8211b6faf08091dd99ed8e5e3f08ab9cf1f Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Fri, 20 Jun 2014 12:22:31 -0400 Subject: [PATCH 006/517] Escape control and nonprintable characters in docker ps The docker ps command displays the user-entered command running in a container. If that command contained \n, \t, or other control characters, they were interpreted literally, and newlines and tabs would be printed in the output. Escape the command string to make things more readable. Docker-DCO-1.1-Signed-off-by: Matt Heon (github: mheon) --- api/client/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client/commands.go b/api/client/commands.go index 0cdf3f1ac..8f76b6e26 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1476,6 +1476,7 @@ func (cli *DockerCli) CmdPs(args ...string) error { outCommand = out.Get("Command") ports = engine.NewTable("", 0) ) + outCommand = strconv.Quote(outCommand) if !*noTrunc { outCommand = utils.Trunc(outCommand, 20) } From 21e258d7328e56fd451e6ed0324cff094ec70798 Mon Sep 17 00:00:00 2001 From: Senthil Kumar Selvaraj Date: Tue, 24 Jun 2014 11:57:24 +0530 Subject: [PATCH 007/517] Fix typo in README.md --- daemon/graphdriver/devmapper/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index c8ab1d1ee..d97baaeab 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -17,7 +17,7 @@ stored in the `$graph/devicemapper/json` file (encoded as Json). In order to support multiple devicemapper graphs on a system the thin pool will be named something like: `docker-0:33-19478248-pool`, where -the `0:30` part is the minor/major device nr and `19478248` is the +the `0:33` part is the minor/major device nr and `19478248` is the inode number of the $graph directory. On the thin pool docker automatically creates a base thin device, From 6e74754a504e80378ed56ec765d62b762f80fcb4 Mon Sep 17 00:00:00 2001 From: Jonathan Camp Date: Mon, 26 May 2014 09:40:24 +0200 Subject: [PATCH 008/517] Replace dashes in link name with underscores Docker-DCO-1.1-Signed-off-by: Jonathan Camp (github: kung-foo) --- links/links.go | 2 +- links/links_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/links/links.go b/links/links.go index 7665a06a1..45c1935c2 100644 --- a/links/links.go +++ b/links/links.go @@ -49,7 +49,7 @@ func (l *Link) Alias() string { func (l *Link) ToEnv() []string { env := []string{} - alias := strings.ToUpper(l.Alias()) + alias := strings.Replace(strings.ToUpper(l.Alias()), "-", "_", -1) if p := l.getDefaultPort(); p != nil { env = append(env, fmt.Sprintf("%s_PORT=%s://%s:%s", alias, p.Proto(), l.ChildIP, p.Port())) diff --git a/links/links_test.go b/links/links_test.go index e66f9bfb7..e79ef1a13 100644 --- a/links/links_test.go +++ b/links/links_test.go @@ -6,6 +6,36 @@ import ( "testing" ) +func TestLinkNaming(t *testing.T) { + ports := make(nat.PortSet) + ports[nat.Port("6379/tcp")] = struct{}{} + + link, err := NewLink("172.0.17.3", "172.0.17.2", "/db/docker-1", nil, ports, nil) + if err != nil { + t.Fatal(err) + } + + rawEnv := link.ToEnv() + env := make(map[string]string, len(rawEnv)) + for _, e := range rawEnv { + parts := strings.Split(e, "=") + if len(parts) != 2 { + t.FailNow() + } + env[parts[0]] = parts[1] + } + + value, ok := env["DOCKER_1_PORT"] + + if !ok { + t.Fatalf("DOCKER_1_PORT not found in env") + } + + if value != "tcp://172.0.17.2:6379" { + t.Fatalf("Expected 172.0.17.2:6379, got %s", env["DOCKER_1_PORT"]) + } +} + func TestLinkNew(t *testing.T) { ports := make(nat.PortSet) ports[nat.Port("6379/tcp")] = struct{}{} From d420134fa2ea8ceedf2a9a64263d09b308263c2c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:45:42 -0400 Subject: [PATCH 009/517] devmapper: fix reloadPool() to also specify '1 skip_block_zeroing' createPool() and reloadPool() should be consistent with the thin-pool table params they use. Since createPool() specifies '1 skip_block_zeroing' reloadPool() should too. Otherwise, if the pool is reloaded (as is done when resizing loopback devices) block zeroing will be enabled after the reload completes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/devmapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/devmapper.go b/daemon/graphdriver/devmapper/devmapper.go index a6602c276..2590ec0fc 100644 --- a/daemon/graphdriver/devmapper/devmapper.go +++ b/daemon/graphdriver/devmapper/devmapper.go @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768" + params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From 2470a5ed999d9dc8d4b9bd250727ac13964db5b3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 23 Jun 2014 15:36:08 -0400 Subject: [PATCH 010/517] devmapper: use RAMInBytes() rather than FromHumanSize() Device Mapper needs device sizes in binary (1024) multiples. Otherwise kernel checks can find that the specified thin-pool device sizes aren't a multiple of the specified thin-pool blocksize. The name for "RAMInBytes" is likely too narrow given the new consumers but... Also add "tebibyte" support to RAMInBytes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 6 +++--- pkg/units/size.go | 8 +++++--- pkg/units/size_test.go | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index 31c3f391e..b0e0819ba 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -1170,19 +1170,19 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error key = strings.ToLower(key) switch key { case "dm.basesize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.baseFsSize = uint64(size) case "dm.loopdatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.dataLoopbackSize = size case "dm.loopmetadatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } diff --git a/pkg/units/size.go b/pkg/units/size.go index 480ec2f14..e0eec3ec7 100644 --- a/pkg/units/size.go +++ b/pkg/units/size.go @@ -58,11 +58,11 @@ func FromHumanSize(size string) (int64, error) { } // Parses a human-readable string representing an amount of RAM -// in bytes, kibibytes, mebibytes or gibibytes, and returns the -// number of bytes, or -1 if the string is unparseable. +// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and +// returns the number of bytes, or -1 if the string is unparseable. // Units are case-insensitive, and the 'b' suffix is optional. func RAMInBytes(size string) (bytes int64, err error) { - re, error := regexp.Compile("^(\\d+)([kKmMgG])?[bB]?$") + re, error := regexp.Compile("^(\\d+)([kKmMgGtT])?[bB]?$") if error != nil { return -1, error } @@ -86,6 +86,8 @@ func RAMInBytes(size string) (bytes int64, err error) { memLimit *= 1024 * 1024 } else if unit == "g" { memLimit *= 1024 * 1024 * 1024 + } else if unit == "t" { + memLimit *= 1024 * 1024 * 1024 * 1024 } return memLimit, nil diff --git a/pkg/units/size_test.go b/pkg/units/size_test.go index 5240bbd9f..2c4982b16 100644 --- a/pkg/units/size_test.go +++ b/pkg/units/size_test.go @@ -64,7 +64,10 @@ func TestRAMInBytes(t *testing.T) { assertRAMInBytes(t, "32kb", false, 32*1024) assertRAMInBytes(t, "32Kb", false, 32*1024) assertRAMInBytes(t, "32Mb", false, 32*1024*1024) + assertRAMInBytes(t, "32MB", false, 32*1024*1024) assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32G", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32Tb", false, 32*1024*1024*1024*1024) assertRAMInBytes(t, "", true, -1) assertRAMInBytes(t, "hello", true, -1) From 09ee269d998ad04733ef577739fa051df9d3f12e Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:53:18 -0400 Subject: [PATCH 011/517] devmapper: Add option for specifying the thin pool blocksize Add dm.blocksize option that you can use with --storage-opt to set a specific blocksize for the thin provisioning pool. Also change the default dm-thin-pool blocksize from 64K to 512K. This strikes a balance between the desire to have smaller blocksize given docker's use of snapshots versus the desire to have more performance that comes with using a larger blocksize. But if very small files will be used on average the user is encouraged to override this default. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/README.md | 8 ++++++++ daemon/graphdriver/devmapper/deviceset.go | 14 ++++++++++++-- daemon/graphdriver/devmapper/devmapper.go | 8 ++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index c8ab1d1ee..849cfce64 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -126,6 +126,14 @@ Here is the list of supported options: ``docker -d --storage-opt dm.datadev=/dev/sdb1 --storage-opt dm.metadatadev=/dev/sdc1`` + * `dm.blocksize` + + Specifies a custom blocksize to use for the thin pool. + + Example use: + + ``docker -d --storage-opt dm.blocksize=64K`` + * `dm.blkdiscard` Enables or disables the use of blkdiscard when removing diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index b0e0819ba..c42d9c5a7 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -28,6 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { @@ -78,6 +79,7 @@ type DeviceSet struct { dataDevice string metadataDevice string doBlkDiscard bool + thinpBlockSize uint32 } type DiskUsage struct { @@ -510,7 +512,7 @@ func (devices *DeviceSet) ResizePool(size int64) error { } // Reload with the new block sizes - if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil { + if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { return fmt.Errorf("Unable to reload pool: %s", err) } @@ -640,7 +642,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { } defer metadataFile.Close() - if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil { + if err := createPool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { return err } } @@ -1159,6 +1161,7 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error baseFsSize: DefaultBaseFsSize, filesystem: "ext4", doBlkDiscard: true, + thinpBlockSize: DefaultThinpBlockSize, } foundBlkDiscard := false @@ -1206,6 +1209,13 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error if err != nil { return nil, err } + case "dm.blocksize": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + // convert to 512b sectors + devices.thinpBlockSize = uint32(size) >> 9 default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/daemon/graphdriver/devmapper/devmapper.go b/daemon/graphdriver/devmapper/devmapper.go index 2590ec0fc..ee4f2a815 100644 --- a/daemon/graphdriver/devmapper/devmapper.go +++ b/daemon/graphdriver/devmapper/devmapper.go @@ -328,7 +328,7 @@ func BlockDeviceDiscard(path string) error { } // This is the programmatic example of "dmsetup create" -func createPool(poolName string, dataFile, metadataFile *os.File) error { +func createPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceCreate, poolName) if task == nil { return err @@ -339,7 +339,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } @@ -358,7 +358,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return nil } -func reloadPool(poolName string, dataFile, metadataFile *os.File) error { +func reloadPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceReload, poolName) if task == nil { return err @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From f9c078ef38106d00de7774374f3ac71bb0d562d3 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 24 Jun 2014 12:43:45 -0400 Subject: [PATCH 012/517] devmapper: remove extra space in DefaultThinpBlockSize assignment Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/deviceset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/deviceset.go b/daemon/graphdriver/devmapper/deviceset.go index c42d9c5a7..61ba81852 100644 --- a/daemon/graphdriver/devmapper/deviceset.go +++ b/daemon/graphdriver/devmapper/deviceset.go @@ -28,7 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { From 79f217e3501a43326bd532e8eb03182f7488f61c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:06:41 -0400 Subject: [PATCH 013/517] devmapper: document the default DM thin pool blocksize Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/daemon/graphdriver/devmapper/README.md b/daemon/graphdriver/devmapper/README.md index 849cfce64..ed99d85a9 100644 --- a/daemon/graphdriver/devmapper/README.md +++ b/daemon/graphdriver/devmapper/README.md @@ -128,7 +128,8 @@ Here is the list of supported options: * `dm.blocksize` - Specifies a custom blocksize to use for the thin pool. + Specifies a custom blocksize to use for the thin pool. The default + blocksize is 512K. Example use: From a2f3ce2294bbe998df24edd50f2b571d0b21bac8 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:39:16 -0400 Subject: [PATCH 014/517] devmapper: add thin-pool blocksize to the 'docker info' output Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) --- daemon/graphdriver/devmapper/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/graphdriver/devmapper/driver.go b/daemon/graphdriver/devmapper/driver.go index cf82ad62e..29335377f 100644 --- a/daemon/graphdriver/devmapper/driver.go +++ b/daemon/graphdriver/devmapper/driver.go @@ -54,6 +54,7 @@ func (d *Driver) Status() [][2]string { status := [][2]string{ {"Pool Name", s.PoolName}, + {"Pool Blocksize", fmt.Sprintf("%d Kb", s.SectorSize/1024)}, {"Data file", s.DataLoopback}, {"Metadata file", s.MetadataLoopback}, {"Data Space Used", fmt.Sprintf("%.1f Mb", float64(s.Data.Used)/(1024*1024))}, From e4742dbb942b9db5f91dfbfb829525bdcac57ae1 Mon Sep 17 00:00:00 2001 From: Jason Giedymin Date: Fri, 20 Jun 2014 07:39:28 -0400 Subject: [PATCH 015/517] Add H3 menu entries for leftnav: - Add H3 formatting and entries to leftnav, to docs css. Docker-DCO-1.1-Signed-off-by: Jason Giedymin (github: JasonGiedymin) --- docs/theme/mkdocs/css/docs.css | 12 ++++++++++++ docs/theme/mkdocs/css/main.css | 2 +- docs/theme/mkdocs/toc.html | 9 ++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/theme/mkdocs/css/docs.css b/docs/theme/mkdocs/css/docs.css index 0f42e22ef..52f170c14 100644 --- a/docs/theme/mkdocs/css/docs.css +++ b/docs/theme/mkdocs/css/docs.css @@ -24,6 +24,18 @@ height: 100%; } +#leftnav h3 { + font-size: 10px; + font-weight: 700; + color: #394d54; + line-height: 1; + margin: 0px 0 10px 0; + padding-left: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + .content-body { padding: 0px 0px 0px 20px; } diff --git a/docs/theme/mkdocs/css/main.css b/docs/theme/mkdocs/css/main.css index 42a7a18a5..b8fc3b48a 100644 --- a/docs/theme/mkdocs/css/main.css +++ b/docs/theme/mkdocs/css/main.css @@ -902,7 +902,7 @@ div + .form-inline { margin-bottom: 22px; } #leftnav .nav { - margin: 0; + margin: 0, 0, 20px, 0; } #leftnav .nav > li > a { line-height: 22px; diff --git a/docs/theme/mkdocs/toc.html b/docs/theme/mkdocs/toc.html index 96d15c265..1de2a4209 100644 --- a/docs/theme/mkdocs/toc.html +++ b/docs/theme/mkdocs/toc.html @@ -1,5 +1,8 @@ - {% for toc_item in toc %} - {% for toc_item in toc_item.children %} -
  • {{ toc_item.title }}
  • +{% for toc_item in toc %} + {% for toc_h2_item in toc_item.children %} +
  • {{ toc_h2_item.title }}
  • + {% for toc_h3_item in toc_h2_item.children %} +

    {{ toc_h3_item.title }}

    + {% endfor %} {% endfor %} {% endfor %} From 86f06b6deb6e5b9ddb30f781e81cc5b6241e20ae Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sat, 28 Jun 2014 23:09:23 -0600 Subject: [PATCH 016/517] Update bash completion for "docker run --link" and "docker run -v" to complete on running containers and the local filesystem respectively Also, add completion for "docker run -a" ("stdin", "stdout", and "stderr"), "docker run --env-file" (local filesystem), and some other minor code style tweaks. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- contrib/completion/bash/docker | 37 +++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 89395560f..532c2538b 100755 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -485,21 +485,52 @@ _docker_rmi() _docker_run() { case "$prev" in - --cidfile) + -a|--attach) + COMPREPLY=( $( compgen -W 'stdin stdout stderr' -- "$cur" ) ) + return + ;; + --cidfile|--env-file) _filedir + return ;; --volumes-from) __docker_containers_all + return ;; -v|--volume) - # TODO something magical with colons and _filedir ? + case "$cur" in + *:*) + # TODO somehow do _filedir for stuff inside the image, if it's already specified (which is also somewhat difficult to determine) + ;; + '') + COMPREPLY=( $( compgen -W '/' -- "$cur" ) ) + compopt -o nospace + ;; + /*) + _filedir + compopt -o nospace + ;; + esac return ;; -e|--env) COMPREPLY=( $( compgen -e -- "$cur" ) ) + compopt -o nospace return ;; - --entrypoint|-h|--hostname|-m|--memory|-u|--user|-w|--workdir|-c|--cpu-shares|-n|--name|-a|--attach|--link|-p|--publish|--expose|--dns|--lxc-conf) + --link) + case "$cur" in + *:*) + ;; + *) + __docker_containers_running + COMPREPLY=( $( compgen -W "${COMPREPLY[*]}" -S ':' ) ) + compopt -o nospace + ;; + esac + return + ;; + --entrypoint|-h|--hostname|-m|--memory|-u|--user|-w|--workdir|-c|--cpu-shares|-n|--name|-p|--publish|--expose|--dns|--lxc-conf) return ;; *) From dda0ce645995d6dd2b6eb5848c30d305a3d220d2 Mon Sep 17 00:00:00 2001 From: Kato Kazuyoshi Date: Sun, 29 Jun 2014 16:50:53 +0900 Subject: [PATCH 017/517] Skip lxc_template_unit_test.go on non-Linux platforms It doesn't work without lxc-start. Docker-DCO-1.1-Signed-off-by: Kato Kazuyoshi (github: kzys) --- daemon/execdriver/lxc/lxc_template_unit_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/execdriver/lxc/lxc_template_unit_test.go b/daemon/execdriver/lxc/lxc_template_unit_test.go index a9a67c421..9e4d33f9e 100644 --- a/daemon/execdriver/lxc/lxc_template_unit_test.go +++ b/daemon/execdriver/lxc/lxc_template_unit_test.go @@ -1,3 +1,5 @@ +// +build linux + package lxc import ( From da853e98c94c9ce8001d3d1516ba0fd2614db3b0 Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Mon, 30 Jun 2014 10:52:11 +0400 Subject: [PATCH 018/517] Add synchronization in server.Containers This fixes races between ps and other operations Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- server/server.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 0bbb9f31c..8be4dcd7b 100644 --- a/server/server.go +++ b/server/server.go @@ -23,6 +23,7 @@ package server import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -955,22 +956,25 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { } } - for _, container := range srv.daemon.List() { + errLast := errors.New("last container") + writeCont := func(container *daemon.Container) error { + container.Lock() + defer container.Unlock() if !container.State.IsRunning() && !all && n <= 0 && since == "" && before == "" { - continue + return nil } if before != "" && !foundBefore { if container.ID == beforeCont.ID { foundBefore = true } - continue + return nil } if n > 0 && displayed == n { - break + return errLast } if since != "" { if container.ID == sinceCont.ID { - break + return errLast } } displayed++ @@ -997,7 +1001,7 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { out.Set("Status", container.State.String()) str, err := container.NetworkSettings.PortMappingAPI().ToListString() if err != nil { - return job.Error(err) + return err } out.Set("Ports", str) if size { @@ -1006,6 +1010,16 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { out.SetInt64("SizeRootFs", sizeRootFs) } outs.Add(out) + return nil + } + + for _, container := range srv.daemon.List() { + if err := writeCont(container); err != nil { + if err != errLast { + return job.Error(err) + } + break + } } outs.ReverseSort() if _, err := outs.WriteListTo(job.Stdout); err != nil { From f387cc1205b79ffbe30b28f5df13da034d8221fd Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Tue, 1 Jul 2014 21:59:11 +0400 Subject: [PATCH 019/517] Refactoring portallocator Faster, more documented, less code. Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- .../portallocator/portallocator.go | 138 ++++++++---------- 1 file changed, 63 insertions(+), 75 deletions(-) diff --git a/daemon/networkdriver/portallocator/portallocator.go b/daemon/networkdriver/portallocator/portallocator.go index c722ba98b..d4fcc6e72 100644 --- a/daemon/networkdriver/portallocator/portallocator.go +++ b/daemon/networkdriver/portallocator/portallocator.go @@ -12,10 +12,22 @@ type portMap struct { last int } -type ( - protocolMap map[string]*portMap - ipMapping map[string]protocolMap -) +func newPortMap() *portMap { + return &portMap{ + p: map[int]struct{}{}, + } +} + +type protoMap map[string]*portMap + +func newProtoMap() protoMap { + return protoMap{ + "tcp": newPortMap(), + "udp": newPortMap(), + } +} + +type ipMapping map[string]protoMap const ( BeginPortRange = 49153 @@ -62,107 +74,83 @@ func (e ErrPortAlreadyAllocated) Error() string { return fmt.Sprintf("Bind for %s:%d failed: port is already allocated", e.ip, e.port) } +// RequestPort requests new port from global ports pool for specified ip and proto. +// If port is 0 it returns first free port. Otherwise it cheks port availability +// in pool and return that port or error if port is already busy. func RequestPort(ip net.IP, proto string, port int) (int, error) { mutex.Lock() defer mutex.Unlock() - if err := validateProto(proto); err != nil { + if proto != "tcp" && proto != "udp" { + return 0, ErrUnknownProtocol + } + + if ip == nil { + ip = defaultIP + } + ipstr := ip.String() + protomap, ok := globalMap[ipstr] + if !ok { + protomap = newProtoMap() + globalMap[ipstr] = protomap + } + mapping := protomap[proto] + if port > 0 { + if _, ok := mapping.p[port]; !ok { + mapping.p[port] = struct{}{} + return port, nil + } + return 0, NewErrPortAlreadyAllocated(ipstr, port) + } + + port, err := mapping.findPort() + if err != nil { return 0, err } - - ip = getDefault(ip) - - mapping := getOrCreate(ip) - - if port > 0 { - if _, ok := mapping[proto].p[port]; !ok { - mapping[proto].p[port] = struct{}{} - return port, nil - } else { - return 0, NewErrPortAlreadyAllocated(ip.String(), port) - } - } else { - port, err := findPort(ip, proto) - - if err != nil { - return 0, err - } - - return port, nil - } + return port, nil } +// ReleasePort releases port from global ports pool for specified ip and proto. func ReleasePort(ip net.IP, proto string, port int) error { mutex.Lock() defer mutex.Unlock() - ip = getDefault(ip) - - mapping := getOrCreate(ip)[proto] - delete(mapping.p, port) - + if ip == nil { + ip = defaultIP + } + protomap, ok := globalMap[ip.String()] + if !ok { + return nil + } + delete(protomap[proto].p, port) return nil } +// ReleaseAll releases all ports for all ips. func ReleaseAll() error { mutex.Lock() - defer mutex.Unlock() - globalMap = ipMapping{} - + mutex.Unlock() return nil } -func getOrCreate(ip net.IP) protocolMap { - ipstr := ip.String() - - if _, ok := globalMap[ipstr]; !ok { - globalMap[ipstr] = protocolMap{ - "tcp": &portMap{p: map[int]struct{}{}, last: 0}, - "udp": &portMap{p: map[int]struct{}{}, last: 0}, - } - } - - return globalMap[ipstr] -} - -func findPort(ip net.IP, proto string) (int, error) { - mapping := getOrCreate(ip)[proto] - - if mapping.last == 0 { - mapping.p[BeginPortRange] = struct{}{} - mapping.last = BeginPortRange +func (pm *portMap) findPort() (int, error) { + if pm.last == 0 { + pm.p[BeginPortRange] = struct{}{} + pm.last = BeginPortRange return BeginPortRange, nil } - for port := mapping.last + 1; port != mapping.last; port++ { + for port := pm.last + 1; port != pm.last; port++ { if port > EndPortRange { port = BeginPortRange } - if _, ok := mapping.p[port]; !ok { - mapping.p[port] = struct{}{} - mapping.last = port + if _, ok := pm.p[port]; !ok { + pm.p[port] = struct{}{} + pm.last = port return port, nil } - } - return 0, ErrAllPortsAllocated } - -func getDefault(ip net.IP) net.IP { - if ip == nil { - return defaultIP - } - - return ip -} - -func validateProto(proto string) error { - if proto != "tcp" && proto != "udp" { - return ErrUnknownProtocol - } - - return nil -} From 9d4e80222144c0aef3e78202ecb4904b3ce5ed9b Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 18:48:37 +0400 Subject: [PATCH 020/517] Move WriteBroadcaster to separate package as BroadcastWriter Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- daemon/container.go | 9 +- daemon/daemon.go | 7 +- utils/broadcastwriter/broadcastwriter.go | 92 +++++++++++++++ utils/broadcastwriter/broadcastwriter_test.go | 108 ++++++++++++++++++ utils/utils.go | 86 -------------- utils/utils_test.go | 101 ---------------- 6 files changed, 209 insertions(+), 194 deletions(-) create mode 100644 utils/broadcastwriter/broadcastwriter.go create mode 100644 utils/broadcastwriter/broadcastwriter_test.go diff --git a/daemon/container.go b/daemon/container.go index 5b4143868..c7a677460 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -29,6 +29,7 @@ import ( "github.com/dotcloud/docker/pkg/symlink" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" + "github.com/dotcloud/docker/utils/broadcastwriter" ) const DefaultPathEnv = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -66,8 +67,8 @@ type Container struct { ExecDriver string command *execdriver.Command - stdout *utils.WriteBroadcaster - stderr *utils.WriteBroadcaster + stdout *broadcastwriter.BroadcastWriter + stderr *broadcastwriter.BroadcastWriter stdin io.ReadCloser stdinPipe io.WriteCloser @@ -502,10 +503,10 @@ func (container *Container) cleanup() { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.CloseWriters(); err != nil { + if err := container.stdout.Close(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.CloseWriters(); err != nil { + if err := container.stderr.Close(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } if container.command != nil && container.command.Terminal != nil { diff --git a/daemon/daemon.go b/daemon/daemon.go index 9500526f9..0b51715ec 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -34,6 +34,7 @@ import ( "github.com/dotcloud/docker/pkg/truncindex" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" + "github.com/dotcloud/docker/utils/broadcastwriter" ) // Set the max depth to the aufs default that most @@ -169,8 +170,8 @@ func (daemon *Daemon) register(container *Container, updateSuffixarray bool, con container.daemon = daemon // Attach to stdout and stderr - container.stderr = utils.NewWriteBroadcaster() - container.stdout = utils.NewWriteBroadcaster() + container.stderr = broadcastwriter.New() + container.stdout = broadcastwriter.New() // Attach to stdin if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() @@ -255,7 +256,7 @@ func (daemon *Daemon) ensureName(container *Container) error { return nil } -func (daemon *Daemon) LogToDisk(src *utils.WriteBroadcaster, dst, stream string) error { +func (daemon *Daemon) LogToDisk(src *broadcastwriter.BroadcastWriter, dst, stream string) error { log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) if err != nil { return err diff --git a/utils/broadcastwriter/broadcastwriter.go b/utils/broadcastwriter/broadcastwriter.go new file mode 100644 index 000000000..81ca9e5bd --- /dev/null +++ b/utils/broadcastwriter/broadcastwriter.go @@ -0,0 +1,92 @@ +package broadcastwriter + +import ( + "bytes" + "encoding/json" + "io" + "sync" + "time" + + "github.com/dotcloud/docker/utils" +) + +type BroadcastWriter struct { + sync.Mutex + buf *bytes.Buffer + streams map[string](map[io.WriteCloser]struct{}) +} + +func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { + w.Lock() + if _, ok := w.streams[stream]; !ok { + w.streams[stream] = make(map[io.WriteCloser]struct{}) + } + w.streams[stream][writer] = struct{}{} + w.Unlock() +} + +func (w *BroadcastWriter) Write(p []byte) (n int, err error) { + created := time.Now().UTC() + w.Lock() + defer w.Unlock() + if writers, ok := w.streams[""]; ok { + for sw := range writers { + if n, err := sw.Write(p); err != nil || n != len(p) { + // On error, evict the writer + delete(writers, sw) + } + } + } + w.buf.Write(p) + lines := []string{} + for { + line, err := w.buf.ReadString('\n') + if err != nil { + w.buf.Write([]byte(line)) + break + } + lines = append(lines, line) + } + + if len(lines) != 0 { + for stream, writers := range w.streams { + if stream == "" { + continue + } + var lp []byte + for _, line := range lines { + b, err := json.Marshal(&utils.JSONLog{Log: line, Stream: stream, Created: created}) + if err != nil { + utils.Errorf("Error making JSON log line: %s", err) + } + lp = append(lp, b...) + lp = append(lp, '\n') + } + for sw := range writers { + if _, err := sw.Write(lp); err != nil { + delete(writers, sw) + } + } + } + } + return len(p), nil +} + +func (w *BroadcastWriter) Close() error { + w.Lock() + defer w.Unlock() + for _, writers := range w.streams { + for w := range writers { + w.Close() + } + } + w.streams = make(map[string](map[io.WriteCloser]struct{})) + return nil +} + +func New() *BroadcastWriter { + return &BroadcastWriter{ + streams: make(map[string](map[io.WriteCloser]struct{})), + buf: bytes.NewBuffer(nil), + } +} diff --git a/utils/broadcastwriter/broadcastwriter_test.go b/utils/broadcastwriter/broadcastwriter_test.go new file mode 100644 index 000000000..8d946e2f4 --- /dev/null +++ b/utils/broadcastwriter/broadcastwriter_test.go @@ -0,0 +1,108 @@ +package broadcastwriter + +import ( + "bytes" + "errors" + + "testing" +) + +type dummyWriter struct { + buffer bytes.Buffer + failOnWrite bool +} + +func (dw *dummyWriter) Write(p []byte) (n int, err error) { + if dw.failOnWrite { + return 0, errors.New("Fake fail") + } + return dw.buffer.Write(p) +} + +func (dw *dummyWriter) String() string { + return dw.buffer.String() +} + +func (dw *dummyWriter) Close() error { + return nil +} + +func TestBroadcastWriter(t *testing.T) { + writer := New() + + // Test 1: Both bufferA and bufferB should contain "foo" + bufferA := &dummyWriter{} + writer.AddWriter(bufferA, "") + bufferB := &dummyWriter{} + writer.AddWriter(bufferB, "") + writer.Write([]byte("foo")) + + if bufferA.String() != "foo" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + + if bufferB.String() != "foo" { + t.Errorf("Buffer contains %v", bufferB.String()) + } + + // Test2: bufferA and bufferB should contain "foobar", + // while bufferC should only contain "bar" + bufferC := &dummyWriter{} + writer.AddWriter(bufferC, "") + writer.Write([]byte("bar")) + + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + + if bufferB.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferB.String()) + } + + if bufferC.String() != "bar" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + + // Test3: Test eviction on failure + bufferA.failOnWrite = true + writer.Write([]byte("fail")) + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + if bufferC.String() != "barfail" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + // Even though we reset the flag, no more writes should go in there + bufferA.failOnWrite = false + writer.Write([]byte("test")) + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + if bufferC.String() != "barfailtest" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + + writer.Close() +} + +type devNullCloser int + +func (d devNullCloser) Close() error { + return nil +} + +func (d devNullCloser) Write(buf []byte) (int, error) { + return len(buf), nil +} + +// This test checks for races. It is only useful when run with the race detector. +func TestRaceBroadcastWriter(t *testing.T) { + writer := New() + c := make(chan bool) + go func() { + writer.AddWriter(devNullCloser(0), "") + c <- true + }() + writer.Write([]byte("hello")) + <-c +} diff --git a/utils/utils.go b/utils/utils.go index 333468d36..ef28aceca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -265,21 +265,6 @@ func (r *bufReader) Close() error { return closer.Close() } -type WriteBroadcaster struct { - sync.Mutex - buf *bytes.Buffer - streams map[string](map[io.WriteCloser]struct{}) -} - -func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser, stream string) { - w.Lock() - if _, ok := w.streams[stream]; !ok { - w.streams[stream] = make(map[io.WriteCloser]struct{}) - } - w.streams[stream][writer] = struct{}{} - w.Unlock() -} - type JSONLog struct { Log string `json:"log,omitempty"` Stream string `json:"stream,omitempty"` @@ -316,77 +301,6 @@ func WriteLog(src io.Reader, dst io.WriteCloser, format string) error { } } -type LogFormatter struct { - wc io.WriteCloser - timeFormat string -} - -func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { - created := time.Now().UTC() - w.Lock() - defer w.Unlock() - if writers, ok := w.streams[""]; ok { - for sw := range writers { - if n, err := sw.Write(p); err != nil || n != len(p) { - // On error, evict the writer - delete(writers, sw) - } - } - } - w.buf.Write(p) - lines := []string{} - for { - line, err := w.buf.ReadString('\n') - if err != nil { - w.buf.Write([]byte(line)) - break - } - lines = append(lines, line) - } - - if len(lines) != 0 { - for stream, writers := range w.streams { - if stream == "" { - continue - } - var lp []byte - for _, line := range lines { - b, err := json.Marshal(&JSONLog{Log: line, Stream: stream, Created: created}) - if err != nil { - Errorf("Error making JSON log line: %s", err) - } - lp = append(lp, b...) - lp = append(lp, '\n') - } - for sw := range writers { - if _, err := sw.Write(lp); err != nil { - delete(writers, sw) - } - } - } - } - return len(p), nil -} - -func (w *WriteBroadcaster) CloseWriters() error { - w.Lock() - defer w.Unlock() - for _, writers := range w.streams { - for w := range writers { - w.Close() - } - } - w.streams = make(map[string](map[io.WriteCloser]struct{})) - return nil -} - -func NewWriteBroadcaster() *WriteBroadcaster { - return &WriteBroadcaster{ - streams: make(map[string](map[io.WriteCloser]struct{})), - buf: bytes.NewBuffer(nil), - } -} - func GetTotalUsedFds() int { if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) diff --git a/utils/utils_test.go b/utils/utils_test.go index 049c0e30a..6fd09b97d 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -2,7 +2,6 @@ package utils import ( "bytes" - "errors" "io" "io/ioutil" "os" @@ -35,106 +34,6 @@ func TestBufReader(t *testing.T) { } } -type dummyWriter struct { - buffer bytes.Buffer - failOnWrite bool -} - -func (dw *dummyWriter) Write(p []byte) (n int, err error) { - if dw.failOnWrite { - return 0, errors.New("Fake fail") - } - return dw.buffer.Write(p) -} - -func (dw *dummyWriter) String() string { - return dw.buffer.String() -} - -func (dw *dummyWriter) Close() error { - return nil -} - -func TestWriteBroadcaster(t *testing.T) { - writer := NewWriteBroadcaster() - - // Test 1: Both bufferA and bufferB should contain "foo" - bufferA := &dummyWriter{} - writer.AddWriter(bufferA, "") - bufferB := &dummyWriter{} - writer.AddWriter(bufferB, "") - writer.Write([]byte("foo")) - - if bufferA.String() != "foo" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foo" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - // Test2: bufferA and bufferB should contain "foobar", - // while bufferC should only contain "bar" - bufferC := &dummyWriter{} - writer.AddWriter(bufferC, "") - writer.Write([]byte("bar")) - - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - if bufferC.String() != "bar" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - // Test3: Test eviction on failure - bufferA.failOnWrite = true - writer.Write([]byte("fail")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfail" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - // Even though we reset the flag, no more writes should go in there - bufferA.failOnWrite = false - writer.Write([]byte("test")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfailtest" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - writer.CloseWriters() -} - -type devNullCloser int - -func (d devNullCloser) Close() error { - return nil -} - -func (d devNullCloser) Write(buf []byte) (int, error) { - return len(buf), nil -} - -// This test checks for races. It is only useful when run with the race detector. -func TestRaceWriteBroadcaster(t *testing.T) { - writer := NewWriteBroadcaster() - c := make(chan bool) - go func() { - writer.AddWriter(devNullCloser(0), "") - c <- true - }() - writer.Write([]byte("hello")) - <-c -} - func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) { if r := CompareKernelVersion(a, b); r != result { t.Fatalf("Unexpected kernel version comparison result. Found %d, expected %d", r, result) From a4a80b64a7f6366f7fb94cb8cca042a6324a0e99 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 17 Mar 2014 17:03:22 +0100 Subject: [PATCH 021/517] server: Break out setHostConfig() from ContainerStart This will be reused for ContainerCreate Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) --- server/server.go | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/server/server.go b/server/server.go index 3e6de00c1..b443eba32 100644 --- a/server/server.go +++ b/server/server.go @@ -2035,6 +2035,32 @@ func (srv *Server) ImageGetCached(imgID string, config *runconfig.Config) (*imag return match, nil } +func (srv *Server) setHostConfig(container *daemon.Container, hostConfig *runconfig.HostConfig) error { + // Validate the HostConfig binds. Make sure that: + // the source exists + for _, bind := range hostConfig.Binds { + splitBind := strings.Split(bind, ":") + source := splitBind[0] + + // ensure the source exists on the host + _, err := os.Stat(source) + if err != nil && os.IsNotExist(err) { + err = os.MkdirAll(source, 0755) + if err != nil { + return fmt.Errorf("Could not create local directory '%s' for bind mount: %s!", source, err.Error()) + } + } + } + // Register any links from the host config before starting the container + if err := srv.daemon.RegisterLinks(container, hostConfig); err != nil { + return err + } + container.SetHostConfig(hostConfig) + container.ToDisk() + + return nil +} + func (srv *Server) ContainerStart(job *engine.Job) engine.Status { if len(job.Args) < 1 { return job.Errorf("Usage: %s container_id", job.Name) @@ -2056,27 +2082,9 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { // If no environment was set, then no hostconfig was passed. if len(job.Environ()) > 0 { hostConfig := runconfig.ContainerHostConfigFromJob(job) - // Validate the HostConfig binds. Make sure that: - // the source exists - for _, bind := range hostConfig.Binds { - splitBind := strings.Split(bind, ":") - source := splitBind[0] - - // ensure the source exists on the host - _, err := os.Stat(source) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(source, 0755) - if err != nil { - return job.Errorf("Could not create local directory '%s' for bind mount: %s!", source, err.Error()) - } - } - } - // Register any links from the host config before starting the container - if err := srv.daemon.RegisterLinks(container, hostConfig); err != nil { + if err := srv.setHostConfig(container, hostConfig); err != nil { return job.Error(err) } - container.SetHostConfig(hostConfig) - container.ToDisk() } if err := container.Start(); err != nil { return job.Errorf("Cannot start container %s: %s", name, err) From 2b3959c414ffe9403e53617ed1a160c1daabeeaa Mon Sep 17 00:00:00 2001 From: Mrunal Patel Date: Wed, 2 Jul 2014 16:33:14 -0400 Subject: [PATCH 022/517] api.DockerCli: Extract pullImage into separate function This lets us reuse this code later. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) --- api/client/commands.go | 66 +++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/api/client/commands.go b/api/client/commands.go index 2d9c74f43..93470c291 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1894,6 +1894,41 @@ func (cli *DockerCli) CmdTag(args ...string) error { return nil } +func (cli *DockerCli) pullImage(image string) error { + v := url.Values{} + repos, tag := utils.ParseRepositoryTag(image) + // pull only the image tagged 'latest' if no tag was specified + if tag == "" { + tag = "latest" + } + v.Set("fromImage", repos) + v.Set("tag", tag) + + // Resolve the Repository name from fqn to hostname + name + hostname, _, err := registry.ResolveRepositoryName(repos) + if err != nil { + return err + } + + // Load the auth config file, to be able to pull the image + cli.LoadConfigFile() + + // Resolve the Auth config relevant for this server + authConfig := cli.configFile.ResolveAuthConfig(hostname) + buf, err := json.Marshal(authConfig) + if err != nil { + return err + } + + registryAuthHeader := []string{ + base64.URLEncoding.EncodeToString(buf), + } + if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { + return err + } + return nil +} + func (cli *DockerCli) CmdRun(args ...string) error { // FIXME: just use runconfig.Parse already config, hostConfig, cmd, err := runconfig.ParseSubcommand(cli.Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container"), args, nil) @@ -1955,37 +1990,10 @@ func (cli *DockerCli) CmdRun(args ...string) error { if statusCode == 404 { fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", config.Image) - v := url.Values{} - repos, tag := utils.ParseRepositoryTag(config.Image) - // pull only the image tagged 'latest' if no tag was specified - if tag == "" { - tag = "latest" - } - v.Set("fromImage", repos) - v.Set("tag", tag) - - // Resolve the Repository name from fqn to hostname + name - hostname, _, err := registry.ResolveRepositoryName(repos) - if err != nil { - return err - } - - // Load the auth config file, to be able to pull the image - cli.LoadConfigFile() - - // Resolve the Auth config relevant for this server - authConfig := cli.configFile.ResolveAuthConfig(hostname) - buf, err := json.Marshal(authConfig) - if err != nil { - return err - } - - registryAuthHeader := []string{ - base64.URLEncoding.EncodeToString(buf), - } - if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { + if err = cli.pullImage(config.Image); err != nil { return err } + // Retry if stream, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), config, false); err != nil { return err } From 9da6c80533f80060575840e1e2fd80cc11826f7b Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Fri, 27 Jun 2014 16:47:32 +0100 Subject: [PATCH 023/517] Move `docker restart` tests to integration cli Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) --- integration-cli/docker_cli_restart_test.go | 122 +++++++++++++++++++++ integration/container_test.go | 75 ------------- 2 files changed, 122 insertions(+), 75 deletions(-) create mode 100644 integration-cli/docker_cli_restart_test.go diff --git a/integration-cli/docker_cli_restart_test.go b/integration-cli/docker_cli_restart_test.go new file mode 100644 index 000000000..c62f108cc --- /dev/null +++ b/integration-cli/docker_cli_restart_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestDockerRestartStoppedContainer(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "echo", "foobar") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "wait", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\n" { + t.Errorf("container should've printed 'foobar'") + } + + runCmd = exec.Command(dockerBinary, "restart", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\nfoobar\n" { + t.Errorf("container should've printed 'foobar' twice") + } + + deleteAllContainers() + + logDone("restart - echo foobar for stopped container") +} + +func TestDockerRestartRunningContainer(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", "echo foobar && sleep 30 && echo 'should not print this'") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\n" { + t.Errorf("container should've printed 'foobar'") + } + + runCmd = exec.Command(dockerBinary, "restart", "-t", "1", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\nfoobar\n" { + t.Errorf("container should've printed 'foobar' twice") + } + + deleteAllContainers() + + logDone("restart - echo foobar for running container") +} + +// Test that restarting a container with a volume does not create a new volume on restart. Regression test for #819. +func TestDockerRestartWithVolumes(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "-v", "/test", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ len .Volumes }}", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out = strings.Trim(out, " \n\r"); out != "1" { + t.Errorf("expect 1 volume received %s", out) + } + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ .Volumes }}", cleanedContainerID) + volumes, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, volumes) + + runCmd = exec.Command(dockerBinary, "restart", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ len .Volumes }}", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out = strings.Trim(out, " \n\r"); out != "1" { + t.Errorf("expect 1 volume after restart received %s", out) + } + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ .Volumes }}", cleanedContainerID) + volumesAfterRestart, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, volumesAfterRestart) + + if volumes != volumesAfterRestart { + volumes = strings.Trim(volumes, " \n\r") + volumesAfterRestart = strings.Trim(volumesAfterRestart, " \n\r") + t.Errorf("expected volume path: %s Actual path: %s", volumes, volumesAfterRestart) + } + + deleteAllContainers() + + logDone("restart - does not create a new volume on restart") +} diff --git a/integration/container_test.go b/integration/container_test.go index 48b3321a5..e70da5f14 100644 --- a/integration/container_test.go +++ b/integration/container_test.go @@ -71,37 +71,6 @@ func TestKillDifferentUser(t *testing.T) { } } -func TestRestart(t *testing.T) { - daemon := mkDaemon(t) - defer nuke(daemon) - container, _, err := daemon.Create(&runconfig.Config{ - Image: GetTestImage(daemon).ID, - Cmd: []string{"echo", "-n", "foobar"}, - }, - "", - ) - if err != nil { - t.Fatal(err) - } - defer daemon.Destroy(container) - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "foobar" { - t.Error(string(output)) - } - - // Run the container again and check the output - output, err = container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "foobar" { - t.Error(string(output)) - } -} - func TestRestartStdin(t *testing.T) { daemon := mkDaemon(t) defer nuke(daemon) @@ -526,47 +495,3 @@ func TestBindMounts(t *testing.T) { t.Fatal("Container failed to write to bind mount file") } } - -// Test that restarting a container with a volume does not create a new volume on restart. Regression test for #819. -func TestRestartWithVolumes(t *testing.T) { - daemon := mkDaemon(t) - defer nuke(daemon) - - container, _, err := daemon.Create(&runconfig.Config{ - Image: GetTestImage(daemon).ID, - Cmd: []string{"echo", "-n", "foobar"}, - Volumes: map[string]struct{}{"/test": {}}, - }, - "", - ) - if err != nil { - t.Fatal(err) - } - defer daemon.Destroy(container) - - for key := range container.Config.Volumes { - if key != "/test" { - t.Fail() - } - } - - _, err = container.Output() - if err != nil { - t.Fatal(err) - } - - expected := container.Volumes["/test"] - if expected == "" { - t.Fail() - } - // Run the container again to verify the volume path persists - _, err = container.Output() - if err != nil { - t.Fatal(err) - } - - actual := container.Volumes["/test"] - if expected != actual { - t.Fatalf("Expected volume path: %s Actual path: %s", expected, actual) - } -} From f337a52a6ec01684c326a827adbe3d0b8b5f1f53 Mon Sep 17 00:00:00 2001 From: Adrien Folie Date: Wed, 2 Jul 2014 03:52:59 +0200 Subject: [PATCH 024/517] Move TestGetImagesByName Docker-DCO-1.1-Signed-off-by: Adrien Folie (github: folieadrien) --- api/server/server_unit_test.go | 37 ++++++++++++++++++++++ integration-cli/docker_cli_inspect_test.go | 22 +++++++++++++ integration/api_test.go | 25 --------------- 3 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 integration-cli/docker_cli_inspect_test.go diff --git a/api/server/server_unit_test.go b/api/server/server_unit_test.go index 2d14f8955..f8068a29c 100644 --- a/api/server/server_unit_test.go +++ b/api/server/server_unit_test.go @@ -319,6 +319,43 @@ func TestGetImagesHistory(t *testing.T) { } } +func TestGetImagesByName(t *testing.T) { + eng := engine.New() + name := "image_name" + var called bool + eng.Register("image_inspect", func(job *engine.Job) engine.Status { + called = true + if job.Args[0] != name { + t.Fatalf("name != '%s': %#v", name, job.Args[0]) + } + if api.APIVERSION.LessThan("1.12") && !job.GetenvBool("dirty") { + t.Fatal("dirty env variable not set") + } else if api.APIVERSION.GreaterThanOrEqualTo("1.12") && job.GetenvBool("dirty") { + t.Fatal("dirty env variable set when it shouldn't") + } + v := &engine.Env{} + v.SetBool("dirty", true) + if _, err := v.WriteTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequest("GET", "/images/"+name+"/json", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + if r.HeaderMap.Get("Content-Type") != "application/json" { + t.Fatalf("%#v\n", r) + } + var stdoutJson interface{} + if err := json.Unmarshal(r.Body.Bytes(), &stdoutJson); err != nil { + t.Fatalf("%#v", err) + } + if stdoutJson.(map[string]interface{})["dirty"].(float64) != 1 { + t.Fatalf("%#v", stdoutJson) + } +} + func serveRequest(method, target string, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { r := httptest.NewRecorder() req, err := http.NewRequest(method, target, body) diff --git a/integration-cli/docker_cli_inspect_test.go b/integration-cli/docker_cli_inspect_test.go new file mode 100644 index 000000000..7e5dd69ae --- /dev/null +++ b/integration-cli/docker_cli_inspect_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestInspectImage(t *testing.T) { + imageTest := "scratch" + imageTestId := "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158" + imagesCmd := exec.Command(dockerBinary, "inspect", "--format='{{.Id}}'", imageTest) + + out, exitCode, err := runCommandWithOutput(imagesCmd) + if exitCode != 0 || err != nil { + t.Fatalf("failed to inspect image") + } + if id := strings.TrimSuffix(out, "\n"); id != imageTestId { + t.Fatalf("Expected id: %s for image: %s but received id: %s", imageTestId, imageTest, id) + } + logDone("inspect - inspect an image") +} diff --git a/integration/api_test.go b/integration/api_test.go index b2e44717a..f5e5599e3 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -16,7 +16,6 @@ import ( "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/api/server" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" ) @@ -124,30 +123,6 @@ func TestGetImagesJSON(t *testing.T) { } } -func TestGetImagesByName(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - req, err := http.NewRequest("GET", "/images/"+unitTestImageName+"/json", nil) - if err != nil { - t.Fatal(err) - } - - r := httptest.NewRecorder() - if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { - t.Fatal(err) - } - assertHttpNotError(r, t) - - img := &image.Image{} - if err := json.Unmarshal(r.Body.Bytes(), img); err != nil { - t.Fatal(err) - } - if img.ID != unitTestImageID { - t.Errorf("Error inspecting image") - } -} - func TestGetContainersJSON(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() From d013b76e10262624c9f8ee05b9877e26eb58b516 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sun, 29 Jun 2014 18:01:11 -0400 Subject: [PATCH 025/517] Minor updates and fixes to the CONTRIBUTING doc * Fixed some uses of docker v. Docker * Formatting and line wrapping. * Spelling errors and grammar fixes. Docker-DCO-1.1-Signed-off-by: James Turnbull (github: jamtur01) --- CONTRIBUTING.md | 72 ++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d07b972eb..fd15f956a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,12 +6,17 @@ feels wrong or incomplete. ## Reporting Issues -When reporting [issues](https://github.com/dotcloud/docker/issues) -on GitHub please include your host OS (Ubuntu 12.04, Fedora 19, etc), -the output of `uname -a` and the output of `docker version` along with -the output of `docker -D info`. Please include the steps required to reproduce -the problem if possible and applicable. -This information will help us review and fix your issue faster. +When reporting [issues](https://github.com/dotcloud/docker/issues) on +GitHub please include your host OS (Ubuntu 12.04, Fedora 19, etc). +Please include: + +* The output of `uname -a`. +* The output of `docker version`. +* The output of `docker -D info`. + +Please also include the steps required to reproduce the problem if +possible and applicable. This information will help us review and fix +your issue faster. ## Build Environment @@ -34,7 +39,7 @@ received feedback on what to improve. We're trying very hard to keep Docker lean and focused. We don't want it to do everything for everybody. This means that we might decide against incorporating a new feature. However, there might be a way to implement -that feature *on top of* docker. +that feature *on top of* Docker. ### Discuss your design on the mailing list @@ -60,12 +65,12 @@ help prioritize the most common problems and requests. ### Conventions -Fork the repo and make changes on your fork in a feature branch: +Fork the repository and make changes on your fork in a feature branch: -- If it's a bugfix branch, name it XXX-something where XXX is the number of the - issue +- If it's a bug fix branch, name it XXXX-something where XXXX is the number of the + issue. - If it's a feature branch, create an enhancement issue to announce your - intentions, and name it XXX-something where XXX is the number of the issue. + intentions, and name it XXXX-something where XXXX is the number of the issue. Submit unit tests for your changes. Go has a great test framework built in; use it! Take a look at existing tests for inspiration. Run the full test suite on @@ -73,12 +78,12 @@ your branch before submitting a pull request. Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as -well as a clean documentation build. See ``docs/README.md`` for more -information on building the docs and how docs get released. +well as a clean documentation build. See `docs/README.md` for more +information on building the docs and how they get released. Write clean code. Universally formatted code promotes ease of writing, reading, and maintenance. Always run `gofmt -s -w file.go` on each changed file before -committing your changes. Most editors have plugins that do this automatically. +committing your changes. Most editors have plug-ins that do this automatically. Pull requests descriptions should be as clear as possible and include a reference to all the issues that they address. @@ -100,21 +105,22 @@ logical units of work using `git rebase -i` and `git push -f`. After every commit the test suite should be passing. Include documentation changes in the same commit so that a revert would remove all traces of the feature or fix. -Commits that fix or close an issue should include a reference like `Closes #XXX` -or `Fixes #XXX`, which will automatically close the issue when merged. +Commits that fix or close an issue should include a reference like +`Closes #XXXX` or `Fixes #XXXX`, which will automatically close the +issue when merged. -Please do not add yourself to the AUTHORS file, as it is regenerated +Please do not add yourself to the `AUTHORS` file, as it is regenerated regularly from the Git history. ### Merge approval -Docker maintainers use LGTM (looks good to me) in comments on the code review +Docker maintainers use LGTM (Looks Good To Me) in comments on the code review to indicate acceptance. A change requires LGTMs from an absolute majority of the maintainers of each -component affected. For example, if a change affects docs/ and registry/, it -needs an absolute majority from the maintainers of docs/ AND, separately, an -absolute majority of the maintainers of registry. +component affected. For example, if a change affects `docs/` and `registry/`, it +needs an absolute majority from the maintainers of `docs/` AND, separately, an +absolute majority of the maintainers of `registry/`. For more details see [MAINTAINERS.md](hack/MAINTAINERS.md) @@ -137,7 +143,6 @@ San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: @@ -165,20 +170,20 @@ By making a contribution to this project, I certify that: this project or the open source license(s) involved. ``` -then you just add a line to every git commit message: +Then you just add a line to every git commit message: Docker-DCO-1.1-Signed-off-by: Joe Smith (github: github_handle) -using your real name (sorry, no pseudonyms or anonymous contributions.) +Using your real name (sorry, no pseudonyms or anonymous contributions.) -One way to automate this, is customise your get ``commit.template`` by adding -a ``prepare-commit-msg`` hook to your docker checkout: +One way to automate this, is customize your git `commit.template` by adding +a `prepare-commit-msg` hook to your Docker repository: ``` curl -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg ``` -* Note: the above script expects to find your GitHub user name in ``git config --get github.user`` +* Note: the above script expects to find your GitHub user name in `git config --get github.user` #### Small patch exception @@ -194,11 +199,12 @@ If you have any questions, please refer to the FAQ in the [docs](http://docs.doc ### How can I become a maintainer? -* Step 1: learn the component inside out -* Step 2: make yourself useful by contributing code, bugfixes, support etc. -* Step 3: volunteer on the irc channel (#docker@freenode) -* Step 4: propose yourself at a scheduled docker meeting in #docker-dev +* Step 1: Learn the component inside out +* Step 2: Make yourself useful by contributing code, bug fixes, support etc. +* Step 3: Volunteer on the IRC channel (#docker at Freenode) +* Step 4: Propose yourself at a scheduled docker meeting in #docker-dev -Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. -You don't have to be a maintainer to make a difference on the project! +Don't forget: being a maintainer is a time investment. Make sure you +will have time to make yourself available. You don't have to be a +maintainer to make a difference on the project! From 873241790f802351d36c490046435215b33e4551 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 2 Jul 2014 21:19:48 -0700 Subject: [PATCH 026/517] Fix 2 docker.com theme regressions on docs.docker.com - Hover colours of doc nav wrong - Primary Docs Nav not active when user is Logged in. Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) --- docs/theme/mkdocs/css/docs.css | 7 ++----- docs/theme/mkdocs/header.html | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/theme/mkdocs/css/docs.css b/docs/theme/mkdocs/css/docs.css index 0f42e22ef..8d7aca2f0 100644 --- a/docs/theme/mkdocs/css/docs.css +++ b/docs/theme/mkdocs/css/docs.css @@ -56,12 +56,9 @@ ol li { #nav_menu > #docsnav > #main-nav > li > a { color: #253237; } -#nav_menu > #docsnav > #main-nav > li.dd_on_hover > a { - color: #5992a3; -} #nav_menu > #docsnav > #main-nav > li.dd_on_hover { - background: #b1d5df; - color: #5992a3; + background: #d3f1fb; + color: #253237; } #nav_menu > #docsnav > #main-nav > li > span > b { border-top-color: #b1d5df !important; diff --git a/docs/theme/mkdocs/header.html b/docs/theme/mkdocs/header.html index 3560929ca..a3b1d9bd7 100644 --- a/docs/theme/mkdocs/header.html +++ b/docs/theme/mkdocs/header.html @@ -24,7 +24,7 @@ From b885f61a61df1e8d8f71914e17c2a2861167f96e Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 3 Jul 2014 08:28:39 +0300 Subject: [PATCH 027/517] update vendored archive/tar to lower allocations Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) --- hack/vendor.sh | 6 +++--- .../p/go/src/pkg/archive/tar/reader.go | 13 ++++++++----- .../p/go/src/pkg/archive/tar/writer.go | 8 +++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 2ee530a73..b0a89db68 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -53,9 +53,9 @@ clone hg code.google.com/p/go.net 84a4013f96e0 clone hg code.google.com/p/gosqlite 74691fb6f837 -# get Go tip's archive/tar, for xattr support -# TODO after Go 1.3 drops, bump our minimum supported version and drop this vendored dep -clone hg code.google.com/p/go 3458ba248590 +# get Go tip's archive/tar, for xattr support and improved performance +# TODO after Go 1.4 drops, bump our minimum supported version and drop this vendored dep +clone hg code.google.com/p/go 17404efd6b02 mv src/code.google.com/p/go/src/pkg/archive/tar tmp-tar rm -rf src/code.google.com/p/go mkdir -p src/code.google.com/p/go/src/pkg/archive diff --git a/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go b/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go index 920a9b08f..a27559d0f 100644 --- a/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go +++ b/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go @@ -29,10 +29,11 @@ const maxNanoSecondIntSize = 9 // The Next method advances to the next file in the archive (including the first), // and then it can be treated as an io.Reader to access the file's data. type Reader struct { - r io.Reader - err error - pad int64 // amount of padding (ignored) after current file entry - curr numBytesReader // reader for current file entry + r io.Reader + err error + pad int64 // amount of padding (ignored) after current file entry + curr numBytesReader // reader for current file entry + hdrBuff [blockSize]byte // buffer to use in readHeader } // A numBytesReader is an io.Reader with a numBytes method, returning the number @@ -426,7 +427,9 @@ func (tr *Reader) verifyChecksum(header []byte) bool { } func (tr *Reader) readHeader() *Header { - header := make([]byte, blockSize) + header := tr.hdrBuff[:] + copy(header, zeroBlock) + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { return nil } diff --git a/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go b/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go index 6eff6f6f8..d107dbbb5 100644 --- a/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go +++ b/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go @@ -37,8 +37,9 @@ type Writer struct { nb int64 // number of unwritten bytes for current file entry pad int64 // amount of padding to write after current file entry closed bool - usedBinary bool // whether the binary numeric field extension was used - preferPax bool // use pax header instead of binary numeric header + usedBinary bool // whether the binary numeric field extension was used + preferPax bool // use pax header instead of binary numeric header + hdrBuff [blockSize]byte // buffer to use in writeHeader } // NewWriter creates a new Writer writing to w. @@ -160,7 +161,8 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { // subsecond time resolution, but for now let's just capture // too long fields or non ascii characters - header := make([]byte, blockSize) + header := tw.hdrBuff[:] + copy(header, zeroBlock) s := slicer(header) // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax From a8d95b7c573dedad26395372dd0d4ff8882f6cb6 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 19:15:56 +0400 Subject: [PATCH 028/517] Benchmark for BroadcastWriter Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- utils/broadcastwriter/broadcastwriter_test.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/utils/broadcastwriter/broadcastwriter_test.go b/utils/broadcastwriter/broadcastwriter_test.go index 8d946e2f4..d5c915246 100644 --- a/utils/broadcastwriter/broadcastwriter_test.go +++ b/utils/broadcastwriter/broadcastwriter_test.go @@ -106,3 +106,39 @@ func TestRaceBroadcastWriter(t *testing.T) { writer.Write([]byte("hello")) <-c } + +func BenchmarkBroadcastWriter(b *testing.B) { + writer := New() + setUpWriter := func() { + for i := 0; i < 100; i++ { + writer.AddWriter(devNullCloser(0), "stdout") + writer.AddWriter(devNullCloser(0), "stderr") + writer.AddWriter(devNullCloser(0), "") + } + } + testLine := "Line that thinks that it is log line from docker" + var buf bytes.Buffer + for i := 0; i < 100; i++ { + buf.Write([]byte(testLine + "\n")) + } + // line without eol + buf.Write([]byte(testLine)) + testText := buf.Bytes() + b.SetBytes(int64(5 * len(testText))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + setUpWriter() + b.StartTimer() + + for j := 0; j < 5; j++ { + if _, err := writer.Write(testText); err != nil { + b.Fatal(err) + } + } + + b.StopTimer() + writer.Close() + b.StartTimer() + } +} From 7bdd23bfeec1968a3061e63e24af049837baf4c4 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 23:28:08 +0400 Subject: [PATCH 029/517] BroadcastWriter refactoring It became slightly faster and lighter possibly fixes #5923 problems Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- daemon/container.go | 4 +-- utils/broadcastwriter/broadcastwriter.go | 34 +++++++++---------- utils/broadcastwriter/broadcastwriter_test.go | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/daemon/container.go b/daemon/container.go index c7a677460..419314b8b 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -503,10 +503,10 @@ func (container *Container) cleanup() { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.Close(); err != nil { + if err := container.stdout.Clean(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.Close(); err != nil { + if err := container.stderr.Clean(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } if container.command != nil && container.command.Terminal != nil { diff --git a/utils/broadcastwriter/broadcastwriter.go b/utils/broadcastwriter/broadcastwriter.go index 81ca9e5bd..9bd2c1471 100644 --- a/utils/broadcastwriter/broadcastwriter.go +++ b/utils/broadcastwriter/broadcastwriter.go @@ -10,12 +10,16 @@ import ( "github.com/dotcloud/docker/utils" ) +// BroadcastWriter accumulate multiple io.WriteCloser by stream. type BroadcastWriter struct { sync.Mutex buf *bytes.Buffer streams map[string](map[io.WriteCloser]struct{}) } +// AddWriter adds new io.WriteCloser for stream. +// If stream is "", then all writes proceed as is. Otherwise every line from +// input will be packed to serialized utils.JSONLog. func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { w.Lock() if _, ok := w.streams[stream]; !ok { @@ -25,10 +29,11 @@ func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { w.Unlock() } +// Write writes bytes to all writers. Failed writers will be evicted during +// this call. func (w *BroadcastWriter) Write(p []byte) (n int, err error) { created := time.Now().UTC() w.Lock() - defer w.Unlock() if writers, ok := w.streams[""]; ok { for sw := range writers { if n, err := sw.Write(p); err != nil || n != len(p) { @@ -38,49 +43,44 @@ func (w *BroadcastWriter) Write(p []byte) (n int, err error) { } } w.buf.Write(p) - lines := []string{} for { line, err := w.buf.ReadString('\n') if err != nil { w.buf.Write([]byte(line)) break } - lines = append(lines, line) - } - - if len(lines) != 0 { for stream, writers := range w.streams { if stream == "" { continue } - var lp []byte - for _, line := range lines { - b, err := json.Marshal(&utils.JSONLog{Log: line, Stream: stream, Created: created}) - if err != nil { - utils.Errorf("Error making JSON log line: %s", err) - } - lp = append(lp, b...) - lp = append(lp, '\n') + b, err := json.Marshal(utils.JSONLog{Log: line, Stream: stream, Created: created}) + if err != nil { + utils.Errorf("Error making JSON log line: %s", err) + continue } + b = append(b, '\n') for sw := range writers { - if _, err := sw.Write(lp); err != nil { + if _, err := sw.Write(b); err != nil { delete(writers, sw) } } } } + w.Unlock() return len(p), nil } -func (w *BroadcastWriter) Close() error { +// Clean closes and removes all writers. Last non-eol-terminated part of data +// will be saved. +func (w *BroadcastWriter) Clean() error { w.Lock() - defer w.Unlock() for _, writers := range w.streams { for w := range writers { w.Close() } } w.streams = make(map[string](map[io.WriteCloser]struct{})) + w.Unlock() return nil } diff --git a/utils/broadcastwriter/broadcastwriter_test.go b/utils/broadcastwriter/broadcastwriter_test.go index d5c915246..62ca12659 100644 --- a/utils/broadcastwriter/broadcastwriter_test.go +++ b/utils/broadcastwriter/broadcastwriter_test.go @@ -82,7 +82,7 @@ func TestBroadcastWriter(t *testing.T) { t.Errorf("Buffer contains %v", bufferC.String()) } - writer.Close() + writer.Clean() } type devNullCloser int @@ -138,7 +138,7 @@ func BenchmarkBroadcastWriter(b *testing.B) { } b.StopTimer() - writer.Close() + writer.Clean() b.StartTimer() } } From 4318802f645cdd4fa63a894160f153a69a97af59 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 4 Jun 2014 16:38:06 -0400 Subject: [PATCH 030/517] Error if Docker daemon starts with BTRFS graph driver and SELinux enabled The Docker btrfs graph driver does not interact well with SELinux at present. If btrfs mounts the same file in several locations, the same SELinux label will be applied to all mountpoints. In the context of the graph driver, things such as shared libraries become inaccessible to containers due to SELInux, causing all dynamically linked applications to fail when run in a container. Consequently, error when we detect the daemon is being run with SELinux enabled and the btrfs driver. Documentation has been added for this behavior. Docker-DCO-1.1-Signed-off-by: Matthew Heon (github: mheon) --- daemon/daemon.go | 5 +++++ docker/docker.go | 2 +- docs/man/docker.1.md | 2 +- docs/sources/reference/commandline/cli.md | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 23402d951..886340797 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -778,6 +778,11 @@ func NewDaemonFromDirectory(config *daemonconfig.Config, eng *engine.Engine) (*D } utils.Debugf("Using graph driver %s", driver) + // As Docker on btrfs and SELinux are incompatible at present, error on both being enabled + if config.EnableSelinuxSupport && driver.String() == "btrfs" { + return nil, fmt.Errorf("SELinux is not supported with the BTRFS graph driver!") + } + daemonRepo := path.Join(config.Root, "containers") if err := os.MkdirAll(daemonRepo, 0700); err != nil && !os.IsExist(err) { diff --git a/docker/docker.go b/docker/docker.go index 30d43bc6a..5367e759a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -66,7 +66,7 @@ func main() { flCa = flag.String([]string{"-tlscacert"}, dockerConfDir+defaultCaFile, "Trust only remotes providing a certificate signed by the CA given here") flCert = flag.String([]string{"-tlscert"}, dockerConfDir+defaultCertFile, "Path to TLS certificate file") flKey = flag.String([]string{"-tlskey"}, dockerConfDir+defaultKeyFile, "Path to TLS key file") - flSelinuxEnabled = flag.Bool([]string{"-selinux-enabled"}, false, "Enable selinux support") + flSelinuxEnabled = flag.Bool([]string{"-selinux-enabled"}, false, "Enable selinux support. SELinux does not presently support the BTRFS storage driver") ) flag.Var(&flDns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers") flag.Var(&flDnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains") diff --git a/docs/man/docker.1.md b/docs/man/docker.1.md index a7a826ed9..602c6e2ac 100644 --- a/docs/man/docker.1.md +++ b/docs/man/docker.1.md @@ -74,7 +74,7 @@ unix://[/path/to/socket] to use. Print version information and quit. Default is false. **--selinux-enabled**=*true*|*false* - Enable selinux support. Default is false. + Enable selinux support. Default is false. SELinux does not presently support the BTRFS storage driver. # COMMANDS **docker-attach(1)** diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 301593f2f..9a6d27f0e 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -73,7 +73,7 @@ expect an integer, and they can only be specified once. -p, --pidfile="/var/run/docker.pid" Path to use for daemon PID file -r, --restart=true Restart previously running containers -s, --storage-driver="" Force the Docker runtime to use a specific storage driver - --selinux-enabled=false Enable selinux support + --selinux-enabled=false Enable selinux support. SELinux does not presently support the BTRFS storage driver --storage-opt=[] Set storage driver options --tls=false Use TLS; implied by tls-verify flags --tlscacert="/home/sven/.docker/ca.pem" Trust only remotes providing a certificate signed by the CA given here From d55cafb72e25e5ef9b400c1c010e9832ec8b5561 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 3 Jul 2014 18:01:36 +0000 Subject: [PATCH 031/517] Change version to 1.1.0-dev Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9084fa2f7..336c36775 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 +1.1.0-dev From 5c246c931eec7ec31a90fc4ae1577eb54f938427 Mon Sep 17 00:00:00 2001 From: Tobias Gesellchen Date: Thu, 3 Jul 2014 22:24:24 +0200 Subject: [PATCH 032/517] docs: add Groovy Docker-Client link Docker-DCO-1.1-Signed-off-by: Tobias Gesellchen (github: gesellix) --- docs/sources/reference/api/remote_api_client_libraries.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/sources/reference/api/remote_api_client_libraries.md b/docs/sources/reference/api/remote_api_client_libraries.md index d1d26a1dd..3dcebeeda 100644 --- a/docs/sources/reference/api/remote_api_client_libraries.md +++ b/docs/sources/reference/api/remote_api_client_libraries.md @@ -140,5 +140,11 @@ will add the libraries here. https://github.com/spotify/docker-client Active + + Groovy + docker-client + https://github.com/gesellix-docker/docker-client + Active + From 6e4b37fffdfc324141aae9f86268db4e8ed7aeb0 Mon Sep 17 00:00:00 2001 From: Naoki Orii Date: Thu, 3 Jul 2014 14:02:40 -0700 Subject: [PATCH 033/517] doc fix: remove trailing * The trailing `*` makes it seem like there is some kind of annotation --- docs/sources/introduction/understanding-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/introduction/understanding-docker.md b/docs/sources/introduction/understanding-docker.md index c79573a63..18b8d5926 100644 --- a/docs/sources/introduction/understanding-docker.md +++ b/docs/sources/introduction/understanding-docker.md @@ -55,7 +55,7 @@ Docker's portability and lightweight nature also make dynamically managing workloads easy. You can use Docker to quickly scale up or tear down applications and services. Docker's speed means that scaling can be near real time. -*Achieving higher density and running more workloads** +*Achieving higher density and running more workloads* Docker is lightweight and fast. It provides a viable, cost-effective alternative to hypervisor-based virtual machines. This is especially useful in high density From 0a3abe33f0bd9b7dfc36021b6f10d0857a432d40 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 1 Jul 2014 16:51:30 -0400 Subject: [PATCH 034/517] Remove dup tests integration/server_test.go/TestCreateNumberUsername == integration-cli/docker_cli_run_test.go/TestUserByID integration/server_test.go/TestRmi == integration-cli/docker_cli_images_test.go/TestCLIImageTagRemove Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) --- integration/server_test.go | 99 -------------------------------------- 1 file changed, 99 deletions(-) diff --git a/integration/server_test.go b/integration/server_test.go index 151490cdc..f70bbb0b7 100644 --- a/integration/server_test.go +++ b/integration/server_test.go @@ -2,7 +2,6 @@ package docker import ( "bytes" - "strings" "testing" "time" @@ -23,18 +22,6 @@ func TestCreateNumberHostname(t *testing.T) { createTestContainer(eng, config, t) } -func TestCreateNumberUsername(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - config, _, _, err := runconfig.Parse([]string{"-u", "1002", unitTestImageID, "echo test"}, nil) - if err != nil { - t.Fatal(err) - } - - createTestContainer(eng, config, t) -} - func TestCommit(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() @@ -271,92 +258,6 @@ func TestRunWithTooLowMemoryLimit(t *testing.T) { } } -func TestRmi(t *testing.T) { - eng := NewTestEngine(t) - srv := mkServerFromEngine(eng, t) - defer mkDaemonFromEngine(eng, t).Nuke() - - initialImages := getAllImages(eng, t) - - config, hostConfig, _, err := runconfig.Parse([]string{unitTestImageID, "echo", "test"}, nil) - if err != nil { - t.Fatal(err) - } - - containerID := createTestContainer(eng, config, t) - - //To remove - job := eng.Job("start", containerID) - if err := job.ImportEnv(hostConfig); err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("wait", containerID).Run(); err != nil { - t.Fatal(err) - } - - job = eng.Job("commit", containerID) - job.Setenv("repo", "test") - var outputBuffer = bytes.NewBuffer(nil) - job.Stdout.Add(outputBuffer) - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("tag", engine.Tail(outputBuffer, 1), "test", "0.1").Run(); err != nil { - t.Fatal(err) - } - - containerID = createTestContainer(eng, config, t) - - //To remove - job = eng.Job("start", containerID) - if err := job.ImportEnv(hostConfig); err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("wait", containerID).Run(); err != nil { - t.Fatal(err) - } - - job = eng.Job("commit", containerID) - job.Setenv("repo", "test") - if err := job.Run(); err != nil { - t.Fatal(err) - } - - images := getAllImages(eng, t) - - if images.Len()-initialImages.Len() != 2 { - t.Fatalf("Expected 2 new images, found %d.", images.Len()-initialImages.Len()) - } - - if err = srv.DeleteImage(engine.Tail(outputBuffer, 1), engine.NewTable("", 0), true, false, false); err != nil { - t.Fatal(err) - } - - images = getAllImages(eng, t) - - if images.Len()-initialImages.Len() != 1 { - t.Fatalf("Expected 1 new image, found %d.", images.Len()-initialImages.Len()) - } - - for _, image := range images.Data { - if strings.Contains(unitTestImageID, image.Get("Id")) { - continue - } - if image.GetList("RepoTags")[0] == ":" { - t.Fatalf("Expected tagged image, got untagged one.") - } - } -} - func TestImagesFilter(t *testing.T) { eng := NewTestEngine(t) defer nuke(mkDaemonFromEngine(eng, t)) From 804b00cd7d1f084a872211e5043d255c454c8e51 Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Thu, 26 Jun 2014 12:03:23 +0100 Subject: [PATCH 035/517] Relax dns search to accept empty domain In that case /etc/resolv.conf will be generated with no search option. Usage: --dns-search=. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) --- docker/docker.go | 2 +- integration-cli/docker_cli_run_test.go | 109 ++++++++++++++++++++ opts/opts.go | 11 +- opts/opts_test.go | 14 +-- pkg/networkfs/resolvconf/resolvconf.go | 6 +- pkg/networkfs/resolvconf/resolvconf_test.go | 25 +++++ runconfig/parse.go | 2 +- 7 files changed, 158 insertions(+), 11 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 30d43bc6a..147c91d21 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -52,7 +52,7 @@ func main() { flSocketGroup = flag.String([]string{"G", "-group"}, "docker", "Group to assign the unix socket specified by -H when running in daemon mode\nuse '' (the empty string) to disable setting of a group") flEnableCors = flag.Bool([]string{"#api-enable-cors", "-api-enable-cors"}, false, "Enable CORS headers in the remote API") flDns = opts.NewListOpts(opts.ValidateIp4Address) - flDnsSearch = opts.NewListOpts(opts.ValidateDomain) + flDnsSearch = opts.NewListOpts(opts.ValidateDnsSearch) flEnableIptables = flag.Bool([]string{"#iptables", "-iptables"}, true, "Enable Docker's addition of iptables rules") flEnableIpForward = flag.Bool([]string{"#ip-forward", "-ip-forward"}, true, "Enable net.ipv4.ip_forward") flDefaultIp = flag.String([]string{"#ip", "-ip"}, "0.0.0.0", "Default IP address to use when binding container ports") diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index eda1b3f0a..9f52d58d1 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io/ioutil" "os" "os/exec" "reflect" @@ -10,6 +11,8 @@ import ( "strings" "sync" "testing" + + "github.com/dotcloud/docker/pkg/networkfs/resolvconf" ) // "test123" should be printed by docker run @@ -983,3 +986,109 @@ func TestDisallowBindMountingRootToRoot(t *testing.T) { logDone("run - bind mount /:/ as volume should fail") } + +func TestDnsDefaultOptions(t *testing.T) { + cmd := exec.Command(dockerBinary, "run", "busybox", "cat", "/etc/resolv.conf") + + actual, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, actual) + } + + resolvConf, err := ioutil.ReadFile("/etc/resolv.conf") + if os.IsNotExist(err) { + t.Fatalf("/etc/resolv.conf does not exist") + } + + if actual != string(resolvConf) { + t.Fatalf("expected resolv.conf is not the same of actual") + } + + deleteAllContainers() + + logDone("run - dns default options") +} + +func TestDnsOptions(t *testing.T) { + cmd := exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "--dns-search=mydomain", "busybox", "cat", "/etc/resolv.conf") + + out, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actual := strings.Replace(strings.Trim(out, "\r\n"), "\n", " ", -1) + if actual != "nameserver 127.0.0.1 search mydomain" { + t.Fatalf("expected 'nameserver 127.0.0.1 search mydomain', but says: '%s'", actual) + } + + cmd = exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "--dns-search=.", "busybox", "cat", "/etc/resolv.conf") + + out, _, err = runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actual = strings.Replace(strings.Trim(strings.Trim(out, "\r\n"), " "), "\n", " ", -1) + if actual != "nameserver 127.0.0.1" { + t.Fatalf("expected 'nameserver 127.0.0.1', but says: '%s'", actual) + } + + logDone("run - dns options") +} + +func TestDnsOptionsBasedOnHostResolvConf(t *testing.T) { + resolvConf, err := ioutil.ReadFile("/etc/resolv.conf") + if os.IsNotExist(err) { + t.Fatalf("/etc/resolv.conf does not exist") + } + + hostNamservers := resolvconf.GetNameservers(resolvConf) + hostSearch := resolvconf.GetSearchDomains(resolvConf) + + cmd := exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "busybox", "cat", "/etc/resolv.conf") + + out, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + if actualNameservers := resolvconf.GetNameservers([]byte(out)); string(actualNameservers[0]) != "127.0.0.1" { + t.Fatalf("expected '127.0.0.1', but says: '%s'", string(actualNameservers[0])) + } + + actualSearch := resolvconf.GetSearchDomains([]byte(out)) + if len(actualSearch) != len(hostSearch) { + t.Fatalf("expected '%s' search domain(s), but it has: '%s'", len(hostSearch), len(actualSearch)) + } + for i := range actualSearch { + if actualSearch[i] != hostSearch[i] { + t.Fatalf("expected '%s' domain, but says: '%s'", actualSearch[i], hostSearch[i]) + } + } + + cmd = exec.Command(dockerBinary, "run", "--dns-search=mydomain", "busybox", "cat", "/etc/resolv.conf") + + out, _, err = runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actualNameservers := resolvconf.GetNameservers([]byte(out)) + if len(actualNameservers) != len(hostNamservers) { + t.Fatalf("expected '%s' nameserver(s), but it has: '%s'", len(hostNamservers), len(actualNameservers)) + } + for i := range actualNameservers { + if actualNameservers[i] != hostNamservers[i] { + t.Fatalf("expected '%s' nameserver, but says: '%s'", actualNameservers[i], hostNamservers[i]) + } + } + + if actualSearch = resolvconf.GetSearchDomains([]byte(out)); string(actualSearch[0]) != "mydomain" { + t.Fatalf("expected 'mydomain', but says: '%s'", string(actualSearch[0])) + } + + deleteAllContainers() + + logDone("run - dns options based on host resolv.conf") +} diff --git a/opts/opts.go b/opts/opts.go index 67f1c8fd4..d17b57e07 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -137,7 +137,16 @@ func ValidateIp4Address(val string) (string, error) { return "", fmt.Errorf("%s is not an ip4 address", val) } -func ValidateDomain(val string) (string, error) { +// Validates domain for resolvconf search configuration. +// A zero length domain is represented by . +func ValidateDnsSearch(val string) (string, error) { + if val = strings.Trim(val, " "); val == "." { + return val, nil + } + return validateDomain(val) +} + +func validateDomain(val string) (string, error) { alpha := regexp.MustCompile(`[a-zA-Z]`) if alpha.FindString(val) == "" { return "", fmt.Errorf("%s is not a valid domain", val) diff --git a/opts/opts_test.go b/opts/opts_test.go index 299cbfe50..b18088b93 100644 --- a/opts/opts_test.go +++ b/opts/opts_test.go @@ -23,8 +23,9 @@ func TestValidateIP4(t *testing.T) { } -func TestValidateDomain(t *testing.T) { +func TestValidateDnsSearch(t *testing.T) { valid := []string{ + `.`, `a`, `a.`, `1.foo`, @@ -49,7 +50,8 @@ func TestValidateDomain(t *testing.T) { invalid := []string{ ``, - `.`, + ` `, + ` `, `17`, `17.`, `.17`, @@ -65,14 +67,14 @@ func TestValidateDomain(t *testing.T) { } for _, domain := range valid { - if ret, err := ValidateDomain(domain); err != nil || ret == "" { - t.Fatalf("ValidateDomain(`"+domain+"`) got %s %s", ret, err) + if ret, err := ValidateDnsSearch(domain); err != nil || ret == "" { + t.Fatalf("ValidateDnsSearch(`"+domain+"`) got %s %s", ret, err) } } for _, domain := range invalid { - if ret, err := ValidateDomain(domain); err == nil || ret != "" { - t.Fatalf("ValidateDomain(`"+domain+"`) got %s %s", ret, err) + if ret, err := ValidateDnsSearch(domain); err == nil || ret != "" { + t.Fatalf("ValidateDnsSearch(`"+domain+"`) got %s %s", ret, err) } } } diff --git a/pkg/networkfs/resolvconf/resolvconf.go b/pkg/networkfs/resolvconf/resolvconf.go index d6854fb3b..38ae56432 100644 --- a/pkg/networkfs/resolvconf/resolvconf.go +++ b/pkg/networkfs/resolvconf/resolvconf.go @@ -78,8 +78,10 @@ func Build(path string, dns, dnsSearch []string) error { } } if len(dnsSearch) > 0 { - if _, err := content.WriteString("search " + strings.Join(dnsSearch, " ") + "\n"); err != nil { - return err + if searchString := strings.Join(dnsSearch, " "); strings.Trim(searchString, " ") != "." { + if _, err := content.WriteString("search " + searchString + "\n"); err != nil { + return err + } } } diff --git a/pkg/networkfs/resolvconf/resolvconf_test.go b/pkg/networkfs/resolvconf/resolvconf_test.go index fd2071237..6187acbae 100644 --- a/pkg/networkfs/resolvconf/resolvconf_test.go +++ b/pkg/networkfs/resolvconf/resolvconf_test.go @@ -131,3 +131,28 @@ func TestBuild(t *testing.T) { t.Fatalf("Expected to find '%s' got '%s'", expected, content) } } + +func TestBuildWithZeroLengthDomainSearch(t *testing.T) { + file, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + err = Build(file.Name(), []string{"ns1", "ns2", "ns3"}, []string{"."}) + if err != nil { + t.Fatal(err) + } + + content, err := ioutil.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + + if expected := "nameserver ns1\nnameserver ns2\nnameserver ns3\n"; !bytes.Contains(content, []byte(expected)) { + t.Fatalf("Expected to find '%s' got '%s'", expected, content) + } + if notExpected := "search ."; bytes.Contains(content, []byte(notExpected)) { + t.Fatalf("Expected to not find '%s' got '%s'", notExpected, content) + } +} diff --git a/runconfig/parse.go b/runconfig/parse.go index dfd9f4ddd..ae7918671 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -45,7 +45,7 @@ func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Conf flPublish opts.ListOpts flExpose opts.ListOpts flDns opts.ListOpts - flDnsSearch = opts.NewListOpts(opts.ValidateDomain) + flDnsSearch = opts.NewListOpts(opts.ValidateDnsSearch) flVolumesFrom opts.ListOpts flLxcOpts opts.ListOpts flEnvFile opts.ListOpts From 3f5ab612c8d0c1775a864e4414846b301e6498e9 Mon Sep 17 00:00:00 2001 From: "hyeongkyu.lee" Date: Fri, 4 Jul 2014 17:59:26 +0900 Subject: [PATCH 036/517] Update README.md Docker-DCO-1.1-Signed-off-by: Hyeongkyu Lee (github: leeplay) --- daemonconfig/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemonconfig/README.md b/daemonconfig/README.md index 488e7c7ca..b61df1449 100644 --- a/daemonconfig/README.md +++ b/daemonconfig/README.md @@ -1,3 +1,3 @@ -This directory contains code pertaining to the configuration of the docker deamon +This directory contains code pertaining to the configuration of the docker daemon These are the configuration settings that you pass to the docker daemon when you launch it with say: `docker -d -e lxc` From 649b50c0b71a21f8ea6b88c63008a0f909fe4113 Mon Sep 17 00:00:00 2001 From: "hyeongkyu.lee" Date: Fri, 4 Jul 2014 18:21:13 +0900 Subject: [PATCH 037/517] Update README.md Docker-DCO-1.1-Signed-off-by: Hyeongkyu Lee (github: leeplay) --- api/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/README.md b/api/README.md index 3ef33f8c2..453f61a1a 100644 --- a/api/README.md +++ b/api/README.md @@ -1,5 +1,5 @@ This directory contains code pertaining to the Docker API: - - Used by the docker client when comunicating with the docker deamon + - Used by the docker client when communicating with the docker daemon - - Used by third party tools wishing to interface with the docker deamon + - Used by third party tools wishing to interface with the docker daemon From eceacb8334250b9f241b263f28372d5f35ada71d Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 4 Jul 2014 15:56:23 +1000 Subject: [PATCH 038/517] add client OS and arch to docker version Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) --- api/client/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/client/commands.go b/api/client/commands.go index df2125f5f..c27389559 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -391,6 +391,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if dockerversion.GITCOMMIT != "" { fmt.Fprintf(cli.out, "Git commit (client): %s\n", dockerversion.GITCOMMIT) } + fmt.Fprintf(cli.out, "OS/Arch (client): %s/%s\n", runtime.GOOS, runtime.GOARCH) body, _, err := readBody(cli.call("GET", "/version", nil, false)) if err != nil { From 6ad5fe86590f757eca754d5bb4a599faec63ae30 Mon Sep 17 00:00:00 2001 From: Andrew France Date: Fri, 4 Jul 2014 12:05:50 +0100 Subject: [PATCH 039/517] Fix "Error while reading file" Fish completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unbalanced single quote caused Fish to fail to load the completion file with `source: Error while reading file “/etc/fish/completions/docker.fish”`. Docker-DCO-1.1-Signed-off-by: Andrew France (github: Odaeus) --- contrib/completion/fish/docker.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/fish/docker.fish b/contrib/completion/fish/docker.fish index a4a9365f9..ba83526c7 100644 --- a/contrib/completion/fish/docker.fish +++ b/contrib/completion/fish/docker.fish @@ -85,7 +85,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from commit' -l run -d 'Conf complete -c docker -A -f -n '__fish_seen_subcommand_from commit' -a '(__fish_print_docker_containers all)' -d "Container" # cp -complete -c docker -f -n '__fish_docker_no_subcommand' -a cp -d 'Copy files/folders from a container's filesystem to the host path' +complete -c docker -f -n '__fish_docker_no_subcommand' -a cp -d "Copy files/folders from a container's filesystem to the host path" # diff complete -c docker -f -n '__fish_docker_no_subcommand' -a diff -d "Inspect changes on a container's filesystem" From 20a77aeeb8855ca0189fe9e3292d92b165b08905 Mon Sep 17 00:00:00 2001 From: OddBloke Date: Thu, 3 Jul 2014 12:55:21 +0100 Subject: [PATCH 040/517] Explain ADD invalidation more accurately And also move it in to the `ADD` section, rather than being hidden in the `RUN` section. Docker-DCO-1.1-Signed-off-by: Daniel Watkins (github: OddBloke) --- docs/sources/reference/builder.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/sources/reference/builder.md b/docs/sources/reference/builder.md index 91190933c..da7f3ebff 100644 --- a/docs/sources/reference/builder.md +++ b/docs/sources/reference/builder.md @@ -169,9 +169,8 @@ will be reused during the next build. The cache for `RUN` instructions can be invalidated by using the `--no-cache` flag, for example `docker build --no-cache`. -The first encountered `ADD` instruction will invalidate the cache for all -following instructions from the 'Dockerfile' if the contents of the context -have changed. This will also invalidate the cache for `RUN` instructions. +The cache for `RUN` instructions can be invalidated by `ADD` instructions. See +[below](#add) for details. ### Known Issues (RUN) @@ -285,6 +284,11 @@ In the case where `` is a remote file URL, the destination will have permis > or use another tool from within the container as ADD does not support > authentication. +> **Note**: +> The first encountered `ADD` instruction will invalidate the cache for all +> following instructions from the Dockerfile if the contents of `` have +> changed. This includes invalidating the cache for `RUN` instructions. + The copy obeys the following rules: - The `` path must be inside the *context* of the build; From cfbe062eb2ba6f1d24ab3b6f9b965a65931d5b07 Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Fri, 4 Jul 2014 18:52:10 +0100 Subject: [PATCH 041/517] Add info to Mount a Host File as a Data Volume Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) --- docs/sources/userguide/dockervolumes.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/sources/userguide/dockervolumes.md b/docs/sources/userguide/dockervolumes.md index 93ac37b1c..fc9a7e0f8 100644 --- a/docs/sources/userguide/dockervolumes.md +++ b/docs/sources/userguide/dockervolumes.md @@ -71,6 +71,21 @@ read-only. Here we've mounted the same `/src/webapp` directory but we've added the `ro` option to specify that the mount should be read-only. +### Mount a Host File as a Data Volume + +As well as directories, the `-v` flag can be used to mount a single file from the host into a container. + + $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash + +This will drop you into a bash shell in a new container, you will have your bash history from your host and when +you exit the container the host will have the history of the commands typed while in the container. + +> **Note:** +> The two-way binding of the mounted file will only be preserved as long as the inode doesn't change. Many +> tools used to edit files including `vi` and `sed --in-place` may result in a inode change. In the case where you +> want to edit the file, it is often best to mount the parent directory. + + ## Creating and mounting a Data Volume Container If you have some persistent data that you want to share between From 87df57810d39bdd50167338e5ff8d0e6bc6e23e2 Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Sat, 5 Jul 2014 03:46:41 +0100 Subject: [PATCH 042/517] Update File mount info for docker v1.1.0 Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) --- docs/sources/userguide/dockervolumes.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/sources/userguide/dockervolumes.md b/docs/sources/userguide/dockervolumes.md index fc9a7e0f8..f0725d4ad 100644 --- a/docs/sources/userguide/dockervolumes.md +++ b/docs/sources/userguide/dockervolumes.md @@ -78,13 +78,12 @@ As well as directories, the `-v` flag can be used to mount a single file from th $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash This will drop you into a bash shell in a new container, you will have your bash history from your host and when -you exit the container the host will have the history of the commands typed while in the container. +you exit the container, the host will have the history of the commands typed while in the container. > **Note:** -> The two-way binding of the mounted file will only be preserved as long as the inode doesn't change. Many -> tools used to edit files including `vi` and `sed --in-place` may result in a inode change. In the case where you -> want to edit the file, it is often best to mount the parent directory. - +> Many tools used to edit files including `vi` and `sed --in-place` may result in a inode change. Since docker v1.1.0 +> this will produce a error such as "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where you +> want to edit the mounted file, it is often easiest to instead mount the parent directory. ## Creating and mounting a Data Volume Container From ac392bc0d775455f71da3c71956f9f6ae7a87f6d Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sat, 5 Jul 2014 16:20:14 -0400 Subject: [PATCH 043/517] docker save: fix the 'repositories' file For various use cases, the 'repositories' file does not match expected behavior. Like, docker save busybox:latest | tar t Before: [...] busybox:latest/ busybox:latest/VERSION busybox:latest/json busybox:latest/layer.tar # note, the layer name, and lack of 'repositories' file Now: [...] a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/ a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/VERSION a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/json a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/layer.tar repositories # and the repositories file is correct for the single tagged # image. #> {"busybox":{"latest":"a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721"}} and docker save a9eb17255234 | tar t Before: [...] a9eb17255234/ a9eb17255234/VERSION a9eb17255234/json a9eb17255234/layer.tar # Note the truncated layer name Now: [...] a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/ a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/VERSION a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/json a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/layer.tar # There is no 'repositories' file, because there is no named repo Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- server/server.go | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 7e453a3f0..0d083ad9a 100644 --- a/server/server.go +++ b/server/server.go @@ -351,29 +351,52 @@ func (srv *Server) ImageExport(job *engine.Job) engine.Status { utils.Debugf("Serializing %s", name) + rootRepoMap := map[string]graph.Repository{} rootRepo, err := srv.daemon.Repositories().Get(name) if err != nil { return job.Error(err) } if rootRepo != nil { + // this is a base repo name, like 'busybox' + for _, id := range rootRepo { if err := srv.exportImage(job.Eng, id, tempdir); err != nil { return job.Error(err) } } - - // write repositories - rootRepoMap := map[string]graph.Repository{} rootRepoMap[name] = rootRepo + } else { + img, err := srv.daemon.Repositories().LookupImage(name) + if err != nil { + return job.Error(err) + } + if img != nil { + // This is a named image like 'busybox:latest' + repoName, repoTag := utils.ParseRepositoryTag(name) + if err := srv.exportImage(job.Eng, img.ID, tempdir); err != nil { + return job.Error(err) + } + // check this length, because a lookup of a truncated has will not have a tag + // and will not need to be added to this map + if len(repoTag) > 0 { + rootRepoMap[repoName] = graph.Repository{repoTag: img.ID} + } + } else { + // this must be an ID that didn't get looked up just right? + if err := srv.exportImage(job.Eng, name, tempdir); err != nil { + return job.Error(err) + } + } + } + // write repositories, if there is something to write + if len(rootRepoMap) > 0 { rootRepoJson, _ := json.Marshal(rootRepoMap) if err := ioutil.WriteFile(path.Join(tempdir, "repositories"), rootRepoJson, os.FileMode(0644)); err != nil { return job.Error(err) } } else { - if err := srv.exportImage(job.Eng, name, tempdir); err != nil { - return job.Error(err) - } + utils.Debugf("There were no repositories to write") } fs, err := archive.Tar(tempdir, archive.Uncompressed) From 29a20672c216e2f8aa90af765ac5c47e3350b3d9 Mon Sep 17 00:00:00 2001 From: Ian Bull Date: Thu, 3 Jul 2014 22:40:02 -0700 Subject: [PATCH 044/517] Fix the parent/child relationship in the docs In the Docker Linking Docs, the parent child relationship was backwards. The Web container should be able to access the DB, not other way around. Furthermore, the output of 'docker ps' was wrong (it showed that the DB could access the Web). This fixes both typos. Docker-DCO-1.1-Signed-off-by: Ian Bull (github: irbull) --- docs/sources/userguide/dockerlinks.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sources/userguide/dockerlinks.md b/docs/sources/userguide/dockerlinks.md index 20a5c1a17..91c6cb355 100644 --- a/docs/sources/userguide/dockerlinks.md +++ b/docs/sources/userguide/dockerlinks.md @@ -149,16 +149,16 @@ Let's look at our linked containers using `docker ps`. $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 349169744e49 training/postgres:latest su postgres -c '/usr About a minute ago Up About a minute 5432/tcp db - aed84ee21bde training/webapp:latest python app.py 16 hours ago Up 2 minutes 0.0.0.0:49154->5000/tcp db/web,web + 349169744e49 training/postgres:latest su postgres -c '/usr About a minute ago Up About a minute 5432/tcp db, web/db + aed84ee21bde training/webapp:latest python app.py 16 hours ago Up 2 minutes 0.0.0.0:49154->5000/tcp web -We can see our named containers, `db` and `web`, and we can see that the `web` -containers also shows `db/web` in the `NAMES` column. This tells us that the +We can see our named containers, `db` and `web`, and we can see that the `db` +containers also shows `web/db` in the `NAMES` column. This tells us that the `web` container is linked to the `db` container in a parent/child relationship. So what does linking the containers do? Well we've discovered the link creates a parent-child relationship between the two containers. The parent container, -here `db`, can access information on the child container `web`. To do this +here `web`, can access information on the child container `db`. To do this Docker creates a secure tunnel between the containers without the need to expose any ports externally on the container. You'll note when we started the `db` container we did not use either of the `-P` or `-p` flags. As we're From 1d4862b7a2d0c3b0641ce6d539eabecee686c794 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Sun, 6 Jul 2014 15:36:34 +0400 Subject: [PATCH 045/517] Moved last build tests to integration-cli Also removed skipped tests on "viz" and "tree" because they blocked integration/buildfile_test.go removing. Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) --- integration-cli/docker_cli_build_test.go | 167 +++++++++ integration/buildfile_test.go | 414 ----------------------- integration/commands_test.go | 128 ------- 3 files changed, 167 insertions(+), 542 deletions(-) delete mode 100644 integration/buildfile_test.go diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 2f2f2b6e0..df0cfcc88 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -1564,3 +1564,170 @@ func TestDockerignoringDockerfile(t *testing.T) { } logDone("build - test .dockerignore of Dockerfile") } + +func TestBuildLineBreak(t *testing.T) { + name := "testbuildlinebreak" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +RUN sh -c 'echo root:testpass \ + > /tmp/passwd' +RUN mkdir -p /var/run/sshd +RUN [ "$(cat /tmp/passwd)" = "root:testpass" ] +RUN [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - line break with \\") +} + +func TestBuildEOLInLine(t *testing.T) { + name := "testbuildeolinline" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +RUN sh -c 'echo root:testpass > /tmp/passwd' +RUN echo "foo \n bar"; echo "baz" +RUN mkdir -p /var/run/sshd +RUN [ "$(cat /tmp/passwd)" = "root:testpass" ] +RUN [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - end of line in dockerfile instruction") +} + +func TestBuildCommentsShebangs(t *testing.T) { + name := "testbuildcomments" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +# This is an ordinary comment. +RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh +RUN [ ! -x /hello.sh ] +# comment with line break \ +RUN chmod +x /hello.sh +RUN [ -x /hello.sh ] +RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] +RUN [ "$(/hello.sh)" = "hello world" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - comments and shebangs") +} + +func TestBuildUsersAndGroups(t *testing.T) { + name := "testbuildusers" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox + +# Make sure our defaults work +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)" = '0:0/root:root' ] + +# TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) +USER root +RUN [ "$(id -G):$(id -Gn)" = '0 10:root wheel' ] + +# Setup dockerio user and group +RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd +RUN echo 'dockerio:x:1001:' >> /etc/group + +# Make sure we can switch to our user and all the information is exactly as we expect it to be +USER dockerio +RUN id -G +RUN id -Gn +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] + +# Switch back to root and double check that worked exactly as we might expect it to +USER root +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0 10:root wheel' ] + +# Add a "supplementary" group for our dockerio user +RUN echo 'supplementary:x:1002:dockerio' >> /etc/group + +# ... and then go verify that we get it like we expect +USER dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001 1002:dockerio supplementary' ] +USER 1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001 1002:dockerio supplementary' ] + +# super test the new "user:group" syntax +USER dockerio:dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER 1001:dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER dockerio:1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER 1001:1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER dockerio:supplementary +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER dockerio:1002 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER 1001:supplementary +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER 1001:1002 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] + +# make sure unknown uid/gid still works properly +USER 1042:1043 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1042:1043/1042:1043/1043:1043' ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - users and groups") +} + +func TestBuildEnvUsage(t *testing.T) { + name := "testbuildenvusage" + defer deleteImages(name) + dockerfile := `FROM busybox +ENV FOO /foo/baz +ENV BAR /bar +ENV BAZ $BAR +ENV FOOPATH $PATH:$FOO +RUN [ "$BAR" = "$BAZ" ] +RUN [ "$FOOPATH" = "$PATH:/foo/baz" ] +ENV FROM hello/docker/world +ENV TO /docker/world/hello +ADD $FROM $TO +RUN [ "$(cat $TO)" = "hello" ]` + ctx, err := fakeContext(dockerfile, map[string]string{ + "hello/docker/world": "hello", + }) + if err != nil { + t.Fatal(err) + } + _, err = buildImageFromContext(name, ctx, true) + if err != nil { + t.Fatal(err) + } + logDone("build - environment variables usage") +} + +func TestBuildAddScript(t *testing.T) { + name := "testbuildaddscript" + defer deleteImages(name) + dockerfile := ` +FROM busybox +ADD test /test +RUN ["chmod","+x","/test"] +RUN ["/test"] +RUN [ "$(cat /testfile)" = 'test!' ]` + ctx, err := fakeContext(dockerfile, map[string]string{ + "test": "#!/bin/sh\necho 'test!' > /testfile", + }) + if err != nil { + t.Fatal(err) + } + _, err = buildImageFromContext(name, ctx, true) + if err != nil { + t.Fatal(err) + } + logDone("build - add and run script") +} diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go deleted file mode 100644 index 147ae353a..000000000 --- a/integration/buildfile_test.go +++ /dev/null @@ -1,414 +0,0 @@ -package docker - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" - "github.com/dotcloud/docker/server" - "github.com/dotcloud/docker/utils" -) - -// A testContextTemplate describes a build context and how to test it -type testContextTemplate struct { - // Contents of the Dockerfile - dockerfile string - // Additional files in the context, eg [][2]string{"./passwd", "gordon"} - files [][2]string - // Additional remote files to host on a local HTTP server. - remoteFiles [][2]string -} - -func (context testContextTemplate) Archive(dockerfile string, t *testing.T) archive.Archive { - input := []string{"Dockerfile", dockerfile} - for _, pair := range context.files { - input = append(input, pair[0], pair[1]) - } - a, err := archive.Generate(input...) - if err != nil { - t.Fatal(err) - } - return a -} - -// A table of all the contexts to build and test. -// A new docker runtime will be created and torn down for each context. -var testContexts = []testContextTemplate{ - { - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - - // Exactly the same as above, except uses a line split with a \ to test - // multiline support. - { - ` -from {IMAGE} -run sh -c 'echo root:testpass \ - > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - - // Line containing literal "\n" - { - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run echo "foo \n bar"; echo "baz" -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - { - ` -from {IMAGE} -add foo /usr/lib/bla/bar -run [ "$(cat /usr/lib/bla/bar)" = 'hello' ] -add http://{SERVERADDR}/baz /usr/lib/baz/quux -run [ "$(cat /usr/lib/baz/quux)" = 'world!' ] -`, - [][2]string{{"foo", "hello"}}, - [][2]string{{"/baz", "world!"}}, - }, - - { - ` -from {IMAGE} -add f / -run [ "$(cat /f)" = "hello" ] -add f /abc -run [ "$(cat /abc)" = "hello" ] -add f /x/y/z -run [ "$(cat /x/y/z)" = "hello" ] -add f /x/y/d/ -run [ "$(cat /x/y/d/f)" = "hello" ] -add d / -run [ "$(cat /ga)" = "bu" ] -add d /somewhere -run [ "$(cat /somewhere/ga)" = "bu" ] -add d /anotherplace/ -run [ "$(cat /anotherplace/ga)" = "bu" ] -add d /somewheeeere/over/the/rainbooow -run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] -`, - [][2]string{ - {"f", "hello"}, - {"d/ga", "bu"}, - }, - nil, - }, - - { - ` -from {IMAGE} -add http://{SERVERADDR}/x /a/b/c -run [ "$(cat /a/b/c)" = "hello" ] -add http://{SERVERADDR}/x?foo=bar / -run [ "$(cat /x)" = "hello" ] -add http://{SERVERADDR}/x /d/ -run [ "$(cat /d/x)" = "hello" ] -add http://{SERVERADDR} /e -run [ "$(cat /e)" = "blah" ] -`, - nil, - [][2]string{{"/x", "hello"}, {"/", "blah"}}, - }, - - // Comments, shebangs, and executability, oh my! - { - ` -FROM {IMAGE} -# This is an ordinary comment. -RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh -RUN [ ! -x /hello.sh ] -RUN chmod +x /hello.sh -RUN [ -x /hello.sh ] -RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] -RUN [ "$(/hello.sh)" = "hello world" ] -`, - nil, - nil, - }, - - // Users and groups - { - ` -FROM {IMAGE} - -# Make sure our defaults work -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)" = '0:0/root:root' ] - -# TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) -USER root -RUN [ "$(id -G):$(id -Gn)" = '0:root' ] - -# Setup dockerio user and group -RUN echo 'dockerio:x:1000:1000::/bin:/bin/false' >> /etc/passwd -RUN echo 'dockerio:x:1000:' >> /etc/group - -# Make sure we can switch to our user and all the information is exactly as we expect it to be -USER dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] - -# Switch back to root and double check that worked exactly as we might expect it to -USER root -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0:root' ] - -# Add a "supplementary" group for our dockerio user -RUN echo 'supplementary:x:1001:dockerio' >> /etc/group - -# ... and then go verify that we get it like we expect -USER dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] -USER 1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] - -# super test the new "user:group" syntax -USER dockerio:dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER 1000:dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER dockerio:1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER 1000:1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER dockerio:supplementary -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER dockerio:1001 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER 1000:supplementary -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER 1000:1001 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] - -# make sure unknown uid/gid still works properly -USER 1042:1043 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1042:1043/1042:1043/1043:1043' ] -`, - nil, - nil, - }, - - // Environment variable - { - ` -from {IMAGE} -env FOO BAR -run [ "$FOO" = "BAR" ] -`, - nil, - nil, - }, - - // Environment overwriting - { - ` -from {IMAGE} -env FOO BAR -run [ "$FOO" = "BAR" ] -env FOO BAZ -run [ "$FOO" = "BAZ" ] -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -ENTRYPOINT /bin/echo -CMD Hello world -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -VOLUME /test -CMD Hello world -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -env FOO /foo/baz -env BAR /bar -env BAZ $BAR -env FOOPATH $PATH:$FOO -run [ "$BAR" = "$BAZ" ] -run [ "$FOOPATH" = "$PATH:/foo/baz" ] -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -env FOO /bar -env TEST testdir -env BAZ /foobar -add testfile $BAZ/ -add $TEST $FOO -run [ "$(cat /foobar/testfile)" = "test1" ] -run [ "$(cat /bar/withfile)" = "test2" ] -`, - [][2]string{ - {"testfile", "test1"}, - {"testdir/withfile", "test2"}, - }, - nil, - }, - - // JSON! - { - ` -FROM {IMAGE} -RUN ["/bin/echo","hello","world"] -CMD ["/bin/true"] -ENTRYPOINT ["/bin/echo","your command -->"] -`, - nil, - nil, - }, - { - ` -FROM {IMAGE} -ADD test /test -RUN ["chmod","+x","/test"] -RUN ["/test"] -RUN [ "$(cat /testfile)" = 'test!' ] -`, - [][2]string{ - {"test", "#!/bin/sh\necho 'test!' > /testfile"}, - }, - nil, - }, - { - ` -FROM {IMAGE} -# what \ -RUN mkdir /testing -RUN touch /testing/other -`, - nil, - nil, - }, -} - -// FIXME: test building with 2 successive overlapping ADD commands - -func constructDockerfile(template string, ip net.IP, port string) string { - serverAddr := fmt.Sprintf("%s:%s", ip, port) - replacer := strings.NewReplacer("{IMAGE}", unitTestImageID, "{SERVERADDR}", serverAddr) - return replacer.Replace(template) -} - -func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { - mux := http.NewServeMux() - for _, file := range files { - name, contents := file[0], file[1] - mux.HandleFunc(name, func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(contents)) - }) - } - - // This is how httptest.NewServer sets up a net.Listener, except that our listener must accept remote - // connections (from the container). - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - - s := httptest.NewUnstartedServer(mux) - s.Listener = listener - s.Start() - return s, nil -} - -func TestBuild(t *testing.T) { - for _, ctx := range testContexts { - _, err := buildImage(ctx, t, nil, true) - if err != nil { - t.Fatal(err) - } - } -} - -func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, useCache bool) (*image.Image, error) { - if eng == nil { - eng = NewTestEngine(t) - runtime := mkDaemonFromEngine(eng, t) - // FIXME: we might not need runtime, why not simply nuke - // the engine? - defer nuke(runtime) - } - srv := mkServerFromEngine(eng, t) - - httpServer, err := mkTestingFileServer(context.remoteFiles) - if err != nil { - t.Fatal(err) - } - defer httpServer.Close() - - idx := strings.LastIndex(httpServer.URL, ":") - if idx < 0 { - t.Fatalf("could not get port from test http server address %s", httpServer.URL) - } - port := httpServer.URL[idx+1:] - - iIP := eng.Hack_GetGlobalVar("httpapi.bridgeIP") - if iIP == nil { - t.Fatal("Legacy bridgeIP field not set in engine") - } - ip, ok := iIP.(net.IP) - if !ok { - panic("Legacy bridgeIP field in engine does not cast to net.IP") - } - dockerfile := constructDockerfile(context.dockerfile, ip, port) - - buildfile := server.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, false, ioutil.Discard, utils.NewStreamFormatter(false), nil, nil) - id, err := buildfile.Build(context.Archive(dockerfile, t)) - if err != nil { - return nil, err - } - - job := eng.Job("image_inspect", id) - buffer := bytes.NewBuffer(nil) - image := &image.Image{} - job.Stdout.Add(buffer) - if err := job.Run(); err != nil { - return nil, err - } - err = json.NewDecoder(buffer).Decode(image) - return image, err -} diff --git a/integration/commands_test.go b/integration/commands_test.go index 47e986005..28210e753 100644 --- a/integration/commands_test.go +++ b/integration/commands_test.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path" - "regexp" "strings" "testing" "time" @@ -15,7 +14,6 @@ import ( "github.com/dotcloud/docker/api/client" "github.com/dotcloud/docker/daemon" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" ) @@ -604,132 +602,6 @@ func TestRunErrorBindNonExistingSource(t *testing.T) { }) } -func TestImagesViz(t *testing.T) { - t.Skip("Image viz is deprecated") - stdout, stdoutPipe := io.Pipe() - - cli := client.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr, nil) - defer cleanup(globalEngine, t) - - image := buildTestImages(t, globalEngine) - - c := make(chan struct{}) - go func() { - defer close(c) - if err := cli.CmdImages("--viz"); err != nil { - t.Fatal(err) - } - stdoutPipe.Close() - }() - - setTimeout(t, "Reading command output time out", 2*time.Second, func() { - cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout)) - if err != nil { - t.Fatal(err) - } - cmdOutput := string(cmdOutputBytes) - - regexpStrings := []string{ - "digraph docker {", - fmt.Sprintf("base -> \"%s\" \\[style=invis]", unitTestImageIDShort), - fmt.Sprintf("label=\"%s\\\\n%s:latest\"", unitTestImageIDShort, unitTestImageName), - fmt.Sprintf("label=\"%s\\\\n%s:%s\"", utils.TruncateID(image.ID), "test", "latest"), - "base \\[style=invisible]", - } - - compiledRegexps := []*regexp.Regexp{} - for _, regexpString := range regexpStrings { - regexp, err := regexp.Compile(regexpString) - if err != nil { - fmt.Println("Error in regex string: ", err) - return - } - compiledRegexps = append(compiledRegexps, regexp) - } - - for _, regexp := range compiledRegexps { - if !regexp.MatchString(cmdOutput) { - t.Fatalf("images --viz content '%s' did not match regexp '%s'", cmdOutput, regexp) - } - } - }) -} - -func TestImagesTree(t *testing.T) { - t.Skip("Image tree is deprecated") - stdout, stdoutPipe := io.Pipe() - - cli := client.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr, nil) - defer cleanup(globalEngine, t) - - image := buildTestImages(t, globalEngine) - - c := make(chan struct{}) - go func() { - defer close(c) - if err := cli.CmdImages("--tree"); err != nil { - t.Fatal(err) - } - stdoutPipe.Close() - }() - - setTimeout(t, "Reading command output time out", 2*time.Second, func() { - cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout)) - if err != nil { - t.Fatal(err) - } - cmdOutput := string(cmdOutputBytes) - regexpStrings := []string{ - fmt.Sprintf("└─%s Virtual Size: \\d+.\\d+ MB Tags: %s:latest", unitTestImageIDShort, unitTestImageName), - "(?m) └─[0-9a-f]+.*", - "(?m) └─[0-9a-f]+.*", - "(?m) └─[0-9a-f]+.*", - fmt.Sprintf("(?m)^ └─%s Virtual Size: \\d+.\\d+ MB Tags: test:latest", utils.TruncateID(image.ID)), - } - - compiledRegexps := []*regexp.Regexp{} - for _, regexpString := range regexpStrings { - regexp, err := regexp.Compile(regexpString) - if err != nil { - fmt.Println("Error in regex string: ", err) - return - } - compiledRegexps = append(compiledRegexps, regexp) - } - - for _, regexp := range compiledRegexps { - if !regexp.MatchString(cmdOutput) { - t.Fatalf("images --tree content '%s' did not match regexp '%s'", cmdOutput, regexp) - } - } - }) -} - -func buildTestImages(t *testing.T, eng *engine.Engine) *image.Image { - - var testBuilder = testContextTemplate{ - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - } - image, err := buildImage(testBuilder, t, eng, true) - if err != nil { - t.Fatal(err) - } - - if err := eng.Job("tag", image.ID, "test").Run(); err != nil { - t.Fatal(err) - } - - return image -} - // #2098 - Docker cidFiles only contain short version of the containerId //sudo docker run --cidfile /tmp/docker_test.cid ubuntu echo "test" // TestRunCidFile tests that run --cidfile returns the longid From 95870ef3349920efc981a703d52f5832b79ee18e Mon Sep 17 00:00:00 2001 From: Felix Rabe Date: Sun, 6 Jul 2014 14:47:37 +0200 Subject: [PATCH 046/517] run.md: Fix references to cli There are now no other occurrences of `#cli-` in this document. TODO: Find other places with wrong links. --- docs/sources/reference/run.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/reference/run.md b/docs/sources/reference/run.md index a539ab0d1..82fe047cb 100644 --- a/docs/sources/reference/run.md +++ b/docs/sources/reference/run.md @@ -11,7 +11,7 @@ its own networking, and its own isolated process tree. The defaults related to the binary to run, the networking to expose, and more, but `docker run` gives final control to the operator who starts the container from the image. That's the main reason -[*run*](/reference/commandline/cli/#cli-run) has more options than any +[*run*](/reference/commandline/cli/#run) has more options than any other `docker` command. ## General Form @@ -21,7 +21,7 @@ The basic `docker run` command takes this form: $ docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] To learn how to interpret the types of `[OPTIONS]`, -see [*Option types*](/reference/commandline/cli/#cli-options). +see [*Option types*](/reference/commandline/cli/#option-types). The list of `[OPTIONS]` breaks down into two groups: From ebbfdf0a0f99aac67b8ad5d789d8a400be346c30 Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Mon, 7 Jul 2014 12:01:04 +0100 Subject: [PATCH 047/517] Reformat to keep line length within 80 characters Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) --- docs/sources/userguide/dockervolumes.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/sources/userguide/dockervolumes.md b/docs/sources/userguide/dockervolumes.md index f0725d4ad..959a01c50 100644 --- a/docs/sources/userguide/dockervolumes.md +++ b/docs/sources/userguide/dockervolumes.md @@ -73,17 +73,21 @@ option to specify that the mount should be read-only. ### Mount a Host File as a Data Volume -As well as directories, the `-v` flag can be used to mount a single file from the host into a container. +As well as directories, the `-v` flag can be used to mount a single file from +the host into a container. $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash -This will drop you into a bash shell in a new container, you will have your bash history from your host and when -you exit the container, the host will have the history of the commands typed while in the container. +This will drop you into a bash shell in a new container, you will have your bash +history from your host and when you exit the container, the host will have the +history of the commands typed while in the container. > **Note:** -> Many tools used to edit files including `vi` and `sed --in-place` may result in a inode change. Since docker v1.1.0 -> this will produce a error such as "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where you -> want to edit the mounted file, it is often easiest to instead mount the parent directory. +> Many tools used to edit files including `vi` and `sed --in-place` may result +> in a inode change. Since docker v1.1.0 this will produce a error such as +> "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where +> you want to edit the mounted file, it is often easiest to instead mount the +> parent directory. ## Creating and mounting a Data Volume Container From 2f13d3a8e541b918135ea0bf0153f56927dd0511 Mon Sep 17 00:00:00 2001 From: Andrea Turli Date: Mon, 7 Jul 2014 15:14:41 +0200 Subject: [PATCH 048/517] add jclouds-docker client reference Docker-DCO-1.1-Signed-off-by: Andrea Turli (github: andreaturli) Docker-DCO-1.1-Signed-off-by: Andrea Turli (github: ) --- docs/sources/reference/api/remote_api_client_libraries.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/sources/reference/api/remote_api_client_libraries.md b/docs/sources/reference/api/remote_api_client_libraries.md index 3dcebeeda..8f5080436 100644 --- a/docs/sources/reference/api/remote_api_client_libraries.md +++ b/docs/sources/reference/api/remote_api_client_libraries.md @@ -146,5 +146,11 @@ will add the libraries here. https://github.com/gesellix-docker/docker-client Active + + Java + jclouds-docker + https://github.com/jclouds/jclouds-labs/tree/master/docker + Active + From 963ee6ce795ae6608ee9acce4bc6298356d7f02f Mon Sep 17 00:00:00 2001 From: Chewey Date: Mon, 7 Jul 2014 17:52:18 +0400 Subject: [PATCH 049/517] Update running_riak_service.md --- docs/sources/examples/running_riak_service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/examples/running_riak_service.md b/docs/sources/examples/running_riak_service.md index 5909b7e2b..59dba0063 100644 --- a/docs/sources/examples/running_riak_service.md +++ b/docs/sources/examples/running_riak_service.md @@ -14,7 +14,7 @@ Create an empty file called `Dockerfile`: $ touch Dockerfile Next, define the parent image you want to use to build your image on top -of. We'll use [Ubuntu](https://registry.hub.docker.cm/_/ubuntu/) (tag: +of. We'll use [Ubuntu](https://registry.hub.docker.com/_/ubuntu/) (tag: `latest`), which is available on [Docker Hub](https://hub.docker.com): # Riak From 3589f5b9c2d9e5cb6d057713aa8d591295c5f523 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 08:09:25 -0600 Subject: [PATCH 050/517] Fix the systemd socket activation socket permissions Fixes #6836 Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- contrib/init/systemd/socket-activation/docker.socket | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/init/systemd/socket-activation/docker.socket b/contrib/init/systemd/socket-activation/docker.socket index 3635c8938..9db504915 100644 --- a/contrib/init/systemd/socket-activation/docker.socket +++ b/contrib/init/systemd/socket-activation/docker.socket @@ -3,6 +3,9 @@ Description=Docker Socket for the API [Socket] ListenStream=/var/run/docker.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker [Install] WantedBy=sockets.target From e895817ad72c681b265bc647d18f201ceced90c5 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 08:32:09 -0600 Subject: [PATCH 051/517] Update the Travis GitHub username regex to match the GitHub username requirements Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- hack/make/validate-dco | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hack/make/validate-dco b/hack/make/validate-dco index 6dbbe2250..6e915c9aa 100644 --- a/hack/make/validate-dco +++ b/hack/make/validate-dco @@ -9,13 +9,22 @@ notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')" : ${adds:=0} : ${dels:=0} +# "Username may only contain alphanumeric characters or dashes and cannot begin with a dash" +githubUsernameRegex='[a-zA-Z][a-zA-Z-]+' + +# https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md#sign-your-work +dcoPrefix='Docker-DCO-1.1-Signed-off-by:' +dcoRegex="^$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)> \\(github: ($githubUsernameRegex)\\)$" + +check_dco() { + grep -qE "$dcoRegex" +} + if [ $adds -eq 0 -a $dels -eq 0 ]; then echo '0 adds, 0 deletions; nothing to validate! :)' elif [ -z "$notDocs" -a $adds -le 1 -a $dels -le 1 ]; then echo 'Congratulations! DCO small-patch-exception material!' else - dcoPrefix='Docker-DCO-1.1-Signed-off-by:' - dcoRegex="^$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)> \\(github: (\S+)\\)$" commits=( $(validate_log --format='format:%H%n') ) badCommits=() for commit in "${commits[@]}"; do @@ -23,7 +32,7 @@ else # no content (ie, Merge commit, etc) continue fi - if ! git log -1 --format='format:%B' "$commit" | grep -qE "$dcoRegex"; then + if ! git log -1 --format='format:%B' "$commit" | check_dco; then badCommits+=( "$commit" ) fi done From bbbf64242726dee184f82b3bb355327c9c85b274 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 09:57:50 -0600 Subject: [PATCH 052/517] Fix DCO parsing for numeric github handles like cpuguy83 Turns out, "alphanumeric" actually means both "alpha" AND "numeric". Dur. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- hack/make/validate-dco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/make/validate-dco b/hack/make/validate-dco index 6e915c9aa..8b7812982 100644 --- a/hack/make/validate-dco +++ b/hack/make/validate-dco @@ -10,7 +10,7 @@ notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')" : ${dels:=0} # "Username may only contain alphanumeric characters or dashes and cannot begin with a dash" -githubUsernameRegex='[a-zA-Z][a-zA-Z-]+' +githubUsernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+' # https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md#sign-your-work dcoPrefix='Docker-DCO-1.1-Signed-off-by:' From 3e2ad6ffee68be7f3462fd1311408c7880cd8d88 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 10:04:23 -0600 Subject: [PATCH 053/517] Update Travis to use pretty Docker-based workers These start up much faster and the only caveat is that we can't use "sudo" (which we don't currently use anyhow). Also, I've updated the Go version here to match what's in the Dockerfile. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ae03d6cde..1e8d41184 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,12 @@ language: go -go: 1.2 +# This should match the version in the Dockerfile. +go: 1.2.1 + +# Let us have pretty experimental Docker-based Travis workers. +# (These spin up much faster than the VM-based ones.) +sudo: false # Disable the normal go build. install: true From 9f61e233c6da72f692190f1f5eb672341253f3e7 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 10:35:38 -0600 Subject: [PATCH 054/517] Standardize "curl" arguments across the repo to be or include "-sSL" - https://botbot.me/freenode/docker-dev/msg/17315692/ - https://botbot.me/freenode/docker-dev/msg/17315994/ Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- README.md | 2 +- contrib/mkimage-alpine.sh | 4 ++-- docs/sources/examples/running_riak_service.md | 2 +- docs/sources/installation/google.md | 2 +- docs/sources/installation/ubuntulinux.md | 4 ++-- hack/RELEASE-CHECKLIST.md | 2 +- hack/install.sh | 6 +++--- hack/release.sh | 5 ++++- 10 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d07b972eb..cd673b998 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -175,7 +175,7 @@ One way to automate this, is customise your get ``commit.template`` by adding a ``prepare-commit-msg`` hook to your docker checkout: ``` -curl -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg +curl -sSL -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg ``` * Note: the above script expects to find your GitHub user name in ``git config --get github.user`` diff --git a/Dockerfile b/Dockerfile index 283e0a326..d2e32a4ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ RUN cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go -RUN curl -s https://go.googlecode.com/files/go1.2.1.src.tar.gz | tar -v -C /usr/local -xz +RUN curl -sSL https://go.googlecode.com/files/go1.2.1.src.tar.gz | tar -v -C /usr/local -xz ENV PATH /usr/local/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/dotcloud/docker/vendor RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1 diff --git a/README.md b/README.md index 3c378de6f..08d839c3c 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Here's a typical Docker build process: FROM ubuntu:12.04 RUN apt-get update RUN apt-get install -q -y python python-pip curl -RUN curl -L https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xzv +RUN curl -sSL https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xzv RUN cd helloflask-master && pip install -r requirements.txt ``` diff --git a/contrib/mkimage-alpine.sh b/contrib/mkimage-alpine.sh index 0bf328efa..b9869ae61 100755 --- a/contrib/mkimage-alpine.sh +++ b/contrib/mkimage-alpine.sh @@ -19,12 +19,12 @@ tmp() { } apkv() { - curl -s $REPO/$ARCH/APKINDEX.tar.gz | tar -Oxz | + curl -sSL $REPO/$ARCH/APKINDEX.tar.gz | tar -Oxz | grep '^P:apk-tools-static$' -A1 | tail -n1 | cut -d: -f2 } getapk() { - curl -s $REPO/$ARCH/apk-tools-static-$(apkv).apk | + curl -sSL $REPO/$ARCH/apk-tools-static-$(apkv).apk | tar -xz -C $TMP sbin/apk.static } diff --git a/docs/sources/examples/running_riak_service.md b/docs/sources/examples/running_riak_service.md index 5909b7e2b..b2f9d99c4 100644 --- a/docs/sources/examples/running_riak_service.md +++ b/docs/sources/examples/running_riak_service.md @@ -59,7 +59,7 @@ After that, we install and setup a few dependencies: Next, we add Basho's APT repository: - RUN curl -s http://apt.basho.com/gpg/basho.apt.key | apt-key add -- + RUN curl -sSL http://apt.basho.com/gpg/basho.apt.key | apt-key add -- RUN echo "deb http://apt.basho.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/basho.list RUN apt-get update diff --git a/docs/sources/installation/google.md b/docs/sources/installation/google.md index b6c1b3d27..6c1e9b4f5 100644 --- a/docs/sources/installation/google.md +++ b/docs/sources/installation/google.md @@ -12,7 +12,7 @@ page_keywords: Docker, Docker documentation, installation, google, Google Comput 2. Download and configure the [Google Cloud SDK][3] to use your project with the following commands: - $ curl https://sdk.cloud.google.com | bash + $ curl -sSL https://sdk.cloud.google.com | bash $ gcloud auth login $ gcloud config set project diff --git a/docs/sources/installation/ubuntulinux.md b/docs/sources/installation/ubuntulinux.md index 5d1b6c3fb..5ddc791f4 100644 --- a/docs/sources/installation/ubuntulinux.md +++ b/docs/sources/installation/ubuntulinux.md @@ -63,7 +63,7 @@ continue installation.* > > There is also a simple `curl` script available to help with this process. > -> $ curl -s https://get.docker.io/ubuntu/ | sudo sh +> $ curl -sSL https://get.docker.io/ubuntu/ | sudo sh To verify that everything has worked as expected: @@ -134,7 +134,7 @@ continue installation.* > > There is also a simple `curl` script available to help with this process. > -> $ curl -s https://get.docker.io/ubuntu/ | sudo sh +> $ curl -sSL https://get.docker.io/ubuntu/ | sudo sh Now verify that the installation has worked by downloading the `ubuntu` image and launching a container. diff --git a/hack/RELEASE-CHECKLIST.md b/hack/RELEASE-CHECKLIST.md index 2fe1a3ce9..6dbff22bc 100644 --- a/hack/RELEASE-CHECKLIST.md +++ b/hack/RELEASE-CHECKLIST.md @@ -181,7 +181,7 @@ Announcing on IRC in both `#docker` and `#docker-dev` is a great way to get help testing! An easy way to get some useful links for sharing: ```bash -echo "Ubuntu/Debian install script: curl -sLS https://test.docker.io/ | sh" +echo "Ubuntu/Debian: https://test.docker.io/ubuntu or curl -sSL https://test.docker.io/ | sh" echo "Linux 64bit binary: https://test.docker.io/builds/Linux/x86_64/docker-${VERSION#v}" echo "Darwin/OSX 64bit client binary: https://test.docker.io/builds/Darwin/x86_64/docker-${VERSION#v}" echo "Darwin/OSX 32bit client binary: https://test.docker.io/builds/Darwin/i386/docker-${VERSION#v}" diff --git a/hack/install.sh b/hack/install.sh index 43248cf2c..641ec6415 100755 --- a/hack/install.sh +++ b/hack/install.sh @@ -2,7 +2,7 @@ set -e # # This script is meant for quick & easy install via: -# 'curl -sL https://get.docker.io/ | sh' +# 'curl -sSL https://get.docker.io/ | sh' # or: # 'wget -qO- https://get.docker.io/ | sh' # @@ -54,7 +54,7 @@ fi curl='' if command_exists curl; then - curl='curl -sL' + curl='curl -sSL' elif command_exists wget; then curl='wget -qO-' elif command_exists busybox && busybox --list-modules | grep -q wget; then @@ -133,7 +133,7 @@ case "$lsb_dist" in if [ -z "$curl" ]; then apt_get_update ( set -x; $sh_c 'sleep 3; apt-get install -y -q curl' ) - curl='curl -sL' + curl='curl -sSL' fi ( set -x diff --git a/hack/release.sh b/hack/release.sh index 8642a4edb..2a6b3992e 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -282,10 +282,13 @@ if [ ! -e /usr/lib/apt/methods/https ]; then apt-get update apt-get install -y apt-transport-https fi + # Add the repository to your APT sources echo deb $(s3_url)/ubuntu docker main > /etc/apt/sources.list.d/docker.list + # Then import the repository key apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 + # Install docker apt-get update ; apt-get install -y lxc-docker @@ -318,7 +321,7 @@ release_binaries() { cat < Date: Mon, 7 Jul 2014 10:50:48 -0600 Subject: [PATCH 055/517] Update Makefile to default BINDDIR to nothing if DOCKER_HOST is set If "DOCKER_HOST" is set, we can usually assume the user is connecting to a remote Docker and thus not bind mount anything by default (meaning the Makefile will more often DWIM for our users). Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2d07b39c3..444ac3c6a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ .PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli validate # to allow `make BINDDIR=. shell` or `make BINDDIR= test` -BINDDIR := bundles +# (default to no bind mount if DOCKER_HOST is set) +BINDDIR := $(if $(DOCKER_HOST),,bundles) # to allow `make DOCSPORT=9000 docs` DOCSPORT := 8000 From 600f65b24786d15a382877da4c3ae77d88e252bb Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 7 Jul 2014 14:58:27 -0400 Subject: [PATCH 056/517] docker save: more integration tests Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) --- integration-cli/docker_cli_save_load_test.go | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/integration-cli/docker_cli_save_load_test.go b/integration-cli/docker_cli_save_load_test.go index fb94cad9d..c70769e03 100644 --- a/integration-cli/docker_cli_save_load_test.go +++ b/integration-cli/docker_cli_save_load_test.go @@ -59,6 +59,63 @@ func TestSaveAndLoadRepoStdout(t *testing.T) { logDone("load - load a repo using stdout") } +func TestSaveSingleTag(t *testing.T) { + repoName := "foobar-save-single-tag-test" + + tagCmdFinal := fmt.Sprintf("%v tag busybox:latest %v:latest", dockerBinary, repoName) + tagCmd := exec.Command("bash", "-c", tagCmdFinal) + out, _, err := runCommandWithOutput(tagCmd) + errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err)) + + idCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName) + idCmd := exec.Command("bash", "-c", idCmdFinal) + out, _, err = runCommandWithOutput(idCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err)) + + cleanedImageID := stripTrailingCharacters(out) + + saveCmdFinal := fmt.Sprintf("%v save %v:latest | tar t | grep -E '(^repositories$|%v)'", dockerBinary, repoName, cleanedImageID) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + errorOut(err, t, fmt.Sprintf("failed to save repo with image ID and 'repositories' file: %v %v", out, err)) + + deleteImages(repoName) + + logDone("save - save a specific image:tag") +} + +func TestSaveImageId(t *testing.T) { + repoName := "foobar-save-image-id-test" + + tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v:latest", dockerBinary, repoName) + tagCmd := exec.Command("bash", "-c", tagCmdFinal) + out, _, err := runCommandWithOutput(tagCmd) + errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err)) + + idLongCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName) + idLongCmd := exec.Command("bash", "-c", idLongCmdFinal) + out, _, err = runCommandWithOutput(idLongCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err)) + + cleanedLongImageID := stripTrailingCharacters(out) + + idShortCmdFinal := fmt.Sprintf("%v images -q %v", dockerBinary, repoName) + idShortCmd := exec.Command("bash", "-c", idShortCmdFinal) + out, _, err = runCommandWithOutput(idShortCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo short ID: %v %v", out, err)) + + cleanedShortImageID := stripTrailingCharacters(out) + + saveCmdFinal := fmt.Sprintf("%v save %v | tar t | grep %v", dockerBinary, cleanedShortImageID, cleanedLongImageID) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + errorOut(err, t, fmt.Sprintf("failed to save repo with image ID: %v %v", out, err)) + + deleteImages(repoName) + + logDone("save - save a image by ID") +} + // save a repo and try to load it using flags func TestSaveAndLoadRepoFlags(t *testing.T) { runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true") From ea583fda97617a349ef026dd2d99a0d5e4b0980d Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Sun, 6 Jul 2014 10:38:30 +0000 Subject: [PATCH 057/517] We haven't required lxc and aufs for years now... Well, maybe not years, but internet years... Docker-DCO-1.1-Signed-off-by: Timothy Hobbs (github: timthelion) --- hack/ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/ROADMAP.md b/hack/ROADMAP.md index c38be56f4..ff797b380 100644 --- a/hack/ROADMAP.md +++ b/hack/ROADMAP.md @@ -31,9 +31,9 @@ We are working on a plugin API which will make Docker very, very customization-f ## Broader kernel support -Our goal is to make Docker run everywhere, but currently Docker requires Linux version 3.8 or higher with lxc and aufs support. If you’re deploying new machines for the purpose of running Docker, this is a fairly easy requirement to meet. However, if you’re adding Docker to an existing deployment, you may not have the flexibility to update and patch the kernel. +Our goal is to make Docker run everywhere, but currently Docker requires Linux version 3.8 or higher with cgroups support. If you’re deploying new machines for the purpose of running Docker, this is a fairly easy requirement to meet. However, if you’re adding Docker to an existing deployment, you may not have the flexibility to update and patch the kernel. -Expanding Docker’s kernel support is a priority. This includes running on older kernel versions, but also on kernels with no AUFS support, or with incomplete lxc capabilities. +Expanding Docker’s kernel support is a priority. This includes running on older kernel versions, specifically focusing on versions already popular in server deployments such as those used by RHEL and the OpenVZ stack. ## Cross-architecture support From 627805f5f8895c3a3887c312f7b8f49a89e51732 Mon Sep 17 00:00:00 2001 From: Guilherme Salgado Date: Wed, 2 Jul 2014 10:01:19 +0200 Subject: [PATCH 058/517] Convert TestGetImagesJSON into several unit tests Docker-DCO-1.1-Signed-off-by: Guilherme Salgado (github: gsalgado) --- api/server/server_unit_test.go | 172 ++++++++++++++++++++++++++++++--- integration/api_test.go | 104 -------------------- 2 files changed, 157 insertions(+), 119 deletions(-) diff --git a/api/server/server_unit_test.go b/api/server/server_unit_test.go index f8068a29c..c51ea6fc0 100644 --- a/api/server/server_unit_test.go +++ b/api/server/server_unit_test.go @@ -7,11 +7,13 @@ import ( "io" "net/http" "net/http/httptest" + "reflect" "strings" "testing" "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/pkg/version" ) func TestGetBoolParam(t *testing.T) { @@ -111,8 +113,105 @@ func TestGetInfo(t *testing.T) { if v.GetInt("Containers") != 1 { t.Fatalf("%#v\n", v) } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) + assertContentType(r, "application/json", t) +} + +func TestGetImagesJSON(t *testing.T) { + eng := engine.New() + var called bool + eng.Register("images", func(job *engine.Job) engine.Status { + called = true + v := createEnvFromGetImagesJSONStruct(sampleImage) + if _, err := v.WriteTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequest("GET", "/images/json", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + assertHttpNotError(r, t) + assertContentType(r, "application/json", t) + var observed getImagesJSONStruct + if err := json.Unmarshal(r.Body.Bytes(), &observed); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(observed, sampleImage) { + t.Errorf("Expected %#v but got %#v", sampleImage, observed) + } +} + +func TestGetImagesJSONFilter(t *testing.T) { + eng := engine.New() + filter := "nothing" + eng.Register("images", func(job *engine.Job) engine.Status { + filter = job.Getenv("filter") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?filter=aaaa", nil, eng, t) + if filter != "aaaa" { + t.Errorf("%#v", filter) + } +} + +func TestGetImagesJSONFilters(t *testing.T) { + eng := engine.New() + filter := "nothing" + eng.Register("images", func(job *engine.Job) engine.Status { + filter = job.Getenv("filters") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?filters=nnnn", nil, eng, t) + if filter != "nnnn" { + t.Errorf("%#v", filter) + } +} + +func TestGetImagesJSONAll(t *testing.T) { + eng := engine.New() + allFilter := "-1" + eng.Register("images", func(job *engine.Job) engine.Status { + allFilter = job.Getenv("all") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?all=1", nil, eng, t) + if allFilter != "1" { + t.Errorf("%#v", allFilter) + } +} + +func TestGetImagesJSONLegacyFormat(t *testing.T) { + eng := engine.New() + var called bool + eng.Register("images", func(job *engine.Job) engine.Status { + called = true + outsLegacy := engine.NewTable("Created", 0) + outsLegacy.Add(createEnvFromGetImagesJSONStruct(sampleImage)) + if _, err := outsLegacy.WriteListTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequestUsingVersion("GET", "/images/json", "1.6", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + assertHttpNotError(r, t) + assertContentType(r, "application/json", t) + images := engine.NewTable("Created", 0) + if _, err := images.ReadListFrom(r.Body.Bytes()); err != nil { + t.Fatal(err) + } + if images.Len() != 1 { + t.Fatalf("Expected 1 image, %d found", images.Len()) + } + image := images.Data[0] + if image.Get("Tag") != "test-tag" { + t.Errorf("Expected tag 'test-tag', found '%s'", image.Get("Tag")) + } + if image.Get("Repository") != "test-name" { + t.Errorf("Expected repository 'test-name', found '%s'", image.Get("Repository")) } } @@ -123,12 +222,12 @@ func TestGetContainersByName(t *testing.T) { eng.Register("container_inspect", func(job *engine.Job) engine.Status { called = true if job.Args[0] != name { - t.Fatalf("name != '%s': %#v", name, job.Args[0]) + t.Errorf("name != '%s': %#v", name, job.Args[0]) } if api.APIVERSION.LessThan("1.12") && !job.GetenvBool("dirty") { - t.Fatal("dirty env variable not set") + t.Errorf("dirty env variable not set") } else if api.APIVERSION.GreaterThanOrEqualTo("1.12") && job.GetenvBool("dirty") { - t.Fatal("dirty env variable set when it shouldn't") + t.Errorf("dirty env variable set when it shouldn't") } v := &engine.Env{} v.SetBool("dirty", true) @@ -141,9 +240,7 @@ func TestGetContainersByName(t *testing.T) { if !called { t.Fatal("handler was not called") } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) - } + assertContentType(r, "application/json", t) var stdoutJson interface{} if err := json.Unmarshal(r.Body.Bytes(), &stdoutJson); err != nil { t.Fatalf("%#v", err) @@ -178,21 +275,19 @@ func TestGetEvents(t *testing.T) { if !called { t.Fatal("handler was not called") } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) - } + assertContentType(r, "application/json", t) var stdout_json struct { Since int Until int } if err := json.Unmarshal(r.Body.Bytes(), &stdout_json); err != nil { - t.Fatalf("%#v", err) + t.Fatal(err) } if stdout_json.Since != 1 { - t.Fatalf("since != 1: %#v", stdout_json.Since) + t.Errorf("since != 1: %#v", stdout_json.Since) } if stdout_json.Until != 0 { - t.Fatalf("until != 0: %#v", stdout_json.Until) + t.Errorf("until != 0: %#v", stdout_json.Until) } } @@ -357,12 +452,16 @@ func TestGetImagesByName(t *testing.T) { } func serveRequest(method, target string, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { + return serveRequestUsingVersion(method, target, api.APIVERSION, body, eng, t) +} + +func serveRequestUsingVersion(method, target string, version version.Version, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { r := httptest.NewRecorder() req, err := http.NewRequest(method, target, body) if err != nil { t.Fatal(err) } - if err := ServeRequest(eng, api.APIVERSION, r, req); err != nil { + if err := ServeRequest(eng, version, r, req); err != nil { t.Fatal(err) } return r @@ -388,3 +487,46 @@ func toJson(data interface{}, t *testing.T) io.Reader { } return &buf } + +func assertContentType(recorder *httptest.ResponseRecorder, content_type string, t *testing.T) { + if recorder.HeaderMap.Get("Content-Type") != content_type { + t.Fatalf("%#v\n", recorder) + } +} + +// XXX: Duplicated from integration/utils_test.go, but maybe that's OK as that +// should die as soon as we converted all integration tests? +// assertHttpNotError expect the given response to not have an error. +// Otherwise the it causes the test to fail. +func assertHttpNotError(r *httptest.ResponseRecorder, t *testing.T) { + // Non-error http status are [200, 400) + if r.Code < http.StatusOK || r.Code >= http.StatusBadRequest { + t.Fatal(fmt.Errorf("Unexpected http error: %v", r.Code)) + } +} + +func createEnvFromGetImagesJSONStruct(data getImagesJSONStruct) *engine.Env { + v := &engine.Env{} + v.SetList("RepoTags", data.RepoTags) + v.Set("Id", data.Id) + v.SetInt64("Created", data.Created) + v.SetInt64("Size", data.Size) + v.SetInt64("VirtualSize", data.VirtualSize) + return v +} + +type getImagesJSONStruct struct { + RepoTags []string + Id string + Created int64 + Size int64 + VirtualSize int64 +} + +var sampleImage getImagesJSONStruct = getImagesJSONStruct{ + RepoTags: []string{"test-name:test-tag"}, + Id: "ID", + Created: 999, + Size: 777, + VirtualSize: 666, +} diff --git a/integration/api_test.go b/integration/api_test.go index f5e5599e3..0da96ccd1 100644 --- a/integration/api_test.go +++ b/integration/api_test.go @@ -9,7 +9,6 @@ import ( "net" "net/http" "net/http/httptest" - "strings" "testing" "time" @@ -20,109 +19,6 @@ import ( "github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" ) -func TestGetImagesJSON(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - job := eng.Job("images") - initialImages, err := job.Stdout.AddListTable() - if err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("GET", "/images/json?all=0", nil) - if err != nil { - t.Fatal(err) - } - - r := httptest.NewRecorder() - - if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { - t.Fatal(err) - } - assertHttpNotError(r, t) - - images := engine.NewTable("Created", 0) - if _, err := images.ReadListFrom(r.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images.Len() != initialImages.Len() { - t.Errorf("Expected %d image, %d found", initialImages.Len(), images.Len()) - } - - found := false - for _, img := range images.Data { - if strings.Contains(img.GetList("RepoTags")[0], unitTestImageName) { - found = true - break - } - } - if !found { - t.Errorf("Expected image %s, %+v found", unitTestImageName, images) - } - - r2 := httptest.NewRecorder() - - // all=1 - - initialImages = getAllImages(eng, t) - - req2, err := http.NewRequest("GET", "/images/json?all=true", nil) - if err != nil { - t.Fatal(err) - } - if err := server.ServeRequest(eng, api.APIVERSION, r2, req2); err != nil { - t.Fatal(err) - } - assertHttpNotError(r2, t) - - images2 := engine.NewTable("Id", 0) - if _, err := images2.ReadListFrom(r2.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images2.Len() != initialImages.Len() { - t.Errorf("Expected %d image, %d found", initialImages.Len(), images2.Len()) - } - - found = false - for _, img := range images2.Data { - if img.Get("Id") == unitTestImageID { - found = true - break - } - } - if !found { - t.Errorf("Retrieved image Id differs, expected %s, received %+v", unitTestImageID, images2) - } - - r3 := httptest.NewRecorder() - - // filter=a - req3, err := http.NewRequest("GET", "/images/json?filter=aaaaaaaaaa", nil) - if err != nil { - t.Fatal(err) - } - - if err := server.ServeRequest(eng, api.APIVERSION, r3, req3); err != nil { - t.Fatal(err) - } - assertHttpNotError(r3, t) - - images3 := engine.NewTable("Id", 0) - if _, err := images3.ReadListFrom(r3.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images3.Len() != 0 { - t.Errorf("Expected 0 image, %d found", images3.Len()) - } -} - func TestGetContainersJSON(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() From fd66e7746519ab32bf3f6f336052a4ff763e213b Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Mon, 7 Jul 2014 13:44:22 -0700 Subject: [PATCH 059/517] Update the footer version with Enterprise and Partners sections Docker-DCO-1.1-Signed-off-by: Roberto Hashioka (github: rogaha) --- docs/theme/mkdocs/footer.html | 49 ++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/docs/theme/mkdocs/footer.html b/docs/theme/mkdocs/footer.html index 1e2d73cf1..0b887b82d 100644 --- a/docs/theme/mkdocs/footer.html +++ b/docs/theme/mkdocs/footer.html @@ -1,13 +1,13 @@