Bump version to v1.6.0

Docker-DCO-1.1-Signed-off-by: Jessie Frazelle <jess@docker.com> (github: jfrazelle)

Docker-DCO-1.1-Signed-off-by: Jessie Frazelle <princess@docker.com> (github: jfrazelle)

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 1.6.0 (2015-04-07)
+
+#### Builder
++ Building images from an image ID
++ build containers with resource constraints, ie `docker build --cpu-shares=100 --memory=1024m...`
++ `commit --change` to apply specified Dockerfile instructions while committing the image
++ `import --change` to apply specified Dockerfile instructions while importing the image
+
+#### Client
++ Windows Support
+
+#### Runtime
++ Container and image Labels
++ `--cgroup-parent` for specifying a parent cgroup to place container cgroup within
++ Logging drivers, `json-file` or `syslog`
++ Pulling images by ID
++ `--ulimit` to set the ulimit on a container
++ `--default-ulimit` option on the daemon which applies to all created containers (and overwritten by `--ulimit` on run)
+
 ## 1.5.0 (2015-02-10)
 
 #### Builder

VERSION

@@ -1 +1 @@
-1.5.0-dev
+1.6.0-rc2

api/client/commands.go

@@ -769,18 +769,6 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 	cmd.Require(flag.Min, 1)
 	utils.ParseFlags(cmd, args, true)
 
-	hijacked := make(chan io.Closer)
-	// Block the return until the chan gets closed
-	defer func() {
-		log.Debugf("CmdStart() returned, defer waiting for hijack to finish.")
-		if _, ok := <-hijacked; ok {
-			log.Errorf("Hijack did not finish (chan still open)")
-		}
-		if *openStdin || *attach {
-			cli.in.Close()
-		}
-	}()
-
 	if *attach || *openStdin {
 		if cmd.NArg() > 1 {
 			return fmt.Errorf("You cannot start and attach multiple containers at once.")
@@ -816,26 +804,34 @@ func (cli *DockerCli) CmdStart(args ...string) error {
 		v.Set("stdout", "1")
 		v.Set("stderr", "1")
 
+		hijacked := make(chan io.Closer)
+		// Block the return until the chan gets closed
+		defer func() {
+			log.Debugf("CmdStart() returned, defer waiting for hijack to finish.")
+			if _, ok := <-hijacked; ok {
+				log.Errorf("Hijack did not finish (chan still open)")
+			}
+			cli.in.Close()
+		}()
 		cErr = promise.Go(func() error {
 			return cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), tty, in, cli.out, cli.err, hijacked, nil)
 		})
-	} else {
+
-		close(hijacked)
+		// Acknowledge the hijack before starting
+		select {
+		case closer := <-hijacked:
+			// Make sure that the hijack gets closed when returning (results
+			// in closing the hijack chan and freeing server's goroutines)
+			if closer != nil {
+				defer closer.Close()
+			}
+		case err := <-cErr:
+			if err != nil {
+				return err
+			}
+		}
 	}
 
-	// Acknowledge the hijack before starting
-	select {
-	case closer := <-hijacked:
-		// Make sure that the hijack gets closed when returning (results
-		// in closing the hijack chan and freeing server's goroutines)
-		if closer != nil {
-			defer closer.Close()
-		}
-	case err := <-cErr:
-		if err != nil {
-			return err
-		}
-	}
 	var encounteredError error
 	for _, name := range cmd.Args() {
 		_, _, err := readBody(cli.call("POST", "/containers/"+name+"/start", nil, false))

builder/evaluator.go

@@ -312,7 +312,11 @@ func (b *Builder) dispatch(stepN int, ast *parser.Node) error {
 		var str string
 		str = ast.Value
 		if _, ok := replaceEnvAllowed[cmd]; ok {
-			str = b.replaceEnv(ast.Value)
+			var err error
+			str, err = ProcessWord(ast.Value, b.Config.Env)
+			if err != nil {
+				return err
+			}
 		}
 		strList[i+l] = str
 		msgList[i] = ast.Value

builder/parser/line_parsers.go

@@ -90,7 +90,7 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 				if blankOK || len(word) > 0 {
 					words = append(words, word)
 
-					// Look for = and if no there assume
+					// Look for = and if not there assume
 					// we're doing the old stuff and
 					// just read the rest of the line
 					if !strings.Contains(word, "=") {
@@ -107,12 +107,15 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 				quote = ch
 				blankOK = true
 				phase = inQuote
-				continue
 			}
 			if ch == '\\' {
 				if pos+1 == len(rest) {
 					continue // just skip \ at end
 				}
+				// If we're not quoted and we see a \, then always just
+				// add \ plus the char to the word, even if the char
+				// is a quote.
+				word += string(ch)
 				pos++
 				ch = rune(rest[pos])
 			}
@@ -122,15 +125,17 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 		if phase == inQuote {
 			if ch == quote {
 				phase = inWord
-				continue
 			}
-			if ch == '\\' {
+			// \ is special except for ' quotes - can't escape anything for '
+			if ch == '\\' && quote != '\'' {
 				if pos+1 == len(rest) {
 					phase = inWord
 					continue // just skip \ at end
 				}
 				pos++
-				ch = rune(rest[pos])
+				nextCh := rune(rest[pos])
+				word += string(ch)
+				ch = nextCh
 			}
 			word += string(ch)
 		}

builder/parser/testfiles/env/Dockerfile

@@ -7,6 +7,14 @@ ENV name=value\ value2
 ENV name="value'quote space'value2"
 ENV name='value"double quote"value2'
 ENV name=value\ value2 name2=value2\ value3
+ENV name="a\"b"
+ENV name="a\'b"
+ENV name='a\'b'
+ENV name='a\'b''
+ENV name='a\"b'
+ENV name="''"
+# don't put anything after the next line - it must be the last line of the
+# Dockerfile and it must end with \
 ENV name=value \
     name1=value1 \
     name2="value2a \

builder/parser/testfiles/env/result

@@ -2,9 +2,15 @@
 (env "name" "value")
 (env "name" "value")
 (env "name" "value" "name2" "value2")
-(env "name" "value value1")
+(env "name" "\"value value1\"")
-(env "name" "value value2")
+(env "name" "value\\ value2")
-(env "name" "value'quote space'value2")
+(env "name" "\"value'quote space'value2\"")
-(env "name" "value\"double quote\"value2")
+(env "name" "'value\"double quote\"value2'")
-(env "name" "value value2" "name2" "value2 value3")
+(env "name" "value\\ value2" "name2" "value2\\ value3")
-(env "name" "value" "name1" "value1" "name2" "value2a            value2b" "name3" "value3an\"value3b\"" "name4" "value4a\\nvalue4b")
+(env "name" "\"a\\\"b\"")
+(env "name" "\"a\\'b\"")
+(env "name" "'a\\'b'")
+(env "name" "'a\\'b''")
+(env "name" "'a\\\"b'")
+(env "name" "\"''\"")
+(env "name" "value" "name1" "value1" "name2" "\"value2a            value2b\"" "name3" "\"value3a\\n\\\"value3b\\\"\"" "name4" "\"value4a\\\\nvalue4b\"")

builder/shell_parser.go

@@ -0,0 +1,209 @@
+package builder
+
+// This will take a single word and an array of env variables and
+// process all quotes (" and ') as well as $xxx and ${xxx} env variable
+// tokens.  Tries to mimic bash shell process.
+// It doesn't support all flavors of ${xx:...} formats but new ones can
+// be added by adding code to the "special ${} format processing" section
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+)
+
+type shellWord struct {
+	word string
+	envs []string
+	pos  int
+}
+
+func ProcessWord(word string, env []string) (string, error) {
+	sw := &shellWord{
+		word: word,
+		envs: env,
+		pos:  0,
+	}
+	return sw.process()
+}
+
+func (sw *shellWord) process() (string, error) {
+	return sw.processStopOn('\000')
+}
+
+// Process the word, starting at 'pos', and stop when we get to the
+// end of the word or the 'stopChar' character
+func (sw *shellWord) processStopOn(stopChar rune) (string, error) {
+	var result string
+	var charFuncMapping = map[rune]func() (string, error){
+		'\'': sw.processSingleQuote,
+		'"':  sw.processDoubleQuote,
+		'$':  sw.processDollar,
+	}
+
+	for sw.pos < len(sw.word) {
+		ch := sw.peek()
+		if stopChar != '\000' && ch == stopChar {
+			sw.next()
+			break
+		}
+		if fn, ok := charFuncMapping[ch]; ok {
+			// Call special processing func for certain chars
+			tmp, err := fn()
+			if err != nil {
+				return "", err
+			}
+			result += tmp
+		} else {
+			// Not special, just add it to the result
+			ch = sw.next()
+			if ch == '\\' {
+				// '\' escapes, except end of line
+				ch = sw.next()
+				if ch == '\000' {
+					continue
+				}
+			}
+			result += string(ch)
+		}
+	}
+
+	return result, nil
+}
+
+func (sw *shellWord) peek() rune {
+	if sw.pos == len(sw.word) {
+		return '\000'
+	}
+	return rune(sw.word[sw.pos])
+}
+
+func (sw *shellWord) next() rune {
+	if sw.pos == len(sw.word) {
+		return '\000'
+	}
+	ch := rune(sw.word[sw.pos])
+	sw.pos++
+	return ch
+}
+
+func (sw *shellWord) processSingleQuote() (string, error) {
+	// All chars between single quotes are taken as-is
+	// Note, you can't escape '
+	var result string
+
+	sw.next()
+
+	for {
+		ch := sw.next()
+		if ch == '\000' || ch == '\'' {
+			break
+		}
+		result += string(ch)
+	}
+	return result, nil
+}
+
+func (sw *shellWord) processDoubleQuote() (string, error) {
+	// All chars up to the next " are taken as-is, even ', except any $ chars
+	// But you can escape " with a \
+	var result string
+
+	sw.next()
+
+	for sw.pos < len(sw.word) {
+		ch := sw.peek()
+		if ch == '"' {
+			sw.next()
+			break
+		}
+		if ch == '$' {
+			tmp, err := sw.processDollar()
+			if err != nil {
+				return "", err
+			}
+			result += tmp
+		} else {
+			ch = sw.next()
+			if ch == '\\' {
+				chNext := sw.peek()
+
+				if chNext == '\000' {
+					// Ignore \ at end of word
+					continue
+				}
+
+				if chNext == '"' || chNext == '$' {
+					// \" and \$ can be escaped, all other \'s are left as-is
+					ch = sw.next()
+				}
+			}
+			result += string(ch)
+		}
+	}
+
+	return result, nil
+}
+
+func (sw *shellWord) processDollar() (string, error) {
+	sw.next()
+	ch := sw.peek()
+	if ch == '{' {
+		sw.next()
+		name := sw.processName()
+		ch = sw.peek()
+		if ch == '}' {
+			// Normal ${xx} case
+			sw.next()
+			return sw.getEnv(name), nil
+		}
+		return "", fmt.Errorf("Unsupported ${} substitution: %s", sw.word)
+	} else {
+		// $xxx case
+		name := sw.processName()
+		if name == "" {
+			return "$", nil
+		}
+		return sw.getEnv(name), nil
+	}
+}
+
+func (sw *shellWord) processName() string {
+	// Read in a name (alphanumeric or _)
+	// If it starts with a numeric then just return $#
+	var name string
+
+	for sw.pos < len(sw.word) {
+		ch := sw.peek()
+		if len(name) == 0 && unicode.IsDigit(ch) {
+			ch = sw.next()
+			return string(ch)
+		}
+		if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) && ch != '_' {
+			break
+		}
+		ch = sw.next()
+		name += string(ch)
+	}
+
+	return name
+}
+
+func (sw *shellWord) getEnv(name string) string {
+	for _, env := range sw.envs {
+		i := strings.Index(env, "=")
+		if i < 0 {
+			if name == env {
+				// Should probably never get here, but just in case treat
+				// it like "var" and "var=" are the same
+				return ""
+			}
+			continue
+		}
+		if name != env[:i] {
+			continue
+		}
+		return env[i+1:]
+	}
+	return ""
+}

builder/shell_parser_test.go

@@ -0,0 +1,51 @@
+package builder
+
+import (
+	"bufio"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestShellParser(t *testing.T) {
+	file, err := os.Open("words")
+	if err != nil {
+		t.Fatalf("Can't open 'words': %s", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	envs := []string{"PWD=/home", "SHELL=bash"}
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// Trim comments and blank lines
+		i := strings.Index(line, "#")
+		if i >= 0 {
+			line = line[:i]
+		}
+		line = strings.TrimSpace(line)
+
+		if line == "" {
+			continue
+		}
+
+		words := strings.Split(line, "|")
+		if len(words) != 2 {
+			t.Fatalf("Error in 'words' - should be 2 words:%q", words)
+		}
+
+		words[0] = strings.TrimSpace(words[0])
+		words[1] = strings.TrimSpace(words[1])
+
+		newWord, err := ProcessWord(words[0], envs)
+
+		if err != nil {
+			newWord = "error"
+		}
+
+		if newWord != words[1] {
+			t.Fatalf("Error. Src: %s  Calc: %s  Expected: %s", words[0], newWord, words[1])
+		}
+	}
+}

builder/support.go

@@ -1,50 +1,9 @@
 package builder
 
 import (
-	"regexp"
 	"strings"
 )
 
-var (
-	// `\\\\+|[^\\]|\b|\A` - match any number of "\\" (ie, properly-escaped backslashes), or a single non-backslash character, or a word boundary, or beginning-of-line
-	// `\$` - match literal $
-	// `[[:alnum:]_]+` - match things like `$SOME_VAR`
-	// `{[[:alnum:]_]+}` - match things like `${SOME_VAR}`
-	tokenEnvInterpolation = regexp.MustCompile(`(\\|\\\\+|[^\\]|\b|\A)\$([[:alnum:]_]+|{[[:alnum:]_]+})`)
-	// this intentionally punts on more exotic interpolations like ${SOME_VAR%suffix} and lets the shell handle those directly
-)
-
-// handle environment replacement. Used in dispatcher.
-func (b *Builder) replaceEnv(str string) string {
-	for _, match := range tokenEnvInterpolation.FindAllString(str, -1) {
-		idx := strings.Index(match, "\\$")
-		if idx != -1 {
-			if idx+2 >= len(match) {
-				str = strings.Replace(str, match, "\\$", -1)
-				continue
-			}
-
-			prefix := match[:idx]
-			stripped := match[idx+2:]
-			str = strings.Replace(str, match, prefix+"$"+stripped, -1)
-			continue
-		}
-
-		match = match[strings.Index(match, "$"):]
-		matchKey := strings.Trim(match, "${}")
-
-		for _, keyval := range b.Config.Env {
-			tmp := strings.SplitN(keyval, "=", 2)
-			if tmp[0] == matchKey {
-				str = strings.Replace(str, match, tmp[1], -1)
-				break
-			}
-		}
-	}
-
-	return str
-}
-
 func handleJsonArgs(args []string, attributes map[string]bool) []string {
 	if len(args) == 0 {
 		return []string{}

builder/words

@@ -0,0 +1,43 @@
+hello                    |     hello
+he'll'o                  |     hello
+he'llo                   |     hello
+he\'llo                  |     he'llo
+he\\'llo                 |     he\llo
+abc\tdef                 |     abctdef
+"abc\tdef"               |     abc\tdef
+'abc\tdef'               |     abc\tdef
+hello\                   |     hello
+hello\\                  |     hello\
+"hello                   |     hello
+"hello\"                 |     hello"
+"hel'lo"                 |     hel'lo
+'hello                   |     hello
+'hello\'                 |     hello\
+"''"                     |     ''
+$.                       |     $.
+$1                       |     
+he$1x                    |     hex
+he$.x                    |     he$.x
+he$pwd.                  |     he.
+he$PWD                   |     he/home
+he\$PWD                  |     he$PWD
+he\\$PWD                 |     he\/home
+he\${}                   |     he${}
+he\${}xx                 |     he${}xx
+he${}                    |     he
+he${}xx                  |     hexx
+he${hi}                  |     he
+he${hi}xx                |     hexx
+he${PWD}                 |     he/home
+he${.}                   |     error
+'he${XX}'                |     he${XX}
+"he${PWD}"               |     he/home
+"he'$PWD'"               |     he'/home'
+"$PWD"                   |     /home
+'$PWD'                   |     $PWD
+'\$PWD'                  |     \$PWD
+'"hello"'                |     "hello"
+he\$PWD                  |     he$PWD
+"he\$PWD"                |     he$PWD
+'he\$PWD'                |     he\$PWD
+he${PWD                  |     error

contrib/completion/bash/docker

@@ -325,7 +325,7 @@ _docker_cp() {
 			(( counter++ ))
 
 			if [ $cword -eq $counter ]; then
-				_filedir
+				_filedir -d
 				return
 			fi
 			;;

daemon/config.go

@@ -83,7 +83,7 @@ func (config *Config) InstallFlags() {
 	opts.LabelListVar(&config.Labels, []string{"-label"}, "Set key=value labels to the daemon")
 	config.Ulimits = make(map[string]*ulimit.Ulimit)
 	opts.UlimitMapVar(config.Ulimits, []string{"-default-ulimit"}, "Set default ulimits for containers")
-	flag.StringVar(&config.LogConfig.Type, []string{"-log-driver"}, "json-file", "Containers logging driver(json-file/none)")
+	flag.StringVar(&config.LogConfig.Type, []string{"-log-driver"}, "json-file", "Containers logging driver")
 }
 
 func getDefaultNetworkMtu() int {

daemon/container.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/daemon/execdriver"
 	"github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/daemon/logger/jsonfilelog"
+	"github.com/docker/docker/daemon/logger/syslog"
 	"github.com/docker/docker/engine"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/links"
@@ -1380,6 +1381,12 @@ func (container *Container) startLogging() error {
 			return err
 		}
 		l = dl
+	case "syslog":
+		dl, err := syslog.New(container.ID[:12])
+		if err != nil {
+			return err
+		}
+		l = dl
 	case "none":
 		return nil
 	default:

daemon/daemon.go

@@ -1012,7 +1012,8 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error)
 	}
 
 	sysInfo := sysinfo.New(false)
-	ed, err := execdrivers.NewDriver(config.ExecDriver, config.Root, sysInitPath, sysInfo)
+	const runDir = "/var/run/docker"
+	ed, err := execdrivers.NewDriver(config.ExecDriver, runDir, config.Root, sysInitPath, sysInfo)
 	if err != nil {
 		return nil, err
 	}

daemon/execdriver/execdrivers/execdrivers.go

@@ -10,13 +10,13 @@ import (
 	"github.com/docker/docker/pkg/sysinfo"
 )
 
-func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
+func NewDriver(name, root, libPath, initPath string, sysInfo *sysinfo.SysInfo) (execdriver.Driver, error) {
 	switch name {
 	case "lxc":
 		// we want to give the lxc driver the full docker root because it needs
 		// to access and write config and template files in /var/lib/docker/containers/*
 		// to be backwards compatible
-		return lxc.NewDriver(root, initPath, sysInfo.AppArmor)
+		return lxc.NewDriver(root, libPath, initPath, sysInfo.AppArmor)
 	case "native":
 		return native.NewDriver(path.Join(root, "execdriver", "native"), initPath)
 	}

daemon/execdriver/lxc/driver.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/docker/daemon/execdriver"
 	sysinfo "github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/term"
+	"github.com/docker/docker/pkg/version"
 	"github.com/docker/docker/utils"
 	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/cgroups"
@@ -35,6 +36,7 @@ var ErrExec = errors.New("Unsupported: Exec is not supported by the lxc driver")
 
 type driver struct {
 	root             string // root path for the driver to use
+	libPath          string
 	initPath         string
 	apparmor         bool
 	sharedRoot       bool
@@ -48,7 +50,10 @@ type activeContainer struct {
 	cmd       *exec.Cmd
 }
 
-func NewDriver(root, initPath string, apparmor bool) (*driver, error) {
+func NewDriver(root, libPath, initPath string, apparmor bool) (*driver, error) {
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return nil, err
+	}
 	// setup unconfined symlink
 	if err := linkLxcStart(root); err != nil {
 		return nil, err
@@ -60,6 +65,7 @@ func NewDriver(root, initPath string, apparmor bool) (*driver, error) {
 	return &driver{
 		apparmor:         apparmor,
 		root:             root,
+		libPath:          libPath,
 		initPath:         initPath,
 		sharedRoot:       rootIsShared(),
 		activeContainers: make(map[string]*activeContainer),
@@ -115,6 +121,13 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 		"-n", c.ID,
 		"-f", configPath,
 	}
+
+	// From lxc>=1.1 the default behavior is to daemonize containers after start
+	lxcVersion := version.Version(d.version())
+	if lxcVersion.GreaterThanOrEqualTo(version.Version("1.1")) {
+		params = append(params, "-F")
+	}
+
 	if c.Network.ContainerID != "" {
 		params = append(params,
 			"--share-net", c.Network.ContainerID,
@@ -658,7 +671,7 @@ func rootIsShared() bool {
 }
 
 func (d *driver) containerDir(containerId string) string {
-	return path.Join(d.root, "containers", containerId)
+	return path.Join(d.libPath, "containers", containerId)
 }
 
 func (d *driver) generateLXCConfig(c *execdriver.Command) (string, error) {
@@ -688,7 +701,7 @@ func (d *driver) generateEnvConfig(c *execdriver.Command) error {
 	if err != nil {
 		return err
 	}
-	p := path.Join(d.root, "containers", c.ID, "config.env")
+	p := path.Join(d.libPath, "containers", c.ID, "config.env")
 	c.Mounts = append(c.Mounts, execdriver.Mount{
 		Source:      p,
 		Destination: "/.dockerenv",
@@ -780,5 +793,8 @@ func (d *driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo
 }
 
 func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
+	if _, ok := d.activeContainers[id]; !ok {
+		return nil, fmt.Errorf("%s is not a key in active containers", id)
+	}
 	return execdriver.Stats(d.containerDir(id), d.activeContainers[id].container.Cgroups.Memory, d.machineMemory)
 }

daemon/execdriver/lxc/lxc_template_unit_test.go

@@ -39,7 +39,7 @@ func TestLXCConfig(t *testing.T) {
 		cpu    = cpuMin + rand.Intn(cpuMax-cpuMin)
 	)
 
-	driver, err := NewDriver(root, "", false)
+	driver, err := NewDriver(root, root, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func TestCustomLxcConfig(t *testing.T) {
 
 	os.MkdirAll(path.Join(root, "containers", "1"), 0777)
 
-	driver, err := NewDriver(root, "", false)
+	driver, err := NewDriver(root, root, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -194,7 +194,7 @@ func TestCustomLxcConfigMounts(t *testing.T) {
 	}
 	os.MkdirAll(path.Join(root, "containers", "1"), 0777)
 
-	driver, err := NewDriver(root, "", false)
+	driver, err := NewDriver(root, root, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -248,7 +248,7 @@ func TestCustomLxcConfigMisc(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 	os.MkdirAll(path.Join(root, "containers", "1"), 0777)
-	driver, err := NewDriver(root, "", true)
+	driver, err := NewDriver(root, root, "", true)
 
 	if err != nil {
 		t.Fatal(err)
@@ -313,7 +313,7 @@ func TestCustomLxcConfigMiscOverride(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 	os.MkdirAll(path.Join(root, "containers", "1"), 0777)
-	driver, err := NewDriver(root, "", false)
+	driver, err := NewDriver(root, root, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}

daemon/execdriver/native/driver.go

@@ -64,7 +64,6 @@ func NewDriver(root, initPath string) (*driver, error) {
 		root,
 		cgm,
 		libcontainer.InitPath(reexec.Self(), DriverName),
-		libcontainer.TmpfsRoot,
 	)
 	if err != nil {
 		return nil, err

daemon/graphdriver/btrfs/version.go

@@ -5,20 +5,22 @@ package btrfs
 /*
 #include <btrfs/version.h>
 
-// because around version 3.16, they did not define lib version yet
+// around version 3.16, they did not define lib version yet
-int my_btrfs_lib_version() {
+#ifndef BTRFS_LIB_VERSION
-#ifdef BTRFS_LIB_VERSION
+#define BTRFS_LIB_VERSION -1
-  return BTRFS_LIB_VERSION;
+#endif
-#else
+
-  return -1;
+// upstream had removed it, but now it will be coming back
+#ifndef BTRFS_BUILD_VERSION
+#define BTRFS_BUILD_VERSION "-"
 #endif
-}
 */
 import "C"
 
 func BtrfsBuildVersion() string {
 	return string(C.BTRFS_BUILD_VERSION)
 }
+
 func BtrfsLibVersion() int {
 	return int(C.BTRFS_LIB_VERSION)
 }

daemon/graphdriver/btrfs/version_none.go

@@ -8,6 +8,7 @@ package btrfs
 func BtrfsBuildVersion() string {
 	return "-"
 }
+
 func BtrfsLibVersion() int {
 	return -1
 }

daemon/graphdriver/btrfs/version_test.go

@@ -1,4 +1,4 @@
-// +build linux
+// +build linux,!btrfs_noversion
 
 package btrfs
 
@@ -6,8 +6,8 @@ import (
 	"testing"
 )
 
-func TestBuildVersion(t *testing.T) {
+func TestLibVersion(t *testing.T) {
-	if len(BtrfsBuildVersion()) == 0 {
+	if BtrfsLibVersion() <= 0 {
-		t.Errorf("expected output from btrfs build version, but got empty string")
+		t.Errorf("expected output from btrfs lib version > 0")
 	}
 }

daemon/logger/syslog/syslog.go

@@ -0,0 +1,54 @@
+package syslog
+
+import (
+	"fmt"
+	"log/syslog"
+	"os"
+	"path"
+	"sync"
+
+	"github.com/docker/docker/daemon/logger"
+)
+
+type Syslog struct {
+	writer *syslog.Writer
+	tag    string
+	mu     sync.Mutex
+}
+
+func New(tag string) (logger.Logger, error) {
+	log, err := syslog.New(syslog.LOG_USER, path.Base(os.Args[0]))
+	if err != nil {
+		return nil, err
+	}
+	return &Syslog{
+		writer: log,
+		tag:    tag,
+	}, nil
+}
+
+func (s *Syslog) Log(msg *logger.Message) error {
+	logMessage := fmt.Sprintf("%s: %s", s.tag, string(msg.Line))
+	if msg.Source == "stderr" {
+		if err := s.writer.Err(logMessage); err != nil {
+			return err
+		}
+
+	} else {
+		if err := s.writer.Info(logMessage); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Syslog) Close() error {
+	if s.writer != nil {
+		return s.writer.Close()
+	}
+	return nil
+}
+
+func (s *Syslog) Name() string {
+	return "Syslog"
+}

daemon/networkdriver/portallocator/portallocator.go

@@ -70,10 +70,11 @@ func NewErrPortAlreadyAllocated(ip string, port int) ErrPortAlreadyAllocated {
 
 func init() {
 	const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
+	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", beginPortRange, endPortRange)
 
 	file, err := os.Open(portRangeKernelParam)
 	if err != nil {
-		log.Warnf("Failed to read %s kernel parameter: %v", portRangeKernelParam, err)
+		log.Warnf("port allocator - %s due to error: %v", portRangeFallback, err)
 		return
 	}
 	var start, end int
@@ -82,7 +83,7 @@ func init() {
 		if err == nil {
 			err = fmt.Errorf("unexpected count of parsed numbers (%d)", n)
 		}
-		log.Errorf("Failed to parse port range from %s: %v", portRangeKernelParam, err)
+		log.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err)
 		return
 	}
 	beginPortRange = start

daemon/volumes.go

@@ -24,6 +24,7 @@ type Mount struct {
 	Writable    bool
 	copyData    bool
 	from        *Container
+	isBind      bool
 }
 
 func (mnt *Mount) Export(resource string) (io.ReadCloser, error) {
@@ -79,7 +80,7 @@ func (m *Mount) initialize() error {
 	if hostPath, exists := m.container.Volumes[m.MountToPath]; exists {
 		// If this is a bind-mount/volumes-from, maybe it was passed in at start instead of create
 		// We need to make sure bind-mounts/volumes-from passed on start can override existing ones.
-		if !m.volume.IsBindMount && m.from == nil {
+		if (!m.volume.IsBindMount && !m.isBind) && m.from == nil {
 			return nil
 		}
 		if m.volume.Path == hostPath {
@@ -172,6 +173,7 @@ func (container *Container) parseVolumeMountConfig() (map[string]*Mount, error)
 			volume:      vol,
 			MountToPath: mountToPath,
 			Writable:    writable,
+			isBind:      true, // in case the volume itself is a normal volume, but is being mounted in as a bindmount here
 		}
 	}
 

docs/man/docker-create.1.md

@@ -121,7 +121,7 @@ IMAGE [COMMAND] [ARG...]
 **--lxc-conf**=[]
    (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
 
-**--log-driver**="|*json-file*|*none*"
+**--log-driver**="|*json-file*|*syslog*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: `docker logs` command works only for `json-file` logging driver.
 

docs/man/docker-run.1.md

@@ -222,7 +222,7 @@ which interface and port to use.
 **--lxc-conf**=[]
    (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1"
 
-**--log-driver**="|*json-file*|*none*"
+**--log-driver**="|*json-file*|*syslog*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: `docker logs` command works only for `json-file` logging driver.
 

docs/man/docker.1.md

@@ -89,7 +89,7 @@ unix://[/path/to/socket] to use.
 **--label**="[]"
   Set key=value labels to the daemon (displayed in `docker info`)
 
-**--log-driver**="*json-file*|*none*"
+**--log-driver**="*json-file*|*syslog*|*none*"
   Container's logging driver. Default is `default`.
   **Warning**: `docker logs` command works only for `json-file` logging driver.
 

docs/sources/reference/api/docker_remote_api_v1.18.md

@@ -259,7 +259,7 @@ Json Parameters:
         `Ulimits: { "Name": "nofile", "Soft": 1024, "Hard", 2048 }}`
   -   **LogConfig** - Logging configuration to container, format
         `{ "Type": "<driver_name>", "Config": {"key1": "val1"}}
-        Available types: `json-file`, `none`.
+        Available types: `json-file`, `syslog`, `none`.
         `json-file` logging driver.
   -   **CgroupParent** - Path to cgroups under which the cgroup for the container will be created. If the path is not absolute, the path is considered to be relative to the cgroups path of the init process. Cgroups will be created if they do not already exist.
 

docs/sources/reference/builder.md

@@ -146,6 +146,17 @@ The instructions that handle environment variables in the `Dockerfile` are:
 `ONBUILD` instructions are **NOT** supported for environment replacement, even
 the instructions above.
 
+Environment variable subtitution will use the same value for each variable
+throughout the entire command.  In other words, in this example:
+
+    ENV abc=hello
+    ENV abc=bye def=$abc
+    ENV ghi=$abc
+
+will result in `def` having a value of `hello`, not `bye`.  However, 
+`ghi` will have a value of `bye` because it is not part of the same command
+that set `abc` to `bye`.
+
 ## The `.dockerignore` file
 
 If a file named `.dockerignore` exists in the source repository, then it

docs/sources/reference/run.md

@@ -657,6 +657,11 @@ this driver.
 Default logging driver for Docker. Writes JSON messages to file. `docker logs`
 command is available only for this logging driver
 
+## Logging driver: syslog
+
+Syslog logging driver for Docker. Writes log messages to syslog. `docker logs`
+command is not available for this logging driver
+
 ## Overriding Dockerfile image defaults
 
 When a developer builds an image from a [*Dockerfile*](/reference/builder)

hack/make.sh

@@ -265,9 +265,6 @@ main() {
 		bundle $SCRIPTDIR/make/$bundle
 		echo
 	done
-
-	# if we get all the way through successfully, let's delete our autogenerated code!
-	rm -r autogen
 }
 
 main "$@"

hack/vendor.sh

@@ -75,7 +75,7 @@ rm -rf src/github.com/docker/distribution
 mkdir -p src/github.com/docker/distribution
 mv tmp-digest src/github.com/docker/distribution/digest
 
-clone git github.com/docker/libcontainer 4a72e540feb67091156b907c4700e580a99f5a9d
+clone git github.com/docker/libcontainer a6044b701c166fe538fc760f9e2dcea3d737cd2a
 # see src/github.com/docker/libcontainer/update-vendor.sh which is the "source of truth" for libcontainer deps (just like this file)
 rm -rf src/github.com/docker/libcontainer/vendor
 eval "$(grep '^clone ' src/github.com/docker/libcontainer/update-vendor.sh | grep -v 'github.com/codegangsta/cli' | grep -v 'github.com/Sirupsen/logrus')"

integration-cli/docker_api_containers_test.go

@@ -516,3 +516,40 @@ func TestBuildApiDockerfileSymlink(t *testing.T) {
 
 	logDone("container REST API - check build w/bad Dockerfile symlink path")
 }
+
+// #9981 - Allow a docker created volume (ie, one in /var/lib/docker/volumes) to be used to overwrite (via passing in Binds on api start) an existing volume
+func TestPostContainerBindNormalVolume(t *testing.T) {
+	defer deleteAllContainers()
+
+	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "create", "-v", "/foo", "--name=one", "busybox"))
+	if err != nil {
+		t.Fatal(err, out)
+	}
+
+	fooDir, err := inspectFieldMap("one", "Volumes", "/foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	out, _, err = runCommandWithOutput(exec.Command(dockerBinary, "create", "-v", "/foo", "--name=two", "busybox"))
+	if err != nil {
+		t.Fatal(err, out)
+	}
+
+	bindSpec := map[string][]string{"Binds": {fooDir + ":/foo"}}
+	_, err = sockRequest("POST", "/containers/two/start", bindSpec)
+	if err != nil && !strings.Contains(err.Error(), "204 No Content") {
+		t.Fatal(err)
+	}
+
+	fooDir2, err := inspectFieldMap("two", "Volumes", "/foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fooDir2 != fooDir {
+		t.Fatal("expected volume path to be %s, got: %s", fooDir, fooDir2)
+	}
+
+	logDone("container REST API - can use path from normal volume as bind-mount to overwrite another volume")
+}

integration-cli/docker_cli_build_test.go

@@ -241,9 +241,18 @@ func TestBuildEnvironmentReplacementEnv(t *testing.T) {
 
 	_, err := buildImage(name,
 		`
-  FROM scratch
+  FROM busybox
-  ENV foo foo
+  ENV foo zzz
   ENV bar ${foo}
+  ENV abc1='$foo'
+  ENV env1=$foo env2=${foo} env3="$foo" env4="${foo}"
+  RUN [ "$abc1" = '$foo' ] && (echo "$abc1" | grep -q foo)
+  ENV abc2="\$foo"
+  RUN [ "$abc2" = '$foo' ] && (echo "$abc2" | grep -q foo)
+  ENV abc3 '$foo'
+  RUN [ "$abc3" = '$foo' ] && (echo "$abc3" | grep -q foo)
+  ENV abc4 "\$foo"
+  RUN [ "$abc4" = '$foo' ] && (echo "$abc4" | grep -q foo)
   `, true)
 
 	if err != nil {
@@ -262,13 +271,19 @@ func TestBuildEnvironmentReplacementEnv(t *testing.T) {
 	}
 
 	found := false
+	envCount := 0
 
 	for _, env := range envResult {
 		parts := strings.SplitN(env, "=", 2)
 		if parts[0] == "bar" {
 			found = true
-			if parts[1] != "foo" {
+			if parts[1] != "zzz" {
-				t.Fatalf("Could not find replaced var for env `bar`: got %q instead of `foo`", parts[1])
+				t.Fatalf("Could not find replaced var for env `bar`: got %q instead of `zzz`", parts[1])
+			}
+		} else if strings.HasPrefix(parts[0], "env") {
+			envCount++
+			if parts[1] != "zzz" {
+				t.Fatalf("%s should be 'foo' but instead its %q", parts[0], parts[1])
 			}
 		}
 	}
@@ -277,6 +292,10 @@ func TestBuildEnvironmentReplacementEnv(t *testing.T) {
 		t.Fatal("Never found the `bar` env variable")
 	}
 
+	if envCount != 4 {
+		t.Fatalf("Didn't find all env vars - only saw %d\n%s", envCount, envResult)
+	}
+
 	logDone("build - env environment replacement")
 }
 
@@ -363,8 +382,8 @@ func TestBuildHandleEscapes(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, ok := result[`\\\\\\${FOO}`]; !ok {
+	if _, ok := result[`\\\${FOO}`]; !ok {
-		t.Fatal(`Could not find volume \\\\\\${FOO} set from env foo in volumes table`)
+		t.Fatal(`Could not find volume \\\${FOO} set from env foo in volumes table`, result)
 	}
 
 	logDone("build - handle escapes")
@@ -1939,6 +1958,7 @@ func TestBuildCancelationKillsSleep(t *testing.T) {
 
 	name := "testbuildcancelation"
 	defer deleteImages(name)
+	defer deleteAllContainers()
 
 	// (Note: one year, will never finish)
 	ctx, err := fakeContext("FROM busybox\nRUN sleep 31536000", nil)
@@ -2256,7 +2276,7 @@ func TestBuildRelativeWorkdir(t *testing.T) {
 
 func TestBuildWorkdirWithEnvVariables(t *testing.T) {
 	name := "testbuildworkdirwithenvvariables"
-	expected := "/test1/test2/$MISSING_VAR"
+	expected := "/test1/test2"
 	defer deleteImages(name)
 	_, err := buildImage(name,
 		`FROM busybox
@@ -4025,9 +4045,9 @@ ENV    abc=zzz TO=/docker/world/hello
 ADD    $FROM $TO
 RUN    [ "$(cat $TO)" = "hello" ]
 ENV    abc "zzz"
-RUN    [ $abc = \"zzz\" ]
+RUN    [ $abc = "zzz" ]
 ENV    abc 'yyy'
-RUN    [ $abc = \'yyy\' ]
+RUN    [ $abc = 'yyy' ]
 ENV    abc=
 RUN    [ "$abc" = "" ]
 
@@ -4043,13 +4063,34 @@ RUN    [ "$abc" = "'foo'" ]
 ENV    abc=\"foo\"
 RUN    [ "$abc" = "\"foo\"" ]
 ENV    abc "foo"
-RUN    [ "$abc" = "\"foo\"" ]
+RUN    [ "$abc" = "foo" ]
 ENV    abc 'foo'
-RUN    [ "$abc" = "'foo'" ]
+RUN    [ "$abc" = 'foo' ]
 ENV    abc \'foo\'
-RUN    [ "$abc" = "\\'foo\\'" ]
+RUN    [ "$abc" = "'foo'" ]
 ENV    abc \"foo\"
-RUN    [ "$abc" = "\\\"foo\\\"" ]
+RUN    [ "$abc" = '"foo"' ]
+
+ENV    e1=bar
+ENV    e2=$e1
+ENV    e3=$e11
+ENV    e4=\$e1
+ENV    e5=\$e11
+RUN    [ "$e0,$e1,$e2,$e3,$e4,$e5" = ',bar,bar,,$e1,$e11' ]
+
+ENV    ee1 bar
+ENV    ee2 $ee1
+ENV    ee3 $ee11
+ENV    ee4 \$ee1
+ENV    ee5 \$ee11
+RUN    [ "$ee1,$ee2,$ee3,$ee4,$ee5" = 'bar,bar,,$ee1,$ee11' ]
+
+ENV    eee1="foo"
+ENV    eee2='foo'
+ENV    eee3 "foo"
+ENV    eee4 'foo'
+RUN    [ "$eee1,$eee2,$eee3,$eee4" = 'foo,foo,foo,foo' ]
+
 `
 	ctx, err := fakeContext(dockerfile, map[string]string{
 		"hello/docker/world": "hello",

integration-cli/docker_cli_pull_test.go

@@ -55,8 +55,6 @@ func TestPullImageWithAliases(t *testing.T) {
 
 // pulling library/hello-world should show verified message
 func TestPullVerified(t *testing.T) {
-	t.Skip("problems verifying library/hello-world (to be fixed)")
-
 	// Image must be pulled from central repository to get verified message
 	// unless keychain is manually updated to contain the daemon's sign key.
 

integration-cli/docker_cli_run_test.go

@@ -413,6 +413,7 @@ func TestRunLinkToContainerNetMode(t *testing.T) {
 }
 
 func TestRunModeNetContainerHostname(t *testing.T) {
+	testRequires(t, ExecSupport)
 	defer deleteAllContainers()
 	cmd := exec.Command(dockerBinary, "run", "-i", "-d", "--name", "parent", "busybox", "top")
 	out, _, err := runCommandWithOutput(cmd)

integration-cli/docker_cli_start_test.go

@@ -240,3 +240,49 @@ func TestStartMultipleContainers(t *testing.T) {
 
 	logDone("start - start multiple containers continue on one failed")
 }
+
+func TestStartAttachMultipleContainers(t *testing.T) {
+
+	var cmd *exec.Cmd
+
+	defer deleteAllContainers()
+	// run  multiple containers to test
+	for _, container := range []string{"test1", "test2", "test3"} {
+		cmd = exec.Command(dockerBinary, "run", "-d", "--name", container, "busybox", "top")
+		if out, _, err := runCommandWithOutput(cmd); err != nil {
+			t.Fatal(out, err)
+		}
+	}
+
+	// stop all the containers
+	for _, container := range []string{"test1", "test2", "test3"} {
+		cmd = exec.Command(dockerBinary, "stop", container)
+		if out, _, err := runCommandWithOutput(cmd); err != nil {
+			t.Fatal(out, err)
+		}
+	}
+
+	// test start and attach multiple containers at once, expected error
+	for _, option := range []string{"-a", "-i", "-ai"} {
+		cmd = exec.Command(dockerBinary, "start", option, "test1", "test2", "test3")
+		out, _, err := runCommandWithOutput(cmd)
+		if !strings.Contains(out, "You cannot start and attach multiple containers at once.") || err == nil {
+			t.Fatal("Expected error but got none")
+		}
+	}
+
+	// confirm the state of all the containers be stopped
+	for container, expected := range map[string]string{"test1": "false", "test2": "false", "test3": "false"} {
+		cmd = exec.Command(dockerBinary, "inspect", "-f", "{{.State.Running}}", container)
+		out, _, err := runCommandWithOutput(cmd)
+		if err != nil {
+			t.Fatal(out, err)
+		}
+		out = strings.Trim(out, "\r\n")
+		if out != expected {
+			t.Fatal("Container running state wrong")
+		}
+	}
+
+	logDone("start - error on start and attach multiple containers at once")
+}

integration-cli/docker_test_vars.go

@@ -17,11 +17,13 @@ var (
 	privateRegistryURL = "127.0.0.1:5000"
 
 	dockerBasePath       = "/var/lib/docker"
-	execDriverPath       = dockerBasePath + "/execdriver/native"
 	volumesConfigPath    = dockerBasePath + "/volumes"
 	volumesStoragePath   = dockerBasePath + "/vfs/dir"
 	containerStoragePath = dockerBasePath + "/containers"
 
+	runtimePath    = "/var/run/docker"
+	execDriverPath = runtimePath + "/execdriver/native"
+
 	workingDirectory string
 )
 

integration-cli/test_vars_windows.go

@@ -6,6 +6,6 @@ const (
 	// identifies if test suite is running on a unix platform
 	isUnixCli = false
 
-	// this is the expected file permission set on windows: gh#11047
+	// this is the expected file permission set on windows: gh#11395
-	expectedFileChmod = "-rwx------"
+	expectedFileChmod = "-rwxr-xr-x"
 )

pkg/archive/archive_windows.go

@@ -28,10 +28,9 @@ func CanonicalTarNameForPath(p string) (string, error) {
 // chmodTarEntry is used to adjust the file permissions used in tar header based
 // on the platform the archival is done.
 func chmodTarEntry(perm os.FileMode) os.FileMode {
-	// Clear r/w on grp/others: no precise equivalen of group/others on NTFS.
+	perm &= 0755
-	perm &= 0711
 	// Add the x bit: make everything +x from windows
-	perm |= 0100
+	perm |= 0111
 
 	return perm
 }

pkg/archive/archive_windows_test.go

@@ -51,11 +51,11 @@ func TestChmodTarEntry(t *testing.T) {
 	cases := []struct {
 		in, expected os.FileMode
 	}{
-		{0000, 0100},
+		{0000, 0111},
-		{0777, 0711},
+		{0777, 0755},
-		{0644, 0700},
+		{0644, 0755},
-		{0755, 0711},
+		{0755, 0755},
-		{0444, 0500},
+		{0444, 0555},
 	}
 	for _, v := range cases {
 		if out := chmodTarEntry(v.in); out != v.expected {

pkg/archive/changes.go

@@ -220,8 +220,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
 				oldStat.Gid() != newStat.Gid() ||
 				oldStat.Rdev() != newStat.Rdev() ||
 				// Don't look at size for dirs, its not a good measure of change
-				(oldStat.Size() != newStat.Size() && oldStat.Mode()&syscall.S_IFDIR != syscall.S_IFDIR) ||
+				(oldStat.Mode()&syscall.S_IFDIR != syscall.S_IFDIR &&
-				!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) ||
+					(!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) ||
 				bytes.Compare(oldChild.capability, newChild.capability) != 0 {
 				change := Change{
 					Path: newChild.path(),

pkg/archive/changes_test.go

@@ -218,7 +218,6 @@ func TestChangesDirsMutated(t *testing.T) {
 	expectedChanges := []Change{
 		{"/dir1", ChangeDelete},
 		{"/dir2", ChangeModify},
-		{"/dir3", ChangeModify},
 		{"/dirnew", ChangeAdd},
 		{"/file1", ChangeDelete},
 		{"/file2", ChangeModify},

pkg/ioutils/readers.go

@@ -2,8 +2,11 @@ package ioutils
 
 import (
 	"bytes"
+	"crypto/rand"
 	"io"
+	"math/big"
 	"sync"
+	"time"
 )
 
 type readCloserWrapper struct {
@@ -42,20 +45,40 @@ func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader {
 	}
 }
 
+// bufReader allows the underlying reader to continue to produce
+// output by pre-emptively reading from the wrapped reader.
+// This is achieved by buffering this data in bufReader's
+// expanding buffer.
 type bufReader struct {
 	sync.Mutex
-	buf      *bytes.Buffer
+	buf                  *bytes.Buffer
-	reader   io.Reader
+	reader               io.Reader
-	err      error
+	err                  error
-	wait     sync.Cond
+	wait                 sync.Cond
-	drainBuf []byte
+	drainBuf             []byte
+	reuseBuf             []byte
+	maxReuse             int64
+	resetTimeout         time.Duration
+	bufLenResetThreshold int64
+	maxReadDataReset     int64
 }
 
 func NewBufReader(r io.Reader) *bufReader {
+	var timeout int
+	if randVal, err := rand.Int(rand.Reader, big.NewInt(120)); err == nil {
+		timeout = int(randVal.Int64()) + 180
+	} else {
+		timeout = 300
+	}
 	reader := &bufReader{
-		buf:      &bytes.Buffer{},
+		buf:                  &bytes.Buffer{},
-		drainBuf: make([]byte, 1024),
+		drainBuf:             make([]byte, 1024),
-		reader:   r,
+		reuseBuf:             make([]byte, 4096),
+		maxReuse:             1000,
+		resetTimeout:         time.Second * time.Duration(timeout),
+		bufLenResetThreshold: 100 * 1024,
+		maxReadDataReset:     10 * 1024 * 1024,
+		reader:               r,
 	}
 	reader.wait.L = &reader.Mutex
 	go reader.drain()
@@ -74,14 +97,94 @@ func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer *
 }
 
 func (r *bufReader) drain() {
+	var (
+		duration       time.Duration
+		lastReset      time.Time
+		now            time.Time
+		reset          bool
+		bufLen         int64
+		dataSinceReset int64
+		maxBufLen      int64
+		reuseBufLen    int64
+		reuseCount     int64
+	)
+	reuseBufLen = int64(len(r.reuseBuf))
+	lastReset = time.Now()
 	for {
 		n, err := r.reader.Read(r.drainBuf)
+		dataSinceReset += int64(n)
 		r.Lock()
+		bufLen = int64(r.buf.Len())
+		if bufLen > maxBufLen {
+			maxBufLen = bufLen
+		}
+
+		// Avoid unbounded growth of the buffer over time.
+		// This has been discovered to be the only non-intrusive
+		// solution to the unbounded growth of the buffer.
+		// Alternative solutions such as compression, multiple
+		// buffers, channels and other similar pieces of code
+		// were reducing throughput, overall Docker performance
+		// or simply crashed Docker.
+		// This solution releases the buffer when specific
+		// conditions are met to avoid the continuous resizing
+		// of the buffer for long lived containers.
+		//
+		// Move data to the front of the buffer if it's
+		// smaller than what reuseBuf can store
+		if bufLen > 0 && reuseBufLen >= bufLen {
+			n, _ := r.buf.Read(r.reuseBuf)
+			r.buf.Write(r.reuseBuf[0:n])
+			// Take action if the buffer has been reused too many
+			// times and if there's data in the buffer.
+			// The timeout is also used as means to avoid doing
+			// these operations more often or less often than
+			// required.
+			// The various conditions try to detect heavy activity
+			// in the buffer which might be indicators of heavy
+			// growth of the buffer.
+		} else if reuseCount >= r.maxReuse && bufLen > 0 {
+			now = time.Now()
+			duration = now.Sub(lastReset)
+			timeoutReached := duration >= r.resetTimeout
+
+			// The timeout has been reached and the
+			// buffered data couldn't be moved to the front
+			// of the buffer, so the buffer gets reset.
+			if timeoutReached && bufLen > reuseBufLen {
+				reset = true
+			}
+			// The amount of buffered data is too high now,
+			// reset the buffer.
+			if timeoutReached && maxBufLen >= r.bufLenResetThreshold {
+				reset = true
+			}
+			// Reset the buffer if a certain amount of
+			// data has gone through the buffer since the
+			// last reset.
+			if timeoutReached && dataSinceReset >= r.maxReadDataReset {
+				reset = true
+			}
+			// The buffered data is moved to a fresh buffer,
+			// swap the old buffer with the new one and
+			// reset all counters.
+			if reset {
+				newbuf := &bytes.Buffer{}
+				newbuf.ReadFrom(r.buf)
+				r.buf = newbuf
+				lastReset = now
+				reset = false
+				dataSinceReset = 0
+				maxBufLen = 0
+				reuseCount = 0
+			}
+		}
 		if err != nil {
 			r.err = err
 		} else {
 			r.buf.Write(r.drainBuf[0:n])
 		}
+		reuseCount++
 		r.wait.Signal()
 		r.Unlock()
 		if err != nil {

pkg/ioutils/readers_test.go

@@ -32,3 +32,61 @@ func TestBufReader(t *testing.T) {
 		t.Error(string(output))
 	}
 }
+
+type repeatedReader struct {
+	readCount int
+	maxReads  int
+	data      []byte
+}
+
+func newRepeatedReader(max int, data []byte) *repeatedReader {
+	return &repeatedReader{0, max, data}
+}
+
+func (r *repeatedReader) Read(p []byte) (int, error) {
+	if r.readCount >= r.maxReads {
+		return 0, io.EOF
+	}
+	r.readCount++
+	n := copy(p, r.data)
+	return n, nil
+}
+
+func testWithData(data []byte, reads int) {
+	reader := newRepeatedReader(reads, data)
+	bufReader := NewBufReader(reader)
+	io.Copy(ioutil.Discard, bufReader)
+}
+
+func Benchmark1M10BytesReads(b *testing.B) {
+	reads := 1000000
+	readSize := int64(10)
+	data := make([]byte, readSize)
+	b.SetBytes(readSize * int64(reads))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		testWithData(data, reads)
+	}
+}
+
+func Benchmark1M1024BytesReads(b *testing.B) {
+	reads := 1000000
+	readSize := int64(1024)
+	data := make([]byte, readSize)
+	b.SetBytes(readSize * int64(reads))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		testWithData(data, reads)
+	}
+}
+
+func Benchmark10k32KBytesReads(b *testing.B) {
+	reads := 10000
+	readSize := int64(32 * 1024)
+	data := make([]byte, readSize)
+	b.SetBytes(readSize * int64(reads))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		testWithData(data, reads)
+	}
+}

pkg/term/term_windows.go

@@ -3,6 +3,7 @@ package term
 
 import (
 	"io"
+	"os"
 
 	"github.com/docker/docker/pkg/term/winconsole"
 )
@@ -114,5 +115,23 @@ func GetFdInfo(in interface{}) (uintptr, bool) {
 }
 
 func StdStreams() (stdIn io.ReadCloser, stdOut, stdErr io.Writer) {
-	return winconsole.StdStreams()
+	var shouldEmulateANSI bool
+	switch {
+	case os.Getenv("ConEmuANSI") == "ON":
+		// ConEmu shell, ansi emulated by default and ConEmu does an extensively
+		// good emulation.
+		shouldEmulateANSI = false
+	case os.Getenv("MSYSTEM") != "":
+		// MSYS (mingw) cannot fully emulate well and still shows escape characters
+		// mostly because it's still running on cmd.exe window.
+		shouldEmulateANSI = true
+	default:
+		shouldEmulateANSI = true
+	}
+
+	if shouldEmulateANSI {
+		return winconsole.StdStreams()
+	}
+
+	return os.Stdin, os.Stdout, os.Stderr
 }

project/make/validate-toml

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source "$(dirname "$BASH_SOURCE")/.validate"
+
+IFS=$'\n'
+files=( $(validate_diff --diff-filter=ACMR --name-only -- 'MAINTAINERS' || true) )
+unset IFS
+
+badFiles=()
+for f in "${files[@]}"; do
+	# we use "git show" here to validate that what's committed is formatted
+	if [ "$(git show "$VALIDATE_HEAD:$f" | tomlv)" ]; then
+		badFiles+=( "$f" )
+	fi
+done
+
+if [ ${#badFiles[@]} -eq 0 ]; then
+	echo 'Congratulations!  All toml source files have valid syntax.'
+else
+	{
+		echo "These files are not valid toml:"
+		for f in "${badFiles[@]}"; do
+			echo " - $f"
+		done
+		echo
+		echo 'Please reformat the above files as valid toml'
+		echo
+	} >&2
+	false
+fi

vendor/src/github.com/docker/libcontainer/cgroups/fs/apply_raw.go

@@ -99,12 +99,11 @@ func (m *Manager) Apply(pid int) error {
 		// created then join consists of writing the process pids to cgroup.procs
 		p, err := d.path(name)
 		if err != nil {
+			if cgroups.IsNotFound(err) {
+				continue
+			}
 			return err
 		}
-		if !cgroups.PathExists(p) {
-			continue
-		}
-
 		paths[name] = p
 	}
 	m.Paths = paths
@@ -174,9 +173,6 @@ func (m *Manager) Freeze(state configs.FreezerState) error {
 	if err != nil {
 		return err
 	}
-	if !cgroups.PathExists(dir) {
-		return cgroups.NewNotFoundError("freezer")
-	}
 
 	prevState := m.Cgroups.Freezer
 	m.Cgroups.Freezer = state
@@ -201,9 +197,6 @@ func (m *Manager) GetPids() ([]int, error) {
 	if err != nil {
 		return nil, err
 	}
-	if !cgroups.PathExists(dir) {
-		return nil, cgroups.NewNotFoundError("devices")
-	}
 
 	return cgroups.ReadProcsFile(dir)
 }

vendor/src/github.com/docker/libcontainer/cgroups/fs/blkio.go

@@ -17,12 +17,8 @@ type BlkioGroup struct {
 
 func (s *BlkioGroup) Apply(d *data) error {
 	dir, err := d.join("blkio")
-	if err != nil {
+	if err != nil && !cgroups.IsNotFound(err) {
-		if cgroups.IsNotFound(err) {
+		return err
-			return nil
-		} else {
-			return err
-		}
 	}
 
 	if err := s.Set(dir, d.c); err != nil {

vendor/src/github.com/docker/libcontainer/cgroups/fs/cpu.go

@@ -18,11 +18,7 @@ func (s *CpuGroup) Apply(d *data) error {
 	// on a container basis
 	dir, err := d.join("cpu")
 	if err != nil {
-		if cgroups.IsNotFound(err) {
+		return err
-			return nil
-		} else {
-			return err
-		}
 	}
 
 	if err := s.Set(dir, d.c); err != nil {

vendor/src/github.com/docker/libcontainer/cgroups/fs/devices.go

@@ -11,11 +11,7 @@ type DevicesGroup struct {
 func (s *DevicesGroup) Apply(d *data) error {
 	dir, err := d.join("devices")
 	if err != nil {
-		if cgroups.IsNotFound(err) {
+		return err
-			return nil
-		} else {
-			return err
-		}
 	}
 
 	if err := s.Set(dir, d.c); err != nil {

vendor/src/github.com/docker/libcontainer/cgroups/fs/freezer.go

@@ -13,12 +13,8 @@ type FreezerGroup struct {
 
 func (s *FreezerGroup) Apply(d *data) error {
 	dir, err := d.join("freezer")
-	if err != nil {
+	if err != nil && !cgroups.IsNotFound(err) {
-		if cgroups.IsNotFound(err) {
+		return err
-			return nil
-		} else {
-			return err
-		}
 	}
 
 	if err := s.Set(dir, d.c); err != nil {

vendor/src/github.com/docker/libcontainer/cgroups/fs/memory.go

@@ -16,12 +16,9 @@ type MemoryGroup struct {
 
 func (s *MemoryGroup) Apply(d *data) error {
 	dir, err := d.join("memory")
-	if err != nil {
+	// only return an error for memory if it was specified
-		if cgroups.IsNotFound(err) {
+	if err != nil && (d.c.Memory != 0 || d.c.MemoryReservation != 0 || d.c.MemorySwap != 0) {
-			return nil
+		return err
-		} else {
-			return err
-		}
 	}
 	defer func() {
 		if err != nil {

vendor/src/github.com/docker/libcontainer/cgroups/systemd/apply_systemd.go

@@ -91,7 +91,7 @@ func UseSystemd() bool {
 		ddf := newProp("DefaultDependencies", false)
 		if _, err := theConn.StartTransientUnit("docker-systemd-test-default-dependencies.scope", "replace", ddf); err != nil {
 			if dbusError, ok := err.(dbus.Error); ok {
-				if dbusError.Name == "org.freedesktop.DBus.Error.PropertyReadOnly" {
+				if strings.Contains(dbusError.Name, "org.freedesktop.DBus.Error.PropertyReadOnly") {
 					hasTransientDefaultDependencies = false
 				}
 			}

vendor/src/github.com/docker/libcontainer/init_linux.go

@@ -91,7 +91,7 @@ func populateProcessEnvironment(env []string) error {
 
 // finalizeNamespace drops the caps, sets the correct user
 // and working dir, and closes any leaked file descriptors
-// before execing the command inside the namespace
+// before executing the command inside the namespace
 func finalizeNamespace(config *initConfig) error {
 	// Ensure that all non-standard fds we may have accidentally
 	// inherited are marked close-on-exec so they stay out of the

vendor/src/github.com/docker/libcontainer/netlink/netlink_linux.go

@@ -659,7 +659,7 @@ func networkSetNsAction(iface *net.Interface, rtattr *RtAttr) error {
 }
 
 // Move a particular network interface to a particular network namespace
-// specified by PID. This is idential to running: ip link set dev $name netns $pid
+// specified by PID. This is identical to running: ip link set dev $name netns $pid
 func NetworkSetNsPid(iface *net.Interface, nspid int) error {
 	data := uint32Attr(syscall.IFLA_NET_NS_PID, uint32(nspid))
 	return networkSetNsAction(iface, data)
@@ -673,7 +673,7 @@ func NetworkSetNsFd(iface *net.Interface, fd int) error {
 	return networkSetNsAction(iface, data)
 }
 
-// Rname a particular interface to a different name
+// Rename a particular interface to a different name
 // !!! Note that you can't rename an active interface. You need to bring it down before renaming it.
 // This is identical to running: ip link set dev ${oldName} name ${newName}
 func NetworkChangeName(iface *net.Interface, newName string) error {

vendor/src/github.com/docker/libcontainer/rootfs_linux.go

@@ -186,7 +186,9 @@ func reOpenDevNull(rootfs string) error {
 func createDevices(config *configs.Config) error {
 	oldMask := syscall.Umask(0000)
 	for _, node := range config.Devices {
-		if err := createDeviceNode(config.Rootfs, node); err != nil {
+		// containers running in a user namespace are not allowed to mknod
+		// devices so we can just bind mount it from the host.
+		if err := createDeviceNode(config.Rootfs, node, config.Namespaces.Contains(configs.NEWUSER)); err != nil {
 			syscall.Umask(oldMask)
 			return err
 		}
@@ -196,20 +198,13 @@ func createDevices(config *configs.Config) error {
 }
 
 // Creates the device node in the rootfs of the container.
-func createDeviceNode(rootfs string, node *configs.Device) error {
+func createDeviceNode(rootfs string, node *configs.Device, bind bool) error {
 	dest := filepath.Join(rootfs, node.Path)
 	if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil {
 		return err
 	}
-	if err := mknodDevice(dest, node); err != nil {
+
-		if os.IsExist(err) {
+	if bind {
-			return nil
-		}
-		if err != syscall.EPERM {
-			return err
-		}
-		// containers running in a user namespace are not allowed to mknod
-		// devices so we can just bind mount it from the host.
 		f, err := os.Create(dest)
 		if err != nil && !os.IsExist(err) {
 			return err
@@ -219,6 +214,12 @@ func createDeviceNode(rootfs string, node *configs.Device) error {
 		}
 		return syscall.Mount(node.Path, dest, "bind", syscall.MS_BIND, "")
 	}
+	if err := mknodDevice(dest, node); err != nil {
+		if os.IsExist(err) {
+			return nil
+		}
+		return err
+	}
 	return nil
 }
 

vendor/src/github.com/docker/libcontainer/update-vendor.sh

@@ -44,6 +44,6 @@ clone git github.com/codegangsta/cli 1.1.0
 clone git github.com/coreos/go-systemd v2
 clone git github.com/godbus/dbus v2
 clone git github.com/Sirupsen/logrus v0.6.6
-clone git github.com/syndtr/gocapability e55e583369
+clone git github.com/syndtr/gocapability 8e4cdcb
 
 # intentionally not vendoring Docker itself...  that'd be a circle :)

vendor/src/github.com/syndtr/gocapability/capability/capability_linux.go

@@ -417,10 +417,6 @@ func (c *capsV3) Load() (err error) {
 }
 
 func (c *capsV3) Apply(kind CapType) (err error) {
-	err = initLastCap()
-	if err != nil {
-		return
-	}
 	if kind&BOUNDS == BOUNDS {
 		var data [2]capData
 		err = capget(&c.hdr, &data[0])
@@ -428,7 +424,7 @@ func (c *capsV3) Apply(kind CapType) (err error) {
 			return
 		}
 		if (1<<uint(CAP_SETPCAP))&data[0].effective != 0 {
-			for i := Cap(0); i <= capLastCap; i++ {
+			for i := Cap(0); i <= CAP_LAST_CAP; i++ {
 				if c.Get(BOUNDING, i) {
 					continue
 				}