Merge branch 'migrate-aufs-containers' into v0.7-rc3

Conflicts:
	runtime.go

CONTRIBUTING.md

@@ -3,6 +3,10 @@
 Want to hack on Docker? Awesome! Here are instructions to get you started. They are probably not perfect, please let us know if anything feels
 wrong or incomplete.
 
+## Build Environment
+
+For instructions on setting up your development environment, please see our dedicated [dev environment setup docs](http://docs.docker.io/en/latest/contributing/devenvironment/).
+
 ## Contribution guidelines
 
 ### Pull requests are always welcome

FIXME

@@ -15,23 +15,16 @@ to put them - so we put them here :)
 * Run linter on codebase
 * Unify build commands and regular commands
 * Move source code into src/ subdir for clarity
-* Clean up the Makefile, it's a mess
 * docker build: on non-existent local path for ADD, don't show full absolute path on the host
-* mount into /dockerinit rather than /sbin/init
 * docker tag foo REPO:TAG
 * use size header for progress bar in pull
 * Clean up context upload in build!!!
 * Parallel pull
-* Ensure /proc/sys/net/ipv4/ip_forward is 1
-* Force DNS to public!
 * Always generate a resolv.conf per container, to avoid changing resolv.conf under thne container's feet
-* Save metadata with import/export
+* Save metadata with import/export (#1974)
 * Upgrade dockerd without stopping containers
-* bring back git revision info, looks like it was lost
-* Simple command to remove all untagged images
+* Simple command to remove all untagged images (`docker rmi $(docker images | awk '/^<none>/ { print $3 }')`)
 * Simple command to clean up containers for disk space
-* Caching after an ADD
-* entry point config
-* bring back git revision info, looks like it was lost
+* Caching after an ADD (#880)
 * Clean up the ProgressReader api, it's a PITA to use
 * Use netlink instead of iproute2/iptables (#925)

commands_test.go

@@ -400,7 +400,7 @@ func TestRunDetach(t *testing.T) {
 	})
 
 	// wait for CmdRun to return
-	setTimeout(t, "Waiting for CmdRun timed out", 5*time.Second, func() {
+	setTimeout(t, "Waiting for CmdRun timed out", 15*time.Second, func() {
 		<-ch
 	})
 
@@ -458,7 +458,7 @@ func TestAttachDetach(t *testing.T) {
 	})
 
 	// wait for CmdRun to return
-	setTimeout(t, "Waiting for CmdAttach timed out", 5*time.Second, func() {
+	setTimeout(t, "Waiting for CmdAttach timed out", 15*time.Second, func() {
 		<-ch
 	})
 

contrib/MAINTAINERS

@@ -1,2 +1,2 @@
-Kawsar Saiyeed <kawsar.saiyeed@projiris.com> (@KSid)
 Tianon Gravi <admwiggin@gmail.com> (@tianon)
+Kawsar Saiyeed <kawsar.saiyeed@projiris.com> (@KSid)

contrib/docker.bash

@@ -341,7 +341,7 @@ _docker_pull()
 
 _docker_push()
 {
-	return
+	__docker_image_repos
 }
 
 _docker_restart()

docs/sources/use/builder.rst

@@ -321,7 +321,7 @@ the command given by ``CMD`` is executed.
     # VERSION               0.0.1
 
     FROM      ubuntu
-    MAINTAINER Guillaume J. Charmes "guillaume@dotcloud.com"
+    MAINTAINER Guillaume J. Charmes <guillaume@dotcloud.com>
 
     # make sure the package repository is up to date
     RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list

docs/theme/docker/layout.html

@@ -6,6 +6,9 @@
     <title>{{ meta['title'] if meta and meta['title'] else title }} - Docker Documentation</title>
     <meta name="description" content="{{ meta['description'] if meta }}" />
     <meta name="keywords" content="{{ meta['keywords'] if meta }}" />
+    <!-- Swiftype tags: https://swiftype.com/documentation/meta_tags -->
+    <meta property='st:popularity' content='4' />
+    <meta property='st:type' content='docker_doc' />
 
     {%- set url_root = pathto('', 1) %}
     {%- if url_root == '#' %}{% set url_root = '' %}{% endif %}
@@ -168,6 +171,9 @@
   Swiftype.resultContainingElement = '#st-results-container';
   Swiftype.attachElement = '#st-search-input';
   Swiftype.renderStyle = "overlay";
+  // from https://swiftype.com/questions/how-can-i-make-more-popular-content-rank-higher 
+  // Use "page" for now -- they don't subgroup by document type yet.
+  Swiftype.searchFunctionalBoosts = {"page": {"popularity": "linear"}};
 
   var script = document.createElement('script');
   script.type = 'text/javascript';

hack/make.sh

@@ -45,7 +45,7 @@ fi
 
 # Use these flags when compiling the tests and final binary
 LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-all"'
-BUILDFLAGS='-tags netgo'
+BUILDFLAGS='-tags netgo -a'
 
 
 bundle() {

registry/registry.go

@@ -160,16 +160,16 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s
 	}
 	req.Header.Set("Authorization", "Token "+strings.Join(token, ", "))
 	res, err := doWithCookies(r.client, req)
-	if err != nil || res.StatusCode != 200 {
-		if res != nil {
-			if res.StatusCode == 401 {
-				return nil, ErrLoginRequired
-			}
-			return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res)
-		}
+	if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		if res.StatusCode == 401 {
+			return nil, ErrLoginRequired
+		}
+		return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res)
+	}
 
 	jsonString, err := ioutil.ReadAll(res.Body)
 	if err != nil {
@@ -240,6 +240,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (
 		return nil, err
 	}
 	if res.StatusCode != 200 {
+		res.Body.Close()
 		return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)",
 			res.StatusCode, imgID)
 	}

runtime.go

@@ -321,7 +321,56 @@ func (runtime *Runtime) restore() error {
 		}
 		return len(ic.Links) < len(jc.Links)
 	})
+
+	deviceSet := runtime.config.DeviceSet
 	for _, container := range containers {
+
+		// Perform a migration for aufs containers
+		if !deviceSet.HasDevice(container.ID) {
+			contents, err := ioutil.ReadDir(container.rwPath())
+			if err != nil {
+				if !os.IsNotExist(err) {
+					utils.Debugf("[migration] Error reading rw dir %s", err)
+				}
+				continue
+			}
+
+			if len(contents) > 0 {
+				utils.Debugf("[migration] Begin migration of %s", container.ID)
+
+				image, err := runtime.graph.Get(container.Image)
+				if err != nil {
+					utils.Debugf("[migratoin] Failed to get image %s", err)
+					continue
+				}
+
+				unmount := func() {
+					if err := image.Unmount(runtime, container.RootfsPath(), container.ID); err != nil {
+						utils.Debugf("[migraton] Failed to unmount image %s", err)
+					}
+				}
+
+				if err := image.Mount(runtime, container.RootfsPath(), container.rwPath(), container.ID); err != nil {
+					utils.Debugf("[migratoin] Failed to mount image %s", err)
+					continue
+				}
+
+				if err := image.applyLayer(container.rwPath(), container.RootfsPath()); err != nil {
+					utils.Debugf("[migration] Failed to apply layer %s", err)
+					unmount()
+					continue
+				}
+
+				unmount()
+
+				if err := os.RemoveAll(container.rwPath()); err != nil {
+					utils.Debugf("[migration] Failed to remove rw dir %s", err)
+				}
+
+				utils.Debugf("[migration] End migration of %s", container.ID)
+			}
+		}
+
 		if err := runtime.Register(container); err != nil {
 			utils.Debugf("Failed to register container %s: %s", container.ID, err)
 			continue

runtime_test.go

@@ -98,11 +98,25 @@ func removeDev(name string) {
 		syscall.Close(fd)
 	}
 	if err := devmapper.RemoveDevice(name); err != nil {
-		panic(fmt.Errorf("Unable to remove existing device %s: %s", name, err))
+		log.Fatalf("Unable to remove device %s needed to get a freash unit test environment", name)
 	}
 }
 
 func cleanupDevMapper() {
+	// Unmount any leftover mounts from previous unit test runs
+	if data, err := ioutil.ReadFile("/proc/mounts"); err == nil {
+		for _, line := range strings.Split(string(data), "\n") {
+			cols := strings.Split(line, " ")
+			if len(cols) >= 2 && strings.HasPrefix(cols[0], "/dev/mapper/docker-unit-tests-devices") {
+				err = syscall.Unmount(cols[1], 0)
+				if err != nil {
+					log.Fatalf("Unable to unmount %s needed to get a freash unit test environment: %s", cols[1], err)
+				}
+			}
+		}
+	}
+
+	// Remove any leftover devmapper devices from previous unit run tests
 	infos, _ := ioutil.ReadDir("/dev/mapper")
 	if infos != nil {
 		hasPool := false