Merge pull request #1974 from dotcloud/1155-offline-image-transfer

Implement offline image transfers

api.go

@@ -534,6 +534,26 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http
 	return nil
 }
 
+func getImagesGet(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	name := vars["name"]
+	if version > 1.0 {
+		w.Header().Set("Content-Type", "application/x-tar")
+	}
+	err := srv.ImageExport(name, w)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func postImagesLoad(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	err := srv.ImageLoad(r.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	if err := parseForm(r); err != nil {
 		return nil
@@ -1036,6 +1056,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/images/json":                    getImagesJSON,
 			"/images/viz":                     getImagesViz,
 			"/images/search":                  getImagesSearch,
+			"/images/{name:.*}/get":           getImagesGet,
 			"/images/{name:.*}/history":       getImagesHistory,
 			"/images/{name:.*}/json":          getImagesByName,
 			"/containers/ps":                  getContainersJSON,
@@ -1052,6 +1073,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) {
 			"/build":                        postBuild,
 			"/images/create":                postImagesCreate,
 			"/images/{name:.*}/insert":      postImagesInsert,
+			"/images/load":                  postImagesLoad,
 			"/images/{name:.*}/push":        postImagesPush,
 			"/images/{name:.*}/tag":         postImagesTag,
 			"/containers/create":            postContainersCreate,

commands.go

@@ -92,6 +92,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 		{"insert", "Insert a file in an image"},
 		{"inspect", "Return low-level information on a container"},
 		{"kill", "Kill a running container"},
+		{"load", "Load an image from a tar archive"},
 		{"login", "Register or Login to the docker registry server"},
 		{"logs", "Fetch the logs of a container"},
 		{"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"},
@@ -102,6 +103,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error {
 		{"rm", "Remove one or more containers"},
 		{"rmi", "Remove one or more images"},
 		{"run", "Run a command in a new container"},
+		{"save", "Save an image to a tar archive"},
 		{"search", "Search for an image in the docker index"},
 		{"start", "Start a stopped container"},
 		{"stop", "Stop a running container"},
@@ -1961,6 +1963,42 @@ func (cli *DockerCli) CmdCp(args ...string) error {
 	return nil
 }
 
+func (cli *DockerCli) CmdSave(args ...string) error {
+	cmd := Subcmd("save", "IMAGE DESTINATION", "Save an image to a tar archive")
+	if err := cmd.Parse(args); err != nil {
+		cmd.Usage()
+		return nil
+	}
+
+	if cmd.NArg() != 1 {
+		cmd.Usage()
+		return nil
+	}
+
+	image := cmd.Arg(0)
+
+	if err := cli.stream("GET", "/images/"+image+"/get", nil, cli.out, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cli *DockerCli) CmdLoad(args ...string) error {
+	cmd := Subcmd("load", "SOURCE", "Load an image from a tar archive")
+
+	if cmd.NArg() != 0 {
+		cmd.Usage()
+		return nil
+	}
+
+	err := cli.stream("POST", "/images/load", cli.in, cli.out, nil)
+	if err != nil {
+		fmt.Println("Send failed", err)
+	}
+
+	return nil
+}
+
 func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, error) {
 	var params io.Reader
 	if data != nil {

docs/sources/api/docker_remote_api_v1.7.rst

@@ -1171,6 +1171,53 @@ Monitor Docker's events
         :statuscode 200: no error
         :statuscode 500: server error
 
+Get a tarball containing all images and tags in a repository
+************************************************************
+
+.. http:get:: /images/(name)/get
+
+  Get a tarball containing all images and metadata for the repository specified by ``name``.
+
+  **Example request**
+
+  .. sourcecode:: http
+  
+           GET /images/ubuntu/get
+
+       **Example response**:
+
+       .. sourcecode:: http
+
+          HTTP/1.1 200 OK
+    Content-Type: application/x-tar
+
+    Binary data stream
+        :statuscode 200: no error
+        :statuscode 500: server error
+
+Load a tarball with a set of images and tags into docker
+********************************************************
+
+.. http:post:: /images/load
+
+  Load a set of images and tags into the docker repository.
+
+  **Example request**
+
+  .. sourcecode:: http
+
+           POST /images/load
+
+         Tarball in body
+
+       **Example response**:
+
+       .. sourcecode:: http
+
+          HTTP/1.1 200 OK
+
+        :statuscode 200: no error
+        :statuscode 500: server error
 
 3. Going further
 ================

docs/sources/commandline/cli.rst

@@ -559,6 +559,18 @@ Known Issues (kill)
 * :issue:`197` indicates that ``docker kill`` may leave directories
   behind and make it difficult to remove the container.
 
+.. _cli_load:
+
+``load``
+--------
+
+::
+
+    Usage: docker load < repository.tar
+
+    Loads a tarred repository from the standard input stream.
+    Restores both images and tags.
+
 .. _cli_login:
 
 ``login``
@@ -852,6 +864,17 @@ Known Issues (run -volumes-from)
   could indicate a permissions problem with AppArmor. Please see the
   issue for a workaround.
 
+.. _cli_save:
+
+``save``
+
+::
+
+    Usage: docker save image > repository.tar
+
+    Streams a tarred repository to the standard output stream.
+    Contains all parent layers, and all tags + versions.
+
 .. _cli_search:
 
 ``search``

server.go

@@ -197,6 +197,185 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error {
 	return fmt.Errorf("No such container: %s", name)
 }
 
+// ImageExport exports all images with the given tag. All versions
+// containing the same tag are exported. The resulting output is an
+// uncompressed tar ball.
+// name is the set of tags to export.
+// out is the writer where the images are written to.
+func (srv *Server) ImageExport(name string, out io.Writer) error {
+	// get image json
+	tempdir, err := ioutil.TempDir("", "docker-export-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempdir)
+
+	utils.Debugf("Serializing %s", name)
+
+	rootRepo := srv.runtime.repositories.Repositories[name]
+	for _, rootImage := range rootRepo {
+		image, _ := srv.ImageInspect(rootImage)
+		for i := image; i != nil; {
+			// temporary directory
+			tmpImageDir := path.Join(tempdir, i.ID)
+			if err := os.Mkdir(tmpImageDir, os.ModeDir); err != nil {
+				return err
+			}
+			defer os.RemoveAll(tmpImageDir)
+
+			var version = "1.0"
+			var versionBuf = []byte(version)
+
+			if err := ioutil.WriteFile(path.Join(tmpImageDir, "VERSION"), versionBuf, os.ModeAppend); err != nil {
+				return err
+			}
+
+			// serialize json
+			b, err := json.Marshal(i)
+			if err != nil {
+				return err
+			}
+			if err := ioutil.WriteFile(path.Join(tmpImageDir, "json"), b, os.ModeAppend); err != nil {
+				return err
+			}
+
+			// serialize filesystem
+			fs, err := archive.Tar(path.Join(srv.runtime.graph.Root, i.ID, "layer"), archive.Uncompressed)
+			if err != nil {
+				return err
+			}
+
+			fsTar, err := os.Create(path.Join(tmpImageDir, "layer.tar"))
+			if err != nil {
+				return err
+			}
+			if _, err = io.Copy(fsTar, fs); err != nil {
+				return err
+			}
+			fsTar.Close()
+
+			// find parent
+			if i.Parent != "" {
+				i, err = srv.ImageInspect(i.Parent)
+				if err != nil {
+					return err
+				}
+			} else {
+				i = nil
+			}
+		}
+	}
+
+	// write repositories
+	rootRepoMap := map[string]Repository{}
+	rootRepoMap[name] = rootRepo
+	rootRepoJson, _ := json.Marshal(rootRepoMap)
+
+	if err := ioutil.WriteFile(path.Join(tempdir, "repositories"), rootRepoJson, os.ModeAppend); err != nil {
+		return err
+	}
+
+	fs, err := archive.Tar(tempdir, archive.Uncompressed)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(out, fs); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Loads a set of images into the repository. This is the complementary of ImageExport.
+// The input stream is an uncompressed tar ball containing images and metadata.
+func (srv *Server) ImageLoad(in io.Reader) error {
+	tmpImageDir, err := ioutil.TempDir("", "docker-import-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpImageDir)
+
+	var (
+		repoTarFile = path.Join(tmpImageDir, "repo.tar")
+		repoDir     = path.Join(tmpImageDir, "repo")
+	)
+
+	tarFile, err := os.Create(repoTarFile)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(tarFile, in); err != nil {
+		return err
+	}
+	tarFile.Close()
+
+	repoFile, err := os.Open(repoTarFile)
+	if err != nil {
+		return err
+	}
+	if err := os.Mkdir(repoDir, os.ModeDir); err != nil {
+		return err
+	}
+	if err := archive.Untar(repoFile, repoDir); err != nil {
+		return err
+	}
+	repositoriesJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", "repositories"))
+	if err != nil {
+		return err
+	}
+	repositories := map[string]Repository{}
+	if err := json.Unmarshal(repositoriesJson, &repositories); err != nil {
+		return err
+	}
+
+	for imageName, tagMap := range repositories {
+		for tag, address := range tagMap {
+			if err := srv.recursiveLoad(address, tmpImageDir); err != nil {
+				return err
+			}
+			if err := srv.runtime.repositories.Set(imageName, tag, address, true); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (srv *Server) recursiveLoad(address, tmpImageDir string) error {
+	if _, err := srv.ImageInspect(address); err != nil {
+		utils.Debugf("Loading %s", address)
+
+		imageJson, err := ioutil.ReadFile(path.Join(tmpImageDir, "repo", address, "json"))
+		if err != nil {
+			return err
+			utils.Debugf("Error reading json", err)
+		}
+
+		layer, err := os.Open(path.Join(tmpImageDir, "repo", address, "layer.tar"))
+		if err != nil {
+			utils.Debugf("Error reading embedded tar", err)
+			return err
+		}
+		img, err := NewImgJSON(imageJson)
+		if err != nil {
+			utils.Debugf("Error unmarshalling json", err)
+			return err
+		}
+		if img.Parent != "" {
+			if !srv.runtime.graph.Exists(img.Parent) {
+				if err := srv.recursiveLoad(img.Parent, tmpImageDir); err != nil {
+					return err
+				}
+			}
+		}
+		if err := srv.runtime.graph.Register(imageJson, layer, img); err != nil {
+			return err
+		}
+	}
+	utils.Debugf("Completed processing %s", address)
+	return nil
+}
+
 func (srv *Server) ImagesSearch(term string) ([]registry.SearchResult, error) {
 	r, err := registry.NewRegistry(srv.runtime.config.Root, nil, srv.HTTPRequestFactory(nil))
 	if err != nil {