From 36fbf4b86469ca6fe3677d47c9a1976bcdd111e4 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Fri, 17 Apr 2015 15:30:22 -0700 Subject: [PATCH 1/5] Shallow clone using git to build images. Signed-off-by: David Calavera --- builder/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/job.go b/builder/job.go index 115d89a4b..7ea1ae30b 100644 --- a/builder/job.go +++ b/builder/job.go @@ -114,7 +114,7 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer os.RemoveAll(root) - if output, err := exec.Command("git", "clone", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { + if output, err := exec.Command("git", "clone", "--depth", "1", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { return fmt.Errorf("Error trying to use git: %s (%s)", err, output) } From 9fb7204a41804131c2492f9d50d7451e123a05e5 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 21 Apr 2015 10:58:38 -0700 Subject: [PATCH 2/5] Do not try to shallow git history when the protocol doesn't allow it. This only happens with the old git http dumb protocol, but that's what we use in our integration tests. We check the Content-Type header advertised in http requests to make sure the http transport is the git smart transport: See this commit as a reference: https://github.com/git/git/commit/4656bf47fca857df51b5d6f4b7b052192b3b2317 Signed-off-by: David Calavera --- builder/job.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/builder/job.go b/builder/job.go index 7ea1ae30b..8c031a8bd 100644 --- a/builder/job.go +++ b/builder/job.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "os/exec" "strings" @@ -114,7 +115,9 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer os.RemoveAll(root) - if output, err := exec.Command("git", "clone", "--depth", "1", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { + clone := cloneArgs(buildConfig.RemoteURL, root) + + if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { return fmt.Errorf("Error trying to use git: %s (%s)", err, output) } @@ -239,3 +242,21 @@ func Commit(d *daemon.Daemon, name string, c *daemon.ContainerCommitConfig) (str return img.ID, nil } + +func cloneArgs(remoteURL, root string) []string { + args := []string{"clone", "--recursive"} + shallow := true + + if strings.HasPrefix(remoteURL, "http") { + res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) + if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { + shallow = false + } + } + + if shallow { + args = append(args, "--depth", "1") + } + + return append(args, remoteURL, root) +} From 3117bf3ef562bc43ef007d8afe3815c58aac6e85 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 21 Apr 2015 14:24:29 -0700 Subject: [PATCH 3/5] Test that we set the right arguments for git cloning depending on the transport. Signed-off-by: David Calavera --- builder/job_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 builder/job_test.go diff --git a/builder/job_test.go b/builder/job_test.go new file mode 100644 index 000000000..79421a068 --- /dev/null +++ b/builder/job_test.go @@ -0,0 +1,56 @@ +package builder + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" +) + +func TestCloneArgsSmartHttp(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + serverURL, _ := url.Parse(server.URL) + + serverURL.Path = "/repo.git" + gitURL := serverURL.String() + + mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("service") + w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q)) + }) + + args := cloneArgs(gitURL, "/tmp") + exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} + +func TestCloneArgsDumbHttp(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + serverURL, _ := url.Parse(server.URL) + + serverURL.Path = "/repo.git" + gitURL := serverURL.String() + + mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + }) + + args := cloneArgs(gitURL, "/tmp") + exp := []string{"clone", "--recursive", gitURL, "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} +func TestCloneArgsGit(t *testing.T) { + args := cloneArgs("git://github.com/docker/docker", "/tmp") + exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} From 8bd5a95e1e39541dfc5dc635c5c8e7604fe10028 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Thu, 23 Apr 2015 09:35:19 -0700 Subject: [PATCH 4/5] Document the extra `depth` argument in git contexts. Signed-off-by: David Calavera --- docs/sources/reference/commandline/cli.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index a87116204..26659c8ff 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -636,12 +636,13 @@ refer to any of the files in the context. For example, your build can use an [*ADD*](/reference/builder/#add) instruction to reference a file in the context. -The `URL` parameter can specify the location of a Git repository; in this -case, the repository is the context. The Git repository is recursively -cloned with its submodules. The system does a fresh `git clone -recursive` -in a temporary directory on your local host. Then, this clone is sent to -the Docker daemon as the context. Local clones give you the ability to -access private repositories using local user credentials, VPN's, and so forth. +The `URL` parameter can specify the location of a Git repository; +the repository acts as the build context. The system recursively clones the repository +and its submodules using a `git clone --depth 1 --recursive` command. +This command runs in a temporary directory on your local host. +After the command succeeds, the directory is sent to the Docker daemon as the context. +Local clones give you the ability to access private repositories using +local user credentials, VPN's, and so forth. Instead of specifying a context, you can pass a single Dockerfile in the `URL` or pipe the file in via `STDIN`. To pipe a Dockerfile from `STDIN`: From 1cfb307d70558110e8e88a7391ae0e6cf6ebf174 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Thu, 23 Apr 2015 10:19:34 -0700 Subject: [PATCH 5/5] Remove duplicated git clone logic. Signed-off-by: David Calavera --- api/client/build.go | 11 +----- builder/job.go | 32 +--------------- utils/git.go | 47 ++++++++++++++++++++++++ builder/job_test.go => utils/git_test.go | 2 +- 4 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 utils/git.go rename builder/job_test.go => utils/git_test.go (98%) diff --git a/api/client/build.go b/api/client/build.go index 63cc63bc9..eb39058b6 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -95,20 +95,11 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } else { root := cmd.Arg(0) if urlutil.IsGitURL(root) { - remoteURL := cmd.Arg(0) - if !urlutil.IsGitTransport(remoteURL) { - remoteURL = "https://" + remoteURL - } - - root, err = ioutil.TempDir("", "docker-build-git") + root, err = utils.GitClone(root) if err != nil { return err } defer os.RemoveAll(root) - - if output, err := exec.Command("git", "clone", "--recursive", remoteURL, root).CombinedOutput(); err != nil { - return fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } } if _, err := os.Stat(root); err != nil { return err diff --git a/builder/job.go b/builder/job.go index 8c031a8bd..7991cba21 100644 --- a/builder/job.go +++ b/builder/job.go @@ -5,9 +5,7 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" - "os/exec" "strings" "sync" @@ -23,6 +21,7 @@ import ( "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + "github.com/docker/docker/utils" ) // whitelist of commands allowed for a commit/import @@ -106,21 +105,12 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { if buildConfig.RemoteURL == "" { context = ioutil.NopCloser(buildConfig.Context) } else if urlutil.IsGitURL(buildConfig.RemoteURL) { - if !urlutil.IsGitTransport(buildConfig.RemoteURL) { - buildConfig.RemoteURL = "https://" + buildConfig.RemoteURL - } - root, err := ioutil.TempDir("", "docker-build-git") + root, err := utils.GitClone(buildConfig.RemoteURL) if err != nil { return err } defer os.RemoveAll(root) - clone := cloneArgs(buildConfig.RemoteURL, root) - - if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { - return fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } - c, err := archive.Tar(root, archive.Uncompressed) if err != nil { return err @@ -242,21 +232,3 @@ func Commit(d *daemon.Daemon, name string, c *daemon.ContainerCommitConfig) (str return img.ID, nil } - -func cloneArgs(remoteURL, root string) []string { - args := []string{"clone", "--recursive"} - shallow := true - - if strings.HasPrefix(remoteURL, "http") { - res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) - if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { - shallow = false - } - } - - if shallow { - args = append(args, "--depth", "1") - } - - return append(args, remoteURL, root) -} diff --git a/utils/git.go b/utils/git.go new file mode 100644 index 000000000..18e002d18 --- /dev/null +++ b/utils/git.go @@ -0,0 +1,47 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "net/http" + "os/exec" + "strings" + + "github.com/docker/docker/pkg/urlutil" +) + +func GitClone(remoteURL string) (string, error) { + if !urlutil.IsGitTransport(remoteURL) { + remoteURL = "https://" + remoteURL + } + root, err := ioutil.TempDir("", "docker-build-git") + if err != nil { + return "", err + } + + clone := cloneArgs(remoteURL, root) + + if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { + return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) + } + + return root, nil +} + +func cloneArgs(remoteURL, root string) []string { + args := []string{"clone", "--recursive"} + shallow := true + + if strings.HasPrefix(remoteURL, "http") { + res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) + if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { + shallow = false + } + } + + if shallow { + args = append(args, "--depth", "1") + } + + return append(args, remoteURL, root) +} diff --git a/builder/job_test.go b/utils/git_test.go similarity index 98% rename from builder/job_test.go rename to utils/git_test.go index 79421a068..a82841ae1 100644 --- a/builder/job_test.go +++ b/utils/git_test.go @@ -1,4 +1,4 @@ -package builder +package utils import ( "fmt"