From 1f61084d83aea37b212468aaa975020094b7f7c9 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 23 Jul 2015 14:19:58 -0700 Subject: [PATCH] Fix uses of "int" where "int64" should be used instead Some structures use int for sizes and UNIX timestamps. On some platforms, int is 32 bits, so this can lead to the year 2038 issues and overflows when dealing with large containers or layers. Consistently use int64 to store sizes and UNIX timestamps in api/types/types.go. Update related to code accordingly (i.e. strconv.FormatInt instead of strconv.Itoa). Use int64 in progressreader package to avoid integer overflow when dealing with large quantities. Update related code accordingly. Signed-off-by: Aaron Lehmann --- api/client/build.go | 2 +- api/client/ps/custom_test.go | 2 +- api/types/types.go | 12 ++++++------ builder/internals.go | 2 +- builder/job.go | 4 ++-- builder/job_test.go | 8 ++++---- daemon/list.go | 6 +++--- graph/graph.go | 6 +++--- graph/graph_unix.go | 2 +- graph/graph_windows.go | 2 +- graph/import.go | 2 +- graph/list.go | 12 ++++++------ graph/pull_v1.go | 4 ++-- graph/pull_v2.go | 4 ++-- graph/push_v1.go | 2 +- graph/push_v2.go | 2 +- pkg/jsonmessage/jsonmessage.go | 6 +++--- pkg/progressreader/progressreader.go | 12 ++++++------ pkg/progressreader/progressreader_test.go | 4 ++-- pkg/tarsum/versioning.go | 10 +++++----- registry/registry_test.go | 2 +- registry/session.go | 6 +++--- 22 files changed, 56 insertions(+), 56 deletions(-) diff --git a/api/client/build.go b/api/client/build.go index bd4f9cff5..6dc59aab2 100644 --- a/api/client/build.go +++ b/api/client/build.go @@ -462,7 +462,7 @@ func getContextFromURL(out io.Writer, remoteURL, dockerfileName string) (absCont In: response.Body, Out: out, Formatter: streamformatter.NewStreamFormatter(), - Size: int(response.ContentLength), + Size: response.ContentLength, NewLines: true, ID: "", Action: fmt.Sprintf("Downloading build context from remote url: %s", remoteURL), diff --git a/api/client/ps/custom_test.go b/api/client/ps/custom_test.go index e65753753..8676518e5 100644 --- a/api/client/ps/custom_test.go +++ b/api/client/ps/custom_test.go @@ -27,7 +27,7 @@ func TestContainerContextID(t *testing.T) { {types.Container{Image: "ubuntu"}, true, "ubuntu", imageHeader, ctx.Image}, {types.Container{Image: ""}, true, "", imageHeader, ctx.Image}, {types.Container{Command: "sh -c 'ls -la'"}, true, `"sh -c 'ls -la'"`, commandHeader, ctx.Command}, - {types.Container{Created: int(unix)}, true, time.Unix(unix, 0).String(), createdAtHeader, ctx.CreatedAt}, + {types.Container{Created: unix}, true, time.Unix(unix, 0).String(), createdAtHeader, ctx.CreatedAt}, {types.Container{Ports: []types.Port{{PrivatePort: 8080, PublicPort: 8080, Type: "tcp"}}}, true, "8080/tcp", portsHeader, ctx.Ports}, {types.Container{Status: "RUNNING"}, true, "RUNNING", statusHeader, ctx.Status}, {types.Container{SizeRw: 10}, true, "10 B", sizeHeader, ctx.Size}, diff --git a/api/types/types.go b/api/types/types.go index 844ddd0f2..a4e0ccf87 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -70,9 +70,9 @@ type Image struct { ParentId string RepoTags []string RepoDigests []string - Created int - Size int - VirtualSize int + Created int64 + Size int64 + VirtualSize int64 Labels map[string]string } @@ -112,10 +112,10 @@ type Container struct { Names []string Image string Command string - Created int + Created int64 Ports []Port - SizeRw int `json:",omitempty"` - SizeRootFs int `json:",omitempty"` + SizeRw int64 `json:",omitempty"` + SizeRootFs int64 `json:",omitempty"` Labels map[string]string Status string HostConfig struct { diff --git a/builder/internals.go b/builder/internals.go index b23b49e8f..6fac57293 100644 --- a/builder/internals.go +++ b/builder/internals.go @@ -330,7 +330,7 @@ func calcCopyInfo(b *builder, cmdName string, cInfos *[]*copyInfo, origPath stri In: resp.Body, Out: b.OutOld, Formatter: b.StreamFormatter, - Size: int(resp.ContentLength), + Size: resp.ContentLength, NewLines: true, ID: "", Action: "Downloading", diff --git a/builder/job.go b/builder/job.go index d3b51aaec..5e3a3c8bd 100644 --- a/builder/job.go +++ b/builder/job.go @@ -139,7 +139,7 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer f.Body.Close() ct := f.Header.Get("Content-Type") - clen := int(f.ContentLength) + clen := f.ContentLength contentType, bodyReader, err := inspectResponse(ct, f.Body, clen) defer bodyReader.Close() @@ -316,7 +316,7 @@ func Commit(name string, d *daemon.Daemon, c *CommitConfig) (string, error) { // - an io.Reader for the response body // - an error value which will be non-nil either when something goes wrong while // reading bytes from r or when the detected content-type is not acceptable. -func inspectResponse(ct string, r io.ReadCloser, clen int) (string, io.ReadCloser, error) { +func inspectResponse(ct string, r io.ReadCloser, clen int64) (string, io.ReadCloser, error) { plen := clen if plen <= 0 || plen > maxPreambleLength { plen = maxPreambleLength diff --git a/builder/job_test.go b/builder/job_test.go index 9ce74b225..2f432d70f 100644 --- a/builder/job_test.go +++ b/builder/job_test.go @@ -31,7 +31,7 @@ func TestInspectEmptyResponse(t *testing.T) { func TestInspectResponseBinary(t *testing.T) { ct := "application/octet-stream" br := ioutil.NopCloser(bytes.NewReader(binaryContext)) - contentType, bReader, err := inspectResponse(ct, br, len(binaryContext)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(binaryContext))) if err != nil { t.Fatal(err) } @@ -56,7 +56,7 @@ func TestResponseUnsupportedContentType(t *testing.T) { content := []byte(textPlainDockerfile) ct := "application/json" br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bReader, err := inspectResponse(ct, br, len(textPlainDockerfile)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(textPlainDockerfile))) if err == nil { t.Fatal("Should have returned an error on content-type 'application/json'") @@ -77,7 +77,7 @@ func TestInspectResponseTextSimple(t *testing.T) { content := []byte(textPlainDockerfile) ct := "text/plain" br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bReader, err := inspectResponse(ct, br, len(content)) + contentType, bReader, err := inspectResponse(ct, br, int64(len(content))) if err != nil { t.Fatal(err) } @@ -96,7 +96,7 @@ func TestInspectResponseTextSimple(t *testing.T) { func TestInspectResponseEmptyContentType(t *testing.T) { content := []byte(textPlainDockerfile) br := ioutil.NopCloser(bytes.NewReader(content)) - contentType, bodyReader, err := inspectResponse("", br, len(content)) + contentType, bodyReader, err := inspectResponse("", br, int64(len(content))) if err != nil { t.Fatal(err) } diff --git a/daemon/list.go b/daemon/list.go index 14e333a7c..1cf7da4c9 100644 --- a/daemon/list.go +++ b/daemon/list.go @@ -152,7 +152,7 @@ func (daemon *Daemon) Containers(config *ContainersConfig) ([]*types.Container, } else { newC.Command = fmt.Sprintf("%s", container.Path) } - newC.Created = int(container.Created.Unix()) + newC.Created = container.Created.Unix() newC.Status = container.State.String() newC.HostConfig.NetworkMode = string(container.hostConfig.NetworkMode) @@ -185,8 +185,8 @@ func (daemon *Daemon) Containers(config *ContainersConfig) ([]*types.Container, if config.Size { sizeRw, sizeRootFs := container.GetSize() - newC.SizeRw = int(sizeRw) - newC.SizeRootFs = int(sizeRootFs) + newC.SizeRw = sizeRw + newC.SizeRootFs = sizeRootFs } newC.Labels = container.Config.Labels containers = append(containers, newC) diff --git a/graph/graph.go b/graph/graph.go index daf088dae..059495922 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -190,7 +190,7 @@ func (graph *Graph) Get(name string) (*image.Image, error) { } img.Size = size - if err := graph.saveSize(graph.imageRoot(id), int(img.Size)); err != nil { + if err := graph.saveSize(graph.imageRoot(id), img.Size); err != nil { return nil, err } } @@ -490,8 +490,8 @@ func (graph *Graph) loadImage(id string) (*image.Image, error) { } // saveSize stores the `size` in the provided graph `img` directory `root`. -func (graph *Graph) saveSize(root string, size int) error { - if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.Itoa(size)), 0600); err != nil { +func (graph *Graph) saveSize(root string, size int64) error { + if err := ioutil.WriteFile(filepath.Join(root, layersizeFileName), []byte(strconv.FormatInt(size, 10)), 0600); err != nil { return fmt.Errorf("Error storing image size in %s/%s: %s", root, layersizeFileName, err) } return nil diff --git a/graph/graph_unix.go b/graph/graph_unix.go index a07807d22..f03dfd1cf 100644 --- a/graph/graph_unix.go +++ b/graph/graph_unix.go @@ -95,7 +95,7 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, img.Size); err != nil { return err } diff --git a/graph/graph_windows.go b/graph/graph_windows.go index a19b356f7..df41f50a9 100644 --- a/graph/graph_windows.go +++ b/graph/graph_windows.go @@ -95,7 +95,7 @@ func (graph *Graph) storeImage(img *image.Image, layerData archive.ArchiveReader } } - if err := graph.saveSize(root, int(img.Size)); err != nil { + if err := graph.saveSize(root, img.Size); err != nil { return err } diff --git a/graph/import.go b/graph/import.go index 4e09cf029..5c11a76ca 100644 --- a/graph/import.go +++ b/graph/import.go @@ -45,7 +45,7 @@ func (s *TagStore) Import(src string, repo string, tag string, inConfig io.ReadC In: resp.Body, Out: outStream, Formatter: sf, - Size: int(resp.ContentLength), + Size: resp.ContentLength, NewLines: true, ID: "", Action: "Importing", diff --git a/graph/list.go b/graph/list.go index 72eae6342..13ed73873 100644 --- a/graph/list.go +++ b/graph/list.go @@ -99,9 +99,9 @@ func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, newImage := new(types.Image) newImage.ParentId = image.Parent newImage.ID = image.ID - newImage.Created = int(image.Created.Unix()) - newImage.Size = int(image.Size) - newImage.VirtualSize = int(s.graph.GetParentsSize(image) + image.Size) + newImage.Created = image.Created.Unix() + newImage.Size = image.Size + newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size newImage.Labels = image.ContainerConfig.Labels if utils.DigestReference(ref) { @@ -136,9 +136,9 @@ func (s *TagStore) Images(filterArgs, filter string, all bool) ([]*types.Image, newImage.RepoTags = []string{":"} newImage.RepoDigests = []string{"@"} newImage.ID = image.ID - newImage.Created = int(image.Created.Unix()) - newImage.Size = int(image.Size) - newImage.VirtualSize = int(s.graph.GetParentsSize(image) + image.Size) + newImage.Created = image.Created.Unix() + newImage.Size = image.Size + newImage.VirtualSize = s.graph.GetParentsSize(image) + image.Size newImage.Labels = image.ContainerConfig.Labels images = append(images, newImage) diff --git a/graph/pull_v1.go b/graph/pull_v1.go index 5eb75372f..0280f1f95 100644 --- a/graph/pull_v1.go +++ b/graph/pull_v1.go @@ -256,7 +256,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro out.Write(p.sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil)) var ( imgJSON []byte - imgSize int + imgSize int64 err error img *image.Image ) @@ -290,7 +290,7 @@ func (p *v1Puller) pullImage(imgID, endpoint string, token []string) (bool, erro status = fmt.Sprintf("Pulling fs layer [retries: %d]", j) } out.Write(p.sf.FormatProgress(stringid.TruncateID(id), status, nil)) - layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, int64(imgSize)) + layer, err := p.session.GetRemoteImageLayer(img.ID, endpoint, imgSize) if uerr, ok := err.(*url.Error); ok { err = uerr.Err } diff --git a/graph/pull_v2.go b/graph/pull_v2.go index 6c2fa2065..d98614109 100644 --- a/graph/pull_v2.go +++ b/graph/pull_v2.go @@ -167,7 +167,7 @@ func (p *v2Puller) download(di *downloadInfo) { In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)), Out: out, Formatter: p.sf, - Size: int(di.size), + Size: di.size, NewLines: false, ID: stringid.TruncateID(di.img.ID), Action: "Downloading", @@ -264,7 +264,7 @@ func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) { In: d.tmpFile, Out: out, Formatter: p.sf, - Size: int(d.size), + Size: d.size, NewLines: false, ID: stringid.TruncateID(d.img.ID), Action: "Extracting", diff --git a/graph/push_v1.go b/graph/push_v1.go index d473040f8..6de293403 100644 --- a/graph/push_v1.go +++ b/graph/push_v1.go @@ -289,7 +289,7 @@ func (p *v1Pusher) pushImage(imgID, ep string, token []string) (checksum string, In: layerData, Out: p.out, Formatter: p.sf, - Size: int(layerData.Size), + Size: layerData.Size, NewLines: false, ID: stringid.TruncateID(imgData.ID), Action: "Pushing", diff --git a/graph/push_v2.go b/graph/push_v2.go index 869347df6..3d74ad46f 100644 --- a/graph/push_v2.go +++ b/graph/push_v2.go @@ -239,7 +239,7 @@ func (p *v2Pusher) pushV2Image(bs distribution.BlobService, img *image.Image) (d In: ioutil.NopCloser(tf), Out: out, Formatter: p.sf, - Size: int(size), + Size: size, NewLines: false, ID: stringid.TruncateID(img.ID), Action: "Pushing", diff --git a/pkg/jsonmessage/jsonmessage.go b/pkg/jsonmessage/jsonmessage.go index 7db1626e4..0c93ee670 100644 --- a/pkg/jsonmessage/jsonmessage.go +++ b/pkg/jsonmessage/jsonmessage.go @@ -23,8 +23,8 @@ func (e *JSONError) Error() string { type JSONProgress struct { terminalFd uintptr - Current int `json:"current,omitempty"` - Total int `json:"total,omitempty"` + Current int64 `json:"current,omitempty"` + Total int64 `json:"total,omitempty"` Start int64 `json:"start,omitempty"` } @@ -64,7 +64,7 @@ func (p *JSONProgress) String() string { numbersBox = fmt.Sprintf("%8v/%v", current, total) if p.Current > 0 && p.Start > 0 && percentage < 50 { - fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) + fromStart := time.Now().UTC().Sub(time.Unix(p.Start, 0)) perEntry := fromStart / time.Duration(p.Current) left := time.Duration(p.Total-p.Current) * perEntry left = (left / time.Second) * time.Second diff --git a/pkg/progressreader/progressreader.go b/pkg/progressreader/progressreader.go index d55fadd71..f48442b59 100644 --- a/pkg/progressreader/progressreader.go +++ b/pkg/progressreader/progressreader.go @@ -14,9 +14,9 @@ type Config struct { In io.ReadCloser // Stream to read from Out io.Writer // Where to send progress bar to Formatter *streamformatter.StreamFormatter - Size int - Current int - LastUpdate int + Size int64 + Current int64 + LastUpdate int64 NewLines bool ID string Action string @@ -29,11 +29,11 @@ func New(newReader Config) *Config { func (config *Config) Read(p []byte) (n int, err error) { read, err := config.In.Read(p) - config.Current += read - updateEvery := 1024 * 512 //512kB + config.Current += int64(read) + updateEvery := int64(1024 * 512) //512kB if config.Size > 0 { // Update progress for every 1% read if 1% < 512kB - if increment := int(0.01 * float64(config.Size)); increment < updateEvery { + if increment := int64(0.01 * float64(config.Size)); increment < updateEvery { updateEvery = increment } } diff --git a/pkg/progressreader/progressreader_test.go b/pkg/progressreader/progressreader_test.go index fdf40cbb3..21d9b0f05 100644 --- a/pkg/progressreader/progressreader_test.go +++ b/pkg/progressreader/progressreader_test.go @@ -20,7 +20,7 @@ func TestOutputOnPrematureClose(t *testing.T) { In: reader, Out: writer, Formatter: streamformatter.NewStreamFormatter(), - Size: len(content), + Size: int64(len(content)), NewLines: true, ID: "Test", Action: "Read", @@ -60,7 +60,7 @@ func TestCompleteSilently(t *testing.T) { In: reader, Out: writer, Formatter: streamformatter.NewStreamFormatter(), - Size: len(content), + Size: int64(len(content)), NewLines: true, ID: "Test", Action: "Read", diff --git a/pkg/tarsum/versioning.go b/pkg/tarsum/versioning.go index 8988b9f58..288228685 100644 --- a/pkg/tarsum/versioning.go +++ b/pkg/tarsum/versioning.go @@ -95,17 +95,17 @@ func (f tarHeaderSelectFunc) selectHeaders(h *tar.Header) (orderedHeaders [][2]s func v0TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) { return [][2]string{ {"name", h.Name}, - {"mode", strconv.Itoa(int(h.Mode))}, + {"mode", strconv.FormatInt(h.Mode, 10)}, {"uid", strconv.Itoa(h.Uid)}, {"gid", strconv.Itoa(h.Gid)}, - {"size", strconv.Itoa(int(h.Size))}, - {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"size", strconv.FormatInt(h.Size, 10)}, + {"mtime", strconv.FormatInt(h.ModTime.UTC().Unix(), 10)}, {"typeflag", string([]byte{h.Typeflag})}, {"linkname", h.Linkname}, {"uname", h.Uname}, {"gname", h.Gname}, - {"devmajor", strconv.Itoa(int(h.Devmajor))}, - {"devminor", strconv.Itoa(int(h.Devminor))}, + {"devmajor", strconv.FormatInt(h.Devmajor, 10)}, + {"devminor", strconv.FormatInt(h.Devminor, 10)}, } } diff --git a/registry/registry_test.go b/registry/registry_test.go index 88b08dffa..d9ac5c6f2 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -185,7 +185,7 @@ func TestGetRemoteImageJSON(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, size, 154, "Expected size 154") + assertEqual(t, size, int64(154), "Expected size 154") if len(json) <= 0 { t.Fatal("Expected non-empty json") } diff --git a/registry/session.go b/registry/session.go index 9bec7c1b2..a9c4daf3a 100644 --- a/registry/session.go +++ b/registry/session.go @@ -240,7 +240,7 @@ func (r *Session) LookupRemoteImage(imgID, registry string) error { } // GetRemoteImageJSON retrieves an image's JSON metadata from the registry. -func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error) { +func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int64, error) { res, err := r.client.Get(registry + "images/" + imgID + "/json") if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) @@ -250,9 +250,9 @@ func (r *Session) GetRemoteImageJSON(imgID, registry string) ([]byte, int, error return nil, -1, httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } // if the size header is not present, then set it to '-1' - imageSize := -1 + imageSize := int64(-1) if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { - imageSize, err = strconv.Atoi(hdr) + imageSize, err = strconv.ParseInt(hdr, 10, 64) if err != nil { return nil, -1, err }