From dc8d4a706100f8090fb7fc8dc030d1f08df41e7b Mon Sep 17 00:00:00 2001 From: Jonathan Boulle Date: Mon, 8 Dec 2014 15:37:14 -0800 Subject: [PATCH] *: numerous updates to match spec Changes to schema: - removing FilesetManifest - update example manifest to match that in spec - splitting out App into a dedicated type embedded in AppImageManifest Changes to tools: - update aci writer (no more filesets) - update actool build (no more filesets) - update actool validate (no more filesets) - update ace validator to use new types - update rocket stage0/stage1 to new schema - update ace app manifests to new spec - fix a bug in build_aci (double compressing aci) --- app-container/ace/app_manifest_main.json | 98 ++++++------ app-container/ace/app_manifest_sidekick.json | 34 ++-- app-container/ace/build_aci | 5 +- app-container/ace/validator.go | 6 +- app-container/aci/layout.go | 106 +++---------- app-container/aci/writer.go | 45 +----- app-container/actool/build.go | 53 +++---- app-container/actool/validate.go | 7 +- app-container/examples/app.json | 159 +++++++++++-------- app-container/schema/app.go | 59 ++----- app-container/schema/container.go | 10 +- app-container/schema/doc.go | 2 +- app-container/schema/fileset.go | 75 --------- app-container/schema/types/ackind.go | 2 +- app-container/schema/types/ackind_test.go | 2 +- app-container/schema/types/annotations.go | 9 -- app-container/schema/types/app.go | 52 ++++++ app-container/schema/types/errors.go | 2 +- app-container/schema/types/labels.go | 59 +++++++ metadatasvc/main.go | 20 +-- path/path.go | 4 +- stage0/run.go | 12 +- stage1/container.go | 29 ++-- 23 files changed, 379 insertions(+), 471 deletions(-) delete mode 100644 app-container/schema/fileset.go create mode 100644 app-container/schema/types/app.go create mode 100644 app-container/schema/types/labels.go diff --git a/app-container/ace/app_manifest_main.json b/app-container/ace/app_manifest_main.json index ea614c1..592433b 100644 --- a/app-container/ace/app_manifest_main.json +++ b/app-container/ace/app_manifest_main.json @@ -1,56 +1,60 @@ { "acVersion": "1.0.0", - "acKind": "AppManifest", + "acKind": "AppImageManifest", "name": "coreos.com/ace-validator-main", - "version": "1.0.0", - "os": "linux", - "arch": "amd64", - "exec": [ - "/ace-validator", "main" + "labels": [ + { "name": "version", "val": "1.0.0" }, + { "name": "os", "val": "linux" }, + { "name": "arch", "val": "amd64" } ], - "eventHandlers": [ - { - "name": "pre-start", - "exec": [ - "/ace-validator", "prestart" - ] + "app": { + "exec": [ + "/ace-validator", "main" + ], + "eventHandlers": [ + { + "name": "pre-start", + "exec": [ + "/ace-validator", "prestart" + ] + }, + { + "name": "post-stop", + "exec": [ + "/ace-validator", "poststop" + ] + } + ], + "user": "0", + "group": "0", + "environment": { + "IN_ACE_VALIDATOR": "correct" }, - { - "name": "post-stop", - "exec": [ - "/ace-validator", "poststop" - ] - } - ], - "user": "0", - "group": "0", - "environment": { - "IN_ACE_VALIDATOR": "correct" + "mountPoints": [ + { + "name": "database", + "path": "/db", + "readOnly": false + } + ], + "ports": [ + { + "name": "www", + "protocol": "tcp", + "port": 80 + } + ], + "isolators": [ + { + "name": "private-network", + "val": "true" + }, + { + "name": "memory/limit", + "val": "1G" + } + ] }, - "mountPoints": [ - { - "name": "database", - "path": "/db", - "readOnly": false - } - ], - "ports": [ - { - "name": "www", - "protocol": "tcp", - "port": 80 - } - ], - "isolators": [ - { - "name": "private-network", - "val": "true" - }, - { - "name": "memory/limit", - "val": "1G" - } - ], "annotations": { "created": "2014-10-27T19:32:27.67021798Z", "authors": "Carly Container , Nat Network ", diff --git a/app-container/ace/app_manifest_sidekick.json b/app-container/ace/app_manifest_sidekick.json index de202e9..51e79c1 100644 --- a/app-container/ace/app_manifest_sidekick.json +++ b/app-container/ace/app_manifest_sidekick.json @@ -1,20 +1,24 @@ { "acVersion": "1.0.0", - "acKind": "AppManifest", + "acKind": "AppImageManifest", "name": "coreos.com/ace-validator-sidekick", - "version": "1.0.0", - "os": "linux", - "arch": "amd64", - "exec": [ - "/ace-validator", "sidekick" + "labels": [ + { "name": "version", "val": "1.0.0" }, + { "name": "os", "val": "linux" }, + { "name": "arch", "val": "amd64" } ], - "user": "0", - "group": "0", - "mountPoints": [ - { - "name": "database", - "path": "/db", - "readOnly": false - } - ] + "app": { + "exec": [ + "/ace-validator", "sidekick" + ], + "user": "0", + "group": "0", + "mountPoints": [ + { + "name": "database", + "path": "/db", + "readOnly": false + } + ] + } } diff --git a/app-container/ace/build_aci b/app-container/ace/build_aci index f351143..e2af186 100755 --- a/app-container/ace/build_aci +++ b/app-container/ace/build_aci @@ -23,9 +23,8 @@ for typ in main sidekick; do touch -a -m -d 1415660606 ${path} done ../actool build --overwrite --app-manifest app rootfs/ ../ace-validator-${typ}.aci - HASH=sha256-$(sha256sum ../ace-validator-${typ}.aci|awk '{print $1}') - gzip -f ../ace-validator-${typ}.aci - mv ../ace-validator-${typ}.aci.gz ../ace-validator-${typ}.aci + # TODO(jonboulle): create uncompressed instead, then gzip? + HASH=sha256-$(gzip -d -f ../ace-validator-${typ}.aci -c | sha256sum - | awk '{print $1}') gpg --cipher-algo AES256 --output ace-validator-${typ}.sig --detach-sig ../ace-validator-${typ}.aci mv ace-validator-${typ}.sig ../ popd >/dev/null diff --git a/app-container/ace/validator.go b/app-container/ace/validator.go index 1ef0823..05d6ee3 100644 --- a/app-container/ace/validator.go +++ b/app-container/ace/validator.go @@ -307,7 +307,7 @@ func validateContainerMetadata(crm *schema.ContainerRuntimeManifest) results { return append(r, validateContainerAnnotations(crm)...) } -func validateAppAnnotations(crm *schema.ContainerRuntimeManifest, app *schema.AppManifest) results { +func validateAppAnnotations(crm *schema.ContainerRuntimeManifest, app *schema.AppImageManifest) results { r := results{} // build a map of expected annotations by merging app.Annotations @@ -358,7 +358,7 @@ func validateAppAnnotations(crm *schema.ContainerRuntimeManifest, app *schema.Ap return r } -func validateAppMetadata(crm *schema.ContainerRuntimeManifest, a schema.App) results { +func validateAppMetadata(crm *schema.ContainerRuntimeManifest, a schema.RuntimeApp) results { appName := a.Name r := results{} @@ -367,7 +367,7 @@ func validateAppMetadata(crm *schema.ContainerRuntimeManifest, a schema.App) res return append(r, err) } - app := &schema.AppManifest{} + app := &schema.AppImageManifest{} if err = json.Unmarshal(am, app); err != nil { return append(r, fmt.Errorf("failed to JSON-decode %q manifest: %v", string(appName), err)) } diff --git a/app-container/aci/layout.go b/app-container/aci/layout.go index c07a2af..ead3326 100644 --- a/app-container/aci/layout.go +++ b/app-container/aci/layout.go @@ -5,24 +5,13 @@ package aci Image Layout The on-disk layout of an app container is straightforward. -It includes a rootfs with all of the files that will exist in the root of the app and an app manifest describing how to execute the app. -The layout must contain either an app manifest, an app manifest and a fileset manifest, or a fileset only. -In the latter case, the layout/image is known as a "fileset image". +It includes a rootfs with all of the files that will exist in the root of the app and a manifest describing the image. +The layout must contain an app image manifest. -/app +/manifest /rootfs/ /rootfs/usr/bin/mysql -/app -/fileset -/rootfs/bin/httpd -/rootfs/config - - -/fileset -/rootfs/ -/rootfs/bin/bash - */ import ( @@ -42,7 +31,7 @@ import ( var ( ErrNoRootFS = errors.New("no rootfs found in layout") - ErrNoManifest = errors.New("no app or fileset manifest found in layout") + ErrNoManifest = errors.New("no app image manifest found in layout") ) // ValidateLayout takes a directory and validates that the layout of the directory @@ -58,8 +47,8 @@ func ValidateLayout(dir string) error { return fmt.Errorf("given path %q is not a directory", dir) } var flist []string - var amOK, fsmOK, rfsOK bool - var am, fsm io.Reader + var amOK, rfsOK bool + var am io.Reader walkLayout := func(fpath string, fi os.FileInfo, err error) error { rpath := strings.TrimPrefix(fpath, dir) name := filepath.Base(rpath) @@ -71,12 +60,6 @@ func ValidateLayout(dir string) error { return err } amOK = true - case "fileset": - fsm, err = os.Open(fpath) - if err != nil { - return err - } - fsmOK = true case "rootfs": if !fi.IsDir() { return errors.New("rootfs is not a directory") @@ -90,7 +73,7 @@ func ValidateLayout(dir string) error { if err := filepath.Walk(dir, walkLayout); err != nil { return err } - return validate(amOK, am, fsmOK, fsm, rfsOK, flist) + return validate(amOK, am, rfsOK, flist) } // ValidateLayout takes a *tar.Reader and validates that the layout of the @@ -99,8 +82,8 @@ func ValidateLayout(dir string) error { // the validation, it will abort and return the first one. func ValidateArchive(tr *tar.Reader) error { var flist []string - var amOK, fsmOK, rfsOK bool - var fsm, am bytes.Buffer + var amOK, rfsOK bool + var am bytes.Buffer Tar: for { hdr, err := tr.Next() @@ -112,12 +95,6 @@ Tar: return err } switch hdr.Name { - case "fileset": - _, err := io.Copy(&fsm, tr) - if err != nil { - return err - } - fsmOK = true case "app": _, err := io.Copy(&am, tr) if err != nil { @@ -133,12 +110,11 @@ Tar: flist = append(flist, hdr.Name) } } - return validate(amOK, &am, fsmOK, &fsm, rfsOK, flist) + return validate(amOK, &am, rfsOK, flist) } -// TODO(jonboulle): find a cleaner way to communicate instead of all these args. -func validate(amOK bool, am io.Reader, fsmOK bool, fsm io.Reader, rfsOK bool, files []string) error { - if !amOK && !fsmOK { +func validate(amOK bool, am io.Reader, rfsOK bool, files []string) error { + if !amOK { return ErrNoManifest } if amOK { @@ -149,75 +125,29 @@ func validate(amOK bool, am io.Reader, fsmOK bool, fsm io.Reader, rfsOK bool, fi if err != nil { return fmt.Errorf("error reading app manifest: %v", err) } - var a schema.AppManifest + var a schema.AppImageManifest if err := a.UnmarshalJSON(b); err != nil { return fmt.Errorf("app manifest validation failed: %v", err) } } - var rfsfiles []string for _, f := range files { - switch { - case strings.HasPrefix(f, "rootfs"): - rfsfiles = append(rfsfiles, strings.TrimPrefix(f, "rootfs")) - default: + if !strings.HasPrefix(f, "rootfs") { return fmt.Errorf("unrecognized file path in layout: %q", f) } } - if fsmOK { - b, err := ioutil.ReadAll(fsm) - if err != nil { - return fmt.Errorf("error reading fileset manifest: %v", err) - } - var f schema.FilesetManifest - if err := f.UnmarshalJSON(b); err != nil { - return fmt.Errorf("fileset manifest validation failed: %v", err) - } - // TODO(jonboulle): this is not quite correct since it does not - // deal with dependent filesets. Maybe filesAreSuperset()? - return filesEqual(f.Files, rfsfiles) - } return nil } -// validateAppManifest ensures that the given io.Reader represents a valid -// AppManifest. -func validateAppManifest(r io.Reader) error { +// validateAppImageManifest ensures that the given io.Reader represents a valid +// AppImageManifest. +func validateAppImageManifest(r io.Reader) error { b, err := ioutil.ReadAll(r) if err != nil { return fmt.Errorf("error reading app manifest: %v", err) } - var am schema.AppManifest + var am schema.AppImageManifest if err = json.Unmarshal(b, &am); err != nil { return fmt.Errorf("error unmarshaling app manifest: %v", err) } return nil } - -// validateFilesetManifest ensures that the given io.Reader represents a valid -// FilesetManifest. -func validateFilesetManifest(r io.Reader) error { - b, err := ioutil.ReadAll(r) - if err != nil { - return fmt.Errorf("error reading app manifest: %v", err) - } - var am schema.FilesetManifest - if err = json.Unmarshal(b, &am); err != nil { - return fmt.Errorf("error unmarshaling app manifest: %v", err) - } - return nil -} - -func filesEqual(a, b []string) error { - na := len(a) - nb := len(b) - if na != nb { - return fmt.Errorf("fileset has different filecount to rootfs (%d != %d)", na, nb) - } - - for i := range a { - if a[i] != b[i] { - return fmt.Errorf("file mismatch %s != %s", a[i], b[i]) - } - } - return nil -} diff --git a/app-container/aci/writer.go b/app-container/aci/writer.go index 1ed9686..b04a462 100644 --- a/app-container/aci/writer.go +++ b/app-container/aci/writer.go @@ -5,15 +5,14 @@ import ( "bytes" "encoding/json" "io" - "strings" "time" "github.com/coreos/rocket/app-container/schema" ) // ArchiveWriter writes App Container Images. Users wanting to create an ACI or -// Fileset ACI should create an ArchiveWriter and add files to it; the ACI will -// be written to the underlying tar.Writer +// should create an ArchiveWriter and add files to it; the ACI will be written +// to the underlying tar.Writer type ArchiveWriter interface { AddFile(path string, hdr *tar.Header, r io.Reader) error Close() error @@ -21,18 +20,13 @@ type ArchiveWriter interface { type appArchiveWriter struct { *tar.Writer - am *schema.AppManifest -} - -type fsArchiveWriter struct { - appArchiveWriter - fsm *schema.FilesetManifest + am *schema.AppImageManifest } // NewAppWriter creates a new ArchiveWriter which will generate an App // Container Image based on the given manifest and write it to the given // tar.Writer -func NewAppWriter(am schema.AppManifest, w *tar.Writer) ArchiveWriter { +func NewAppWriter(am schema.AppImageManifest, w *tar.Writer) ArchiveWriter { aw := &appArchiveWriter{ w, &am, @@ -40,23 +34,6 @@ func NewAppWriter(am schema.AppManifest, w *tar.Writer) ArchiveWriter { return aw } -// NewFilesetWriter creates a new ArchiveWriter which will generate a Fileset -// ACI by the given name and write it to the given tar.Writer. -func NewFilesetWriter(name string, w *tar.Writer) (ArchiveWriter, error) { - fsm, err := schema.NewFilesetManifest(name) - if err != nil { - return nil, err - } - aw := &fsArchiveWriter{ - appArchiveWriter{ - w, - nil, - }, - fsm, - } - return aw, nil -} - func (aw *appArchiveWriter) AddFile(path string, hdr *tar.Header, r io.Reader) error { err := aw.Writer.WriteHeader(hdr) if err != nil { @@ -73,14 +50,6 @@ func (aw *appArchiveWriter) AddFile(path string, hdr *tar.Header, r io.Reader) e return nil } -func (aw *fsArchiveWriter) AddFile(path string, hdr *tar.Header, r io.Reader) error { - relpath := strings.TrimPrefix(path, "rootfs") - if relpath != "/" { - aw.fsm.Files = append(aw.fsm.Files, relpath) - } - return aw.appArchiveWriter.AddFile(path, hdr, r) -} - func (aw *appArchiveWriter) addFileNow(path string, contents []byte) error { buf := bytes.NewBuffer(contents) now := time.Now() @@ -113,9 +82,3 @@ func (aw *appArchiveWriter) Close() error { } return aw.Writer.Close() } -func (aw *fsArchiveWriter) Close() error { - if err := aw.addManifest("fileset", aw.fsm); err != nil { - return err - } - return aw.Writer.Close() -} diff --git a/app-container/actool/build.go b/app-container/actool/build.go index 5600639..1faab78 100644 --- a/app-container/actool/build.go +++ b/app-container/actool/build.go @@ -13,23 +13,20 @@ import ( ) var ( - buildFilesetName string - buildAppManifest string - buildRootfs bool - buildOverwrite bool - cmdBuild = &Command{ + buildAppImageManifest string + buildRootfs bool + buildOverwrite bool + cmdBuild = &Command{ Name: "build", - Description: "Build a Fileset ACI from the target directory", - Summary: "Build a Fileset ACI from the target directory", + Description: "Build an ACI from the target directory", + Summary: "Build an ACI from the target directory", Usage: "[--overwrite] --name=NAME DIRECTORY OUTPUT_FILE", Run: runBuild, } ) func init() { - cmdBuild.Flags.StringVar(&buildFilesetName, "fileset-name", "", - "Build a Fileset Image, by this name (e.g. example.com/reduce-worker)") - cmdBuild.Flags.StringVar(&buildAppManifest, "app-manifest", "", + cmdBuild.Flags.StringVar(&buildAppImageManifest, "app-manifest", "", "Build an App Image with this App Manifest") cmdBuild.Flags.BoolVar(&buildRootfs, "rootfs", true, "Whether the supplied directory is a rootfs. If false, it will be assume the supplied directory already contains a rootfs/ subdirectory.") @@ -100,11 +97,8 @@ func runBuild(args []string) (exit int) { stderr("build: Must provide directory and output file") return 1 } - switch { - case buildFilesetName != "" && buildAppManifest == "": - case buildFilesetName == "" && buildAppManifest != "": - default: - stderr("build: must specify either --fileset-name or --app-manifest") + if buildAppImageManifest == "" { + stderr("build: must specify --app-manifest") return 1 } @@ -142,26 +136,17 @@ func runBuild(args []string) (exit int) { } }() - var aw aci.ArchiveWriter - if buildFilesetName != "" { - aw, err = aci.NewFilesetWriter(buildFilesetName, tr) - if err != nil { - stderr("build: Unable to create FilesetWriter: %v", err) - return 1 - } - } else { - b, err := ioutil.ReadFile(buildAppManifest) - if err != nil { - stderr("build: Unable to read App Manifest: %v", err) - return 1 - } - var am schema.AppManifest - if err := am.UnmarshalJSON(b); err != nil { - stderr("build: Unable to load App Manifest: %v", err) - return 1 - } - aw = aci.NewAppWriter(am, tr) + b, err := ioutil.ReadFile(buildAppImageManifest) + if err != nil { + stderr("build: Unable to read App Manifest: %v", err) + return 1 } + var am schema.AppImageManifest + if err := am.UnmarshalJSON(b); err != nil { + stderr("build: Unable to load App Manifest: %v", err) + return 1 + } + aw := aci.NewAppWriter(am, tr) err = filepath.Walk(root, buildWalker(root, aw, buildRootfs)) if err != nil { diff --git a/app-container/actool/validate.go b/app-container/actool/validate.go index 72ea2fe..36133c9 100644 --- a/app-container/actool/validate.go +++ b/app-container/actool/validate.go @@ -120,15 +120,12 @@ func runValidate(args []string) (exit int) { return 1 } switch k.ACKind { - case "AppManifest": - m := schema.AppManifest{} + case "AppImageManifest": + m := schema.AppImageManifest{} err = m.UnmarshalJSON(b) case "ContainerRuntimeManifest": m := schema.ContainerRuntimeManifest{} err = m.UnmarshalJSON(b) - case "FilesetManifest": - m := schema.FilesetManifest{} - err = m.UnmarshalJSON(b) default: // Should not get here; schema.Kind unmarshal should fail panic("bad ACKind") diff --git a/app-container/examples/app.json b/app-container/examples/app.json index 8d84415..b47e094 100644 --- a/app-container/examples/app.json +++ b/app-container/examples/app.json @@ -1,83 +1,106 @@ { - "acVersion": "1.0.0", - "acKind": "AppManifest", + "acKind": "AppImageManifest", + "acVersion": "0.1.0", "name": "example.com/reduce-worker", - "version": "1.0.0", - "os": "linux", - "arch": "amd64", - "exec": [ - "/usr/bin/reduce-worker" - ], - "user": "100", - "group": "300", - "eventHandlers": [ + "labels": [ { - "name": "pre-start", - "exec": [ - "/usr/bin/data-downloader" - ] + "name": "version", + "val": "1.0.0" }, { - "name": "post-stop", - "exec": [ - "/usr/bin/deregister-worker" - ] - } - ], - "environment": { - "REDUCE_WORKER_DEBUG": "true" - }, - "mountPoints": [ - { - "name": "database", - "path": "/var/lib/db", - "readOnly": false - } - ], - "ports": [ - { - "name": "health", - "protocol": "tcp", - "port": 4000, - "socketActivated": true - } - ], - "isolators": [ - { - "name": "private-network", - "val": "true" + "name": "arch", + "val": "amd64" }, { - "name": "cpu/shares", - "val": "20" - }, - { - "name": "memory/limit", - "val": "1G" - }, - { - "name": "capabilities/bounding-set", - "val": "CAP_NET_BIND_SERVICECAP_SYS_ADMIN" + "name": "os", + "val": "linux" } ], - "files": { - "/usr/bin/ls": { - "hash": "sha256-908540d22dae9d8e6e3c6b13e21ddd12817406fd5c948eae4a744a6ccf94f96d", - "type": "file", - "mode": "0755", - "uid": 1000, - "gid": 500, - "xattrs": { - "foo": "bar" + "app": { + "exec": [ + "/usr/bin/reduce-worker" + ], + "user": "100", + "group": "300", + "eventHandlers": [ + { + "exec": [ + "/usr/bin/data-downloader" + ], + "name": "pre-start" }, - "mtime": "2010-07-10T17:14:51.5678Z", - "ctime": "2010-07-10T17:20:03.9212Z" - } + { + "exec": [ + "/usr/bin/deregister-worker" + ], + "name": "post-stop" + } + ], + "environment": { + "REDUCE_WORKER_DEBUG": "true" + }, + "isolators": [ + { + "name": "private-network", + "val": "true" + }, + { + "name": "cpu/shares", + "val": "20" + }, + { + "name": "memory/limit", + "val": "1G" + }, + { + "name": "capabilities/bounding-set", + "val": "CAP_NET_BIND_SERVICECAP_SYS_ADMIN" + } + ], + "mountPoints": [ + { + "name": "database", + "path": "/var/lib/db", + "readOnly": false + } + ], + "ports": [ + { + "name": "health", + "port": 4000, + "protocol": "tcp", + "socketActivated": true + } + ] }, + "dependencies": [ + { + "hash": "sha256-...", + "labels": [ + { + "name": "os", + "val": "linux" + }, + { + "name": "env", + "val": "canary" + } + ], + "name": "example.com/reduce-worker-base", + "root": "/" + } + ], + "pathWhitelist": [ + "/etc/ca/example.com/crt", + "/usr/bin/map-reduce-worker", + "/opt/libs/reduce-toolkit.so", + "/etc/reduce-worker.conf", + "/etc/systemd/system/" + ], "annotations": { + "authors": "Carly Container , Nat Network <[nat@example.com](mailto:nat@example.com)>", "created": "2014-10-27T19:32:27.67021798Z", - "authors": "Carly Container , Nat Network ", - "homepage": "https://example.com", - "documentation": "https://example.com/docs" + "documentation": "https://example.com/docs", + "homepage": "https://example.com" } } diff --git a/app-container/schema/app.go b/app-container/schema/app.go index 007ad41..1181cb8 100644 --- a/app-container/schema/app.go +++ b/app-container/schema/app.go @@ -11,60 +11,48 @@ const ( ACIExtension = ".aci" ) -type AppManifest struct { - ACVersion types.SemVer `json:"acVersion"` - ACKind types.ACKind `json:"acKind"` - Name types.ACName `json:"name"` - Version types.ACName `json:"version"` - OS types.ACName `json:"os"` - Arch types.ACName `json:"arch"` - Exec []string `json:"exec"` - EventHandlers []types.EventHandler `json:"eventHandlers"` - User string `json:"user"` - Group string `json:"group"` - Environment map[string]string `json:"environment"` - MountPoints []types.MountPoint `json:"mountPoints"` - Ports []types.Port `json:"ports"` - Isolators []types.Isolator `json:"isolators"` - Annotations types.Annotations `json:"annotations"` +type AppImageManifest struct { + ACKind types.ACKind `json:"acKind"` + ACVersion types.SemVer `json:"acVersion"` + Name types.ACName `json:"name"` + Labels types.Labels `json:"labels"` + App types.App `json:"app"` + Annotations types.Annotations `json:"annotations"` } // appManifest is a model to facilitate extra validation during the -// unmarshalling of the AppManifest -type appManifest AppManifest +// unmarshalling of the AppImageManifest +type appManifest AppImageManifest -func (am *AppManifest) UnmarshalJSON(data []byte) error { +func (am *AppImageManifest) UnmarshalJSON(data []byte) error { a := appManifest{} err := json.Unmarshal(data, &a) if err != nil { return err } - nam := AppManifest(a) + nam := AppImageManifest(a) if err := nam.assertValid(); err != nil { return err } - if nam.Environment == nil { - nam.Environment = make(map[string]string) - } *am = nam return nil } -func (am AppManifest) MarshalJSON() ([]byte, error) { +func (am AppImageManifest) MarshalJSON() ([]byte, error) { if err := am.assertValid(); err != nil { return nil, err } return json.Marshal(appManifest(am)) } -// assertValid performs extra assertions on an AppManifest to ensure that +// assertValid performs extra assertions on an AppImageManifest to ensure that // fields are set appropriately, etc. It is used exclusively when marshalling -// and unmarshalling an AppManifest. Most field-specific validation is +// and unmarshalling an AppImageManifest. Most field-specific validation is // performed through the individual types being marshalled; assertValid() // should only deal with higher-level validation. -func (am *AppManifest) assertValid() error { - if am.ACKind != "AppManifest" { - return types.ACKindError(`missing or bad ACKind (must be "AppManifest")`) +func (am *AppImageManifest) assertValid() error { + if am.ACKind != "AppImageManifest" { + return types.ACKindError(`missing or bad ACKind (must be "AppImageManifest")`) } if am.ACVersion.Empty() { return errors.New(`acVersion must be set`) @@ -72,18 +60,5 @@ func (am *AppManifest) assertValid() error { if am.Name.Empty() { return errors.New(`name must be set`) } - if am.Version.Empty() { - return errors.New(`version must be set`) - } - if am.OS.String() != "linux" { - return errors.New(`missing or bad OS (must be "linux")`) - } - if am.Arch.String() != "amd64" { - return errors.New(`missing or bad Arch (must be "amd64")`) - } - if len(am.Exec) < 1 { - return errors.New(`Exec cannot be empty`) - } - // TODO(jonboulle): assert hashes is not empty? return nil } diff --git a/app-container/schema/container.go b/app-container/schema/container.go index d7f898c..8f1d412 100644 --- a/app-container/schema/container.go +++ b/app-container/schema/container.go @@ -53,12 +53,12 @@ func (cm *ContainerRuntimeManifest) assertValid() error { return nil } -type AppList []App +type AppList []RuntimeApp // Get retrieves an app by the specified name from the AppList; if there is -// no such app, nil is returned. The returned *App MUST be considered +// no such app, nil is returned. The returned *RuntimeApp MUST be considered // read-only. -func (al AppList) Get(name types.ACName) *App { +func (al AppList) Get(name types.ACName) *RuntimeApp { for _, a := range al { if name.Equals(a.Name) { aa := a @@ -68,8 +68,8 @@ func (al AppList) Get(name types.ACName) *App { return nil } -// App describes an application referenced in a ContainerRuntimeManifest -type App struct { +// RuntimeApp describes an application referenced in a ContainerRuntimeManifest +type RuntimeApp struct { Name types.ACName `json:"name"` ImageID types.Hash `json:"imageID"` Isolators []types.Isolator `json:"isolators"` diff --git a/app-container/schema/doc.go b/app-container/schema/doc.go index a936aa5..292d05d 100644 --- a/app-container/schema/doc.go +++ b/app-container/schema/doc.go @@ -5,7 +5,7 @@ package schema Package schema provides definitions for the JSON schema of the different manifests in the App Container Standard. The manifests are canonically represented in their respective structs: - - `AppManifest` + - `AppImageManifest` - `ContainerRuntimeManifest` Validation is performed through serialization: if a blob of JSON data will diff --git a/app-container/schema/fileset.go b/app-container/schema/fileset.go deleted file mode 100644 index 01f876f..0000000 --- a/app-container/schema/fileset.go +++ /dev/null @@ -1,75 +0,0 @@ -package schema - -import ( - "encoding/json" - "errors" - - "github.com/coreos/rocket/app-container/schema/types" -) - -type FilesetManifest struct { - ACVersion types.SemVer `json:"acVersion"` - ACKind types.ACKind `json:"acKind"` - Name types.ACName `json:"name"` - OS string `json:"os"` - Arch string `json:"arch"` - Dependencies []Dependency `json:"dependencies"` - Files []string `json:"files"` -} - -type Dependency struct { - Name types.ACName `json:"name"` - Hash types.Hash `json:"hash"` - Root string `json:"root"` -} - -func NewFilesetManifest(name string) (*FilesetManifest, error) { - n, err := types.NewACName(name) - if err != nil { - return nil, err - } - fsm := FilesetManifest{ - ACVersion: AppContainerVersion, - ACKind: "FilesetManifest", - OS: "linux", - Arch: "amd64", - Name: *n, - } - return &fsm, nil -} - -type fileSetManifest FilesetManifest - -func (fsm *FilesetManifest) assertValid() error { - if fsm.ACKind != "FilesetManifest" { - return types.ACKindError(`missing or bad ACKind (must be "FilesetManifest")`) - } - if fsm.OS != "linux" { - return errors.New(`missing or bad OS (must be "linux")`) - } - if fsm.Arch != "amd64" { - return errors.New(`missing or bad Arch (must be "amd64")`) - } - return nil -} - -func (fsm *FilesetManifest) UnmarshalJSON(data []byte) error { - f := fileSetManifest{} - err := json.Unmarshal(data, &f) - if err != nil { - return err - } - nfsm := FilesetManifest(f) - if err := nfsm.assertValid(); err != nil { - return err - } - *fsm = nfsm - return nil -} - -func (fsm FilesetManifest) MarshalJSON() ([]byte, error) { - if err := fsm.assertValid(); err != nil { - return nil, err - } - return json.Marshal(fileSetManifest(fsm)) -} diff --git a/app-container/schema/types/ackind.go b/app-container/schema/types/ackind.go index d2bd493..6b21bf4 100644 --- a/app-container/schema/types/ackind.go +++ b/app-container/schema/types/ackind.go @@ -21,7 +21,7 @@ func (a ACKind) String() string { func (a ACKind) assertValid() error { s := a.String() switch s { - case "AppManifest", "ContainerRuntimeManifest", "FilesetManifest": + case "AppImageManifest", "ContainerRuntimeManifest": return nil case "": return ErrNoACKind diff --git a/app-container/schema/types/ackind_test.go b/app-container/schema/types/ackind_test.go index 783654b..f6b7430 100644 --- a/app-container/schema/types/ackind_test.go +++ b/app-container/schema/types/ackind_test.go @@ -30,7 +30,7 @@ func TestACKindMarshalBad(t *testing.T) { func TestACKindMarshalGood(t *testing.T) { for i, in := range []string{ - "AppManifest", + "AppImageManifest", "ContainerRuntimeManifest", } { a := ACKind(in) diff --git a/app-container/schema/types/annotations.go b/app-container/schema/types/annotations.go index 537dec4..877c0d4 100644 --- a/app-container/schema/types/annotations.go +++ b/app-container/schema/types/annotations.go @@ -10,15 +10,6 @@ type Annotations map[ACName]string type annotations Annotations -/* -struct { - Created Date `json:"created"` - Authors []string `json:"authors"` - Homepage URL `json:"homepage"` - Documentation URL `json:"documentation"` -} -*/ - func (a Annotations) assertValid() error { if c, ok := a["created"]; ok { if _, err := NewDate(c); err != nil { diff --git a/app-container/schema/types/app.go b/app-container/schema/types/app.go new file mode 100644 index 0000000..7fdad3a --- /dev/null +++ b/app-container/schema/types/app.go @@ -0,0 +1,52 @@ +package types + +import ( + "encoding/json" + "errors" +) + +type App struct { + Exec []string `json:"exec"` + EventHandlers []EventHandler `json:"eventHandlers"` + User string `json:"user"` + Group string `json:"group"` + Environment map[string]string `json:"environment"` + MountPoints []MountPoint `json:"mountPoints"` + Ports []Port `json:"ports"` + Isolators []Isolator `json:"isolators"` +} + +// app is a model to facilitate extra validation during the +// unmarshalling of the App +type app App + +func (a *App) UnmarshalJSON(data []byte) error { + ja := app{} + err := json.Unmarshal(data, &ja) + if err != nil { + return err + } + na := App(ja) + if err := na.assertValid(); err != nil { + return err + } + if na.Environment == nil { + na.Environment = make(map[string]string) + } + *a = na + return nil +} + +func (a App) MarshalJSON() ([]byte, error) { + if err := a.assertValid(); err != nil { + return nil, err + } + return json.Marshal(app(a)) +} + +func (a *App) assertValid() error { + if len(a.Exec) < 1 { + return errors.New(`Exec cannot be empty`) + } + return nil +} diff --git a/app-container/schema/types/errors.go b/app-container/schema/types/errors.go index 096711e..4cdd561 100644 --- a/app-container/schema/types/errors.go +++ b/app-container/schema/types/errors.go @@ -21,7 +21,7 @@ func (e ACNameError) Error() string { return string(e) } -// An AMStartedOnError is returned when the wrong StartedOn is set in an AppManifest +// An AMStartedOnError is returned when the wrong StartedOn is set in an AppImageManifest type AMStartedOnError string func (e AMStartedOnError) Error() string { diff --git a/app-container/schema/types/labels.go b/app-container/schema/types/labels.go new file mode 100644 index 0000000..1e154a4 --- /dev/null +++ b/app-container/schema/types/labels.go @@ -0,0 +1,59 @@ +package types + +import ( + "encoding/json" + "errors" +) + +// TODO(jonboulle): this is awkward since it's inconsistent with the way we do +// things elsewhere (i.e. using strict types instead of string types), but it's +// tricky because Labels needs to be able to catch arbitrary key-values. +// Clean this up somehow? +type Labels []Label + +type labels Labels + +type Label struct { + Name ACName `json:"name"` + Value string `json:"val"` +} + +func (l Labels) assertValid() error { + if os, ok := l.get("os"); ok && os != "linux" { + return errors.New(`bad os (must be "linux")`) + } + if arch, ok := l.get("arch"); ok && arch != "amd64" { + return errors.New(`bad arch (must be "amd64")`) + } + + return nil +} + +func (l Labels) MarshalJSON() ([]byte, error) { + if err := l.assertValid(); err != nil { + return nil, err + } + return json.Marshal(labels(l)) +} + +func (l *Labels) UnmarshalJSON(data []byte) error { + var jl labels + if err := json.Unmarshal(data, &jl); err != nil { + return err + } + nl := Labels(jl) + if err := l.assertValid(); err != nil { + return err + } + *l = nl + return nil +} + +func (l Labels) get(name string) (val string, ok bool) { + for _, lbl := range l { + if lbl.Name.String() == name { + return lbl.Value, true + } + } + return "", false +} diff --git a/metadatasvc/main.go b/metadatasvc/main.go index 153ab4a..8535fb3 100644 --- a/metadatasvc/main.go +++ b/metadatasvc/main.go @@ -21,7 +21,7 @@ import ( type metadata struct { manifest schema.ContainerRuntimeManifest - apps map[string]*schema.AppManifest + apps map[string]*schema.AppImageManifest } var ( @@ -79,7 +79,7 @@ func handleRegisterContainer(w http.ResponseWriter, r *http.Request) { } m := &metadata{ - apps: make(map[string]*schema.AppManifest), + apps: make(map[string]*schema.AppImageManifest), } if err := json.NewDecoder(r.Body).Decode(&m.manifest); err != nil { @@ -124,7 +124,7 @@ func handleRegisterApp(w http.ResponseWriter, r *http.Request) { an := mux.Vars(r)["app"] - app := &schema.AppManifest{} + app := &schema.AppImageManifest{} if err := json.NewDecoder(r.Body).Decode(&app); err != nil { w.WriteHeader(http.StatusBadRequest) fmt.Fprintf(w, "JSON-decoding failed: %v", err) @@ -150,7 +150,7 @@ func containerGet(h func(w http.ResponseWriter, r *http.Request, m *metadata)) f } } -func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppManifest)) func(http.ResponseWriter, *http.Request) { +func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppImageManifest)) func(http.ResponseWriter, *http.Request) { return containerGet(func(w http.ResponseWriter, r *http.Request, m *metadata) { appname := mux.Vars(r)["app"] @@ -209,7 +209,7 @@ func handleContainerUID(w http.ResponseWriter, r *http.Request, m *metadata) { w.Write([]byte(uid)) } -func mergeAppAnnotations(am *schema.AppManifest, cm *schema.ContainerRuntimeManifest) types.Annotations { +func mergeAppAnnotations(am *schema.AppImageManifest, cm *schema.ContainerRuntimeManifest) types.Annotations { merged := make(types.Annotations) for k, v := range am.Annotations { @@ -225,7 +225,7 @@ func mergeAppAnnotations(am *schema.AppManifest, cm *schema.ContainerRuntimeMani return merged } -func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppManifest) { +func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppImageManifest) { w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) @@ -234,7 +234,7 @@ func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, a } } -func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppManifest) { +func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppImageManifest) { k, err := types.NewACName(mux.Vars(r)["name"]) if err != nil { w.WriteHeader(http.StatusNotFound) @@ -256,7 +256,7 @@ func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, am w.Write([]byte(v)) } -func handleAppManifest(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppManifest) { +func handleAppImageManifest(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppImageManifest) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -265,7 +265,7 @@ func handleAppManifest(w http.ResponseWriter, r *http.Request, m *metadata, am * } } -func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppManifest) { +func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, am *schema.AppImageManifest) { w.Header().Add("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) a := m.manifest.Apps.Get(am.Name) @@ -401,7 +401,7 @@ func main() { mr.HandleFunc("/apps/{app:.*}/annotations/", logReq(appGet(handleAppAnnotations))) mr.HandleFunc("/apps/{app:.*}/annotations/{name}", logReq(appGet(handleAppAnnotation))) - mr.HandleFunc("/apps/{app:.*}/image/manifest", logReq(appGet(handleAppManifest))) + mr.HandleFunc("/apps/{app:.*}/image/manifest", logReq(appGet(handleAppImageManifest))) mr.HandleFunc("/apps/{app:.*}/image/id", logReq(appGet(handleAppID))) acRtr.HandleFunc("/container/hmac/sign", logReq(handleContainerSign)).Methods("POST") diff --git a/path/path.go b/path/path.go index c12e68a..4713e83 100644 --- a/path/path.go +++ b/path/path.go @@ -50,8 +50,8 @@ func RelAppRootfsPath(imageID types.Hash) string { return filepath.Join(RelAppImagePath(imageID), "rootfs") } -// AppManifestPath returns the path to the app's manifest file inside the expanded ACI. +// AppImageManifestPath returns the path to the app's manifest file inside the expanded ACI. // id should be the app image ID. -func AppManifestPath(root string, imageID types.Hash) string { +func AppImageManifestPath(root string, imageID types.Hash) string { return filepath.Join(AppImagePath(root, imageID), "app") } diff --git a/stage0/run.go b/stage0/run.go index 2003a1c..2c28c8d 100644 --- a/stage0/run.go +++ b/stage0/run.go @@ -133,10 +133,10 @@ func Setup(cfg Config) (string, error) { if cm.Apps.Get(am.Name) != nil { return "", fmt.Errorf("error: multiple apps with name %s", am.Name) } - a := schema.App{ + a := schema.RuntimeApp{ Name: am.Name, ImageID: img, - Isolators: am.Isolators, + Isolators: am.App.Isolators, Annotations: am.Annotations, } cm.Apps = append(cm.Apps, a) @@ -250,8 +250,8 @@ func unpackBuiltinRootfs(dir string) error { // setupImage attempts to load the image by the given hash from the store, // verifies that the image matches the given hash and extracts the image // into a directory in the given dir. -// It returns the AppManifest that the image contains -func setupImage(cfg Config, img types.Hash, dir string) (*schema.AppManifest, error) { +// It returns the AppImageManifest that the image contains +func setupImage(cfg Config, img types.Hash, dir string) (*schema.AppImageManifest, error) { log.Println("Loading image", img.String()) rs, err := cfg.Store.ReadStream(img.String()) @@ -289,7 +289,7 @@ func setupImage(cfg Config, img types.Hash, dir string) (*schema.AppManifest, er return nil, fmt.Errorf("error creating tmp directory: %v", err) } - mpath := rktpath.AppManifestPath(dir, img) + mpath := rktpath.AppImageManifestPath(dir, img) f, err := os.Open(mpath) if err != nil { return nil, fmt.Errorf("error opening app manifest: %v", err) @@ -298,7 +298,7 @@ func setupImage(cfg Config, img types.Hash, dir string) (*schema.AppManifest, er if err != nil { return nil, fmt.Errorf("error reading app manifest: %v", err) } - var am schema.AppManifest + var am schema.AppImageManifest if err := json.Unmarshal(b, &am); err != nil { return nil, fmt.Errorf("error unmarshaling app manifest: %v", err) } diff --git a/stage1/container.go b/stage1/container.go index 44d2472..7aa6f60 100644 --- a/stage1/container.go +++ b/stage1/container.go @@ -18,11 +18,11 @@ import ( rktpath "github.com/coreos/rocket/path" ) -// Container encapsulates a ContainerRuntimeManifest and AppManifests +// Container encapsulates a ContainerRuntimeManifest and AppImageManifests type Container struct { Root string // root directory where the container will be located Manifest *schema.ContainerRuntimeManifest - Apps map[string]*schema.AppManifest + Apps map[string]*schema.AppImageManifest } // LoadContainer loads a Container Runtime Manifest (as prepared by stage0) and @@ -30,7 +30,7 @@ type Container struct { func LoadContainer(root string) (*Container, error) { c := &Container{ Root: root, - Apps: make(map[string]*schema.AppManifest), + Apps: make(map[string]*schema.AppImageManifest), } buf, err := ioutil.ReadFile(rktpath.ContainerManifestPath(c.Root)) @@ -45,13 +45,13 @@ func LoadContainer(root string) (*Container, error) { c.Manifest = cm for _, app := range c.Manifest.Apps { - ampath := rktpath.AppManifestPath(c.Root, app.ImageID) + ampath := rktpath.AppImageManifestPath(c.Root, app.ImageID) buf, err := ioutil.ReadFile(ampath) if err != nil { return nil, fmt.Errorf("failed reading app manifest %q: %v", ampath, err) } - am := &schema.AppManifest{} + am := &schema.AppImageManifest{} if err = json.Unmarshal(buf, am); err != nil { return nil, fmt.Errorf("failed unmarshalling app manifest %q: %v", ampath, err) } @@ -66,9 +66,10 @@ func LoadContainer(root string) (*Container, error) { } // appToSystemd transforms the provided app manifest into systemd units -func (c *Container) appToSystemd(am *schema.AppManifest, id types.Hash) error { +func (c *Container) appToSystemd(am *schema.AppImageManifest, id types.Hash) error { name := am.Name.String() - execStart := strings.Join(am.Exec, " ") + app := am.App + execStart := strings.Join(app.Exec, " ") opts := []*unit.UnitOption{ &unit.UnitOption{"Unit", "Description", name}, &unit.UnitOption{"Unit", "DefaultDependencies", "false"}, @@ -78,11 +79,11 @@ func (c *Container) appToSystemd(am *schema.AppManifest, id types.Hash) error { &unit.UnitOption{"Service", "Restart", "no"}, &unit.UnitOption{"Service", "RootDirectory", rktpath.RelAppRootfsPath(id)}, &unit.UnitOption{"Service", "ExecStart", execStart}, - &unit.UnitOption{"Service", "User", am.User}, - &unit.UnitOption{"Service", "Group", am.Group}, + &unit.UnitOption{"Service", "User", app.User}, + &unit.UnitOption{"Service", "Group", app.Group}, } - for _, eh := range am.EventHandlers { + for _, eh := range app.EventHandlers { var typ string switch eh.Name { case "pre-start": @@ -96,7 +97,7 @@ func (c *Container) appToSystemd(am *schema.AppManifest, id types.Hash) error { opts = append(opts, &unit.UnitOption{"Service", typ, exec}) } - env := am.Environment + env := app.Environment env["AC_APP_NAME"] = name for ek, ev := range env { ee := fmt.Sprintf(`"%s=%s"`, ek, ev) @@ -104,7 +105,7 @@ func (c *Container) appToSystemd(am *schema.AppManifest, id types.Hash) error { } saPorts := []types.Port{} - for _, p := range am.Ports { + for _, p := range app.Ports { if p.SocketActivated { saPorts = append(saPorts, p) } @@ -184,7 +185,7 @@ func (c *Container) ContainerToSystemd() error { // appToNspawnArgs transforms the given app manifest, with the given associated // app image id, into a subset of applicable systemd-nspawn argument -func (c *Container) appToNspawnArgs(am *schema.AppManifest, id types.Hash) ([]string, error) { +func (c *Container) appToNspawnArgs(am *schema.AppImageManifest, id types.Hash) ([]string, error) { args := []string{} name := am.Name.String() @@ -195,7 +196,7 @@ func (c *Container) appToNspawnArgs(am *schema.AppManifest, id types.Hash) ([]st } } - for _, mp := range am.MountPoints { + for _, mp := range am.App.MountPoints { key := mp.Name vol, ok := vols[key] if !ok {