cri-o update fixes

ClearLinux now has cri-o version 1.14.1 which allows multiple plugin
locations and also creates /opt/cni/bin by default. We no longer need
the hacks for them

Fixes: #82
Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

.gitignore

@@ -0,0 +1,3 @@
+.vagrant
+OVMF.fd
+

clr-k8s-examples/1-core-metrics/aggregated-metrics-reader.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]

clr-k8s-examples/1-core-metrics/auth-delegator.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/auth-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/metrics-apiservice.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apiregistration.k8s.io/v1beta1
 kind: APIService
 metadata:

clr-k8s-examples/1-core-metrics/metrics-server-deployment.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -22,10 +23,19 @@ spec:
         k8s-app: metrics-server
     spec:
       serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
       containers:
       - name: metrics-server
-        image: gcr.io/google_containers/metrics-server-amd64:v0.2.0
+        image: k8s.gcr.io/metrics-server-amd64:v0.3.1
         imagePullPolicy: Always
-        command:
-        - /metrics-server
-        - --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true
+        args:
+        - --logtostderr
+        - --kubelet-insecure-tls
+        - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+

clr-k8s-examples/1-core-metrics/metrics-server-service.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:

clr-k8s-examples/1-core-metrics/resource-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -9,15 +10,6 @@ rules:
   - pods
   - nodes
   - nodes/stats
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - "extensions"
-  resources:
-  - deployments
   verbs:
   - get
   - list

clr-k8s-examples/1-core-metrics/versions

@@ -1 +1 @@
-f90c6705d2381ea2db1a6343da6c400bd2ef4cb2
+92d8412788e27ee669d38f21f20bad5342211884

clr-k8s-examples/7-rook/000-operator.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph-system
-  labels:
-    kata: "false"
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition

clr-k8s-examples/7-rook/001-cluster.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph
-  labels:
-    kata: "false"
 ---
 apiVersion: v1
 kind: ServiceAccount

clr-k8s-examples/8-kata/deploy/kata-cleanup.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubelet-kata-cleanup
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kubelet-kata-cleanup
+  template:
+    metadata:
+        labels:
+          name: kubelet-kata-cleanup
+    spec:
+      serviceAccountName: kata-label-node
+      nodeSelector:
+          katacontainers.io/kata-runtime: cleanup
+      containers:
+      - name: kube-kata-cleanup
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      volumes:
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-deploy.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kata-deploy
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kata-deploy
+  template:
+    metadata:
+        labels:
+          name: kata-deploy
+    spec:
+      serviceAccountName: kata-label-node
+      containers:
+      - name: kube-kata
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        lifecycle:
+          preStop:
+            exec:
+              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: crio-conf
+          mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
+        - name: kata-artifacts
+          mountPath: /opt/kata/
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+        - name: local-bin
+          mountPath: /usr/local/bin/
+      volumes:
+        - name: crio-conf
+          hostPath:
+            path: /etc/crio/
+        - name: containerd-conf
+          hostPath:
+            path: /etc/containerd/
+        - name: kata-artifacts
+          hostPath:
+            path: /opt/kata/
+            type: DirectoryOrCreate
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin/
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-rbac.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kata-label-node
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: node-labeler
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kata-label-node-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-labeler
+subjects:
+- kind: ServiceAccount
+  name: kata-label-node
+  namespace: kube-system
+

clr-k8s-examples/8-kata/fire-runtimeClass.yaml

@@ -1,6 +0,0 @@
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
-metadata:
-    name: fire
-spec:
-    runtimeHandler: fire

clr-k8s-examples/8-kata/kata-fc-runtimeClass.yaml

@@ -0,0 +1,5 @@
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-fc
+handler: kata-fc

clr-k8s-examples/8-kata/kata-qemu-runtimeClass.yaml

@@ -0,0 +1,12 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata
+handler: kata
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-qemu
+handler: kata-qemu

clr-k8s-examples/8-kata/kata-runtimeClass.yaml

@@ -1,6 +0,0 @@
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
-metadata:
-    name: kata
-spec:
-    runtimeHandler: kata

clr-k8s-examples/8-kata/runtimeclass_crd.yaml

@@ -1,26 +0,0 @@
-kind: CustomResourceDefinition
-apiVersion: apiextensions.k8s.io/v1beta1
-metadata:
-  name: runtimeclasses.node.k8s.io
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  group: node.k8s.io
-  version: v1alpha1
-  versions:
-    - name: v1alpha1
-      served: true
-      storage: true
-  names:
-    plural:   runtimeclasses
-    singular: runtimeclass
-    kind:     RuntimeClass
-  scope: Cluster
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            runtimeHandler:
-              type: string
-              pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$'

clr-k8s-examples/9-multi-network/Dockerfile

@@ -0,0 +1,33 @@
+# Build multus plugin
+FROM golang:1.10 AS multus
+RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
+WORKDIR /go/src/github.com/intel/multus-cni
+RUN ./build
+
+# Build sriov plugin
+FROM golang:1.10 AS sriov-cni
+RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
+WORKDIR /go/src/github.com/intel-corp/sriov-cni
+RUN ./build
+
+# Build sriov device plugin
+FROM golang:1.10 AS sriov-dp
+RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
+WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
+RUN make
+
+# Build vfioveth plugin
+FROM busybox as vfioveth
+RUN wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+COPY cni/vfioveth /bin/vfioveth
+RUN chmod +x /bin/vfioveth /bin/jq
+
+# Final image
+FROM centos/systemd
+WORKDIR /tmp/cni/bin
+COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
+COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=vfioveth /bin/vfioveth .
+COPY --from=vfioveth /bin/jq .
+WORKDIR /usr/bin
+COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -0,0 +1,76 @@
+# Multi-Network
+
+## Daemonset
+
+We launch a `Daemonset` with an `initContainer` which sets up the CNI
+directories on the host with the necessary binaries and configuration files.
+
+> NOTE: SR-IOV devices are not necessary to test multi-network capability
+
+### Customization
+
+The device plugin will register the SR-IOV enabled devices on the host, specified as
+`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+file is provided, which enables SR-IOV for the above `rootDevices`
+
+> NOTE: This assumes homogenous nodes in the cluster
+
+### Pre-req (SR-IOV only)
+
+One each SR-IOV node make sure `VT-d` is enabled in the BIOS and `intel_iommu=on` on kernel commandline.
+Setup systemd to bring up VFs on designated interfaces bound to network driver or `vfio-pci`
+
+```bash
+# Make sure vfio-pci is loaded on boot
+echo 'vfio-pci' | sudo tee /etc/modules-load.d/sriov.conf
+sudo systemctl restart systemd-modules-load.service
+
+sudo cp systemd/sriov.sh /usr/bin/sriov.sh
+sudo cp systemd/sriov.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now sriov.service
+```
+
+### Install
+
+To install and configure `multus-cni` on all nodes, along with
+`sriov-cni`, `vfioveth-cni` and `sriov-network-device-plugin`
+
+```bash
+kubectl apply -f .
+kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov_*"
+```
+
+## Tests
+
+### Default only
+
+To test if default connectivity is working
+
+```bash
+kubectl apply -f test/pod.yaml
+kubectl exec test -- ip a                        # should see one interface only
+```
+
+### Bridge
+
+To test multus with second interface created by `bridge` plugin
+
+```bash
+kubectl apply -f test/bridge
+kubectl exec test-bridge -- ip a                 # should see two interfaces
+ip a show mynet                                  # bridge created on host if it doesnt exist already
+```
+
+### SR-IOV
+
+To test multus with second interface created by `sriov` plugin
+
+```bash
+kubectl apply -f test/sriov
+
+kubectl exec test-sriov -- ip a                  # second interface is a VF
+
+kubectl exec test-sriov-dpdk -- ip a             # veth pair with details of VF
+kubectl exec test-sriov-dpdk -- ls -l /dev/vfio
+```

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -0,0 +1,67 @@
+#!/bin/bash -x
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+exec 3>&1
+exec &>>/var/log/$(basename $0).log
+
+PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH"
+CNI_CONF=$(cat /dev/stdin)
+
+get_peer_name() {
+	echo "$1-vdev"
+}
+
+get_mac_with_vfpci() {
+	local pf=$(readlink /sys/devices/pci*/*/$1/physfn | awk '{print substr($1,4)}')
+	local pfName=$(ls /sys/devices/pci*/*/$pf/net/ | head -1)
+	local idx=$(ls -l /sys/devices/pci*/*/$pf | awk -v vf=$1 'substr($11,4)==vf {print substr($9,7)}')
+	local mac=$(ip link show dev $pfName | awk -v idx="$idx" '$1=="vf" && $2==idx {print substr($4,1,17)}')
+	echo $mac
+}
+
+ipam() {
+	local plugin=$(echo $CNI_CONF | jq -r '.ipam.type')
+	local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.')
+	echo $res
+}
+
+add_pair_ns() {
+	vfpci=$(echo $CNI_CONF | jq -r '.deviceID')
+	mac=$(get_mac_with_vfpci $vfpci)
+	peer=$(get_peer_name $CNI_IFNAME)
+	ip=$1
+
+	mkdir -p /var/run/netns/
+	ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID
+
+	ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer
+	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up
+	ip netns exec $CNI_CONTAINERID ip link set $peer up
+	ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
+}
+
+delete_pair_ns() {
+	ip netns exec $CNI_CONTAINERID ip link del $CNI_IFNAME
+}
+
+case $CNI_COMMAND in
+ADD)
+	res=$(ipam)
+	ip=$(echo $res | jq -r '.ip4.ip')
+	add_pair_ns $ip
+	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	;;
+DEL)
+	set +o errexit
+	ipam
+	delete_pair_ns
+	set -o errexit
+	;;
+*)
+	echo "CNI_COMMAND=[ADD|DEL] only supported"
+	exit 1
+	;;
+esac

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -0,0 +1,212 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: k8s.cni.cncf.io
+  # version name to use for REST API: /apis/<group>/<version>
+  version: v1
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: network-attachment-definitions
+    # singular name to be used as an alias on the CLI and for display
+    singular: network-attachment-definition
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: NetworkAttachmentDefinition
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - net-attach-def
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: multus-sa-secret
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: multus-sa
+type: kubernetes.io/service-account-token
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-pod-networks-lister
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["update"]
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs: ["get"]
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-scripts
+  namespace: kube-system
+data:
+  install-certs.sh: |
+    # Copied from Calico
+    # https://github.com/projectcalico/cni-plugin/blob/master/k8s-install/scripts/install-cni.sh
+    touch /host/etc/cni/net.d/multus-kubeconfig
+    chmod 600 /host/etc/cni/net.d/multus-kubeconfig
+    SERVICE_ACCOUNT_PATH=/var/run/secrets/multus/serviceaccount
+    KUBE_CA_FILE=$SERVICE_ACCOUNT_PATH/ca.crt
+    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+    SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+    cat > /host/etc/cni/net.d/multus-kubeconfig <<EOF
+    # Kubeconfig file for Multus CNI plugin.
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - name: local
+      cluster:
+        server: https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+        $TLS_CFG
+    users:
+    - name: multus
+      user:
+        token: "${SERVICEACCOUNT_TOKEN}"
+    contexts:
+    - name: multus-context
+      context:
+        cluster: local
+        user: multus
+    current-context: multus-context
+    EOF
+  install-multus-conf.sh: |
+    # copied from https://github.com/intel/multus-cni/blob/master/images/entrypoint.sh
+    rm -f /host/etc/cni/net.d/00-multus.conf
+    MASTER_PLUGIN="$(ls /host/etc/cni/net.d | grep -E '\.conf(list)?$' | head -1)"
+    MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
+    cat > /host/etc/cni/net.d/00-multus.conf <<EOF
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "logFile": "/var/log/multus.log",
+      "logLevel": "debug",
+      "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
+      "delegates": [
+                    $MASTER_PLUGIN_JSON
+       ]
+    }
+    EOF
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-networks-lister
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sriov-device-plugin
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: sriov-device-plugin
+  template:
+    metadata:
+      labels:
+        name: sriov-device-plugin
+    spec:
+      initContainers:
+      - name: multus
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "bash", "-c" ]
+        args:
+        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+          /tmp/multus/install-multus-conf.sh;
+          /tmp/multus/install-certs.sh;
+          echo "Restarting crio kubelet";
+          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart kubelet;
+        volumeMounts:
+        - name: usr-bin
+          mountPath: /host/usr/bin
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: multus-sa
+          mountPath: /var/run/secrets/multus/serviceaccount
+        - name: multus-scripts
+          mountPath: /tmp/multus
+        - name: cni-conf
+          mountPath: /host/etc/cni/net.d
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      containers:
+      - name: sriovdp
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "sh", "-c" ]
+        args:
+        - /usr/bin/sriovdp --logtostderr -v 10;
+          sleep infinity;
+        volumeMounts:
+        - name: net
+          mountPath: /sys/class/net
+          readOnly: true
+        - name: dp-sock
+          mountPath: /var/lib/kubelet/device-plugins/
+          readOnly: false
+        - mountPath: /etc/pcidp
+          name: sriov-config
+          readOnly: false
+      volumes:
+      - name: sriov-config
+        configMap:
+          name: sriov-config
+      - name: usr-bin
+        hostPath:
+          path: /usr/bin
+      - name: cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: multus-sa
+        secret:
+          secretName: multus-sa-secret
+      - name: multus-scripts
+        configMap:
+          defaultMode: 511
+          name: multus-scripts
+      - name: cni-conf
+        hostPath:
+          path: /etc/cni/net.d
+      - name: dbus
+        hostPath:
+          path: /var/run/dbus
+      - name: systemd
+        hostPath:
+          path: /run/systemd
+      - name: net
+        hostPath:
+          path: /sys/class/net
+      - name: dp-sock
+        hostPath:
+          path: /var/lib/kubelet/device-plugins/
+      hostNetwork: true
+      hostPID: true
+

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -0,0 +1,25 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: sriov-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+      "resourceList":
+      [
+        {
+          "resourceName": "sriov_netdevice",
+          "rootDevices": ["07:00.0"],
+          "sriovMode": true,
+          "deviceType": "netdevice"
+        },
+        {
+          "resourceName": "sriov_vfio",
+          "rootDevices": ["07:00.1"],
+          "sriovMode": true,
+          "deviceType": "vfio"
+        }
+      ]
+    }

clr-k8s-examples/9-multi-network/systemd/sriov.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Create VFs on ens785f0 (netdev) ens785f1 (vfio) interfaces
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/sriov.sh ens785f0
+ExecStart=/usr/bin/sriov.sh -b ens785f1
+
+[Install]
+WantedBy=default.target

clr-k8s-examples/9-multi-network/systemd/sriov.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+OPTIND=1
+bind="false"
+
+while getopts ":b" opt; do
+	case ${opt} in
+	b)
+		bind="true"
+		;;
+	\?)
+		echo "Usage: sriov.sh [-b] ens785f0 ens785f1 ..."
+		echo "-b	Bind to vfio-pci"
+		exit
+		;;
+	esac
+done
+shift $((OPTIND - 1))
+
+setup_pf() {
+	local pf=$1
+	echo "Resetting PF $pf"
+	echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
+	echo "Enabling $NUM_VFS VFs for $pf"
+	echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs
+	ip link set $pf up
+	sleep 1
+}
+
+setup_vfs() {
+	local pf=$1
+	local pfpci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_numvfs)
+	for ((idx = 0; idx < NUM_VFS; idx++)); do
+		ip link set dev $pf vf $idx state enable
+		if [ $bind != "true" ]; then continue; fi
+
+		local vfn="virtfn$idx"
+		local vfpci=$(ls -l /sys/devices/pci*/*/$pfpci | awk -v vfn=$vfn 'vfn==$9 {print substr($11,4)}')
+		# Capture and set MAC of the VF before unbinding from linux, for later use in CNI
+		local mac=$(cat /sys/bus/pci*/*/$vfpci/net/*/address)
+		ip link set dev $pf vf $idx mac $mac
+		# Bind VF to vfio-pci
+		echo $vfpci >/sys/bus/pci*/*/$vfpci/driver/unbind
+		echo "vfio-pci" >/sys/devices/pci*/*/$vfpci/driver_override
+		echo $vfpci >/sys/bus/pci/drivers/vfio-pci/bind
+	done
+}
+
+for pf in "$@"; do
+	setup_pf $pf
+	setup_vfs $pf
+done

clr-k8s-examples/9-multi-network/test/bridge/0-bridge-net.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: mynet
+spec:
+  config: '{
+        "name": "mynet",
+        "type": "bridge",
+        "bridge": "mynet",
+        "ipam": {
+            "type": "host-local",
+            "subnet": "198.18.0.0/24"
+        }
+}'
+

clr-k8s-examples/9-multi-network/test/bridge/1-pod-bridge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-bridge
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "mynet", "interface": "mynet" }
+    ]'
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/pod.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_netdevice
+spec:
+  config: '{
+    "type": "sriov",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-dpdk
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_vfio
+spec:
+  config: '{
+    "type": "vfioveth",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+

clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_netdevice: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov-dpdk
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+

clr-k8s-examples/README.md

@@ -1,13 +1,24 @@
 # How to setup the cluster
 
+## Prerequisite
+This setup currently will work with k8s 1.14 & above. Any version of k8s before that might work, but is not guaranteed.
+
 ## Sample multi-node vagrant setup
 
-To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using libvirt, but you can use any hypervisor that you are familiar with.
+To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
 
 * Install vagrant on the distro you are using. Steps can be found at [Vagrant docs](https://www.vagrantup.com/intro/getting-started/install.html#installing-vagrant)
 * `vagrant up --provider=libvirt`
 
 Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
+Customize the setup using environment variables. E.g., `NODES=1 MEMORY=8192 CPUS=8 vagrant up --provider=libvirt`
+
+To login to the master node and change to this directory
+
+```bash
+vagrant ssh clr-01
+cd clr-k8s-examples
+```
 
 ## Setup the nodes in the cluster
 
@@ -20,18 +31,17 @@ This script ensures the following
 * Customizes the system to ensure correct defaults are setup (IP Forwarding, Swap off,...)
 * Ensures all the dependencies are loaded on boot (kernel modules)
 
-> NOTE: This step is done automatically if using vagrant. 
+> NOTE: This step is done automatically if using vagrant.
 
 ### Enabling experimental firecracker support
 
-> EXPERIMENTAL: Optionally run [`setup_firecracker.sh`](setup_firecracker.sh) to be
+> EXPERIMENTAL: Optionally run [`setup_kata_firecracker.sh`](setup_kata_firecracker.sh) to be
 able to use firecracker VMM with Kata.
 
 The firecracker setup switches the setup to use a sparse file backed loop device for
 devicemapper storage. This should not be used for production.
 
-> NOTE: This step is done automatically if using vagrant. 
-
+> NOTE: This step is done automatically if using vagrant.
 
 ## Bring up the master
 
@@ -40,13 +50,12 @@ master and also uses kubelet config via [`kubeadm.yaml`](kubeadm.yaml)
 to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
-There are two flavors of install -
-- `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
-- `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
+There are different flavors to install, run `./create_stack.sh help` to get
+more information.
 
 ```bash
-# default is 'all'
-./create_stack.sh [minimal|all]
+# default shows help
+./create_stack.sh <subcommand>
 ```
 
 ## Join Workers to the cluster
@@ -73,54 +82,23 @@ runtime class set to "kata" will launch the POD/Deployment with Kata.
 
 An example is
 
-`kubectl apply -f tests/test-deploy-kata.yaml`
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-qemu.yaml`
 
 ### Running Kata Workloads with Firecracker
 
-> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "fire" will launch the POD/Deployment
+> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "kata-fc" will launch the POD/Deployment
 with firecracker as the isolation mechanism for Kata.
 
 An example is
 
-`kubectl apply -f tests/test-deploy-fire.yaml`
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-fc.yaml`
 
-## Making Kata the default runtime
+## Making Kata the default runtime using admission controller
 
-Today in `crio.conf` runc is the default runtime when a user does not specify
-`runtimeClass` in the pod spec. If you want to run a cluster where kata is used
-by default, except for workloads we know for sure will not work with kata, use
-the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
-and sample admission controller we created by running -
-
-`kubectl apply -f admit-kata/`
-
-The [admission webhook](admit-kata/webhook-registration.yaml)
-is setup to exclude certian namespaces from being run with Kata using filters on namespace labels.
-
-```yaml
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-```
-
-The rook operators for example are marked as such
-
-```yaml
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: rook-ceph-system
-  labels:
-    kata: "false"
-```
-
-Pods not explicitly excluded by the namespace filter are dynamically tagged to
-run with Kata with some [exceptions](https://github.com/mcastelino/kubewebhook/blob/topic/hack-kata/examples/pod-annotate/main.go#L25) -
-
-* `hostNetwork: true`
-* `rook-ceph` and `rook-ceph-system` namespaces (buggy)
-
-Other pod properties will be added as exceptions in future.
+If you want to run a cluster where kata is used
+by default, except for workloads we know for sure will not work with kata, using
+[admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
+and sample admission controller, follow `admit-kata` [README.md](admit-kata/README.md)
 
 ## Accessing control plane services
 

clr-k8s-examples/Vagrantfile

@@ -4,21 +4,22 @@
 require 'fileutils'
 require 'ipaddr'
 require 'securerandom'
-DISK_UUID = SecureRandom.urlsafe_base64(9)
 
 $num_instances = (ENV['NODES'] || 3).to_i
 $cpus = (ENV['CPUS'] || 2).to_i
 $memory = (ENV['MEMORY'] || 4096).to_i
 $disks = 2
+# Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
+$disk_prefix = File.basename(File.dirname(__FILE__), "/")
 $disk_size = "10G"
-$box = "gmmaha/clearlinux"
-$loader = File.join(File.dirname(__FILE__), "OVMF.fd")
+$box = "AntonioMeireles/ClearLinux"
+File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-base_ip = IPAddr.new("192.52.100.10")
-hosts = {}
-proxy_ip_list = ""
-#DISK_UUID = Time.now.utc.to_i
-driveletters = ('a'..'z').to_a
+$base_ip = IPAddr.new("192.52.100.10")
+$hosts = {}
+$proxy_ip_list = ""
+$driveletters = ('a'..'z').to_a
+$setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
 
 if not File.exists?($loader)
   system('curl -O https://download.clearlinux.org/image/OVMF.fd')
@@ -43,39 +44,47 @@ Vagrant.configure("2") do |config|
   # boxes at https://vagrantcloud.com/search.
   config.vm.box = $box
 
+  # Mount the current dir at home folder instead of default
+  config.vm.synced_folder './', '/vagrant', disabled: true
+  config.vm.synced_folder './', '/home/clear/' + File.basename(Dir.getwd), type: 'rsync',
+    rsync__args: ["--verbose", "--archive", "--delete", "-zz", "--copy-links"]
   #Setup proxies for all machines
   (1..$num_instances).each do |i|
-    base_ip = base_ip.succ
-    hosts["clr-%02d" % i] = base_ip.to_s
+    $base_ip = $base_ip.succ
+    $hosts["clr-%02d" % i] = $base_ip.to_s
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     proxy_ip_list = ("#{proxy_ip_list},#{vm_name},#{ip}")
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     config.vm.define vm_name do |c|
       c.vm.hostname = vm_name
       c.vm.network :private_network, ip: ip, autostart: true
       c.vm.provider :libvirt do |lv|
-        lv.loader = $loader
         lv.cpu_mode = "host-passthrough"
         lv.nested = true
+        lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
         (1..$disks).each do |d|
-          lv.storage :file, :device => "hd#{driveletters[d]}", :path => "disk-#{vm_name}-#{d}-#{DISK_UUID}.disk", :size => $disk_size, :type => "raw"
+          lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end
       end
-      if Vagrant.has_plugin?("vagrant-proxyconf")
-        c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
-        c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
-        c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+      if ENV['http_proxy'] || ENV['HTTP_PROXY']
+        if Vagrant.has_plugin?("vagrant-proxyconf")
+          c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
+          c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
+          c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+        end
       end
-      # Bad hack for the vagrant libvirt boxes. WIll be removed once they are fixed.
-      c.vm.provision "shell", privileged: false, inline: "sudo usermod --password vagrant root"
       c.vm.provision "shell", privileged: false, path: "setup_system.sh"
-      c.vm.provision "shell", privileged: false, path: "setup_firecracker.sh"
+      if $setup_fc
+        c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+      end
+      # Include shells bundle to get bash completion and add kubectl's commands to vagrant's shell
+      c.vm.provision "shell", privileged: false, inline: 'sudo -E swupd bundle-add shells; echo "source <(kubectl completion bash)" >> $HOME/.bashrc'
     end
   end
 end

clr-k8s-examples/admit-kata/README.md

@@ -0,0 +1,39 @@
+# Kata Admission controller webhook
+
+Implement a simple admission controller webhook to annotate pods with the 
+Kata runtime class.
+
+## How to build the admission controller
+
+First build the admission controller image and the associated
+Kubernetes yaml files required to instantiate the admission
+controller.
+
+```bash
+$ docker build -t katadocker/kata-webhook-example:latest .
+$ ./create_certs.sh
+```
+
+> **Note:**
+> Image needs to be published for the webhook needs to work. Alternately
+> on a single machine cluster change the `imagePullPolicy` to use the locally
+> built image.
+
+## Making Kata the default runtime using an admission controller
+
+Today in `crio.conf` `runc` is the default runtime when a user does not specify
+`runtimeClass` in the pod spec. If you want to run a cluster where Kata is used
+by default, except for workloads we know for sure will not work with Kata, use
+the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
+and sample admission controller we created by running
+
+```bash
+$ kubectl apply -f deploy/
+```
+
+The webhook mutates pods to use the kata runtime class for all pods except
+those with 
+
+* `hostNetwork: true` 
+* namespace: `rook-ceph` and `rook-ceph-system`
+

clr-k8s-examples/admit-kata/create-certs.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+WEBHOOK_NS=${1:-"default"}
+WEBHOOK_NAME=${2:-"pod-annotate"}
+WEBHOOK_SVC="${WEBHOOK_NAME}-webhook"
+
+# Create certs for our webhook
+openssl genrsa -out webhookCA.key 2048
+openssl req -new -key ./webhookCA.key -subj "/CN=${WEBHOOK_SVC}.${WEBHOOK_NS}.svc" -out ./webhookCA.csr 
+openssl x509 -req -days 365 -in webhookCA.csr -signkey webhookCA.key -out webhook.crt
+
+# Create certs secrets for k8s
+kubectl create secret generic \
+    ${WEBHOOK_SVC}-certs \
+    --from-file=key.pem=./webhookCA.key \
+    --from-file=cert.pem=./webhook.crt \
+    --dry-run -o yaml > ./deploy/webhook-certs.yaml
+
+# Set the CABundle on the webhook registration
+CA_BUNDLE=$(cat ./webhook.crt | base64 -w0)
+sed "s/CA_BUNDLE/${CA_BUNDLE}/" ./deploy/webhook-registration.yaml.tpl > ./deploy/webhook-registration.yaml
+
+# Clean
+rm ./webhookCA* && rm ./webhook.crt

clr-k8s-examples/admit-kata/deploy/webhook-registration.yaml.tpl

@@ -0,0 +1,24 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pod-annotate-webhook
+  labels:
+    app: pod-annotate-webhook
+    kind: mutator
+webhooks:
+  - name: pod-annotate-webhook.kata.xyz
+    clientConfig:
+      service:
+        name: pod-annotate-webhook
+        namespace: default
+        path: "/mutate"
+      caBundle: CA_BUNDLE
+    rules:
+      - operations: [ "CREATE" ]
+        apiGroups: [""]
+        apiVersions: ["v1"]
+        resources: ["pods"]

clr-k8s-examples/admit-kata/deploy/webhook.yaml

@@ -1,3 +1,7 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -13,11 +17,12 @@ spec:
     spec:
       containers:
         - name: pod-annotate-webhook
-          image: mcastelino/kubewebhook-pod-annotate-example:1.0
+          image: katadocker/kata-webhook-example:latest
           imagePullPolicy: Always
           args:
             - -tls-cert-file=/etc/webhook/certs/cert.pem
             - -tls-key-file=/etc/webhook/certs/key.pem
+            - -exclude-namespaces=rook-ceph-system,rook-ceph
           volumeMounts:
             - name: webhook-certs
               mountPath: /etc/webhook/certs

clr-k8s-examples/admit-kata/versions

@@ -0,0 +1,2 @@
+https://github.com/kata-containers/tests/tree/master/kata-webhook
+Commit: 5ad2cec

clr-k8s-examples/admit-kata/webhook-certs.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-data:
-  cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd2hVVVBKbnZ2Y2c0SUhueHdrSjJGYkJXc3NWb0I4dWp4S3B4dlZvdnRERFdLTlVOCmNoejNtQXRxQm9pcDBVSWpyWm1SMGd0b3FhSThSaXhxY1A5b1V5cDFTUk1UK2E3VXk2OG9LN3dRdjJ5eUI2eDIKMTFTaWNraEdpbzFNSlA3TEJFekZSVjlJb21yY2ZKdm4vdUk0Vk1lZnJjWXBqdlhwTjh0VHU3VkFnL1JMODZXYwpJeDhURWVtTko2QStRZ1Y3RUJLTVlQUTAwM0Z6NUdkR1ovOXNoUTVGQ0hCdUFYd2lRc0x6elArTlJFRmhtczJnCkdWT3FVeUFQR1FkWVlVRlZUd0tLbGlvRTVtcldTRTBMZnpDS3M2MUdwZ0pwWWNpN0RVdWRQelM4SHJUU2k1WXUKN002NlAwUThwaGZpN2NvcmJFcHQrci9DZTVHaTRXVlVLd3dSZlFJREFRQUJBb0lCQUNsY3NxNWpwQ3RQRTE4ZwpmZnlIaTREOXpzeFVzK0lOYlQ2SmtLbnBJWWVHYk8zUTBnZkMwcVdOc1MvcFNqKzFsOEwwbXBZb0prc1lyejNKClUzbjNoTFZNdENnNXI2VWp0R2dnVVRCWTZUelNubkp5Ti9XV0xQU3NJSlNBbm55MzdHNWpLaHVwdmVJWFlod1EKWWZYNUlrWTNNUG1vNU14NTdoVWMrU0JrOVhYcGFIOGZTbTVFeFl1Zkd5WVpsWWh1L1BDdSt2ZWlYRXZHbFRheQpYZmZUUmkvQjFOTlhha3JLTzY1RjBPMks3U1ljNkx3Q3krRVU4UDA0ZXR2NEoxOFdXVHJqZ01Tenh3OUNZb1BLClNsT0xLVE1xeXptQmtETGV1NURNUGh3U1dia0Y5ejArQS94bTNjcVp1cC9yczIxYWxCa29TTTZUNGx0WmR4NUkKc21pR3NPVUNnWUVBNTE1Vy9veXNLYjJpSVZWOFFhb09LWjFsQWlJaDh2d3NsMUFRbzF3QW00VStrRnFqSVlIUgp1UDBmQjFlZW5jSHFkdm1DOEZrNnhhTGZxL2RxNGtHaHF6MDVHdjFoUDVyRXlTVXFxYmFYTGEvVUgzN2doVkJjCmsyclVtTmJYV2l2NWtJVzh4T1hOSklUenIzUUo1Y0VQMjNrVE96UWNwN3AwWHlzU05YU3RSOWNDZ1lFQTFyNk4KRWdWajNlY3hFR2tvVFB2aXc3d2E0Ti83UXFCaytWNTZPQytNamIvNHJiWDBNeEtlQjlPRDc1OXlPYlF4YjhxRgpweFZCdGVSTUN2OWtEcDJmTHBWTlB0YVBvQit3RUNQVm1aa2k5cFpITVUwMGw0QU9uVWM2SVZmZzd0cWllTzYyCnFJb20vWnVINXBlU2t2QjJVbktZTnNZdlA2VnNMaVRITStEUmRzc0NnWUVBamwxb1hqMGsrcElySHlQQXo0N2EKSkJVclFBTE1yUDBxV3ZqekkvSEtsZWVKTmIzdnZ3Qm9rZEYrdEQvZjQrYWNaUlRtMHdtMHRrT2dLZXFXSkI2SwpZaG5MOTZXTm4xdVdWc2E5MTZ2NG5pNGc4amhaNHU0dDZLL0ZuVDRsU09EaU1XRjVaQmFiQWl1azNvTWlTL05kCjFJaE5veEpQeDQxZGFlblF6SCs4MkFVQ2dZRUF2NjRDdWEwNkFSNlYrdklDV1FVVTJtWVREOXFkcWxFRkVGbTIKZW1SbTd6Z0Z2dmlFNnZtWk9aOGhTMGhsYXdCZWlFeWJsQkl6UHlweWZmYU0xMGIyaVZ3WFFSbS94Y3ZER2dVQwpha0g0cFdacVVhVjZaaWlWNHVsckI4d3JLTWphOFZzU2k4b1ZGNVkyYml1cFY2TnYyaFFUcmdDa3VBanVVUm5lCi9YMlZPcFVDZ1lCY2M5R1c5SlhLT242aGNML3lub1NiL3pQYTNrM3lFMlZVNWZiamZpK0JQVWxKRzg5T1JDSU0KSGo1c2lVN2pEeGk2VmM5eGo3dDNNVXh4cXlKYTBWaVd5bGZ6WkVzSVNUQmpub04xbSszM3VqbTV6RSs3NGxQUwpPUm4yNWhrTlIvejY0QWZiZjd6Si9jaEVZSndoQS9zK0NjcVptQ0VWaEc4RjgranRyLzhUQVE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
-kind: Secret
-metadata:
-  creationTimestamp: null
-  name: pod-annotate-webhook-certs

clr-k8s-examples/admit-kata/webhook-registration.yaml

@@ -1,24 +0,0 @@
-apiVersion: admissionregistration.k8s.io/v1beta1
-kind: MutatingWebhookConfiguration
-metadata:
-  name: pod-annotate-webhook
-  labels:
-    app: pod-annotate-webhook
-    kind: mutator
-webhooks:
-  - name: pod-annotate-webhook.pod.xyz
-    clientConfig:
-      service:
-        name: pod-annotate-webhook
-        namespace: default
-        path: "/mutate"
-      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-    rules:
-      - operations: [ "CREATE" ]
-        apiGroups: [""]
-        apiVersions: ["v1"]
-        resources: ["pods"]
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-

clr-k8s-examples/create_stack.sh

@@ -8,12 +8,23 @@ CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 
 function print_usage_exit() {
-	echo $"Usage: $0 [minimal|all]"
-	exit 1
+	exit_code=${1:-0}
+	cat <<EOT
+Usage: $0 [subcommand]
+
+Subcommands:
+
+$(
+		for cmd in "${!command_handlers[@]}"; do
+			printf "\t%s:|\t%s\n" "${cmd}" "${command_help[${cmd}]:-Not-documented}"
+		done | sort | column -t -s "|"
+	)
+EOT
+	exit "${exit_code}"
 }
 
 function finish() {
-	cd $CUR_DIR
+	cd "${CUR_DIR}"
 }
 trap finish EXIT
 
@@ -22,29 +33,26 @@ function cluster_init() {
 	#to enable the RuntimeClass featuregate
 	sudo -E kubeadm init --config=./kubeadm.yaml
 
+	rm -rf "${HOME}/.kube"
+	mkdir -p "${HOME}/.kube"
+	sudo cp -i /etc/kubernetes/admin.conf "${HOME}/.kube/config"
+	sudo chown "$(id -u):$(id -g)" "${HOME}/.kube/config"
+
 	# If this an interactive terminal then wait for user to join workers
 	if [ -t 0 ]; then
 		read -p "Join other nodes. Press enter to continue"
 	fi
 
-	rm -rf $HOME/.kube
-	mkdir -p $HOME/.kube
-	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
-	sudo chown $(id -u):$(id -g) $HOME/.kube/config
-
 	#Ensure single node k8s works
-	if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
+	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
 		kubectl taint nodes --all node-role.kubernetes.io/master-
 	fi
 }
 
-function runtimeclass_kata() {
-	#Add support for kata runtime
-	kubectl apply -f 8-kata/runtimeclass_crd.yaml
-	while [[ $(kubectl get crd runtimeclasses.node.k8s.io >/dev/null 2>&1) || $? -ne 0 ]]; do
-		echo "Waiting for runtime class CRD"
-		sleep 2
-	done
+function kata() {
+	# Install kata artifacts using kata-deploy
+	kubectl apply -f 8-kata/deploy/kata-rbac.yaml
+	kubectl apply -f 8-kata/deploy/kata-deploy.yaml
 	kubectl apply -f 8-kata/
 }
 
@@ -100,8 +108,8 @@ function miscellaneous() {
 
 function minimal() {
 	cluster_init
-	runtimeclass_kata
 	cni
+	kata
 	metrics
 }
 
@@ -112,20 +120,25 @@ function all() {
 	miscellaneous
 }
 
-cd $SCRIPT_DIR
-if [[ "$#" -eq 0 ]]; then
-	all
-	exit
-fi
+declare -A command_handlers
+command_handlers[init]=cluster_init
+command_handlers[cni]=cni
+command_handlers[minimal]=minimal
+command_handlers[all]=all
+command_handlers[help]=print_usage_exit
 
-case "$1" in
-minimal)
-	minimal
-	;;
-all)
-	all
-	;;
-*)
-	print_usage_exit
-	;;
-esac
+declare -A command_help
+command_help[init]="Only inits a cluster using kubeadm"
+command_help[cni]="Setup network for running cluster"
+command_help[minimal]="init + cni +  kata + metrics"
+command_help[all]="minimal + storage + monitoring + miscellaneous"
+command_help[help]="show this message"
+
+cd "${SCRIPT_DIR}"
+
+cmd_handler=${command_handlers[${1:-none}]:-unimplemented}
+if [ "${cmd_handler}" != "unimplemented" ]; then
+	"${cmd_handler}"
+else
+	print_usage_exit 1
+fi

clr-k8s-examples/kubeadm.yaml

@@ -1,15 +1,10 @@
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
-nodeRegistration:
-  criSocket: /var/run/crio/crio.sock
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-featureGates:
-  RuntimeClass: true
-# Kata does not work with static
-# https://github.com/kata-containers/runtime/issues/878
-# cpuManagerPolicy: static
+# Allowing for CPU pinning and isolation in case of guaranteed QoS class
+cpuManagerPolicy: static
 systemReserved:
   cpu: 500m
   memory: 256M
@@ -17,11 +12,9 @@ kubeReserved:
   cpu: 500m
   memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16
   serviceSubnet: 10.96.0.0/12
-apiServerExtraArgs:
-  feature-gates: RuntimeClass=true

clr-k8s-examples/reset_stack.sh

@@ -3,7 +3,10 @@
 set -o nounset
 
 #Cleanup
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster() {
+	sudo -E kubeadm reset -f
+}
+reset_cluster
 
 for ctr in $(sudo crictl ps --quiet); do
 	sudo crictl stop "$ctr"
@@ -15,9 +18,10 @@ for pod in $(sudo crictl pods --quiet); do
 done
 
 #Forcefull cleanup all artifacts
-#This is needed is things really go wrong
+#This is needed if things really go wrong
 sudo systemctl stop kubelet
-sudo systemctl stop crio
+systemctl is-active crio && sudo systemctl stop crio
+systemctl is-active containerd && sudo systemctl stop containerd
 sudo pkill -9 qemu
 sudo pkill -9 kata
 sudo pkill -9 kube
@@ -39,8 +43,11 @@ sudo -E bash -c "rm -r /var/run/kata-containers/*"
 sudo rm -rf /var/lib/rook
 
 sudo systemctl daemon-reload
-sudo systemctl enable kubelet crio
-sudo systemctl restart crio
+sudo systemctl is-active crio && sudo systemctl stop crio
+sudo systemctl is-active containerd && sudo systemctl stop containerd
+sudo systemctl is-enabled crio && sudo systemctl restart crio
+sudo systemctl is-enabled containerd && sudo systemctl restart containerd
+
 sudo systemctl restart kubelet
 
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster

clr-k8s-examples/setup_firecracker.sh

@@ -1,92 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-
-sudo mkdir -p /etc/kata-containers
-
-# Setup a configuration to be used by firecracker
-cat <<EOT | sudo tee /etc/kata-containers/configuration_firecracker.toml
-[hypervisor.firecracker]
-path = "/usr/bin/firecracker"
-kernel = "/usr//share/kata-containers/vmlinux.container"
-image = "/usr//share/kata-containers/kata-containers.img"
-kernel_params = ""
-default_vcpus = 1
-default_memory = 4096
-default_maxvcpus = 0
-default_bridges = 1
-block_device_driver = "virtio-mmio"
-disable_block_device_use = false
-enable_debug = true
-use_vsock = true
-
-[shim.kata]
-path = "/usr//libexec/kata-containers/kata-shim"
-
-[agent.kata]
-
-[runtime]
-internetworking_model="tcfilter"
-EOT
-
-# Firecracker can only work with devicemapper
-# Setup a sparse disk to be used for devicemapper
-sudo rm -f /var/lib/crio/devicemapper/disk.img
-sudo mkdir -p /var/lib/crio/devicemapper
-sudo truncate /var/lib/crio/devicemapper/disk.img --size 10G
-
-# Ensure that this disk is loop mounted at each boot
-sudo mkdir -p /etc/systemd/system
-
-cat <<EOT | sudo tee /etc/systemd/system/devicemapper.service
-[Unit]
-Description=Setup CRIO devicemapper
-DefaultDependencies=no
-After=systemd-udev-settle.service
-Before=lvm2-activation-early.service
-Wants=systemd-udev-settle.service
-
-[Service]
-ExecStart=-/sbin/losetup /dev/loop8 /var/lib/crio/devicemapper/disk.img
-RemainAfterExit=true
-Type=oneshot
-
-[Install]
-WantedBy=local-fs.target
-EOT
-
-sudo systemctl daemon-reload
-sudo systemctl enable --now devicemapper
-
-# For now till we address https://github.com/kubernetes-sigs/cri-o/issues/1991
-# use a shell script to expose firecracker through kata
-cat <<EOT | sudo tee /usr/bin/kata-runtime-fire
-#!/bin/bash
-
-/usr/bin/kata-runtime --kata-config /etc/kata-containers/configuration_firecracker.toml "\$@"
-EOT
-
-sudo chmod +x /usr/bin/kata-runtime-fire
-
-# Add firecracker as a second runtime
-# Also setup crio to use devicemapper
-
-sudo mkdir -p /etc/crio/
-sudo cp /usr/share/defaults/crio/crio.conf /etc/crio/crio.conf
-
-echo -e "\n[crio.runtime.runtimes.kata]\nruntime_path = \"/usr/bin/kata-runtime\"" | sudo tee -a /etc/crio/crio.conf
-echo -e "\n[crio.runtime.runtimes.fire]\nruntime_path = \"/usr/bin/kata-runtime-fire\"" | sudo tee -a /etc/crio/crio.conf
-
-sudo sed -i 's|\(\[crio\.runtime\]\)|\1\nmanage_network_ns_lifecycle = true|' /etc/crio/crio.conf
-
-sudo sed -i 's/storage_driver = \"overlay\"/storage_driver = \"devicemapper\"\
-storage_option = [\
-  \"dm.basesize=8G\",\
-  \"dm.directlvm_device=\/dev\/loop8\",\
-  \"dm.directlvm_device_force=true\",\
-  \"dm.fs=ext4\"\
-]/g' /etc/crio/crio.conf
-
-sudo systemctl restart crio || true

clr-k8s-examples/setup_kata_firecracker.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Firecracker can only work with devicemapper
+# Setup a sparse disk to be used for devicemapper
+sudo rm -f /var/lib/crio/devicemapper/disk.img
+sudo mkdir -p /var/lib/crio/devicemapper
+sudo truncate /var/lib/crio/devicemapper/disk.img --size 10G
+
+# Ensure that this disk is loop mounted at each boot
+sudo mkdir -p /etc/systemd/system
+
+cat <<EOT | sudo tee /etc/systemd/system/devicemapper.service
+[Unit]
+Description=Setup CRIO devicemapper
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=-/sbin/losetup /dev/loop8 /var/lib/crio/devicemapper/disk.img
+RemainAfterExit=true
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+EOT
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now devicemapper
+
+sudo sed -i 's/storage_driver = \"overlay\"/storage_driver = \"devicemapper\"\
+storage_option = [\
+  \"dm.basesize=8G\",\
+  \"dm.directlvm_device=\/dev\/loop8\",\
+  \"dm.directlvm_device_force=true\",\
+  \"dm.override_udev_sync_check=true",\
+  \"dm.fs=ext4\"\
+]/g' /etc/crio/crio.conf
+
+sudo systemctl restart crio || true

clr-k8s-examples/setup_system.sh

@@ -22,6 +22,8 @@ fi
 sudo mkdir -p /etc/sysctl.d/
 cat <<EOT | sudo bash -c "cat > /etc/sysctl.d/60-k8s.conf"
 net.ipv4.ip_forward=1
+net.ipv4.conf.default.rp_filter=1
+net.ipv4.conf.all.rp_filter=1
 EOT
 sudo systemctl restart systemd-sysctl
 
@@ -33,6 +35,10 @@ vhost_vsock
 overlay
 EOT
 
+# Make sure /etc/hosts file exists
+if [ ! -f /etc/hosts ]; then
+  sudo touch /etc/hosts
+fi
 hostcount=$(grep '127.0.0.1 localhost' /etc/hosts | wc -l)
 if [ "$hostcount" == "0" ]; then
 	echo "127.0.0.1 localhost $(hostname)" | sudo bash -c "cat >> /etc/hosts"
@@ -46,8 +52,6 @@ sudo systemctl daemon-reload
 echo "The following kubelet command may complain... it is not an error"
 sudo systemctl enable --now kubelet crio || true
 
-sudo mkdir -p /usr/libexec/cni /opt/cni
-[ ! -e /opt/cni/bin/cni ] && sudo ln -s /usr/libexec/cni /opt/cni/bin
 #Ensure that the system is ready without requiring a reboot
 sudo swapoff -a
 sudo systemctl restart systemd-modules-load.service

clr-k8s-examples/tests/autoscale/load-gen.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: load-generator
+spec:
+  containers:
+  - command: ["/bin/sh", "-c"]
+    args:
+    - while true; do wget -q -O- http://php-apache-test; done;
+    image: busybox
+    imagePullPolicy: Always
+    name: load-generator

clr-k8s-examples/tests/autoscale/php.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-test
+  name: php-apache-test
+spec:
+  selector:
+    matchLabels:
+      run: php-apache-test
+  template:
+    metadata:
+      labels:
+        run: php-apache-test
+    spec:
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        name: php-apache-test
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: php-apache-test
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-test
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: php-apache-test
+spec:
+  maxReplicas: 10
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: php-apache-test
+  targetCPUUtilizationPercentage: 50
+

clr-k8s-examples/tests/autoscale/test-autoscale.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+kubectl apply -f $SCRIPT_DIR
+watch kubectl describe hpa

clr-k8s-examples/tests/cpumanager/test-cpumanager.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+input="test-cpumanager.yaml.tmpl"
+
+filename() {
+	echo "test-cpumanager-$1.yaml"
+}
+
+for runtimeclass in runc kata-qemu kata-fc; do
+	output=$(filename $runtimeclass)
+	cp $input $output
+	sed -i "s/__runtimeclass__/$runtimeclass/g" $output
+	if [ $runtimeclass == "runc" ]; then continue; fi
+
+	insertline="\ \ runtimeClassName: $runtimeclass"
+	sed -i "/spec:/a $insertline" $output
+done
+kubectl apply -f .

clr-k8s-examples/tests/cpumanager/test-cpumanager.yaml.tmpl

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-guaranteed-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        cpu: 1
+        memory: 500Mi # For kata to run
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+      limits:
+        cpu: 2
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+      limits:
+        cpu: 1
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-besteffort-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-fc.yaml

@@ -2,19 +2,19 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    run: php-apache-fire
-  name: php-apache-fire
+    run: php-apache-kata-fc
+  name: php-apache-kata-fc
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: php-apache-fire
+      run: php-apache-kata-fc
   template:
     metadata:
       labels:
-        run: php-apache-fire
+        run: php-apache-kata-fc
     spec:
-      runtimeClassName: fire
+      runtimeClassName: kata-fc
       containers:
       - image: k8s.gcr.io/hpa-example
         imagePullPolicy: Always
@@ -30,13 +30,13 @@ spec:
 apiVersion: v1 
 kind: Service
 metadata:
-  name: php-apache-fire
+  name: php-apache-kata-fc
 spec:
   ports:
   - port: 80
     protocol: TCP
     targetPort: 80
   selector:
-    run: php-apache-fire
+    run: php-apache-kata-fc
   sessionAffinity: None
   type: ClusterIP

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-qemu.yaml

@@ -2,22 +2,19 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    run: php-apache-kata
-  name: php-apache-kata
+    run: php-apache-kata-qemu
+  name: php-apache-kata-qemu
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: php-apache-kata
+      run: php-apache-kata-qemu
   template:
     metadata:
-      annotations:
-        io.kubernetes.cri-o.TrustedSandbox: "false"
-        io.kubernetes.cri.untrusted-workload: "true"
       labels:
-        run: php-apache-kata
+        run: php-apache-kata-qemu
     spec:
-      runtimeClassName: kata
+      runtimeClassName: kata-qemu
       containers:
       - image: k8s.gcr.io/hpa-example
         imagePullPolicy: Always
@@ -33,13 +30,13 @@ spec:
 apiVersion: v1 
 kind: Service
 metadata:
-  name: php-apache-kata
+  name: php-apache-kata-qemu
 spec:
   ports:
   - port: 80
     protocol: TCP
     targetPort: 80
   selector:
-    run: php-apache-kata
+    run: php-apache-kata-qemu
   sessionAffinity: None
   type: ClusterIP

clr-k8s-examples/tests/test-autoscale.sh

@@ -1,6 +0,0 @@
-kubectl run php-apache-test --image=k8s.gcr.io/hpa-example --requests=cpu=200m --expose --port=80
-kubectl autoscale deployment php-apache-test --cpu-percent=50 --min=1 --max=10
-kubectl get hpa
-
-#kubectl run -i --tty load-generator --image=busybox /bin/sh
-# while true; do wget -q -O- http://php-apache-test.default.svc.cluster.local; done

clr-k8s-examples/tests/test-cpumanager-kata.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: test-cpumanager-kata
-spec:
-  runtimeClassName: kata
-  restartPolicy: Never
-  containers:
-  - name: taskset
-    image: busybox
-    command: [ "taskset", "-p", "1" ]
-    resources:
-      limits:
-        cpu: 1
-        memory: 500Mi

clr-k8s-examples/tests/test-cpumanager-runc.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: test-cpumanager-runc
-spec:
-  restartPolicy: Never
-  containers:
-  - name: taskset
-    image: busybox
-    command: [ "taskset", "-p", "1" ]
-    resources:
-      limits:
-        cpu: 1
-        memory: 100Mi