Add swupd update back

Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

.gitignore

@@ -0,0 +1,3 @@
+.vagrant
+OVMF.fd
+

Makefile

@@ -5,5 +5,5 @@ TARGET ?= $(PREFIX)/share/
 all:
 
 install:
-        install -m 0755 -d $(DESTDIR)/$(TARGET)/clr-k8s-examples
+	install -m 0755 -d $(DESTDIR)/$(TARGET)/clr-k8s-examples
-        cp -r clr-k8s-examples/* $(DESTDIR)/$(TARGET)/clr-k8s-examples/
+	cp -r clr-k8s-examples/* $(DESTDIR)/$(TARGET)/clr-k8s-examples/

clr-k8s-examples/0-canal/canal.yaml

@@ -1,8 +1,8 @@
-# Canal Version v3.3.0
+# Canal Version v3.3.2
-# https://docs.projectcalico.org/v3.3/releases#v3.3.0
+# https://docs.projectcalico.org/v3.3/releases#v3.3.2
 # This manifest includes the following component versions:
-#   calico/node:v3.3.0
+#   calico/node:v3.3.2
-#   calico/cni:v3.3.0
+#   calico/cni:v3.3.2
 #   coreos/flannel:v0.9.1
 
 # This ConfigMap is used to configure a self-hosted Canal installation.
@@ -115,7 +115,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: quay.io/calico/node:v3.3.0
+          image: quay.io/calico/node:v3.3.2
           env:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
@@ -159,6 +159,8 @@ spec:
               value: "info"
             - name: FELIX_HEALTHENABLED
               value: "true"
+            - name: FELIX_IGNORELOOSERPF
+              value: "true"
           securityContext:
             privileged: true
           resources:
@@ -194,7 +196,7 @@ spec:
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: quay.io/calico/cni:v3.3.0
+          image: quay.io/calico/cni:v3.3.2
           command: ["/install-cni.sh"]
           env:
             # Name of the CNI config file to create.

clr-k8s-examples/1-core-metrics/aggregated-metrics-reader.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]

clr-k8s-examples/1-core-metrics/auth-delegator.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/auth-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/metrics-apiservice.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apiregistration.k8s.io/v1beta1
 kind: APIService
 metadata:

clr-k8s-examples/1-core-metrics/metrics-server-deployment.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -22,10 +23,19 @@ spec:
         k8s-app: metrics-server
     spec:
       serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
       containers:
       - name: metrics-server
-        image: gcr.io/google_containers/metrics-server-amd64:v0.2.0
+        image: k8s.gcr.io/metrics-server-amd64:v0.3.1
         imagePullPolicy: Always
-        command:
+        args:
-        - /metrics-server
+        - --logtostderr
-        - --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true
+        - --kubelet-insecure-tls
+        - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+

clr-k8s-examples/1-core-metrics/metrics-server-service.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:

clr-k8s-examples/1-core-metrics/resource-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -9,15 +10,6 @@ rules:
   - pods
   - nodes
   - nodes/stats
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - "extensions"
-  resources:
-  - deployments
   verbs:
   - get
   - list

clr-k8s-examples/1-core-metrics/versions

@@ -1 +1 @@
-f90c6705d2381ea2db1a6343da6c400bd2ef4cb2
+92d8412788e27ee669d38f21f20bad5342211884

clr-k8s-examples/7-rook/000-operator.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph-system
-  labels:
-    kata: "false"
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition

clr-k8s-examples/7-rook/001-cluster.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph
-  labels:
-    kata: "false"
 ---
 apiVersion: v1
 kind: ServiceAccount

clr-k8s-examples/8-kata/deploy/kata-cleanup.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubelet-kata-cleanup
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kubelet-kata-cleanup
+  template:
+    metadata:
+        labels:
+          name: kubelet-kata-cleanup
+    spec:
+      serviceAccountName: kata-label-node
+      nodeSelector:
+          katacontainers.io/kata-runtime: cleanup
+      containers:
+      - name: kube-kata-cleanup
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      volumes:
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-deploy.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kata-deploy
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kata-deploy
+  template:
+    metadata:
+        labels:
+          name: kata-deploy
+    spec:
+      serviceAccountName: kata-label-node
+      containers:
+      - name: kube-kata
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        lifecycle:
+          preStop:
+            exec:
+              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: crio-conf
+          mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
+        - name: kata-artifacts
+          mountPath: /opt/kata/
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+        - name: local-bin
+          mountPath: /usr/local/bin/
+      volumes:
+        - name: crio-conf
+          hostPath:
+            path: /etc/crio/
+        - name: containerd-conf
+          hostPath:
+            path: /etc/containerd/
+        - name: kata-artifacts
+          hostPath:
+            path: /opt/kata/
+            type: DirectoryOrCreate
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin/
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-rbac.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kata-label-node
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: node-labeler
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kata-label-node-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-labeler
+subjects:
+- kind: ServiceAccount
+  name: kata-label-node
+  namespace: kube-system
+

clr-k8s-examples/8-kata/kata-fc-runtimeClass.yaml

@@ -0,0 +1,5 @@
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-fc
+handler: kata-fc

clr-k8s-examples/8-kata/kata-qemu-runtimeClass.yaml

@@ -0,0 +1,12 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata
+handler: kata
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1beta1
+metadata:
+    name: kata-qemu
+handler: kata-qemu

clr-k8s-examples/8-kata/kata-runtimeClass.yaml

@@ -1,6 +0,0 @@
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
-metadata:
-    name: kata
-spec:
-    runtimeHandler: kata

clr-k8s-examples/8-kata/runtimeclass_crd.yaml

@@ -1,26 +0,0 @@
-kind: CustomResourceDefinition
-apiVersion: apiextensions.k8s.io/v1beta1
-metadata:
-  name: runtimeclasses.node.k8s.io
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  group: node.k8s.io
-  version: v1alpha1
-  versions:
-    - name: v1alpha1
-      served: true
-      storage: true
-  names:
-    plural:   runtimeclasses
-    singular: runtimeclass
-    kind:     RuntimeClass
-  scope: Cluster
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            runtimeHandler:
-              type: string
-              pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$'

clr-k8s-examples/9-multi-network/Dockerfile

@@ -0,0 +1,33 @@
+# Build multus plugin
+FROM golang:1.10 AS multus
+RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
+WORKDIR /go/src/github.com/intel/multus-cni
+RUN ./build
+
+# Build sriov plugin
+FROM golang:1.10 AS sriov-cni
+RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
+WORKDIR /go/src/github.com/intel-corp/sriov-cni
+RUN ./build
+
+# Build sriov device plugin
+FROM golang:1.10 AS sriov-dp
+RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
+WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
+RUN make
+
+# Build vfioveth plugin
+FROM busybox as vfioveth
+RUN wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+COPY cni/vfioveth /bin/vfioveth
+RUN chmod +x /bin/vfioveth /bin/jq
+
+# Final image
+FROM centos/systemd
+WORKDIR /tmp/cni/bin
+COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
+COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=vfioveth /bin/vfioveth .
+COPY --from=vfioveth /bin/jq .
+WORKDIR /usr/bin
+COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -0,0 +1,76 @@
+# Multi-Network
+
+## Daemonset
+
+We launch a `Daemonset` with an `initContainer` which sets up the CNI
+directories on the host with the necessary binaries and configuration files.
+
+> NOTE: SR-IOV devices are not necessary to test multi-network capability
+
+### Customization
+
+The device plugin will register the SR-IOV enabled devices on the host, specified as
+`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+file is provided, which enables SR-IOV for the above `rootDevices`
+
+> NOTE: This assumes homogenous nodes in the cluster
+
+### Pre-req (SR-IOV only)
+
+One each SR-IOV node make sure `VT-d` is enabled in the BIOS and `intel_iommu=on` on kernel commandline.
+Setup systemd to bring up VFs on designated interfaces bound to network driver or `vfio-pci`
+
+```bash
+# Make sure vfio-pci is loaded on boot
+echo 'vfio-pci' | sudo tee /etc/modules-load.d/sriov.conf
+sudo systemctl restart systemd-modules-load.service
+
+sudo cp systemd/sriov.sh /usr/bin/sriov.sh
+sudo cp systemd/sriov.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now sriov.service
+```
+
+### Install
+
+To install and configure `multus-cni` on all nodes, along with
+`sriov-cni`, `vfioveth-cni` and `sriov-network-device-plugin`
+
+```bash
+kubectl apply -f .
+kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov_*"
+```
+
+## Tests
+
+### Default only
+
+To test if default connectivity is working
+
+```bash
+kubectl apply -f test/pod.yaml
+kubectl exec test -- ip a                        # should see one interface only
+```
+
+### Bridge
+
+To test multus with second interface created by `bridge` plugin
+
+```bash
+kubectl apply -f test/bridge
+kubectl exec test-bridge -- ip a                 # should see two interfaces
+ip a show mynet                                  # bridge created on host if it doesnt exist already
+```
+
+### SR-IOV
+
+To test multus with second interface created by `sriov` plugin
+
+```bash
+kubectl apply -f test/sriov
+
+kubectl exec test-sriov -- ip a                  # second interface is a VF
+
+kubectl exec test-sriov-dpdk -- ip a             # veth pair with details of VF
+kubectl exec test-sriov-dpdk -- ls -l /dev/vfio
+```

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -0,0 +1,67 @@
+#!/bin/bash -x
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+exec 3>&1
+exec &>>/var/log/$(basename $0).log
+
+PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH"
+CNI_CONF=$(cat /dev/stdin)
+
+get_peer_name() {
+	echo "$1-vdev"
+}
+
+get_mac_with_vfpci() {
+	local pf=$(readlink /sys/devices/pci*/*/$1/physfn | awk '{print substr($1,4)}')
+	local pfName=$(ls /sys/devices/pci*/*/$pf/net/ | head -1)
+	local idx=$(ls -l /sys/devices/pci*/*/$pf | awk -v vf=$1 'substr($11,4)==vf {print substr($9,7)}')
+	local mac=$(ip link show dev $pfName | awk -v idx="$idx" '$1=="vf" && $2==idx {print substr($4,1,17)}')
+	echo $mac
+}
+
+ipam() {
+	local plugin=$(echo $CNI_CONF | jq -r '.ipam.type')
+	local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.')
+	echo $res
+}
+
+add_pair_ns() {
+	vfpci=$(echo $CNI_CONF | jq -r '.deviceID')
+	mac=$(get_mac_with_vfpci $vfpci)
+	peer=$(get_peer_name $CNI_IFNAME)
+	ip=$1
+
+	mkdir -p /var/run/netns/
+	ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID
+
+	ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer
+	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up
+	ip netns exec $CNI_CONTAINERID ip link set $peer up
+	ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
+}
+
+delete_pair_ns() {
+	ip netns exec $CNI_CONTAINERID ip link del $CNI_IFNAME
+}
+
+case $CNI_COMMAND in
+ADD)
+	res=$(ipam)
+	ip=$(echo $res | jq -r '.ip4.ip')
+	add_pair_ns $ip
+	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	;;
+DEL)
+	set +o errexit
+	ipam
+	delete_pair_ns
+	set -o errexit
+	;;
+*)
+	echo "CNI_COMMAND=[ADD|DEL] only supported"
+	exit 1
+	;;
+esac

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -0,0 +1,212 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: k8s.cni.cncf.io
+  # version name to use for REST API: /apis/<group>/<version>
+  version: v1
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: network-attachment-definitions
+    # singular name to be used as an alias on the CLI and for display
+    singular: network-attachment-definition
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: NetworkAttachmentDefinition
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - net-attach-def
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: multus-sa-secret
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: multus-sa
+type: kubernetes.io/service-account-token
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-pod-networks-lister
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["update"]
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs: ["get"]
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-scripts
+  namespace: kube-system
+data:
+  install-certs.sh: |
+    # Copied from Calico
+    # https://github.com/projectcalico/cni-plugin/blob/master/k8s-install/scripts/install-cni.sh
+    touch /host/etc/cni/net.d/multus-kubeconfig
+    chmod 600 /host/etc/cni/net.d/multus-kubeconfig
+    SERVICE_ACCOUNT_PATH=/var/run/secrets/multus/serviceaccount
+    KUBE_CA_FILE=$SERVICE_ACCOUNT_PATH/ca.crt
+    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+    SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+    cat > /host/etc/cni/net.d/multus-kubeconfig <<EOF
+    # Kubeconfig file for Multus CNI plugin.
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - name: local
+      cluster:
+        server: https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+        $TLS_CFG
+    users:
+    - name: multus
+      user:
+        token: "${SERVICEACCOUNT_TOKEN}"
+    contexts:
+    - name: multus-context
+      context:
+        cluster: local
+        user: multus
+    current-context: multus-context
+    EOF
+  install-multus-conf.sh: |
+    # copied from https://github.com/intel/multus-cni/blob/master/images/entrypoint.sh
+    rm -f /host/etc/cni/net.d/00-multus.conf
+    MASTER_PLUGIN="$(ls /host/etc/cni/net.d | grep -E '\.conf(list)?$' | head -1)"
+    MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
+    cat > /host/etc/cni/net.d/00-multus.conf <<EOF
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "logFile": "/var/log/multus.log",
+      "logLevel": "debug",
+      "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
+      "delegates": [
+                    $MASTER_PLUGIN_JSON
+       ]
+    }
+    EOF
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-networks-lister
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sriov-device-plugin
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: sriov-device-plugin
+  template:
+    metadata:
+      labels:
+        name: sriov-device-plugin
+    spec:
+      initContainers:
+      - name: multus
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "bash", "-c" ]
+        args:
+        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+          /tmp/multus/install-multus-conf.sh;
+          /tmp/multus/install-certs.sh;
+          echo "Restarting crio kubelet";
+          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart kubelet;
+        volumeMounts:
+        - name: usr-bin
+          mountPath: /host/usr/bin
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: multus-sa
+          mountPath: /var/run/secrets/multus/serviceaccount
+        - name: multus-scripts
+          mountPath: /tmp/multus
+        - name: cni-conf
+          mountPath: /host/etc/cni/net.d
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      containers:
+      - name: sriovdp
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "sh", "-c" ]
+        args:
+        - /usr/bin/sriovdp --logtostderr -v 10;
+          sleep infinity;
+        volumeMounts:
+        - name: net
+          mountPath: /sys/class/net
+          readOnly: true
+        - name: dp-sock
+          mountPath: /var/lib/kubelet/device-plugins/
+          readOnly: false
+        - mountPath: /etc/pcidp
+          name: sriov-config
+          readOnly: false
+      volumes:
+      - name: sriov-config
+        configMap:
+          name: sriov-config
+      - name: usr-bin
+        hostPath:
+          path: /usr/bin
+      - name: cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: multus-sa
+        secret:
+          secretName: multus-sa-secret
+      - name: multus-scripts
+        configMap:
+          defaultMode: 511
+          name: multus-scripts
+      - name: cni-conf
+        hostPath:
+          path: /etc/cni/net.d
+      - name: dbus
+        hostPath:
+          path: /var/run/dbus
+      - name: systemd
+        hostPath:
+          path: /run/systemd
+      - name: net
+        hostPath:
+          path: /sys/class/net
+      - name: dp-sock
+        hostPath:
+          path: /var/lib/kubelet/device-plugins/
+      hostNetwork: true
+      hostPID: true
+

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -0,0 +1,25 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: sriov-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+      "resourceList":
+      [
+        {
+          "resourceName": "sriov_netdevice",
+          "rootDevices": ["07:00.0"],
+          "sriovMode": true,
+          "deviceType": "netdevice"
+        },
+        {
+          "resourceName": "sriov_vfio",
+          "rootDevices": ["07:00.1"],
+          "sriovMode": true,
+          "deviceType": "vfio"
+        }
+      ]
+    }

clr-k8s-examples/9-multi-network/systemd/sriov.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Create VFs on ens785f0 (netdev) ens785f1 (vfio) interfaces
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/sriov.sh ens785f0
+ExecStart=/usr/bin/sriov.sh -b ens785f1
+
+[Install]
+WantedBy=default.target

clr-k8s-examples/9-multi-network/systemd/sriov.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+OPTIND=1
+bind="false"
+
+while getopts ":b" opt; do
+	case ${opt} in
+	b)
+		bind="true"
+		;;
+	\?)
+		echo "Usage: sriov.sh [-b] ens785f0 ens785f1 ..."
+		echo "-b	Bind to vfio-pci"
+		exit
+		;;
+	esac
+done
+shift $((OPTIND - 1))
+
+setup_pf() {
+	local pf=$1
+	echo "Resetting PF $pf"
+	echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
+	echo "Enabling $NUM_VFS VFs for $pf"
+	echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs
+	ip link set $pf up
+	sleep 1
+}
+
+setup_vfs() {
+	local pf=$1
+	local pfpci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_numvfs)
+	for ((idx = 0; idx < NUM_VFS; idx++)); do
+		ip link set dev $pf vf $idx state enable
+		if [ $bind != "true" ]; then continue; fi
+
+		local vfn="virtfn$idx"
+		local vfpci=$(ls -l /sys/devices/pci*/*/$pfpci | awk -v vfn=$vfn 'vfn==$9 {print substr($11,4)}')
+		# Capture and set MAC of the VF before unbinding from linux, for later use in CNI
+		local mac=$(cat /sys/bus/pci*/*/$vfpci/net/*/address)
+		ip link set dev $pf vf $idx mac $mac
+		# Bind VF to vfio-pci
+		echo $vfpci >/sys/bus/pci*/*/$vfpci/driver/unbind
+		echo "vfio-pci" >/sys/devices/pci*/*/$vfpci/driver_override
+		echo $vfpci >/sys/bus/pci/drivers/vfio-pci/bind
+	done
+}
+
+for pf in "$@"; do
+	setup_pf $pf
+	setup_vfs $pf
+done

clr-k8s-examples/9-multi-network/test/bridge/0-bridge-net.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: mynet
+spec:
+  config: '{
+        "name": "mynet",
+        "type": "bridge",
+        "bridge": "mynet",
+        "ipam": {
+            "type": "host-local",
+            "subnet": "198.18.0.0/24"
+        }
+}'
+

clr-k8s-examples/9-multi-network/test/bridge/1-pod-bridge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-bridge
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "mynet", "interface": "mynet" }
+    ]'
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/pod.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_netdevice
+spec:
+  config: '{
+    "type": "sriov",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-dpdk
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_vfio
+spec:
+  config: '{
+    "type": "vfioveth",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+

clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_netdevice: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov-dpdk
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+

clr-k8s-examples/README.md

@@ -1,13 +1,24 @@
 # How to setup the cluster
 
+## Prerequisite
+This setup currently will work with k8s 1.14 & above. Any version of k8s before that might work, but is not guaranteed.
+
 ## Sample multi-node vagrant setup
 
-To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using libvirt, but you can use any hypervisor that you are familiar with.
+To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
 
 * Install vagrant on the distro you are using. Steps can be found at [Vagrant docs](https://www.vagrantup.com/intro/getting-started/install.html#installing-vagrant)
 * `vagrant up --provider=libvirt`
 
 Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
+Customize the setup using environment variables. E.g., `NODES=1 MEMORY=8192 CPUS=8 vagrant up --provider=libvirt`
+
+To login to the master node and change to this directory
+
+```bash
+vagrant ssh clr-01
+cd clr-k8s-examples
+```
 
 ## Setup the nodes in the cluster
 
@@ -20,6 +31,18 @@ This script ensures the following
 * Customizes the system to ensure correct defaults are setup (IP Forwarding, Swap off,...)
 * Ensures all the dependencies are loaded on boot (kernel modules)
 
+> NOTE: This step is done automatically if using vagrant.
+
+### Enabling experimental firecracker support
+
+> EXPERIMENTAL: Optionally run [`setup_kata_firecracker.sh`](setup_kata_firecracker.sh) to be
+able to use firecracker VMM with Kata.
+
+The firecracker setup switches the setup to use a sparse file backed loop device for
+devicemapper storage. This should not be used for production.
+
+> NOTE: This step is done automatically if using vagrant.
+
 ## Bring up the master
 
 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -27,6 +50,14 @@ master and also uses kubelet config via [`kubeadm.yaml`](kubeadm.yaml)
 to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
+There are different flavors to install, run `./create_stack.sh help` to get
+more information.
+
+```bash
+# default shows help
+./create_stack.sh <subcommand>
+```
+
 ## Join Workers to the cluster
 
 ```bash
@@ -51,45 +82,23 @@ runtime class set to "kata" will launch the POD/Deployment with Kata.
 
 An example is
 
-`kubectl apply -f tests/test-deploy-kata.yaml`
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-qemu.yaml`
 
-## Making Kata the default runtime
+### Running Kata Workloads with Firecracker
 
-Today in `crio.conf` runc is the default runtime when a user does not specify
+> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "kata-fc" will launch the POD/Deployment
-`runtimeClass` in the pod spec. If you want to run a cluster where kata is used
+with firecracker as the isolation mechanism for Kata.
-by default, except for workloads we know for sure will not work with kata, use
-the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
-and sample admission controller we created by running -
 
-`kubectl apply -f admit-kata/`
+An example is
 
-The [admission webhook](admit-kata/webhook-registration.yaml)
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-fc.yaml`
-is setup to exclude certian namespaces from being run with Kata using filters on namespace labels.
 
-```yaml
+## Making Kata the default runtime using admission controller
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-```
 
-The rook operators for example are marked as such
+If you want to run a cluster where kata is used
-
+by default, except for workloads we know for sure will not work with kata, using
-```yaml
+[admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
-apiVersion: v1
+and sample admission controller, follow `admit-kata` [README.md](admit-kata/README.md)
-kind: Namespace
-metadata:
-  name: rook-ceph-system
-  labels:
-    kata: "false"
-```
-
-Pods not explicitly excluded by the namespace filter are dynamically tagged to
-run with Kata with some [exceptions](https://github.com/mcastelino/kubewebhook/blob/topic/hack-kata/examples/pod-annotate/main.go#L25) -
-
-* `hostNetwork: true`
-* `rook-ceph` and `rook-ceph-system` namespaces (buggy)
-
-Other pod properties will be added as exceptions in future.
 
 ## Accessing control plane services
 

clr-k8s-examples/Vagrantfile

@@ -4,21 +4,26 @@
 require 'fileutils'
 require 'ipaddr'
 require 'securerandom'
-DISK_UUID = SecureRandom.urlsafe_base64(9)
 
 $num_instances = (ENV['NODES'] || 3).to_i
 $cpus = (ENV['CPUS'] || 2).to_i
 $memory = (ENV['MEMORY'] || 4096).to_i
 $disks = 2
+# Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
+$disk_prefix = File.basename(File.dirname(__FILE__), "/")
 $disk_size = "10G"
-$box = "gmmaha/clearlinux"
+$box = "AntonioMeireles/ClearLinux"
-$loader = File.join(File.dirname(__FILE__), "OVMF.fd")
+File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("192.52.100.10")
-hosts = {}
+$hosts = {}
-proxy_ip_list = ""
+$proxy_ip_list = ""
-#DISK_UUID = Time.now.utc.to_i
+$driveletters = ('a'..'z').to_a
-driveletters = ('a'..'z').to_a
+$setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
+$runner = ENV.has_key?('RUNNER') ? ENV['RUNNER'].to_s : "containerd".to_s
+if !(["crio","containerd"].include? $runner)
+  abort("it's either crio or containerd. Cannot do anything else")
+end
 
 if not File.exists?($loader)
   system('curl -O https://download.clearlinux.org/image/OVMF.fd')
@@ -42,40 +47,57 @@ Vagrant.configure("2") do |config|
   # Every Vagrant development environment requires a box. You can search for
   # boxes at https://vagrantcloud.com/search.
   config.vm.box = $box
+  config.vm.box_version = "30260"
 
+  # Mount the current dir at home folder instead of default
+  config.vm.synced_folder './', '/vagrant', disabled: true
+  config.vm.synced_folder './', '/home/clear/' + File.basename(Dir.getwd), type: 'rsync',
+    rsync__args: ["--verbose", "--archive", "--delete", "-zz", "--copy-links"]
   #Setup proxies for all machines
   (1..$num_instances).each do |i|
-    base_ip = base_ip.succ
+    $base_ip = $base_ip.succ
-    hosts["clr-%02d" % i] = base_ip.to_s
+    $hosts["clr-%02d" % i] = $base_ip.to_s
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     proxy_ip_list = ("#{proxy_ip_list},#{vm_name},#{ip}")
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     config.vm.define vm_name do |c|
       c.vm.hostname = vm_name
       c.vm.network :private_network, ip: ip, autostart: true
       c.vm.provider :libvirt do |lv|
-        lv.loader = $loader
         lv.cpu_mode = "host-passthrough"
         lv.nested = true
+        lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
         (1..$disks).each do |d|
-          lv.storage :file, :device => "hd#{driveletters[d]}", :path => "disk-#{vm_name}-#{d}-#{DISK_UUID}.disk", :size => $disk_size, :type => "raw"
+          lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end
       end
-      if Vagrant.has_plugin?("vagrant-proxyconf")
+      if ENV['http_proxy'] || ENV['HTTP_PROXY']
-        c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
+        if Vagrant.has_plugin?("vagrant-proxyconf")
-        c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
+          c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
-        c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+          c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
+          c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+        end
       end
-      # Bad hack for the vagrant libvirt boxes. WIll be removed once they are fixed.
+      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner}
-      c.vm.provision "shell", privileged: false, inline: "sudo usermod --password vagrant root"
+      if $setup_fc
-
+        if $runner == "crio".to_s
-      c.vm.provision "shell", privileged: false, path: "setup_system.sh"
+        c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+        else
+        # Wish we could use device mapper snapshotter with containerd, but it
+        # does not exist on any released containerd version. Failing for now
+        # when we use FC with containerd
+        abort("Cannot use containerd with FC for now.")
+        #c.vm.provision "shell", privileged: false, path: "containerd_devmapper_setup.sh"
+        end
+      end
+      # Include shells bundle to get bash completion and add kubectl's commands to vagrant's shell
+      c.vm.provision "shell", privileged: false, inline: 'sudo -E swupd bundle-add shells; echo "source <(kubectl completion bash)" >> $HOME/.bashrc'
     end
   end
 end

clr-k8s-examples/admit-kata/README.md

@@ -0,0 +1,39 @@
+# Kata Admission controller webhook
+
+Implement a simple admission controller webhook to annotate pods with the 
+Kata runtime class.
+
+## How to build the admission controller
+
+First build the admission controller image and the associated
+Kubernetes yaml files required to instantiate the admission
+controller.
+
+```bash
+$ docker build -t katadocker/kata-webhook-example:latest .
+$ ./create_certs.sh
+```
+
+> **Note:**
+> Image needs to be published for the webhook needs to work. Alternately
+> on a single machine cluster change the `imagePullPolicy` to use the locally
+> built image.
+
+## Making Kata the default runtime using an admission controller
+
+Today in `crio.conf` `runc` is the default runtime when a user does not specify
+`runtimeClass` in the pod spec. If you want to run a cluster where Kata is used
+by default, except for workloads we know for sure will not work with Kata, use
+the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
+and sample admission controller we created by running
+
+```bash
+$ kubectl apply -f deploy/
+```
+
+The webhook mutates pods to use the kata runtime class for all pods except
+those with 
+
+* `hostNetwork: true` 
+* namespace: `rook-ceph` and `rook-ceph-system`
+

clr-k8s-examples/admit-kata/create-certs.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+WEBHOOK_NS=${1:-"default"}
+WEBHOOK_NAME=${2:-"pod-annotate"}
+WEBHOOK_SVC="${WEBHOOK_NAME}-webhook"
+
+# Create certs for our webhook
+openssl genrsa -out webhookCA.key 2048
+openssl req -new -key ./webhookCA.key -subj "/CN=${WEBHOOK_SVC}.${WEBHOOK_NS}.svc" -out ./webhookCA.csr 
+openssl x509 -req -days 365 -in webhookCA.csr -signkey webhookCA.key -out webhook.crt
+
+# Create certs secrets for k8s
+kubectl create secret generic \
+    ${WEBHOOK_SVC}-certs \
+    --from-file=key.pem=./webhookCA.key \
+    --from-file=cert.pem=./webhook.crt \
+    --dry-run -o yaml > ./deploy/webhook-certs.yaml
+
+# Set the CABundle on the webhook registration
+CA_BUNDLE=$(cat ./webhook.crt | base64 -w0)
+sed "s/CA_BUNDLE/${CA_BUNDLE}/" ./deploy/webhook-registration.yaml.tpl > ./deploy/webhook-registration.yaml
+
+# Clean
+rm ./webhookCA* && rm ./webhook.crt

clr-k8s-examples/admit-kata/deploy/webhook-registration.yaml.tpl

@@ -0,0 +1,24 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pod-annotate-webhook
+  labels:
+    app: pod-annotate-webhook
+    kind: mutator
+webhooks:
+  - name: pod-annotate-webhook.kata.xyz
+    clientConfig:
+      service:
+        name: pod-annotate-webhook
+        namespace: default
+        path: "/mutate"
+      caBundle: CA_BUNDLE
+    rules:
+      - operations: [ "CREATE" ]
+        apiGroups: [""]
+        apiVersions: ["v1"]
+        resources: ["pods"]

clr-k8s-examples/admit-kata/deploy/webhook.yaml

@@ -1,3 +1,7 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -13,11 +17,12 @@ spec:
     spec:
       containers:
         - name: pod-annotate-webhook
-          image: mcastelino/kubewebhook-pod-annotate-example:1.0
+          image: katadocker/kata-webhook-example:latest
           imagePullPolicy: Always
           args:
             - -tls-cert-file=/etc/webhook/certs/cert.pem
             - -tls-key-file=/etc/webhook/certs/key.pem
+            - -exclude-namespaces=rook-ceph-system,rook-ceph
           volumeMounts:
             - name: webhook-certs
               mountPath: /etc/webhook/certs

clr-k8s-examples/admit-kata/versions

@@ -0,0 +1,2 @@
+https://github.com/kata-containers/tests/tree/master/kata-webhook
+Commit: 5ad2cec

clr-k8s-examples/admit-kata/webhook-certs.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-data:
-  cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd2hVVVBKbnZ2Y2c0SUhueHdrSjJGYkJXc3NWb0I4dWp4S3B4dlZvdnRERFdLTlVOCmNoejNtQXRxQm9pcDBVSWpyWm1SMGd0b3FhSThSaXhxY1A5b1V5cDFTUk1UK2E3VXk2OG9LN3dRdjJ5eUI2eDIKMTFTaWNraEdpbzFNSlA3TEJFekZSVjlJb21yY2ZKdm4vdUk0Vk1lZnJjWXBqdlhwTjh0VHU3VkFnL1JMODZXYwpJeDhURWVtTko2QStRZ1Y3RUJLTVlQUTAwM0Z6NUdkR1ovOXNoUTVGQ0hCdUFYd2lRc0x6elArTlJFRmhtczJnCkdWT3FVeUFQR1FkWVlVRlZUd0tLbGlvRTVtcldTRTBMZnpDS3M2MUdwZ0pwWWNpN0RVdWRQelM4SHJUU2k1WXUKN002NlAwUThwaGZpN2NvcmJFcHQrci9DZTVHaTRXVlVLd3dSZlFJREFRQUJBb0lCQUNsY3NxNWpwQ3RQRTE4ZwpmZnlIaTREOXpzeFVzK0lOYlQ2SmtLbnBJWWVHYk8zUTBnZkMwcVdOc1MvcFNqKzFsOEwwbXBZb0prc1lyejNKClUzbjNoTFZNdENnNXI2VWp0R2dnVVRCWTZUelNubkp5Ti9XV0xQU3NJSlNBbm55MzdHNWpLaHVwdmVJWFlod1EKWWZYNUlrWTNNUG1vNU14NTdoVWMrU0JrOVhYcGFIOGZTbTVFeFl1Zkd5WVpsWWh1L1BDdSt2ZWlYRXZHbFRheQpYZmZUUmkvQjFOTlhha3JLTzY1RjBPMks3U1ljNkx3Q3krRVU4UDA0ZXR2NEoxOFdXVHJqZ01Tenh3OUNZb1BLClNsT0xLVE1xeXptQmtETGV1NURNUGh3U1dia0Y5ejArQS94bTNjcVp1cC9yczIxYWxCa29TTTZUNGx0WmR4NUkKc21pR3NPVUNnWUVBNTE1Vy9veXNLYjJpSVZWOFFhb09LWjFsQWlJaDh2d3NsMUFRbzF3QW00VStrRnFqSVlIUgp1UDBmQjFlZW5jSHFkdm1DOEZrNnhhTGZxL2RxNGtHaHF6MDVHdjFoUDVyRXlTVXFxYmFYTGEvVUgzN2doVkJjCmsyclVtTmJYV2l2NWtJVzh4T1hOSklUenIzUUo1Y0VQMjNrVE96UWNwN3AwWHlzU05YU3RSOWNDZ1lFQTFyNk4KRWdWajNlY3hFR2tvVFB2aXc3d2E0Ti83UXFCaytWNTZPQytNamIvNHJiWDBNeEtlQjlPRDc1OXlPYlF4YjhxRgpweFZCdGVSTUN2OWtEcDJmTHBWTlB0YVBvQit3RUNQVm1aa2k5cFpITVUwMGw0QU9uVWM2SVZmZzd0cWllTzYyCnFJb20vWnVINXBlU2t2QjJVbktZTnNZdlA2VnNMaVRITStEUmRzc0NnWUVBamwxb1hqMGsrcElySHlQQXo0N2EKSkJVclFBTE1yUDBxV3ZqekkvSEtsZWVKTmIzdnZ3Qm9rZEYrdEQvZjQrYWNaUlRtMHdtMHRrT2dLZXFXSkI2SwpZaG5MOTZXTm4xdVdWc2E5MTZ2NG5pNGc4amhaNHU0dDZLL0ZuVDRsU09EaU1XRjVaQmFiQWl1azNvTWlTL05kCjFJaE5veEpQeDQxZGFlblF6SCs4MkFVQ2dZRUF2NjRDdWEwNkFSNlYrdklDV1FVVTJtWVREOXFkcWxFRkVGbTIKZW1SbTd6Z0Z2dmlFNnZtWk9aOGhTMGhsYXdCZWlFeWJsQkl6UHlweWZmYU0xMGIyaVZ3WFFSbS94Y3ZER2dVQwpha0g0cFdacVVhVjZaaWlWNHVsckI4d3JLTWphOFZzU2k4b1ZGNVkyYml1cFY2TnYyaFFUcmdDa3VBanVVUm5lCi9YMlZPcFVDZ1lCY2M5R1c5SlhLT242aGNML3lub1NiL3pQYTNrM3lFMlZVNWZiamZpK0JQVWxKRzg5T1JDSU0KSGo1c2lVN2pEeGk2VmM5eGo3dDNNVXh4cXlKYTBWaVd5bGZ6WkVzSVNUQmpub04xbSszM3VqbTV6RSs3NGxQUwpPUm4yNWhrTlIvejY0QWZiZjd6Si9jaEVZSndoQS9zK0NjcVptQ0VWaEc4RjgranRyLzhUQVE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
-kind: Secret
-metadata:
-  creationTimestamp: null
-  name: pod-annotate-webhook-certs

clr-k8s-examples/admit-kata/webhook-registration.yaml

@@ -1,24 +0,0 @@
-apiVersion: admissionregistration.k8s.io/v1beta1
-kind: MutatingWebhookConfiguration
-metadata:
-  name: pod-annotate-webhook
-  labels:
-    app: pod-annotate-webhook
-    kind: mutator
-webhooks:
-  - name: pod-annotate-webhook.pod.xyz
-    clientConfig:
-      service:
-        name: pod-annotate-webhook
-        namespace: default
-        path: "/mutate"
-      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-    rules:
-      - operations: [ "CREATE" ]
-        apiGroups: [""]
-        apiVersions: ["v1"]
-        resources: ["pods"]
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-

clr-k8s-examples/containerd_devmapper_setup.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+sudo rm -rf /var/lib/containerd/devmapper/data-disk.img
+sudo rm -rf /var/lib/containerd/devmapper/meta-disk.img
+sudo mkdir -p /var/lib/containerd/devmapper
+sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img
+sudo truncate --size 10G /var/lib/containerd/devmapper/meta-disk.img
+
+sudo mkdir -p /etc/systemd/system
+
+cat<<EOT | sudo tee /etc/systemd/system/containerd-devmapper.service
+[Unit]
+Description=Setup containerd devmapper device
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=-/sbin/losetup /dev/loop20 /var/lib/containerd/devmapper/data-disk.img
+ExecStart=-/sbin/losetup /dev/loop21 /var/lib/containerd/devmapper/meta-disk.img
+
+[Install]
+WantedBy=local-fs.target
+EOT
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now containerd-devmapper
+
+# Time to setup the thin pool for consumption.
+# The table arguments are such.
+# start block in the virtual device
+# length of the segment (block device size in bytes / Sector size (512)
+# metadata device
+# block data device
+# data_block_size Currently set it 512 (128KB)
+# low_water_mark. Copied this from containerd snapshotter test setup
+# no. of feature arguments
+# Skip zeroing blocks for new volumes.
+sudo dmsetup create contd-thin-pool \
+  --table "0 2097152 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
+
+sudo mkdir -p /etc/containerd/
+if [ -f /etc/containerd/config.toml ]
+then
+  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"512MB\"|' /etc/containerd/config.toml
+else
+  cat<<EOT | sudo tee /etc/containerd/config.toml
+[plugins]
+  [plugins.devmapper]
+    pool_name = "contd-thin-pool"
+    base_image_size = "512MB"
+EOT
+fi
+
+sudo systemctl restart containerd

clr-k8s-examples/create_stack.sh

@@ -6,75 +6,139 @@ set -o nounset
 
 CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_DIR
 
-function finish {
+function print_usage_exit() {
-  cd $CUR_DIR
+	exit_code=${1:-0}
+	cat <<EOT
+Usage: $0 [subcommand]
+
+Subcommands:
+
+$(
+		for cmd in "${!command_handlers[@]}"; do
+			printf "\t%s:|\t%s\n" "${cmd}" "${command_help[${cmd}]:-Not-documented}"
+		done | sort | column -t -s "|"
+	)
+EOT
+	exit "${exit_code}"
+}
+
+function finish() {
+	cd "${CUR_DIR}"
 }
 trap finish EXIT
 
-#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
+function cluster_init() {
-#to enable the RuntimeClass featuregate
+	#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
-sudo -E kubeadm init --config=./kubeadm.yaml
+	#to enable the RuntimeClass featuregate
+	sudo -E kubeadm init --config=./kubeadm.yaml
 
-# If this an interactive terminal then wait for user to join workers
+	rm -rf "${HOME}/.kube"
-if [ -t 0 ]; then
+	mkdir -p "${HOME}/.kube"
-read -p "Join other nodes. Press enter to continue"
+	sudo cp -i /etc/kubernetes/admin.conf "${HOME}/.kube/config"
+	sudo chown "$(id -u):$(id -g)" "${HOME}/.kube/config"
+
+	# If this an interactive terminal then wait for user to join workers
+	if [ -t 0 ]; then
+		read -p "Join other nodes. Press enter to continue"
+	fi
+
+	#Ensure single node k8s works
+	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
+		kubectl taint nodes --all node-role.kubernetes.io/master-
+	fi
+}
+
+function kata() {
+	# Install kata artifacts using kata-deploy
+	kubectl apply -f 8-kata/deploy/kata-rbac.yaml
+	kubectl apply -f 8-kata/deploy/kata-deploy.yaml
+	kubectl apply -f 8-kata/
+}
+
+function cni() {
+	kubectl apply -f 0-canal/rbac.yaml
+	kubectl apply -f 0-canal/canal.yaml
+}
+
+function metrics() {
+	kubectl apply -f 1-core-metrics/
+}
+
+function storage() {
+	#Start rook before any other component that requires storage
+	ROOK_URL=7-rook
+	kubectl apply -f ${ROOK_URL}/000-operator.yaml
+	while [[ $(kubectl get crd clusters.ceph.rook.io pools.ceph.rook.io >/dev/null 2>&1) || $? -ne 0 ]]; do
+		echo "Waiting for Rook CRDs"
+		sleep 2
+	done
+	kubectl apply -f ${ROOK_URL}/001-cluster.yaml
+	kubectl apply -f ${ROOK_URL}/002-storageclass.yaml
+}
+
+function monitoring() {
+	#Just to allow the CRD to be created. Ideally wait and then run second time
+	kubectl apply -f 4-kube-prometheus/
+	while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com >/dev/null 2>&1) || $? -ne 0 ]]; do
+		echo "Waiting for Prometheus CRDs"
+		sleep 2
+	done
+	kubectl apply -f 4-kube-prometheus/
+
+	#Expose the dashboards
+	#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
+	#kubectl --namespace monitoring port-forward svc/grafana 3000 &
+	#kubectl --namespace monitoring port-forward svc/alertmanager-main 9093 &
+}
+
+function miscellaneous() {
+	kubectl apply -f 2-dashboard/
+	kubectl apply -f 3-efk/
+	#Create an ingress load balancer
+	kubectl apply -f 5-ingres-lb/
+
+	#Create a bare metal load balancer.
+	#kubectl apply -f 6-metal-lb/metallb.yaml
+
+	#The config map should be properly modified to pick a range that can live
+	#on this subnet behind the same gateway (i.e. same L2 domain)
+	#kubectl apply -f 6-metal-lb/example-layer2-config.yaml
+}
+
+function minimal() {
+	cluster_init
+	cni
+	kata
+	metrics
+}
+
+function all() {
+	minimal
+	storage
+	monitoring
+	miscellaneous
+}
+
+declare -A command_handlers
+command_handlers[init]=cluster_init
+command_handlers[cni]=cni
+command_handlers[minimal]=minimal
+command_handlers[all]=all
+command_handlers[help]=print_usage_exit
+
+declare -A command_help
+command_help[init]="Only inits a cluster using kubeadm"
+command_help[cni]="Setup network for running cluster"
+command_help[minimal]="init + cni +  kata + metrics"
+command_help[all]="minimal + storage + monitoring + miscellaneous"
+command_help[help]="show this message"
+
+cd "${SCRIPT_DIR}"
+
+cmd_handler=${command_handlers[${1:-none}]:-unimplemented}
+if [ "${cmd_handler}" != "unimplemented" ]; then
+	"${cmd_handler}"
+else
+	print_usage_exit 1
 fi
-
-rm -rf $HOME/.kube
-mkdir -p $HOME/.kube
-sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-
-#Add support for kata runtime
-kubectl apply -f 8-kata/runtimeclass_crd.yaml
-while [[ $(kubectl get crd runtimeclasses.node.k8s.io > /dev/null 2>&1) || $? -ne 0 ]];
-do echo "Waiting for runtime class CRD"; sleep 2; done
-kubectl apply -f 8-kata/kata-runtimeClass.yaml
-
-kubectl apply -f 0-canal/rbac.yaml
-kubectl apply -f 0-canal/canal.yaml
-
-#Ensure single node k8s works
-if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
-kubectl taint nodes --all node-role.kubernetes.io/master-
-fi
-
-#Start rook before any other component that requires storage
-ROOK_URL=7-rook
-kubectl apply -f ${ROOK_URL}/000-operator.yaml
-while [[ $(kubectl get crd clusters.ceph.rook.io pools.ceph.rook.io > /dev/null 2>&1) || $? -ne 0 ]];
-do echo "Waiting for Rook CRDs"; sleep 2; done
-kubectl apply -f ${ROOK_URL}/001-cluster.yaml
-kubectl apply -f ${ROOK_URL}/002-storageclass.yaml
-
-kubectl apply -f 1-core-metrics/
-
-kubectl apply -f 2-dashboard/
-
-kubectl apply -f 3-efk/
-
-#Just to allow the CRD to be created. Ideally wait and then run second time
-kubectl apply -f 4-kube-prometheus/
-while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com > /dev/null 2>&1) || $? -ne 0 ]];
-do echo "Waiting for Prometheus CRDs"; sleep 2; done
-kubectl apply -f 4-kube-prometheus/
-#Create an ingress load balancer
-kubectl apply -f 5-ingres-lb/
-
-
-#Create a bare metal load balancer.
-#kubectl apply -f 6-metal-lb/metallb.yaml
-
-#The config map should be properly modified to pick a range that can live
-#on this subnet behind the same gateway (i.e. same L2 domain)
-#kubectl apply -f 6-metal-lb/example-layer2-config.yaml
-
-
-#Expose the dashboards
-#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
-#kubectl --namespace monitoring port-forward svc/grafana 3000 &
-#kubectl --namespace monitoring port-forward svc/alertmanager-main 9093 &
-#kubectl proxy &
-

clr-k8s-examples/kubeadm.yaml

@@ -1,19 +1,20 @@
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
-nodeRegistration:
-  criSocket: /var/run/crio/crio.sock
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-featureGates:
+# Allowing for CPU pinning and isolation in case of guaranteed QoS class
-  RuntimeClass: true
+cpuManagerPolicy: static
+systemReserved:
+  cpu: 500m
+  memory: 256M
+kubeReserved:
+  cpu: 500m
+  memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
-kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16
   serviceSubnet: 10.96.0.0/12
-apiServerExtraArgs:
-  feature-gates: RuntimeClass=true

clr-k8s-examples/reset_stack.sh

@@ -3,7 +3,10 @@
 set -o nounset
 
 #Cleanup
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster() {
+	sudo -E kubeadm reset -f
+}
+reset_cluster
 
 for ctr in $(sudo crictl ps --quiet); do
 	sudo crictl stop "$ctr"
@@ -14,11 +17,11 @@ for pod in $(sudo crictl pods --quiet); do
 	sudo crictl rmp "$pod"
 done
 
-
 #Forcefull cleanup all artifacts
-#This is needed is things really go wrong
+#This is needed if things really go wrong
 sudo systemctl stop kubelet
-sudo systemctl stop crio
+systemctl is-active crio && sudo systemctl stop crio
+systemctl is-active containerd && sudo systemctl stop containerd
 sudo pkill -9 qemu
 sudo pkill -9 kata
 sudo pkill -9 kube
@@ -40,9 +43,11 @@ sudo -E bash -c "rm -r /var/run/kata-containers/*"
 sudo rm -rf /var/lib/rook
 
 sudo systemctl daemon-reload
-sudo systemctl enable kubelet crio
+sudo systemctl is-active crio && sudo systemctl stop crio
-sudo systemctl restart crio
+sudo systemctl is-active containerd && sudo systemctl stop containerd
+sudo systemctl is-enabled crio && sudo systemctl restart crio
+sudo systemctl is-enabled containerd && sudo systemctl restart containerd
+
 sudo systemctl restart kubelet
 
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster
-

clr-k8s-examples/setup_kata_firecracker.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Firecracker can only work with devicemapper
+# Setup a sparse disk to be used for devicemapper
+sudo rm -f /var/lib/crio/devicemapper/disk.img
+sudo mkdir -p /var/lib/crio/devicemapper
+sudo truncate /var/lib/crio/devicemapper/disk.img --size 10G
+
+# Ensure that this disk is loop mounted at each boot
+sudo mkdir -p /etc/systemd/system
+
+cat <<EOT | sudo tee /etc/systemd/system/devicemapper.service
+[Unit]
+Description=Setup CRIO devicemapper
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=-/sbin/losetup /dev/loop8 /var/lib/crio/devicemapper/disk.img
+RemainAfterExit=true
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+EOT
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now devicemapper
+
+sudo sed -i 's/storage_driver = \"overlay\"/storage_driver = \"devicemapper\"\
+storage_option = [\
+  \"dm.basesize=8G\",\
+  \"dm.directlvm_device=\/dev\/loop8\",\
+  \"dm.directlvm_device_force=true\",\
+  \"dm.override_udev_sync_check=true",\
+  \"dm.fs=ext4\"\
+]/g' /etc/crio/crio.conf
+
+sudo systemctl restart crio || true

clr-k8s-examples/setup_system.sh

@@ -5,9 +5,10 @@ set -o nounset
 
 ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12"
 ADD_NO_PROXY+=",$(hostname -I | sed 's/[[:space:]]/,/g')"
+: ${RUNNER:="containerd"}
 
 #Install kubernetes and crio
-sudo -E swupd update
+sudo swupd update
 sudo -E swupd bundle-add cloud-native-basic storage-utils
 
 #Permanently disable swap
@@ -20,22 +21,26 @@ else
 fi
 
 sudo mkdir -p /etc/sysctl.d/
-cat <<EOT | sudo tee /etc/sysctl.d/60-k8s.conf
+cat <<EOT | sudo bash -c "cat > /etc/sysctl.d/60-k8s.conf"
 net.ipv4.ip_forward=1
 EOT
 sudo systemctl restart systemd-sysctl
 
 #Ensure the modules we need are preloaded
 sudo mkdir -p /etc/modules-load.d/
-cat <<EOT | sudo tee /etc/modules-load.d/k8s.conf
+cat <<EOT | sudo bash -c "cat > /etc/modules-load.d/k8s.conf"
 br_netfilter
 vhost_vsock
 overlay
 EOT
 
+# Make sure /etc/hosts file exists
+if [ ! -f /etc/hosts ]; then
+  sudo touch /etc/hosts
+fi
 hostcount=$(grep '127.0.0.1 localhost' /etc/hosts | wc -l)
 if [ "$hostcount" == "0" ]; then
-	echo "127.0.0.1 localhost `hostname`" | sudo tee -a /etc/hosts
+	echo "127.0.0.1 localhost $(hostname)" | sudo bash -c "cat >> /etc/hosts"
 else
 	echo "/etc/hosts already configured"
 fi
@@ -43,32 +48,40 @@ fi
 sudo systemctl daemon-reload
 # This will fail at this point, but puts it into a retry loop that
 # will therefore startup later once we have configured with kubeadm.
-sudo systemctl enable --now kubelet crio || true
+echo "The following kubelet command may complain... it is not an error"
+sudo systemctl enable --now kubelet $RUNNER || true
 
-sudo mkdir -p /usr/libexec/cni /opt/cni
-sudo ln -s /usr/libexec/cni /opt/cni/bin
 #Ensure that the system is ready without requiring a reboot
 sudo swapoff -a
-sudo modprobe br_netfilter vhost_vsock overlay
+sudo systemctl restart systemd-modules-load.service
 
 set +o nounset
 if [[ ${http_proxy} ]] || [[ ${HTTP_PROXY} ]]; then
-  echo "Setting up proxy stuff...."
+	echo "Setting up proxy stuff...."
-  # Setup IP for users too
+	# Setup IP for users too
-  sed_val=${ADD_NO_PROXY//\//\\/}
+	sed_val=${ADD_NO_PROXY//\//\\/}
-  sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
+	[ -f /etc/environment ] && sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
-  sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+	if [ -f /etc/profile.d/proxy.sh ]; then
+		sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+	else
+		echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
+	fi
 
-  services=('crio' 'docker' 'kubelet')
+	services=($RUNNER 'kubelet')
-  for s in "${services[@]}"; do
+	for s in "${services[@]}"; do
-    sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
+		sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
-    cat << EOF | sudo tee "/etc/systemd/system/${s}.service.d/proxy.conf"
+		cat <<EOF | sudo bash -c "cat > /etc/systemd/system/${s}.service.d/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="SOCKS_PROXY=${socks_proxy}"
 Environment="NO_PROXY=${no_proxy},${ADD_NO_PROXY}"
 EOF
-  done
+	done
 fi
 set -o nounset
+
+# We have potentially modified their env files, we need to restart the services.
+sudo systemctl daemon-reload
+sudo systemctl restart $RUNNER || true
+sudo systemctl restart kubelet || true

clr-k8s-examples/tests/autoscale/load-gen.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: load-generator
+spec:
+  containers:
+  - command: ["/bin/sh", "-c"]
+    args:
+    - while true; do wget -q -O- http://php-apache-test; done;
+    image: busybox
+    imagePullPolicy: Always
+    name: load-generator

clr-k8s-examples/tests/autoscale/php.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-test
+  name: php-apache-test
+spec:
+  selector:
+    matchLabels:
+      run: php-apache-test
+  template:
+    metadata:
+      labels:
+        run: php-apache-test
+    spec:
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        name: php-apache-test
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: php-apache-test
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-test
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: php-apache-test
+spec:
+  maxReplicas: 10
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: php-apache-test
+  targetCPUUtilizationPercentage: 50
+

clr-k8s-examples/tests/autoscale/test-autoscale.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+kubectl apply -f $SCRIPT_DIR
+watch kubectl describe hpa

clr-k8s-examples/tests/cpumanager/test-cpumanager.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+input="test-cpumanager.yaml.tmpl"
+
+filename() {
+	echo "test-cpumanager-$1.yaml"
+}
+
+for runtimeclass in runc kata-qemu kata-fc; do
+	output=$(filename $runtimeclass)
+	cp $input $output
+	sed -i "s/__runtimeclass__/$runtimeclass/g" $output
+	if [ $runtimeclass == "runc" ]; then continue; fi
+
+	insertline="\ \ runtimeClassName: $runtimeclass"
+	sed -i "/spec:/a $insertline" $output
+done
+kubectl apply -f .

clr-k8s-examples/tests/cpumanager/test-cpumanager.yaml.tmpl

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-guaranteed-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        cpu: 1
+        memory: 500Mi # For kata to run
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+      limits:
+        cpu: 2
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+      limits:
+        cpu: 1
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-besteffort-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-fc.yaml

@@ -2,22 +2,19 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    run: php-apache-kata
+    run: php-apache-kata-fc
-  name: php-apache-kata
+  name: php-apache-kata-fc
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: php-apache-kata
+      run: php-apache-kata-fc
   template:
     metadata:
-      annotations:
-        io.kubernetes.cri-o.TrustedSandbox: "false"
-        io.kubernetes.cri.untrusted-workload: "true"
       labels:
-        run: php-apache-kata
+        run: php-apache-kata-fc
     spec:
-      runtimeClassName: kata
+      runtimeClassName: kata-fc
       containers:
       - image: k8s.gcr.io/hpa-example
         imagePullPolicy: Always
@@ -33,13 +30,13 @@ spec:
 apiVersion: v1 
 kind: Service
 metadata:
-  name: php-apache-kata
+  name: php-apache-kata-fc
 spec:
   ports:
   - port: 80
     protocol: TCP
     targetPort: 80
   selector:
-    run: php-apache-kata
+    run: php-apache-kata-fc
   sessionAffinity: None
   type: ClusterIP

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-qemu.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-kata-qemu
+  name: php-apache-kata-qemu
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: php-apache-kata-qemu
+  template:
+    metadata:
+      labels:
+        run: php-apache-kata-qemu
+    spec:
+      runtimeClassName: kata-qemu
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        imagePullPolicy: Always
+        name: php-apache
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1 
+kind: Service
+metadata:
+  name: php-apache-kata-qemu
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-kata-qemu
+  sessionAffinity: None
+  type: ClusterIP

clr-k8s-examples/tests/e2e/run_e2e.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Runs upstream k8s e2e tests against existing cloud native basic cluster
+# Requires cluster to already be up
+# To specify a parameter for --ginkgo.focus as described below, provide a focus as the first argument to this script
+#     https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#building-kubernetes-and-running-the-tests
+# One example would be Feature:Performance. The script will add square brackets for you
+# For other examples of values, see
+#     https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#kinds-of-tests
+
+set -o errexit
+set -o pipefail
+
+if [ ! -z $1 ]
+then
+    FOCUS=$1
+    echo Running e2e tests where spec matches $1
+else
+    echo Running all e2e tests, this will take a long time
+fi
+
+GO_INSTALLED=$(sudo swupd bundle-list | grep go-basic)
+if [ -z $GO_INSTALLED ]
+then
+    echo Installing go-basic bundle
+    sudo swupd bundle-add go-basic
+else
+    echo Skipping go-basic bundle installation
+fi
+
+if [ -z $GOPATH ] ; then GOPATH=$HOME/go; fi
+if [ -z $GOBIN ] ; then GOBIN=$HOME/go/bin; fi
+
+echo Getting kubetest
+go get -u k8s.io/test-infra/kubetest
+
+cd $GOPATH/src/k8s.io
+
+if [ -d kubernetes ]
+then
+    cd kubernetes
+    echo Checking status of existing k8s repo clone
+    git status kubernetes
+else
+    echo Cloning upstream k8s repo
+    git clone https://github.com/kubernetes/kubernetes.git
+    cd kubernetes
+fi
+
+PATH=$PATH:$GOBIN
+
+API_SERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"kubernetes\")].cluster.server}")
+CLIENT_VERSION=$(kubectl version --short | grep -E 'Client' | sed 's/Client Version: //')
+
+echo Running kubetest
+
+if [ -z $FOCUS ]
+then
+    echo sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER" --extract=$CLIENT_VERSION --provider=local 
+    sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER" --extract=$CLIENT_VERSION --provider=local
+else
+    echo sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER --ginkgo.focus=\[$FOCUS\]" --extract=$CLIENT_VERSION --provider=local
+    sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER --ginkgo.focus=\[$FOCUS\]" --extract=$CLIENT_VERSION --provider=local
+fi
+

clr-k8s-examples/tests/test-autoscale.sh

@@ -1,6 +0,0 @@
-kubectl run php-apache-test --image=k8s.gcr.io/hpa-example --requests=cpu=200m --expose --port=80
-kubectl autoscale deployment php-apache-test --cpu-percent=50 --min=1 --max=10
-kubectl get hpa
-
-#kubectl run -i --tty load-generator --image=busybox /bin/sh
-# while true; do wget -q -O- http://php-apache-test.default.svc.cluster.local; done