cri-o update fixes

ClearLinux now has cri-o version 1.14.1 which allows multiple plugin
locations and also creates /opt/cni/bin by default. We no longer need
the hacks for them

Fixes: #82
Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

clr-k8s-examples/7-rook/000-operator.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph-system
-  labels:
-    kata: "false"
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition

clr-k8s-examples/7-rook/001-cluster.yaml

@@ -2,8 +2,6 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: rook-ceph
-  labels:
-    kata: "false"
 ---
 apiVersion: v1
 kind: ServiceAccount

clr-k8s-examples/8-kata/deploy/kata-cleanup.yaml

@@ -0,0 +1,46 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubelet-kata-cleanup
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kubelet-kata-cleanup
+  template:
+    metadata:
+        labels:
+          name: kubelet-kata-cleanup
+    spec:
+      serviceAccountName: kata-label-node
+      nodeSelector:
+          katacontainers.io/kata-runtime: cleanup
+      containers:
+      - name: kube-kata-cleanup
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh reset" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      volumes:
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-deploy.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kata-deploy
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: kata-deploy
+  template:
+    metadata:
+        labels:
+          name: kata-deploy
+    spec:
+      serviceAccountName: kata-label-node
+      containers:
+      - name: kube-kata
+        image: katadocker/kata-deploy
+        imagePullPolicy: Always
+        lifecycle:
+          preStop:
+            exec:
+              command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"]
+        command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install" ]
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: crio-conf
+          mountPath: /etc/crio/
+        - name: containerd-conf
+          mountPath: /etc/containerd/
+        - name: kata-artifacts
+          mountPath: /opt/kata/
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+        - name: local-bin
+          mountPath: /usr/local/bin/
+      volumes:
+        - name: crio-conf
+          hostPath:
+            path: /etc/crio/
+        - name: containerd-conf
+          hostPath:
+            path: /etc/containerd/
+        - name: kata-artifacts
+          hostPath:
+            path: /opt/kata/
+            type: DirectoryOrCreate
+        - name: dbus
+          hostPath:
+            path: /var/run/dbus
+        - name: systemd
+          hostPath:
+            path: /run/systemd
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin/
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

clr-k8s-examples/8-kata/deploy/kata-rbac.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kata-label-node
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: node-labeler
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kata-label-node-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-labeler
+subjects:
+- kind: ServiceAccount
+  name: kata-label-node
+  namespace: kube-system
+

clr-k8s-examples/8-kata/kata-fc-runtimeClass.yaml

@@ -1,6 +1,5 @@
 kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
+apiVersion: node.k8s.io/v1beta1
 metadata:
     name: kata-fc
-spec:
+handler: kata-fc
-    runtimeHandler: kata-fc

clr-k8s-examples/8-kata/kata-qemu-runtimeClass.yaml

@@ -1,14 +1,12 @@
 ---
 kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
+apiVersion: node.k8s.io/v1beta1
 metadata:
     name: kata
-spec:
+handler: kata
-    runtimeHandler: kata-qemu
 ---
 kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
+apiVersion: node.k8s.io/v1beta1
 metadata:
     name: kata-qemu
-spec:
+handler: kata-qemu
-    runtimeHandler: kata-qemu

clr-k8s-examples/8-kata/runtimeclass_crd.yaml

@@ -1,26 +0,0 @@
-kind: CustomResourceDefinition
-apiVersion: apiextensions.k8s.io/v1beta1
-metadata:
-  name: runtimeclasses.node.k8s.io
-  labels:
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  group: node.k8s.io
-  version: v1alpha1
-  versions:
-    - name: v1alpha1
-      served: true
-      storage: true
-  names:
-    plural:   runtimeclasses
-    singular: runtimeclass
-    kind:     RuntimeClass
-  scope: Cluster
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            runtimeHandler:
-              type: string
-              pattern: '^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)?$'

clr-k8s-examples/9-multi-network/README.md

@@ -15,6 +15,22 @@ file is provided, which enables SR-IOV for the above `rootDevices`
 
 > NOTE: This assumes homogenous nodes in the cluster
 
+### Pre-req (SR-IOV only)
+
+One each SR-IOV node make sure `VT-d` is enabled in the BIOS and `intel_iommu=on` on kernel commandline.
+Setup systemd to bring up VFs on designated interfaces bound to network driver or `vfio-pci`
+
+```bash
+# Make sure vfio-pci is loaded on boot
+echo 'vfio-pci' | sudo tee /etc/modules-load.d/sriov.conf
+sudo systemctl restart systemd-modules-load.service
+
+sudo cp systemd/sriov.sh /usr/bin/sriov.sh
+sudo cp systemd/sriov.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now sriov.service
+```
+
 ### Install
 
 To install and configure `multus-cni` on all nodes, along with

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -140,10 +140,8 @@ spec:
         - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
           /tmp/multus/install-multus-conf.sh;
           /tmp/multus/install-certs.sh;
-          systemctl stop kubelet;
+          echo "Restarting crio kubelet";
-          echo "Restarting crio/containerd, kubelet";
+          systemctl restart crio; # Needed when crio manages ns lifecycle
-          systemctl restart containerd;
-          systemctl restart crio;
           systemctl restart kubelet;
         volumeMounts:
         - name: usr-bin

clr-k8s-examples/README.md

@@ -1,5 +1,8 @@
 # How to setup the cluster
 
+## Prerequisite
+This setup currently will work with k8s 1.14 & above. Any version of k8s before that might work, but is not guaranteed.
+
 ## Sample multi-node vagrant setup
 
 To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
@@ -8,6 +11,7 @@ To be able to test this tool, you can create a 3-node vagrant setup. In this tut
 * `vagrant up --provider=libvirt`
 
 Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
+Customize the setup using environment variables. E.g., `NODES=1 MEMORY=8192 CPUS=8 vagrant up --provider=libvirt`
 
 To login to the master node and change to this directory
 
@@ -46,14 +50,12 @@ master and also uses kubelet config via [`kubeadm.yaml`](kubeadm.yaml)
 to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
-There are two flavors of install -
+There are different flavors to install, run `./create_stack.sh help` to get
-
+more information.
-* `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
-* `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
 
 ```bash
-# default is 'all'
+# default shows help
-./create_stack.sh [minimal|all]
+./create_stack.sh <subcommand>
 ```
 
 ## Join Workers to the cluster
@@ -91,43 +93,12 @@ An example is
 
 `kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-fc.yaml`
 
-## Making Kata the default runtime
+## Making Kata the default runtime using admission controller
 
-Today in `crio.conf` runc is the default runtime when a user does not specify
+If you want to run a cluster where kata is used
-`runtimeClass` in the pod spec. If you want to run a cluster where kata is used
+by default, except for workloads we know for sure will not work with kata, using
-by default, except for workloads we know for sure will not work with kata, use
+[admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
-the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
+and sample admission controller, follow `admit-kata` [README.md](admit-kata/README.md)
-and sample admission controller we created by running -
-
-`kubectl apply -f admit-kata/`
-
-The [admission webhook](admit-kata/webhook-registration.yaml)
-is setup to exclude certian namespaces from being run with Kata using filters on namespace labels.
-
-```yaml
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-```
-
-The rook operators for example are marked as such
-
-```yaml
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: rook-ceph-system
-  labels:
-    kata: "false"
-```
-
-Pods not explicitly excluded by the namespace filter are dynamically tagged to
-run with Kata with some [exceptions](https://github.com/mcastelino/kubewebhook/blob/topic/hack-kata/examples/pod-annotate/main.go#L25) -
-
-* `hostNetwork: true`
-* `rook-ceph` and `rook-ceph-system` namespaces (buggy)
-
-Other pod properties will be added as exceptions in future.
 
 ## Accessing control plane services
 

clr-k8s-examples/Vagrantfile

@@ -10,17 +10,16 @@ $cpus = (ENV['CPUS'] || 2).to_i
 $memory = (ENV['MEMORY'] || 4096).to_i
 $disks = 2
 # Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
-disk_prefix = File.basename(File.dirname(__FILE__), "/")
+$disk_prefix = File.basename(File.dirname(__FILE__), "/")
-#DISK_UUID = SecureRandom.urlsafe_base64(9)
 $disk_size = "10G"
 $box = "AntonioMeireles/ClearLinux"
-$loader = File.join(File.dirname(__FILE__), "OVMF.fd")
+File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("192.52.100.10")
-hosts = {}
+$hosts = {}
-proxy_ip_list = ""
+$proxy_ip_list = ""
-#DISK_UUID = Time.now.utc.to_i
+$driveletters = ('a'..'z').to_a
-driveletters = ('a'..'z').to_a
+$setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
 
 if not File.exists?($loader)
   system('curl -O https://download.clearlinux.org/image/OVMF.fd')
@@ -47,38 +46,43 @@ Vagrant.configure("2") do |config|
 
   # Mount the current dir at home folder instead of default
   config.vm.synced_folder './', '/vagrant', disabled: true
-  config.vm.synced_folder './', '/home/clear/' + File.basename(Dir.getwd), type: 'rsync'
+  config.vm.synced_folder './', '/home/clear/' + File.basename(Dir.getwd), type: 'rsync',
+    rsync__args: ["--verbose", "--archive", "--delete", "-zz", "--copy-links"]
   #Setup proxies for all machines
   (1..$num_instances).each do |i|
-    base_ip = base_ip.succ
+    $base_ip = $base_ip.succ
-    hosts["clr-%02d" % i] = base_ip.to_s
+    $hosts["clr-%02d" % i] = $base_ip.to_s
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     proxy_ip_list = ("#{proxy_ip_list},#{vm_name},#{ip}")
   end
 
-  hosts.each do |vm_name, ip|
+  $hosts.each do |vm_name, ip|
     config.vm.define vm_name do |c|
       c.vm.hostname = vm_name
       c.vm.network :private_network, ip: ip, autostart: true
       c.vm.provider :libvirt do |lv|
-        lv.loader = $loader
         lv.cpu_mode = "host-passthrough"
         lv.nested = true
+        lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
         (1..$disks).each do |d|
-          lv.storage :file, :device => "hd#{driveletters[d]}", :path => "disk-#{disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
+          lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end
       end
-      if Vagrant.has_plugin?("vagrant-proxyconf")
+      if ENV['http_proxy'] || ENV['HTTP_PROXY']
-        c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
+        if Vagrant.has_plugin?("vagrant-proxyconf")
-        c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
+          c.proxy.http = (ENV['http_proxy']||ENV['HTTP_PROXY'])
-        c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+          c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
+          c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
+        end
       end
       c.vm.provision "shell", privileged: false, path: "setup_system.sh"
-      c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+      if $setup_fc
+        c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+      end
       # Include shells bundle to get bash completion and add kubectl's commands to vagrant's shell
       c.vm.provision "shell", privileged: false, inline: 'sudo -E swupd bundle-add shells; echo "source <(kubectl completion bash)" >> $HOME/.bashrc'
     end

clr-k8s-examples/admit-kata/README.md

@@ -0,0 +1,39 @@
+# Kata Admission controller webhook
+
+Implement a simple admission controller webhook to annotate pods with the 
+Kata runtime class.
+
+## How to build the admission controller
+
+First build the admission controller image and the associated
+Kubernetes yaml files required to instantiate the admission
+controller.
+
+```bash
+$ docker build -t katadocker/kata-webhook-example:latest .
+$ ./create_certs.sh
+```
+
+> **Note:**
+> Image needs to be published for the webhook needs to work. Alternately
+> on a single machine cluster change the `imagePullPolicy` to use the locally
+> built image.
+
+## Making Kata the default runtime using an admission controller
+
+Today in `crio.conf` `runc` is the default runtime when a user does not specify
+`runtimeClass` in the pod spec. If you want to run a cluster where Kata is used
+by default, except for workloads we know for sure will not work with Kata, use
+the [admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#admission-webhooks)
+and sample admission controller we created by running
+
+```bash
+$ kubectl apply -f deploy/
+```
+
+The webhook mutates pods to use the kata runtime class for all pods except
+those with 
+
+* `hostNetwork: true` 
+* namespace: `rook-ceph` and `rook-ceph-system`
+

clr-k8s-examples/admit-kata/create-certs.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+WEBHOOK_NS=${1:-"default"}
+WEBHOOK_NAME=${2:-"pod-annotate"}
+WEBHOOK_SVC="${WEBHOOK_NAME}-webhook"
+
+# Create certs for our webhook
+openssl genrsa -out webhookCA.key 2048
+openssl req -new -key ./webhookCA.key -subj "/CN=${WEBHOOK_SVC}.${WEBHOOK_NS}.svc" -out ./webhookCA.csr 
+openssl x509 -req -days 365 -in webhookCA.csr -signkey webhookCA.key -out webhook.crt
+
+# Create certs secrets for k8s
+kubectl create secret generic \
+    ${WEBHOOK_SVC}-certs \
+    --from-file=key.pem=./webhookCA.key \
+    --from-file=cert.pem=./webhook.crt \
+    --dry-run -o yaml > ./deploy/webhook-certs.yaml
+
+# Set the CABundle on the webhook registration
+CA_BUNDLE=$(cat ./webhook.crt | base64 -w0)
+sed "s/CA_BUNDLE/${CA_BUNDLE}/" ./deploy/webhook-registration.yaml.tpl > ./deploy/webhook-registration.yaml
+
+# Clean
+rm ./webhookCA* && rm ./webhook.crt

clr-k8s-examples/admit-kata/deploy/webhook-registration.yaml.tpl

@@ -0,0 +1,24 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: pod-annotate-webhook
+  labels:
+    app: pod-annotate-webhook
+    kind: mutator
+webhooks:
+  - name: pod-annotate-webhook.kata.xyz
+    clientConfig:
+      service:
+        name: pod-annotate-webhook
+        namespace: default
+        path: "/mutate"
+      caBundle: CA_BUNDLE
+    rules:
+      - operations: [ "CREATE" ]
+        apiGroups: [""]
+        apiVersions: ["v1"]
+        resources: ["pods"]

clr-k8s-examples/admit-kata/deploy/webhook.yaml

@@ -1,3 +1,7 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -18,6 +22,7 @@ spec:
           args:
             - -tls-cert-file=/etc/webhook/certs/cert.pem
             - -tls-key-file=/etc/webhook/certs/key.pem
+            - -exclude-namespaces=rook-ceph-system,rook-ceph
           volumeMounts:
             - name: webhook-certs
               mountPath: /etc/webhook/certs

clr-k8s-examples/admit-kata/versions

@@ -0,0 +1,2 @@
+https://github.com/kata-containers/tests/tree/master/kata-webhook
+Commit: 5ad2cec

clr-k8s-examples/admit-kata/webhook-certs.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-data:
-  cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-  key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBd2hVVVBKbnZ2Y2c0SUhueHdrSjJGYkJXc3NWb0I4dWp4S3B4dlZvdnRERFdLTlVOCmNoejNtQXRxQm9pcDBVSWpyWm1SMGd0b3FhSThSaXhxY1A5b1V5cDFTUk1UK2E3VXk2OG9LN3dRdjJ5eUI2eDIKMTFTaWNraEdpbzFNSlA3TEJFekZSVjlJb21yY2ZKdm4vdUk0Vk1lZnJjWXBqdlhwTjh0VHU3VkFnL1JMODZXYwpJeDhURWVtTko2QStRZ1Y3RUJLTVlQUTAwM0Z6NUdkR1ovOXNoUTVGQ0hCdUFYd2lRc0x6elArTlJFRmhtczJnCkdWT3FVeUFQR1FkWVlVRlZUd0tLbGlvRTVtcldTRTBMZnpDS3M2MUdwZ0pwWWNpN0RVdWRQelM4SHJUU2k1WXUKN002NlAwUThwaGZpN2NvcmJFcHQrci9DZTVHaTRXVlVLd3dSZlFJREFRQUJBb0lCQUNsY3NxNWpwQ3RQRTE4ZwpmZnlIaTREOXpzeFVzK0lOYlQ2SmtLbnBJWWVHYk8zUTBnZkMwcVdOc1MvcFNqKzFsOEwwbXBZb0prc1lyejNKClUzbjNoTFZNdENnNXI2VWp0R2dnVVRCWTZUelNubkp5Ti9XV0xQU3NJSlNBbm55MzdHNWpLaHVwdmVJWFlod1EKWWZYNUlrWTNNUG1vNU14NTdoVWMrU0JrOVhYcGFIOGZTbTVFeFl1Zkd5WVpsWWh1L1BDdSt2ZWlYRXZHbFRheQpYZmZUUmkvQjFOTlhha3JLTzY1RjBPMks3U1ljNkx3Q3krRVU4UDA0ZXR2NEoxOFdXVHJqZ01Tenh3OUNZb1BLClNsT0xLVE1xeXptQmtETGV1NURNUGh3U1dia0Y5ejArQS94bTNjcVp1cC9yczIxYWxCa29TTTZUNGx0WmR4NUkKc21pR3NPVUNnWUVBNTE1Vy9veXNLYjJpSVZWOFFhb09LWjFsQWlJaDh2d3NsMUFRbzF3QW00VStrRnFqSVlIUgp1UDBmQjFlZW5jSHFkdm1DOEZrNnhhTGZxL2RxNGtHaHF6MDVHdjFoUDVyRXlTVXFxYmFYTGEvVUgzN2doVkJjCmsyclVtTmJYV2l2NWtJVzh4T1hOSklUenIzUUo1Y0VQMjNrVE96UWNwN3AwWHlzU05YU3RSOWNDZ1lFQTFyNk4KRWdWajNlY3hFR2tvVFB2aXc3d2E0Ti83UXFCaytWNTZPQytNamIvNHJiWDBNeEtlQjlPRDc1OXlPYlF4YjhxRgpweFZCdGVSTUN2OWtEcDJmTHBWTlB0YVBvQit3RUNQVm1aa2k5cFpITVUwMGw0QU9uVWM2SVZmZzd0cWllTzYyCnFJb20vWnVINXBlU2t2QjJVbktZTnNZdlA2VnNMaVRITStEUmRzc0NnWUVBamwxb1hqMGsrcElySHlQQXo0N2EKSkJVclFBTE1yUDBxV3ZqekkvSEtsZWVKTmIzdnZ3Qm9rZEYrdEQvZjQrYWNaUlRtMHdtMHRrT2dLZXFXSkI2SwpZaG5MOTZXTm4xdVdWc2E5MTZ2NG5pNGc4amhaNHU0dDZLL0ZuVDRsU09EaU1XRjVaQmFiQWl1azNvTWlTL05kCjFJaE5veEpQeDQxZGFlblF6SCs4MkFVQ2dZRUF2NjRDdWEwNkFSNlYrdklDV1FVVTJtWVREOXFkcWxFRkVGbTIKZW1SbTd6Z0Z2dmlFNnZtWk9aOGhTMGhsYXdCZWlFeWJsQkl6UHlweWZmYU0xMGIyaVZ3WFFSbS94Y3ZER2dVQwpha0g0cFdacVVhVjZaaWlWNHVsckI4d3JLTWphOFZzU2k4b1ZGNVkyYml1cFY2TnYyaFFUcmdDa3VBanVVUm5lCi9YMlZPcFVDZ1lCY2M5R1c5SlhLT242aGNML3lub1NiL3pQYTNrM3lFMlZVNWZiamZpK0JQVWxKRzg5T1JDSU0KSGo1c2lVN2pEeGk2VmM5eGo3dDNNVXh4cXlKYTBWaVd5bGZ6WkVzSVNUQmpub04xbSszM3VqbTV6RSs3NGxQUwpPUm4yNWhrTlIvejY0QWZiZjd6Si9jaEVZSndoQS9zK0NjcVptQ0VWaEc4RjgranRyLzhUQVE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
-kind: Secret
-metadata:
-  creationTimestamp: null
-  name: pod-annotate-webhook-certs

clr-k8s-examples/admit-kata/webhook-registration.yaml

@@ -1,24 +0,0 @@
-apiVersion: admissionregistration.k8s.io/v1beta1
-kind: MutatingWebhookConfiguration
-metadata:
-  name: pod-annotate-webhook
-  labels:
-    app: pod-annotate-webhook
-    kind: mutator
-webhooks:
-  - name: pod-annotate-webhook.pod.xyz
-    clientConfig:
-      service:
-        name: pod-annotate-webhook
-        namespace: default
-        path: "/mutate"
-      caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMwakNDQWJvQ0NRQ1IrV2diYXlJeHJ6QU5CZ2txaGtpRzl3MEJBUXNGQURBck1Ta3dKd1lEVlFRRERDQncKYjJRdFlXNXViM1JoZEdVdGQyVmlhRzl2YXk1a1pXWmhkV3gwTG5OMll6QWVGdzB4T0RBM01EZ3hOVFEzTXpGYQpGdzB4T1RBM01EZ3hOVFEzTXpGYU1Dc3hLVEFuQmdOVkJBTU1JSEJ2WkMxaGJtNXZkR0YwWlMxM1pXSm9iMjlyCkxtUmxabUYxYkhRdWMzWmpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXdoVVUKUEpudnZjZzRJSG54d2tKMkZiQldzc1ZvQjh1anhLcHh2Vm92dEREV0tOVU5jaHozbUF0cUJvaXAwVUlqclptUgowZ3RvcWFJOFJpeHFjUDlvVXlwMVNSTVQrYTdVeTY4b0s3d1F2Mnl5QjZ4MjExU2lja2hHaW8xTUpQN0xCRXpGClJWOUlvbXJjZkp2bi91STRWTWVmcmNZcGp2WHBOOHRUdTdWQWcvUkw4NldjSXg4VEVlbU5KNkErUWdWN0VCS00KWVBRMDAzRno1R2RHWi85c2hRNUZDSEJ1QVh3aVFzTHp6UCtOUkVGaG1zMmdHVk9xVXlBUEdRZFlZVUZWVHdLSwpsaW9FNW1yV1NFMExmekNLczYxR3BnSnBZY2k3RFV1ZFB6UzhIclRTaTVZdTdNNjZQMFE4cGhmaTdjb3JiRXB0CityL0NlNUdpNFdWVUt3d1JmUUlEQVFBQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0lZdHQrUTl1bzNKblEKdzBMRExjbUIwb0xyM1VFaFBOaVk4emNKZHd1eTRSbEg0NXRjYllJcmVhZHZieUlRbjBxUEQzd1ZvTk00R01wRwpXREtrY1VsYnhVMURSbmdyY2FrcytkM2prT2NueEtIRGNIemJ6bkh2SStaTkFpMU8yeERibWN3VlhQTGJxb2FnCmNCalJtV3QzVGFIN2FsS2ZtbkpYbit2NDZEV3IvQm1GZ0pvcld3TjdYM1V5LzFxcitrVVZxc3lGTHBqelRLZXoKVnYweklJZ1hGTmJXbzQ4cFRNenkxTWRjb0RObDVJRlZrczVnQUkzR05nQlJWdFlOYUN1bUpKNGxKM21tUGtTUAp4OW93ZE9qLzVJTFdzdWVkV1UzT2c2UE96TmdPSjRYVXlLS3ZWSG9BdU11MEtvaVk0QUF6VHFXSjNsR1M5L2VwCm1qQW9BTFdSCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-    rules:
-      - operations: [ "CREATE" ]
-        apiGroups: [""]
-        apiVersions: ["v1"]
-        resources: ["pods"]
-    namespaceSelector:
-      matchExpressions:
-        -  {key: "kata", operator: NotIn, values: ["false"]}
-

clr-k8s-examples/create_stack.sh

@@ -8,12 +8,23 @@ CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 
 function print_usage_exit() {
-	echo $"Usage: $0 [minimal|all]"
+	exit_code=${1:-0}
-	exit 1
+	cat <<EOT
+Usage: $0 [subcommand]
+
+Subcommands:
+
+$(
+		for cmd in "${!command_handlers[@]}"; do
+			printf "\t%s:|\t%s\n" "${cmd}" "${command_help[${cmd}]:-Not-documented}"
+		done | sort | column -t -s "|"
+	)
+EOT
+	exit "${exit_code}"
 }
 
 function finish() {
-	cd $CUR_DIR
+	cd "${CUR_DIR}"
 }
 trap finish EXIT
 
@@ -22,10 +33,10 @@ function cluster_init() {
 	#to enable the RuntimeClass featuregate
 	sudo -E kubeadm init --config=./kubeadm.yaml
 
-	rm -rf $HOME/.kube
+	rm -rf "${HOME}/.kube"
-	mkdir -p $HOME/.kube
+	mkdir -p "${HOME}/.kube"
-	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+	sudo cp -i /etc/kubernetes/admin.conf "${HOME}/.kube/config"
-	sudo chown $(id -u):$(id -g) $HOME/.kube/config
+	sudo chown "$(id -u):$(id -g)" "${HOME}/.kube/config"
 
 	# If this an interactive terminal then wait for user to join workers
 	if [ -t 0 ]; then
@@ -33,18 +44,15 @@ function cluster_init() {
 	fi
 
 	#Ensure single node k8s works
-	if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
+	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
 		kubectl taint nodes --all node-role.kubernetes.io/master-
 	fi
 }
 
-function runtimeclass_kata() {
+function kata() {
-	#Add support for kata runtime
+	# Install kata artifacts using kata-deploy
-	kubectl apply -f 8-kata/runtimeclass_crd.yaml
+	kubectl apply -f 8-kata/deploy/kata-rbac.yaml
-	while [[ $(kubectl get crd runtimeclasses.node.k8s.io >/dev/null 2>&1) || $? -ne 0 ]]; do
+	kubectl apply -f 8-kata/deploy/kata-deploy.yaml
-		echo "Waiting for runtime class CRD"
-		sleep 2
-	done
 	kubectl apply -f 8-kata/
 }
 
@@ -100,8 +108,8 @@ function miscellaneous() {
 
 function minimal() {
 	cluster_init
-	runtimeclass_kata
 	cni
+	kata
 	metrics
 }
 
@@ -112,20 +120,25 @@ function all() {
 	miscellaneous
 }
 
-cd $SCRIPT_DIR
+declare -A command_handlers
-if [[ "$#" -eq 0 ]]; then
+command_handlers[init]=cluster_init
-	all
+command_handlers[cni]=cni
-	exit
+command_handlers[minimal]=minimal
-fi
+command_handlers[all]=all
+command_handlers[help]=print_usage_exit
 
-case "$1" in
+declare -A command_help
-minimal)
+command_help[init]="Only inits a cluster using kubeadm"
-	minimal
+command_help[cni]="Setup network for running cluster"
-	;;
+command_help[minimal]="init + cni +  kata + metrics"
-all)
+command_help[all]="minimal + storage + monitoring + miscellaneous"
-	all
+command_help[help]="show this message"
-	;;
+
-*)
+cd "${SCRIPT_DIR}"
-	print_usage_exit
+
-	;;
+cmd_handler=${command_handlers[${1:-none}]:-unimplemented}
-esac
+if [ "${cmd_handler}" != "unimplemented" ]; then
+	"${cmd_handler}"
+else
+	print_usage_exit 1
+fi

clr-k8s-examples/kubeadm.yaml

@@ -1,15 +1,10 @@
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
-nodeRegistration:
-  criSocket: /var/run/crio/crio.sock
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-featureGates:
+# Allowing for CPU pinning and isolation in case of guaranteed QoS class
-  RuntimeClass: true
+cpuManagerPolicy: static
-# Kata does not work with static
-# https://github.com/kata-containers/runtime/issues/878
-# cpuManagerPolicy: static
 systemReserved:
   cpu: 500m
   memory: 256M
@@ -17,11 +12,9 @@ kubeReserved:
   cpu: 500m
   memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16
   serviceSubnet: 10.96.0.0/12
-apiServerExtraArgs:
-  feature-gates: RuntimeClass=true

clr-k8s-examples/reset_stack.sh

@@ -3,7 +3,10 @@
 set -o nounset
 
 #Cleanup
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster() {
+	sudo -E kubeadm reset -f
+}
+reset_cluster
 
 for ctr in $(sudo crictl ps --quiet); do
 	sudo crictl stop "$ctr"
@@ -15,9 +18,10 @@ for pod in $(sudo crictl pods --quiet); do
 done
 
 #Forcefull cleanup all artifacts
-#This is needed is things really go wrong
+#This is needed if things really go wrong
 sudo systemctl stop kubelet
-sudo systemctl stop crio
+systemctl is-active crio && sudo systemctl stop crio
+systemctl is-active containerd && sudo systemctl stop containerd
 sudo pkill -9 qemu
 sudo pkill -9 kata
 sudo pkill -9 kube
@@ -39,8 +43,11 @@ sudo -E bash -c "rm -r /var/run/kata-containers/*"
 sudo rm -rf /var/lib/rook
 
 sudo systemctl daemon-reload
-sudo systemctl enable kubelet crio
+sudo systemctl is-active crio && sudo systemctl stop crio
-sudo systemctl restart crio
+sudo systemctl is-active containerd && sudo systemctl stop containerd
+sudo systemctl is-enabled crio && sudo systemctl restart crio
+sudo systemctl is-enabled containerd && sudo systemctl restart containerd
+
 sudo systemctl restart kubelet
 
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster

clr-k8s-examples/setup_kata_firecracker.sh

@@ -4,33 +4,6 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-sudo mkdir -p /etc/kata-containers
-
-# Setup a configuration to be used by firecracker
-cat <<EOT | sudo tee /etc/kata-containers/configuration_firecracker.toml
-[hypervisor.firecracker]
-path = "/usr/bin/firecracker"
-kernel = "/usr//share/kata-containers/vmlinux.container"
-image = "/usr//share/kata-containers/kata-containers.img"
-kernel_params = ""
-default_vcpus = 1
-default_memory = 4096
-default_maxvcpus = 0
-default_bridges = 1
-block_device_driver = "virtio-mmio"
-disable_block_device_use = false
-enable_debug = true
-use_vsock = true
-
-[shim.kata]
-path = "/usr//libexec/kata-containers/kata-shim"
-
-[agent.kata]
-
-[runtime]
-internetworking_model="tcfilter"
-EOT
-
 # Firecracker can only work with devicemapper
 # Setup a sparse disk to be used for devicemapper
 sudo rm -f /var/lib/crio/devicemapper/disk.img
@@ -60,32 +33,12 @@ EOT
 sudo systemctl daemon-reload
 sudo systemctl enable --now devicemapper
 
-# For now till we address https://github.com/kubernetes-sigs/cri-o/issues/1991
-# use a shell script to expose firecracker through kata
-cat <<EOT | sudo tee /usr/bin/kata-fc
-#!/bin/bash
-
-/usr/bin/kata-runtime --kata-config /etc/kata-containers/configuration_firecracker.toml "\$@"
-EOT
-
-sudo chmod +x /usr/bin/kata-fc
-
-# Add firecracker as a second runtime
-# Also setup crio to use devicemapper
-
-sudo mkdir -p /etc/crio/
-sudo cp /usr/share/defaults/crio/crio.conf /etc/crio/crio.conf
-
-echo -e "\n[crio.runtime.runtimes.kata-qemu]\nruntime_path = \"/usr/bin/kata-runtime\"" | sudo tee -a /etc/crio/crio.conf
-echo -e "\n[crio.runtime.runtimes.kata-fc]\nruntime_path = \"/usr/bin/kata-fc\"" | sudo tee -a /etc/crio/crio.conf
-
-sudo sed -i 's|\(\[crio\.runtime\]\)|\1\nmanage_network_ns_lifecycle = true|' /etc/crio/crio.conf
-
 sudo sed -i 's/storage_driver = \"overlay\"/storage_driver = \"devicemapper\"\
 storage_option = [\
   \"dm.basesize=8G\",\
   \"dm.directlvm_device=\/dev\/loop8\",\
   \"dm.directlvm_device_force=true\",\
+  \"dm.override_udev_sync_check=true",\
   \"dm.fs=ext4\"\
 ]/g' /etc/crio/crio.conf
 

clr-k8s-examples/setup_system.sh

@@ -22,6 +22,8 @@ fi
 sudo mkdir -p /etc/sysctl.d/
 cat <<EOT | sudo bash -c "cat > /etc/sysctl.d/60-k8s.conf"
 net.ipv4.ip_forward=1
+net.ipv4.conf.default.rp_filter=1
+net.ipv4.conf.all.rp_filter=1
 EOT
 sudo systemctl restart systemd-sysctl
 
@@ -50,8 +52,6 @@ sudo systemctl daemon-reload
 echo "The following kubelet command may complain... it is not an error"
 sudo systemctl enable --now kubelet crio || true
 
-sudo mkdir -p /usr/libexec/cni /opt/cni
-[ ! -e /opt/cni/bin/cni ] && sudo ln -s /usr/libexec/cni /opt/cni/bin
 #Ensure that the system is ready without requiring a reboot
 sudo swapoff -a
 sudo systemctl restart systemd-modules-load.service

clr-k8s-examples/tests/cpumanager/test-cpumanager.yaml.tmpl

@@ -12,7 +12,7 @@ spec:
     resources:
       limits:
         cpu: 1
-        memory: 100Mi
+        memory: 500Mi # For kata to run
 ---
 apiVersion: v1
 kind: Pod

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-qemu.yaml

@@ -11,9 +11,6 @@ spec:
       run: php-apache-kata-qemu
   template:
     metadata:
-      annotations:
-        io.kubernetes.cri-o.TrustedSandbox: "false"
-        io.kubernetes.cri.untrusted-workload: "true"
       labels:
         run: php-apache-kata-qemu
     spec: