Update to latest metrics-server 0.3.x works

With kubelet service file fixed and correct args set in
metrics-server-deployment.yaml, autoscale seems to be working reliably.

Fixes: #5

Signed-off-by: Saikrishna Edupuganti <saikrishna.edupuganti@intel.com>

.gitignore

@@ -0,0 +1,3 @@
+.vagrant
+OVMF.fd
+

clr-k8s-examples/1-core-metrics/aggregated-metrics-reader.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]

clr-k8s-examples/1-core-metrics/auth-delegator.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/auth-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:

clr-k8s-examples/1-core-metrics/metrics-apiservice.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: apiregistration.k8s.io/v1beta1
 kind: APIService
 metadata:

clr-k8s-examples/1-core-metrics/metrics-server-deployment.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -22,10 +23,19 @@ spec:
         k8s-app: metrics-server
     spec:
       serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
       containers:
       - name: metrics-server
-        image: gcr.io/google_containers/metrics-server-amd64:v0.2.0
+        image: k8s.gcr.io/metrics-server-amd64:v0.3.1
         imagePullPolicy: Always
-        command:
-        - /metrics-server
-        - --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true
+        args:
+        - --logtostderr
+        - --kubelet-insecure-tls
+        - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+

clr-k8s-examples/1-core-metrics/metrics-server-service.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Service
 metadata:

clr-k8s-examples/1-core-metrics/resource-reader.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -9,15 +10,6 @@ rules:
   - pods
   - nodes
   - nodes/stats
-  - namespaces
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - "extensions"
-  resources:
-  - deployments
   verbs:
   - get
   - list

clr-k8s-examples/1-core-metrics/versions

@@ -1 +1 @@
-f90c6705d2381ea2db1a6343da6c400bd2ef4cb2
+92d8412788e27ee669d38f21f20bad5342211884

clr-k8s-examples/8-kata/kata-fc-runtimeClass.yaml

@@ -1,6 +1,6 @@
 kind: RuntimeClass
 apiVersion: node.k8s.io/v1alpha1
 metadata:
-    name: fire
+    name: kata-fc
 spec:
-    runtimeHandler: fire
+    runtimeHandler: kata-fc

clr-k8s-examples/8-kata/kata-qemu-runtimeClass.yaml

@@ -0,0 +1,14 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1alpha1
+metadata:
+    name: kata
+spec:
+    runtimeHandler: kata-qemu
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1alpha1
+metadata:
+    name: kata-qemu
+spec:
+    runtimeHandler: kata-qemu

clr-k8s-examples/8-kata/kata-runtimeClass.yaml

@@ -1,6 +0,0 @@
-kind: RuntimeClass
-apiVersion: node.k8s.io/v1alpha1
-metadata:
-    name: kata
-spec:
-    runtimeHandler: kata

clr-k8s-examples/9-multi-network/Dockerfile

@@ -0,0 +1,33 @@
+# Build multus plugin
+FROM golang:1.10 AS multus
+RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
+WORKDIR /go/src/github.com/intel/multus-cni
+RUN ./build
+
+# Build sriov plugin
+FROM golang:1.10 AS sriov-cni
+RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
+WORKDIR /go/src/github.com/intel-corp/sriov-cni
+RUN ./build
+
+# Build sriov device plugin
+FROM golang:1.10 AS sriov-dp
+RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
+WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
+RUN make
+
+# Build vfioveth plugin
+FROM busybox as vfioveth
+RUN wget -O /bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+COPY cni/vfioveth /bin/vfioveth
+RUN chmod +x /bin/vfioveth /bin/jq
+
+# Final image
+FROM centos/systemd
+WORKDIR /tmp/cni/bin
+COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
+COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=vfioveth /bin/vfioveth .
+COPY --from=vfioveth /bin/jq .
+WORKDIR /usr/bin
+COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -0,0 +1,60 @@
+# Multi-Network
+
+## Daemonset
+
+We launch a `Daemonset` with an `initContainer` which sets up the CNI
+directories on the host with the necessary binaries and configuration files.
+
+> NOTE: SR-IOV devices are not necessary to test multi-network capability
+
+### Customization
+
+The device plugin will register the SR-IOV enabled devices on the host, specified as
+`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+file is provided, which enables SR-IOV for the above `rootDevices`
+
+> NOTE: This assumes homogenous nodes in the cluster
+
+### Install
+
+To install and configure `multus-cni` on all nodes, along with
+`sriov-cni`, `vfioveth-cni` and `sriov-network-device-plugin`
+
+```bash
+kubectl apply -f .
+kubectl get nodes -o json | jq '.items[].status.allocatable' # should list "intel.com/sriov_*"
+```
+
+## Tests
+
+### Default only
+
+To test if default connectivity is working
+
+```bash
+kubectl apply -f test/pod.yaml
+kubectl exec test -- ip a                        # should see one interface only
+```
+
+### Bridge
+
+To test multus with second interface created by `bridge` plugin
+
+```bash
+kubectl apply -f test/bridge
+kubectl exec test-bridge -- ip a                 # should see two interfaces
+ip a show mynet                                  # bridge created on host if it doesnt exist already
+```
+
+### SR-IOV
+
+To test multus with second interface created by `sriov` plugin
+
+```bash
+kubectl apply -f test/sriov
+
+kubectl exec test-sriov -- ip a                  # second interface is a VF
+
+kubectl exec test-sriov-dpdk -- ip a             # veth pair with details of VF
+kubectl exec test-sriov-dpdk -- ls -l /dev/vfio
+```

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -0,0 +1,67 @@
+#!/bin/bash -x
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+exec 3>&1
+exec &>>/var/log/$(basename $0).log
+
+PATH="$CNI_PATH:$(dirname "${BASH_SOURCE[0]}"):$PATH"
+CNI_CONF=$(cat /dev/stdin)
+
+get_peer_name() {
+	echo "$1-vdev"
+}
+
+get_mac_with_vfpci() {
+	local pf=$(readlink /sys/devices/pci*/*/$1/physfn | awk '{print substr($1,4)}')
+	local pfName=$(ls /sys/devices/pci*/*/$pf/net/ | head -1)
+	local idx=$(ls -l /sys/devices/pci*/*/$pf | awk -v vf=$1 'substr($11,4)==vf {print substr($9,7)}')
+	local mac=$(ip link show dev $pfName | awk -v idx="$idx" '$1=="vf" && $2==idx {print substr($4,1,17)}')
+	echo $mac
+}
+
+ipam() {
+	local plugin=$(echo $CNI_CONF | jq -r '.ipam.type')
+	local res=$(echo $"$CNI_CONF" | "$plugin" | jq -c '.')
+	echo $res
+}
+
+add_pair_ns() {
+	vfpci=$(echo $CNI_CONF | jq -r '.deviceID')
+	mac=$(get_mac_with_vfpci $vfpci)
+	peer=$(get_peer_name $CNI_IFNAME)
+	ip=$1
+
+	mkdir -p /var/run/netns/
+	ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID
+
+	ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer
+	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up
+	ip netns exec $CNI_CONTAINERID ip link set $peer up
+	ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
+}
+
+delete_pair_ns() {
+	ip netns exec $CNI_CONTAINERID ip link del $CNI_IFNAME
+}
+
+case $CNI_COMMAND in
+ADD)
+	res=$(ipam)
+	ip=$(echo $res | jq -r '.ip4.ip')
+	add_pair_ns $ip
+	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	;;
+DEL)
+	set +o errexit
+	ipam
+	delete_pair_ns
+	set -o errexit
+	;;
+*)
+	echo "CNI_COMMAND=[ADD|DEL] only supported"
+	exit 1
+	;;
+esac

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -0,0 +1,214 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: k8s.cni.cncf.io
+  # version name to use for REST API: /apis/<group>/<version>
+  version: v1
+  # either Namespaced or Cluster
+  scope: Namespaced
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: network-attachment-definitions
+    # singular name to be used as an alias on the CLI and for display
+    singular: network-attachment-definition
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: NetworkAttachmentDefinition
+    # shortNames allow shorter string to match your resource on the CLI
+    shortNames:
+    - net-attach-def
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: multus-sa-secret
+  namespace: kube-system
+  annotations:
+    kubernetes.io/service-account.name: multus-sa
+type: kubernetes.io/service-account-token
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-pod-networks-lister
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["pods/status"]
+  verbs: ["update"]
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs: ["get"]
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-scripts
+  namespace: kube-system
+data:
+  install-certs.sh: |
+    # Copied from Calico
+    # https://github.com/projectcalico/cni-plugin/blob/master/k8s-install/scripts/install-cni.sh
+    touch /host/etc/cni/net.d/multus-kubeconfig
+    chmod 600 /host/etc/cni/net.d/multus-kubeconfig
+    SERVICE_ACCOUNT_PATH=/var/run/secrets/multus/serviceaccount
+    KUBE_CA_FILE=$SERVICE_ACCOUNT_PATH/ca.crt
+    TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+    SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_PATH/token)
+    cat > /host/etc/cni/net.d/multus-kubeconfig <<EOF
+    # Kubeconfig file for Multus CNI plugin.
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - name: local
+      cluster:
+        server: https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+        $TLS_CFG
+    users:
+    - name: multus
+      user:
+        token: "${SERVICEACCOUNT_TOKEN}"
+    contexts:
+    - name: multus-context
+      context:
+        cluster: local
+        user: multus
+    current-context: multus-context
+    EOF
+  install-multus-conf.sh: |
+    # copied from https://github.com/intel/multus-cni/blob/master/images/entrypoint.sh
+    rm -f /host/etc/cni/net.d/00-multus.conf
+    MASTER_PLUGIN="$(ls /host/etc/cni/net.d | grep -E '\.conf(list)?$' | head -1)"
+    MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
+    cat > /host/etc/cni/net.d/00-multus.conf <<EOF
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "logFile": "/var/log/multus.log",
+      "logLevel": "debug",
+      "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
+      "delegates": [
+                    $MASTER_PLUGIN_JSON
+       ]
+    }
+    EOF
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-networks-lister
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sriov-device-plugin
+  namespace: kube-system
+spec:
+  selector:
+      matchLabels:
+        name: sriov-device-plugin
+  template:
+    metadata:
+      labels:
+        name: sriov-device-plugin
+    spec:
+      initContainers:
+      - name: multus
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "bash", "-c" ]
+        args:
+        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+          /tmp/multus/install-multus-conf.sh;
+          /tmp/multus/install-certs.sh;
+          systemctl stop kubelet;
+          echo "Restarting crio/containerd, kubelet";
+          systemctl restart containerd;
+          systemctl restart crio;
+          systemctl restart kubelet;
+        volumeMounts:
+        - name: usr-bin
+          mountPath: /host/usr/bin
+        - name: cni-bin
+          mountPath: /host/opt/cni/bin
+        - name: multus-sa
+          mountPath: /var/run/secrets/multus/serviceaccount
+        - name: multus-scripts
+          mountPath: /tmp/multus
+        - name: cni-conf
+          mountPath: /host/etc/cni/net.d
+        - name: dbus
+          mountPath: /var/run/dbus
+        - name: systemd
+          mountPath: /run/systemd
+      containers:
+      - name: sriovdp
+        image: krsna1729/multus-sriov:k8s-1.13
+        command: [ "sh", "-c" ]
+        args:
+        - /usr/bin/sriovdp --logtostderr -v 10;
+          sleep infinity;
+        volumeMounts:
+        - name: net
+          mountPath: /sys/class/net
+          readOnly: true
+        - name: dp-sock
+          mountPath: /var/lib/kubelet/device-plugins/
+          readOnly: false
+        - mountPath: /etc/pcidp
+          name: sriov-config
+          readOnly: false
+      volumes:
+      - name: sriov-config
+        configMap:
+          name: sriov-config
+      - name: usr-bin
+        hostPath:
+          path: /usr/bin
+      - name: cni-bin
+        hostPath:
+          path: /opt/cni/bin
+      - name: multus-sa
+        secret:
+          secretName: multus-sa-secret
+      - name: multus-scripts
+        configMap:
+          defaultMode: 511
+          name: multus-scripts
+      - name: cni-conf
+        hostPath:
+          path: /etc/cni/net.d
+      - name: dbus
+        hostPath:
+          path: /var/run/dbus
+      - name: systemd
+        hostPath:
+          path: /run/systemd
+      - name: net
+        hostPath:
+          path: /sys/class/net
+      - name: dp-sock
+        hostPath:
+          path: /var/lib/kubelet/device-plugins/
+      hostNetwork: true
+      hostPID: true
+

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -0,0 +1,25 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: sriov-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+      "resourceList":
+      [
+        {
+          "resourceName": "sriov_netdevice",
+          "rootDevices": ["07:00.0"],
+          "sriovMode": true,
+          "deviceType": "netdevice"
+        },
+        {
+          "resourceName": "sriov_vfio",
+          "rootDevices": ["07:00.1"],
+          "sriovMode": true,
+          "deviceType": "vfio"
+        }
+      ]
+    }

clr-k8s-examples/9-multi-network/systemd/sriov.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Create VFs on ens785f0 (netdev) ens785f1 (vfio) interfaces
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/sriov.sh ens785f0
+ExecStart=/usr/bin/sriov.sh -b ens785f1
+
+[Install]
+WantedBy=default.target

clr-k8s-examples/9-multi-network/systemd/sriov.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+OPTIND=1
+bind="false"
+
+while getopts ":b" opt; do
+	case ${opt} in
+	b)
+		bind="true"
+		;;
+	\?)
+		echo "Usage: sriov.sh [-b] ens785f0 ens785f1 ..."
+		echo "-b	Bind to vfio-pci"
+		exit
+		;;
+	esac
+done
+shift $((OPTIND - 1))
+
+setup_pf() {
+	local pf=$1
+	echo "Resetting PF $pf"
+	echo 0 | tee /sys/class/net/$pf/device/sriov_numvfs
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_totalvfs)
+	echo "Enabling $NUM_VFS VFs for $pf"
+	echo $NUM_VFS | tee /sys/class/net/$pf/device/sriov_numvfs
+	ip link set $pf up
+	sleep 1
+}
+
+setup_vfs() {
+	local pf=$1
+	local pfpci=$(readlink /sys/devices/pci*/*/*/net/$pf/device | awk '{print substr($1,10)}')
+	local NUM_VFS=$(cat /sys/class/net/$pf/device/sriov_numvfs)
+	for ((idx = 0; idx < NUM_VFS; idx++)); do
+		ip link set dev $pf vf $idx state enable
+		if [ $bind != "true" ]; then continue; fi
+
+		local vfn="virtfn$idx"
+		local vfpci=$(ls -l /sys/devices/pci*/*/$pfpci | awk -v vfn=$vfn 'vfn==$9 {print substr($11,4)}')
+		# Capture and set MAC of the VF before unbinding from linux, for later use in CNI
+		local mac=$(cat /sys/bus/pci*/*/$vfpci/net/*/address)
+		ip link set dev $pf vf $idx mac $mac
+		# Bind VF to vfio-pci
+		echo $vfpci >/sys/bus/pci*/*/$vfpci/driver/unbind
+		echo "vfio-pci" >/sys/devices/pci*/*/$vfpci/driver_override
+		echo $vfpci >/sys/bus/pci/drivers/vfio-pci/bind
+	done
+}
+
+for pf in "$@"; do
+	setup_pf $pf
+	setup_vfs $pf
+done

clr-k8s-examples/9-multi-network/test/bridge/0-bridge-net.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: mynet
+spec:
+  config: '{
+        "name": "mynet",
+        "type": "bridge",
+        "bridge": "mynet",
+        "ipam": {
+            "type": "host-local",
+            "subnet": "198.18.0.0/24"
+        }
+}'
+

clr-k8s-examples/9-multi-network/test/bridge/1-pod-bridge.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-bridge
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name": "mynet", "interface": "mynet" }
+    ]'
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/pod.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/9-multi-network/test/sriov/0-sriov-net.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_netdevice
+spec:
+  config: '{
+    "type": "sriov",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+---
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-dpdk
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov_vfio
+spec:
+  config: '{
+    "type": "vfioveth",
+    "name": "sriov-net",
+    "ipam": {
+        "type": "host-local",
+        "subnet": "198.19.0.0/24",
+        "rangeStart": "198.19.0.100",
+        "rangeEnd": "198.19.0.200",
+        "gateway": "198.19.0.1"
+    }
+}'
+

clr-k8s-examples/9-multi-network/test/sriov/1-pod-sriov.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_netdevice: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sriov-dpdk
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+

clr-k8s-examples/README.md

@@ -2,13 +2,20 @@
 
 ## Sample multi-node vagrant setup
 
-To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using libvirt, but you can use any hypervisor that you are familiar with.
+To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
 
 * Install vagrant on the distro you are using. Steps can be found at [Vagrant docs](https://www.vagrantup.com/intro/getting-started/install.html#installing-vagrant)
 * `vagrant up --provider=libvirt`
 
 Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
 
+To login to the master node and change to this directory
+
+```bash
+vagrant ssh clr-01
+cd clr-k8s-examples
+```
+
 ## Setup the nodes in the cluster
 
 Run [`setup_system.sh`](setup_system.sh) once on each and every node (master and workers)
@@ -20,18 +27,17 @@ This script ensures the following
 * Customizes the system to ensure correct defaults are setup (IP Forwarding, Swap off,...)
 * Ensures all the dependencies are loaded on boot (kernel modules)
 
-> NOTE: This step is done automatically if using vagrant. 
+> NOTE: This step is done automatically if using vagrant.
 
 ### Enabling experimental firecracker support
 
-> EXPERIMENTAL: Optionally run [`setup_firecracker.sh`](setup_firecracker.sh) to be
+> EXPERIMENTAL: Optionally run [`setup_kata_firecracker.sh`](setup_kata_firecracker.sh) to be
 able to use firecracker VMM with Kata.
 
 The firecracker setup switches the setup to use a sparse file backed loop device for
 devicemapper storage. This should not be used for production.
 
-> NOTE: This step is done automatically if using vagrant. 
-
+> NOTE: This step is done automatically if using vagrant.
 
 ## Bring up the master
 
@@ -41,8 +47,9 @@ to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
 There are two flavors of install -
-- `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
-- `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
+
+* `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
+* `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
 
 ```bash
 # default is 'all'
@@ -73,16 +80,16 @@ runtime class set to "kata" will launch the POD/Deployment with Kata.
 
 An example is
 
-`kubectl apply -f tests/test-deploy-kata.yaml`
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-qemu.yaml`
 
 ### Running Kata Workloads with Firecracker
 
-> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "fire" will launch the POD/Deployment
+> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "kata-fc" will launch the POD/Deployment
 with firecracker as the isolation mechanism for Kata.
 
 An example is
 
-`kubectl apply -f tests/test-deploy-fire.yaml`
+`kubectl apply -f tests/deploy-svc-ing/test-deploy-kata-fc.yaml`
 
 ## Making Kata the default runtime
 

clr-k8s-examples/Vagrantfile

@@ -4,14 +4,16 @@
 require 'fileutils'
 require 'ipaddr'
 require 'securerandom'
-DISK_UUID = SecureRandom.urlsafe_base64(9)
 
 $num_instances = (ENV['NODES'] || 3).to_i
 $cpus = (ENV['CPUS'] || 2).to_i
 $memory = (ENV['MEMORY'] || 4096).to_i
 $disks = 2
+# Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
+disk_prefix = File.basename(File.dirname(__FILE__), "/")
+#DISK_UUID = SecureRandom.urlsafe_base64(9)
 $disk_size = "10G"
-$box = "gmmaha/clearlinux"
+$box = "AntonioMeireles/ClearLinux"
 $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
 base_ip = IPAddr.new("192.52.100.10")
@@ -43,6 +45,9 @@ Vagrant.configure("2") do |config|
   # boxes at https://vagrantcloud.com/search.
   config.vm.box = $box
 
+  # Mount the current dir at home folder instead of default
+  config.vm.synced_folder './', '/vagrant', disabled: true
+  config.vm.synced_folder './', '/home/clear/' + File.basename(Dir.getwd), type: 'rsync'
   #Setup proxies for all machines
   (1..$num_instances).each do |i|
     base_ip = base_ip.succ
@@ -64,7 +69,7 @@ Vagrant.configure("2") do |config|
         lv.cpus = $cpus
         lv.memory = $memory
         (1..$disks).each do |d|
-          lv.storage :file, :device => "hd#{driveletters[d]}", :path => "disk-#{vm_name}-#{d}-#{DISK_UUID}.disk", :size => $disk_size, :type => "raw"
+          lv.storage :file, :device => "hd#{driveletters[d]}", :path => "disk-#{disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end
       end
       if Vagrant.has_plugin?("vagrant-proxyconf")
@@ -72,10 +77,10 @@ Vagrant.configure("2") do |config|
         c.proxy.https = (ENV['https_proxy']||ENV['HTTPS_PROXY'])
         c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
       end
-      # Bad hack for the vagrant libvirt boxes. WIll be removed once they are fixed.
-      c.vm.provision "shell", privileged: false, inline: "sudo usermod --password vagrant root"
       c.vm.provision "shell", privileged: false, path: "setup_system.sh"
-      c.vm.provision "shell", privileged: false, path: "setup_firecracker.sh"
+      c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+      # Include shells bundle to get bash completion and add kubectl's commands to vagrant's shell
+      c.vm.provision "shell", privileged: false, inline: 'sudo -E swupd bundle-add shells; echo "source <(kubectl completion bash)" >> $HOME/.bashrc'
     end
   end
 end

clr-k8s-examples/admit-kata/webhook.yaml

@@ -13,7 +13,7 @@ spec:
     spec:
       containers:
         - name: pod-annotate-webhook
-          image: mcastelino/kubewebhook-pod-annotate-example:1.0
+          image: katadocker/kata-webhook-example:latest
           imagePullPolicy: Always
           args:
             - -tls-cert-file=/etc/webhook/certs/cert.pem

clr-k8s-examples/create_stack.sh

@@ -22,16 +22,16 @@ function cluster_init() {
 	#to enable the RuntimeClass featuregate
 	sudo -E kubeadm init --config=./kubeadm.yaml
 
-	# If this an interactive terminal then wait for user to join workers
-	if [ -t 0 ]; then
-		read -p "Join other nodes. Press enter to continue"
-	fi
-
 	rm -rf $HOME/.kube
 	mkdir -p $HOME/.kube
 	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 	sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
+	# If this an interactive terminal then wait for user to join workers
+	if [ -t 0 ]; then
+		read -p "Join other nodes. Press enter to continue"
+	fi
+
 	#Ensure single node k8s works
 	if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
 		kubectl taint nodes --all node-role.kubernetes.io/master-

clr-k8s-examples/setup_kata_firecracker.sh

@@ -62,13 +62,13 @@ sudo systemctl enable --now devicemapper
 
 # For now till we address https://github.com/kubernetes-sigs/cri-o/issues/1991
 # use a shell script to expose firecracker through kata
-cat <<EOT | sudo tee /usr/bin/kata-runtime-fire
+cat <<EOT | sudo tee /usr/bin/kata-fc
 #!/bin/bash
 
 /usr/bin/kata-runtime --kata-config /etc/kata-containers/configuration_firecracker.toml "\$@"
 EOT
 
-sudo chmod +x /usr/bin/kata-runtime-fire
+sudo chmod +x /usr/bin/kata-fc
 
 # Add firecracker as a second runtime
 # Also setup crio to use devicemapper
@@ -76,8 +76,8 @@ sudo chmod +x /usr/bin/kata-runtime-fire
 sudo mkdir -p /etc/crio/
 sudo cp /usr/share/defaults/crio/crio.conf /etc/crio/crio.conf
 
-echo -e "\n[crio.runtime.runtimes.kata]\nruntime_path = \"/usr/bin/kata-runtime\"" | sudo tee -a /etc/crio/crio.conf
-echo -e "\n[crio.runtime.runtimes.fire]\nruntime_path = \"/usr/bin/kata-runtime-fire\"" | sudo tee -a /etc/crio/crio.conf
+echo -e "\n[crio.runtime.runtimes.kata-qemu]\nruntime_path = \"/usr/bin/kata-runtime\"" | sudo tee -a /etc/crio/crio.conf
+echo -e "\n[crio.runtime.runtimes.kata-fc]\nruntime_path = \"/usr/bin/kata-fc\"" | sudo tee -a /etc/crio/crio.conf
 
 sudo sed -i 's|\(\[crio\.runtime\]\)|\1\nmanage_network_ns_lifecycle = true|' /etc/crio/crio.conf
 

clr-k8s-examples/setup_system.sh

@@ -33,6 +33,10 @@ vhost_vsock
 overlay
 EOT
 
+# Make sure /etc/hosts file exists
+if [ ! -f /etc/hosts ]; then
+  sudo touch /etc/hosts
+fi
 hostcount=$(grep '127.0.0.1 localhost' /etc/hosts | wc -l)
 if [ "$hostcount" == "0" ]; then
 	echo "127.0.0.1 localhost $(hostname)" | sudo bash -c "cat >> /etc/hosts"

clr-k8s-examples/tests/autoscale/load-gen.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: load-generator
+spec:
+  containers:
+  - command: ["/bin/sh", "-c"]
+    args:
+    - while true; do wget -q -O- http://php-apache-test; done;
+    image: busybox
+    imagePullPolicy: Always
+    name: load-generator

clr-k8s-examples/tests/autoscale/php.yaml

@@ -0,0 +1,54 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-test
+  name: php-apache-test
+spec:
+  selector:
+    matchLabels:
+      run: php-apache-test
+  template:
+    metadata:
+      labels:
+        run: php-apache-test
+    spec:
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        name: php-apache-test
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: php-apache-test
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-test
+  sessionAffinity: None
+  type: ClusterIP
+---
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: php-apache-test
+spec:
+  maxReplicas: 10
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: php-apache-test
+  targetCPUUtilizationPercentage: 50
+

clr-k8s-examples/tests/autoscale/test-autoscale.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+kubectl apply -f $SCRIPT_DIR
+watch kubectl describe hpa

clr-k8s-examples/tests/cpumanager/test-cpumanager.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+input="test-cpumanager.yaml.tmpl"
+
+filename() {
+	echo "test-cpumanager-$1.yaml"
+}
+
+for runtimeclass in runc kata-qemu kata-fc; do
+	output=$(filename $runtimeclass)
+	cp $input $output
+	sed -i "s/__runtimeclass__/$runtimeclass/g" $output
+	if [ $runtimeclass == "runc" ]; then continue; fi
+
+	insertline="\ \ runtimeClassName: $runtimeclass"
+	sed -i "/spec:/a $insertline" $output
+done
+kubectl apply -f .

clr-k8s-examples/tests/cpumanager/test-cpumanager.yaml.tmpl

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-guaranteed-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      limits:
+        cpu: 1
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+      limits:
+        cpu: 2
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-limit-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+      limits:
+        cpu: 1
+        memory: 500Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-integer-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 1
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-burstable-float-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]
+    resources:
+      requests:
+        cpu: 500m
+        memory: 100Mi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-besteffort-__runtimeclass__
+spec:
+  restartPolicy: Never
+  containers:
+  - name: busy
+    image: busybox
+    command: [ "top" ]

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-fc.yaml

@@ -2,19 +2,19 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    run: php-apache-fire
-  name: php-apache-fire
+    run: php-apache-kata-fc
+  name: php-apache-kata-fc
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: php-apache-fire
+      run: php-apache-kata-fc
   template:
     metadata:
       labels:
-        run: php-apache-fire
+        run: php-apache-kata-fc
     spec:
-      runtimeClassName: fire
+      runtimeClassName: kata-fc
       containers:
       - image: k8s.gcr.io/hpa-example
         imagePullPolicy: Always
@@ -30,13 +30,13 @@ spec:
 apiVersion: v1 
 kind: Service
 metadata:
-  name: php-apache-fire
+  name: php-apache-kata-fc
 spec:
   ports:
   - port: 80
     protocol: TCP
     targetPort: 80
   selector:
-    run: php-apache-fire
+    run: php-apache-kata-fc
   sessionAffinity: None
   type: ClusterIP

clr-k8s-examples/tests/deploy-svc-ing/test-deploy-kata-qemu.yaml

@@ -2,22 +2,22 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
-    run: php-apache-kata
-  name: php-apache-kata
+    run: php-apache-kata-qemu
+  name: php-apache-kata-qemu
 spec:
   replicas: 1
   selector:
     matchLabels:
-      run: php-apache-kata
+      run: php-apache-kata-qemu
   template:
     metadata:
       annotations:
         io.kubernetes.cri-o.TrustedSandbox: "false"
         io.kubernetes.cri.untrusted-workload: "true"
       labels:
-        run: php-apache-kata
+        run: php-apache-kata-qemu
     spec:
-      runtimeClassName: kata
+      runtimeClassName: kata-qemu
       containers:
       - image: k8s.gcr.io/hpa-example
         imagePullPolicy: Always
@@ -33,13 +33,13 @@ spec:
 apiVersion: v1 
 kind: Service
 metadata:
-  name: php-apache-kata
+  name: php-apache-kata-qemu
 spec:
   ports:
   - port: 80
     protocol: TCP
     targetPort: 80
   selector:
-    run: php-apache-kata
+    run: php-apache-kata-qemu
   sessionAffinity: None
   type: ClusterIP

clr-k8s-examples/tests/test-autoscale.sh

@@ -1,6 +0,0 @@
-kubectl run php-apache-test --image=k8s.gcr.io/hpa-example --requests=cpu=200m --expose --port=80
-kubectl autoscale deployment php-apache-test --cpu-percent=50 --min=1 --max=10
-kubectl get hpa
-
-#kubectl run -i --tty load-generator --image=busybox /bin/sh
-# while true; do wget -q -O- http://php-apache-test.default.svc.cluster.local; done

clr-k8s-examples/tests/test-cpumanager-kata.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: test-cpumanager-kata
-spec:
-  runtimeClassName: kata
-  restartPolicy: Never
-  containers:
-  - name: taskset
-    image: busybox
-    command: [ "taskset", "-p", "1" ]
-    resources:
-      limits:
-        cpu: 1
-        memory: 500Mi

clr-k8s-examples/tests/test-cpumanager-runc.yaml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: test-cpumanager-runc
-spec:
-  restartPolicy: Never
-  containers:
-  - name: taskset
-    image: busybox
-    command: [ "taskset", "-p", "1" ]
-    resources:
-      limits:
-        cpu: 1
-        memory: 100Mi