Update README.md

Fix create_stack for version 1.25
kubernetes 1.25 changed the key for the NoSchedule taint to 'control-plane'. Fix the script to handle both pre and post version 1.25 Signed-off-by: Peter W. Morreale <pwmorreale@gmail.com>
2026-08-18 21:16:16 +00:00 · 2025-08-07 13:34:33 -07:00 · 2022-11-15 14:39:32 -07:00 · 2022-08-31 09:37:52 -07:00 · 2022-08-29 12:17:33 -07:00 · 2022-08-16 17:18:48 -06:00
113 changed files with 3419 additions and 1526 deletions
@@ -0,0 +1,56 @@
+pipeline {
+	agent {
+		label 'clearlinux'
+	}
+	options {
+		timeout(time: 1, unit: "HOURS")
+	}
+	triggers {
+		cron('H */12 * * *')
+	}
+	environment {
+		CLR_K8S_PATH="${env.WORKSPACE}/clr-k8s-examples"
+	}
+	stages {
+		stage('Setup system') {
+			steps {
+				dir(path: "$CLR_K8S_PATH") {
+					sh './setup_system.sh'
+				}
+			}
+		}
+		stage('Init') {
+			steps {
+				dir(path: "$CLR_K8S_PATH") {
+					sh './create_stack.sh init'
+					sh 'mkdir -p $HOME/.kube'
+					sh 'sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config'
+					sh 'sudo chown $(id -u):$(id -g) $HOME/.kube/config'
+					sh 'kubectl version'
+				}
+			}
+		}
+		stage('CNI') {
+			steps {
+				dir(path: "$CLR_K8S_PATH") {
+					sh './create_stack.sh cni'
+					sh 'kubectl rollout status deployment/coredns -n kube-system --timeout=5m'
+					sh 'kubectl get pods -n kube-system'
+				}
+			}
+		}
+		stage('Reset Stack') {
+			steps {
+				dir(path: "$CLR_K8S_PATH") {
+					sh './reset_stack.sh'
+				}
+			}
+		}
+	}
+	post {
+		always {
+			sh 'uname -a'
+			sh 'swupd info'
+		}
+	}
+}
@@ -1,3 +1,11 @@
+## DISCONTINUATION OF PROJECT.
+
+This project will no longer be maintained by Intel.
+
+Intel will not provide or guarantee development of or support for this project, including but not limited to, maintenance, bug fixes, new releases or updates. Patches to this project are no longer accepted by Intel. If you have an ongoing need to use this project, are interested in independently developing it, or would like to maintain patches for the community, please create your own fork of the project.
+
+Contact: webadmin@linux.intel.com
+
 ## Cloud Native Setup

 Automation around setting up the cloud-native content (Kubernetes) on Clear Linux.
@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+
@@ -0,0 +1,2 @@
+resources:
+  - canal/canal.yaml
@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+
@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+
@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml
@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml
@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml
@@ -0,0 +1,5 @@
+ipam:
+  mode: "cluster-pool"
+  operator:
+    clusterPoolIPv4PodCIDR: "10.244.0.0/16"
+    clusterPoolIPv4MaskSize: 24
@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml
@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml
@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml
@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml
@@ -0,0 +1,8 @@
+resources:
+  - metrics-server/deploy/1.8+/aggregated-metrics-reader.yaml
+  - metrics-server/deploy/1.8+/auth-delegator.yaml
+  - metrics-server/deploy/1.8+/auth-reader.yaml
+  - metrics-server/deploy/1.8+/metrics-apiservice.yaml
+  - metrics-server/deploy/1.8+/metrics-server-deployment.yaml
+  - metrics-server/deploy/1.8+/metrics-server-service.yaml
+  - metrics-server/deploy/1.8+/resource-reader.yaml
@@ -0,0 +1,8 @@
+resources:
+  - components.yaml
+patchesJson6902:
+  - target:
+      version: v1
+      kind: Deployment
+      name: metrics-server
+    path: patch_metricstls.yaml
@@ -0,0 +1,3 @@
+- op: add
+  path: "/spec/template/spec/containers/0/args/-"
+  value: --kubelet-insecure-tls
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kubernetes-dashboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-user
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: admin-user
+  namespace: kubernetes-dashboard
@@ -0,0 +1,4 @@
+resources:
+  - dashboard/aio/deploy/recommended.yaml
+  - dashboard-admin.yaml
+
@@ -0,0 +1,8 @@
+resources:
+  - instrumentation-addons/fluentd-elasticsearch/create-logging-namespace.yaml
+  - instrumentation-addons/fluentd-elasticsearch/es-service.yaml
+  - instrumentation-addons/fluentd-elasticsearch/es-statefulset.yaml
+  - instrumentation-addons/fluentd-elasticsearch/fluentd-es-configmap.yaml
+  - instrumentation-addons/fluentd-elasticsearch/fluentd-es-ds.yaml
+  - instrumentation-addons/fluentd-elasticsearch/kibana-deployment.yaml
+  - instrumentation-addons/fluentd-elasticsearch/kibana-service.yaml
@@ -0,0 +1,76 @@
+resources:
+  - kube-prometheus/manifests/kubeStateMetrics-serviceAccount.yaml
+  - kube-prometheus/manifests/blackboxExporter-clusterRole.yaml
+  - kube-prometheus/manifests/nodeExporter-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheus-prometheusRule.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml
+  - kube-prometheus/manifests/prometheus-serviceMonitor.yaml
+  - kube-prometheus/manifests/grafana-dashboardSources.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-prometheusRule.yaml
+  - kube-prometheus/manifests/kubePrometheus-prometheusRule.yaml
+  - kube-prometheus/manifests/prometheus-clusterRole.yaml
+  - kube-prometheus/manifests/blackboxExporter-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheus-roleSpecificNamespaces.yaml
+  - kube-prometheus/manifests/alertmanager-service.yaml
+  - kube-prometheus/manifests/prometheusAdapter-serviceMonitor.yaml
+  - kube-prometheus/manifests/nodeExporter-prometheusRule.yaml
+  - kube-prometheus/manifests/nodeExporter-service.yaml
+  - kube-prometheus/manifests/prometheus-roleConfig.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-clusterRole.yaml
+  - kube-prometheus/manifests/prometheusOperator-deployment.yaml
+  - kube-prometheus/manifests/prometheusOperator-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheusAdapter-deployment.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
+  - kube-prometheus/manifests/prometheusAdapter-configMap.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-prometheusRule.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-deployment.yaml
+  - kube-prometheus/manifests/blackboxExporter-configuration.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/blackboxExporter-serviceAccount.yaml
+  - kube-prometheus/manifests/grafana-dashboardDefinitions.yaml
+  - kube-prometheus/manifests/prometheusOperator-service.yaml
+  - kube-prometheus/manifests/grafana-service.yaml
+  - kube-prometheus/manifests/prometheus-prometheus.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml
+  - kube-prometheus/manifests/alertmanager-alertmanager.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml
+  - kube-prometheus/manifests/grafana-dashboardDatasources.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorCoreDNS.yaml
+  - kube-prometheus/manifests/alertmanager-serviceMonitor.yaml
+  - kube-prometheus/manifests/grafana-deployment.yaml
+  - kube-prometheus/manifests/grafana-serviceAccount.yaml
+  - kube-prometheus/manifests/alertmanager-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml
+  - kube-prometheus/manifests/prometheusOperator-prometheusRule.yaml
+  - kube-prometheus/manifests/alertmanager-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/prometheus-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheus-service.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleServerResources.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheus-roleBindingConfig.yaml
+  - kube-prometheus/manifests/nodeExporter-daemonset.yaml
+  - kube-prometheus/manifests/prometheus-roleBindingSpecificNamespaces.yaml
+  - kube-prometheus/manifests/nodeExporter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheusOperator-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheusOperator-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheusAdapter-roleBindingAuthReader.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleBindingDelegator.yaml
+  - kube-prometheus/manifests/prometheusAdapter-serviceAccount.yaml
+  - kube-prometheus/manifests/blackboxExporter-deployment.yaml
+  - kube-prometheus/manifests/alertmanager-prometheusRule.yaml
+  - kube-prometheus/manifests/prometheus-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRole.yaml
+  - kube-prometheus/manifests/grafana-serviceMonitor.yaml
+  - kube-prometheus/manifests/nodeExporter-clusterRole.yaml
+  - kube-prometheus/manifests/prometheusAdapter-service.yaml
+  - kube-prometheus/manifests/prometheus-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/blackboxExporter-service.yaml
+  - kube-prometheus/manifests/nodeExporter-serviceMonitor.yaml
+  - kube-prometheus/manifests/blackboxExporter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/alertmanager-secret.yaml
+  - kube-prometheus/manifests/prometheusAdapter-apiService.yaml
+  - kube-prometheus/manifests/prometheusOperator-clusterRole.yaml
+  - kube-prometheus/manifests/grafana-config.yaml
+  - kube-prometheus/manifests/prometheusAdapter-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-service.yaml
@@ -0,0 +1,2 @@
+resources:
+  - ingress-nginx/deploy/static/provider/baremetal/deploy.yaml
@@ -6,7 +6,7 @@ patchesJson6902:
  # adds "networking.k8s.io" to ClusterRole's apiGroups
  - target:
      group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
      kind: ClusterRole
      name: nginx-ingress-clusterrole
    path: patch_clusterrole.yaml
@@ -0,0 +1,3 @@
+resources:
+  - ingress-nginx/deploy/static/mandatory.yaml
+  - ingress-nginx/deploy/static/provider/baremetal/service-nodeport.yaml
@@ -0,0 +1,6 @@
+resources:
+  - metallb/manifests/example-layer2-config.yaml
+  - metallb/manifests/metallb.yaml
+
+patchesStrategicMerge:
+  - patch_configmap.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: my-ip-space
+      protocol: layer2
+      addresses:
+      - 10.0.0.240/28
@@ -1,7 +1,11 @@
 resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
  - rook/cluster/examples/kubernetes/ceph/operator.yaml
  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
  - rook/cluster/examples/kubernetes/ceph/storageclass.yaml

 patchesStrategicMerge:
-  - patch_operator.yaml
+  - patch_operator.yaml
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -1,5 +1,5 @@
 # operator
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: rook-ceph-operator
@@ -1,3 +1,4 @@
+---
 apiVersion: ceph.rook.io/v1
 kind: CephCluster
 metadata:
@@ -6,4 +7,4 @@ metadata:
 spec:
  storage:
    directories:
-     - path: /var/lib/rook
+      - path: /var/lib/rook
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -0,0 +1,10 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  storage:
+    directories:
+      - path: /var/lib/rook
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  storage:
+    directories:
+      - path: /var/lib/rook
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  storage:
+    directories:
+      - path: /var/lib/rook
@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+  storage:
+    directories:
+      - path: /var/lib/rook
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -0,0 +1,6 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  - probe_timeout.yaml
@@ -0,0 +1,16 @@
+---
+apiVersion:  ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  healthCheck:
+    startupProbe:
+      osd:
+        probe:
+           timeoutSeconds: 120
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 10
+           successThreshold: 1
@@ -0,0 +1,9 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
+  - probe_timeout.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -0,0 +1,30 @@
+---
+apiVersion:  ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  healthCheck:
+    startupProbe:
+      mon:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1
+      mgr:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1
+      osd:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1
@@ -0,0 +1,3 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
@@ -0,0 +1,8 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1
@@ -4,3 +4,8 @@ resources:
  - packaging/kata-deploy/k8s-1.14/kata-fc-runtimeClass.yaml
  - packaging/kata-deploy/k8s-1.14/kata-qemu-runtimeClass.yaml

+images:
+  # change 'latest' to specified version
+  - name: katadocker/kata-deploy
+    newName: katadocker/kata-deploy
+    newTag: 1.8.2
@@ -0,0 +1,11 @@
+resources:
+  - packaging/kata-deploy/kata-deploy.yaml
+  - packaging/kata-deploy/kata-rbac.yaml
+  - packaging/kata-deploy/k8s-1.14/kata-fc-runtimeClass.yaml
+  - packaging/kata-deploy/k8s-1.14/kata-qemu-runtimeClass.yaml
+
+images:
+  # change 'latest' to specified version
+  - name: katadocker/kata-deploy
+    newName: katadocker/kata-deploy
+    newTag: 1.9.1
@@ -1,20 +1,28 @@
 # Build multus plugin
-FROM golang:1.10 AS multus
-RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
-WORKDIR /go/src/github.com/intel/multus-cni
-RUN ./build
+FROM busybox AS multus
+ARG MULTUS_VER=3.4.2
+RUN wget -O multus.tgz https://github.com/intel/multus-cni/releases/download/v${MULTUS_VER}/multus-cni_${MULTUS_VER}_linux_amd64.tar.gz
+RUN tar xvzf multus.tgz --strip-components=1 -C /bin

 # Build sriov plugin
-FROM golang:1.10 AS sriov-cni
-RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
-WORKDIR /go/src/github.com/intel-corp/sriov-cni
-RUN ./build
+FROM golang AS sriov-cni
+ARG SRIOV_CNI_VER=2.3
+RUN wget -qO sriov-cni.tgz https://github.com/intel/sriov-cni/archive/v${SRIOV_CNI_VER}.tar.gz
+RUN mkdir -p sriov-cni && \
+    tar xzf sriov-cni.tgz --strip-components=1 -C sriov-cni && \
+    cd sriov-cni && \
+    make && \
+    cp build/sriov /bin

 # Build sriov device plugin
-FROM golang:1.10 AS sriov-dp
-RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
-WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
-RUN make
+FROM golang AS sriov-dp
+ARG SRIOV_DP_VER=3.2
+RUN wget -qO sriov-dp.tgz https://github.com/intel/sriov-network-device-plugin/archive/v${SRIOV_DP_VER}.tar.gz
+RUN mkdir -p sriov-dp && \
+    tar xzf sriov-dp.tgz --strip-components=1 -C sriov-dp && \
+    cd sriov-dp && \
+    make && \
+    cp build/sriovdp /bin

 # Build vfioveth plugin
 FROM busybox as vfioveth
@@ -25,9 +33,9 @@ RUN chmod +x /bin/vfioveth /bin/jq
 # Final image
 FROM centos/systemd
 WORKDIR /tmp/cni/bin
-COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
-COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=multus /bin/multus-cni .
+COPY --from=sriov-cni /bin/sriov .
 COPY --from=vfioveth /bin/vfioveth .
 COPY --from=vfioveth /bin/jq .
 WORKDIR /usr/bin
-COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .
+COPY --from=sriov-dp /bin/sriovdp .
@@ -9,11 +9,10 @@ directories on the host with the necessary binaries and configuration files.

 ### Customization

-The device plugin will register the SR-IOV enabled devices on the host, specified as
-`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
-file is provided, which enables SR-IOV for the above `rootDevices`
-
-> NOTE: This assumes homogenous nodes in the cluster
+The device plugin will register the SR-IOV enabled devices on the host, specified with
+`selectors` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+file is provided, which enables SR-IOV for the above devices. More config options
+are listed [here](https://github.com/intel/sriov-network-device-plugin#configurations).

 ### Pre-req (SR-IOV only)

@@ -38,7 +38,7 @@ add_pair_ns() {
 	ln -sfT $CNI_NETNS /var/run/netns/$CNI_CONTAINERID

 	ip netns exec $CNI_CONTAINERID ip link add $CNI_IFNAME type veth peer name $peer
-	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up
+	ip netns exec $CNI_CONTAINERID ip link set $CNI_IFNAME addr $mac up alias $vfpci
 	ip netns exec $CNI_CONTAINERID ip link set $peer up
 	ip netns exec $CNI_CONTAINERID ip addr add $ip dev $CNI_IFNAME
 }
@@ -50,9 +50,9 @@ delete_pair_ns() {
 case $CNI_COMMAND in
 ADD)
 	res=$(ipam)
-	ip=$(echo $res | jq -r '.ip4.ip')
+	ip=$(echo $res | jq -r '.ips[0].address')
 	add_pair_ns $ip
-	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	echo '{"cniVersion":"0.3.1"}' | jq -c --arg ip $ip '.ips[0].address = $ip' >&3
 	;;
 DEL)
 	set +o errexit
@@ -1,26 +1,35 @@
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
  name: network-attachment-definitions.k8s.cni.cncf.io
 spec:
-  # group name to use for REST API: /apis/<group>/<version>
  group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
  scope: Namespaced
  names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
    plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
    singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
    kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
    shortNames:
    - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -40,17 +49,43 @@ type: kubernetes.io/service-account-token
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: multus-pod-networks-lister
+  name: multus-pod-updater
 rules:
- apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get"]
- apiGroups: [""]
-  resources: ["pods/status"]
-  verbs: ["update"]
- apiGroups: ["k8s.cni.cncf.io"]
-  resources: ["*"]
-  verbs: ["get"]
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-updater
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -94,8 +129,9 @@ data:
    MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
    cat > /host/etc/cni/net.d/00-multus.conf <<EOF
    {
+      "cniVersion": "0.3.1",
      "name": "multus-cni-network",
-      "type": "multus",
+      "type": "multus-cni",
      "logFile": "/var/log/multus.log",
      "logLevel": "debug",
      "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
@@ -105,19 +141,6 @@ data:
    }
    EOF
 ---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus-rb
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus-pod-networks-lister
-subjects:
- kind: ServiceAccount
-  name: multus-sa
-  namespace: kube-system
---
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -134,14 +157,14 @@ spec:
    spec:
      initContainers:
      - name: multus
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
        command: [ "bash", "-c" ]
        args:
-        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+        - cp /tmp/cni/bin/{multus-cni,sriov,vfioveth,jq} /host/opt/cni/bin/;
          /tmp/multus/install-multus-conf.sh;
          /tmp/multus/install-certs.sh;
          echo "Restarting crio kubelet";
-          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart crio;
          systemctl restart kubelet;
        volumeMounts:
        - name: usr-bin
@@ -160,7 +183,7 @@ spec:
          mountPath: /run/systemd
      containers:
      - name: sriovdp
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
        command: [ "sh", "-c" ]
        args:
        - /usr/bin/sriovdp --logtostderr -v 10;
@@ -209,4 +232,3 @@ spec:
          path: /var/lib/kubelet/device-plugins/
      hostNetwork: true
      hostPID: true
-
@@ -11,15 +11,15 @@ data:
      [
        {
          "resourceName": "sriov_netdevice",
-          "rootDevices": ["07:00.0"],
-          "sriovMode": true,
-          "deviceType": "netdevice"
+          "selectors": {
+                  "drivers": ["i40evf", "iavf"]
+          }
        },
        {
          "resourceName": "sriov_vfio",
-          "rootDevices": ["07:00.1"],
-          "sriovMode": true,
-          "deviceType": "vfio"
+          "selectors": {
+                  "drivers": ["vfio-pci"]
+          }
        }
      ]
    }
@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1711
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:17.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1811
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:18.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1911
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:19.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-2002
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:20.02
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
@@ -0,0 +1,17 @@
+FROM ubuntu:bionic as ubuntu-build
+RUN apt-get update && \
+    apt-get -y install \
+        build-essential \
+        git \
+        libnuma-dev
+
+ARG DPDK_VER='master'
+ENV DPDK_DIR='/dpdk'
+ENV RTE_TARGET='x86_64-native-linuxapp-gcc'
+RUN git clone -b $DPDK_VER -q --depth 1 http://dpdk.org/git/dpdk-stable $DPDK_DIR 2>&1
+RUN cd ${DPDK_DIR} && \
+    sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \
+    sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \
+    make config T=x86_64-native-linuxapp-gcc && \
+    make -j $CPUS
+ENV PATH="$PATH:$DPDK_DIR/build/app/"
@@ -119,7 +119,7 @@ patchesJson6902:
  # adds "networking.k8s.io" to ClusterRole's apiGroups
  - target:
      group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
      kind: ClusterRole
      name: nginx-ingress-clusterrole
    path: patch_clusterrole.yaml
@@ -137,7 +137,7 @@ itself contains the operation to perform, target path and value. The `rules/3/ap
 operation (in this case "add") at the `apiGroups:` list found under the 4th list item of `rules:`. 

 ```yaml
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: nginx-ingress-clusterrole
@@ -3,6 +3,11 @@
 ## Prerequisite
 This setup currently will work with Kubernetes 1.14 & above. Any version of Kubernetes before that might work, but is not guaranteed.

+## QUICK NOTE
+The version of Kubernetes* was bumped from 1.17.7 to 1.19.4 in Clear Linux* OS release 34090. The [guide](https://docs.01.org/clearlinux/latest/guides/clear/k8s-migration.html) and the Clear Linux OS bundle k8s-migration were created to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
+
+The new Clear Linux OS bundle k8s-migration was added in Clear Linux* OS release 34270. Please follow the guide for an upgrade.
+
 ## Sample multi-node vagrant setup

 To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
@@ -13,8 +18,8 @@ Follow instructions in the [Vagrant docs](https://www.vagrantup.com/intro/gettin

 Or, follow our [detailed steps](vagrant.md)

-Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
-Customize the setup using environment variables. E.g., `NODES=1 MEMORY=8192 CPUS=8 vagrant up --provider=libvirt`
+Now you have a 3 node cluster up and running. Each of them have 4 vCPU, 8GB Memory, 2x10GB disks, 1 additional private network.
+Customize the setup using environment variables. E.g., `NODES=2 MEMORY=16384 CPUS=8 vagrant up --provider=libvirt`

 To login to the master node and change to this directory

@@ -38,6 +43,14 @@ This script ensures the following
 script uses the runtime specified in the `RUNNER` environment variable and defaults to `crio`. To use the
 `containerd` runtime, set the `RUNNER` environment variable to `containerd`.

+In case of vagrant, if you want to spin up VM's using different environment variable than declared in [`setup_system.sh`],
+specify when performing vagrant up. E.g., `RUNNER=containerd vagrant up`
+
+### Specify a version of Clear Linux
+
+To specify a particular version of Clear Linux to use, set the CLRK8S_CLR_VER environment variable to the desired
+version before starting setup_system.sh (e.g. `CLRK8S_CLR_VER=31400 ./setup_system.sh`)
+
 ### Configuration for high numbers of pods per node

 In order to enable running greater than 110 pods per node, set the environment
@@ -55,6 +68,28 @@ devicemapper storage. This should not be used for production.

 > NOTE: This step is done automatically if using vagrant.

+### For HA, setup the load balancer node
+
+Ideally, the load balancer node will be a separate node. However, one of the
+master nodes can also serve as the load balancer for the cluster. [HAProxy](https://www.haproxy.org/)
+is used in these instructions.
+
+```bash
+sudo swupd bundle-add haproxy
+sudo systemctl enable haproxy
+```
+
+Edit the master IP addresses and load balancer address and ports in [`haproxy.cfg.example`](haproxy.cfg.example)
+to match the IPs for the new cluster. If using a master node for the load balancer
+make sure that the `frontend bind` port is different than the Kubernetes API port, 6443.
+If using a separate machine for load balancing, the port can be 6443 if desired.
+
+```bash
+sudo mkdir -p /etc/haproxy
+sudo cp haproxy.cfg.example /etc/haproxy/haproxy.cfg
+sudo systemctl start haproxy
+```
+
 ## Bring up the master

 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -65,6 +100,12 @@ you need to setup other cluster wide properties.
 There are different flavors to install, run `./create_stack.sh help` to get
 more information.

+> NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export
+the necessary environment variables if needed to be changed. By default it will use
+`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the
+Vagrant. If creating an HA cluster, make sure to specify `LOAD_BALANCER_IP` and
+`LOAD_BALANCER_PORT`.
+
 ```bash
 # default shows help
 ./create_stack.sh <subcommand>
@@ -73,6 +114,13 @@ more information.
 In order to enable running greater than 110 pods per node, set the environment
 variable `HIGH_POD_COUNT` to any non-empty value.

+If creating an HA cluster, join the other master nodes to the cluster.
+
+```bash
+kubeadm join <load-balancer-ip>:<load-balancer-port> --token <token> --discovery-token-ca-cert-hash <hash> \
+    --control-plane --certificate-key <certificate-key> --cri-socket=/run/crio/crio.sock
+```
+
 ## Join Workers to the cluster

 ```bash
@@ -81,6 +129,9 @@ kubeadm join <master-ip>:<master-port> --token <token> --discovery-token-ca-cert

 Note: Remember to append `--cri-socket=/run/crio/crio.sock` to the join command generated by the master.

+If creating an HA cluster, join the other worker nodes to the cluster. The same way,
+but replacing the `<master-ip>:<master-port>` with `<load-balancer-ip>:<load-balancer-port>`.
+
 On workers just use the join command that the master spits out. There nothing
 else you need to run on the worker. All the other Kubernetes customizations are pushed
 in from master via the values setup in the `kubeadm.yaml` file.
@@ -149,3 +200,18 @@ Grafana is available at this URL http://localhost:3000 . Default credentials are
 ## Cleaning up the cluster (Hard reset to a clean state)

 Run `reset_stack.sh` on all the nodes
+
+## Additional Components
+
+### Rook
+The default Rook configuration provided is intended for testing purposes only
+and is not suitable for a production environment. By default Rook is configured
+to provide local storage (/var/lib/rook) and will be provisioned differently
+depending on whether or not you startup a single node Kubernetes cluster, or
+a multiple node Kubernetes cluster. 
+
+- When starting up a single node Kubernetes cluster, Rook will be configured
+to start up a single replica, and will allow multiple monitors on the same node.
+- When multiple Kubernetes worker nodes are detected, Rook will be configured
+to startup a replica on each available node and will schedule monitor processes
+on separate nodes providing greater reliability.
@@ -6,8 +6,8 @@ require 'ipaddr'
 require 'securerandom'

 $num_instances = (ENV['NODES'] || 3).to_i
-$cpus = (ENV['CPUS'] || 2).to_i
-$memory = (ENV['MEMORY'] || 4096).to_i
+$cpus = (ENV['CPUS'] || 4).to_i
+$memory = (ENV['MEMORY'] || 8192).to_i
 $disks = 2
 # Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
 $disk_prefix = File.basename(File.dirname(__FILE__), "/")
@@ -16,13 +16,14 @@ $box = "AntonioMeireles/ClearLinux"
 $box_ver = (ENV['CLEAR_VBOX_VER'])
 File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-$base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("10.10.100.10")
 $hosts = {}
 $proxy_ip_list = "192.168.121.0/24"
 $driveletters = ('a'..'z').to_a
 $setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
 $runner = ENV.has_key?('RUNNER') ? ENV['RUNNER'].to_s : "crio".to_s
 $high_pod_count = ENV.has_key?('HIGH_POD_COUNT') ? ENV['HIGH_POD_COUNT'].to_s : ""
+$CLRK8S_CLR_VER = ENV.has_key?('CLRK8S_CLR_VER') ? ENV['CLRK8S_CLR_VER'].to_s : ""
 if !(["crio","containerd"].include? $runner)
  abort("it's either crio or containerd. Cannot do anything else")
 end
@@ -75,6 +76,7 @@ Vagrant.configure("2") do |config|
        lv.loader = $loader
        lv.cpus = $cpus
        lv.memory = $memory
+        lv.machine_virtual_size = 40
        (1..$disks).each do |d|
          lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
        end
@@ -97,7 +99,7 @@ Vagrant.configure("2") do |config|
      c.vm.provider :virtualbox do |_, override|
        override.vm.provision "shell", privileged: true, inline: "sudo mkdir -p /etc/profile.d; echo export MASTER_IP=#{$hosts["clr-01"]} > /etc/profile.d/cnsetup.sh"
      end
-      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner, "HIGH_POD_COUNT" => $high_pod_count}
+      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner, "HIGH_POD_COUNT" => $high_pod_count, "CLRK8S_CLR_VER" => $CLRK8S_CLR_VER}
      if $setup_fc
        if $runner == "crio".to_s
        c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0

-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
  name: pod-annotate-webhook
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0

-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Deployment
 metadata:
  name: pod-annotate-webhook
@@ -44,18 +44,21 @@ sudo systemctl enable --now containerd-devmapper
 # no. of feature arguments
 # Skip zeroing blocks for new volumes.
 sudo dmsetup create contd-thin-pool \
-  --table "0 2097152 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
+  --table "0 20971520 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"

 sudo mkdir -p /etc/containerd/
 if [ -f /etc/containerd/config.toml ]
 then
-  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"512MB\"|' /etc/containerd/config.toml
+  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"4096MB\"|' /etc/containerd/config.toml
 else
  cat<<EOT | sudo tee /etc/containerd/config.toml
 [plugins]
  [plugins.devmapper]
    pool_name = "contd-thin-pool"
-    base_image_size = "512MB"
+    base_image_size = "4096MB"
+  [plugins.cri]
+    [plugins.cri.containerd]
+      snapshotter = "devmapper"
 EOT
 fi

@@ -12,14 +12,31 @@ CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 : ${TOKEN:=}
 : ${MASTER_IP:=}
+: ${CERT_SANS:=}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
+LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""}
+LOAD_BALANCER_PORT="${LOAD_BALANCER_PORT:-6444}"

 # versions
-CANAL_VER="${CLRK8S_CANAL_VER:-v3.9}"
+CANAL_VER="${CLRK8S_CANAL_VER:-v3.24}"
+CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.9.13}"
+FLANNEL_VER="${CLRK8S_FLANNEL_VER:-v0.16.3}"
+CILIUM_VAL_OVERRIDE=""
 K8S_VER="${CLRK8S_K8S_VER:-}"
-ROOK_VER="${CLRK8S_ROOK_VER:-v1.1.1}"
-METRICS_VER="${CLRK8S_METRICS_VER:-v0.3.5}"
-PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-f458e85e5d7675f7bc253072e1b4c8892b51af0f}"
+KATA_VER="${CLRK8S_KATA_VER:-2.4.0}"
+ROOK_VER="${CLRK8S_ROOK_VER:-v1.8.10}"
+METRICS_VER="${CLRK8S_METRICS_VER:-v0.6.1}"
+DASHBOARD_VER="${CLRK8S_DASHBOARD_VER:-v2.6.1}"
+INGRES_VER="${CLRK8S_INGRES_VER:-controller-v1.3.0}"
+EFK_VER="${CLRK8S_EFK_VER:-193692c92eb4667b8f4fb7d4cdf0462e229b5f13}"
+METALLB_VER="${CLRK8S_METALLB_VER:-v0.8.3}"
+NPD_VER="${CLRK8S_NPD_VER:-v0.6.6}"
+PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-v0.10.0}"
+CNI=${CLRK8S_CNI:-"canal"}
+if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi
+
+NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}"
+mode="multinode"

 function print_usage_exit() {
 	exit_code=${1:-0}
@@ -45,17 +62,30 @@ trap finish EXIT
 function cluster_init() {
 	# Config replacements
 	if ! [ -z ${TOKEN} ]; then
-		sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm/base/kubeadm.yaml
+		sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml
 	fi
 	if ! [ -z ${MASTER_IP} ]; then
-		sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm/base/kubeadm.yaml
+		sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml
 	fi
-	if [[ -n "$K8S_VER" && $(grep -c kubernetesVersion ./kubeadm/base/kubeadm.yaml) -eq 0 ]]; then
-		sed -i "s/ClusterConfiguration/ClusterConfiguration\nkubernetesVersion: ${K8S_VER}/g" ./kubeadm/base/kubeadm.yaml
+	if [[ -n "$K8S_VER" && $(grep -c kubernetesVersion ./kubeadm.yaml) -eq 0 ]]; then
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\nkubernetesVersion: ${K8S_VER}/g" ./kubeadm.yaml
+	fi
+	if [[ -n "$CERT_SANS" ]]; then
+		if [[ $(grep -c certSANs ./kubeadm.yaml) -gt 0 ]]; then
+			sed -i '/certSANs/,/[a-zA-Z]*:/{//!d}' ./kubeadm.yaml
+		else
+			sed -i "/ClusterConfiguration/a apiServer:\\n  certSANs:" ./kubeadm.yaml
+		fi
+		for CERT_SAN in ${CERT_SANS[@]}; do
+			sed -i "/certSANs/a \ \ - ${CERT_SAN}" ./kubeadm.yaml
+		done
 	fi
-	# Config patches
 	if [[ -n "${HIGH_POD_COUNT}" ]]; then
-		echo "$(kubectl kustomize ./kubeadm/high-pod-count/)" > ./kubeadm/base/kubeadm.yaml
+		# increase limits in kubelet
+		sed -i "/KubeletConfiguration/a maxOpenFiles\: 1048576" ./kubeadm.yaml
+		sed -i "/KubeletConfiguration/a maxPods\: 5000" ./kubeadm.yaml
+		# increase the address range per node
+		sed -i "/ClusterConfiguration/a controllerManager:\\n  extraArgs:\\n    node-cidr-mask-size: \"20\"" ./kubeadm.yaml
 	fi
 	#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
 	#to enable the RuntimeClass featuregate
@@ -63,7 +93,13 @@ function cluster_init() {
 		echo "/var/lib/etcd exists! skipping init."
 		return
 	fi
-	sudo -E kubeadm init --config=./kubeadm/base/kubeadm.yaml
+
+	if [[ -n "${LOAD_BALANCER_IP}" ]]; then
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\ncontrolPlaneEndpoint: ${LOAD_BALANCER_IP}:${LOAD_BALANCER_PORT}/g" ./kubeadm.yaml
+	fi
+	# upload-certs will automatically upload certificates that should be shared
+	# across control-plane nodes in HA clusters. It is harmless in non-HA cases.
+	sudo -E kubeadm init --upload-certs --config=./kubeadm.yaml

 	rm -rf "${HOME}/.kube"
 	mkdir -p "${HOME}/.kube"
@@ -79,59 +115,97 @@ function cluster_init() {
 		fi
 	fi

-	#Ensure single node k8s works
+	#Ensure single node k8s works both pre and post v1.25
 	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
-		kubectl taint nodes --all node-role.kubernetes.io/master-
+		minor=$(kubeadm version -o short | cut -f 2 -d "." )
+		if [ $minor -ge "25" ]; then
+			kubectl taint nodes --all  node-role.kubernetes.io/control-plane-
+		elif [ $minor -eq "24" ]; then
+			kubectl taint nodes --all  node-role.kubernetes.io/control-plane-
+			kubectl taint nodes --all  node-role.kubernetes.io/master-
+		else
+			kubectl taint nodes --all  node-role.kubernetes.io/master-
+		fi
+		mode="standalone"
 	fi
 }

 function kata() {
-	KATA_VER=${1:-1.8.2-kernel-config}
-	KATA_URL="https://github.com/kata-containers/packaging.git"
+	KATA_VER=${1:-$KATA_VER}
+	KATA_URL="https://github.com/kata-containers/kata-containers.git"
 	KATA_DIR="8-kata"
 	get_repo "${KATA_URL}" "${KATA_DIR}/overlays/${KATA_VER}"
-	set_repo_version "${KATA_VER}" "${KATA_DIR}/overlays/${KATA_VER}/packaging"
-	kubectl apply -k "${KATA_DIR}/overlays/${KATA_VER}"
+	set_repo_version "${KATA_VER}" "${KATA_DIR}/overlays/${KATA_VER}/kata-containers"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"

 }

 function cni() {
-	# note version is not semver
-	CANAL_VER=${1:-$CANAL_VER}
-	CANAL_URL="https://docs.projectcalico.org/${CANAL_VER}/manifests"
-	if [[ "$CANAL_VER" == "v3.3" ]]; then
-		CANAL_URL="https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/canal"
-	fi
-	CANAL_DIR="0-canal"
+	case "$CNI" in
+	canal)
+		# note version is not semver
+		CANAL_VER=${1:-$CANAL_VER}
+		CANAL_URL="https://projectcalico.docs.tigera.io/archive/${CANAL_VER}/manifests"
+		if [[ "$CANAL_VER" == "v3.3" ]]; then
+			CANAL_URL="https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/canal"
+		fi
+		CANAL_DIR="0-canal"

-	# canal manifests are not kept in repo but in docs site so use curl
-	mkdir -p "${CANAL_DIR}/overlays/${CANAL_VER}/canal"
-	curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
-	if [[ "$CANAL_VER" == "v3.3" ]]; then
-		curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/rbac.yaml" "$CANAL_URL/rbac.yaml"
-	fi
-	# canal doesnt pass kustomize validation
-	kubectl apply -k "${CANAL_DIR}/overlays/${CANAL_VER}" --validate=false
+		# canal manifests are not kept in repo but in docs site so use curl
+		mkdir -p "${CANAL_DIR}/overlays/${CANAL_VER}/canal"
+		curl -L -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
+		if [[ "$CANAL_VER" == "v3.3" ]]; then
+			curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/rbac.yaml" "$CANAL_URL/rbac.yaml"
+		fi
+		# canal doesnt pass kustomize validation
+		kubectl apply -k "${CANAL_DIR}/overlays/${CANAL_VER}" --validate=false
+		;;
+	flannel)
+		FLANNEL_VER=${1:-$FLANNEL_VER}
+		FLANNEL_URL="https://github.com/flannel-io/flannel"
+		FLANNEL_DIR="0-flannel"

+		get_repo "${FLANNEL_URL}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
+		set_repo_version "${FLANNEL_VER}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}/flannel"
+		kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
+		;;
+	cilium)
+		local podsubnet=$(grep -Po 'podSubnet:\ \K[^*]*' ${SCRIPT_DIR}/kubeadm.yaml)
+		CILIUM_VER=${1:-$CILIUM_VER}
+		CILIUM_URL="https://github.com/cilium/cilium.git"
+		CILIUM_DIR="0-cilium"
+
+		get_repo "${CILIUM_URL}" "${CILIUM_DIR}/overlays/${CILIUM_VER}"
+		set_repo_version "${CILIUM_VER}" "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/"
+		if [ -f "${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml" ]; then
+			CILIUM_VAL_OVERRIDE="--values ${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml"
+		fi
+		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set containerRuntime.integration="$RUNNER" --set hubble.enabled=false --set ipam.operator.clusterPoolIPv4PodCIDR="${podsubnet}" | kubectl apply -f -
+		;;
+	*)
+		echo"Unknown cni $CNI"
+		exit 1
+		;;
+	esac
 }

 function metrics() {
 	METRICS_VER="${1:-$METRICS_VER}"
-	METRICS_URL="https://github.com/kubernetes-incubator/metrics-server.git"
+	METRICS_URL="https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_VER}/components.yaml"
 	METRICS_DIR="1-core-metrics"
-	get_repo "${METRICS_URL}" "${METRICS_DIR}/overlays/${METRICS_VER}"
-	set_repo_version "${METRICS_VER}" "${METRICS_DIR}/overlays/${METRICS_VER}/metrics-server"
-	kubectl apply -k "${METRICS_DIR}/overlays/${METRICS_VER}"
+
+	curl -L ${METRICS_URL} --output - >${METRICS_DIR}/overlays/${METRICS_VER}/components.yaml
+	kubectl apply -k "${METRICS_DIR}/overlays/${METRICS_VER}/"

 }
 function wait_on_pvc() {
 	# create and destroy pvc until successful
 	while [[ $(kubectl get pvc test-pv-claim --no-headers | grep Bound -c) -ne 1 ]]; do
 		sleep 30
-		kubectl delete pvc test-pv-claim
-		create_pvc
-		sleep 10
 	done
+	kubectl delete pvc test-pv-claim
 }
 function create_pvc() {
 	kubectl apply -f - <<HERE
@@ -156,10 +230,20 @@ function storage() {
 	ROOK_URL="https://github.com/rook/rook.git"
 	ROOK_DIR=7-rook

+	# This function might be called standalone, so good to check the mode we are in.
+	if [ "$(kubectl get nodes --no-headers | wc -l)" -eq 1 ]; then
+		mode="standalone"
+	fi
+
 	# get and apply rook
-	get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}"
-	set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/rook"
-	kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}"
+	get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
+	set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook"
+	kubectl apply -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/crds.yaml -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/common.yaml -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/operator.yaml
+	while [[ $(kubectl get po -n rook-ceph --field-selector=status.phase=Running | grep -e 'operator' -c) -lt 1 ]]; do
+		echo "Waiting on operator"
+		sleep 10
+	done
+	kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
 	# wait for the rook OSDs to run which means rooks should be ready
 	while [[ $(kubectl get po --all-namespaces | grep -e 'osd.*Running.*' -c) -lt 1 ]]; do
 		echo "Waiting for Rook OSD"
@@ -180,13 +264,15 @@ function monitoring() {
 	PROMETHEUS_DIR="4-kube-prometheus"
 	get_repo "${PROMETHEUS_URL}" "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}"
 	set_repo_version "${PROMETHEUS_VER}" "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/kube-prometheus"
-	kubectl apply -k "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}"
+	kubectl apply --server-side -f "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/kube-prometheus/manifests/setup/"

-	while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com >/dev/null 2>&1) || $? -ne 0 ]]; do
-		echo "Waiting for Prometheus CRDs"
-		sleep 2
+	while ! [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com) ]]; do
+		echo "Waiting for prometheus crds"
+		sleep 10
 	done

+	kubectl apply -k "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/"
+
 	#Expose the dashboards
 	#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
 	#kubectl --namespace monitoring port-forward svc/grafana 3000 &
@@ -194,16 +280,18 @@ function monitoring() {
 }

 function dashboard() {
-	DASHBOARD_VER=${1:-v2.0.0-beta2}
+	DASHBOARD_VER=${1:-$DASHBOARD_VER}
 	DASHBOARD_URL="https://github.com/kubernetes/dashboard.git"
 	DASHBOARD_DIR="2-dashboard"
 	get_repo "${DASHBOARD_URL}" "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}"
 	set_repo_version "${DASHBOARD_VER}" "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}/dashboard"
 	kubectl apply -k "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}"
+
+	echo 'Run "kubectl -n kubernetes-dashboard create token admin-user" to create an admin token'
 }

 function ingres() {
-	INGRES_VER=${1:-nginx-0.25.1}
+	INGRES_VER=${1:-$INGRES_VER}
 	INGRES_URL="https://github.com/kubernetes/ingress-nginx.git"
 	INGRES_DIR="5-ingres-lb"
 	get_repo "${INGRES_URL}" "${INGRES_DIR}/overlays/${INGRES_VER}"
@@ -212,17 +300,17 @@ function ingres() {
 }

 function efk() {
-	EFK_VER=${1:-v1.15.1}
-	EFK_URL="https://github.com/kubernetes/kubernetes.git"
+	EFK_VER=${1:-$EFK_VER}
+	EFK_URL="https://github.com/kubernetes-sigs/instrumentation-addons.git"
 	EFK_DIR="3-efk"
 	get_repo "${EFK_URL}" "${EFK_DIR}/overlays/${EFK_VER}"
-	set_repo_version "${EFK_VER}" "${EFK_DIR}/overlays/${EFK_VER}/kubernetes"
+	set_repo_version "${EFK_VER}" "${EFK_DIR}/overlays/${EFK_VER}/instrumentation-addons"
 	kubectl apply -k "${EFK_DIR}/overlays/${EFK_VER}"

 }

 function metallb() {
-	METALLB_VER=${1:-v0.8.1}
+	METALLB_VER=${1:-$METALLB_VER}
 	METALLB_URL="https://github.com/danderson/metallb.git"
 	METALLB_DIR="6-metal-lb"
 	get_repo "${METALLB_URL}" "${METALLB_DIR}/overlays/${METALLB_VER}"
@@ -230,9 +318,8 @@ function metallb() {
 	kubectl apply -k "${METALLB_DIR}/overlays/${METALLB_VER}"

 }
-
 function npd() {
-	NPD_VER=${1:-v0.6.6}
+	NPD_VER=${1:-$NPD_VER}
 	NPD_URL="https://github.com/kubernetes/node-problem-detector.git"
 	NPD_DIR="node-problem-detector"
 	get_repo "${NPD_URL}" "${NPD_DIR}/overlays/${NPD_VER}"
@@ -240,6 +327,16 @@ function npd() {
 	kubectl apply -k "${NPD_DIR}/overlays/${NPD_VER}"
 }

+# node feature discovery
+function nfd() {
+	NFD_VER=${1:-$NFD_VER}
+	NFD_URL="https://github.com/kubernetes-sigs/node-feature-discovery.git"
+	NFD_DIR="node-feature-discovery"
+	get_repo "${NFD_URL}" "${NFD_DIR}/overlays/${NFD_VER}"
+	set_repo_version "${NFD_VER}" "${NFD_DIR}/overlays/${NFD_VER}/node-feature-discovery"
+	kubectl apply -k "${NFD_DIR}/overlays/${NFD_VER}"
+}
+
 function miscellaneous() {

 	# dashboard
@@ -268,7 +365,7 @@ function minimal() {

 function all() {
 	minimal
-	storage
+  storage
 	monitoring
 	miscellaneous
 }
@@ -287,7 +384,7 @@ function set_repo_version() {
 	pushd "$(pwd)"
 	cd "${path}"
 	git fetch origin "${ver}"
-	git checkout "${ver}"
+	git -c advice.detachedHead=false checkout "${ver}"
 	popd

 }
@@ -306,6 +403,12 @@ command_handlers[storage]=storage
 command_handlers[monitoring]=monitoring
 command_handlers[metallb]=metallb
 command_handlers[npd]=npd
+command_handlers[nfd]=nfd
+command_handlers[kata]=kata
+command_handlers[metrics]=metrics
+command_handlers[dashboard]=dashboard
+command_handlers[efk]=efk
+command_handlers[ingres]=ingres

 declare -A command_help
 command_help[init]="Only inits a cluster using kubeadm"
@@ -313,6 +416,7 @@ command_help[cni]="Setup network for running cluster"
 command_help[minimal]="init + cni +  kata + metrics"
 command_help[all]="minimal + storage + monitoring + miscellaneous"
 command_help[help]="show this message"
+command_help[nfd]="node feature discovery"

 cd "${SCRIPT_DIR}"

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+
+###
+# update_checker.sh
+# Parses create_stack.sh for urls and versions
+# Curls urls for latest version and reports the comparison
+##
+
+# first argument is path to create_stach.sh
+COMPONENT_FILE="${1:-$create_stack.sh}"
+# set CLRK8S_DEBUG=1 for debug output
+DEBUG=${CLRK8S_DEBUG:-0}
+# set CLRK8S_NO_COLOR=1 for no colors
+NO_COLOR=${CLRK8S_NO_COLOR:-""}
+# set CLRK8S_ALL=1 for all results, not just changed
+ALL=${CLRK8S_ALL:-""}
+# add components to skip (not check)
+# - canal doesn't use git repo tags for revisions
+declare -a COMPONENT_SKIP=( CANAL )
+
+# internal vars
+declare -A COMPONENT_VER
+declare -A COMPONENT_URL
+LATEST_URL=""
+
+# usage prints help and exit
+function usage(){
+	echo "Compare default component versions to latest release"
+	echo "usage: update_checker.sh <path to create_stack.sh>"
+	exit 0
+}
+# log echoes to stdout
+function log(){
+	echo "$1"
+}
+# debug echoes to stdout if debug is enabled
+function debug(){
+	if [[ "${DEBUG}" -ne 0 ]]; then
+	  echo "$1"
+	fi
+}
+# extract_component_data scans file for component versions and urls and add them to maps
+function extract_component_data(){
+	file=${1:-$COMPONENT_FILE}
+	name=""
+	version=""
+	url=""
+	while read -r line
+	do
+		# versions
+	  if [[ $line =~ "_VER=" ]]; then
+			debug "Found component version $line"
+			name=${line%%_*}
+			if [[ "$COMPONENT_SKIP" =~ (^|[[:space:]])"$name"($|[[:space:]]) ]]; then
+				debug "Skipping component $name"
+				continue
+			fi
+			versions=${line#*=}
+			if [[ $versions =~ ":-" ]]; then
+				version=${line#*:-}
+			fi
+			# cleanup value
+			version=${version%\}}
+			version=${version%\}\"}
+			if [[ -n "$name" && ${COMPONENT_VER[$name]} == "" ]]; then
+				debug "Adding component $name=$version to COMPONENT_VER"
+				COMPONENT_VER[$name]=$version
+	  	fi
+	  fi
+
+	  # urls
+	  if [[ $line =~ "_URL=" ]]; then
+	    debug "Found component URL $line"
+			name=${line%%_*}
+			if [[ $COMPONENT_SKIP =~ (^|[[:space:]])"$name"($|[[:space:]]) ]]; then
+				debug "Skipping component $name"
+				continue
+			fi
+			urls=${line#*=}
+
+			if [[ $urls =~ ":-" ]]; then
+				urls=${line#*:-}
+			fi
+			# cleanup value
+			url=${urls%\"}
+			url=${url#\"}
+			if [[ -n "$name" && ${COMPONENT_URL[$name]} == "" ]]; then
+				debug "Adding component $name=$url to COMPONENT_URL"
+				COMPONENT_URL[$name]=$url
+	  	fi
+	  fi
+
+	done < $file
+
+}
+# resolve_latest_url extracts the real release/latest url from a repo url
+function resolve_latest_url(){
+	repo=$1
+	url=${repo%.git*}/releases/latest
+	LATEST_URL=$(curl -Ls -o /dev/null -w %{url_effective} $url)
+	if [[ "$?" -gt 0 ]]; then
+		echo "curl error, exiting."
+		exit 1
+	fi
+}
+# function_exists checks if a function exists
+function function_exists() {
+    declare -f -F "$1" > /dev/null
+    return $?
+}
+function report(){
+	if [[ -z $NO_COLOR ]]; then
+		BOLD="\e[1m\e[33m"
+		BOLD_OFF="\e[0m"
+  fi
+  mode="changed"
+  out=""
+  if [[ -n "$1" ]]; then
+  	mode="all"
+  fi
+	out+="\n"
+	out+="Components ($mode)\n"
+	out+="--------------------------\n"
+	echo -e $out
+	out="NAME CURRENT LATEST\n"
+	# loop thru each url, get latest version and report
+	for k in "${!COMPONENT_URL[@]}";
+	do
+		name=$k
+		resolve_latest_url "${COMPONENT_URL[$k]}"
+		latest_url="$LATEST_URL"
+		latest_ver="${latest_url#*tag/}"
+		current_ver="${COMPONENT_VER[$name]}"
+		if [[ "${current_ver}" != "${latest_ver}" ]]; then
+			out+="$name $current_ver $BOLD $latest_ver $BOLD_OFF\n";
+		fi
+		if [[ "${current_ver}" == "${latest_ver}" && -n $ALL ]]; then
+			out+="$name $current_ver $latest_ver\n";
+		fi
+	done;
+	echo -e "${out}" | column -t
+	if [[ "${#COMPONENT_SKIP[@]}" -gt 0 ]]; then
+		echo "---"
+		echo "WARNING: Skipped comparisions for the following components:"
+		for s in "${!COMPONENT_SKIP[@]}"
+		do
+			echo "${COMPONENT_SKIP[$s]}"
+		done
+	fi
+}
+###
+# Main
+##
+
+# print help if no args
+if [[ "$#" -eq 0 ]]; then
+	usage
+fi
+
+# get the versions
+extract_component_data "$1"
+# output
+report "${ALL}"
+
+
+
@@ -0,0 +1,34 @@
+global
+    log /dev/log    local0
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy-master.sock mode 660 level admin
+    stats timeout 30s
+    user haproxy
+    group haproxy
+    daemon
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
+    ssl-default-bind-options no-sslv3
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+    timeout tunnel  4h
+frontend kubernetes
+    bind 10.0.0.100:6444
+    option tcplog
+    mode tcp
+    default_backend kubernetes-master-nodes
+
+backend kubernetes-master-nodes
+    mode tcp
+    balance source
+    option tcp-check
+    server master-1 10.0.0.100:6443 check fall 3 rise 2
+    server master-2 10.0.0.101:6443 check fall 3 rise 2
+    server master-3 10.0.0.102:6443 check fall 3 rise 2
@@ -1,15 +1,9 @@
-apiVersion: kubeadm.k8s.io/v1beta1
+apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
-metadata:
-  name: init-config
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-metadata:
-  name: kubelet-config
 cgroupDriver: systemd
-# Allowing for CPU pinning and isolation in case of guaranteed QoS class
-cpuManagerPolicy: static
 systemReserved:
  cpu: 500m
  memory: 256M
@@ -17,11 +11,10 @@ kubeReserved:
  cpu: 500m
  memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1beta1
+apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
-metadata:
-  name: cluster-config
 networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
+
@@ -1,2 +0,0 @@
-resources:
- kubeadm.yaml
@@ -1,3 +0,0 @@
-[
-  {"op": "add", "path": "/controllerManager", "value": {"extraArgs": {"node-cidr-mask-size": "20"}}}
-]
@@ -1,4 +0,0 @@
-[
-  {"op": "add", "path": "/maxOpenFiles", "value": 1048576},
-  {"op": "add", "path": "/maxPods", "value": 5000}
-]
@@ -1,17 +0,0 @@
-bases:
- ../base/
-patchesJson6902:
-# increase limits in kubelet
- target:
-    group: kubelet.config.k8s.io
-    version: v1beta1
-    kind: KubeletConfiguration
-    name: kubelet-config
-  path: kubelet-config.json
-# increase the address range per node
- target:
-    group: kubeadm.k8s.io
-    version: v1beta1
-    kind: ClusterConfiguration
-    name: cluster-config
-  path: cluster-config.json
@@ -0,0 +1,5 @@
+resources:
+  - node-feature-discovery/nfd-daemonset-combined.yaml.template
+  - node-feature-discovery/nfd-worker-daemonset.yaml.template
+
+
@@ -52,3 +52,4 @@ sudo systemctl is-enabled containerd && sudo systemctl restart containerd
 sudo systemctl restart kubelet

 reset_cluster
+sudo -E bash -c "rm -rf /var/lib/containerd/*"
@@ -1,19 +1,24 @@
 #!/usr/bin/env bash
-
 set -o errexit
 set -o nounset

 # global vars
 CLRK8S_OS=${CLRK8S_OS:-""}
+CLR_VER=${CLRK8S_CLR_VER:-""}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}

 # set no proxy
-ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12"
-ADD_NO_PROXY+=",$(hostname -I | sed 's/[[:space:]]/,/g')"
-: "${RUNNER:=crio}"
+NO_PROXY_ARRAY=(.svc 10.0.0.0/8 )
+NO_PROXY_ARRAY+=( $(hostname -I | awk -F. '{print $1"."$2"."$3".0/24"}'))
+
+if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-crio}"; fi

 # update os version
 function upate_os_version() {
+	if [[ -n "${CLR_VER}" ]]; then
+		sudo swupd repair -m "${CLR_VER}" --picky --force
+		return
+	fi
 	sudo swupd update
 }

@@ -24,13 +29,10 @@ function add_os_deps() {

 # permanently disable swap
 function disable_swap() {
-	swapcount=$(sudo grep '^/dev/\([0-9a-z]*\).*' /proc/swaps | wc -l)
-
-	if [ "$swapcount" != "0" ]; then
-		sudo systemctl mask "$(sed -n -e 's#^/dev/\([0-9a-z]*\).*#dev-\1.swap#p' /proc/swaps)" 2>/dev/null
-	else
-		echo "Swap not enabled"
-	fi
+	# disable current swap
+	sudo swapoff -a
+	# permanently disable swap
+	sudo systemctl mask swap.target
 }

 # enable ip forwarding
@@ -132,37 +134,42 @@ function enable_kubelet_runner() {

 # ensure that the system is ready without requiring a reboot
 function ensure_system_ready() {
-	sudo swapoff -a
 	sudo systemctl restart systemd-modules-load.service
 }

 # add proxy if found
 function setup_proxy() {
 	set +o nounset
+	set +o errexit
 	if [[ ${http_proxy} ]] || [[ ${HTTP_PROXY} ]]; then
 		echo "Setting up proxy stuff...."
 		# Setup IP for users too
+		for ip in "${NO_PROXY_ARRAY[@]}"
+                do
+                   result=`grep no_proxy /etc/profile.d/proxy.sh | grep $ip`
+                   [ -z "$result" ] && ADD_NO_PROXY+="$ip,"
+                done
 		sed_val=${ADD_NO_PROXY//\//\\/}
 		[ -f /etc/environment ] && sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
 		if [ -f /etc/profile.d/proxy.sh ]; then
-			sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+			sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/profile.d/proxy.sh
 		else
 			echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
 		fi

-		services=("${RUNNER}" 'kubelet')
-		for s in "${services[@]}"; do
-			sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
-			cat <<EOF | sudo bash -c "cat > /etc/systemd/system/${s}.service.d/proxy.conf"
-[Service]
-Environment="HTTP_PROXY=${http_proxy}"
-Environment="HTTPS_PROXY=${https_proxy}"
-Environment="SOCKS_PROXY=${socks_proxy}"
-Environment="NO_PROXY=${no_proxy},${ADD_NO_PROXY}"
+		sudo mkdir -p /etc/systemd/system.conf.d
+		cat <<EOF | sudo bash -c "cat > /etc/systemd/system.conf.d/proxy.conf"
+[Manager]
+DefaultEnvironment="HTTP_PROXY=${http_proxy}"
+DefaultEnvironment="HTTPS_PROXY=${https_proxy}"
+DefaultEnvironment="SOCKS_PROXY=${socks_proxy}"
+DefaultEnvironment="NO_PROXY=${no_proxy},${ADD_NO_PROXY}"
 EOF
-		done
-	fi
-	set -o nounset
+
+        sudo systemctl daemon-reexec
+        fi
+        set -o nounset
+        set -o errexit
 }

 # init for performing any pre tasks
@@ -181,12 +188,12 @@ fi

 echo "Init..."
 init
+echo "Disabling swap..."
+disable_swap
 echo "Setting OS Version..."
 upate_os_version
 echo "Adding OS Dependencies..."
 add_os_deps
-echo "Disabling swap..."
-disable_swap
 echo "Enabling IP Forwarding..."
 enable_ip_forwarding
 echo "Setting up modules to load..."
@@ -199,7 +206,7 @@ if [[ -n "${HIGH_POD_COUNT}" ]]; then
 fi
 echo "Reloading daemons..."
 daemon_reload
-echo "Enabling Kublet runner..."
+echo "Enabling kubelet runner..."
 enable_kubelet_runner
 echo "Ensuring system is ready..."
 ensure_system_ready
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
  name: php-apache-kata
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
  name: php-apache-runc
@@ -1,132 +1,83 @@
+* [Metric testing for scaling on Kubernetes.](#metric-testing-for-scaling-on-kubernetes)
+    * [Results storage and analysis](#results-storage-and-analysis)
+* [Developers](#developers)
+    * [Metrics gathering](#metrics-gathering)
+        * [`collectd` statistics](#collectd-statistics)
+        * [privileged statistics pods](#privileged-statistics-pods)
+    * [Configuring constant 'loads'](#configuring-constant-loads)
+
 # Metric testing for scaling on Kubernetes.

 This folder contains tools to aid in measuring the scaling capabilities of
 Kubernetes clusters.

+Primarily these tools were designed to measure scaling of large number of pods on a single node, but
+the code is structured to handle multiple nodes, and may also be useful in that scenario.
+
 The tools tend to take one of two forms:

- Tools to take measurements
+- Tools to launch jobs and take measurements
 - Tools to analyse results

 For more details, see individual sub-folders. A brief summary of available tools
 is below:

-| Tool | Description |
+| Folder | Description |
 | ---- | ----------- |
+| collectd | `collectd` based statistics/metrics gathering daemonset code |
 | lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis |
-| scaling | Tests to measure scaling, such as linear or parallel launching of pods |
-| report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results |
-
+| lib/[cpu-load*](lib/cpu-load.md) | Helper functions to enable CPU load generation on a cluster whilst under test |
+| [report](report/README.md) | Rmarkdown based report generator, used to produce a PDF comparison report of one or more sets of results |
+| [scaling](scaling/README.md) | Tests to measure scaling, such as linear or parallel launching of pods |

 ## Results storage and analysis

-The tools generate JSON formatted results files via the `lib/json.bash` functions. The `metrics_json_save()`
-function in that file has the ability to also `curl` or `socat` the JSON results to a database defined
+The tools generate JSON formatted results files via the [`lib/json.bash`](lib/json.bash) functions. The `metrics_json_save()`
+function has the ability to also `curl` or `socat` the JSON results to a database defined
 by environment variables (see the file source for details). This method has been used to store results in
 Elasticsearch and InfluxDB databases for instance, but should be adaptable to use with any REST API that accepts
 JSON input.

-## Scaling execution
-This section describes a complete step-by-step scaling execution up to results reporting by using `scaling/k8s_scale.sh` tool which launches a series of workloads and take memory metric measurements after each launch.
+## Prerequisites
+
+There are some basic pre-requisites required in order to run the test and process the results:

-**Requirements**
 * A Kubernetes cluster up and running (tested on v1.15.3).
 * `bc` and `jq` packages.
 * Docker (only for report generation).

-The steps to execute a run of the scaling framework are listed below, which need to be executed on the master node of a Kubernetes cluster to avoid network issues:
-1. Clone `cloud-native-setup` repository into a preferred directory and change directory up to `cloud-native-setup/metrics`:
-   ```sh
-   $ git clone https://github.com/clearlinux/cloud-native-setup.git
-   $ cd cloud-native-setup/metrics
-   ```
-2. Launch the execution by:
-   ```sh
-   $ ./scaling/k8s_scale.sh
-   INFO: Initialising
-   command: bc: yes
-   command: jq: yes
-   INFO: Checking Kubernetes accessible
-   INFO: 1 Kubernetes nodes in 'Ready' state found
-   starting kubectl proxy
-   Starting to serve on 127.0.0.1:8090
-   daemonset.apps/stats created
-   Waiting for daemon set "stats" rollout to finish: 0 of 1 updated pods are available...
-   daemon set "stats" successfully rolled out
-   INFO: Running test
-   INFO: And grab some stats
-   INFO: idle [98.49] free [29031100] launch [0] node [clr-30f01b5149ba4ab8b05a7ee03b6812a5] inodes_free [31103039]
-   INFO: Testing replicas 1 of 20
-   INFO: Content of runtime_command=:/@RUNTIMECLASS@/d
-   ...
-   ```
-The above execution might take about 4min because it launch up to 20 pods by default and takes measurements for CPU utilization, memory utilization and pod boot time, finally it will generate a `k8s-scaling.json` result file at `result` directory.
+# Developers

-**Note**: to test the launch of pods concurrently, `k8s_parallel.sh` may be used. For quicker testing, `k8s_scale_rapid.sh` can be used in place of `k8s_scale.sh`. The rest of the launch instructions remain consistent other than script name.
+Below are some architecture and internal details of how the code is structured and configured. This will be
+helpful for improving, modifying or submitting fixes to the code base.

-**Note**: by default the scaling framework makes call to the Kubernetes API directly so, if facing connectivity issues verify that `kubelet` service's proxies and `no_proxy` environment variable are properly setup.
+## Metrics gathering

-**Note**: by default the scaling framework uses default values for all its required variables, which can be checked through `scaling/k8s_scale.sh -h` and updated when launching the execution, i.e.:
-```
-$ ./scaling/k8s_scale.sh -h
-Usage: ./scaling/k8s_scale.sh [-h] [options]
-   Description:
-	Launch a series of workloads and take memory metric measurements after
-	each launch.
-   Options:
-		-h,    Help page.
+Metrics can be gathered using either a daemonset deployment of privileged pods used to gather statistics
+directly from the nodes using a combination of `mpstat`, `free` and `df`, or a daemonset deployment based
+around `collectd`. The general recommendation is to use the `collectd` based collection if possible, as it
+is more efficient, as the system does not have to poll and wait for results, and thus executes the test
+cycle faster. The `collectd` results are collected asyncronously, and the report generator code later
+aligns the results with the pod execution in the timeline.

-Environment variables:
-	Name (default)
-		Description
-	TEST_NAME (k8s scaling)
-		Can be set to over-ride the default JSON results filename
-	NUM_PODS (20)
-		Number of pods to launch
-	STEP (1)
-		Number of pods to launch per cycle
-	wait_time (30)
-		Seconds to wait for pods to become ready
-	delete_wait_time (600)
-		Seconds to wait for all pods to be deleted
-	settle_time (5)
-		Seconds to wait after pods ready before taking measurements
-	use_api (yes)
-		specify yes or no to use the API to launch pods
-	grace (30)
-		specify the grace period in seconds for workload pod termination
+### `collectd` statistics

-$ use_api=no ./scaling/k8s_scale.sh
-```
+The `collected` based code can be found in the `collectd` subdirectory. It uses the `collected` configuration
+found in the `collectd.conf` file to gather statistics, and store the results on the nodes themselves whilst
+tests are running. At the end of the test, the results are copied from the nodes and stored in the results
+directory for later processing.

-The steps to generate the result report are listed below:
+The `collectd` statistics are only configured and gathered if the environment variable `SMF_USE_COLLECTD`
+is set to non-empty by the test code (that is, it is only enabled upon request).

-1. Having the `results/k8s-scaling.json` result file, create a subdirectory in the `results` directory with a preferred name and copy the `k8s-scaling.json` file into it, so the file distribution looks like:
-   ```sh
-   $ tree result
-   results/
-   └── scaling
-       └── k8s-scaling.json
-   ```
+### privileged statistics pods

-**Note**: if `k8s_scale_rapid.sh` was run instead of `k8s_scale.sh`, that the `<node_name>.tar.gz` files that appear in the results directory also need to be copied into the newly created subdirectory. And the results file is named `k8s-rapid.json` rather than `k8s-scaling.json`.
-If k8s_parallel.sh was run, the results file is named `k8s-parallel.json` rather than `k8s-scaling.json`.
+The privileged statistics pods `YAML` can be found in the [`scaling/stats.yaml`](scaling/stats.yaml) file.
+An example of how to invoke and use this daemonset to extract statistics can be found in the
+[`scaling/k8s_scale.sh`](scaling/k8s_scale.sh) file.

-2. Launch the report generation by:
-   ```sh
-   ./report/makereport.sh
-   ```
-   **Note**: the first time you launch the report generation it will build a docker container to generate the reports and this process can take several minutes. Subsequent runs will be much faster.
+## Configuring constant 'loads'

-   The above execution will generate a `report/output` directory with the final reports, such as:
-   ```sh
-   $ tree report/output/
-   report/output/
-   ├── dut-1.png
-   ├── metrics_report.pdf
-   ├── scaling-1.png
-   ├── scaling-2.png
-   ├── scaling-3.png
-   └── scaling-4.png
-   ```
-More details about result reporting can be reviewed at [`report`](./report) directory.
+The framework includes some tooling to assist in setting up constant pre-defined 'loads' across the cluster
+to aid evaluation of their impacts on the scaling metrics. See the [cpu-load documentation](lib/cpu-load.md)
+for more information.
@@ -12,31 +12,37 @@ collectd_pod="collectd"
 init_stats() {
 	local wait_time=$1

-	# create collectd-config configmap
+	# create collectd-config configmap, delete old if there is one
+	kubectl get configmap collectd-config >/dev/null 2>&1 && kubectl delete configmap collectd-config
 	kubectl create configmap collectd-config --from-file=${COLLECTD_DIR}/collectd.conf

+	# if there is collectd daemonset already running, delete it
+	# to make sure that the latest configmap will be used.
+	kubectl get daemonset collectd >/dev/null 2>&1 && kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${collectd_pod}"
+
 	# Launch our stats gathering pod
 	kubectl apply -f ${COLLECTD_DIR}/${collectd_pod}.yaml
 	kubectl rollout status --timeout=${wait_time}s daemonset/${collectd_pod}

+	# clear existing collectd output
+	while read -u 3 name node; do
+		kubectl exec -ti $name -- sh -c "rm -rf /mnt/opt/collectd/run/localhost/*"
+	done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"')
+
 	# attempting to provide buffer for collectd to be installed and running,
-    # and CPU collection to build adequate history
+	# and CPU collection to build adequate history
 	sleep 12
 }

 cleanup_stats() {
-	local delete_wait_time=$1
-
 	# attempting to provide buffer for collectd CPU collection to record adequate history
 	sleep 6

 	# get logs before shutting down stats daemonset
 	while read -u 3 name node; do
-		kubectl exec -ti $name -- sh -c "cd /opt/collectd; tar -czvf localhost.tar.gz localhost"
-		# make a backup on the host in-case collection fail
-		kubectl exec -ti $name -- sh -c "mkdir -p /mnt/opt/collectd"
-		kubectl exec -ti $name -- sh -c "cp /opt/collectd/localhost.tar.gz /mnt/opt/collectd/localhost.tar.gz"
-		kubectl cp $name:/opt/collectd/localhost.tar.gz ${RESULT_DIR}/${node}.tar.gz
+		kubectl exec -ti $name -- sh -c "cd /mnt/opt/collectd/run; rm -f ../localhost.tar.gz; tar -czvf ../localhost.tar.gz localhost"
+		kubectl cp $name:/mnt/opt/collectd/localhost.tar.gz ${RESULT_DIR}/${node}.tar.gz
+		kubectl exec -ti $name -- sh -c "rm -rf /mnt/opt/collectd/run"
 	done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"')

 	kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${collectd_pod}" || true
@@ -9,13 +9,15 @@ LoadPlugin memory
 LoadPlugin cpufreq
 LoadPlugin df

+Hostname localhost
+
 <Plugin "cpu">
    ReportByCpu true
    ReportByState true
    ValuesPercentage true
 </Plugin>
 <Plugin "csv">
-    DataDir "/opt/collectd"
+    DataDir "/mnt/opt/collectd/run"
    StoreRates true
 </Plugin>
 <Plugin "interface">
@@ -23,6 +25,7 @@ LoadPlugin df
    Interface "/^ens/"
    Interface "/^enp/"
    Interface "/^em/"
+    Interface "/^eth/"
    IgnoreSelected false
 </Plugin>
 <Plugin "aggregation">
@@ -10,6 +10,7 @@ RESULT_DIR="${LIB_DIR}/../results"

 source ${LIB_DIR}/json.bash
 source ${LIB_DIR}/k8s-api.bash
+source ${LIB_DIR}/cpu-load.bash
 source /etc/os-release || source /usr/lib/os-release

 die() {
@@ -67,6 +68,48 @@ init_env()
 	# been deliberately injected into the cluster under test.
 }

+framework_init() {
+	info "Initialising"
+
+	check_cmds "${cmds[@]}"
+
+	info "Checking k8s accessible"
+	local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
+	if [ "$worked" != 0 ]; then
+		die "kubectl failed to get nodes"
+	fi
+
+	info $(get_num_nodes) "k8s nodes in 'Ready' state found"
+
+	k8s_api_init
+
+	# Launch our stats gathering pod
+	if [ -n "$SMF_USE_COLLECTD" ]; then
+		info "Setting up collectd"
+		init_stats $wait_time
+	fi
+
+	# And now we can set up our results storage then...
+	metrics_json_init "k8s"
+	save_config
+
+	# Initialise the cpu load generators now - after json init, as they may
+	# produce some json results (config) data.
+	cpu_load_init
+
+}
+
+framework_shutdown() {
+	metrics_json_save
+	k8s_api_shutdown
+	cpu_load_shutdown
+
+	if [ -n "$SMF_USE_COLLECTD" ]; then
+		cleanup_stats
+	fi
+
+}
+
 # finds elements in $1 that are not in $2
 find_unique_pods() {
 	local list_a=$1
@@ -86,3 +129,22 @@ find_unique_pods() {
 				fi
 		done
 }
+
+# waits for process to complete within a given time range
+waitForProcess(){
+    wait_time="$1"
+    sleep_time="$2"
+    cmd="$3"
+    proc_info_msg="$4"
+
+    while [ "$wait_time" -gt 0 ]; do
+        if eval "$cmd"; then
+            return 0
+        else
+            info "$proc_info_msg"
+            sleep "$sleep_time"
+            wait_time=$((wait_time-sleep_time))
+        fi
+    done
+    return 1
+}
@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Helper routines for setting up a constant CPU load on the cluster/nodes
+
+CPULOAD_DIR=${THIS_FILE%/*}
+
+# Default to testing all cores
+SMF_CPU_LOAD_NODES_NCPU=${SMF_CPU_LOAD_NODES_NCPU:-0}
+# Default to 100% load (yes, this might kill your node)
+SMF_CPU_LOAD_NODES_PERCENT=${SMF_CPU_LOAD_NODES_PERCENT:-}
+# Default to not setting any limits or requests, so no cpuset limiting and
+# no cpu core pinning
+SMF_CPU_LOAD_NODES_LIMIT=${SMF_CPU_LOAD_NODES_LIMIT:-}
+SMF_CPU_LOAD_NODES_REQUEST=${SMF_CPU_LOAD_NODES_REQUEST:-}
+
+cpu_load_post_deploy_sleep=${cpu_load_post_deploy_sleep:-30}
+
+cpu_per_node_daemonset=cpu-load
+clean_up_cpu_per_node=false
+
+# Use a DaemonSet to place one cpu stressor on each node.
+cpu_per_node_init() {
+	info "Generating per-node CPU load daemonset"
+
+	local ds_template=${CPULOAD_DIR}/cpu_load_daemonset.yaml.in
+	local ds_yaml=${ds_template%\.in}
+
+	# Grab a copy of the template
+	cp -f ${ds_template} ${ds_yaml}
+
+	# If a setting is not used (defined), then delete its relevant
+	# lines from the YAML. Note, the YAML is constructed when necessary
+	# with comments on the correct lines to ensure all necessary lines are
+	# deleted
+	if [ -z "$SMF_CPU_LOAD_NODES_NCPU" ]; then
+		sed -i '/CPU_NCPU/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_PERCENT}" ]; then
+		sed -i '/CPU_PERCENT/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_LIMIT}" ]; then
+		sed -i '/CPU_LIMIT/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_REQUEST}" ]; then
+		sed -i '/CPU_REQUEST/d' ${ds_yaml}
+	fi
+
+	# And then finally replace all the remaining defined parts with the
+	# real values.
+	sed -i \
+		-e "s|@CPU_NCPU@|${SMF_CPU_LOAD_NODES_NCPU}|g" \
+		-e "s|@CPU_PERCENT@|${SMF_CPU_LOAD_NODES_PERCENT}|g" \
+		-e "s|@CPU_LIMIT@|${SMF_CPU_LOAD_NODES_LIMIT}|g" \
+		-e "s|@CPU_REQUEST@|${SMF_CPU_LOAD_NODES_REQUEST}|g" \
+		${ds_yaml}
+
+	# Launch the daemonset...
+	info "Deploying cpu-load-per-node daemonset"
+	kubectl apply -f ${ds_yaml}
+	kubectl rollout status --timeout=${wait_time}s daemonset/${cpu_per_node_daemonset}
+	clean_up_cpu_per_node=yes
+	info "cpu-load-per-node daemonset Deployed"
+	if [ -n "$cpu_load_post_deploy_sleep" ]; then
+		info "Sleeping ${cpu_load_post_deploy_sleep}s for cpu-load to settle"
+		sleep ${cpu_load_post_deploy_sleep}
+	fi
+
+	# And store off our config into the JSON results
+	metrics_json_start_array
+	local json="$(cat << EOF
+	{
+		"LOAD_NODES_NCPU": "${SMF_CPU_LOAD_NODES_NCPU}",
+		"LOAD_NODES_PERCENT": "${SMF_CPU_LOAD_NODES_PERCENT}",
+		"LOAD_NODES_LIMIT": "${SMF_CPU_LOAD_NODES_LIMIT}",
+		"LOAD_NODES_REQUEST": "${SMF_CPU_LOAD_NODES_REQUEST}"
+	}
+EOF
+)"
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "cpu-load"
+}
+
+cpu_load_init() {
+	info "Check if we need CPU load generators..."
+	# This is defaulted of off (not defined), unless the high level test requests it.
+	if [ -n "$SMF_CPU_LOAD_NODES" ]; then
+		info "Initialising per-node CPU load"
+		cpu_per_node_init
+	fi
+}
+
+cpu_load_shutdown() {
+	if [ "$clean_up_cpu_per_node" = "yes" ]; then
+		info "Cleaning up cpu per node load daemonset"
+		kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${cpu_per_node_daemonset}" || true
+	fi
+}
@@ -0,0 +1,81 @@
+# `cpu-load` stack stresser
+
+The `cpu-load` stress functionality of the scaling framework allows you to optionally add a constant CPU stress
+load to cluster under test whilst the tests are running. This aids impact analysis of CPU load.
+
+The `cpu-load` functionality utilises the [`stress-ng`](https://kernel.ubuntu.com/git/cking/stress-ng.git/) tool
+to generate the CPU load. Some of the configuration parameters are taken directoy from the `stress-ng` command line.
+
+## Configuration
+
+`cpu-load` is configured via a number of environment variables:
+
+| Tool | Description |
+| ---- | ----------- |
+| collectd | `collectd` based statistics/metrics gathering daemonset code |
+| lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis |
+| report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results |
+| scaling | Tests to measure scaling, such as linear or parallel launching of pods |
+
+| Variable | Description | Default |
+| -------- | ----------- | ------- |
+| `SMF_CPU_LOAD_NODES` | Set to non-empty to deploy `cpu-load` stressor | unset (off) |
+| `SMF_CPU_LOAD_NODES_NCPU` | Number of stressor threads to launch per node | 0 (one per cpu) |
+| `SMF_CPU_LOAD_NODES_PERCENT` | Percentage of CPU to load | unset (100%) |
+| `SMF_CPU_LOAD_NODES_LIMIT` | k8s cpu resource limit to set | unset (none) |
+| `SMF_CPU_LOAD_NODES_REQUEST` | k8s cpu resource request to set | unset (none) |
+| `cpu_load_post_deploy_sleep` | Seconds to sleep for `cpu-load` deployment to settle | 30 |
+
+`SMF_CPU_LOAD_NODES` must be set to a non-empty string to enable the `cpu-load` functionality. `cpu-load` uses
+a daemonSet to deploy one `stress-ng` single container pod to each active node in the cluster.
+
+
+Any of the `SMF_CPU_LOAD_NODES_*` variables can be set, or unset, and the daemonSet pods will be configured
+appropriately.
+
+## Examples
+
+The combinations of settings available allow a lot of flexibility. Below are some common example setups:
+
+### 50% CPU load on all cores of all nodes (`stress-ng`)
+
+Here we allow `stress-ng` to spawn workers to cover all the CPUs on each node, but ask it to restrict its
+bandwidth use to 50% of the CPU. We do not use the k8s limits.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+#export SMF_CPU_LOAD_NODES_NCPU=
+export SMF_CPU_LOAD_NODES_PERCENT=50
+#export SMF_CPU_LOAD_NODES_LIMIT=999m
+#export SMF_CPU_LOAD_NODES_REQUEST=999m
+```
+
+### 50% CPU load on 1 un-pinned core of all nodes (k8s `limits`)
+
+Here we set `stress-ng` to run a single worker thread at 100% CPU, but use the k8s resource limits to restrict
+actual CPU usage to 50%. Because the k8s limit and request are not whole interger units, if the static policy is
+in place on the k8s cluster, the pods will be classified as Guaranteed QoS, but will *not* get pinned to a specific
+cpuset.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+export SMF_CPU_LOAD_NODES_NCPU=1
+export SMF_CPU_LOAD_NODES_PERCENT=100
+export SMF_CPU_LOAD_NODES_LIMIT=500m
+export SMF_CPU_LOAD_NODES_REQUEST=500m
+```
+
+### 50% CPU load pinned to 1 core, on all nodes
+
+Here we set `stress-ng` to run a single worker thread at 50% CPU, and use the k8s resource limits to classify the
+pod as Guaranteed, and as we are using whole integer units of CPU resource requests, if the static policy manager is
+in play, the thread will be pinned to a single cpu cpuset.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+export SMF_CPU_LOAD_NODES_NCPU=1
+export SMF_CPU_LOAD_NODES_PERCENT=50
+export SMF_CPU_LOAD_NODES_LIMIT=1
+export SMF_CPU_LOAD_NODES_REQUEST=1
+```
+
@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cpu-load
+spec:
+  selector:
+    matchLabels:
+      name: cpu-load-pods
+  template:
+    metadata:
+      labels:
+        name: cpu-load-pods
+    spec:
+      hostNetwork: true
+      terminationGracePeriodSeconds: 0
+      containers:
+        - name: cpu-load
+          imagePullPolicy: IfNotPresent
+          image: polinux/stress-ng
+          command: ["stress-ng"]
+          args:                 # comment fields here so we can *delete* sections on demand
+            - "--cpu"
+            - "@CPU_NCPU@"
+            - "-l"              #CPU_PERCENT
+            - "@CPU_PERCENT@"   #CPU_PERCENT
+          resources:
+            limits:
+              cpu: @CPU_LIMIT@
+            requests:
+              cpu: @CPU_REQUEST@
@@ -100,7 +100,7 @@ setup() {
 }

 run() {
-	docker run -ti --rm -v ${HOSTINPUTDIR}:${GUESTINPUTDIR} -v ${HOSTOUTPUTDIR}:${GUESTOUTPUTDIR} ${extra_volumes} ${IMAGE} ${extra_command}
+	docker run ${extra_opts} --rm -v ${HOSTINPUTDIR}:${GUESTINPUTDIR} -v ${HOSTOUTPUTDIR}:${GUESTOUTPUTDIR} ${extra_volumes} ${IMAGE} ${extra_command}
 	ls -la ${HOSTOUTPUTDIR}/*
 }

@@ -113,6 +113,7 @@ main() {
 			# In debug mode, run a shell instead of the default report generation
 			extra_command="bash"
 			extra_volumes="-v ${HOSTSCRIPTDIR}:${GUESTSCRIPTDIR}"
+			extra_opts="-ti"
 			;;
 		esac
 	done
@@ -13,10 +13,10 @@
 # We would have used the 'verse' base, that already has some of the docs processing
 # installed, but I could not figure out how to add in the extra bits we needed to
 # the lite tex version is uses.
-FROM rocker/tidyverse
+FROM rocker/tidyverse:3.6.0

 # Version of the Dockerfile
-LABEL DOCKERFILE_VERSION="1.0"
+LABEL DOCKERFILE_VERSION="1.1"

 # Without this some of the package installs stop to try and ask questions...
 ENV DEBIAN_FRONTEND=noninteractive
@@ -13,95 +13,105 @@ library(gridExtra)					# together.
 suppressMessages(suppressWarnings(library(ggpubr)))	# for ggtexttable.
 suppressMessages(library(jsonlite))			# to load the data.

-# A list of all the known results files we might find the information inside.
-resultsfiles=c(
-	"k8s-parallel.json",
-	"k8s-scaling.json",
-	"k8s-rapid.json"
-	)
+render_dut_details <- function()
+{
+	# A list of all the known results files we might find the information inside.
+	resultsfiles=c(
+		"k8s-parallel.json",
+		"k8s-scaling.json",
+		"k8s-rapid.json"
+		)

-data=c()
-stats=c()
-stats_names=c()
+	data=c()
+	stats=c()
+	stats_names=c()

-# For each set of results
-for (currentdir in resultdirs) {
-	count=1
-	dirstats=c()
-	for (resultsfile in resultsfiles) {
-		fname=paste(inputdir, currentdir, resultsfile, sep="/")
-		if ( !file.exists(fname)) {
-			#warning(paste("Skipping non-existent file: ", fname))
-			next
-		}
-
-		# Derive the name from the test result dirname
-		datasetname=basename(currentdir)
-
-		# Import the data
-		fdata=fromJSON(fname)
-
-		if (length(fdata$'kubectl-version') != 0 ) {
-			# We have kata-runtime data
-			dirstats=tibble("Client Ver"=as.character(fdata$'kubectl-version'$clientVersion$gitVersion))
-			dirstats=cbind(dirstats, "Server Ver"=as.character(fdata$'kubectl-version'$serverVersion$gitVersion))
-			numnodes= nrow(fdata$'kubectl-get-nodes'$items)
-			dirstats=cbind(dirstats, "No. nodes"=as.character(numnodes))
-
-			if (numnodes != 0) {
-				first_node=fdata$'kubectl-get-nodes'$items[1,]
-				dirstats=cbind(dirstats, "- Node0 name"=as.character(first_node$metadata$name))
-
-				havekata=first_node$metadata$labels$'katacontainers.io/kata-runtime'
-				if ( is.null(havekata) ) {
-					dirstats=cbind(dirstats, "  Have Kata"=as.character('false'))
-				} else {
-					dirstats=cbind(dirstats, "  Have Kata"=as.character(havekata))
-				}
-
-				dirstats=cbind(dirstats, "  CPUs"=as.character(first_node$status$capacity$cpu))
-				dirstats=cbind(dirstats, "  Memory"=as.character(first_node$status$capacity$memory))
-				dirstats=cbind(dirstats, "  MaxPods"=as.character(first_node$status$capacity$pods))
-				dirstats=cbind(dirstats, "  PodCIDR"=as.character(first_node$spec$podCIDR))
-
-				dirstats=cbind(dirstats, "  runtime"=as.character(first_node$status$nodeInfo$containerRuntimeVersion))
-				dirstats=cbind(dirstats, "  kernel"=as.character(first_node$status$nodeInfo$kernelVersion))
-				dirstats=cbind(dirstats, "  kubeProxy"=as.character(first_node$status$nodeInfo$kubeProxyVersion))
-				dirstats=cbind(dirstats, "  Kubelet"=as.character(first_node$status$nodeInfo$kubeletVersion))
-				dirstats=cbind(dirstats, "  OS"=as.character(first_node$status$nodeInfo$osImage))
+	# For each set of results
+	for (currentdir in resultdirs) {
+		count=1
+		dirstats=c()
+		datasetname=c()
+		for (resultsfile in resultsfiles) {
+			fname=paste(inputdir, currentdir, resultsfile, sep="/")
+			if ( !file.exists(fname)) {
+				#warning(paste("Skipping non-existent file: ", fname))
+				next
 			}

-			break
+			# Derive the name from the test result dirname
+			datasetname=basename(currentdir)
+
+			# Import the data
+			fdata=fromJSON(fname)
+
+			if (length(fdata$'kubectl-version') != 0 ) {
+				# We have kata-runtime data
+				dirstats=tibble("Client Ver"=as.character(fdata$'kubectl-version'$clientVersion$gitVersion))
+				dirstats=cbind(dirstats, "Server Ver"=as.character(fdata$'kubectl-version'$serverVersion$gitVersion))
+				numnodes= nrow(fdata$'kubectl-get-nodes'$items)
+				dirstats=cbind(dirstats, "No. nodes"=as.character(numnodes))
+
+				if (numnodes != 0) {
+					first_node=fdata$'kubectl-get-nodes'$items[1,]
+					dirstats=cbind(dirstats, "- Node0 name"=as.character(first_node$metadata$name))
+
+					havekata=first_node$metadata$labels$'katacontainers.io/kata-runtime'
+					if ( is.null(havekata) ) {
+						dirstats=cbind(dirstats, "  Have Kata"=as.character('false'))
+					} else {
+						dirstats=cbind(dirstats, "  Have Kata"=as.character(havekata))
+					}
+
+					dirstats=cbind(dirstats, "  CPUs"=as.character(first_node$status$capacity$cpu))
+					dirstats=cbind(dirstats, "  Memory"=as.character(first_node$status$capacity$memory))
+					dirstats=cbind(dirstats, "  MaxPods"=as.character(first_node$status$capacity$pods))
+					dirstats=cbind(dirstats, "  PodCIDR"=as.character(first_node$spec$podCIDR))
+
+					dirstats=cbind(dirstats, "  runtime"=as.character(first_node$status$nodeInfo$containerRuntimeVersion))
+					dirstats=cbind(dirstats, "  kernel"=as.character(first_node$status$nodeInfo$kernelVersion))
+					dirstats=cbind(dirstats, "  kubeProxy"=as.character(first_node$status$nodeInfo$kubeProxyVersion))
+					dirstats=cbind(dirstats, "  Kubelet"=as.character(first_node$status$nodeInfo$kubeletVersion))
+					dirstats=cbind(dirstats, "  OS"=as.character(first_node$status$nodeInfo$osImage))
+				}
+
+				break
+			}
 		}
+
+		if ( length(dirstats) == 0 ) {
+			cat(paste("No valid data found for directory ", currentdir, "\n\n"))
+		}
+
+		# use plyr rbind.fill so we can combine disparate version info frames
+		stats=rbind.fill(stats, dirstats)
+		stats_names=rbind(stats_names, datasetname)
 	}

-	if ( length(dirstats) == 0 ) {
-		warning(paste("No valid data found for directory ", currentdir))
+	if ( length(stats_names) == 0 ) {
+		cat("No system details found\n\n")
+		return()
 	}

-	# use plyr rbind.fill so we can combine disparate version info frames
-	stats=rbind.fill(stats, dirstats)
-	stats_names=rbind(stats_names, datasetname)
+	rownames(stats) = stats_names
+
+	# Rotate the tibble so we get data dirs as the columns
+	spun_stats = as_tibble(cbind(What=names(stats), t(stats)))
+
+	# Build us a text table of numerical results
+	# Set up as left hand justify, so the node data indent renders.
+	tablefontsize=8
+	tbody.style = tbody_style(hjust=0, x=0.1, size=tablefontsize)
+	stats_plot = suppressWarnings(ggtexttable(data.frame(spun_stats, check.names=FALSE),
+		theme=ttheme(base_size=tablefontsize, tbody.style=tbody.style),
+		rows=NULL
+		))
+
+	# It may seem odd doing a grid of 1x1, but it should ensure we get a uniform format and
+	# layout to match the other charts and tables in the report.
+	master_plot = grid.arrange(
+		stats_plot,
+		nrow=1,
+		ncol=1 )
 }

-rownames(stats) = stats_names
-
-# Rotate the tibble so we get data dirs as the columns
-spun_stats = as_tibble(cbind(What=names(stats), t(stats)))
-
-# Build us a text table of numerical results
-# Set up as left hand justify, so the node data indent renders.
-tablefontsize=8
-tbody.style = tbody_style(hjust=0, x=0.1, size=tablefontsize)
-stats_plot = suppressWarnings(ggtexttable(data.frame(spun_stats, check.names=FALSE),
-	theme=ttheme(base_size=tablefontsize, tbody.style=tbody.style),
-	rows=NULL
-	))
-
-# It may seem odd doing a grid of 1x1, but it should ensure we get a uniform format and
-# layout to match the other charts and tables in the report.
-master_plot = grid.arrange(
-	stats_plot,
-	nrow=1,
-	ncol=1 )
-
+render_dut_details()
@@ -0,0 +1,26 @@
+
+library('elasticsearchr')
+
+for_scaling <- query('{
+"bool": {
+	"must": [
+	{ "match":
+		{
+			"test.testname": "k8s scaling"
+		}
+	}
+	]
+}
+}')
+
+these_fields <- select_fields('{
+	"includes": [
+		"date.Date",
+		"k8s-scaling.BootResults.launch_time.Result",
+		"k8s-scaling.BootResults.n_pods.Result"
+	]
+}')
+
+sort_by_date <- sort_on('[{"date.Date": {"order": "asc"}}]')
+
+x=elastic("http://192.168.0.111:9200", "logtest") %search% (for_scaling + sort_by_date + these_fields)
@@ -32,7 +32,7 @@ This [test](https://github.com/clearlinux/cloud-native-setup/metrics/scaling/k8s
 measures the time taken to launch and delete pods in parallel using a deployment. The times
 are how long it takes for the whole deployment operation to complete.

-```{r parallel, echo=FALSE, fig.cap="K8S parallel pods"}
+```{r parallel, echo=FALSE, fig.cap="K8S parallel pods", results='asis'}
 source('parallel.R')
 ```

@@ -57,7 +57,7 @@ This table describes the test system details, as derived from the information co
 in the test results files.


-```{r dut, echo=FALSE, fig.cap="System configuration details"}
+```{r dut, echo=FALSE, fig.cap="System configuration details", results='asis'}
 source('dut-details.R')
 ```

@@ -67,6 +67,6 @@ source('dut-details.R')

 This table describes node details within the Kubernetes cluster that have been used for test.

-```{r node, echo=FALSE, fig.cap="Node information within Kubernetes cluster"}
+```{r node, echo=FALSE, fig.cap="Node information within Kubernetes cluster", results='asis'}
 source('node-info.R')
 ```
@@ -13,83 +13,96 @@ library(gridExtra)                                      # together.
 suppressMessages(suppressWarnings(library(ggpubr)))     # for ggtexttable.
 suppressMessages(library(jsonlite))                     # to load the data.

-# A list of all the known results files we might find the information inside.
-resultsfiles=c(
-	       "k8s-scaling.json"
-	       )
+render_node_info <- function()
+{
+	# A list of all the known results files we might find the information inside.
+	resultsfiles=c(
+		       "k8s-scaling.json"
+		       )

-stats=c()
-stats_names=c()
-max_char_name_node=18
+	stats=c()
+	stats_names=c()
+	datasetname=c()
+	complete_data=c()
+	max_char_name_node=18

-# list for each dirstats
-dirstats_list=list()
-j=1
+	# list for each dirstats
+	dirstats_list=list()
+	j=1

-# For each set of results
-for (currentdir in resultdirs) {
-	dirstats=c()
-	for (resultsfile in resultsfiles) {
-		fname=paste(inputdir, currentdir, resultsfile, sep="/")
-		if ( !file.exists(fname)) {
-			next
-		}
-
-		# Derive the name from the test result dirname
-		datasetname=basename(currentdir)
-
-		# Import the data
-		fdata=fromJSON(fname)
-
-		if (length(fdata$'kubectl-version') != 0 ) {
-			numnodes= nrow(fdata$'kubectl-get-nodes'$items)
-			for (i in 1:numnodes) {
-				node_i=fdata$'kubectl-get-nodes'$items[i,]
-				node_info=fdata$'socketsPerNode'[i,]
-
-				# Substring node name so it fits properly into final table
-				node_name=node_i$metadata$name
-				if ( nchar(node_name) >= max_char_name_node) {
-					dirstats=tibble("Node \nname"=as.character(substring(node_name, 1, max_char_name_node)))
-				} else {
-					dirstats=tibble("Node \nname"=as.character(node_name))
-				}
-
-				dirstats=cbind(dirstats, "CPUs"=as.character(node_i$status$capacity$cpu))
-				dirstats=cbind(dirstats, "Memory"=as.character(node_i$status$capacity$memory))
-				dirstats=cbind(dirstats, "Max \nPods"=as.character(node_i$status$capacity$pods))
-				dirstats=cbind(dirstats, "Count \nsockets"=as.character(node_info$num_sockets))
-				dirstats=cbind(dirstats, "Have \nhypervisor"=as.character(node_info$hypervisor))
-
-				dirstats=cbind(dirstats, "kernel"=as.character(node_i$status$nodeInfo$kernelVersion))
-				dirstats=cbind(dirstats, "OS"=as.character(node_i$status$nodeInfo$osImage))
-				dirstats=cbind(dirstats, "Test"=as.character(datasetname))
-
-				dirstats_list[[j]]=dirstats
-				j=j+1
+	# For each set of results
+	for (currentdir in resultdirs) {
+		dirstats=c()
+		for (resultsfile in resultsfiles) {
+			fname=paste(inputdir, currentdir, resultsfile, sep="/")
+			if ( !file.exists(fname)) {
+				next
+			}
+
+			# Derive the name from the test result dirname
+			datasetname=basename(currentdir)
+
+			# Import the data
+			fdata=fromJSON(fname)
+
+			if (length(fdata$'kubectl-version') != 0 ) {
+				numnodes= nrow(fdata$'kubectl-get-nodes'$items)
+				for (i in 1:numnodes) {
+					node_i=fdata$'kubectl-get-nodes'$items[i,]
+					node_info=fdata$'socketsPerNode'[i,]
+
+					# Substring node name so it fits properly into final table
+					node_name=node_i$metadata$name
+					if ( nchar(node_name) >= max_char_name_node) {
+						dirstats=tibble("Node \nname"=as.character(substring(node_name, 1, max_char_name_node)))
+					} else {
+						dirstats=tibble("Node \nname"=as.character(node_name))
+					}
+
+					dirstats=cbind(dirstats, "CPUs"=as.character(node_i$status$capacity$cpu))
+					dirstats=cbind(dirstats, "Memory"=as.character(node_i$status$capacity$memory))
+					dirstats=cbind(dirstats, "Max \nPods"=as.character(node_i$status$capacity$pods))
+					dirstats=cbind(dirstats, "Count \nsockets"=as.character(node_info$num_sockets))
+					dirstats=cbind(dirstats, "Have \nhypervisor"=as.character(node_info$hypervisor))
+
+					dirstats=cbind(dirstats, "kernel"=as.character(node_i$status$nodeInfo$kernelVersion))
+					dirstats=cbind(dirstats, "OS"=as.character(node_i$status$nodeInfo$osImage))
+					dirstats=cbind(dirstats, "Test"=as.character(datasetname))
+
+					dirstats_list[[j]]=dirstats
+					j=j+1
+				}
+				complete_data = do.call(rbind, dirstats_list)
 			}
-			complete_data = do.call(rbind, dirstats_list)
 		}
+
+		if ( length(complete_data) == 0 ) {
+			cat(paste("No valid data found for directory ", currentdir, "\n\n"))
+		}
+
+		# use plyr rbind.fill so we can combine disparate version info frames
+		stats=rbind.fill(stats, complete_data)
+		stats_names=rbind(stats_names, datasetname)
 	}

-	if ( length(complete_data) == 0 ) {
-		warning(paste("No valid data found for directory ", currentdir))
+	if ( length(stats_names) == 0 ) {
+		cat("No node stats found\n\n");
+		return()
 	}

-	# use plyr rbind.fill so we can combine disparate version info frames
-	stats=rbind.fill(stats, complete_data)
-	stats_names=rbind(stats_names, datasetname)
+	# Build us a text table of numerical results
+	# Set up as left hand justify, so the node data indent renders.
+	tablefontsize=8
+	tbody.style = tbody_style(hjust=0, x=0.1, size=tablefontsize)
+	stats_plot = suppressWarnings(ggtexttable(data.frame(complete_data, check.names=FALSE),
+						  theme=ttheme(base_size=tablefontsize, tbody.style=tbody.style),
+						  rows=NULL))
+
+	# It may seem odd doing a grid of 1x1, but it should ensure we get a uniform format and
+	# layout to match the other charts and tables in the report.
+	master_plot = grid.arrange(stats_plot,
+				   nrow=1,
+				   ncol=1 )
 }
-# Build us a text table of numerical results
-# Set up as left hand justify, so the node data indent renders.
-tablefontsize=8
-tbody.style = tbody_style(hjust=0, x=0.1, size=tablefontsize)
-stats_plot = suppressWarnings(ggtexttable(data.frame(complete_data, check.names=FALSE),
-					  theme=ttheme(base_size=tablefontsize, tbody.style=tbody.style),
-					  rows=NULL))

-# It may seem odd doing a grid of 1x1, but it should ensure we get a uniform format and
-# layout to match the other charts and tables in the report.
-master_plot = grid.arrange(stats_plot,
-			   nrow=1,
-			   ncol=1 )
+render_node_info()
--- a/Show More
+++ b/Show More