Updating flannel (#299)

Running setup_system.sh on a system setting the OS version to keep setup_system from upgrading the OS causes an error because a package has been removed. Need to add the --force flag to the
sudo swupd repair -m "${CLR_VER}" --picky command

Closes issue #297

clr-k8s-examples/0-cilium/overlays/v1.6.4/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml

clr-k8s-examples/0-flannel/overlays/960b3243b9a7faccdfe7b3c09097105e68030ea7/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/1-core-metrics/overlays/v0.3.6/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - metrics-server/deploy/1.8+/aggregated-metrics-reader.yaml
+  - metrics-server/deploy/1.8+/auth-delegator.yaml
+  - metrics-server/deploy/1.8+/auth-reader.yaml
+  - metrics-server/deploy/1.8+/metrics-apiservice.yaml
+  - metrics-server/deploy/1.8+/metrics-server-deployment.yaml
+  - metrics-server/deploy/1.8+/metrics-server-service.yaml
+  - metrics-server/deploy/1.8+/resource-reader.yaml

clr-k8s-examples/5-ingres-lb/overlays/nginx-0.25.0/kustomization.yaml

@@ -6,7 +6,7 @@ patchesJson6902:
   # adds "networking.k8s.io" to ClusterRole's apiGroups
   - target:
       group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
       kind: ClusterRole
       name: nginx-ingress-clusterrole
     path: patch_clusterrole.yaml

clr-k8s-examples/5-ingres-lb/overlays/nginx-0.26.1/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - ingress-nginx/deploy/static/mandatory.yaml
+  - ingress-nginx/deploy/static/provider/baremetal/service-nodeport.yaml

clr-k8s-examples/6-metal-lb/overlays/v0.8.3/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - metallb/manifests/example-layer2-config.yaml
+  - metallb/manifests/metallb.yaml
+
+patchesStrategicMerge:
+  - patch_configmap.yaml

clr-k8s-examples/6-metal-lb/overlays/v0.8.3/patch_configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: my-ip-space
+      protocol: layer2
+      addresses:
+      - 10.0.0.240/28

clr-k8s-examples/7-rook/overlays/v0.8.3/patch_operator.yaml

@@ -1,5 +1,5 @@
 # operator
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: rook-ceph-operator

clr-k8s-examples/7-rook/overlays/v1.1.7/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+  - rook/cluster/examples/kubernetes/ceph/common.yaml
+  - rook/cluster/examples/kubernetes/ceph/operator.yaml
+  - rook/cluster/examples/kubernetes/ceph/cluster.yaml
+  - rook/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.1.7/patch_cephcluster.yaml

@@ -0,0 +1,9 @@
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  storage:
+    directories:
+     - path: /var/lib/rook

clr-k8s-examples/8-kata/overlays/1.9.1-kernel-config/kustomization.yaml

@@ -0,0 +1,11 @@
+resources:
+  - packaging/kata-deploy/kata-deploy.yaml
+  - packaging/kata-deploy/kata-rbac.yaml
+  - packaging/kata-deploy/k8s-1.14/kata-fc-runtimeClass.yaml
+  - packaging/kata-deploy/k8s-1.14/kata-qemu-runtimeClass.yaml
+
+images:
+  # change 'latest' to specified version
+  - name: katadocker/kata-deploy
+    newName: katadocker/kata-deploy
+    newTag: 1.9.1

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   # name must match the spec fields below, and be in the form: <plural>.<group>

clr-k8s-examples/DEVELOP.md

@@ -119,7 +119,7 @@ patchesJson6902:
   # adds "networking.k8s.io" to ClusterRole's apiGroups
   - target:
       group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
       kind: ClusterRole
       name: nginx-ingress-clusterrole
     path: patch_clusterrole.yaml
@@ -137,7 +137,7 @@ itself contains the operation to perform, target path and value. The `rules/3/ap
 operation (in this case "add") at the `apiGroups:` list found under the 4th list item of `rules:`. 
 
 ```yaml
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: nginx-ingress-clusterrole

clr-k8s-examples/README.md

@@ -38,6 +38,9 @@ This script ensures the following
 script uses the runtime specified in the `RUNNER` environment variable and defaults to `crio`. To use the
 `containerd` runtime, set the `RUNNER` environment variable to `containerd`.
 
+In case of vagrant, if you want to spin up VM's using different environment variable than declared in [`setup_system.sh`],
+specify when performing vagrant up. E.g., `RUNNER=containerd vagrant up`
+
 ### Specify a version of Clear Linux
 
 To specify a particular version of Clear Linux to use, set the CLRK8S_CLR_VER environment variable to the desired

clr-k8s-examples/admit-kata/deploy/webhook-registration.yaml.tpl

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   name: pod-annotate-webhook

clr-k8s-examples/admit-kata/deploy/webhook.yaml

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Deployment
 metadata:
   name: pod-annotate-webhook

clr-k8s-examples/containerd_devmapper_setup.sh

@@ -44,18 +44,21 @@ sudo systemctl enable --now containerd-devmapper
 # no. of feature arguments
 # Skip zeroing blocks for new volumes.
 sudo dmsetup create contd-thin-pool \
-  --table "0 2097152 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
+  --table "0 20971520 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
 
 sudo mkdir -p /etc/containerd/
 if [ -f /etc/containerd/config.toml ]
 then
-  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"512MB\"|' /etc/containerd/config.toml
+  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"4096MB\"|' /etc/containerd/config.toml
 else
   cat<<EOT | sudo tee /etc/containerd/config.toml
 [plugins]
   [plugins.devmapper]
     pool_name = "contd-thin-pool"
-    base_image_size = "512MB"
+    base_image_size = "4096MB"
+  [plugins.cri]
+    [plugins.cri.containerd]
+      snapshotter = "devmapper"
 EOT
 fi
 

clr-k8s-examples/create_stack.sh

@@ -17,20 +17,21 @@ HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
 
 # versions
 CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}"
-CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.6}"
-FLANNEL_VER="${CLRK8S_FLANNEL_VER:-16b0fe66285d1ad1f42b154ab852682f6fafb1a7}"
+CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.6.4}"
+FLANNEL_VER="${CLRK8S_FLANNEL_VER:-960b3243b9a7faccdfe7b3c09097105e68030ea7}"
 K8S_VER="${CLRK8S_K8S_VER:-}"
-KATA_VER="${CLRK8S_KATA_VER:-1.8.2-kernel-config}"
-ROOK_VER="${CLRK8S_ROOK_VER:-v1.1.1}"
-METRICS_VER="${CLRK8S_METRICS_VER:-v0.3.5}"
+KATA_VER="${CLRK8S_KATA_VER:-1.9.1-kernel-config}"
+ROOK_VER="${CLRK8S_ROOK_VER:-v1.1.7}"
+METRICS_VER="${CLRK8S_METRICS_VER:-v0.3.6}"
 DASHBOARD_VER="${CLRK8S_DASHBOARD_VER:-v2.0.0-beta2}"
-INGRES_VER="${CLRK8S_INGRES_VER:-nginx-0.25.1}"
+INGRES_VER="${CLRK8S_INGRES_VER:-nginx-0.26.1}"
 EFK_VER="${CLRK8S_EFK_VER:-v1.15.1}"
-METALLB_VER="${CLRK8S_METALLB_VER:-v0.8.1}"
+METALLB_VER="${CLRK8S_METALLB_VER:-v0.8.3}"
 NPD_VER="${CLRK8S_NPD_VER:-v0.6.6}"
 PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-f458e85e5d7675f7bc253072e1b4c8892b51af0f}"
 CNI=${CLRK8S_CNI:-"canal"}
-RUNNER=${CLRK8S_RUNNER:-"crio"}
+if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi
+
 NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}"
 
 function print_usage_exit() {
@@ -144,10 +145,10 @@ function cni() {
 		FLANNEL_VER=${1:-$FLANNEL_VER}
 		FLANNEL_URL="https://github.com/coreos/flannel"
 		FLANNEL_DIR="0-flannel"
-		
+
 		get_repo "${FLANNEL_URL}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
-        	set_repo_version "${FLANNEL_VER}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}/flannel"
-        	kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
+		set_repo_version "${FLANNEL_VER}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}/flannel"
+		kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
 		;;
 	cilium)
 		CILIUM_VER=${1:-$CILIUM_VER}

clr-k8s-examples/kubeadm.yaml

@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1beta1
+apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
@@ -13,9 +13,10 @@ kubeReserved:
   cpu: 500m
   memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1beta1
+apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16
   serviceSubnet: 10.96.0.0/12
+

clr-k8s-examples/reset_stack.sh

@@ -52,3 +52,4 @@ sudo systemctl is-enabled containerd && sudo systemctl restart containerd
 sudo systemctl restart kubelet
 
 reset_cluster
+sudo -E bash -c "rm -rf /var/lib/containerd/*"

clr-k8s-examples/setup_system.sh

@@ -9,15 +9,14 @@ CLR_VER=${CLRK8S_CLR_VER:-""}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
 
 # set no proxy
-ADD_NO_PROXY=".svc,10.244.0.0/16,10.96.0.0/12"
+ADD_NO_PROXY=".svc,10.0.0.0/8,192.168.0.0/16"
 ADD_NO_PROXY+=",$(hostname -I | sed 's/[[:space:]]/,/g')"
-: "${RUNNER:=crio}"
+if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-crio}"; fi
 
 # update os version
 function upate_os_version() {
-	if [[ -n "${CLR_VER}" ]];
-	then
-		sudo swupd repair -m "${CLR_VER}" --picky
+	if [[ -n "${CLR_VER}" ]]; then
+		sudo swupd repair -m "${CLR_VER}" --picky --force
 		return
 	fi
 	sudo swupd update
@@ -151,7 +150,7 @@ function setup_proxy() {
 		sed_val=${ADD_NO_PROXY//\//\\/}
 		[ -f /etc/environment ] && sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
 		if [ -f /etc/profile.d/proxy.sh ]; then
-			sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+			sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/profile.d/proxy.sh
 		else
 			echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
 		fi

clr-k8s-examples/tests/deploy-svc-ing/test-ingress-kata.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
   name: php-apache-kata

clr-k8s-examples/tests/deploy-svc-ing/test-ingress-runc.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
   name: php-apache-runc

metrics/README.md

@@ -13,10 +13,11 @@ is below:
 
 | Tool | Description |
 | ---- | ----------- |
+| collectd | `collectd` based statistics/metrics gathering daemonset code |
 | lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis |
-| scaling | Tests to measure scaling, such as linear or parallel launching of pods |
+| lib/cpu-load* | Routines to enable CPU load generation on a cluster |
 | report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results |
-
+| scaling | Tests to measure scaling, such as linear or parallel launching of pods |
 
 ## Results storage and analysis
 
@@ -130,3 +131,30 @@ If k8s_parallel.sh was run, the results file is named `k8s-parallel.json` rather
    └── scaling-4.png
    ```
 More details about result reporting can be reviewed at [`report`](./report) directory.
+
+# Developers
+
+This section provides some details of how the code is structured and configured. This may be of use whilst modifying
+existing or creating new tests.
+
+## Metrics gathering
+
+Metrics can be gathered using either a daemonset deployment of privileged pods used to gather statistics directly from the nodes using a combination of `mpstat`, `free` and `df`, or a daemonset deployment based around `collectd`.
+
+### `collectd` statistics
+
+The `collected` based code can be found in the `collectd` subdirectory. It uses the `collected` configuration found in the `collectd.conf` file to gather statistics, and store the results on the nodes themselves whilst tests are running. At the end of the test, the results are copied from the nodes and stored in the results directory for later processing.
+
+The `collectd` statistics are only configured and gathered if the environment variable `SMF_USE_COLLECTD` is set to non-empty by the test code (that is, only enabled upon request).
+
+### privileged statistics pods
+
+The privileged statistics pods `YAML` can be found in the `scaling/stats.yaml` file. An example of how to invoke and use this daemonset to extract statistics can be found in the `scaling/k8s_scale.sh` file.
+
+## Configuring constant 'loads'
+
+The framework includes some tooling to assist in setting up constant pre-defined 'loads' across the cluster to aid evaluation of their impacts on the scaling metrics.
+
+### CPU load generator
+
+Details of how to configure a constant CPU load are detailed in the [cpu-load documentation](lib/cpu-load.md).

metrics/collectd/collectd.bash

@@ -25,8 +25,6 @@ init_stats() {
 }
 
 cleanup_stats() {
-	local delete_wait_time=$1
-
 	# attempting to provide buffer for collectd CPU collection to record adequate history
 	sleep 6
 

metrics/collectd/collectd.conf

@@ -9,6 +9,8 @@ LoadPlugin memory
 LoadPlugin cpufreq
 LoadPlugin df
 
+Hostname localhost
+
 <Plugin "cpu">
     ReportByCpu true
     ReportByState true
@@ -23,6 +25,7 @@ LoadPlugin df
     Interface "/^ens/"
     Interface "/^enp/"
     Interface "/^em/"
+    Interface "/^eth/"
     IgnoreSelected false
 </Plugin>
 <Plugin "aggregation">

metrics/lib/common.bash

@@ -10,6 +10,7 @@ RESULT_DIR="${LIB_DIR}/../results"
 
 source ${LIB_DIR}/json.bash
 source ${LIB_DIR}/k8s-api.bash
+source ${LIB_DIR}/cpu-load.bash
 source /etc/os-release || source /usr/lib/os-release
 
 die() {
@@ -67,6 +68,48 @@ init_env()
 	# been deliberately injected into the cluster under test.
 }
 
+framework_init() {
+	info "Initialising"
+
+	check_cmds "${cmds[@]}"
+
+	info "Checking k8s accessible"
+	local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
+	if [ "$worked" != 0 ]; then
+		die "kubectl failed to get nodes"
+	fi
+
+	info $(get_num_nodes) "k8s nodes in 'Ready' state found"
+
+	k8s_api_init
+
+	# Launch our stats gathering pod
+	if [ -n "$SMF_USE_COLLECTD" ]; then
+		info "Setting up collectd"
+		init_stats $wait_time
+	fi
+
+	# And now we can set up our results storage then...
+	metrics_json_init "k8s"
+	save_config
+
+	# Initialise the cpu load generators now - after json init, as they may
+	# produce some json results (config) data.
+	cpu_load_init
+
+}
+
+framework_shutdown() {
+	metrics_json_save
+	k8s_api_shutdown
+	cpu_load_shutdown
+
+	if [ -n "$SMF_USE_COLLECTD" ]; then
+		cleanup_stats
+	fi
+
+}
+
 # finds elements in $1 that are not in $2
 find_unique_pods() {
 	local list_a=$1
@@ -86,3 +129,22 @@ find_unique_pods() {
 				fi
 		done
 }
+
+# waits for process to complete within a given time range
+waitForProcess(){
+    wait_time="$1"
+    sleep_time="$2"
+    cmd="$3"
+    proc_info_msg="$4"
+
+    while [ "$wait_time" -gt 0 ]; do
+        if eval "$cmd"; then
+            return 0
+        else
+            info "$proc_info_msg"
+            sleep "$sleep_time"
+            wait_time=$((wait_time-sleep_time))
+        fi
+    done
+    return 1
+}

metrics/lib/cpu-load.bash

@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Helper routines for setting up a constant CPU load on the cluster/nodes
+
+CPULOAD_DIR=${THIS_FILE%/*}
+
+# Default to testing all cores
+SMF_CPU_LOAD_NODES_NCPU=${SMF_CPU_LOAD_NODES_NCPU:-0}
+# Default to 100% load (yes, this might kill your node)
+SMF_CPU_LOAD_NODES_PERCENT=${SMF_CPU_LOAD_NODES_PERCENT:-}
+# Default to not setting any limits or requests, so no cpuset limiting and
+# no cpu core pinning
+SMF_CPU_LOAD_NODES_LIMIT=${SMF_CPU_LOAD_NODES_LIMIT:-}
+SMF_CPU_LOAD_NODES_REQUEST=${SMF_CPU_LOAD_NODES_REQUEST:-}
+
+cpu_load_post_deploy_sleep=${cpu_load_post_deploy_sleep:-30}
+
+cpu_per_node_daemonset=cpu-load
+clean_up_cpu_per_node=false
+
+# Use a DaemonSet to place one cpu stressor on each node.
+cpu_per_node_init() {
+	info "Generating per-node CPU load daemonset"
+
+	local ds_template=${CPULOAD_DIR}/cpu_load_daemonset.yaml.in
+	local ds_yaml=${ds_template%\.in}
+
+	# Grab a copy of the template
+	cp -f ${ds_template} ${ds_yaml}
+
+	# If a setting is not used (defined), then delete its relevant
+	# lines from the YAML. Note, the YAML is constructed when necessary
+	# with comments on the correct lines to ensure all necessary lines are
+	# deleted
+	if [ -z "$SMF_CPU_LOAD_NODES_NCPU" ]; then
+		sed -i '/CPU_NCPU/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_PERCENT}" ]; then
+		sed -i '/CPU_PERCENT/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_LIMIT}" ]; then
+		sed -i '/CPU_LIMIT/d' ${ds_yaml}
+	fi
+
+	if [ -z "${SMF_CPU_LOAD_NODES_REQUEST}" ]; then
+		sed -i '/CPU_REQUEST/d' ${ds_yaml}
+	fi
+
+	# And then finally replace all the remaining defined parts with the
+	# real values.
+	sed -i \
+		-e "s|@CPU_NCPU@|${SMF_CPU_LOAD_NODES_NCPU}|g" \
+		-e "s|@CPU_PERCENT@|${SMF_CPU_LOAD_NODES_PERCENT}|g" \
+		-e "s|@CPU_LIMIT@|${SMF_CPU_LOAD_NODES_LIMIT}|g" \
+		-e "s|@CPU_REQUEST@|${SMF_CPU_LOAD_NODES_REQUEST}|g" \
+		${ds_yaml}
+
+	# Launch the daemonset...
+	info "Deploying cpu-load-per-node daemonset"
+	kubectl apply -f ${ds_yaml}
+	kubectl rollout status --timeout=${wait_time}s daemonset/${cpu_per_node_daemonset}
+	clean_up_cpu_per_node=yes
+	info "cpu-load-per-node daemonset Deployed"
+	if [ -n "$cpu_load_post_deploy_sleep" ]; then
+		info "Sleeping ${cpu_load_post_deploy_sleep}s for cpu-load to settle"
+		sleep ${cpu_load_post_deploy_sleep}
+	fi
+
+	# And store off our config into the JSON results
+	metrics_json_start_array
+	local json="$(cat << EOF
+	{
+		"LOAD_NODES_NCPU": "${SMF_CPU_LOAD_NODES_NCPU}",
+		"LOAD_NODES_PERCENT": "${SMF_CPU_LOAD_NODES_PERCENT}",
+		"LOAD_NODES_LIMIT": "${SMF_CPU_LOAD_NODES_LIMIT}",
+		"LOAD_NODES_REQUEST": "${SMF_CPU_LOAD_NODES_REQUEST}"
+	}
+EOF
+)"
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "cpu-load"
+}
+
+cpu_load_init() {
+	info "Check if we need CPU load generators..."
+	# This is defaulted of off (not defined), unless the high level test requests it.
+	if [ -n "$SMF_CPU_LOAD_NODES" ]; then
+		info "Initialising per-node CPU load"
+		cpu_per_node_init
+	fi
+}
+
+cpu_load_shutdown() {
+	if [ "$clean_up_cpu_per_node" = "yes" ]; then
+		info "Cleaning up cpu per node load daemonset"
+		kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${cpu_per_node_daemonset}" || true
+	fi
+}

metrics/lib/cpu-load.md

@@ -0,0 +1,81 @@
+# `cpu-load` stack stresser
+
+The `cpu-load` stress functionality of the scaling framework allows you to optionally add a constant CPU stress
+load to cluster under test whilst the tests are running. This aids impact analysis of CPU load.
+
+The `cpu-load` functionality utilises the [`stress-ng`](https://kernel.ubuntu.com/git/cking/stress-ng.git/) tool
+to generate the CPU load. Some of the configuration parameters are taken directoy from the `stress-ng` command line.
+
+## Configuration
+
+`cpu-load` is configured via a number of environment variables:
+
+| Tool | Description |
+| ---- | ----------- |
+| collectd | `collectd` based statistics/metrics gathering daemonset code |
+| lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis |
+| report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results |
+| scaling | Tests to measure scaling, such as linear or parallel launching of pods |
+
+| Variable | Description | Default |
+| -------- | ----------- | ------- |
+| `SMF_CPU_LOAD_NODES` | Set to non-empty to deploy `cpu-load` stressor | unset (off) |
+| `SMF_CPU_LOAD_NODES_NCPU` | Number of stressor threads to launch per node | 0 (one per cpu) |
+| `SMF_CPU_LOAD_NODES_PERCENT` | Percentage of CPU to load | unset (100%) |
+| `SMF_CPU_LOAD_NODES_LIMIT` | k8s cpu resource limit to set | unset (none) |
+| `SMF_CPU_LOAD_NODES_REQUEST` | k8s cpu resource request to set | unset (none) |
+| `cpu_load_post_deploy_sleep` | Seconds to sleep for `cpu-load` deployment to settle | 30 |
+
+`SMF_CPU_LOAD_NODES` must be set to a non-empty string to enable the `cpu-load` functionality. `cpu-load` uses
+a daemonSet to deploy one `stress-ng` single container pod to each active node in the cluster.
+
+
+Any of the `SMF_CPU_LOAD_NODES_*` variables can be set, or unset, and the daemonSet pods will be configured
+appropriately.
+
+## Examples
+
+The combinations of settings available allow a lot of flexibility. Below are some common example setups:
+
+### 50% CPU load on all cores of all nodes (`stress-ng`)
+
+Here we allow `stress-ng` to spawn workers to cover all the CPUs on each node, but ask it to restrict its
+bandwidth use to 50% of the CPU. We do not use the k8s limits.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+#export SMF_CPU_LOAD_NODES_NCPU=
+export SMF_CPU_LOAD_NODES_PERCENT=50
+#export SMF_CPU_LOAD_NODES_LIMIT=999m
+#export SMF_CPU_LOAD_NODES_REQUEST=999m
+```
+
+### 50% CPU load on 1 un-pinned core of all nodes (k8s `limits`)
+
+Here we set `stress-ng` to run a single worker thread at 100% CPU, but use the k8s resource limits to restrict
+actual CPU usage to 50%. Because the k8s limit and request are not whole interger units, if the static policy is
+in place on the k8s cluster, the pods will be classified as Guaranteed QoS, but will *not* get pinned to a specific
+cpuset.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+export SMF_CPU_LOAD_NODES_NCPU=1
+export SMF_CPU_LOAD_NODES_PERCENT=100
+export SMF_CPU_LOAD_NODES_LIMIT=500m
+export SMF_CPU_LOAD_NODES_REQUEST=500m
+```
+
+### 50% CPU load pinned to 1 core, on all nodes
+
+Here we set `stress-ng` to run a single worker thread at 50% CPU, and use the k8s resource limits to classify the
+pod as Guaranteed, and as we are using whole integer units of CPU resource requests, if the static policy manager is
+in play, the thread will be pinned to a single cpu cpuset.
+
+```bash
+export SMF_CPU_LOAD_NODES=true
+export SMF_CPU_LOAD_NODES_NCPU=1
+export SMF_CPU_LOAD_NODES_PERCENT=50
+export SMF_CPU_LOAD_NODES_LIMIT=1
+export SMF_CPU_LOAD_NODES_REQUEST=1
+```
+

metrics/lib/cpu_load_daemonset.yaml.in

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cpu-load
+spec:
+  selector:
+    matchLabels:
+      name: cpu-load-pods
+  template:
+    metadata:
+      labels:
+        name: cpu-load-pods
+    spec:
+      hostNetwork: true
+      terminationGracePeriodSeconds: 0
+      containers:
+        - name: cpu-load
+          imagePullPolicy: IfNotPresent
+          image: polinux/stress-ng
+          command: ["stress-ng"]
+          args:                 # comment fields here so we can *delete* sections on demand
+            - "--cpu"
+            - "@CPU_NCPU@"
+            - "-l"              #CPU_PERCENT
+            - "@CPU_PERCENT@"   #CPU_PERCENT
+          resources:
+            limits:
+              cpu: @CPU_LIMIT@
+            requests:
+              cpu: @CPU_REQUEST@

metrics/report/makereport.sh

@@ -100,7 +100,7 @@ setup() {
 }
 
 run() {
-	docker run -ti --rm -v ${HOSTINPUTDIR}:${GUESTINPUTDIR} -v ${HOSTOUTPUTDIR}:${GUESTOUTPUTDIR} ${extra_volumes} ${IMAGE} ${extra_command}
+	docker run ${extra_opts} --rm -v ${HOSTINPUTDIR}:${GUESTINPUTDIR} -v ${HOSTOUTPUTDIR}:${GUESTOUTPUTDIR} ${extra_volumes} ${IMAGE} ${extra_command}
 	ls -la ${HOSTOUTPUTDIR}/*
 }
 
@@ -113,6 +113,7 @@ main() {
 			# In debug mode, run a shell instead of the default report generation
 			extra_command="bash"
 			extra_volumes="-v ${HOSTSCRIPTDIR}:${GUESTSCRIPTDIR}"
+			extra_opts="-ti"
 			;;
 		esac
 	done

metrics/report/report_dockerfile/Dockerfile

@@ -13,7 +13,7 @@
 # We would have used the 'verse' base, that already has some of the docs processing
 # installed, but I could not figure out how to add in the extra bits we needed to
 # the lite tex version is uses.
-FROM rocker/tidyverse
+FROM rocker/tidyverse:latest
 
 # Version of the Dockerfile
 LABEL DOCKERFILE_VERSION="1.0"

metrics/report/report_dockerfile/collectd_scaling.R

@@ -30,15 +30,6 @@ cpustats=c()		# Statistics for cpu usage
 bootstats=c()		# Statistics for boot (launch) times
 inodestats=c()		# Statistics for inode usage
 
-# values for scaling the secondary y axes on some graphs
-mem_scale=1
-cpu_scale=1
-inode_scale=1
-ip_scale=1
-oct_scale=1
-drop_scale=1
-error_scale=1
-
 # iterate over every set of results (test run)
 for (currentdir in resultdirs) {
 	# For every results file we are interested in evaluating
@@ -277,43 +268,79 @@ for (currentdir in resultdirs) {
 					next
 				}
 
-				max_free_mem=max(node_mem_free_data$value)
-				min_free_mem=min(node_mem_free_data$value)
+				# get the epoch time of first and last pod launch
+				start_time=local_bootdata$epoch[1]
+				end_time=local_bootdata$epoch[length(local_bootdata$epoch)]
+
+				# get value closest to first pod launch
+				mem_start_index=Position(function(x) x > start_time, node_mem_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(mem_start_index)) {
+					mem_start_index = 1
+				} else if (mem_start_index > 1) {
+					mem_start_index = mem_start_index - 1
+				}
+				max_free_mem=node_mem_free_data$value[mem_start_index]
+
+				# get value closest to last pod launch
+				mem_end_index=Position(function(x) x > end_time, node_mem_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(mem_end_index)) {
+					mem_end_index = length(node_mem_free_data$epoch)
+				} else if (mem_end_index > 1) {
+					mem_end_index = mem_end_index - 1
+				}
+				min_free_mem=node_mem_free_data$value[mem_end_index]
+
 				memtotal = memtotal + (max_free_mem - min_free_mem)
-				max_idle_cpu=max(node_cpu_idle_data$value)
-				min_idle_cpu=min(node_cpu_idle_data$value)
+
+				# get value closest to first pod launch
+				cpu_start_index=Position(function(x) x > start_time, node_cpu_idle_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(cpu_start_index)) {
+					cpu_start_index = 1
+				} else if (cpu_start_index > 1) {
+					cpu_start_index = cpu_start_index - 1
+				}
+				max_idle_cpu=node_cpu_idle_data$value[cpu_start_index]
+
+				# get value closest to last pod launch
+				cpu_end_index=Position(function(x) x > end_time, node_cpu_idle_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(cpu_end_index)) {
+					cpu_end_index = length(node_cpu_idle_data$epoch)
+				} else if (cpu_end_index > 1) {
+					cpu_end_index = cpu_end_index - 1
+				}
+				min_idle_cpu=node_cpu_idle_data$value[cpu_end_index]
+
 				cputotal = cputotal + (max_idle_cpu - min_idle_cpu)
-				max_free_inode=max(node_inode_free_data$value)
-				min_free_inode=min(node_inode_free_data$value)
+
+				# get value closest to first pod launch
+				inode_start_index=Position(function(x) x > start_time, node_inode_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(inode_start_index)) {
+					inode_start_index = 1
+				} else if (inode_start_index > 1) {
+					inode_start_index = inode_start_index - 1
+				}
+				max_free_inode=node_inode_free_data$value[inode_start_index]
+
+				# get value closest to last pod launch
+				inode_end_index=Position(function(x) x > end_time, node_inode_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(inode_end_index)) {
+					inode_end_index = length(node_cpu_idle_data$epoch)
+				} else if (inode_end_index > 1) {
+					inode_end_index = inode_end_index - 1
+				}
+				min_free_inode=node_inode_free_data$value[inode_end_index]
+
 				inodetotal = inodetotal + (max_free_inode - min_free_inode)
 			}
 
 			num_pods = local_bootdata$n_pods[length(local_bootdata$n_pods)]
 
-			# calculate scaling for secondary y axis
-			# the two y scales, in R, must be mathematically related
-			mem_scale = max(c(mem_scale,
-							  (max(mem_free_data$value) / (1024*1024*1024)) / num_pods))
-			cpu_scale = max(c(cpu_scale,
-							  max(cpu_idle_data$value) / num_pods))
-			inode_scale = max(c(inode_scale,
-								max(inode_free_data$value) / num_pods))
-			ip_scale = max(c(ip_scale,
-							 max(c(max(interface_packets_data$tx, na.rm=TRUE),
-								   max(interface_packets_data$rx, na.rm=TRUE))) / num_pods))
-			oct_scale = max(c(oct_scale,
-							  max(c(max(interface_octets_data$tx, na.rm=TRUE),
-									max(interface_octets_data$rx, na.rm=TRUE))) / num_pods))
-			# drops and scale are often 0, so providing 1 so we won't scale by infinity
-			drop_scale = max(c(drop_scale,
-							   max(c(1,
-									 max(interface_dropped_data$tx, na.rm=TRUE),
-									 max(interface_dropped_data$rx, na.rm=TRUE))) / num_pods))
-			error_scale = max(c(error_scale,
-								max(c(1,
-									  max(interface_errors_data$tx, na.rm=TRUE),
-									  max(interface_errors_data$rx, na.rm=TRUE))) / num_pods))
-
 			# We get data in b, but want the graphs in Gb.
 			memtotal = memtotal / (1024*1024*1024)
 			gb_per_pod = memtotal/num_pods
@@ -380,13 +407,14 @@ memfreedata$mem_free_gb = memfreedata$value/(1024*1024*1024)
 # And show the boot times in seconds, not ms
 podbootdata$launch_time_s = podbootdata$launch_time/1000.0
 
+
 ########### Output memory page ##############
 mem_stats_plot = suppressWarnings(ggtexttable(data.frame(memstats),
 	theme=ttheme(base_size=10),
 	rows=NULL
 	))
 
-#mem_sec_axis_scale=
+mem_scale = (max(memfreedata$value) / (1024*1024*1024)) / max(podbootdata$n_pods)
 mem_line_plot <- ggplot() +
 	geom_line(data=memfreedata,
 			  aes(s_offset, mem_free_gb, colour=interaction(testname, node),
@@ -407,6 +435,7 @@ mem_line_plot <- ggplot() +
 	ylab("System Avail (Gb)") +
 	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./mem_scale, name="pods")) +
 	ggtitle("System Memory free") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page1 = grid.arrange(
@@ -424,6 +453,7 @@ cpu_stats_plot = suppressWarnings(ggtexttable(data.frame(cpustats),
 	rows=NULL
 	))
 
+cpu_scale = max(cpuidledata$value) / max(podbootdata$n_pods)
 cpu_line_plot <- ggplot() +
 	geom_line(data=cpuidledata,
 			  aes(x=s_offset, y=value, colour=interaction(testname, node),
@@ -444,6 +474,7 @@ cpu_line_plot <- ggplot() +
 	xlab("seconds") +
 	ylab("System CPU Idle (%)") +
 	ggtitle("System CPU usage") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page2 = grid.arrange(
@@ -468,6 +499,7 @@ boot_line_plot <- ggplot() +
 	xlab("pods") +
 	ylab("Boot time (s)") +
 	ggtitle("Pod boot time") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page3 = grid.arrange(
@@ -485,6 +517,7 @@ inode_stats_plot = suppressWarnings(ggtexttable(data.frame(inodestats),
 	rows=NULL
 	))
 
+inode_scale = max(inodefreedata$value) / max(podbootdata$n_pods)
 inode_line_plot <- ggplot() +
 	geom_line(data=inodefreedata,
 			  aes(x=s_offset, y=value, colour=interaction(testname, node),
@@ -505,6 +538,7 @@ inode_line_plot <- ggplot() +
 	ylab("inodes free") +
 	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./inode_scale, name="pods")) +
 	ggtitle("inodes free") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page4 = grid.arrange(
@@ -517,6 +551,8 @@ page4 = grid.arrange(
 cat("\n\n\\pagebreak\n")
 
 ########## Output interface page packets and octets ##############
+ip_scale = max(c(max(ifpacketdata$tx, na.rm=TRUE),
+				 max(ifpacketdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_packet_line_plot <- ggplot() +
 	geom_line(data=ifpacketdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -545,8 +581,11 @@ interface_packet_line_plot <- ggplot() +
 	ylab("packets") +
 	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./ip_scale, name="pods")) +
 	ggtitle("interface packets") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
+oct_scale = max(c(max(ifoctetdata$tx, na.rm=TRUE),
+				  max(ifoctetdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_octet_line_plot <- ggplot() +
 	geom_line(data=ifoctetdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -575,9 +614,9 @@ interface_octet_line_plot <- ggplot() +
 	ylab("octets") +
 	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./oct_scale, name="pods")) +
 	ggtitle("interface octets") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
-
 page5 = grid.arrange(
 	interface_packet_line_plot,
 	interface_octet_line_plot,
@@ -588,6 +627,10 @@ page5 = grid.arrange(
 cat("\n\n\\pagebreak\n")
 
 ########## Output interface page drops and errors ##############
+# drops are often 0, so providing 1 so we won't scale by infinity
+drop_scale = max(c(1,
+				   max(ifdropdata$tx, na.rm=TRUE),
+				   max(ifdropdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_drop_line_plot <- ggplot() +
 	geom_line(data=ifdropdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -614,10 +657,15 @@ interface_drop_line_plot <- ggplot() +
 	labs(colour="") +
 	xlab("seconds") +
 	ylab("drops") +
-	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./drop_scale, name="pods")) +
+	scale_y_continuous(breaks=pretty_breaks(), sec.axis=sec_axis(~ ./drop_scale, name="pods", labels=comma)) +
 	ggtitle("interface drops") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
+# errors are often 0, so providing 1 so we won't scale by infinity
+error_scale = max(c(1,
+					max(iferrordata$tx, na.rm=TRUE),
+					max(iferrordata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_error_line_plot <- ggplot() +
 	geom_line(data=iferrordata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -644,8 +692,9 @@ interface_error_line_plot <- ggplot() +
 	labs(colour="") +
 	xlab("seconds") +
 	ylab("errors") +
-	scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./error_scale, name="pods")) +
+	scale_y_continuous(breaks=pretty_breaks(), sec.axis=sec_axis(~ ./error_scale, name="pods", labels=comma)) +
 	ggtitle("interface errors") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page6 = grid.arrange(

metrics/report/report_dockerfile/pdf.Rmd

@@ -8,6 +8,10 @@ author: "Auto generated"
 date: "`r format(Sys.time(), '%d %B, %Y')`"
 output:
   pdf_document:
+# Shrink the page margins so we get bigger/better resolution on the graphs
+# Keep the top and bottom margins reasonable, as we are really interested in
+# gaining 'width', and if we trim the bottom too much, we lose the page numbers.
+geometry: "left=1cm, right=1cm, top=2cm, bottom=2cm"
 urlcolor: blue
 ---
 

metrics/report/report_dockerfile/tidy_scaling.R

@@ -190,13 +190,13 @@ for (currentdir in resultdirs) {
 				"avg_inode"=round(inodetotal/num_pods, 4)
 				)
 			inodestats=rbind(inodestats, local_inodes)
-		}
 
-		# And collect up our rows into our global table of all results
-		# These two tables *should* be the source of all the data we need to
-		# process and plot (apart from the stats....)
-		bootdata=rbind(bootdata, local_bootdata, make.row.names=FALSE)
-		nodedata=rbind(nodedata, local_nodedata, make.row.names=FALSE)
+			# And collect up our rows into our global table of all results
+			# These two tables *should* be the source of all the data we need to
+			# process and plot (apart from the stats....)
+			bootdata=rbind(bootdata, local_bootdata, make.row.names=FALSE)
+			nodedata=rbind(nodedata, local_nodedata, make.row.names=FALSE)
+		}
 	}
 }
 
@@ -229,6 +229,7 @@ mem_line_plot <- ggplot(data=nodedata, aes(n_pods,
 	ylab("System Avail (Gb)") +
 	scale_y_continuous(labels=comma) +
 	ggtitle("System Memory free") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page1 = grid.arrange(
@@ -256,6 +257,7 @@ cpu_line_plot <- ggplot(data=nodedata, aes(n_pods,
 	xlab("pods") +
 	ylab("System CPU Idle (%)") +
 	ggtitle("System CPU usage") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page2 = grid.arrange(
@@ -279,6 +281,7 @@ boot_line_plot <- ggplot() +
 	xlab("pods") +
 	ylab("Boot time (s)") +
 	ggtitle("Pod boot time") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page3 = grid.arrange(
@@ -307,6 +310,7 @@ inode_line_plot <- ggplot(data=nodedata, aes(n_pods,
 	ylab("inodes free") +
 	scale_y_continuous(labels=comma) +
 	ggtitle("inodes free") +
+	theme(legend.position="bottom") +
 	theme(axis.text.x=element_text(angle=90))
 
 page4 = grid.arrange(

metrics/scaling/common.bash

@@ -13,6 +13,7 @@ deployment="busybox"
 stats_pod="stats"
 
 NUM_PODS=${NUM_PODS:-20}
+NUM_DEPLOYMENTS=${NUM_DEPLOYMENTS:-20}
 STEP=${STEP:-1}
 
 LABEL=${LABEL:-magiclabel}
@@ -24,6 +25,8 @@ delete_wait_time=${delete_wait_time:-600}
 settle_time=${settle_time:-5}
 use_api=${use_api:-yes}
 grace=${grace:-30}
+proc_wait_time=${proc_wait_time:-20}
+proc_sleep_time=2
 
 declare -a new_pods
 declare -A node_basemem

metrics/scaling/k8s_parallel.sh

@@ -126,14 +126,10 @@ init() {
 	# a nice way to do it (unless you want to parse 'descibe nodes')
 	# Have a read of https://github.com/kubernetes/kubernetes/issues/25353
 
-	k8s_api_init
+	framework_init
 
 	# Ensure we pre-cache the container image etc.
 	warmup
-
-	# And now we can set up our results storage then...
-	metrics_json_init "k8s"
-	save_config
 }
 
 save_config(){
@@ -218,11 +214,8 @@ cleanup() {
 
 	# First try to save any results we got
 	metrics_json_end_array "BootResults"
-	metrics_json_save
-
 	kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time}
-
-	k8s_api_shutdown
+	framework_shutdown
 }
 
 show_vars()
@@ -253,8 +246,8 @@ help()
 	usage=$(cat << EOF
 Usage: $0 [-h] [options]
    Description:
-	Launch a series of workloads and take memory metric measurements after
-	each launch.
+	Launch a series of workloads in a parallel manner and take memory metric measurements
+	after each launch.
    Options:
         -h,    Help page.
 EOF

metrics/scaling/k8s_scale.sh

@@ -169,7 +169,7 @@ EOF
 	if [ $n_pods -eq 0 ]; then
 		local pods_per_gb=0
 	else
-		local pods_per_gb=$(bc -l <<< "scale=2; ($total_mem_used/1024) / $n_pods")
+		local pods_per_gb=$(printf "%0f" $(bc -l <<< "scale=2; ($total_mem_used/1024) / $n_pods"))
 	fi
 	local mem_json="$(cat << EOF
 			"memory": {
@@ -209,7 +209,7 @@ init() {
 	# FIXME - check the node(s) can run enough pods - check 'max-pods' in the
 	# kubelet config - from 'kubectl describe node -o json' ?
 
-	k8s_api_init
+	framework_init
 
 	# Launch our stats gathering pod
 	kubectl apply -f ${SCRIPT_PATH}/${stats_pod}.yaml
@@ -218,10 +218,6 @@ init() {
 	# FIXME - we should probably 'warm up' the cluster with the container image(s) we will
 	# use for testing, otherwise the download time will likely be included in the first pod
 	# boot time.
-
-	# And now we can set up our results storage then...
-	metrics_json_init "k8s"
-	save_config
 }
 
 save_config(){
@@ -347,9 +343,7 @@ EOF
 )"
 
 	metrics_json_add_fragment "$json"
-	metrics_json_save
-
-	k8s_api_shutdown
+	framework_shutdown
 }
 
 show_vars()
@@ -380,8 +374,8 @@ help()
 	usage=$(cat << EOF
 Usage: $0 [-h] [options]
    Description:
-	Launch a series of workloads and take memory metric measurements after
-	each launch.
+	Launch a series of workloads in a linear manner and take memory metric measurements
+	after each launch.
    Options:
 		-h,    Help page.
 EOF

metrics/scaling/k8s_scale_nc.sh

@@ -195,7 +195,7 @@ EOF
 	if [ $n_pods -eq 0 ]; then
 		local pods_per_gb=0
 	else
-		local pods_per_gb=$(bc -l <<< "scale=2; ($total_mem_used/1024) / $n_pods")
+		local pods_per_gb=$(printf "%0f" $(bc -l <<< "scale=2; ($total_mem_used/1024) / $n_pods"))
 	fi
 	local mem_json="$(cat << EOF
 			"memory": {
@@ -235,7 +235,7 @@ init() {
 	# FIXME - check the node(s) can run enough pods - check 'max-pods' in the
 	# kubelet config - from 'kubectl describe node -o json' ?
 
-	k8s_api_init
+	framework_init
 
 	# Launch our stats gathering pod
 	kubectl apply -f ${SCRIPT_PATH}/${stats_pod}.yaml
@@ -244,10 +244,6 @@ init() {
 	# FIXME - we should probably 'warm up' the cluster with the container image(s) we will
 	# use for testing, otherwise the download time will likely be included in the first pod
 	# boot time.
-
-	# And now we can set up our results storage then...
-	metrics_json_init "k8s"
-	save_config
 }
 
 save_config(){
@@ -410,9 +406,7 @@ EOF
 )"
 
 	metrics_json_add_fragment "$json"
-	metrics_json_save
-
-	k8s_api_shutdown
+	framework_shutdown
 }
 
 show_vars()

metrics/scaling/k8s_scale_net.sh

@@ -0,0 +1,285 @@
+#!/bin/bash
+# Copyright (c) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+# Pull in some common, useful, items
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+source "${SCRIPT_PATH}/common.bash"
+
+LABELVALUE=${LABELVALUE:-scale_net}
+
+# Set some default metrics env vars
+TEST_ARGS="runtime=${RUNTIME}"
+TEST_NAME="k8s scaling net"
+input_yaml="${SCRIPT_PATH}/net-serve.yaml.in"
+input_json="${SCRIPT_PATH}/net-serve.json.in"
+name_base_depl="net-serve"
+
+# $1 is the launch time in seconds this pod/container took to start up.
+# $2 is the number of pod/containers under test
+# $3 is the time to pod network measure
+grab_stats(){
+    local launch_time_ms=$1
+    local n_pods=$2
+    local net_time=$3
+
+    info "And grab some stats"
+
+    local date_json="$(cat << EOF
+        "date": {
+            "ns": $(date +%s%N),
+            "Date": "$(date -u +"%Y-%m-%dT%T.%3N")"
+        }
+EOF
+)"
+    metrics_json_add_array_fragment "$date_json"
+
+    local pods_json="$(cat << EOF
+	                "n_pods": {
+                            "Result": ${n_pods},
+                            "Units" : "int"
+                    }
+EOF
+)"
+    metrics_json_add_array_fragment "$pods_json"
+
+    local time_to_pod_net_json="$(cat << EOF
+                        "time_to_pod_net": {
+                                "Result": ${net_time},
+                                "Units" : "ms"
+                            }
+EOF
+)"
+    metrics_json_add_array_fragment "$time_to_pod_net_json"
+
+    local launch_json="$(cat << EOF
+                        "launch_time": {
+                                "Result": $launch_time_ms,
+                                "Units" : "ms"
+                            }
+EOF
+)"
+    metrics_json_add_array_fragment "$launch_json"
+
+    info "launch [$launch_time_ms]"
+
+    metrics_json_close_array_element
+}
+
+init() {
+    info "Initialising"
+
+    local cmds=("bc" "jq")
+    check_cmds "${cmds[@]}"
+
+    info "Checking Kubernetes accessible"
+    local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
+    if [ "$worked" != 0 ]; then
+        die "kubectl failed to get nodes"
+    fi
+
+    info $(get_num_nodes) "Kubernetes nodes in 'Ready' state found"
+
+    framework_init
+}
+
+save_config() {
+    metrics_json_start_array
+
+    local json="$(cat << EOF
+    {
+        "testname": "${TEST_NAME}",
+        "NUM_DEPLOYMENTS": ${NUM_DEPLOYMENTS},
+        "STEP": ${STEP},
+        "wait_time": ${wait_time},
+        "delete_wait_time": ${delete_wait_time},
+        "settle_time": ${settle_time}
+    }
+EOF
+)"
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "Config"
+}
+
+run() {
+	info "Running test"
+	local header_post="Content-Type: application/json"
+	local base_curl=${API_ADDRESS}:${API_PORT}/apis/apps/v1/namespaces/default/deployments
+
+	trap cleanup EXIT QUIT KILL
+
+	metrics_json_start_array
+
+	for reqs in $(seq ${STEP} ${STEP} ${NUM_DEPLOYMENTS}); do
+		local deployment="${name_base_depl}${reqs}"
+		info "Testing replicas ${reqs} of ${NUM_DEPLOYMENTS}"
+		# Generate the next yaml file
+
+		local runtime_command
+		if [ -n "$RUNTIME" ]; then
+			runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g"
+		else
+			runtime_command="/@RUNTIMECLASS@/d"
+		fi
+
+		local input_template
+		local generated_file
+		if [ "$use_api" != "no" ]; then
+			input_template=$input_json
+			generated_file=$generated_json
+		else
+			input_template=$input_yaml
+			generated_file=$generated_yaml
+		fi
+
+		sed -e $runtime_command \
+			-e "s|@DEPLOYMENT@|${deployment}|g" \
+			-e "s|@LABEL@|${LABEL}|g" \
+			-e "s|@LABELVALUE@|${LABELVALUE}|g" \
+			-e "s|@GRACE@|${grace}|g" \
+			< ${input_template} > ${generated_file}
+
+		info "Applying changes"
+		local start_time=$(date +%s%N)
+
+		if [ "$use_api" != "no" ]; then
+			curl -s ${base_curl} -XPOST -H "${header_post}" -d@${generated_file} > /dev/null
+		else
+			kubectl apply -f ${generated_file}
+		fi
+
+		kubectl rollout status --timeout=${wait_time}s deployment/${deployment}
+		kubectl expose --port=8080 deployment $deployment
+
+        # Check service exposed
+        cmd="kubectl get services $deployment -n default --no-headers=true"
+        waitForProcess "$proc_wait_time" "$proc_sleep_time" "$cmd" "Waiting for service"
+
+		IP=$(kubectl get services $deployment -n default --no-headers=true | awk '{printf $3}')
+		end_net=$(date +%s%N)
+		info "IP: $IP"
+
+		# service health check
+        cmd="curl --noproxy \"*\" http://$IP:8080/healthz"
+        waitForProcess "$proc_wait_time" "$proc_sleep_time" "$cmd" "http server is not ready yet!!"
+
+		RESP=$(curl -s --noproxy "*" http://$IP:8080/echo?msg=curl%20request%20to%20$deployment)
+		local end_time=$(date +%s%N)
+		info "http reply: $RESP"
+
+		local total_milliseconds=$(( (end_time - start_time) / 1000000 ))
+		local net_diff=$(( (end_net - start_time) / 1000000 ))
+		info "Took $total_milliseconds ms ($end_time - $start_time)"
+		info "Net took $net_diff ms"
+
+		kubectl delete service $deployment
+		if [ $? -ne 0 ]; then
+			echo "kubectl delete service failed"
+			exit
+		fi
+
+		sleep ${settle_time}
+		grab_stats $total_milliseconds $reqs $net_diff
+	done
+}
+
+cleanup() {
+    info "Cleaning up"
+
+    # First try to save any results we got
+    metrics_json_end_array "BootResults"
+
+    local start_time=$(date +%s%N)
+
+    for reqs in $(seq ${STEP} ${STEP} ${NUM_DEPLOYMENTS}); do
+        local deployment="${name_base_depl}${reqs}"
+        kubectl delete deployment --wait=true --timeout=${delete_wait_time}s ${deployment} || true
+    done
+
+    for x in $(seq 1 ${delete_wait_time}); do
+        local npods=$(kubectl get pods -l=${LABEL}=${LABELVALUE} -o=name | wc -l)
+        if [ $npods -eq 0 ]; then
+            echo "All pods have terminated at cycle $x"
+            local alldied=true
+            break;
+        fi
+        sleep 1
+    done
+
+    if [ -z "$alldied" ]; then
+        echo "ERROR: Not all pods died!"
+    fi
+
+    local end_time=$(date +%s%N)
+    local total_milliseconds=$(( (end_time - start_time) / 1000000 ))
+    info "Delete Took $total_milliseconds ms ($end_time - $start_time)"
+
+    local json="$(cat << EOF
+        "Delete": {
+            "Result": ${total_milliseconds},
+            "Units" : "ms"
+        }
+EOF
+)"
+
+    metrics_json_add_fragment "$json"
+    framework_shutdown
+}
+
+show_vars() {
+	echo -e "\nEnvironment variables:"
+	echo -e "\tName (default)"
+	echo -e "\t\tDescription"
+	echo -e "\tNUM_DEPLOYMENTS (${NUM_DEPLOYMENTS})"
+	echo -e "\t\tNumber of deployments to launch"
+	echo -e "\tSTEP (${STEP})"
+	echo -e "\t\tNumber of pods to launch per cycle"
+	echo -e "\twait_time (${wait_time})"
+	echo -e "\t\tSeconds to wait for pods to become ready"
+	echo -e "\tproc_wait_time (${proc_wait_time})"
+	echo -e "\t\tSeconds to wait for net server process to become ready"
+	echo -e "\tdelete_wait_time (${delete_wait_time})"
+	echo -e "\t\tSeconds to wait for all pods to be deleted"
+	echo -e "\tsettle_time (${settle_time})"
+	echo -e "\t\tSeconds to wait after pods ready before taking measurements"
+	echo -e "\tuse_api (${use_api})"
+	echo -e "\t\tspecify yes or no to use the API to launch pods"
+	echo -e "\tgrace (${grace})"
+	echo -e "\t\tspecify the grace period in seconds for workload pod termination"
+}
+
+help() {
+    usage=$(cat << EOF
+    Usage: $0 [-h] [options]
+    Description:
+        Launch a series of workloads and take time to pod network  metric measurements after
+        each launch.
+    Options:
+        -h,    Help page.
+EOF
+)
+    echo "$usage"
+    show_vars
+}
+
+main() {
+    local OPTIND
+    while getopts "h" opt;do
+        case ${opt} in
+            h)
+                help
+                exit 0;
+                ;;
+        esac
+    done
+    shift $((OPTIND-1))
+    init
+    run
+}
+
+main "$@"

metrics/scaling/k8s_scale_rapid.sh

@@ -15,6 +15,8 @@ source "${SCRIPT_PATH}/../collectd/collectd.bash"
 NUM_PODS=${NUM_PODS:-20}
 STEP=${STEP:-1}
 
+SMF_USE_COLLECTD=true
+
 LABELVALUE=${LABELVALUE:-gandalf}
 
 pod_command="[\"tail\", \"-f\", \"/dev/null\"]"
@@ -64,27 +66,7 @@ EOF
 }
 
 init() {
-	info "Initialising"
-
-	local cmds=("bc" "jq")
-	check_cmds "${cmds[@]}"
-
-	info "Checking k8s accessible"
-	local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
-	if [ "$worked" != 0 ]; then
-		die "kubectl failed to get nodes"
-	fi
-
-	info $(get_num_nodes) "k8s nodes in 'Ready' state found"
-
-	k8s_api_init
-
-	# Launch our stats gathering pod
-	init_stats $wait_time
-
-	# And now we can set up our results storage then...
-	metrics_json_init "k8s"
-	save_config
+	framework_init
 }
 
 save_config(){
@@ -198,11 +180,7 @@ EOF
 )"
 
 	metrics_json_add_fragment "$json"
-	metrics_json_save
-
-	cleanup_stats $delete_wait_time
-
-	k8s_api_shutdown
+	framework_shutdown
 }
 
 show_vars()

metrics/scaling/net-serve.json.in

@@ -0,0 +1,40 @@
+{
+    "apiVersion": "apps/v1",
+    "kind": "Deployment",
+    "metadata": {
+        "labels": {
+            "run": "net-serve"
+        },
+        "name": "@DEPLOYMENT@"
+    },
+    "spec": {
+        "replicas": 1,
+        "selector": {
+            "matchLabels": {
+                "run": "net-serve"
+            }
+        },
+        "template": {
+            "metadata": {
+                "labels": {
+                    "run": "net-serve",
+                    "@LABEL@": "@LABELVALUE@"
+                }
+            },
+            "spec": {
+                "terminationGracePeriodSeconds": @GRACE@,
+                "runtimeClassName": "@RUNTIMECLASS@",
+                "automountServiceAccountToken": false,
+                "containers": [{
+                    "name": "net-serve",
+                    "image": "gcr.io/kubernetes-e2e-test-images/agnhost:2.8",
+                    "imagePullPolicy": "IfNotPresent",
+                    "args": [
+                        "netexec"
+                    ]
+                }],
+                "restartPolicy": "Always"
+            }
+        }
+    }
+}

metrics/scaling/net-serve.yaml.in

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: net-serve
+  name: @DEPLOYMENT@
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: net-serve
+  template:
+    metadata:
+      labels:
+        run: net-serve
+        @LABEL@: @LABELVALUE@
+    spec:
+      terminationGracePeriodSeconds: @GRACE@
+      runtimeClassName: @RUNTIMECLASS@
+      automountServiceAccountToken: false
+      containers:
+      - name: net-serve
+        image: gcr.io/kubernetes-e2e-test-images/agnhost:2.8
+        imagePullPolicy: IfNotPresent
+        args:
+        - netexec
+      restartPolicy: Always