Initial commit of metrics for scaling

metrics/README.md

@@ -0,0 +1,18 @@
+# Metric testing for scaling on Kubernetes.
+
+The usage is
+
+```bash
+cd metrics/scaling
+./k8s-scale.sh
+```
+
+The default container runtime is not Kata containers, but Kata can be specified
+setting the environment variable:
+
+```bash
+use_kata_runtime=yes
+```
+
+These tests currently only support single node deployments, but PRs to add
+multi-node support will follow shortly.

metrics/lib/common.bash

@@ -0,0 +1,339 @@
+#!/bin/bash
+#
+# Copyright (c) 2017,2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+THIS_FILE=$(readlink -f ${BASH_SOURCE[0]})
+LIB_DIR=${THIS_FILE%/*}
+RESULT_DIR="${LIB_DIR}/../results"
+
+source ${LIB_DIR}/kata-common.bash
+source ${LIB_DIR}/json.bash
+source /etc/os-release || source /usr/lib/os-release
+KATA_KSM_THROTTLER="${KATA_KSM_THROTTLER:-no}"
+
+# Set variables to reasonable defaults if unset or empty
+DOCKER_EXE="${DOCKER_EXE:-docker}"
+RUNTIME="${RUNTIME:-kata-runtime}"
+
+KSM_BASE="/sys/kernel/mm/ksm"
+KSM_ENABLE_FILE="${KSM_BASE}/run"
+KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan"
+KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs"
+
+# The settings we use for an 'aggresive' KSM setup
+# Scan 1000 pages every 50ms - 20,000 pages/s
+KSM_AGGRESIVE_PAGES=1000
+KSM_AGGRESIVE_SLEEP=50
+
+# If we fail for any reason, exit through here and we should log that to the correct
+# place and return the correct code to halt the run
+die(){
+	msg="$*"
+	echo "ERROR: $msg" >&2
+	exit 1
+}
+
+# Sometimes we just want to warn about something - let's have a standard
+# method for that, so maybe we can make a standard form that can be searched
+# for in the logs/tooling
+warning(){
+	msg="$*"
+	echo "WARNING: $msg" >&2
+}
+
+info() {
+    echo -e "INFO: $*"
+}
+
+# This function checks existence of commands.
+# They can be received standalone or as an array, e.g.
+#
+# cmds=(“cmd1” “cmd2”)
+# check_cmds "${cmds[@]}"
+check_cmds()
+{
+	local cmd req_cmds=( "$@" )
+	for cmd in "${req_cmds[@]}"; do
+		if ! command -v "$cmd" > /dev/null 2>&1; then
+			die "command $cmd not available"
+		fi
+		echo "command: $cmd: yes"
+	done
+}
+
+# This function performs a docker pull on the image names
+# passed in (notionally as 'about to be used'), to ensure
+#  - that we have the most upto date images
+#  - that any pull/refresh time (for a first pull) does not
+#    happen during the test itself.
+#
+# The image list can be received standalone or as an array, e.g.
+#
+# images=(“img1” “img2”)
+# check_imgs "${images[@]}"
+check_images()
+{
+	local img req_images=( "$@" )
+	for img in "${req_images[@]}"; do
+		echo "docker pull'ing: $img"
+		if ! docker pull "$img"; then
+			die "Failed to docker pull image $img"
+		fi
+		echo "docker pull'd: $img"
+	done
+}
+
+# This function performs a docker build on the image names
+# passed in, to ensure that we have the latest changes from
+# the dockerfiles
+build_dockerfile_image()
+{
+	local image="$1"
+	local dockerfile_path="$2"
+	local dockerfile_dir=${2%/*}
+
+	echo "docker building $image"
+	if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then
+		die "Failed to docker build image $image"
+	fi
+}
+
+# This function verifies that the dockerfile version is
+# equal to the test version in order to build the image or
+# just run the test
+check_dockerfiles_images()
+{
+	local image="$1"
+	local dockerfile_path="$2"
+
+	if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then
+		die "Missing image or dockerfile path variable"
+	fi
+
+	# Verify that dockerfile version is equal to test version
+	check_image=$(docker images "$image" -q)
+	if [ -n "$check_image" ]; then
+		# Check image label
+		check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4)
+		if [ -n "$check_image_version" ]; then
+			echo "$image is not updated"
+			build_dockerfile_image "$image" "$dockerfile_path"
+		else
+			# Check dockerfile label
+			dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2)
+			if [ "$dockerfile_version" != "$check_image_version" ]; then
+				echo "$dockerfile_version is not equal to $check_image_version"
+				build_dockerfile_image "$image" "$dockerfile_path"
+			fi
+		fi
+	else
+		build_dockerfile_image "$image" "$dockerfile_path"
+	fi
+}
+
+# A one time (per uber test cycle) init that tries to get the
+# system to a 'known state' as much as possible
+metrics_onetime_init()
+{
+	# The onetime init must be called once, and only once
+	if [ ! -z "$onetime_init_done" ]; then
+		die "onetime_init() called more than once"
+	fi
+
+	# Restart services
+	sudo systemctl restart docker
+
+	# We want this to be seen in sub shells as well...
+	# otherwise init_env() cannot check us
+	export onetime_init_done=1
+}
+
+# Print a banner to the logs noting clearly which test
+# we are about to run
+test_banner()
+{
+	echo -e "\n===== starting test [$1] ====="
+}
+
+# Initialization/verification environment. This function makes
+# minimal steps for metrics/tests execution.
+init_env()
+{
+	test_banner "${TEST_NAME}"
+
+	cmd=("docker")
+
+	# check dependencies
+	check_cmds "${cmd[@]}"
+
+	# Remove all stopped containers
+	clean_env
+
+	# This clean up is more aggressive, this is in order to
+	# decrease the factors that could affect the metrics results.
+	kill_processes_before_start
+}
+
+# This function checks if there are containers or
+# shim/proxy/hypervisor processes up, if found, they are
+# killed to start test with clean environment.
+kill_processes_before_start() {
+	DOCKER_PROCS=$(${DOCKER_EXE} ps -q)
+	[[ -n "${DOCKER_PROCS}" ]] && clean_env
+	check_processes
+}
+
+# Generate a random name - generally used when creating containers, but can
+# be used for any other appropriate purpose
+random_name() {
+	mktemp -u kata-XXXXXX
+}
+
+# Dump diagnostics about our current system state.
+# Very useful for diagnosing if we have failed a sanity check
+show_system_state() {
+	echo "Showing system state:"
+	echo " --Docker ps--"
+	${DOCKER_EXE} ps -a
+	echo " --${RUNTIME} list--"
+	local RPATH=$(command -v ${RUNTIME})
+	sudo ${RPATH} list
+
+	local processes="kata-proxy kata-shim kata-runtime qemu"
+
+	for p in ${processes}; do
+		echo " --pgrep ${p}--"
+		pgrep -a ${p}
+	done
+
+	# Verify kata-ksm-throttler
+	if [ "$KATA_KSM_THROTTLER" == "yes" ]; then
+		process="kata-ksm-throttler"
+		process_path=$(whereis ${process} | tr -d '[:space:]'| cut -d ':' -f2)
+		echo " --pgrep ${process}--"
+		pgrep -f ${process_path} > /dev/null
+	fi
+}
+
+common_init(){
+
+	# If we are running a kata runtime, go extract its environment
+	# for later use.
+	local iskata=$(is_a_kata_runtime "$RUNTIME")
+
+	if [ "$iskata" == "1" ]; then
+		extract_kata_env
+	else
+		# We know we have nothing to do for runc
+		if [ "$RUNTIME" != "runc" ]; then
+			warning "Unrecognised runtime ${RUNTIME}"
+		fi
+	fi
+}
+
+
+# Save the current KSM settings so we can restore them later
+save_ksm_settings(){
+	echo "saving KSM settings"
+	ksm_stored_run=$(cat ${KSM_ENABLE_FILE})
+	ksm_stored_pages=$(cat ${KSM_ENABLE_FILE})
+	ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE})
+}
+
+set_ksm_aggressive(){
+	echo "setting KSM to aggressive mode"
+	# Flip the run off/on to ensure a restart/rescan
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+	sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}"
+	sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}"
+	sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}"
+}
+
+restore_ksm_settings(){
+	echo "restoring KSM settings"
+	# First turn off the run to ensure if we are then re-enabling
+	# that any changes take effect
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+	sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}"
+	sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}"
+	sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}"
+}
+
+disable_ksm(){
+	echo "disabling KSM"
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+}
+
+# See if KSM is enabled.
+# If so, amend the test name to reflect that
+check_for_ksm(){
+	if [ ! -f ${KSM_ENABLE_FILE} ]; then
+		return
+	fi
+
+	ksm_on=$(< ${KSM_ENABLE_FILE})
+
+	if [ $ksm_on == "1" ]; then
+		TEST_NAME="${TEST_NAME} ksm"
+	fi
+}
+
+# Wait for KSM to settle down, or timeout waiting
+# The basic algorithm is to look at the pages_shared value
+# at the end of every 'full scan', and if the value
+# has changed very little, then we are done (because we presume
+# a full scan has managed to do few new merges)
+#
+# arg1 - timeout in seconds
+wait_ksm_settle(){
+	[[ "$RUNTIME" == "runc" ]] || [[ "$RUNTIME" == "kata-fc" ]] && return
+	local t pcnt
+	local oldscan=-1 newscan
+	local oldpages=-1 newpages
+
+	oldscan=$(cat /sys/kernel/mm/ksm/full_scans)
+
+	# Go around the loop until either we see a small % change
+	# between two full_scans, or we timeout
+	for ((t=0; t<$1; t++)); do
+
+		newscan=$(cat /sys/kernel/mm/ksm/full_scans)
+		newpages=$(cat /sys/kernel/mm/ksm/pages_shared)
+		[[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return
+
+		if (( newscan != oldscan )); then
+			echo -e "\nnew full_scan ($oldscan to $newscan)"
+
+			# Do we have a previous scan to compare with
+			echo "check pages $oldpages to $newpages"
+
+			if (( oldpages != -1 )); then
+				# avoid divide by zero problems
+				if (( $oldpages > 0 )); then
+					pcnt=$(( 100 - ((newpages * 100) / oldpages) ))
+					# abs()
+					pcnt=$(( $pcnt * -1 ))
+
+					echo "$oldpages to $newpages is ${pcnt}%"
+
+					if (( $pcnt <= 5 )); then
+						echo "KSM stabilised at ${t}s"
+						return
+					fi
+				else
+					echo "$oldpages KSM pages... waiting"
+				fi
+			fi
+			oldscan=$newscan
+			oldpages=$newpages
+		else
+			echo -n "."
+		fi
+		sleep 1
+	done
+	echo "Timed out after ${1}s waiting for KSM to settle"
+}
+
+common_init

metrics/lib/json.bash

@@ -0,0 +1,248 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Helper routines for generating JSON formatted results.
+
+declare -a json_result_array
+declare -a json_array_array
+
+# Generate a timestamp in nanoseconds since 1st Jan 1970
+timestamp_ns() {
+	local t
+	local s
+	local n
+	local ns
+
+	t="$(date +%-s:%-N)"
+	s=$(echo $t | awk -F ':' '{print $1}')
+	n=$(echo $t | awk -F ':' '{print $2}')
+	ns=$(( (s * 1000000000) + n ))
+
+	echo $ns
+}
+
+# Generate a timestamp in milliseconds since 1st Jan 1970
+timestamp_ms() {
+	echo $(($(date +%s%N)/1000000))
+}
+
+# Intialise the json subsystem
+# FIXME - if $1 == "k8s", then we skip some data writes, as we have not
+# worked out how to extract useful info yet..
+metrics_json_init() {
+
+
+	# Clear out any previous results
+	json_result_array=()
+
+	despaced_name="$(echo ${TEST_NAME} | sed 's/[ \/]/-/g')"
+	json_filename="${RESULT_DIR}/${despaced_name}.json"
+
+	local json="$(cat << EOF
+	"@timestamp" : $(timestamp_ms)
+EOF
+)"
+	metrics_json_add_fragment "$json"
+
+	local json="$(cat << EOF
+	"env" : {
+		"RuntimeVersion": "$RUNTIME_VERSION",
+		"RuntimeCommit": "$RUNTIME_COMMIT",
+		"RuntimeConfig": "$RUNTIME_CONFIG_PATH",
+		"Hypervisor": "$HYPERVISOR_PATH",
+		"HypervisorVersion": "$HYPERVISOR_VERSION",
+		"Proxy": "$PROXY_PATH",
+		"ProxyVersion": "$PROXY_VERSION",
+		"Shim": "$SHIM_PATH",
+		"ShimVersion": "$SHIM_VERSION",
+		"machinename": "$(uname -n)"
+	}
+EOF
+)"
+
+	metrics_json_add_fragment "$json"
+
+	local json="$(cat << EOF
+	"date" : {
+		"ns": $(timestamp_ns),
+		"Date": "$(date -u +"%Y-%m-%dT%T.%3N")"
+	}
+EOF
+)"
+	metrics_json_add_fragment "$json"
+
+	local json="$(cat << EOF
+	"test" : {
+		"runtime": "${RUNTIME}",
+		"testname": "${TEST_NAME}"
+	}
+EOF
+)"
+	metrics_json_add_fragment "$json"
+
+	# Now add a runtime specific environment section if we can
+	if [ "$1" == "k8s" ]; then
+		# FIXME - add k8s specific data dump here.
+		true
+	else
+		local iskata=$(is_a_kata_runtime "$RUNTIME")
+		if [ "$iskata" == "1" ]; then
+			local rpath="$(get_docker_kata_path $RUNTIME)"
+			local json="$(cat << EOF
+	"kata-env" :
+	$($rpath kata-env --json)
+EOF
+)"
+			metrics_json_add_fragment "$json"
+		else
+			if [ "$RUNTIME" == "runc" ]; then
+				local output=$(docker-runc -v)
+				local runcversion=$(grep version <<< "$output" | sed 's/runc version //')
+				local runccommit=$(grep commit <<< "$output" | sed 's/commit: //')
+				local json="$(cat << EOF
+	"runc-env" :
+	{
+		"Version": {
+			"Semver": "$runcversion",
+			"Commit": "$runccommit"
+		}
+	}
+EOF
+)"
+				metrics_json_add_fragment "$json"
+			else
+				warning "Unrecognised runtime ${RUNTIME} - no env extracted"
+			fi
+		fi
+	fi
+
+	metrics_json_end_of_system
+}
+
+# Save out the final JSON file
+metrics_json_save() {
+
+	if [ ! -d ${RESULT_DIR} ];then
+		mkdir -p ${RESULT_DIR}
+	fi
+
+	local maxelem=$(( ${#json_result_array[@]} - 1 ))
+	local json="$(cat << EOF
+{
+$(for index in $(seq 0 $maxelem); do
+	# After the standard system data, we then place all the test generated
+	# data into its own unique named subsection.
+	if (( index == system_index )); then
+		echo "\"${despaced_name}\" : {"
+	fi
+	if (( index != maxelem )); then
+		echo "${json_result_array[$index]},"
+	else
+		echo "${json_result_array[$index]}"
+	fi
+done)
+	}
+}
+EOF
+)"
+
+	echo "$json" > $json_filename
+
+	# If we have a JSON URL or host/socket pair set up, post the results there as well.
+	# Optionally compress into a single line.
+	if [[ $JSON_TX_ONELINE ]]; then
+		json="$(sed 's/[\n\t]//g' <<< ${json})"
+	fi
+
+	if [[ $JSON_HOST ]]; then
+		echo "socat'ing results to [$JSON_HOST:$JSON_SOCKET]"
+		socat -u - TCP:${JSON_HOST}:${JSON_SOCKET} <<< ${json}
+	fi
+
+	if [[ $JSON_URL ]]; then
+		echo "curl'ing results to [$JSON_URL]"
+		curl -XPOST -H"Content-Type: application/json" "$JSON_URL" -d "@-" <<< ${json}
+	fi
+}
+
+metrics_json_end_of_system() {
+	system_index=$(( ${#json_result_array[@]}))
+}
+
+# Add a top level (complete) JSON fragment to the data
+metrics_json_add_fragment() {
+	local data=$1
+
+	# Place on end of array
+	json_result_array[${#json_result_array[@]}]="$data"
+}
+
+# Prepare to collect up array elements
+metrics_json_start_array() {
+	json_array_array=()
+}
+
+# Add a (complete) element to the current array
+metrics_json_add_array_element() {
+	local data=$1
+
+	# Place on end of array
+	json_array_array[${#json_array_array[@]}]="$data"
+}
+
+# Add a fragment to the current array element
+metrics_json_add_array_fragment() {
+	local data=$1
+
+	# Place on end of array
+	json_array_fragments[${#json_array_fragments[@]}]="$data"
+}
+
+# Turn the currently registered array fragments into an array element
+metrics_json_close_array_element() {
+
+	local maxelem=$(( ${#json_array_fragments[@]} - 1 ))
+	local json="$(cat << EOF
+	{
+		$(for index in $(seq 0 $maxelem); do
+			if (( index != maxelem )); then
+				echo "${json_array_fragments[$index]},"
+			else
+				echo "${json_array_fragments[$index]}"
+			fi
+		done)
+	}
+EOF
+)"
+
+	# And save that to the top level
+	metrics_json_add_array_element "$json"
+
+	# Reset the array fragment array ready for a new one
+	json_array_fragments=()
+}
+
+# Close the current array
+metrics_json_end_array() {
+	local name=$1
+
+	local maxelem=$(( ${#json_array_array[@]} - 1 ))
+	local json="$(cat << EOF
+	"$name": [
+		$(for index in $(seq 0 $maxelem); do
+			if (( index != maxelem )); then
+				echo "${json_array_array[$index]},"
+			else
+				echo "${json_array_array[$index]}"
+			fi
+		done)
+	]
+EOF
+)"
+
+	# And save that to the top level
+	metrics_json_add_fragment "$json"
+}

metrics/lib/kata-common.bash

@@ -0,0 +1,218 @@
+#!/bin/bash
+#
+# Copyright (c) 2018-2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This file contains common functions that
+# are being used by our metrics and integration tests
+
+# Place where virtcontainers keeps its active pod info
+VC_POD_DIR="${VC_POD_DIR:-/var/lib/vc/sbs}"
+
+# Sandbox runtime directory
+RUN_SBS_DIR="${RUN_SBS_DIR:-/run/vc/sbs}"
+
+KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
+
+die() {
+	local msg="$*"
+	echo "ERROR: $msg" >&2
+	exit 1
+}
+
+warn() {
+	local msg="$*"
+	echo "WARNING: $msg"
+}
+
+info() {
+	local msg="$*"
+	echo "INFO: $msg"
+}
+
+# Check if the $1 argument is the name of a 'known'
+# Kata runtime. Of course, the end user can choose any name they
+# want in reality, but this function knows the names of the default
+# and recommended Kata docker runtime install names.
+is_a_kata_runtime(){
+	case "$1" in
+	"kata-runtime") ;&	# fallthrough
+	"kata-qemu") ;&		# fallthrough
+	"kata-fc")
+		echo "1"
+		return
+		;;
+	esac
+
+	echo "0"
+}
+
+
+# Try to find the real runtime path for the docker runtime passed in $1
+get_docker_kata_path(){
+	local jpaths=$(docker info --format "{{json .Runtimes}}" || true)
+	local rpath=$(jq .\"$1\".path <<< "$jpaths")
+	# Now we have to de-quote it..
+	rpath="${rpath%\"}"
+	rpath="${rpath#\"}"
+	echo "$rpath"
+}
+
+# Gets versions and paths of all the components
+# list in kata-env
+extract_kata_env(){
+	local toml
+	local rpath=$(get_docker_kata_path "$RUNTIME")
+	if [ -n "$rpath" ]; then
+		rpath=$(command -v "$rpath" || true)
+	fi
+
+	# If we can execute the path handed back to us
+	if [ -x "$rpath" ]; then
+		# and if the kata-env command does not error out. Bash hack so we can get $? even
+		# when the sub-command fails, but does not invoke the errexit in this parent shell.
+		local is_valid=$( $rpath kata-env >/dev/null 2>&1 && echo $? || echo $? )
+
+		if [ "$is_valid" == "0" ]; then
+			# then we can parse out the data we want
+			local toml="$($rpath kata-env)"
+
+			# The runtime path itself, for kata-runtime, will be contained in the `kata-env`
+			# section. For other runtimes we do not know where the runtime Docker is using lives.
+			RUNTIME_CONFIG_PATH=$(awk '/^  \[Runtime.Config\]$/ {foundit=1} /^    Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			RUNTIME_VERSION=$(awk '/^  \[Runtime.Version\]$/ {foundit=1} /^    Semver =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			RUNTIME_COMMIT=$(awk '/^  \[Runtime.Version\]$/ {foundit=1} /^    Commit =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			RUNTIME_PATH=$(awk '/^\[Runtime\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+
+			SHIM_PATH=$(awk '/^\[Shim\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			SHIM_VERSION=$(awk '/^\[Shim\]$/ {foundit=1} /^  Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+
+			PROXY_PATH=$(awk '/^\[Proxy\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			PROXY_VERSION=$(awk '/^\[Proxy\]$/ {foundit=1} /^  Version =/ { if (foundit==1) {print $5; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+
+			HYPERVISOR_PATH=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			HYPERVISOR_VERSION=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^  Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+
+			INITRD_PATH=$(awk '/^\[Initrd\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+
+			NETMON_PATH=$(awk '/^\[Netmon\]$/ {foundit=1} /^  Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g')
+			return 0
+		fi
+	fi
+
+	# We have not found a command with a 'kata-env' option we can run. Set up some
+	# default values.
+	# We could be more diligent here and search for each individual component,
+	# but if the runtime cannot tell us the exact details it is configured for then
+	# we would be guessing anyway - so, set some defaults that may be true and give
+	# strong hints that we 'made them up'.
+	info "Runtime environment not found - setting defaults"
+	RUNTIME_CONFIG_PATH="/usr/share/defaults/kata-containers/configuration.toml"
+	RUNTIME_VERSION="0.0.0"
+	RUNTIME_COMMIT="unknown"
+	# If docker is broken, disabled or not installed then we may not get a runtime
+	# path from it...
+	if [ -z "$RUNTIME_PATH" ]; then
+		RUNTIME_PATH="/usr/bin/kata-runtime"
+	else
+		RUNTIME_PATH="$rpath"
+	fi
+	SHIM_PATH="/usr/libexec/kata-containers/kata-shim"
+	SHIM_VERSION="0.0.0"
+	PROXY_PATH="/usr/libexec/kata-containers/kata-proxy"
+	PROXY_VERSION="0.0.0"
+	if [ "$KATA_HYPERVISOR" == firecracker ]; then
+		HYPERVISOR_PATH="/usr/bin/firecracker"
+	else
+		# We would use $(${cidir}/kata-arch.sh -d) here but we don't know
+		# that the callee has set up ${cidir} for us.
+		HYPERVISOR_PATH="/usr/bin/qemu-system-$(uname -m)"
+	fi
+	HYPERVISOR_VERSION="0.0.0"
+	INITRD_PATH=""
+	NETMON_PATH="/usr/libexec/kata-containers/kata-netmon"
+}
+
+# Checks that processes are not running
+check_processes() {
+	extract_kata_env
+
+	# Only check the kata-env if we have managed to find the kata executable...
+	if [ -x "$RUNTIME_PATH" ]; then
+		local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}')
+		local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}')
+	else
+		local vsock_configured="false"
+		local vsock_supported="false"
+	fi
+	if [ "$vsock_configured" == true ] && [ "$vsock_supported" == true ]; then
+		general_processes=( ${HYPERVISOR_PATH} ${SHIM_PATH} )
+	else
+		general_processes=( ${PROXY_PATH} ${HYPERVISOR_PATH} ${SHIM_PATH} )
+	fi
+	for i in "${general_processes[@]}"; do
+		if pgrep -f "$i"; then
+			die "Found unexpected ${i} present"
+		fi
+	done
+}
+
+# Checks that pods were not left in a directory
+check_pods_in_dir() {
+    local DIR=$1
+    if [ -d ${DIR} ]; then
+		# Verify that pods were not left
+		pods_number=$(ls ${DIR} | wc -l)
+		if [ ${pods_number} -ne 0 ]; then
+            ls ${DIR}
+			die "${pods_number} pods left and found at ${DIR}"
+		fi
+	else
+		echo "Not ${DIR} directory found"
+	fi
+}
+
+# Checks that pods were not left
+check_pods() {
+	check_pods_in_dir ${VC_POD_DIR}
+}
+
+# Check that runtimes are not running, they should be transient
+check_runtimes() {
+	runtime_number=$(ps --no-header -C ${RUNTIME} | wc -l)
+	if [ ${runtime_number} -ne 0 ]; then
+		die "Unexpected runtime ${RUNTIME} running"
+	fi
+}
+
+# Clean environment, this function will try to remove all
+# stopped/running containers.
+clean_env()
+{
+	# If the timeout has not been set, default it to 30s
+	# Docker has a built in 10s default timeout, so make ours
+	# longer than that.
+	KATA_DOCKER_TIMEOUT=${KATA_DOCKER_TIMEOUT:-30}
+	containers_running=$(timeout ${KATA_DOCKER_TIMEOUT} docker ps -q)
+
+	if [ ! -z "$containers_running" ]; then
+		# First stop all containers that are running
+		# Use kill, as the containers are generally benign, and most
+		# of the time our 'stop' request ends up doing a `kill` anyway
+		sudo timeout ${KATA_DOCKER_TIMEOUT} docker kill $containers_running
+
+		# Remove all containers
+		sudo timeout ${KATA_DOCKER_TIMEOUT} docker rm -f $(docker ps -qa)
+	fi
+}
+
+get_pod_config_dir() {
+	if kubectl get runtimeclass 2> /dev/null | grep -q "kata"; then
+		pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads"
+		info "k8s configured to use runtimeclass"
+	else
+		pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads"
+		info "k8s configured to use trusted and untrusted annotations"
+	fi
+}

metrics/scaling/bb.yaml.in

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+# We use a deployment rather than a pod directly specifically so we can use it to
+# generate more replicas by re-deploying it.
+# The only downside might be, if containers go wrong or die etc., then the deployment
+# is going to try and re-deploy them, and we may not notice or get stuck waiting for
+# them, so we want to be careful in the test code to time out in such cases.
+kind: Deployment
+metadata:
+  labels:
+    run: busybox
+  name: @DEPLOYMENT@
+spec:
+  replicas: @REPLICAS@
+  selector:
+    matchLabels:
+      run: busybox
+  template:
+    metadata:
+      labels:
+        run: busybox
+        @LABEL@: @LABELVALUE@
+    spec:
+      runtimeClassName: @RUNTIMECLASS@
+      containers:
+      - name: bb
+        image: busybox
+        command:
+            - "tail"
+            - "-f"
+            - "/dev/null"
+        stdin: true
+        tty: true
+      restartPolicy: Always

metrics/scaling/k8s_scale.sh

@@ -0,0 +1,267 @@
+#!/bin/bash
+# Copyright (c) 2019 Intel Corporation
+# 
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+# Pull in some common, useful, items
+SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_PATH}/../lib/common.bash"
+
+input_yaml="bb.yaml.in"
+generated_yaml="generated.yaml"
+deployment="busybox"
+
+stats_pod="stats"
+
+export RUNTIME="kata-qemu"
+NUM_PODS=${NUM_PODS:-20}
+STEP=${STEP:-1}
+
+LABEL=${LABEL:-magiclabel}
+LABELVALUE=${LABELVALUE:-gandalf}
+
+# sleep and timeout times for k8s actions, in seconds
+wait_time=${wait_time:-30}
+delete_wait_time=${delete_wait_time:-600}
+settle_time=${settle_time:-5}
+use_kata_runtime=${use_kata_runtime:-no}
+
+# Set some default metrics env vars
+TEST_ARGS="runtime=${RUNTIME}"
+TEST_NAME="k8s scaling"
+
+get_num_nodes() {
+	n=$(kubectl get nodes --no-headers=true | wc -l)
+	echo "$n"
+}
+
+# $1 is the launch time in seconds this pod/container took to start up.
+# $2 is the number of pod/containers under test
+grab_stats() {
+	local launch_time=$1
+	local n_pods=$2
+	info "And grab some stats"
+	# Tell mpstat to measure over a short period, not only so we get slightly de-noised data, but also
+	# if you don't tell it the period, you will get the avg since boot, which is not what we want.
+	local cpu_idle=$(kubectl exec -ti ${stats_pod} -- sh -c "mpstat -u 3 1 | tail -1 | awk '{print \$11}'" | sed 's/\r//')
+	local mem_free=$(kubectl exec -ti ${stats_pod} -- sh -c "free | tail -2 | head -1 | awk '{print \$4}'" | sed 's/\r//')
+
+	info "idle [$cpu_idle] free [$mem_free] launch [$launch_time]"
+
+	# Annoyingly, it seems sometimes once in a while we don't get an answer!
+	# We should really retry, but for now, make the json valid at least
+	cpu_idle=${cpu_idle:-0}
+	mem_free=${mem_free:-0}
+
+	local json="$(cat << EOF
+	{
+		"n_pods": {
+			"Result": ${n_pods},
+			"Units" : "int"
+		},
+		"cpu_idle": {
+			"Result": ${cpu_idle},
+			"Units" : "%"
+		},
+		"launch_time": {
+			"Result": $launch_time,
+			"Units" : "s"
+		},
+		"mem_free": {
+			"Result": ${mem_free},
+			"Units" : "kb"
+		}
+	}
+EOF
+)"
+	metrics_json_add_array_element "$json"
+}
+
+init() {
+	info "Initialising"
+	info "Checking k8s accessible"
+	local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
+	if [ "$worked" != 0 ]; then
+		die "kubectl failed to get nodes"
+	fi
+
+	info $(get_num_nodes) "k8s nodes found"
+	# We could check we have just the one node here - right now this is a single node
+	# test!! - because, our stats gathering is rudimentry, as k8s does not provide
+	# a nice way to do it (unless you want to parse 'descibe nodes')
+	# Have a read of https://github.com/kubernetes/kubernetes/issues/25353
+
+	# FIXME - check the node(s) can run enough pods - check 'max-pods' in the
+	# kubelet config - from 'kubectl describe node -o json' ?
+
+	# Launch our stats gathering pod
+	kubectl apply -f ${stats_pod}.yaml
+	kubectl wait --for=condition=Ready pod "${stats_pod}"
+
+	# FIXME - we should probably 'warm up' the cluster with the container image(s) we will
+	# use for testing, otherwise the download time will likely be included in the first pod
+	# boot time.
+
+	# And now we can set up our results storage then...
+	metrics_json_init "k8s"
+	save_config
+}
+
+save_config(){
+	metrics_json_start_array
+
+	local json="$(cat << EOF
+	{
+		"testname": "${TEST_NAME}",
+		"NUM_PODS": "${NUM_PODS}",
+		"STEP": "${STEP}",
+		"wait_time": "${wait_time}",
+		"delete_wait_time": "${delete_wait_time}",
+		"settle_time": "${settle_time}"
+	}
+EOF
+)"
+	metrics_json_add_array_element "$json"
+	metrics_json_end_array "Config"
+}
+
+run() {
+	info "Running test"
+
+	trap cleanup EXIT QUIT KILL
+
+	metrics_json_start_array
+	for reqs in $(seq ${STEP} ${STEP} ${NUM_PODS}); do
+		info "Testing replicas ${reqs} of ${NUM_PODS}"
+		# Generate the next yaml file
+
+	if [ "$use_kata_runtime" != "no" ]; then
+		sed -e "s|@REPLICAS@|${reqs}|g" \
+		    -e "s|@RUNTIMECLASS@|${RUNTIME}|g" \
+		    -e "s|@DEPLOYMENT@|${deployment}|g" \
+		    -e "s|@LABEL@|${LABEL}|g" \
+		    -e "s|@LABELVALUE@|${LABELVALUE}|g" \
+		    < ${input_yaml} > ${generated_yaml}
+    else
+        sed -e "s|@REPLICAS@|${reqs}|g" \
+		    -e "/@RUNTIMECLASS@/d" \
+		    -e "s|@DEPLOYMENT@|${deployment}|g" \
+		    -e "s|@LABEL@|${LABEL}|g" \
+		    -e "s|@LABELVALUE@|${LABELVALUE}|g" \
+		    < ${input_yaml} > ${generated_yaml}
+    fi
+
+		info "Applying changes"
+		# FIXME - time taken in 'seconds' - we can do better than that, if we
+		# use the date nanosecond stamp, and steal the magic from the density test script.
+        # switch to 'date +%N'
+		local start_time=$(date +%s)
+		kubectl apply -f ${generated_yaml}
+
+		#cmd="kubectl get pods | grep busybox | grep Completed"
+		kubectl rollout status --timeout=${wait_time}s deployment/${deployment}
+		local end_time=$(date +%s)
+		local total_seconds=$(( end_time - start_time ))
+		info "Took $total_seconds ($end_time - $start_time)"
+		sleep ${settle_time}
+		grab_stats $total_seconds $reqs
+	done
+}
+
+cleanup() {
+	info "Cleaning up"
+
+	# First try to save any results we got
+	metrics_json_end_array "BootResults"
+
+	kubectl delete pod --wait=true --timeout=${delete_wait_time}s "${stats_pod}" || true
+	local start_time=$(date +%s)
+	kubectl delete deployment --wait=true --timeout=${delete_wait_time}s "${deployment}" || true
+	for x in $(seq 1 ${delete_wait_time}); do
+		local npods=$(kubectl get pods -l=${LABEL}=${LABELVALUE} -o=name | wc -l)
+		if [ $npods -eq 0 ]; then
+			echo "All pods have terminated at cycle $x"
+			local alldied=true
+			break;
+		fi
+		sleep 1
+	done
+	local end_time=$(date +%s)
+	local total_seconds=$(( end_time - start_time ))
+	if [ -z "$alldied" ]; then
+		echo "ERROR: Not all pods died!"
+	fi
+	info "Delete Took $total_seconds ($end_time - $start_time)"
+
+	local json="$(cat << EOF
+	"Delete": {
+		"Result": ${total_seconds},
+		"Units" : "s"
+	}
+EOF
+)"
+
+	metrics_json_add_fragment "$json"
+	metrics_json_save
+}
+
+show_vars()
+{
+	echo -e "\nEnvironment variables:"
+	echo -e "\tName (default)"
+	echo -e "\t\tDescription"
+	echo -e "\tTEST_NAME (${TEST_NAME})"
+	echo -e "\t\tCan be set to over-ride the default JSON results filename"
+	echo -e "\tNUM_PODS (${NUM_PODS})"
+	echo -e "\t\tNumber of pods to launch"
+	echo -e "\tSTEP (${STEP})"
+	echo -e "\t\tNumber of pods to launch per cycle"
+	echo -e "\twait_time (${wait_time})"
+	echo -e "\t\tSeconds to wait for pods to become ready"
+	echo -e "\tdelete_wait_time (${delete_wait_time})"
+	echo -e "\t\tSeconds to wait for all pods to be deleted"
+	echo -e "\tsettle_time (${settle_time})"
+	echo -e "\t\tSeconds to wait after pods ready before taking measurements"
+    echo -e "\tuse_kata_runtime (${use_kata_runtime})"
+    echo -e "\t\tspecify yes or no to use kata runtime"
+}
+
+help()
+{
+	usage=$(cat << EOF
+Usage: $0 [-h] [options]
+   Description:
+	Launch a series of workloads and take memory metric measurements after
+	each launch.
+   Options:
+        -h,    Help page.
+EOF
+)
+	echo "$usage"
+	show_vars
+}
+
+main() {
+
+	local OPTIND
+	while getopts "h" opt;do
+		case ${opt} in
+		h)
+		    help
+		    exit 0;
+		    ;;
+		esac
+	done
+	shift $((OPTIND-1))
+
+	init
+	run
+	# cleanup will happen at exit due to the shell 'trap' we registered
+	# cleanup
+}
+
+main "$@"
+

metrics/scaling/stats.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: stats
+spec:
+  containers:
+    - name: stats
+      image: busybox
+      securityContext:
+        # Run a priv container so we really do measure what is happening on the
+        # host (node) system
+        privileged: true
+      command:
+          - "tail"
+          - "-f"
+          - "/dev/null"
+      stdin: true
+      tty: true