diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 0000000..0050280 --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,18 @@ +# Metric testing for scaling on Kubernetes. + +The usage is + +```bash +cd metrics/scaling +./k8s-scale.sh +``` + +The default container runtime is not Kata containers, but Kata can be specified +setting the environment variable: + +```bash +use_kata_runtime=yes +``` + +These tests currently only support single node deployments, but PRs to add +multi-node support will follow shortly. diff --git a/metrics/lib/common.bash b/metrics/lib/common.bash new file mode 100755 index 0000000..5fb8768 --- /dev/null +++ b/metrics/lib/common.bash @@ -0,0 +1,339 @@ +#!/bin/bash +# +# Copyright (c) 2017,2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +THIS_FILE=$(readlink -f ${BASH_SOURCE[0]}) +LIB_DIR=${THIS_FILE%/*} +RESULT_DIR="${LIB_DIR}/../results" + +source ${LIB_DIR}/kata-common.bash +source ${LIB_DIR}/json.bash +source /etc/os-release || source /usr/lib/os-release +KATA_KSM_THROTTLER="${KATA_KSM_THROTTLER:-no}" + +# Set variables to reasonable defaults if unset or empty +DOCKER_EXE="${DOCKER_EXE:-docker}" +RUNTIME="${RUNTIME:-kata-runtime}" + +KSM_BASE="/sys/kernel/mm/ksm" +KSM_ENABLE_FILE="${KSM_BASE}/run" +KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan" +KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs" + +# The settings we use for an 'aggresive' KSM setup +# Scan 1000 pages every 50ms - 20,000 pages/s +KSM_AGGRESIVE_PAGES=1000 +KSM_AGGRESIVE_SLEEP=50 + +# If we fail for any reason, exit through here and we should log that to the correct +# place and return the correct code to halt the run +die(){ + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +# Sometimes we just want to warn about something - let's have a standard +# method for that, so maybe we can make a standard form that can be searched +# for in the logs/tooling +warning(){ + msg="$*" + echo "WARNING: $msg" >&2 +} + +info() { + echo -e "INFO: $*" +} + +# This function checks existence of commands. +# They can be received standalone or as an array, e.g. +# +# cmds=(“cmd1” “cmd2”) +# check_cmds "${cmds[@]}" +check_cmds() +{ + local cmd req_cmds=( "$@" ) + for cmd in "${req_cmds[@]}"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + die "command $cmd not available" + fi + echo "command: $cmd: yes" + done +} + +# This function performs a docker pull on the image names +# passed in (notionally as 'about to be used'), to ensure +# - that we have the most upto date images +# - that any pull/refresh time (for a first pull) does not +# happen during the test itself. +# +# The image list can be received standalone or as an array, e.g. +# +# images=(“img1” “img2”) +# check_imgs "${images[@]}" +check_images() +{ + local img req_images=( "$@" ) + for img in "${req_images[@]}"; do + echo "docker pull'ing: $img" + if ! docker pull "$img"; then + die "Failed to docker pull image $img" + fi + echo "docker pull'd: $img" + done +} + +# This function performs a docker build on the image names +# passed in, to ensure that we have the latest changes from +# the dockerfiles +build_dockerfile_image() +{ + local image="$1" + local dockerfile_path="$2" + local dockerfile_dir=${2%/*} + + echo "docker building $image" + if ! docker build --label "$image" --tag "${image}" -f "$dockerfile_path" "$dockerfile_dir"; then + die "Failed to docker build image $image" + fi +} + +# This function verifies that the dockerfile version is +# equal to the test version in order to build the image or +# just run the test +check_dockerfiles_images() +{ + local image="$1" + local dockerfile_path="$2" + + if [ -z "$image" ] || [ -z "$dockerfile_path" ]; then + die "Missing image or dockerfile path variable" + fi + + # Verify that dockerfile version is equal to test version + check_image=$(docker images "$image" -q) + if [ -n "$check_image" ]; then + # Check image label + check_image_version=$(docker image inspect $image | grep -w DOCKERFILE_VERSION | head -1 | cut -d '"' -f4) + if [ -n "$check_image_version" ]; then + echo "$image is not updated" + build_dockerfile_image "$image" "$dockerfile_path" + else + # Check dockerfile label + dockerfile_version=$(grep DOCKERFILE_VERSION $dockerfile_path | cut -d '"' -f2) + if [ "$dockerfile_version" != "$check_image_version" ]; then + echo "$dockerfile_version is not equal to $check_image_version" + build_dockerfile_image "$image" "$dockerfile_path" + fi + fi + else + build_dockerfile_image "$image" "$dockerfile_path" + fi +} + +# A one time (per uber test cycle) init that tries to get the +# system to a 'known state' as much as possible +metrics_onetime_init() +{ + # The onetime init must be called once, and only once + if [ ! -z "$onetime_init_done" ]; then + die "onetime_init() called more than once" + fi + + # Restart services + sudo systemctl restart docker + + # We want this to be seen in sub shells as well... + # otherwise init_env() cannot check us + export onetime_init_done=1 +} + +# Print a banner to the logs noting clearly which test +# we are about to run +test_banner() +{ + echo -e "\n===== starting test [$1] =====" +} + +# Initialization/verification environment. This function makes +# minimal steps for metrics/tests execution. +init_env() +{ + test_banner "${TEST_NAME}" + + cmd=("docker") + + # check dependencies + check_cmds "${cmd[@]}" + + # Remove all stopped containers + clean_env + + # This clean up is more aggressive, this is in order to + # decrease the factors that could affect the metrics results. + kill_processes_before_start +} + +# This function checks if there are containers or +# shim/proxy/hypervisor processes up, if found, they are +# killed to start test with clean environment. +kill_processes_before_start() { + DOCKER_PROCS=$(${DOCKER_EXE} ps -q) + [[ -n "${DOCKER_PROCS}" ]] && clean_env + check_processes +} + +# Generate a random name - generally used when creating containers, but can +# be used for any other appropriate purpose +random_name() { + mktemp -u kata-XXXXXX +} + +# Dump diagnostics about our current system state. +# Very useful for diagnosing if we have failed a sanity check +show_system_state() { + echo "Showing system state:" + echo " --Docker ps--" + ${DOCKER_EXE} ps -a + echo " --${RUNTIME} list--" + local RPATH=$(command -v ${RUNTIME}) + sudo ${RPATH} list + + local processes="kata-proxy kata-shim kata-runtime qemu" + + for p in ${processes}; do + echo " --pgrep ${p}--" + pgrep -a ${p} + done + + # Verify kata-ksm-throttler + if [ "$KATA_KSM_THROTTLER" == "yes" ]; then + process="kata-ksm-throttler" + process_path=$(whereis ${process} | tr -d '[:space:]'| cut -d ':' -f2) + echo " --pgrep ${process}--" + pgrep -f ${process_path} > /dev/null + fi +} + +common_init(){ + + # If we are running a kata runtime, go extract its environment + # for later use. + local iskata=$(is_a_kata_runtime "$RUNTIME") + + if [ "$iskata" == "1" ]; then + extract_kata_env + else + # We know we have nothing to do for runc + if [ "$RUNTIME" != "runc" ]; then + warning "Unrecognised runtime ${RUNTIME}" + fi + fi +} + + +# Save the current KSM settings so we can restore them later +save_ksm_settings(){ + echo "saving KSM settings" + ksm_stored_run=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_pages=$(cat ${KSM_ENABLE_FILE}) + ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE}) +} + +set_ksm_aggressive(){ + echo "setting KSM to aggressive mode" + # Flip the run off/on to ensure a restart/rescan + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}" +} + +restore_ksm_settings(){ + echo "restoring KSM settings" + # First turn off the run to ensure if we are then re-enabling + # that any changes take effect + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" + sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}" + sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}" + sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}" +} + +disable_ksm(){ + echo "disabling KSM" + sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}" +} + +# See if KSM is enabled. +# If so, amend the test name to reflect that +check_for_ksm(){ + if [ ! -f ${KSM_ENABLE_FILE} ]; then + return + fi + + ksm_on=$(< ${KSM_ENABLE_FILE}) + + if [ $ksm_on == "1" ]; then + TEST_NAME="${TEST_NAME} ksm" + fi +} + +# Wait for KSM to settle down, or timeout waiting +# The basic algorithm is to look at the pages_shared value +# at the end of every 'full scan', and if the value +# has changed very little, then we are done (because we presume +# a full scan has managed to do few new merges) +# +# arg1 - timeout in seconds +wait_ksm_settle(){ + [[ "$RUNTIME" == "runc" ]] || [[ "$RUNTIME" == "kata-fc" ]] && return + local t pcnt + local oldscan=-1 newscan + local oldpages=-1 newpages + + oldscan=$(cat /sys/kernel/mm/ksm/full_scans) + + # Go around the loop until either we see a small % change + # between two full_scans, or we timeout + for ((t=0; t<$1; t++)); do + + newscan=$(cat /sys/kernel/mm/ksm/full_scans) + newpages=$(cat /sys/kernel/mm/ksm/pages_shared) + [[ "$newpages" -eq 0 ]] && echo "No need to wait for KSM to settle" && return + + if (( newscan != oldscan )); then + echo -e "\nnew full_scan ($oldscan to $newscan)" + + # Do we have a previous scan to compare with + echo "check pages $oldpages to $newpages" + + if (( oldpages != -1 )); then + # avoid divide by zero problems + if (( $oldpages > 0 )); then + pcnt=$(( 100 - ((newpages * 100) / oldpages) )) + # abs() + pcnt=$(( $pcnt * -1 )) + + echo "$oldpages to $newpages is ${pcnt}%" + + if (( $pcnt <= 5 )); then + echo "KSM stabilised at ${t}s" + return + fi + else + echo "$oldpages KSM pages... waiting" + fi + fi + oldscan=$newscan + oldpages=$newpages + else + echo -n "." + fi + sleep 1 + done + echo "Timed out after ${1}s waiting for KSM to settle" +} + +common_init diff --git a/metrics/lib/json.bash b/metrics/lib/json.bash new file mode 100644 index 0000000..67ca04a --- /dev/null +++ b/metrics/lib/json.bash @@ -0,0 +1,248 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Helper routines for generating JSON formatted results. + +declare -a json_result_array +declare -a json_array_array + +# Generate a timestamp in nanoseconds since 1st Jan 1970 +timestamp_ns() { + local t + local s + local n + local ns + + t="$(date +%-s:%-N)" + s=$(echo $t | awk -F ':' '{print $1}') + n=$(echo $t | awk -F ':' '{print $2}') + ns=$(( (s * 1000000000) + n )) + + echo $ns +} + +# Generate a timestamp in milliseconds since 1st Jan 1970 +timestamp_ms() { + echo $(($(date +%s%N)/1000000)) +} + +# Intialise the json subsystem +# FIXME - if $1 == "k8s", then we skip some data writes, as we have not +# worked out how to extract useful info yet.. +metrics_json_init() { + + + # Clear out any previous results + json_result_array=() + + despaced_name="$(echo ${TEST_NAME} | sed 's/[ \/]/-/g')" + json_filename="${RESULT_DIR}/${despaced_name}.json" + + local json="$(cat << EOF + "@timestamp" : $(timestamp_ms) +EOF +)" + metrics_json_add_fragment "$json" + + local json="$(cat << EOF + "env" : { + "RuntimeVersion": "$RUNTIME_VERSION", + "RuntimeCommit": "$RUNTIME_COMMIT", + "RuntimeConfig": "$RUNTIME_CONFIG_PATH", + "Hypervisor": "$HYPERVISOR_PATH", + "HypervisorVersion": "$HYPERVISOR_VERSION", + "Proxy": "$PROXY_PATH", + "ProxyVersion": "$PROXY_VERSION", + "Shim": "$SHIM_PATH", + "ShimVersion": "$SHIM_VERSION", + "machinename": "$(uname -n)" + } +EOF +)" + + metrics_json_add_fragment "$json" + + local json="$(cat << EOF + "date" : { + "ns": $(timestamp_ns), + "Date": "$(date -u +"%Y-%m-%dT%T.%3N")" + } +EOF +)" + metrics_json_add_fragment "$json" + + local json="$(cat << EOF + "test" : { + "runtime": "${RUNTIME}", + "testname": "${TEST_NAME}" + } +EOF +)" + metrics_json_add_fragment "$json" + + # Now add a runtime specific environment section if we can + if [ "$1" == "k8s" ]; then + # FIXME - add k8s specific data dump here. + true + else + local iskata=$(is_a_kata_runtime "$RUNTIME") + if [ "$iskata" == "1" ]; then + local rpath="$(get_docker_kata_path $RUNTIME)" + local json="$(cat << EOF + "kata-env" : + $($rpath kata-env --json) +EOF +)" + metrics_json_add_fragment "$json" + else + if [ "$RUNTIME" == "runc" ]; then + local output=$(docker-runc -v) + local runcversion=$(grep version <<< "$output" | sed 's/runc version //') + local runccommit=$(grep commit <<< "$output" | sed 's/commit: //') + local json="$(cat << EOF + "runc-env" : + { + "Version": { + "Semver": "$runcversion", + "Commit": "$runccommit" + } + } +EOF +)" + metrics_json_add_fragment "$json" + else + warning "Unrecognised runtime ${RUNTIME} - no env extracted" + fi + fi + fi + + metrics_json_end_of_system +} + +# Save out the final JSON file +metrics_json_save() { + + if [ ! -d ${RESULT_DIR} ];then + mkdir -p ${RESULT_DIR} + fi + + local maxelem=$(( ${#json_result_array[@]} - 1 )) + local json="$(cat << EOF +{ +$(for index in $(seq 0 $maxelem); do + # After the standard system data, we then place all the test generated + # data into its own unique named subsection. + if (( index == system_index )); then + echo "\"${despaced_name}\" : {" + fi + if (( index != maxelem )); then + echo "${json_result_array[$index]}," + else + echo "${json_result_array[$index]}" + fi +done) + } +} +EOF +)" + + echo "$json" > $json_filename + + # If we have a JSON URL or host/socket pair set up, post the results there as well. + # Optionally compress into a single line. + if [[ $JSON_TX_ONELINE ]]; then + json="$(sed 's/[\n\t]//g' <<< ${json})" + fi + + if [[ $JSON_HOST ]]; then + echo "socat'ing results to [$JSON_HOST:$JSON_SOCKET]" + socat -u - TCP:${JSON_HOST}:${JSON_SOCKET} <<< ${json} + fi + + if [[ $JSON_URL ]]; then + echo "curl'ing results to [$JSON_URL]" + curl -XPOST -H"Content-Type: application/json" "$JSON_URL" -d "@-" <<< ${json} + fi +} + +metrics_json_end_of_system() { + system_index=$(( ${#json_result_array[@]})) +} + +# Add a top level (complete) JSON fragment to the data +metrics_json_add_fragment() { + local data=$1 + + # Place on end of array + json_result_array[${#json_result_array[@]}]="$data" +} + +# Prepare to collect up array elements +metrics_json_start_array() { + json_array_array=() +} + +# Add a (complete) element to the current array +metrics_json_add_array_element() { + local data=$1 + + # Place on end of array + json_array_array[${#json_array_array[@]}]="$data" +} + +# Add a fragment to the current array element +metrics_json_add_array_fragment() { + local data=$1 + + # Place on end of array + json_array_fragments[${#json_array_fragments[@]}]="$data" +} + +# Turn the currently registered array fragments into an array element +metrics_json_close_array_element() { + + local maxelem=$(( ${#json_array_fragments[@]} - 1 )) + local json="$(cat << EOF + { + $(for index in $(seq 0 $maxelem); do + if (( index != maxelem )); then + echo "${json_array_fragments[$index]}," + else + echo "${json_array_fragments[$index]}" + fi + done) + } +EOF +)" + + # And save that to the top level + metrics_json_add_array_element "$json" + + # Reset the array fragment array ready for a new one + json_array_fragments=() +} + +# Close the current array +metrics_json_end_array() { + local name=$1 + + local maxelem=$(( ${#json_array_array[@]} - 1 )) + local json="$(cat << EOF + "$name": [ + $(for index in $(seq 0 $maxelem); do + if (( index != maxelem )); then + echo "${json_array_array[$index]}," + else + echo "${json_array_array[$index]}" + fi + done) + ] +EOF +)" + + # And save that to the top level + metrics_json_add_fragment "$json" +} diff --git a/metrics/lib/kata-common.bash b/metrics/lib/kata-common.bash new file mode 100755 index 0000000..6b500b1 --- /dev/null +++ b/metrics/lib/kata-common.bash @@ -0,0 +1,218 @@ +#!/bin/bash +# +# Copyright (c) 2018-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# This file contains common functions that +# are being used by our metrics and integration tests + +# Place where virtcontainers keeps its active pod info +VC_POD_DIR="${VC_POD_DIR:-/var/lib/vc/sbs}" + +# Sandbox runtime directory +RUN_SBS_DIR="${RUN_SBS_DIR:-/run/vc/sbs}" + +KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}" + +die() { + local msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +warn() { + local msg="$*" + echo "WARNING: $msg" +} + +info() { + local msg="$*" + echo "INFO: $msg" +} + +# Check if the $1 argument is the name of a 'known' +# Kata runtime. Of course, the end user can choose any name they +# want in reality, but this function knows the names of the default +# and recommended Kata docker runtime install names. +is_a_kata_runtime(){ + case "$1" in + "kata-runtime") ;& # fallthrough + "kata-qemu") ;& # fallthrough + "kata-fc") + echo "1" + return + ;; + esac + + echo "0" +} + + +# Try to find the real runtime path for the docker runtime passed in $1 +get_docker_kata_path(){ + local jpaths=$(docker info --format "{{json .Runtimes}}" || true) + local rpath=$(jq .\"$1\".path <<< "$jpaths") + # Now we have to de-quote it.. + rpath="${rpath%\"}" + rpath="${rpath#\"}" + echo "$rpath" +} + +# Gets versions and paths of all the components +# list in kata-env +extract_kata_env(){ + local toml + local rpath=$(get_docker_kata_path "$RUNTIME") + if [ -n "$rpath" ]; then + rpath=$(command -v "$rpath" || true) + fi + + # If we can execute the path handed back to us + if [ -x "$rpath" ]; then + # and if the kata-env command does not error out. Bash hack so we can get $? even + # when the sub-command fails, but does not invoke the errexit in this parent shell. + local is_valid=$( $rpath kata-env >/dev/null 2>&1 && echo $? || echo $? ) + + if [ "$is_valid" == "0" ]; then + # then we can parse out the data we want + local toml="$($rpath kata-env)" + + # The runtime path itself, for kata-runtime, will be contained in the `kata-env` + # section. For other runtimes we do not know where the runtime Docker is using lives. + RUNTIME_CONFIG_PATH=$(awk '/^ \[Runtime.Config\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + RUNTIME_VERSION=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Semver =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + RUNTIME_COMMIT=$(awk '/^ \[Runtime.Version\]$/ {foundit=1} /^ Commit =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + RUNTIME_PATH=$(awk '/^\[Runtime\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + + SHIM_PATH=$(awk '/^\[Shim\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + SHIM_VERSION=$(awk '/^\[Shim\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g') + + PROXY_PATH=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + PROXY_VERSION=$(awk '/^\[Proxy\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {print $5; foundit=0} } ' <<< "$toml" | sed 's/"//g') + + HYPERVISOR_PATH=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + HYPERVISOR_VERSION=$(awk '/^\[Hypervisor\]$/ {foundit=1} /^ Version =/ { if (foundit==1) {$1=$2=""; print $0; foundit=0} } ' <<< "$toml" | sed 's/"//g') + + INITRD_PATH=$(awk '/^\[Initrd\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + + NETMON_PATH=$(awk '/^\[Netmon\]$/ {foundit=1} /^ Path =/ { if (foundit==1) {print $3; foundit=0} } ' <<< "$toml" | sed 's/"//g') + return 0 + fi + fi + + # We have not found a command with a 'kata-env' option we can run. Set up some + # default values. + # We could be more diligent here and search for each individual component, + # but if the runtime cannot tell us the exact details it is configured for then + # we would be guessing anyway - so, set some defaults that may be true and give + # strong hints that we 'made them up'. + info "Runtime environment not found - setting defaults" + RUNTIME_CONFIG_PATH="/usr/share/defaults/kata-containers/configuration.toml" + RUNTIME_VERSION="0.0.0" + RUNTIME_COMMIT="unknown" + # If docker is broken, disabled or not installed then we may not get a runtime + # path from it... + if [ -z "$RUNTIME_PATH" ]; then + RUNTIME_PATH="/usr/bin/kata-runtime" + else + RUNTIME_PATH="$rpath" + fi + SHIM_PATH="/usr/libexec/kata-containers/kata-shim" + SHIM_VERSION="0.0.0" + PROXY_PATH="/usr/libexec/kata-containers/kata-proxy" + PROXY_VERSION="0.0.0" + if [ "$KATA_HYPERVISOR" == firecracker ]; then + HYPERVISOR_PATH="/usr/bin/firecracker" + else + # We would use $(${cidir}/kata-arch.sh -d) here but we don't know + # that the callee has set up ${cidir} for us. + HYPERVISOR_PATH="/usr/bin/qemu-system-$(uname -m)" + fi + HYPERVISOR_VERSION="0.0.0" + INITRD_PATH="" + NETMON_PATH="/usr/libexec/kata-containers/kata-netmon" +} + +# Checks that processes are not running +check_processes() { + extract_kata_env + + # Only check the kata-env if we have managed to find the kata executable... + if [ -x "$RUNTIME_PATH" ]; then + local vsock_configured=$($RUNTIME_PATH kata-env | awk '/UseVSock/ {print $3}') + local vsock_supported=$($RUNTIME_PATH kata-env | awk '/SupportVSock/ {print $3}') + else + local vsock_configured="false" + local vsock_supported="false" + fi + if [ "$vsock_configured" == true ] && [ "$vsock_supported" == true ]; then + general_processes=( ${HYPERVISOR_PATH} ${SHIM_PATH} ) + else + general_processes=( ${PROXY_PATH} ${HYPERVISOR_PATH} ${SHIM_PATH} ) + fi + for i in "${general_processes[@]}"; do + if pgrep -f "$i"; then + die "Found unexpected ${i} present" + fi + done +} + +# Checks that pods were not left in a directory +check_pods_in_dir() { + local DIR=$1 + if [ -d ${DIR} ]; then + # Verify that pods were not left + pods_number=$(ls ${DIR} | wc -l) + if [ ${pods_number} -ne 0 ]; then + ls ${DIR} + die "${pods_number} pods left and found at ${DIR}" + fi + else + echo "Not ${DIR} directory found" + fi +} + +# Checks that pods were not left +check_pods() { + check_pods_in_dir ${VC_POD_DIR} +} + +# Check that runtimes are not running, they should be transient +check_runtimes() { + runtime_number=$(ps --no-header -C ${RUNTIME} | wc -l) + if [ ${runtime_number} -ne 0 ]; then + die "Unexpected runtime ${RUNTIME} running" + fi +} + +# Clean environment, this function will try to remove all +# stopped/running containers. +clean_env() +{ + # If the timeout has not been set, default it to 30s + # Docker has a built in 10s default timeout, so make ours + # longer than that. + KATA_DOCKER_TIMEOUT=${KATA_DOCKER_TIMEOUT:-30} + containers_running=$(timeout ${KATA_DOCKER_TIMEOUT} docker ps -q) + + if [ ! -z "$containers_running" ]; then + # First stop all containers that are running + # Use kill, as the containers are generally benign, and most + # of the time our 'stop' request ends up doing a `kill` anyway + sudo timeout ${KATA_DOCKER_TIMEOUT} docker kill $containers_running + + # Remove all containers + sudo timeout ${KATA_DOCKER_TIMEOUT} docker rm -f $(docker ps -qa) + fi +} + +get_pod_config_dir() { + if kubectl get runtimeclass 2> /dev/null | grep -q "kata"; then + pod_config_dir="${BATS_TEST_DIRNAME}/runtimeclass_workloads" + info "k8s configured to use runtimeclass" + else + pod_config_dir="${BATS_TEST_DIRNAME}/untrusted_workloads" + info "k8s configured to use trusted and untrusted annotations" + fi +} diff --git a/metrics/scaling/bb.yaml.in b/metrics/scaling/bb.yaml.in new file mode 100644 index 0000000..4b507bd --- /dev/null +++ b/metrics/scaling/bb.yaml.in @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +# We use a deployment rather than a pod directly specifically so we can use it to +# generate more replicas by re-deploying it. +# The only downside might be, if containers go wrong or die etc., then the deployment +# is going to try and re-deploy them, and we may not notice or get stuck waiting for +# them, so we want to be careful in the test code to time out in such cases. +kind: Deployment +metadata: + labels: + run: busybox + name: @DEPLOYMENT@ +spec: + replicas: @REPLICAS@ + selector: + matchLabels: + run: busybox + template: + metadata: + labels: + run: busybox + @LABEL@: @LABELVALUE@ + spec: + runtimeClassName: @RUNTIMECLASS@ + containers: + - name: bb + image: busybox + command: + - "tail" + - "-f" + - "/dev/null" + stdin: true + tty: true + restartPolicy: Always diff --git a/metrics/scaling/k8s_scale.sh b/metrics/scaling/k8s_scale.sh new file mode 100755 index 0000000..7ca1d64 --- /dev/null +++ b/metrics/scaling/k8s_scale.sh @@ -0,0 +1,267 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +# Pull in some common, useful, items +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +input_yaml="bb.yaml.in" +generated_yaml="generated.yaml" +deployment="busybox" + +stats_pod="stats" + +export RUNTIME="kata-qemu" +NUM_PODS=${NUM_PODS:-20} +STEP=${STEP:-1} + +LABEL=${LABEL:-magiclabel} +LABELVALUE=${LABELVALUE:-gandalf} + +# sleep and timeout times for k8s actions, in seconds +wait_time=${wait_time:-30} +delete_wait_time=${delete_wait_time:-600} +settle_time=${settle_time:-5} +use_kata_runtime=${use_kata_runtime:-no} + +# Set some default metrics env vars +TEST_ARGS="runtime=${RUNTIME}" +TEST_NAME="k8s scaling" + +get_num_nodes() { + n=$(kubectl get nodes --no-headers=true | wc -l) + echo "$n" +} + +# $1 is the launch time in seconds this pod/container took to start up. +# $2 is the number of pod/containers under test +grab_stats() { + local launch_time=$1 + local n_pods=$2 + info "And grab some stats" + # Tell mpstat to measure over a short period, not only so we get slightly de-noised data, but also + # if you don't tell it the period, you will get the avg since boot, which is not what we want. + local cpu_idle=$(kubectl exec -ti ${stats_pod} -- sh -c "mpstat -u 3 1 | tail -1 | awk '{print \$11}'" | sed 's/\r//') + local mem_free=$(kubectl exec -ti ${stats_pod} -- sh -c "free | tail -2 | head -1 | awk '{print \$4}'" | sed 's/\r//') + + info "idle [$cpu_idle] free [$mem_free] launch [$launch_time]" + + # Annoyingly, it seems sometimes once in a while we don't get an answer! + # We should really retry, but for now, make the json valid at least + cpu_idle=${cpu_idle:-0} + mem_free=${mem_free:-0} + + local json="$(cat << EOF + { + "n_pods": { + "Result": ${n_pods}, + "Units" : "int" + }, + "cpu_idle": { + "Result": ${cpu_idle}, + "Units" : "%" + }, + "launch_time": { + "Result": $launch_time, + "Units" : "s" + }, + "mem_free": { + "Result": ${mem_free}, + "Units" : "kb" + } + } +EOF +)" + metrics_json_add_array_element "$json" +} + +init() { + info "Initialising" + info "Checking k8s accessible" + local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? ) + if [ "$worked" != 0 ]; then + die "kubectl failed to get nodes" + fi + + info $(get_num_nodes) "k8s nodes found" + # We could check we have just the one node here - right now this is a single node + # test!! - because, our stats gathering is rudimentry, as k8s does not provide + # a nice way to do it (unless you want to parse 'descibe nodes') + # Have a read of https://github.com/kubernetes/kubernetes/issues/25353 + + # FIXME - check the node(s) can run enough pods - check 'max-pods' in the + # kubelet config - from 'kubectl describe node -o json' ? + + # Launch our stats gathering pod + kubectl apply -f ${stats_pod}.yaml + kubectl wait --for=condition=Ready pod "${stats_pod}" + + # FIXME - we should probably 'warm up' the cluster with the container image(s) we will + # use for testing, otherwise the download time will likely be included in the first pod + # boot time. + + # And now we can set up our results storage then... + metrics_json_init "k8s" + save_config +} + +save_config(){ + metrics_json_start_array + + local json="$(cat << EOF + { + "testname": "${TEST_NAME}", + "NUM_PODS": "${NUM_PODS}", + "STEP": "${STEP}", + "wait_time": "${wait_time}", + "delete_wait_time": "${delete_wait_time}", + "settle_time": "${settle_time}" + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Config" +} + +run() { + info "Running test" + + trap cleanup EXIT QUIT KILL + + metrics_json_start_array + for reqs in $(seq ${STEP} ${STEP} ${NUM_PODS}); do + info "Testing replicas ${reqs} of ${NUM_PODS}" + # Generate the next yaml file + + if [ "$use_kata_runtime" != "no" ]; then + sed -e "s|@REPLICAS@|${reqs}|g" \ + -e "s|@RUNTIMECLASS@|${RUNTIME}|g" \ + -e "s|@DEPLOYMENT@|${deployment}|g" \ + -e "s|@LABEL@|${LABEL}|g" \ + -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + < ${input_yaml} > ${generated_yaml} + else + sed -e "s|@REPLICAS@|${reqs}|g" \ + -e "/@RUNTIMECLASS@/d" \ + -e "s|@DEPLOYMENT@|${deployment}|g" \ + -e "s|@LABEL@|${LABEL}|g" \ + -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + < ${input_yaml} > ${generated_yaml} + fi + + info "Applying changes" + # FIXME - time taken in 'seconds' - we can do better than that, if we + # use the date nanosecond stamp, and steal the magic from the density test script. + # switch to 'date +%N' + local start_time=$(date +%s) + kubectl apply -f ${generated_yaml} + + #cmd="kubectl get pods | grep busybox | grep Completed" + kubectl rollout status --timeout=${wait_time}s deployment/${deployment} + local end_time=$(date +%s) + local total_seconds=$(( end_time - start_time )) + info "Took $total_seconds ($end_time - $start_time)" + sleep ${settle_time} + grab_stats $total_seconds $reqs + done +} + +cleanup() { + info "Cleaning up" + + # First try to save any results we got + metrics_json_end_array "BootResults" + + kubectl delete pod --wait=true --timeout=${delete_wait_time}s "${stats_pod}" || true + local start_time=$(date +%s) + kubectl delete deployment --wait=true --timeout=${delete_wait_time}s "${deployment}" || true + for x in $(seq 1 ${delete_wait_time}); do + local npods=$(kubectl get pods -l=${LABEL}=${LABELVALUE} -o=name | wc -l) + if [ $npods -eq 0 ]; then + echo "All pods have terminated at cycle $x" + local alldied=true + break; + fi + sleep 1 + done + local end_time=$(date +%s) + local total_seconds=$(( end_time - start_time )) + if [ -z "$alldied" ]; then + echo "ERROR: Not all pods died!" + fi + info "Delete Took $total_seconds ($end_time - $start_time)" + + local json="$(cat << EOF + "Delete": { + "Result": ${total_seconds}, + "Units" : "s" + } +EOF +)" + + metrics_json_add_fragment "$json" + metrics_json_save +} + +show_vars() +{ + echo -e "\nEnvironment variables:" + echo -e "\tName (default)" + echo -e "\t\tDescription" + echo -e "\tTEST_NAME (${TEST_NAME})" + echo -e "\t\tCan be set to over-ride the default JSON results filename" + echo -e "\tNUM_PODS (${NUM_PODS})" + echo -e "\t\tNumber of pods to launch" + echo -e "\tSTEP (${STEP})" + echo -e "\t\tNumber of pods to launch per cycle" + echo -e "\twait_time (${wait_time})" + echo -e "\t\tSeconds to wait for pods to become ready" + echo -e "\tdelete_wait_time (${delete_wait_time})" + echo -e "\t\tSeconds to wait for all pods to be deleted" + echo -e "\tsettle_time (${settle_time})" + echo -e "\t\tSeconds to wait after pods ready before taking measurements" + echo -e "\tuse_kata_runtime (${use_kata_runtime})" + echo -e "\t\tspecify yes or no to use kata runtime" +} + +help() +{ + usage=$(cat << EOF +Usage: $0 [-h] [options] + Description: + Launch a series of workloads and take memory metric measurements after + each launch. + Options: + -h, Help page. +EOF +) + echo "$usage" + show_vars +} + +main() { + + local OPTIND + while getopts "h" opt;do + case ${opt} in + h) + help + exit 0; + ;; + esac + done + shift $((OPTIND-1)) + + init + run + # cleanup will happen at exit due to the shell 'trap' we registered + # cleanup +} + +main "$@" + diff --git a/metrics/scaling/stats.yaml b/metrics/scaling/stats.yaml new file mode 100644 index 0000000..cf14710 --- /dev/null +++ b/metrics/scaling/stats.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: stats +spec: + containers: + - name: stats + image: busybox + securityContext: + # Run a priv container so we really do measure what is happening on the + # host (node) system + privileged: true + command: + - "tail" + - "-f" + - "/dev/null" + stdin: true + tty: true