From e73beb18e1e9b5c1e8592ffab6021fe4a4e0c69c Mon Sep 17 00:00:00 2001 From: Graham Whaley Date: Wed, 24 Jul 2019 15:00:56 +0100 Subject: [PATCH] metrics: parallel: Test deployment parallel pod scaling Test how long it takes to launch, and delete, a deployment running up 'n' pods. Used to ascertain if parallel launch times are linear with number of pods or not. Signed-off-by: Graham Whaley --- metrics/report/grabdata.sh | 1 + .../report_dockerfile/metrics_report.Rmd | 11 + metrics/report/report_dockerfile/parallel.R | 124 ++++++++ metrics/scaling/k8s_parallel.sh | 288 ++++++++++++++++++ 4 files changed, 424 insertions(+) create mode 100755 metrics/report/report_dockerfile/parallel.R create mode 100755 metrics/scaling/k8s_parallel.sh diff --git a/metrics/report/grabdata.sh b/metrics/report/grabdata.sh index b67f884..6dec893 100755 --- a/metrics/report/grabdata.sh +++ b/metrics/report/grabdata.sh @@ -72,6 +72,7 @@ run_scaling() { echo "Running scaling tests" (cd scaling; ./k8s_scale.sh) + (cd scaling; ./k8s_parallel.sh) } # Execute metrics scripts diff --git a/metrics/report/report_dockerfile/metrics_report.Rmd b/metrics/report/report_dockerfile/metrics_report.Rmd index 950bbc1..c57612d 100644 --- a/metrics/report/report_dockerfile/metrics_report.Rmd +++ b/metrics/report/report_dockerfile/metrics_report.Rmd @@ -38,3 +38,14 @@ and more idle `busybox` pods on a single node Kubernetes cluster. ```{r, echo=FALSE, fig.cap="K8S scaling"} source('scaling.R') ``` + +\pagebreak + +# Runtime parallel scaling +This [test](https://github.com/clearlinux/cloud-native-setup/metrics/scaling/k8s_parallel.sh) +measures the time taken to launch and delete pods in parallel using a deployment. The times +are how long it takes for the whole deployment operation to complete. + +```{r, echo=FALSE, fig.cap="K8S parallel pods"} +source('parallel.R') +``` diff --git a/metrics/report/report_dockerfile/parallel.R b/metrics/report/report_dockerfile/parallel.R new file mode 100755 index 0000000..954eee8 --- /dev/null +++ b/metrics/report/report_dockerfile/parallel.R @@ -0,0 +1,124 @@ +#!/usr/bin/env Rscript +# Copyright (c) 2018-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Show effects of parallel container launch on boot and deletion times by +# launching and killing off a deployment whilst ramping the number of pods requested. + +suppressMessages(suppressWarnings(library(ggplot2))) # ability to plot nicely. + # So we can plot multiple graphs +library(gridExtra) # together. +suppressMessages(suppressWarnings(library(ggpubr))) # for ggtexttable. +suppressMessages(library(jsonlite)) # to load the data. +suppressMessages(library(scales)) # For de-science notation of axis + +testnames=c( + "k8s-parallel*" +) + +data=c() +stats=c() +rstats=c() +rstats_names=c() +cstats=c() +cstats_names=c() + +skip_points=0 # Should we draw the points as well as lines on the graphs. + +for (currentdir in resultdirs) { + count=1 + dirstats=c() + for (testname in testnames) { + matchdir=paste(inputdir, currentdir, sep="") + matchfile=paste(testname, '\\.json', sep="") + files=list.files(matchdir, pattern=matchfile) + if ( length(files) == 0 ) { + #warning(paste("Pattern [", matchdir, "/", matchfile, "] matched nothing")) + } + for (ffound in files) { + fname=paste(inputdir, currentdir, ffound, sep="") + if ( !file.exists(fname)) { + warning(paste("Skipping non-existent file: ", fname)) + next + } + + # Derive the name from the test result dirname + datasetname=basename(currentdir) + + # Import the data + fdata=fromJSON(fname) + # De-nest the test name specific data + shortname=substr(ffound, 1, nchar(ffound)-nchar(".json")) + fdata=fdata[[shortname]] + + testname=datasetname + + # convert ms to seconds + cdata=data.frame(boot_time=as.numeric(fdata$BootResults$launch_time$Result)/1000) + cdata=cbind(cdata, delete_time=as.numeric(fdata$BootResults$delete_time$Result)/1000) + + # If we have more than 20 items to draw, then do not draw the points on + # the graphs, as they are then too noisy to read. + if (length(cdata[, "boot_time"]) > 20) { + skip_points=1 + } + + cdata=cbind(cdata, count=seq_len(length(cdata[, "boot_time"]))) + cdata=cbind(cdata, testname=rep(testname, length(cdata[, "boot_time"]) )) + cdata=cbind(cdata, dataset=rep(datasetname, length(cdata[, "boot_time"]) )) + + # Store away as a single set + data=rbind(data, cdata) + + count = count + 1 + } + } +} + +# Show how boot time changed +boot_line_plot <- ggplot() + + geom_line( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.2) + + geom_smooth( data=data, aes(count, boot_time, colour=testname, group=dataset), se=FALSE, method="loess", size=0.3) + + xlab("parallel pods") + + ylab("Boot time (s)") + + ggtitle("Pod boot time (detail)") + + #ylim(0, NA) + # For big machines, better to not 0-index + theme(axis.text.x=element_text(angle=90)) + +if ( skip_points == 0 ) { + boot_line_plot = boot_line_plot + geom_point( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.3) +} + + # And get a zero Y index plot. + boot_line_plot_zero = boot_line_plot + ylim(0, NA) + + ggtitle("Pod boot time (0 index)") + +# Show how boot time changed +delete_line_plot <- ggplot() + + geom_line( data=data, aes(count, delete_time, colour=testname, group=dataset), alpha=0.2) + + geom_smooth( data=data, aes(count, delete_time, colour=testname, group=dataset), se=FALSE, method="loess", size=0.3) + + xlab("parallel pods") + + ylab("Delete time (s)") + + ggtitle("Pod deletion time (detail)") + + #ylim(0, NA) + # For big machines, better to not 0-index + theme(axis.text.x=element_text(angle=90)) + +if ( skip_points == 0 ) { + delete_line_plot = delete_line_plot + geom_point( data=data, aes(count, delete_time, colour=testname, group=dataset), alpha=0.3) +} + + # And get a 0 indexed Y axis plot + delete_line_plot_zero = delete_line_plot + ylim(0, NA) + + ggtitle("Pod deletion time (0 index)") + +# See https://www.r-bloggers.com/ggplot2-easy-way-to-mix-multiple-graphs-on-the-same-page/ for +# excellent examples +master_plot = grid.arrange( + boot_line_plot_zero, + delete_line_plot_zero, + boot_line_plot, + delete_line_plot, + nrow=2, + ncol=2 ) + diff --git a/metrics/scaling/k8s_parallel.sh b/metrics/scaling/k8s_parallel.sh new file mode 100755 index 0000000..77136b6 --- /dev/null +++ b/metrics/scaling/k8s_parallel.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +# Measure pod create and delete times whilst launching +# them in parallel - try to measure any effects parallel pod +# launching has. + +set -e + +# Pull in some common, useful, items +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" + +input_yaml="${SCRIPT_PATH}/bb.yaml.in" +input_json="${SCRIPT_PATH}/bb.json.in" +generated_yaml="${SCRIPT_PATH}/generated.yaml" +generated_json="${SCRIPT_PATH}/generated.json" +deployment="busybox" + +NUM_PODS=${NUM_PODS:-20} +STEP=${STEP:-1} + +LABEL=${LABEL:-magiclabel} +LABELVALUE=${LABELVALUE:-parallel} + +# sleep and timeout times for k8s actions, in seconds +wait_time=${wait_time:-30} +delete_wait_time=${delete_wait_time:-600} +use_api=${use_api:-yes} + +# Set some default metrics env vars +TEST_ARGS="runtime=${RUNTIME}" +TEST_NAME="k8s parallel" + +# Kill off a deployment, and wait for it to finish dying off. +# $1 name of deployment +# $2 name of label to watch +# $3 value of label to watch +# $4 timeout to wait, in seconds +kill_deployment() { + kubectl delete deployment --wait=true --timeout=${4}s "${1}" || true + for x in $(seq 1 ${delete_wait_time}); do + # FIXME + local npods=$(kubectl get pods -l=${2}=${3} -o=name | wc -l) + if [ $npods -eq 0 ]; then + info "deployment has terminated" + local alldied=true + break; + fi + sleep 1 + done + + if [ -z "$alldied" ]; then + info "Not all pods died" + fi +} + +# Run up a single pod, and kill it off. This will pre-warm any one-time/first-time +# elements, such as pulling down the container image if necessary. +warmup() { + info "Warming up" + + trap cleanup EXIT QUIT KILL + + local runtime_command + if [ -n "$RUNTIME" ]; then + runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g" + else + runtime_command="/@RUNTIMECLASS@/d" + fi + + # Always use the convenience of kubectl (not the REST API) to + # run the warmup pod, why not. + sed -e "s|@REPLICAS@|${reqs}|g" \ + -e $runtime_command \ + -e "s|@DEPLOYMENT@|${deployment}|g" \ + -e "s|@LABEL@|${LABEL}|g" \ + -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + < ${input_yaml} > ${generated_yaml} + + info "Applying warmup pod" + kubectl apply -f ${generated_yaml} + info "Waiting for warmup" + kubectl rollout status --timeout=${wait_time}s deployment/${deployment} + + info "Killing warmup pod" + kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time} + +} + +# $1 is the launch time in seconds this pod/container took to start up. +# $2 is the delete time in seconds this pod/container took to start up. +# $2 is the number of pod/containers under test +save_stats() { + local launch_time_ms=$1 + local delete_time_ms=$2 + local n_pods=$3 + + local json="$(cat << EOF + { + "n_pods": { + "Result": ${n_pods}, + "Units" : "int" + }, + "launch_time": { + "Result": $launch_time_ms, + "Units" : "ms" + }, + "delete_time": { + "Result": $delete_time_ms, + "Units" : "ms" + } + } +EOF +)" + metrics_json_add_array_element "$json" +} + +init() { + info "Initialising" + info "Checking k8s accessible" + local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? ) + if [ "$worked" != 0 ]; then + die "kubectl failed to get nodes" + fi + + info $(get_num_nodes) "k8s nodes found" + # We could check we have just the one node here - right now this is a single node + # test!! - because, our stats gathering is rudimentry, as k8s does not provide + # a nice way to do it (unless you want to parse 'descibe nodes') + # Have a read of https://github.com/kubernetes/kubernetes/issues/25353 + + k8s_api_init + + # Ensure we pre-cache the container image etc. + warmup + + # And now we can set up our results storage then... + metrics_json_init "k8s" + save_config +} + +save_config(){ + metrics_json_start_array + + local json="$(cat << EOF + { + "testname": "${TEST_NAME}", + "NUM_PODS": "${NUM_PODS}", + "STEP": "${STEP}", + "wait_time": "${wait_time}", + "delete_wait_time": "${delete_wait_time}" + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Config" +} + +run() { + info "Running test" + + trap cleanup EXIT QUIT KILL + + metrics_json_start_array + for reqs in $(seq ${STEP} ${STEP} ${NUM_PODS}); do + info "Testing parallel replicas ${reqs} of ${NUM_PODS}" + # Generate the next yaml file + + local runtime_command + if [ -n "$RUNTIME" ]; then + runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g" + else + runtime_command="/@RUNTIMECLASS@/d" + fi + + local input_template + local generated_file + if [ "$use_api" != "no" ]; then + input_template=$input_json + generated_file=$generated_json + else + input_template=$input_yaml + generated_file=$generated_yaml + fi + + sed -e "s|@REPLICAS@|${reqs}|g" \ + -e $runtime_command \ + -e "s|@DEPLOYMENT@|${deployment}|g" \ + -e "s|@LABEL@|${LABEL}|g" \ + -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + < ${input_template} > ${generated_file} + + info "Applying changes" + local start_time=$(date +%s%N) + + if [ "$use_api" != "no" ]; then + curl -s ${API_ADDRESS}:${API_PORT}/apis/apps/v1/namespaces/default/deployments -XPOST -H 'Content-Type: application/json' -d@${generated_file} > /dev/null + else + kubectl apply -f ${generated_file} + fi + + kubectl rollout status --timeout=${wait_time}s deployment/${deployment} + local end_time=$(date +%s%N) + local total_milliseconds=$(( (end_time - start_time) / 1000000 )) + info "Took $total_milliseconds ms ($end_time - $start_time)" + + # And now remove that deployment, ready to launch the next one + local delete_start_time=$(date +%s%N) + kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time} + local delete_end_time=$(date +%s%N) + local delete_total_milliseconds=$(( (delete_end_time - delete_start_time) / 1000000 )) + info "Delete took $delete_total_milliseconds ms ($delete_end_time - $delete_start_time)" + save_stats $total_milliseconds $delete_total_milliseconds $reqs + done +} + +cleanup() { + info "Cleaning up" + + # First try to save any results we got + metrics_json_end_array "BootResults" + metrics_json_save + + kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time} + + k8s_api_shutdown +} + +show_vars() +{ + echo -e "\nEnvironment variables:" + echo -e "\tName (default)" + echo -e "\t\tDescription" + echo -e "\tTEST_NAME (${TEST_NAME})" + echo -e "\t\tCan be set to over-ride the default JSON results filename" + echo -e "\tNUM_PODS (${NUM_PODS})" + echo -e "\t\tNumber of pods to launch" + echo -e "\tSTEP (${STEP})" + echo -e "\t\tNumber of pods to launch per cycle" + echo -e "\twait_time (${wait_time})" + echo -e "\t\tSeconds to wait for pods to become ready" + echo -e "\tdelete_wait_time (${delete_wait_time})" + echo -e "\t\tSeconds to wait for all pods to be deleted" + echo -e "\tsettle_time (${settle_time})" + echo -e "\t\tSeconds to wait after pods ready before taking measurements" + echo -e "\tuse_api (${use_api})" + echo -e "\t\tspecify yes or no to use the API to launch pods" +} + +help() +{ + usage=$(cat << EOF +Usage: $0 [-h] [options] + Description: + Launch a series of workloads and take memory metric measurements after + each launch. + Options: + -h, Help page. +EOF +) + echo "$usage" + show_vars +} + +main() { + + local OPTIND + while getopts "h" opt;do + case ${opt} in + h) + help + exit 0; + ;; + esac + done + shift $((OPTIND-1)) + + init + run + # cleanup will happen at exit due to the shell 'trap' we registered + # cleanup +} + +main "$@" +