mirror of
https://github.com/clearlinux/cloud-native-setup.git
synced 2026-08-26 18:36:03 +00:00
metrics: parallel: Test deployment parallel pod scaling
Test how long it takes to launch, and delete, a deployment running up 'n' pods. Used to ascertain if parallel launch times are linear with number of pods or not. Signed-off-by: Graham Whaley <graham.whaley@intel.com>
This commit is contained in:
Graham Whaley
committed by
David Lyle
David Lyle
parent
59a1f36e37
commit
e73beb18e1
@@ -72,6 +72,7 @@ run_scaling() {
|
||||
echo "Running scaling tests"
|
||||
|
||||
(cd scaling; ./k8s_scale.sh)
|
||||
(cd scaling; ./k8s_parallel.sh)
|
||||
}
|
||||
|
||||
# Execute metrics scripts
|
||||
|
||||
@@ -38,3 +38,14 @@ and more idle `busybox` pods on a single node Kubernetes cluster.
|
||||
```{r, echo=FALSE, fig.cap="K8S scaling"}
|
||||
source('scaling.R')
|
||||
```
|
||||
|
||||
\pagebreak
|
||||
|
||||
# Runtime parallel scaling
|
||||
This [test](https://github.com/clearlinux/cloud-native-setup/metrics/scaling/k8s_parallel.sh)
|
||||
measures the time taken to launch and delete pods in parallel using a deployment. The times
|
||||
are how long it takes for the whole deployment operation to complete.
|
||||
|
||||
```{r, echo=FALSE, fig.cap="K8S parallel pods"}
|
||||
source('parallel.R')
|
||||
```
|
||||
|
||||
Executable
+124
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env Rscript
|
||||
# Copyright (c) 2018-2019 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Show effects of parallel container launch on boot and deletion times by
|
||||
# launching and killing off a deployment whilst ramping the number of pods requested.
|
||||
|
||||
suppressMessages(suppressWarnings(library(ggplot2))) # ability to plot nicely.
|
||||
# So we can plot multiple graphs
|
||||
library(gridExtra) # together.
|
||||
suppressMessages(suppressWarnings(library(ggpubr))) # for ggtexttable.
|
||||
suppressMessages(library(jsonlite)) # to load the data.
|
||||
suppressMessages(library(scales)) # For de-science notation of axis
|
||||
|
||||
testnames=c(
|
||||
"k8s-parallel*"
|
||||
)
|
||||
|
||||
data=c()
|
||||
stats=c()
|
||||
rstats=c()
|
||||
rstats_names=c()
|
||||
cstats=c()
|
||||
cstats_names=c()
|
||||
|
||||
skip_points=0 # Should we draw the points as well as lines on the graphs.
|
||||
|
||||
for (currentdir in resultdirs) {
|
||||
count=1
|
||||
dirstats=c()
|
||||
for (testname in testnames) {
|
||||
matchdir=paste(inputdir, currentdir, sep="")
|
||||
matchfile=paste(testname, '\\.json', sep="")
|
||||
files=list.files(matchdir, pattern=matchfile)
|
||||
if ( length(files) == 0 ) {
|
||||
#warning(paste("Pattern [", matchdir, "/", matchfile, "] matched nothing"))
|
||||
}
|
||||
for (ffound in files) {
|
||||
fname=paste(inputdir, currentdir, ffound, sep="")
|
||||
if ( !file.exists(fname)) {
|
||||
warning(paste("Skipping non-existent file: ", fname))
|
||||
next
|
||||
}
|
||||
|
||||
# Derive the name from the test result dirname
|
||||
datasetname=basename(currentdir)
|
||||
|
||||
# Import the data
|
||||
fdata=fromJSON(fname)
|
||||
# De-nest the test name specific data
|
||||
shortname=substr(ffound, 1, nchar(ffound)-nchar(".json"))
|
||||
fdata=fdata[[shortname]]
|
||||
|
||||
testname=datasetname
|
||||
|
||||
# convert ms to seconds
|
||||
cdata=data.frame(boot_time=as.numeric(fdata$BootResults$launch_time$Result)/1000)
|
||||
cdata=cbind(cdata, delete_time=as.numeric(fdata$BootResults$delete_time$Result)/1000)
|
||||
|
||||
# If we have more than 20 items to draw, then do not draw the points on
|
||||
# the graphs, as they are then too noisy to read.
|
||||
if (length(cdata[, "boot_time"]) > 20) {
|
||||
skip_points=1
|
||||
}
|
||||
|
||||
cdata=cbind(cdata, count=seq_len(length(cdata[, "boot_time"])))
|
||||
cdata=cbind(cdata, testname=rep(testname, length(cdata[, "boot_time"]) ))
|
||||
cdata=cbind(cdata, dataset=rep(datasetname, length(cdata[, "boot_time"]) ))
|
||||
|
||||
# Store away as a single set
|
||||
data=rbind(data, cdata)
|
||||
|
||||
count = count + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Show how boot time changed
|
||||
boot_line_plot <- ggplot() +
|
||||
geom_line( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.2) +
|
||||
geom_smooth( data=data, aes(count, boot_time, colour=testname, group=dataset), se=FALSE, method="loess", size=0.3) +
|
||||
xlab("parallel pods") +
|
||||
ylab("Boot time (s)") +
|
||||
ggtitle("Pod boot time (detail)") +
|
||||
#ylim(0, NA) + # For big machines, better to not 0-index
|
||||
theme(axis.text.x=element_text(angle=90))
|
||||
|
||||
if ( skip_points == 0 ) {
|
||||
boot_line_plot = boot_line_plot + geom_point( data=data, aes(count, boot_time, colour=testname, group=dataset), alpha=0.3)
|
||||
}
|
||||
|
||||
# And get a zero Y index plot.
|
||||
boot_line_plot_zero = boot_line_plot + ylim(0, NA) +
|
||||
ggtitle("Pod boot time (0 index)")
|
||||
|
||||
# Show how boot time changed
|
||||
delete_line_plot <- ggplot() +
|
||||
geom_line( data=data, aes(count, delete_time, colour=testname, group=dataset), alpha=0.2) +
|
||||
geom_smooth( data=data, aes(count, delete_time, colour=testname, group=dataset), se=FALSE, method="loess", size=0.3) +
|
||||
xlab("parallel pods") +
|
||||
ylab("Delete time (s)") +
|
||||
ggtitle("Pod deletion time (detail)") +
|
||||
#ylim(0, NA) + # For big machines, better to not 0-index
|
||||
theme(axis.text.x=element_text(angle=90))
|
||||
|
||||
if ( skip_points == 0 ) {
|
||||
delete_line_plot = delete_line_plot + geom_point( data=data, aes(count, delete_time, colour=testname, group=dataset), alpha=0.3)
|
||||
}
|
||||
|
||||
# And get a 0 indexed Y axis plot
|
||||
delete_line_plot_zero = delete_line_plot + ylim(0, NA) +
|
||||
ggtitle("Pod deletion time (0 index)")
|
||||
|
||||
# See https://www.r-bloggers.com/ggplot2-easy-way-to-mix-multiple-graphs-on-the-same-page/ for
|
||||
# excellent examples
|
||||
master_plot = grid.arrange(
|
||||
boot_line_plot_zero,
|
||||
delete_line_plot_zero,
|
||||
boot_line_plot,
|
||||
delete_line_plot,
|
||||
nrow=2,
|
||||
ncol=2 )
|
||||
|
||||
Executable
+288
@@ -0,0 +1,288 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) 2019 Intel Corporation
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Measure pod create and delete times whilst launching
|
||||
# them in parallel - try to measure any effects parallel pod
|
||||
# launching has.
|
||||
|
||||
set -e
|
||||
|
||||
# Pull in some common, useful, items
|
||||
SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
|
||||
source "${SCRIPT_PATH}/../lib/common.bash"
|
||||
|
||||
input_yaml="${SCRIPT_PATH}/bb.yaml.in"
|
||||
input_json="${SCRIPT_PATH}/bb.json.in"
|
||||
generated_yaml="${SCRIPT_PATH}/generated.yaml"
|
||||
generated_json="${SCRIPT_PATH}/generated.json"
|
||||
deployment="busybox"
|
||||
|
||||
NUM_PODS=${NUM_PODS:-20}
|
||||
STEP=${STEP:-1}
|
||||
|
||||
LABEL=${LABEL:-magiclabel}
|
||||
LABELVALUE=${LABELVALUE:-parallel}
|
||||
|
||||
# sleep and timeout times for k8s actions, in seconds
|
||||
wait_time=${wait_time:-30}
|
||||
delete_wait_time=${delete_wait_time:-600}
|
||||
use_api=${use_api:-yes}
|
||||
|
||||
# Set some default metrics env vars
|
||||
TEST_ARGS="runtime=${RUNTIME}"
|
||||
TEST_NAME="k8s parallel"
|
||||
|
||||
# Kill off a deployment, and wait for it to finish dying off.
|
||||
# $1 name of deployment
|
||||
# $2 name of label to watch
|
||||
# $3 value of label to watch
|
||||
# $4 timeout to wait, in seconds
|
||||
kill_deployment() {
|
||||
kubectl delete deployment --wait=true --timeout=${4}s "${1}" || true
|
||||
for x in $(seq 1 ${delete_wait_time}); do
|
||||
# FIXME
|
||||
local npods=$(kubectl get pods -l=${2}=${3} -o=name | wc -l)
|
||||
if [ $npods -eq 0 ]; then
|
||||
info "deployment has terminated"
|
||||
local alldied=true
|
||||
break;
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "$alldied" ]; then
|
||||
info "Not all pods died"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run up a single pod, and kill it off. This will pre-warm any one-time/first-time
|
||||
# elements, such as pulling down the container image if necessary.
|
||||
warmup() {
|
||||
info "Warming up"
|
||||
|
||||
trap cleanup EXIT QUIT KILL
|
||||
|
||||
local runtime_command
|
||||
if [ -n "$RUNTIME" ]; then
|
||||
runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g"
|
||||
else
|
||||
runtime_command="/@RUNTIMECLASS@/d"
|
||||
fi
|
||||
|
||||
# Always use the convenience of kubectl (not the REST API) to
|
||||
# run the warmup pod, why not.
|
||||
sed -e "s|@REPLICAS@|${reqs}|g" \
|
||||
-e $runtime_command \
|
||||
-e "s|@DEPLOYMENT@|${deployment}|g" \
|
||||
-e "s|@LABEL@|${LABEL}|g" \
|
||||
-e "s|@LABELVALUE@|${LABELVALUE}|g" \
|
||||
< ${input_yaml} > ${generated_yaml}
|
||||
|
||||
info "Applying warmup pod"
|
||||
kubectl apply -f ${generated_yaml}
|
||||
info "Waiting for warmup"
|
||||
kubectl rollout status --timeout=${wait_time}s deployment/${deployment}
|
||||
|
||||
info "Killing warmup pod"
|
||||
kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time}
|
||||
|
||||
}
|
||||
|
||||
# $1 is the launch time in seconds this pod/container took to start up.
|
||||
# $2 is the delete time in seconds this pod/container took to start up.
|
||||
# $2 is the number of pod/containers under test
|
||||
save_stats() {
|
||||
local launch_time_ms=$1
|
||||
local delete_time_ms=$2
|
||||
local n_pods=$3
|
||||
|
||||
local json="$(cat << EOF
|
||||
{
|
||||
"n_pods": {
|
||||
"Result": ${n_pods},
|
||||
"Units" : "int"
|
||||
},
|
||||
"launch_time": {
|
||||
"Result": $launch_time_ms,
|
||||
"Units" : "ms"
|
||||
},
|
||||
"delete_time": {
|
||||
"Result": $delete_time_ms,
|
||||
"Units" : "ms"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
)"
|
||||
metrics_json_add_array_element "$json"
|
||||
}
|
||||
|
||||
init() {
|
||||
info "Initialising"
|
||||
info "Checking k8s accessible"
|
||||
local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? )
|
||||
if [ "$worked" != 0 ]; then
|
||||
die "kubectl failed to get nodes"
|
||||
fi
|
||||
|
||||
info $(get_num_nodes) "k8s nodes found"
|
||||
# We could check we have just the one node here - right now this is a single node
|
||||
# test!! - because, our stats gathering is rudimentry, as k8s does not provide
|
||||
# a nice way to do it (unless you want to parse 'descibe nodes')
|
||||
# Have a read of https://github.com/kubernetes/kubernetes/issues/25353
|
||||
|
||||
k8s_api_init
|
||||
|
||||
# Ensure we pre-cache the container image etc.
|
||||
warmup
|
||||
|
||||
# And now we can set up our results storage then...
|
||||
metrics_json_init "k8s"
|
||||
save_config
|
||||
}
|
||||
|
||||
save_config(){
|
||||
metrics_json_start_array
|
||||
|
||||
local json="$(cat << EOF
|
||||
{
|
||||
"testname": "${TEST_NAME}",
|
||||
"NUM_PODS": "${NUM_PODS}",
|
||||
"STEP": "${STEP}",
|
||||
"wait_time": "${wait_time}",
|
||||
"delete_wait_time": "${delete_wait_time}"
|
||||
}
|
||||
EOF
|
||||
)"
|
||||
metrics_json_add_array_element "$json"
|
||||
metrics_json_end_array "Config"
|
||||
}
|
||||
|
||||
run() {
|
||||
info "Running test"
|
||||
|
||||
trap cleanup EXIT QUIT KILL
|
||||
|
||||
metrics_json_start_array
|
||||
for reqs in $(seq ${STEP} ${STEP} ${NUM_PODS}); do
|
||||
info "Testing parallel replicas ${reqs} of ${NUM_PODS}"
|
||||
# Generate the next yaml file
|
||||
|
||||
local runtime_command
|
||||
if [ -n "$RUNTIME" ]; then
|
||||
runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g"
|
||||
else
|
||||
runtime_command="/@RUNTIMECLASS@/d"
|
||||
fi
|
||||
|
||||
local input_template
|
||||
local generated_file
|
||||
if [ "$use_api" != "no" ]; then
|
||||
input_template=$input_json
|
||||
generated_file=$generated_json
|
||||
else
|
||||
input_template=$input_yaml
|
||||
generated_file=$generated_yaml
|
||||
fi
|
||||
|
||||
sed -e "s|@REPLICAS@|${reqs}|g" \
|
||||
-e $runtime_command \
|
||||
-e "s|@DEPLOYMENT@|${deployment}|g" \
|
||||
-e "s|@LABEL@|${LABEL}|g" \
|
||||
-e "s|@LABELVALUE@|${LABELVALUE}|g" \
|
||||
< ${input_template} > ${generated_file}
|
||||
|
||||
info "Applying changes"
|
||||
local start_time=$(date +%s%N)
|
||||
|
||||
if [ "$use_api" != "no" ]; then
|
||||
curl -s ${API_ADDRESS}:${API_PORT}/apis/apps/v1/namespaces/default/deployments -XPOST -H 'Content-Type: application/json' -d@${generated_file} > /dev/null
|
||||
else
|
||||
kubectl apply -f ${generated_file}
|
||||
fi
|
||||
|
||||
kubectl rollout status --timeout=${wait_time}s deployment/${deployment}
|
||||
local end_time=$(date +%s%N)
|
||||
local total_milliseconds=$(( (end_time - start_time) / 1000000 ))
|
||||
info "Took $total_milliseconds ms ($end_time - $start_time)"
|
||||
|
||||
# And now remove that deployment, ready to launch the next one
|
||||
local delete_start_time=$(date +%s%N)
|
||||
kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time}
|
||||
local delete_end_time=$(date +%s%N)
|
||||
local delete_total_milliseconds=$(( (delete_end_time - delete_start_time) / 1000000 ))
|
||||
info "Delete took $delete_total_milliseconds ms ($delete_end_time - $delete_start_time)"
|
||||
save_stats $total_milliseconds $delete_total_milliseconds $reqs
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
info "Cleaning up"
|
||||
|
||||
# First try to save any results we got
|
||||
metrics_json_end_array "BootResults"
|
||||
metrics_json_save
|
||||
|
||||
kill_deployment "${deployment}" "${LABEL}" "${LABELVALUE}" ${delete_wait_time}
|
||||
|
||||
k8s_api_shutdown
|
||||
}
|
||||
|
||||
show_vars()
|
||||
{
|
||||
echo -e "\nEnvironment variables:"
|
||||
echo -e "\tName (default)"
|
||||
echo -e "\t\tDescription"
|
||||
echo -e "\tTEST_NAME (${TEST_NAME})"
|
||||
echo -e "\t\tCan be set to over-ride the default JSON results filename"
|
||||
echo -e "\tNUM_PODS (${NUM_PODS})"
|
||||
echo -e "\t\tNumber of pods to launch"
|
||||
echo -e "\tSTEP (${STEP})"
|
||||
echo -e "\t\tNumber of pods to launch per cycle"
|
||||
echo -e "\twait_time (${wait_time})"
|
||||
echo -e "\t\tSeconds to wait for pods to become ready"
|
||||
echo -e "\tdelete_wait_time (${delete_wait_time})"
|
||||
echo -e "\t\tSeconds to wait for all pods to be deleted"
|
||||
echo -e "\tsettle_time (${settle_time})"
|
||||
echo -e "\t\tSeconds to wait after pods ready before taking measurements"
|
||||
echo -e "\tuse_api (${use_api})"
|
||||
echo -e "\t\tspecify yes or no to use the API to launch pods"
|
||||
}
|
||||
|
||||
help()
|
||||
{
|
||||
usage=$(cat << EOF
|
||||
Usage: $0 [-h] [options]
|
||||
Description:
|
||||
Launch a series of workloads and take memory metric measurements after
|
||||
each launch.
|
||||
Options:
|
||||
-h, Help page.
|
||||
EOF
|
||||
)
|
||||
echo "$usage"
|
||||
show_vars
|
||||
}
|
||||
|
||||
main() {
|
||||
|
||||
local OPTIND
|
||||
while getopts "h" opt;do
|
||||
case ${opt} in
|
||||
h)
|
||||
help
|
||||
exit 0;
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
init
|
||||
run
|
||||
# cleanup will happen at exit due to the shell 'trap' we registered
|
||||
# cleanup
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user