From 20cb90c7f760f8fce9484fb284cf31a93715c8d5 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Thu, 3 Oct 2019 15:43:30 -0600 Subject: [PATCH] Adding support for aysnc metrics via collectd This patch adds a new script, scaling/k8s_scaling_rapid.sh, for launching pods and collecting metrics. The goal is to two fold. The first goal is improve the required runtime duration of scaling to large numbers of pods. k8s_scaling.sh can take up to 29 hours to scale to 2900 pods. The is largely due to the overhead of collecting system utilization stats after each new pod is launched. This new script will collect system utilization stats asynchronously. The second goal is to make it easier to collect additional system utilization stats by leveraging the plugins supported by collectd. Instead of using the stats daemon set, a new daemon set that runs collectd on each node is added. collectd configuration is handled by collectd/collectd.conf A configmap is added to the K8s cluster containing collectd.conf, so the user of the script can test new configurations easily. The configmap is created and deleted as part of the test run. The data from collectd is stored on each node in the cluster via the csv plugin and the data is collected to the master node at the end of the test run. Several new pages have been added to the metrics_report.pdf These pages cover the same metrics as k8s_scale.sh results, but are populated with data from collectd. Additionally, network interface results are added. To run the report, in addition to the previous steps of creating a new directory and copying the result json file into it, all the new .tar.gz files must be copied in as well. Signed-off-by: David Lyle --- metrics/README.md | 14 +- metrics/collectd/collectd.bash | 43 ++ metrics/collectd/collectd.conf | 43 ++ metrics/collectd/collectd.yaml | 64 +++ .../report_dockerfile/collectd_scaling.R | 499 ++++++++++++++++++ .../report/report_dockerfile/dut-details.R | 3 +- .../report_dockerfile/metrics_report.Rmd | 17 +- metrics/scaling/k8s_scale_rapid.sh | 299 +++++++++++ 8 files changed, 975 insertions(+), 7 deletions(-) create mode 100644 metrics/collectd/collectd.bash create mode 100644 metrics/collectd/collectd.conf create mode 100644 metrics/collectd/collectd.yaml create mode 100755 metrics/report/report_dockerfile/collectd_scaling.R create mode 100755 metrics/scaling/k8s_scale_rapid.sh diff --git a/metrics/README.md b/metrics/README.md index e934d05..8a6ded4 100644 --- a/metrics/README.md +++ b/metrics/README.md @@ -27,7 +27,7 @@ Elasticsearch and InfluxDB databases for instance, but should be adaptable to us JSON input. ## Scaling execution -This section describes a complete step-by-step scaling execution upto results reporting by using `scaling/k8s_scale.sh` tool which launches a series of workloads and take memory metric measurements after each launch. +This section describes a complete step-by-step scaling execution up to results reporting by using `scaling/k8s_scale.sh` tool which launches a series of workloads and take memory metric measurements after each launch. **Requirements** * A Kubernetes cluster up and running (tested on v1.15.3). @@ -35,7 +35,7 @@ This section describes a complete step-by-step scaling execution upto results re * Docker (only for report generation). The steps to execute a run of the scaling framework are listed below, which need to be executed on the master node of a Kubernetes cluster to avoid network issues: -1. Clone `cloud-native-setup` repository into a preferred directory and change directory upto `cloud-native-setup/metrics`: +1. Clone `cloud-native-setup` repository into a preferred directory and change directory up to `cloud-native-setup/metrics`: ```sh $ git clone https://github.com/clearlinux/cloud-native-setup.git $ cd cloud-native-setup/metrics @@ -60,9 +60,11 @@ The steps to execute a run of the scaling framework are listed below, which need INFO: Content of runtime_command=:/@RUNTIMECLASS@/d ... ``` -The above execution might take about 4min because it launch upto 20 pods by default and takes measurements for CPU utilization, memory utilization and pod boot time, finally it will generate a `k8s-scaling.json` result file at `result` directory. +The above execution might take about 4min because it launch up to 20 pods by default and takes measurements for CPU utilization, memory utilization and pod boot time, finally it will generate a `k8s-scaling.json` result file at `result` directory. -**Note**: by default the scaling framework makes call to the Kubernetes API directly so, if facing conectivity issues verify that `kubelet` service's proxies and `no_proxy` environment variable are properly setup. +**Note**: to test the launch of pods concurrently, `k8s_parallel.sh` may be used. For quicker testing, `k8s_scale_rapid.sh` can be used in place of `k8s_scale.sh`. The rest of the launch instructions remain consistent other than script name. + +**Note**: by default the scaling framework makes call to the Kubernetes API directly so, if facing connectivity issues verify that `kubelet` service's proxies and `no_proxy` environment variable are properly setup. **Note**: by default the scaling framework uses default values for all its required variables, which can be checked through `scaling/k8s_scale.sh -h` and updated when launching the execution, i.e.: ``` @@ -106,6 +108,10 @@ The steps to generate the result report are listed below: └── scaling └── k8s-scaling.json ``` + +**Note**: if `k8s_scale_rapid.sh` was run instead of `k8s_scale.sh`, that the `.tar.gz` files that appear in the results directory also need to be copied into the newly created subdirectory. And the results file is named `k8s-rapid.json` rather than `k8s-scaling.json`. +If k8s_parallel.sh was run, the results file is named `k8s-parallel.json` rather than `k8s-scaling.json`. + 2. Launch the report generation by: ```sh ./report/makereport.sh diff --git a/metrics/collectd/collectd.bash b/metrics/collectd/collectd.bash new file mode 100644 index 0000000..faaa380 --- /dev/null +++ b/metrics/collectd/collectd.bash @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +THIS_FILE=$(readlink -f ${BASH_SOURCE[0]}) +COLLECTD_DIR=${THIS_FILE%/*} + +collectd_pod="collectd" + +init_stats() { + local wait_time=$1 + + # create collectd-config configmap + kubectl create configmap collectd-config --from-file=${COLLECTD_DIR}/collectd.conf + + # Launch our stats gathering pod + kubectl apply -f ${COLLECTD_DIR}/${collectd_pod}.yaml + kubectl rollout status --timeout=${wait_time}s daemonset/${collectd_pod} + + # attempting to provide buffer for collectd to be installed and running, + # and CPU collection to build adequate history + sleep 12 +} + +cleanup_stats() { + local delete_wait_time=$1 + + # attempting to provide buffer for collectd CPU collection to record adequate history + sleep 6 + + # get logs before shutting down stats daemonset + while read -u 3 name node; do + kubectl exec -ti $name -- sh -c "cd /opt/collectd; tar -czvf localhost.tar.gz localhost" + kubectl cp $name:/opt/collectd/localhost.tar.gz ${RESULT_DIR}/${node}.tar.gz + done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"') + + kubectl delete daemonset --wait=true --timeout=${delete_wait_time}s "${collectd_pod}" || true + + # remove configmap + kubectl delete configmap collectd-config +} diff --git a/metrics/collectd/collectd.conf b/metrics/collectd/collectd.conf new file mode 100644 index 0000000..fcc7a0c --- /dev/null +++ b/metrics/collectd/collectd.conf @@ -0,0 +1,43 @@ +Interval 5 + +LoadPlugin aggregation +LoadPlugin cpu +LoadPlugin csv +LoadPlugin interface +LoadPlugin ipc +LoadPlugin memory +LoadPlugin cpufreq +LoadPlugin df + + + ReportByCpu false + ReportByState true + ValuesPercentage "True" + + + DataDir "/opt/collectd" + StoreRates true + + + Interface "/^eno/" + IgnoreSelected false + + + + Plugin "cpu" + Type "percent" + + GroupBy "Host" + GroupBy "TypeInstance" + + CalculateSum true + CalculateAverage true + + + + Device "overlay" + MountPoint "/" + FSType "overlay" + ReportInodes true + IgnoreSelected false + diff --git a/metrics/collectd/collectd.yaml b/metrics/collectd/collectd.yaml new file mode 100644 index 0000000..5947c47 --- /dev/null +++ b/metrics/collectd/collectd.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: collectd +spec: + selector: + matchLabels: + name: collectd-pods + template: + metadata: + labels: + name: collectd-pods + spec: + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + terminationGracePeriodSeconds: 0 + containers: + - name: collectd + image: alpine:latest + imagePullPolicy: IfNotPresent + securityContext: + # Run a priv container so we really do measure what is happening on the + # host (node) system + privileged: true + command: ["/bin/sh", "-c"] + args: + - apk update && apk add collectd; + collectd -f; + volumeMounts: + - name: collectd-config-volume + mountPath: /etc/collectd + - name: proc + mountPath: /mnt/proc + readOnly: true + - name: root + mountPath: /hostfs + readOnly: true + - name: etc + mountPath: /mnt/etc + readOnly: true + - name: opt + mountPath: /mnt/opt + volumes: + - name: collectd-config-volume + configMap: + name: collectd-config + items: + - key: collectd.conf + path: collectd.conf + - name: proc + hostPath: + path: /proc + - name: root + hostPath: + path: / + - name: etc + hostPath: + path: /etc + - name: opt + hostPath: + path: /opt diff --git a/metrics/report/report_dockerfile/collectd_scaling.R b/metrics/report/report_dockerfile/collectd_scaling.R new file mode 100755 index 0000000..f7aa772 --- /dev/null +++ b/metrics/report/report_dockerfile/collectd_scaling.R @@ -0,0 +1,499 @@ +#!/usr/bin/env Rscript +# Copyright (c) 2018-2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +# Show pod scaling data - memory use, boot time, CPU utilisation. + +suppressMessages(suppressWarnings(library(ggplot2))) # ability to plot nicely. + # So we can plot multiple graphs +library(gridExtra) # together. +suppressMessages(suppressWarnings(library(ggpubr))) # for ggtexttable. +suppressMessages(library(jsonlite)) # to load the data. +suppressMessages(library(scales)) # For de-science notation of axis +library(tibble) # tibbles for tidy data + +testnames=c( + "k8s-rapid.*" +) + +podbootdata=c() # Track per-launch data +cpuidledata=c() # Track cpu idle data per nodes +memfreedata=c() # Track mem free data for nodes +inodefreedata=c() # Track inode free data for nodes +ifpacketdata=c() # Track interface packet data for nodes +ifoctetdata=c() # Track interface octets data for nodes +ifdropdata=c() # Track interface dropped data for nodes +iferrordata=c() # Track interface errors data for nodes +memstats=c() # Statistics for memory usage +cpustats=c() # Statistics for cpu usage +bootstats=c() # Statistics for boot (launch) times +inodestats=c() # Statistics for inode usage + +# iterate over every set of results (test run) +for (currentdir in resultdirs) { + # For every results file we are interested in evaluating + for (testname in testnames) { + matchdir=paste(inputdir, currentdir, sep="") + matchfile=paste(testname, '\\.json', sep="") + files=list.files(matchdir, pattern=matchfile) + if ( length(files) == 0 ) { + #warning(paste("Pattern [", matchdir, "/", matchfile, "] matched nothing")) + } + + # For every matching results file + for (ffound in files) { + fname=paste(inputdir, currentdir, ffound, sep="") + if ( !file.exists(fname)) { + warning(paste("Skipping non-existent file: ", fname)) + next + } + # Derive the name from the test result dirname + datasetname=basename(currentdir) + + # Import the data + fdata=fromJSON(fname) + # De-nest the test name specific data + shortname=substr(ffound, 1, nchar(ffound)-nchar(".json")) + fdata=fdata[[shortname]] + testname=datasetname + + # Most of the data we are looking for comes in BootResults, so pick it out to make + # referencing easier + br=fdata$BootResults + + ######################################################## + #### Now extract all the pod launch boot data items #### + ######################################################## + local_bootdata=tibble(launch_time=br$launch_time$Result) + local_bootdata=cbind(local_bootdata, n_pods=br$n_pods$Result) + local_bootdata=cbind(local_bootdata, testname=rep(testname, length(local_bootdata$n_pods))) + local_bootdata=cbind(local_bootdata, ns=br$date$ns) + # get the epoch time in seconds for the boot + local_bootdata$epoch = local_bootdata$ns/1000000000 + local_bootdata$s_offset = local_bootdata$epoch - local_bootdata[1,]$epoch + + + ######################################################## + #### Now extract all node performance information ###### + ######################################################## + nu=br$node_util + + # We need to associate a pod count with each result, but you + # get one result per-node, and the JSON does not carry the pod + # count in that table. Walk the node util structure, assigning the + # n_pods value from the boot results over to the list of node util + # entries associated with it - creating a new 'n_pods' field in the + # node util dataframe. + for (n in seq(length(br$n_pods$Result))) { + nu[[n]]$n_pods = br$n_pods$Result[[n]] + } + + # node_util is a list of nested data frames. I'm sure there is some better R'ish + # way of extracting this data maybe with dplyr, map, select or melt, but I can't + # work it out right now, and at least this is semi-readable... + # + # Basically, we are de-listing and flattening the lists of dataframes into a + # singly 'tidy' dataframe... + nodes=do.call("rbind", lapply(nu, "[", "node")) + noschedule=do.call("rbind", lapply(nu, "[", "noschedule")) + n_pods=do.call("rbind", lapply(nu, "[", "n_pods")) + + # and build our rows + local_nodedata=tibble(node=nodes$node) + local_nodedata=cbind(local_nodedata, n_pods=n_pods) + local_nodedata=cbind(local_nodedata, noschedule=noschedule) + local_nodedata=cbind(local_nodedata, testname=rep(testname, length(local_nodedata$node))) + + # Now Calculate some stats. This gets more complicated as we may have n-nodes, + # and we want to show a 'pod average', so we try to assess for all nodes. If + # we have different 'size' nodes in a cluster, that could throw out the result, + # but the only other option would be to try and show every node separately in the + # table. + + # Get a list of all the nodes + nodes=unique(local_nodedata$node) + + memtotal=0 + cputotal=0 + inodetotal=0 + cpu_idle_data=c() + mem_free_data=c() + inode_free_data=c() + interface_packets_data=c() + interface_octets_data=c() + interface_dropped_data=c() + interface_errors_data=c() + # Calculate per-node totals, and tot them up to a global total. + for (n in nodes) { + # Make a frame with just that nodes data in + thisnode=subset(local_nodedata, node %in% c(n)) + + # check if collectd node data has been untarred yet, if not untar + node_dir=paste(inputdir, currentdir, n, sep="") + if ( !file.exists(node_dir)) { + node_tar=paste(inputdir, currentdir, n, ".tar.gz", sep="") + system(paste("mkdir -p", node_dir)) + system(paste("tar -xzf", node_tar, "-C", node_dir)) + } + # all collectd data is under localhost/ + localhost_dir=paste(node_dir, "localhost", sep="/") + + # grab memory data + memory_dir=paste(localhost_dir, "memory", sep="/") + # filename has date on the end, so look for the right file name + freemem_pattern='^memory\\-free' + files=list.files(memory_dir, pattern=freemem_pattern) + + mem_free_csv=paste(memory_dir, files[1], sep="/") + node_mem_free_data=read.csv(mem_free_csv, header=TRUE, sep=",") + node_mem_free_data=cbind(node_mem_free_data, node=rep(n, length(node_mem_free_data$value))) + node_mem_free_data=cbind(node_mem_free_data, noschedule=rep(thisnode[1,]$noschedule, length(node_mem_free_data$value))) + node_mem_free_data=cbind(node_mem_free_data, testname=rep(testname, length(node_mem_free_data$value))) + node_mem_free_data$s_offset = node_mem_free_data$epoch - local_bootdata[1,]$epoch + + mem_free_data=rbind(mem_free_data, node_mem_free_data) + + # grab CPU data + cpu_dir=paste(localhost_dir, "aggregation-cpu-average", sep="/") + # filename has date on the end, so look for the right file name + percent_idle_pattern='^percent\\-idle' + files=list.files(cpu_dir, pattern=percent_idle_pattern) + + cpu_idle_csv=paste(cpu_dir, files[1], sep="/") + node_cpu_idle_data=read.csv(cpu_idle_csv, header=TRUE, sep=",") + node_cpu_idle_data=cbind(node_cpu_idle_data, node=rep(n, length(node_cpu_idle_data$value))) + node_cpu_idle_data=cbind(node_cpu_idle_data, noschedule=rep(thisnode[1,]$noschedule, length(node_cpu_idle_data$value))) + node_cpu_idle_data=cbind(node_cpu_idle_data, testname=rep(testname, length(node_cpu_idle_data$value))) + node_cpu_idle_data$s_offset = node_cpu_idle_data$epoch - local_bootdata[1,]$epoch + + cpu_idle_data=rbind(cpu_idle_data, node_cpu_idle_data) + + # grab inode data + inode_dir=paste(localhost_dir, "df-root", sep="/") + # filename has date on the end, so look for the right file name + inode_free_pattern='^df_inodes\\-free' + files=list.files(inode_dir, pattern=inode_free_pattern) + inode_free_csv=paste(inode_dir, files[1], sep="/") + node_inode_free_data=read.csv(inode_free_csv, header=TRUE, sep=",") + node_inode_free_data=cbind(node_inode_free_data, node=rep(n, length(node_inode_free_data$value))) + node_inode_free_data=cbind(node_inode_free_data, noschedule=rep(thisnode[1,]$noschedule, length(node_inode_free_data$value))) + node_inode_free_data=cbind(node_inode_free_data, testname=rep(testname, length(node_inode_free_data$value))) + node_inode_free_data$s_offset = node_inode_free_data$epoch - local_bootdata[1,]$epoch + + inode_free_data=rbind(inode_free_data, node_inode_free_data) + + # grab interface data + interface_dir_pattern='^interface\\-eno' + files=list.files(localhost_dir, pattern=interface_dir_pattern) + # TODO: handle multiple interfaces, right now just grabbing first + interface_dir=paste(localhost_dir, files[1], sep="/") + + # filename has date on the end, so look for the right file name + interface_packets_pattern='^if_packets' + files=list.files(interface_dir, pattern=interface_packets_pattern) + interface_packets_csv=paste(interface_dir, files[1], sep="/") + node_interface_packets_data=read.csv(interface_packets_csv, header=TRUE, sep=",") + node_interface_packets_data=cbind(node_interface_packets_data, node=rep(n, length(node_interface_packets_data$epoch))) + node_interface_packets_data=cbind(node_interface_packets_data, noschedule=rep(thisnode[1,]$noschedule, length(node_interface_packets_data$epoch))) + node_interface_packets_data=cbind(node_interface_packets_data, testname=rep(testname, length(node_interface_packets_data$epoch))) + node_interface_packets_data$s_offset = node_interface_packets_data$epoch - local_bootdata[1,]$epoch + + interface_packets_data=rbind(interface_packets_data, node_interface_packets_data) + + # filename has date on the end, so look for the right file name + interface_octets_pattern='^if_octets' + files=list.files(interface_dir, pattern=interface_octets_pattern) + interface_octets_csv=paste(interface_dir, files[1], sep="/") + node_interface_octets_data=read.csv(interface_octets_csv, header=TRUE, sep=",") + node_interface_octets_data=cbind(node_interface_octets_data, node=rep(n, length(node_interface_octets_data$epoch))) + node_interface_octets_data=cbind(node_interface_octets_data, noschedule=rep(thisnode[1,]$noschedule, length(node_interface_octets_data$epoch))) + node_interface_octets_data=cbind(node_interface_octets_data, testname=rep(testname, length(node_interface_octets_data$epoch))) + node_interface_octets_data$s_offset = node_interface_octets_data$epoch - local_bootdata[1,]$epoch + + interface_octets_data=rbind(interface_octets_data, node_interface_octets_data) + + # filename has date on the end, so look for the right file name + interface_dropped_pattern='^if_dropped' + files=list.files(interface_dir, pattern=interface_dropped_pattern) + interface_dropped_csv=paste(interface_dir, files[1], sep="/") + node_interface_dropped_data=read.csv(interface_dropped_csv, header=TRUE, sep=",") + node_interface_dropped_data=cbind(node_interface_dropped_data, node=rep(n, length(node_interface_dropped_data$epoch))) + node_interface_dropped_data=cbind(node_interface_dropped_data, noschedule=rep(thisnode[1,]$noschedule, length(node_interface_dropped_data$epoch))) + node_interface_dropped_data=cbind(node_interface_dropped_data, testname=rep(testname, length(node_interface_dropped_data$epoch))) + node_interface_dropped_data$s_offset = node_interface_dropped_data$epoch - local_bootdata[1,]$epoch + + interface_dropped_data=rbind(interface_dropped_data, node_interface_dropped_data) + + # filename has date on the end, so look for the right file name + interface_errors_pattern='^if_errors' + files=list.files(interface_dir, pattern=interface_errors_pattern) + interface_errors_csv=paste(interface_dir, files[1], sep="/") + node_interface_errors_data=read.csv(interface_errors_csv, header=TRUE, sep=",") + node_interface_errors_data=cbind(node_interface_errors_data, node=rep(n, length(node_interface_errors_data$epoch))) + node_interface_errors_data=cbind(node_interface_errors_data, noschedule=rep(thisnode[1,]$noschedule, length(node_interface_errors_data$epoch))) + node_interface_errors_data=cbind(node_interface_errors_data, testname=rep(testname, length(node_interface_errors_data$epoch))) + node_interface_errors_data$s_offset = node_interface_errors_data$epoch - local_bootdata[1,]$epoch + + interface_errors_data=rbind(interface_errors_data, node_interface_errors_data) + + # Do not use the master (non-schedulable) nodes to calculate + # launched pod metrics + if(thisnode[1,]$noschedule == "true") { + next + } + + max_free_mem=max(node_mem_free_data$value) + min_free_mem=min(node_mem_free_data$value) + memtotal = memtotal + (max_free_mem - min_free_mem) + max_idle_cpu=max(node_cpu_idle_data$value) + min_idle_cpu=min(node_cpu_idle_data$value) + cputotal = cputotal + (max_idle_cpu - min_idle_cpu) + max_free_inode=max(node_inode_free_data$value) + min_free_inode=min(node_inode_free_data$value) + inodetotal = inodetotal + (max_free_inode - min_free_inode) + } + + num_pods = local_bootdata$n_pods[length(local_bootdata$n_pods)] + # We get data in b, but want the graphs in Gb. + memtotal = memtotal / (1024*1024*1024) + gb_per_pod = memtotal/num_pods + pod_per_gb = 1/gb_per_pod + + # Memory usage stats. + local_mems = c( + "Test"=testname, + "n"=num_pods, + "Tot_Gb"=round(memtotal, 3), + "avg_Gb"=round(gb_per_pod, 4), + "n_per_Gb"=round(pod_per_gb, 2) + ) + memstats=rbind(memstats, local_mems) + + # cpu usage stats + local_cpus = c( + "Test"=testname, + "n"=num_pods, + "Tot_CPU"=round(cputotal, 3), + "avg_CPU"=round(cputotal/num_pods, 4) + ) + cpustats=rbind(cpustats, local_cpus) + + # launch (boot) stats + local_boots = c( + "Test"=testname, + "n"=num_pods, + "median"=median(na.omit(local_bootdata)$launch_time)/1000, + "min"=min(na.omit(local_bootdata)$launch_time)/1000, + "max"=max(na.omit(local_bootdata)$launch_time)/1000, + "sd"=round(sd(na.omit(local_bootdata)$launch_time)/1000, 4) + ) + + bootstats=rbind(bootstats, local_boots) + + # inode stats + local_inodes = c( + "Test"=testname, + "n"=num_pods, + "Tot_inode"=round(inodetotal, 3), + "avg_inode"=round(inodetotal/num_pods, 4) + ) + inodestats=rbind(inodestats, local_inodes) + } + + # And collect up our rows into our global table of all results + # These two tables *should* be the source of all the data we need to + # process and plot (apart from the stats....) + podbootdata=rbind(podbootdata, local_bootdata, make.row.names=FALSE) + cpuidledata=rbind(cpuidledata, cpu_idle_data) + memfreedata=rbind(memfreedata, mem_free_data) + inodefreedata=rbind(inodefreedata, inode_free_data) + ifpacketdata=rbind(ifpacketdata, interface_packets_data) + ifoctetdata=rbind(ifoctetdata, interface_octets_data) + ifdropdata=rbind(ifdropdata, interface_dropped_data) + iferrordata=rbind(iferrordata, interface_errors_data) + } +} + +# It's nice to show the graphs in Gb, at least for any decent sized test +# run, so make a new column with that pre-divided data in it for us to use. +memfreedata$mem_free_gb = memfreedata$value/(1024*1024*1024) +# And show the boot times in seconds, not ms +podbootdata$launch_time_s = podbootdata$launch_time/1000.0 + +########### Output memory page ############## +mem_stats_plot = suppressWarnings(ggtexttable(data.frame(memstats), + theme=ttheme(base_size=10), + rows=NULL + )) + +mem_line_plot <- ggplot() + + geom_line(data=memfreedata, aes(s_offset, mem_free_gb, colour=interaction(testname, node), group=interaction(testname, node)), alpha=0.3) + + geom_point(data=memfreedata, aes(s_offset, mem_free_gb, colour=interaction(testname, node), group=interaction(testname, node)),alpha=0.5, size=0.5) + + geom_line( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("System Avail (Gb)") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + ggtitle("System Memory free") + + theme(axis.text.x=element_text(angle=90)) + +page1 = grid.arrange( + mem_line_plot, + mem_stats_plot, + ncol=1 + ) + +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output cpu page ############## +cpu_stats_plot = suppressWarnings(ggtexttable(data.frame(cpustats), + theme=ttheme(base_size=10), + rows=NULL + )) + +cpu_line_plot <- ggplot() + + geom_line(data=cpuidledata, aes(x=s_offset, y=value, colour=interaction(testname, node), group=interaction(testname, node)), alpha=0.3) + + geom_point(data=cpuidledata, aes(x=s_offset, y=value, colour=interaction(testname, node), group=interaction(testname, node)), alpha=0.5, size=0.5) + + geom_line( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + xlab("seconds") + + ylab("System CPU Idle (%)") + + ggtitle("System CPU usage") + + theme(axis.text.x=element_text(angle=90)) + +page2 = grid.arrange( + cpu_line_plot, + cpu_stats_plot, + ncol=1 + ) + +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output boot page ############## +boot_stats_plot = suppressWarnings(ggtexttable(data.frame(bootstats), + theme=ttheme(base_size=10), + rows=NULL + )) + +boot_line_plot <- ggplot() + + geom_line( data=podbootdata, aes(n_pods, launch_time_s, colour=testname, group=testname), alpha=0.2) + + xlab("pods") + + ylab("Boot time (s)") + + ggtitle("Pod boot time") + + theme(axis.text.x=element_text(angle=90)) + +page3 = grid.arrange( + boot_line_plot, + boot_stats_plot, + ncol=1 + ) + +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output inode page ############## +inode_stats_plot = suppressWarnings(ggtexttable(data.frame(inodestats), + theme=ttheme(base_size=10), + rows=NULL + )) + +inode_line_plot <- ggplot() + + geom_line(data=inodefreedata, aes(x=s_offset, y=value, colour=interaction(testname, node), group=interaction(testname, node)), alpha=0.2) + + geom_point(data=inodefreedata, aes(x=s_offset, y=value, colour=interaction(testname, node), group=interaction(testname, node)), alpha=0.5, size=0.5) + + geom_line(data=podbootdata, aes(x=s_offset, y=n_pods*10000, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods*10000, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("inodes free") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ./10000, name="pods")) + + ggtitle("inodes free") + + theme(axis.text.x=element_text(angle=90)) + +page4 = grid.arrange( + inode_line_plot, + inode_stats_plot, + ncol=1 + ) + +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output interface page packets and octets ############## +interface_packet_line_plot <- ggplot() + + geom_line(data=ifpacketdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifpacketdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=ifpacketdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifpacketdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("packets") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + ggtitle("interface packets") + + theme(axis.text.x=element_text(angle=90)) + +interface_octet_line_plot <- ggplot() + + geom_line(data=ifoctetdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifoctetdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=ifoctetdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifoctetdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("octets") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + ggtitle("interface octets") + + theme(axis.text.x=element_text(angle=90)) + + +page5 = grid.arrange( + interface_packet_line_plot, + interface_octet_line_plot, + ncol=1 + ) + +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output interface page drops and errors ############## +interface_drop_line_plot <- ggplot() + + geom_line(data=ifdropdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifdropdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=ifdropdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=ifdropdata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("drops") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + ggtitle("interface drops") + + theme(axis.text.x=element_text(angle=90)) + +interface_error_line_plot <- ggplot() + + geom_line(data=iferrordata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=iferrordata, aes(x=s_offset, y=tx, colour=interaction(testname, node, "tx"), group=interaction(testname, node, "tx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=iferrordata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.2, na.rm=TRUE) + + geom_point(data=iferrordata, aes(x=s_offset, y=rx, colour=interaction(testname, node, "rx"), group=interaction(testname, node, "rx")), alpha=0.5, size=0.5, na.rm=TRUE) + + geom_line(data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.2) + + geom_point( data=podbootdata, aes(x=s_offset, y=n_pods, colour=interaction(testname,"pod count"), group=testname), alpha=0.3, size=0.5) + + labs(colour="") + + xlab("seconds") + + ylab("errors") + + scale_y_continuous(labels=comma, sec.axis=sec_axis(~ ., name="pods")) + + ggtitle("interface errors") + + theme(axis.text.x=element_text(angle=90)) + +page6 = grid.arrange( + interface_drop_line_plot, + interface_error_line_plot, + ncol=1 + ) diff --git a/metrics/report/report_dockerfile/dut-details.R b/metrics/report/report_dockerfile/dut-details.R index ad062ff..7a707e5 100755 --- a/metrics/report/report_dockerfile/dut-details.R +++ b/metrics/report/report_dockerfile/dut-details.R @@ -16,7 +16,8 @@ suppressMessages(library(jsonlite)) # to load the data. # A list of all the known results files we might find the information inside. resultsfiles=c( "k8s-parallel.json", - "k8s-scaling.json" + "k8s-scaling.json", + "k8s-rapid.json" ) data=c() diff --git a/metrics/report/report_dockerfile/metrics_report.Rmd b/metrics/report/report_dockerfile/metrics_report.Rmd index 28599b1..d5245dd 100644 --- a/metrics/report/report_dockerfile/metrics_report.Rmd +++ b/metrics/report/report_dockerfile/metrics_report.Rmd @@ -16,8 +16,8 @@ This report was generated using the data from the **`r resultdirs`** results dir # Runtime scaling This [test](https://github.com/clearlinux/cloud-native-setup/metrics/scaling/k8s_scale.sh) -measures the system memory 'free' reduction, CPU idle % and pod boot time as it launches more -and more idle `busybox` pods on a single node Kubernetes cluster. +measures the system memory 'free' reduction, CPU idle %, free inodes, and pod boot time as +it launches more and more idle `busybox` pods on a Kubernetes cluster. > Note: CPU % is measured as a system whole - 100% represents *all* CPUs on the node. @@ -38,6 +38,19 @@ source('parallel.R') \pagebreak +# Runtime scaling rapid +This [test](https://github.com/clearlinux/cloud-native-setup/metrics/scaling/k8s_scale_fast.sh) +uses collectd to asynchronously measure CPU idle %, free memory, pod boot time, free inodes, +and interface stats as it launches more and more idle `busybox` pods on a Kubernetes cluster. + +> Note: CPU % is measured as a system whole - 100% represents *all* CPUs on the node. + +```{r collectd, echo=FALSE, fig.cap="K8S scaling collectd", results='asis'} +source('collectd_scaling.R') +``` + +\pagebreak + # Test setup details This table describes the test system details, as derived from the information contained diff --git a/metrics/scaling/k8s_scale_rapid.sh b/metrics/scaling/k8s_scale_rapid.sh new file mode 100755 index 0000000..98fe8ef --- /dev/null +++ b/metrics/scaling/k8s_scale_rapid.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +# Pull in some common, useful, items +SCRIPT_PATH=$(dirname "$(readlink -f "$0")") +source "${SCRIPT_PATH}/../lib/common.bash" +source "${SCRIPT_PATH}/common.bash" +source "${SCRIPT_PATH}/../collectd/collectd.bash" + +NUM_PODS=${NUM_PODS:-20} +STEP=${STEP:-1} + +LABELVALUE=${LABELVALUE:-gandalf} + +pod_command="[\"tail\", \"-f\", \"/dev/null\"]" + +# Set some default metrics env vars +TEST_ARGS="runtime=${RUNTIME}" +TEST_NAME="k8s rapid" + +# $1 is the launch time in seconds this pod/container took to start up. +# $2 is the number of pod/containers under test +grab_stats() { + local launch_time_ms=$1 + local n_pods=$2 + local cpu_idle=() + local mem_free=() + local total_mem_used=0 + + info "And grab some stats" + + local date_json="$(cat << EOF + "date": { + "ns": $(date +%s%N), + "Date": "$(date -u +"%Y-%m-%dT%T.%3N")" + } +EOF + )" + metrics_json_add_array_fragment "$date_json" + + local pods_json="$(cat << EOF + "n_pods": { + "Result": ${n_pods}, + "Units" : "int" + } +EOF + )" + metrics_json_add_array_fragment "$pods_json" + + local launch_json="$(cat << EOF + "launch_time": { + "Result": $launch_time_ms, + "Units" : "ms" + } +EOF + )" + metrics_json_add_array_fragment "$launch_json" + + # start the node utilization array + metrics_json_start_nested_array + + # TODO move tracking of noschedule tracking to collectd plugin or pull from system setup data + # grab pods in the collectd daemonset + # use 3 for the file descriptor rather than stdin otherwise the sh commands + # in the middle will read the rest of stdin + while read -u 3 name node; do + # look for taint that prevents scheduling + local noschedule=false + local t_match_values=$(kubectl get node ${node} -o json | jq 'select(.spec.taints) | .spec.taints[].effect == "NoSchedule"') + for v in $t_match_values; do + if [[ $v == true ]]; then + noschedule=true + break + fi + done + + info "launch [$launch_time_ms]" + + local util_json="$(cat << EOF + { + "node": "${node}", + "noschedule": "${noschedule}" + } +EOF + )" + + metrics_json_add_nested_array_element "$util_json" + + done 3< <(kubectl get pods --selector name=collectd-pods -o json | jq -r '.items[] | "\(.metadata.name) \(.spec.nodeName)"') + + metrics_json_end_nested_array "node_util" + + metrics_json_close_array_element +} + +init() { + info "Initialising" + + local cmds=("bc" "jq") + check_cmds "${cmds[@]}" + + info "Checking k8s accessible" + local worked=$( kubectl get nodes > /dev/null 2>&1 && echo $? || echo $? ) + if [ "$worked" != 0 ]; then + die "kubectl failed to get nodes" + fi + + info $(get_num_nodes) "k8s nodes in 'Ready' state found" + + k8s_api_init + + # Launch our stats gathering pod + init_stats $wait_time + + # And now we can set up our results storage then... + metrics_json_init "k8s" + save_config +} + +save_config(){ + metrics_json_start_array + + local json="$(cat << EOF + { + "testname": "${TEST_NAME}", + "NUM_PODS": ${NUM_PODS}, + "STEP": ${STEP}, + "wait_time": ${wait_time}, + "delete_wait_time": ${delete_wait_time}, + "settle_time": ${settle_time} + } +EOF +)" + metrics_json_add_array_element "$json" + metrics_json_end_array "Config" +} + +run() { + info "Running test" + + trap cleanup EXIT QUIT KILL + + metrics_json_start_array + + for reqs in $(seq ${STEP} ${STEP} ${NUM_PODS}); do + info "Testing replicas ${reqs} of ${NUM_PODS}" + # Generate the next yaml file + + local runtime_command + if [ -n "$RUNTIME" ]; then + runtime_command="s|@RUNTIMECLASS@|${RUNTIME}|g" + else + runtime_command="/@RUNTIMECLASS@/d" + fi + + local input_template + local generated_file + if [ "$use_api" != "no" ]; then + input_template=$input_json + generated_file=$generated_json + else + input_template=$input_yaml + generated_file=$generated_yaml + fi + + sed -e "s|@REPLICAS@|${reqs}|g" \ + -e $runtime_command \ + -e "s|@DEPLOYMENT@|${deployment}|g" \ + -e "s|@LABEL@|${LABEL}|g" \ + -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + -e "s|@GRACE@|${grace}|g" \ + -e "s#@PODCOMMAND@#${pod_command}#g" \ + < ${input_template} > ${generated_file} + + info "Applying changes" + local start_time=$(date +%s%N) + if [ "$use_api" != "no" ]; then + # If this is the first launch of the deploy, we need to use a different URL form. + if [ $reqs == ${STEP} ]; then + curl -s ${API_ADDRESS}:${API_PORT}/apis/apps/v1/namespaces/default/deployments -XPOST -H 'Content-Type: application/json' -d@${generated_file} > /dev/null + else + curl -s ${API_ADDRESS}:${API_PORT}/apis/apps/v1/namespaces/default/deployments/${deployment} -XPATCH -H 'Content-Type:application/strategic-merge-patch+json' -d@${generated_file} > /dev/null + fi + else + kubectl apply -f ${generated_file} + fi + + kubectl rollout status --timeout=${wait_time}s deployment/${deployment} + local end_time=$(date +%s%N) + local total_milliseconds=$(( (end_time - start_time) / 1000000 )) + info "Took $total_milliseconds ms ($end_time - $start_time)" + + sleep ${settle_time} + grab_stats $total_milliseconds $reqs + done +} + +cleanup() { + info "Cleaning up" + + # First try to save any results we got + metrics_json_end_array "BootResults" + + local start_time=$(date +%s%N) + kubectl delete deployment --wait=true --timeout=${delete_wait_time}s ${deployment} || true + for x in $(seq 1 ${delete_wait_time}); do + local npods=$(kubectl get pods -l=${LABEL}=${LABELVALUE} -o=name | wc -l) + if [ $npods -eq 0 ]; then + echo "All pods have terminated at cycle $x" + local alldied=true + break; + fi + sleep 1 + done + local end_time=$(date +%s%N) + local total_milliseconds=$(( (end_time - start_time) / 1000000 )) + if [ -z "$alldied" ]; then + echo "ERROR: Not all pods died!" + fi + info "Delete Took $total_milliseconds ms ($end_time - $start_time)" + + local json="$(cat << EOF + "Delete": { + "Result": ${total_milliseconds}, + "Units" : "ms" + } +EOF +)" + + metrics_json_add_fragment "$json" + metrics_json_save + + cleanup_stats $delete_wait_time + + k8s_api_shutdown +} + +show_vars() +{ + echo -e "\nEnvironment variables:" + echo -e "\tName (default)" + echo -e "\t\tDescription" + echo -e "\tTEST_NAME (${TEST_NAME})" + echo -e "\t\tCan be set to over-ride the default JSON results filename" + echo -e "\tNUM_PODS (${NUM_PODS})" + echo -e "\t\tNumber of pods to launch" + echo -e "\tSTEP (${STEP})" + echo -e "\t\tNumber of pods to launch per cycle" + echo -e "\twait_time (${wait_time})" + echo -e "\t\tSeconds to wait for pods to become ready" + echo -e "\tdelete_wait_time (${delete_wait_time})" + echo -e "\t\tSeconds to wait for all pods to be deleted" + echo -e "\tsettle_time (${settle_time})" + echo -e "\t\tSeconds to wait after pods ready before taking measurements" + echo -e "\tuse_api (${use_api})" + echo -e "\t\tspecify yes or no to use the API to launch pods" + echo -e "\tgrace (${grace})" + echo -e "\t\tspecify the grace period in seconds for workload pod termination" +} + +help() +{ + usage=$(cat << EOF +Usage: $0 [-h] [options] + Description: + Launch a series of workloads and take memory metric measurements after + each launch. + Options: + -h, Help page. +EOF +) + echo "$usage" + show_vars +} + +main() { + + local OPTIND + while getopts "h" opt;do + case ${opt} in + h) + help + exit 0; + ;; + esac + done + shift $((OPTIND-1)) + init + run + # cleanup will happen at exit due to the shell 'trap' we registered + # cleanup +} + +main "$@"