From ccddc591dc22cf5b034d357048bf0053b87998ba Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 4 Sep 2019 17:13:04 -0600 Subject: [PATCH] Adding metric for inode usage Adds support for tracking inode usage on the nodes. Results also added to report. The stats pod is updated as well to use net=host. --- .../report/report_dockerfile/tidy_scaling.R | 46 +++++++++++++++++++ metrics/scaling/k8s_scale.sh | 13 +++++- metrics/scaling/stats.yaml | 1 + 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/metrics/report/report_dockerfile/tidy_scaling.R b/metrics/report/report_dockerfile/tidy_scaling.R index 43105f8..3e56d99 100755 --- a/metrics/report/report_dockerfile/tidy_scaling.R +++ b/metrics/report/report_dockerfile/tidy_scaling.R @@ -22,6 +22,7 @@ nodedata=c() # Track node status data memstats=c() # Statistics for memory usage cpustats=c() # Statistics for cpu usage bootstats=c() # Statistics for boot (launch) times +inodestats=c() # Statistics for inode usage # iterate over every set of results (test run) for (currentdir in resultdirs) { @@ -99,6 +100,11 @@ for (currentdir in resultdirs) { free_df=do.call("rbind", lapply(free, "[[", "mem_free")) used=lapply(nu, "[", "mem_used") used_df=do.call("rbind", lapply(used, "[[", "mem_used")) + ifree=lapply(nu, "[", "inode_free") + ifree_df=do.call("rbind", lapply(ifree, "[[", "inode_free")) + iused=lapply(nu, "[", "inode_used") + iused_df=do.call("rbind", lapply(iused, "[[", "inode_used")) + # and build our rows local_nodedata=tibble(node=nodes$node) @@ -107,6 +113,8 @@ for (currentdir in resultdirs) { local_nodedata=cbind(local_nodedata, idle=idle_df$Result) local_nodedata=cbind(local_nodedata, mem_free=free_df$Result) local_nodedata=cbind(local_nodedata, mem_used=used_df$Result) + local_nodedata=cbind(local_nodedata, inode_free=ifree_df$Result) + local_nodedata=cbind(local_nodedata, inode_used=iused_df$Result) local_nodedata=cbind(local_nodedata, testname=rep(testname, length(local_nodedata$node))) # Now Calculate some stats. This gets more complicated as we may have n-nodes, @@ -120,6 +128,7 @@ for (currentdir in resultdirs) { memtotal=0 cputotal=0 + inodetotal=0 # Calculate per-node totals, and tot them up to a global total. for (n in nodes) { # Make a frame with just that nodes data in @@ -133,6 +142,7 @@ for (currentdir in resultdirs) { memtotal = memtotal + thisnode[nrow(thisnode),]$mem_used cpuused = thisnode[1,]$idle - thisnode[nrow(thisnode),]$idle cputotal = cputotal + cpuused + inodetotal = inodetotal + thisnode[nrow(thisnode),]$inode_used } num_pods = local_bootdata$n_pods[length(local_bootdata$n_pods)] @@ -171,6 +181,15 @@ for (currentdir in resultdirs) { ) bootstats=rbind(bootstats, local_boots) + + # inode stats + local_inodes = c( + "Test"=testname, + "n"=num_pods, + "Tot_inode"=round(inodetotal, 3), + "avg_inode"=round(inodetotal/num_pods, 4) + ) + inodestats=rbind(inodestats, local_inodes) } # And collect up our rows into our global table of all results @@ -268,3 +287,30 @@ page3 = grid.arrange( ncol=1 ) +# pagebreak, as the graphs overflow the page otherwise +cat("\n\n\\pagebreak\n") + +########## Output inode page ############## +inode_stats_plot = suppressWarnings(ggtexttable(data.frame(inodestats), + theme=ttheme(base_size=10), + rows=NULL + )) + +inode_line_plot <- ggplot(data=nodedata, aes(n_pods, + inode_free, + colour=(if (length(resultdirs) > 1) testname else node), + group=interaction(testname, node))) + + labs(colour=colour_label) + + geom_line(alpha=0.2) + + geom_point(aes(shape=node), alpha=0.3, size=0.5) + + xlab("pods") + + ylab("inodes free") + + scale_y_continuous(labels=comma) + + ggtitle("inodes free") + + theme(axis.text.x=element_text(angle=90)) + +page4 = grid.arrange( + inode_line_plot, + inode_stats_plot, + ncol=1 + ) diff --git a/metrics/scaling/k8s_scale.sh b/metrics/scaling/k8s_scale.sh index fd27ca1..f305700 100755 --- a/metrics/scaling/k8s_scale.sh +++ b/metrics/scaling/k8s_scale.sh @@ -37,6 +37,7 @@ TEST_NAME="k8s scaling" declare -a new_pods declare -A node_basemem +declare -A node_baseinode # $1 is the launch time in seconds this pod/container took to start up. # $2 is the number of pod/containers under test @@ -96,20 +97,24 @@ EOF # if you don't tell it the period, you will get the avg since boot, which is not what we want. local cpu_idle=$(kubectl exec -ti $name -- sh -c "mpstat -u 3 1 | tail -1 | awk '{print \$11}'" | sed 's/\r//') local mem_free=$(kubectl exec -ti $name -- sh -c "free | tail -2 | head -1 | awk '{print \$4}'" | sed 's/\r//') + local inode_free=$(kubectl exec -ti $name -- sh -c "df -i | awk '/^overlay/ {print \$4}'" | sed 's/\r//') - info "idle [$cpu_idle] free [$mem_free] launch [$launch_time_ms] node [$node]" + info "idle [$cpu_idle] free [$mem_free] launch [$launch_time_ms] node [$node] inodes_free [$inode_free]" # Annoyingly, it seems sometimes once in a while we don't get an answer! # We should really retry, but for now, make the json valid at least cpu_idle=${cpu_idle:-0} mem_free=${mem_free:-0} + inode_free=${inode_free:-0} # If this is the 0 node instance, store away the base memory value if [ $n_pods -eq 0 ]; then node_basemem[$node]=$mem_free + node_baseinode[$node]=$inode_free fi local mem_used=$((node_basemem[$node]-mem_free)) + local inode_used=$((node_baseinode[$node]-inode_free)) # Only account for memory usage on schedulable nodes if [ $noschedule == false ]; then total_mem_used=$((total_mem_used+mem_used)) @@ -130,6 +135,12 @@ EOF "mem_used": { "Result": ${mem_used}, "Units" : "kb" + }, + "inode_free": { + "Result": ${inode_free} + }, + "inode_used": { + "Result": ${inode_used} } } EOF diff --git a/metrics/scaling/stats.yaml b/metrics/scaling/stats.yaml index 51b5b33..317071e 100644 --- a/metrics/scaling/stats.yaml +++ b/metrics/scaling/stats.yaml @@ -11,6 +11,7 @@ spec: labels: name: stats-pods spec: + hostNetwork: true tolerations: - key: node-role.kubernetes.io/master operator: Exists