From af7b595e4a94b80b9fe1fa94eca8e31fbccec1db Mon Sep 17 00:00:00 2001 From: David Lyle Date: Fri, 2 Aug 2019 13:26:55 -0600 Subject: [PATCH] excluding NoSchedule nodes from workload calculations --- metrics/report/report_dockerfile/scaling.R | 53 ++++++++++++---------- metrics/scaling/bb.json.in | 1 + metrics/scaling/bb.yaml.in | 1 + metrics/scaling/k8s_scale.sh | 13 ++++-- metrics/scaling/stats.yaml | 5 ++ 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/metrics/report/report_dockerfile/scaling.R b/metrics/report/report_dockerfile/scaling.R index 8542ffb..e8b9b7d 100755 --- a/metrics/report/report_dockerfile/scaling.R +++ b/metrics/report/report_dockerfile/scaling.R @@ -65,50 +65,51 @@ for (currentdir in resultdirs) { if (i == 1) { # first iteration provide column name for c1 c1=cbind(node=udata$nodename.node) - c2=cbind(udata$nodename.cpu_idle$Result) - c3=cbind(udata$nodename.mem_free$Result)/(1024*1024) - c4=cbind(rep(i, length(udata$nodename.node))) - c5=cbind(rep(testname, length(udata$nodename.node))) - # FIXME figure out how to add schedule here ### - # then add to cdata below as well ### - # then update calculations to exclude schedule=false ### - + c2=cbind(udata$nodename.noschedule) + c3=cbind(udata$nodename.cpu_idle$Result) + c4=cbind(udata$nodename.mem_free$Result)/(1024*1024) + c5=cbind(rep(i, length(udata$nodename.node))) + c6=cbind(rep(testname, length(udata$nodename.node))) # declare formatted utility data - fudata=cbind(c1,c2,c3,c4,c5) + fudata=cbind(c1,c2,c3,c4,c5,c6) } else { # shift to 0 based indexing index=i-1 - sindex=(index*3)+1 - eindex=sindex+2 + sindex=(index*4)+1 + eindex=sindex+3 # grab 3 columns for next row bind row=cbind(udata[,sindex:eindex]) c1=cbind(node=row[,1]) - c2=cbind(row[,2]$Result) - c3=cbind(row[,3]$Result)/(1024*1024) - c4=cbind(rep(i, length(udata$nodename.node))) - c5=cbind(rep(testname, length(udata$nodename.node))) + c2=cbind(row[,2]) + c3=cbind(row[,3]$Result) + c4=cbind(row[,4]$Result)/(1024*1024) + c5=cbind(rep(i, length(udata$nodename.node))) + c6=cbind(rep(testname, length(udata$nodename.node))) # create the new row (which is actually the number of nodes of rows) - frow=cbind(c1,c2,c3,c4,c5) + frow=cbind(c1,c2,c3,c4,c5,c6) fudata=rbind(fudata,frow) } } - colnames(fudata)=c("node", "cpu_idle", "mem_free", "pod", "testname") + colnames(fudata)=c("node", "noschedule", "cpu_idle", "mem_free", "pod", "testname") # fudata is considered a vector for some reason so converting it to a data.frame fudata=as.data.frame(fudata) # get unique node names nodes=unique(fudata$node) - for (nodename in nodes) { - c1=cbind(subset(fudata,node==nodename)["mem_free"]) - # extra work to name the column from a variable - colnames(c1)=paste(nodename,"_avail_gb", sep="") + c1=cbind(subset(fudata,node==nodename)["noschedule"]) + colnames(c1)=paste(nodename,"_noschedule", sep="") cdata=cbind(cdata, c1) - c2=cbind(subset(fudata,node==nodename)["cpu_idle"]) - colnames(c2)=paste(nodename,"_cpu_idle", sep="") + c2=cbind(subset(fudata,node==nodename)["mem_free"]) + # extra work to name the column from a variable + colnames(c2)=paste(nodename,"_avail_gb", sep="") cdata=cbind(cdata, c2) + + c3=cbind(subset(fudata,node==nodename)["cpu_idle"]) + colnames(c3)=paste(nodename,"_cpu_idle", sep="") + cdata=cbind(cdata, c3) } # convert ms to seconds @@ -130,8 +131,12 @@ for (currentdir in resultdirs) { sdata=data.frame(num_containers=length(cdata[, "boot_time"])) sudata=c() # first (which should be 0-containers) - # FIXME - don't calculate nodes with NoSchedule effect on node-role.kubernetes.io/master taint for (nodename in nodes) { + node_noschedule=paste(nodename, "_noschedule", sep="") + # if workloads are not scheduled on this node, don't include it in the calculations below + if(cdata[, node_noschedule][1] == "true") { + next + } node_avail_gb=paste(nodename, "_avail_gb", sep="") # Work out memory reduction by subtracting last (most consumed) from srdata=cbind(mem_consumed=as.numeric(as.character(cdata[, node_avail_gb][1])) - diff --git a/metrics/scaling/bb.json.in b/metrics/scaling/bb.json.in index 066703b..001d4d5 100644 --- a/metrics/scaling/bb.json.in +++ b/metrics/scaling/bb.json.in @@ -22,6 +22,7 @@ } }, "spec": { + "terminationGracePeriodSeconds": @GRACE@, "runtimeClassName": "@RUNTIMECLASS@", "automountServiceAccountToken": false, "containers": [{ diff --git a/metrics/scaling/bb.yaml.in b/metrics/scaling/bb.yaml.in index fa49f75..ce427d4 100644 --- a/metrics/scaling/bb.yaml.in +++ b/metrics/scaling/bb.yaml.in @@ -24,6 +24,7 @@ spec: run: busybox @LABEL@: @LABELVALUE@ spec: + terminationGracePeriodSeconds: @GRACE@ runtimeClassName: @RUNTIMECLASS@ automountServiceAccountToken: false containers: diff --git a/metrics/scaling/k8s_scale.sh b/metrics/scaling/k8s_scale.sh index 5f4197f..a729b79 100755 --- a/metrics/scaling/k8s_scale.sh +++ b/metrics/scaling/k8s_scale.sh @@ -29,6 +29,7 @@ wait_time=${wait_time:-30} delete_wait_time=${delete_wait_time:-600} settle_time=${settle_time:-5} use_api=${use_api:-yes} +grace=${grace:-30} # Set some default metrics env vars TEST_ARGS="runtime=${RUNTIME}" @@ -68,11 +69,11 @@ EOF # in the middle will read the rest of stdin while read -u 3 name node; do # look for taint that prevents scheduling - local schedule=false + local noschedule=false local t_match_values=$(kubectl get node ${node} -o json | jq 'select(.spec.taints) | .spec.taints[].effect == "NoSchedule"') - for v in t_match_values; do - if [[ v == true ]]; then - schedule=true + for v in $t_match_values; do + if [[ $v == true ]]; then + noschedule=true break fi done @@ -91,6 +92,7 @@ EOF local util_json="$(cat << EOF { "node": "${node}", + "noschedule": "${noschedule}", "cpu_idle": { "Result": ${cpu_idle}, "Units" : "%" @@ -198,6 +200,7 @@ run() { -e "s|@DEPLOYMENT@|${deployment}|g" \ -e "s|@LABEL@|${LABEL}|g" \ -e "s|@LABELVALUE@|${LABELVALUE}|g" \ + -e "s|@GRACE@|${grace}|g" \ < ${input_template} > ${generated_file} info "Applying changes" @@ -281,6 +284,8 @@ show_vars() echo -e "\t\tSeconds to wait after pods ready before taking measurements" echo -e "\tuse_api (${use_api})" echo -e "\t\tspecify yes or no to use the API to launch pods" + echo -e "\tgrace (${grace})" + echo -e "\t\tspecify the grace period in seconds for workload pod termination" } help() diff --git a/metrics/scaling/stats.yaml b/metrics/scaling/stats.yaml index 0b0053e..51b5b33 100644 --- a/metrics/scaling/stats.yaml +++ b/metrics/scaling/stats.yaml @@ -11,6 +11,11 @@ spec: labels: name: stats-pods spec: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + terminationGracePeriodSeconds: 0 containers: - name: stats image: busybox