From 7efe99f13900db306e6f10abcd9dc7e8fbc78485 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Mon, 21 Oct 2019 17:58:50 -0600 Subject: [PATCH] disregard collectd tail data from stats Since collectd is started before the pods are launched and shutdown after the last pod is launched, we gather data outside the pod launch window which can adversely influence the per pod launch stats. This is especially true after the last pod launches as all the pods are then deleted before collectd stops collecting metrics. This patch isolates the collectd data used to only coincide with the pod launch window. And additional change in this patch is to improve the secondary y axis scaling. There was an ill-advised check in previously to force the scale to be at least 1. This does not work well when the pod number is significantly higher than say 100 (the max possible cpu idle value). This patch changes the scaling to be across all data to be graphed. The special condition for interface drops and interface errors, where the data is typically 0. We don't scale by 0. Signed-off-by: David Lyle --- .../report_dockerfile/collectd_scaling.R | 123 ++++++++++++------ 1 file changed, 82 insertions(+), 41 deletions(-) diff --git a/metrics/report/report_dockerfile/collectd_scaling.R b/metrics/report/report_dockerfile/collectd_scaling.R index 7a7b4f4..b6c963f 100755 --- a/metrics/report/report_dockerfile/collectd_scaling.R +++ b/metrics/report/report_dockerfile/collectd_scaling.R @@ -30,15 +30,6 @@ cpustats=c() # Statistics for cpu usage bootstats=c() # Statistics for boot (launch) times inodestats=c() # Statistics for inode usage -# values for scaling the secondary y axes on some graphs -mem_scale=1 -cpu_scale=1 -inode_scale=1 -ip_scale=1 -oct_scale=1 -drop_scale=1 -error_scale=1 - # iterate over every set of results (test run) for (currentdir in resultdirs) { # For every results file we are interested in evaluating @@ -277,43 +268,79 @@ for (currentdir in resultdirs) { next } - max_free_mem=max(node_mem_free_data$value) - min_free_mem=min(node_mem_free_data$value) + # get the epoch time of first and last pod launch + start_time=local_bootdata$epoch[1] + end_time=local_bootdata$epoch[length(local_bootdata$epoch)] + + # get value closest to first pod launch + mem_start_index=Position(function(x) x > start_time, node_mem_free_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(mem_start_index)) { + mem_start_index = 1 + } else if (mem_start_index > 1) { + mem_start_index = mem_start_index - 1 + } + max_free_mem=node_mem_free_data$value[mem_start_index] + + # get value closest to last pod launch + mem_end_index=Position(function(x) x > end_time, node_mem_free_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(mem_end_index)) { + mem_end_index = length(node_mem_free_data$epoch) + } else if (mem_end_index > 1) { + mem_end_index = mem_end_index - 1 + } + min_free_mem=node_mem_free_data$value[mem_end_index] + memtotal = memtotal + (max_free_mem - min_free_mem) - max_idle_cpu=max(node_cpu_idle_data$value) - min_idle_cpu=min(node_cpu_idle_data$value) + + # get value closest to first pod launch + cpu_start_index=Position(function(x) x > start_time, node_cpu_idle_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(cpu_start_index)) { + cpu_start_index = 1 + } else if (cpu_start_index > 1) { + cpu_start_index = cpu_start_index - 1 + } + max_idle_cpu=node_cpu_idle_data$value[cpu_start_index] + + # get value closest to last pod launch + cpu_end_index=Position(function(x) x > end_time, node_cpu_idle_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(cpu_end_index)) { + cpu_end_index = length(node_cpu_idle_data$epoch) + } else if (cpu_end_index > 1) { + cpu_end_index = cpu_end_index - 1 + } + min_idle_cpu=node_cpu_idle_data$value[cpu_end_index] + cputotal = cputotal + (max_idle_cpu - min_idle_cpu) - max_free_inode=max(node_inode_free_data$value) - min_free_inode=min(node_inode_free_data$value) + + # get value closest to first pod launch + inode_start_index=Position(function(x) x > start_time, node_inode_free_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(inode_start_index)) { + inode_start_index = 1 + } else if (inode_start_index > 1) { + inode_start_index = inode_start_index - 1 + } + max_free_inode=node_inode_free_data$value[inode_start_index] + + # get value closest to last pod launch + inode_end_index=Position(function(x) x > end_time, node_inode_free_data$epoch) + # take the reading previous to the index as long as a valid index + if (is.na(inode_end_index)) { + inode_end_index = length(node_cpu_idle_data$epoch) + } else if (inode_end_index > 1) { + inode_end_index = inode_end_index - 1 + } + min_free_inode=node_inode_free_data$value[inode_end_index] + inodetotal = inodetotal + (max_free_inode - min_free_inode) } num_pods = local_bootdata$n_pods[length(local_bootdata$n_pods)] - # calculate scaling for secondary y axis - # the two y scales, in R, must be mathematically related - mem_scale = max(c(mem_scale, - (max(mem_free_data$value) / (1024*1024*1024)) / num_pods)) - cpu_scale = max(c(cpu_scale, - max(cpu_idle_data$value) / num_pods)) - inode_scale = max(c(inode_scale, - max(inode_free_data$value) / num_pods)) - ip_scale = max(c(ip_scale, - max(c(max(interface_packets_data$tx, na.rm=TRUE), - max(interface_packets_data$rx, na.rm=TRUE))) / num_pods)) - oct_scale = max(c(oct_scale, - max(c(max(interface_octets_data$tx, na.rm=TRUE), - max(interface_octets_data$rx, na.rm=TRUE))) / num_pods)) - # drops and scale are often 0, so providing 1 so we won't scale by infinity - drop_scale = max(c(drop_scale, - max(c(1, - max(interface_dropped_data$tx, na.rm=TRUE), - max(interface_dropped_data$rx, na.rm=TRUE))) / num_pods)) - error_scale = max(c(error_scale, - max(c(1, - max(interface_errors_data$tx, na.rm=TRUE), - max(interface_errors_data$rx, na.rm=TRUE))) / num_pods)) - # We get data in b, but want the graphs in Gb. memtotal = memtotal / (1024*1024*1024) gb_per_pod = memtotal/num_pods @@ -380,13 +407,14 @@ memfreedata$mem_free_gb = memfreedata$value/(1024*1024*1024) # And show the boot times in seconds, not ms podbootdata$launch_time_s = podbootdata$launch_time/1000.0 + ########### Output memory page ############## mem_stats_plot = suppressWarnings(ggtexttable(data.frame(memstats), theme=ttheme(base_size=10), rows=NULL )) -#mem_sec_axis_scale= +mem_scale = (max(memfreedata$value) / (1024*1024*1024)) / max(podbootdata$n_pods) mem_line_plot <- ggplot() + geom_line(data=memfreedata, aes(s_offset, mem_free_gb, colour=interaction(testname, node), @@ -424,6 +452,7 @@ cpu_stats_plot = suppressWarnings(ggtexttable(data.frame(cpustats), rows=NULL )) +cpu_scale = max(cpuidledata$value) / max(podbootdata$n_pods) cpu_line_plot <- ggplot() + geom_line(data=cpuidledata, aes(x=s_offset, y=value, colour=interaction(testname, node), @@ -485,6 +514,7 @@ inode_stats_plot = suppressWarnings(ggtexttable(data.frame(inodestats), rows=NULL )) +inode_scale = max(inodefreedata$value) / max(podbootdata$n_pods) inode_line_plot <- ggplot() + geom_line(data=inodefreedata, aes(x=s_offset, y=value, colour=interaction(testname, node), @@ -517,6 +547,8 @@ page4 = grid.arrange( cat("\n\n\\pagebreak\n") ########## Output interface page packets and octets ############## +ip_scale = max(c(max(ifpacketdata$tx, na.rm=TRUE), + max(ifpacketdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods) interface_packet_line_plot <- ggplot() + geom_line(data=ifpacketdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"), @@ -547,6 +579,8 @@ interface_packet_line_plot <- ggplot() + ggtitle("interface packets") + theme(axis.text.x=element_text(angle=90)) +oct_scale = max(c(max(ifoctetdata$tx, na.rm=TRUE), + max(ifoctetdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods) interface_octet_line_plot <- ggplot() + geom_line(data=ifoctetdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"), @@ -577,7 +611,6 @@ interface_octet_line_plot <- ggplot() + ggtitle("interface octets") + theme(axis.text.x=element_text(angle=90)) - page5 = grid.arrange( interface_packet_line_plot, interface_octet_line_plot, @@ -588,6 +621,10 @@ page5 = grid.arrange( cat("\n\n\\pagebreak\n") ########## Output interface page drops and errors ############## +# drops are often 0, so providing 1 so we won't scale by infinity +drop_scale = max(c(1, + max(ifdropdata$tx, na.rm=TRUE), + max(ifdropdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods) interface_drop_line_plot <- ggplot() + geom_line(data=ifdropdata, aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"), @@ -618,6 +655,10 @@ interface_drop_line_plot <- ggplot() + ggtitle("interface drops") + theme(axis.text.x=element_text(angle=90)) +# errors are often 0, so providing 1 so we won't scale by infinity +error_scale = max(c(1, + max(iferrordata$tx, na.rm=TRUE), + max(iferrordata$rx, na.rm=TRUE))) / max(podbootdata$n_pods) interface_error_line_plot <- ggplot() + geom_line(data=iferrordata, aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),