Fixed code for no_proxy in setup_system.sh

The sed command for updating no-proxy settings in the proxy.sh file was
not working. I made it match the above command which was working for
/etc/environment.

Signed-off-by: Craig Sterrett <craig.sterrett@intel.com>

clr-k8s-examples/0-canal/overlays/v3.10/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+

clr-k8s-examples/0-cilium/overlays/v1.6/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml

clr-k8s-examples/0-flannel/overlays/16b0fe66285d1ad1f42b154ab852682f6fafb1a7/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/5-ingres-lb/overlays/nginx-0.25.0/kustomization.yaml

@@ -6,7 +6,7 @@ patchesJson6902:
   # adds "networking.k8s.io" to ClusterRole's apiGroups
   - target:
       group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
       kind: ClusterRole
       name: nginx-ingress-clusterrole
     path: patch_clusterrole.yaml

clr-k8s-examples/7-rook/overlays/v0.8.3/patch_operator.yaml

@@ -1,5 +1,5 @@
 # operator
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: rook-ceph-operator

clr-k8s-examples/8-kata/overlays/1.8.2-kernel-config/kustomization.yaml

@@ -4,3 +4,8 @@ resources:
   - packaging/kata-deploy/k8s-1.14/kata-fc-runtimeClass.yaml
   - packaging/kata-deploy/k8s-1.14/kata-qemu-runtimeClass.yaml
 
+images:
+  # change 'latest' to specified version
+  - name: katadocker/kata-deploy
+    newName: katadocker/kata-deploy
+    newTag: 1.8.2

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
+apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   # name must match the spec fields below, and be in the form: <plural>.<group>

clr-k8s-examples/DEVELOP.md

@@ -119,7 +119,7 @@ patchesJson6902:
   # adds "networking.k8s.io" to ClusterRole's apiGroups
   - target:
       group: rbac.authorization.k8s.io
-      version: v1beta1
+      version: v1
       kind: ClusterRole
       name: nginx-ingress-clusterrole
     path: patch_clusterrole.yaml
@@ -137,7 +137,7 @@ itself contains the operation to perform, target path and value. The `rules/3/ap
 operation (in this case "add") at the `apiGroups:` list found under the 4th list item of `rules:`. 
 
 ```yaml
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: nginx-ingress-clusterrole

clr-k8s-examples/README.md

@@ -13,8 +13,8 @@ Follow instructions in the [Vagrant docs](https://www.vagrantup.com/intro/gettin
 
 Or, follow our [detailed steps](vagrant.md)
 
-Now you have a 3 node cluster up and running. Each of them have 2 vCPU, 4GB Memory, 2x10GB disks, 1 additional private network.
+Now you have a 3 node cluster up and running. Each of them have 4 vCPU, 8GB Memory, 2x10GB disks, 1 additional private network.
-Customize the setup using environment variables. E.g., `NODES=1 MEMORY=8192 CPUS=8 vagrant up --provider=libvirt`
+Customize the setup using environment variables. E.g., `NODES=2 MEMORY=16384 CPUS=8 vagrant up --provider=libvirt`
 
 To login to the master node and change to this directory
 
@@ -38,6 +38,11 @@ This script ensures the following
 script uses the runtime specified in the `RUNNER` environment variable and defaults to `crio`. To use the
 `containerd` runtime, set the `RUNNER` environment variable to `containerd`.
 
+### Specify a version of Clear Linux
+
+To specify a particular version of Clear Linux to use, set the CLRK8S_CLR_VER environment variable to the desired
+version before starting setup_system.sh (e.g. `CLRK8S_CLR_VER=31400 ./setup_system.sh`)
+
 ### Configuration for high numbers of pods per node
 
 In order to enable running greater than 110 pods per node, set the environment
@@ -65,6 +70,11 @@ you need to setup other cluster wide properties.
 There are different flavors to install, run `./create_stack.sh help` to get
 more information.
 
+> NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export
+the necessary environment variables if needed to be changed. By default it will use
+`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the 
+Vagrant.
+
 ```bash
 # default shows help
 ./create_stack.sh <subcommand>

clr-k8s-examples/Vagrantfile

@@ -6,8 +6,8 @@ require 'ipaddr'
 require 'securerandom'
 
 $num_instances = (ENV['NODES'] || 3).to_i
-$cpus = (ENV['CPUS'] || 2).to_i
+$cpus = (ENV['CPUS'] || 4).to_i
-$memory = (ENV['MEMORY'] || 4096).to_i
+$memory = (ENV['MEMORY'] || 8192).to_i
 $disks = 2
 # Using folder prefix instead of uuid until vagrant-libvirt fixes disk cleanup
 $disk_prefix = File.basename(File.dirname(__FILE__), "/")
@@ -23,6 +23,7 @@ $driveletters = ('a'..'z').to_a
 $setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
 $runner = ENV.has_key?('RUNNER') ? ENV['RUNNER'].to_s : "crio".to_s
 $high_pod_count = ENV.has_key?('HIGH_POD_COUNT') ? ENV['HIGH_POD_COUNT'].to_s : ""
+$CLRK8S_CLR_VER = ENV.has_key?('CLRK8S_CLR_VER') ? ENV['CLRK8S_CLR_VER'].to_s : ""
 if !(["crio","containerd"].include? $runner)
   abort("it's either crio or containerd. Cannot do anything else")
 end
@@ -97,7 +98,7 @@ Vagrant.configure("2") do |config|
       c.vm.provider :virtualbox do |_, override|
         override.vm.provision "shell", privileged: true, inline: "sudo mkdir -p /etc/profile.d; echo export MASTER_IP=#{$hosts["clr-01"]} > /etc/profile.d/cnsetup.sh"
       end
-      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner, "HIGH_POD_COUNT" => $high_pod_count}
+      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner, "HIGH_POD_COUNT" => $high_pod_count, "CLRK8S_CLR_VER" => $CLRK8S_CLR_VER}
       if $setup_fc
         if $runner == "crio".to_s
         c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"

clr-k8s-examples/admit-kata/deploy/webhook-registration.yaml.tpl

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-apiVersion: admissionregistration.k8s.io/v1beta1
+apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   name: pod-annotate-webhook

clr-k8s-examples/admit-kata/deploy/webhook.yaml

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Deployment
 metadata:
   name: pod-annotate-webhook

clr-k8s-examples/create_stack.sh

@@ -12,14 +12,26 @@ CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 : ${TOKEN:=}
 : ${MASTER_IP:=}
+: ${CERT_SANS:=}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
 
 # versions
-CANAL_VER="${CLRK8S_CANAL_VER:-v3.9}"
+CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}"
+CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.6}"
+FLANNEL_VER="${CLRK8S_FLANNEL_VER:-16b0fe66285d1ad1f42b154ab852682f6fafb1a7}"
 K8S_VER="${CLRK8S_K8S_VER:-}"
+KATA_VER="${CLRK8S_KATA_VER:-1.8.2-kernel-config}"
 ROOK_VER="${CLRK8S_ROOK_VER:-v1.1.1}"
 METRICS_VER="${CLRK8S_METRICS_VER:-v0.3.5}"
+DASHBOARD_VER="${CLRK8S_DASHBOARD_VER:-v2.0.0-beta2}"
+INGRES_VER="${CLRK8S_INGRES_VER:-nginx-0.25.1}"
+EFK_VER="${CLRK8S_EFK_VER:-v1.15.1}"
+METALLB_VER="${CLRK8S_METALLB_VER:-v0.8.1}"
+NPD_VER="${CLRK8S_NPD_VER:-v0.6.6}"
 PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-f458e85e5d7675f7bc253072e1b4c8892b51af0f}"
+CNI=${CLRK8S_CNI:-"canal"}
+RUNNER=${CLRK8S_RUNNER:-"crio"}
+NFD_VER="${CLRK8S_NFD_VER:-v0.4.0}"
 
 function print_usage_exit() {
 	exit_code=${1:-0}
@@ -45,17 +57,30 @@ trap finish EXIT
 function cluster_init() {
 	# Config replacements
 	if ! [ -z ${TOKEN} ]; then
-		sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm/base/kubeadm.yaml
+		sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml
 	fi
 	if ! [ -z ${MASTER_IP} ]; then
-		sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm/base/kubeadm.yaml
+		sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml
 	fi
-	if [[ -n "$K8S_VER" && $(grep -c kubernetesVersion ./kubeadm/base/kubeadm.yaml) -eq 0 ]]; then
+	if [[ -n "$K8S_VER" && $(grep -c kubernetesVersion ./kubeadm.yaml) -eq 0 ]]; then
-		sed -i "s/ClusterConfiguration/ClusterConfiguration\nkubernetesVersion: ${K8S_VER}/g" ./kubeadm/base/kubeadm.yaml
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\nkubernetesVersion: ${K8S_VER}/g" ./kubeadm.yaml
+	fi
+	if [[ -n "$CERT_SANS" ]]; then
+		if [[ $(grep -c certSANs ./kubeadm.yaml) -gt 0 ]]; then
+			sed -i '/certSANs/,/[a-zA-Z]*:/{//!d}' ./kubeadm.yaml
+		else
+			sed -i "/ClusterConfiguration/a apiServer:\\n  certSANs:" ./kubeadm.yaml
+		fi
+		for CERT_SAN in ${CERT_SANS[@]}; do
+			sed -i "/certSANs/a \ \ - ${CERT_SAN}" ./kubeadm.yaml
+		done
 	fi
-	# Config patches
 	if [[ -n "${HIGH_POD_COUNT}" ]]; then
-		echo "$(kubectl kustomize ./kubeadm/high-pod-count/)" > ./kubeadm/base/kubeadm.yaml
+		# increase limits in kubelet
+		sed -i "/KubeletConfiguration/a maxOpenFiles\: 1048576" ./kubeadm.yaml
+		sed -i "/KubeletConfiguration/a maxPods\: 5000" ./kubeadm.yaml
+		# increase the address range per node
+		sed -i "/ClusterConfiguration/a controllerManager:\\n  extraArgs:\\n    node-cidr-mask-size: \"20\"" ./kubeadm.yaml
 	fi
 	#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
 	#to enable the RuntimeClass featuregate
@@ -63,7 +88,7 @@ function cluster_init() {
 		echo "/var/lib/etcd exists! skipping init."
 		return
 	fi
-	sudo -E kubeadm init --config=./kubeadm/base/kubeadm.yaml
+	sudo -E kubeadm init --config=./kubeadm.yaml
 
 	rm -rf "${HOME}/.kube"
 	mkdir -p "${HOME}/.kube"
@@ -86,7 +111,7 @@ function cluster_init() {
 }
 
 function kata() {
-	KATA_VER=${1:-1.8.2-kernel-config}
+	KATA_VER=${1:-$KATA_VER}
 	KATA_URL="https://github.com/kata-containers/packaging.git"
 	KATA_DIR="8-kata"
 	get_repo "${KATA_URL}" "${KATA_DIR}/overlays/${KATA_VER}"
@@ -96,6 +121,8 @@ function kata() {
 }
 
 function cni() {
+	case "$CNI" in
+	canal)
 		# note version is not semver
 		CANAL_VER=${1:-$CANAL_VER}
 		CANAL_URL="https://docs.projectcalico.org/${CANAL_VER}/manifests"
@@ -112,7 +139,30 @@ function cni() {
 		fi
 		# canal doesnt pass kustomize validation
 		kubectl apply -k "${CANAL_DIR}/overlays/${CANAL_VER}" --validate=false
+		;;
+	flannel)
+		FLANNEL_VER=${1:-$FLANNEL_VER}
+		FLANNEL_URL="https://github.com/coreos/flannel"
+		FLANNEL_DIR="0-flannel"
 		
+		get_repo "${FLANNEL_URL}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
+        	set_repo_version "${FLANNEL_VER}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}/flannel"
+        	kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
+		;;
+	cilium)
+		CILIUM_VER=${1:-$CILIUM_VER}
+		CILIUM_URL="https://github.com/cilium/cilium.git"
+		CILIUM_DIR="0-cilium"
+
+		get_repo "${CILIUM_URL}" "${CILIUM_DIR}/overlays/${CILIUM_VER}"
+		set_repo_version "${CILIUM_VER}" "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/"
+		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set global.containerRuntime.integration="$RUNNER" | kubectl apply -f -
+		;;
+	*)
+		echo"Unknown cni $CNI"
+		exit 1
+		;;
+	esac
 }
 
 function metrics() {
@@ -194,7 +244,7 @@ function monitoring() {
 }
 
 function dashboard() {
-	DASHBOARD_VER=${1:-v2.0.0-beta2}
+	DASHBOARD_VER=${1:-$DASHBOARD_VER}
 	DASHBOARD_URL="https://github.com/kubernetes/dashboard.git"
 	DASHBOARD_DIR="2-dashboard"
 	get_repo "${DASHBOARD_URL}" "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}"
@@ -203,7 +253,7 @@ function dashboard() {
 }
 
 function ingres() {
-	INGRES_VER=${1:-nginx-0.25.1}
+	INGRES_VER=${1:-$INGRES_VER}
 	INGRES_URL="https://github.com/kubernetes/ingress-nginx.git"
 	INGRES_DIR="5-ingres-lb"
 	get_repo "${INGRES_URL}" "${INGRES_DIR}/overlays/${INGRES_VER}"
@@ -212,7 +262,7 @@ function ingres() {
 }
 
 function efk() {
-	EFK_VER=${1:-v1.15.1}
+	EFK_VER=${1:-$EFK_VER}
 	EFK_URL="https://github.com/kubernetes/kubernetes.git"
 	EFK_DIR="3-efk"
 	get_repo "${EFK_URL}" "${EFK_DIR}/overlays/${EFK_VER}"
@@ -222,7 +272,7 @@ function efk() {
 }
 
 function metallb() {
-	METALLB_VER=${1:-v0.8.1}
+	METALLB_VER=${1:-$METALLB_VER}
 	METALLB_URL="https://github.com/danderson/metallb.git"
 	METALLB_DIR="6-metal-lb"
 	get_repo "${METALLB_URL}" "${METALLB_DIR}/overlays/${METALLB_VER}"
@@ -230,9 +280,8 @@ function metallb() {
 	kubectl apply -k "${METALLB_DIR}/overlays/${METALLB_VER}"
 
 }
-
 function npd() {
-	NPD_VER=${1:-v0.6.6}
+	NPD_VER=${1:-$NPD_VER}
 	NPD_URL="https://github.com/kubernetes/node-problem-detector.git"
 	NPD_DIR="node-problem-detector"
 	get_repo "${NPD_URL}" "${NPD_DIR}/overlays/${NPD_VER}"
@@ -240,6 +289,16 @@ function npd() {
 	kubectl apply -k "${NPD_DIR}/overlays/${NPD_VER}"
 }
 
+# node feature discovery
+function nfd() {
+	NFD_VER=${1:-$NFD_VER}
+	NFD_URL="https://github.com/kubernetes-sigs/node-feature-discovery.git"
+	NFD_DIR="node-feature-discovery"
+	get_repo "${NFD_URL}" "${NFD_DIR}/overlays/${NFD_VER}"
+	set_repo_version "${NFD_VER}" "${NFD_DIR}/overlays/${NFD_VER}/node-feature-discovery"
+	kubectl apply -k "${NFD_DIR}/overlays/${NFD_VER}"
+}
+
 function miscellaneous() {
 
 	# dashboard
@@ -287,7 +346,7 @@ function set_repo_version() {
 	pushd "$(pwd)"
 	cd "${path}"
 	git fetch origin "${ver}"
-	git checkout "${ver}"
+	git -c advice.detachedHead=false checkout "${ver}"
 	popd
 
 }
@@ -306,6 +365,7 @@ command_handlers[storage]=storage
 command_handlers[monitoring]=monitoring
 command_handlers[metallb]=metallb
 command_handlers[npd]=npd
+command_handlers[nfd]=nfd
 
 declare -A command_help
 command_help[init]="Only inits a cluster using kubeadm"
@@ -313,6 +373,7 @@ command_help[cni]="Setup network for running cluster"
 command_help[minimal]="init + cni +  kata + metrics"
 command_help[all]="minimal + storage + monitoring + miscellaneous"
 command_help[help]="show this message"
+command_help[nfd]="node feature discovery"
 
 cd "${SCRIPT_DIR}"
 

clr-k8s-examples/hack/update_checker.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+
+###
+# update_checker.sh
+# Parses create_stack.sh for urls and versions
+# Curls urls for latest version and reports the comparison
+##
+
+# first argument is path to create_stach.sh
+COMPONENT_FILE="${1:-$create_stack.sh}"
+# set CLRK8S_DEBUG=1 for debug output
+DEBUG=${CLRK8S_DEBUG:-0}
+# set CLRK8S_NO_COLOR=1 for no colors
+NO_COLOR=${CLRK8S_NO_COLOR:-""}
+# set CLRK8S_ALL=1 for all results, not just changed
+ALL=${CLRK8S_ALL:-""}
+# add components to skip (not check)
+# - canal doesn't use git repo tags for revisions
+declare -a COMPONENT_SKIP=( CANAL )
+
+# internal vars
+declare -A COMPONENT_VER
+declare -A COMPONENT_URL
+LATEST_URL=""
+
+# usage prints help and exit
+function usage(){
+	echo "Compare default component versions to latest release"
+	echo "usage: update_checker.sh <path to create_stack.sh>"
+	exit 0
+}
+# log echoes to stdout
+function log(){
+	echo "$1"
+}
+# debug echoes to stdout if debug is enabled
+function debug(){
+	if [[ "${DEBUG}" -ne 0 ]]; then
+	  echo "$1"
+	fi
+}
+# extract_component_data scans file for component versions and urls and add them to maps
+function extract_component_data(){
+	file=${1:-$COMPONENT_FILE}
+	name=""
+	version=""
+	url=""
+	while read -r line
+	do
+		# versions
+	  if [[ $line =~ "_VER=" ]]; then
+			debug "Found component version $line"
+			name=${line%%_*}
+			if [[ "$COMPONENT_SKIP" =~ (^|[[:space:]])"$name"($|[[:space:]]) ]]; then
+				debug "Skipping component $name"
+				continue
+			fi
+			versions=${line#*=}
+			if [[ $versions =~ ":-" ]]; then
+				version=${line#*:-}
+			fi
+			# cleanup value
+			version=${version%\}}
+			version=${version%\}\"}
+			if [[ -n "$name" && ${COMPONENT_VER[$name]} == "" ]]; then
+				debug "Adding component $name=$version to COMPONENT_VER"
+				COMPONENT_VER[$name]=$version
+	  	fi
+	  fi
+
+	  # urls
+	  if [[ $line =~ "_URL=" ]]; then
+	    debug "Found component URL $line"
+			name=${line%%_*}
+			if [[ $COMPONENT_SKIP =~ (^|[[:space:]])"$name"($|[[:space:]]) ]]; then
+				debug "Skipping component $name"
+				continue
+			fi
+			urls=${line#*=}
+
+			if [[ $urls =~ ":-" ]]; then
+				urls=${line#*:-}
+			fi
+			# cleanup value
+			url=${urls%\"}
+			url=${url#\"}
+			if [[ -n "$name" && ${COMPONENT_URL[$name]} == "" ]]; then
+				debug "Adding component $name=$url to COMPONENT_URL"
+				COMPONENT_URL[$name]=$url
+	  	fi
+	  fi
+
+	done < $file
+
+}
+# resolve_latest_url extracts the real release/latest url from a repo url
+function resolve_latest_url(){
+	repo=$1
+	url=${repo%.git*}/releases/latest
+	LATEST_URL=$(curl -Ls -o /dev/null -w %{url_effective} $url)
+	if [[ "$?" -gt 0 ]]; then
+		echo "curl error, exiting."
+		exit 1
+	fi
+}
+# function_exists checks if a function exists
+function function_exists() {
+    declare -f -F "$1" > /dev/null
+    return $?
+}
+function report(){
+	if [[ -z $NO_COLOR ]]; then
+		BOLD="\e[1m\e[33m"
+		BOLD_OFF="\e[0m"
+  fi
+  mode="changed"
+  out=""
+  if [[ -n "$1" ]]; then
+  	mode="all"
+  fi
+	out+="\n"
+	out+="Components ($mode)\n"
+	out+="--------------------------\n"
+	echo -e $out
+	out="NAME CURRENT LATEST\n"
+	# loop thru each url, get latest version and report
+	for k in "${!COMPONENT_URL[@]}";
+	do
+		name=$k
+		resolve_latest_url "${COMPONENT_URL[$k]}"
+		latest_url="$LATEST_URL"
+		latest_ver="${latest_url#*tag/}"
+		current_ver="${COMPONENT_VER[$name]}"
+		if [[ "${current_ver}" != "${latest_ver}" ]]; then
+			out+="$name $current_ver $BOLD $latest_ver $BOLD_OFF\n";
+		fi
+		if [[ "${current_ver}" == "${latest_ver}" && -n $ALL ]]; then
+			out+="$name $current_ver $latest_ver\n";
+		fi
+	done;
+	echo -e "${out}" | column -t
+	if [[ "${#COMPONENT_SKIP[@]}" -gt 0 ]]; then
+		echo "---"
+		echo "WARNING: Skipped comparisions for the following components:"
+		for s in "${!COMPONENT_SKIP[@]}"
+		do
+			echo "${COMPONENT_SKIP[$s]}"
+		done
+	fi
+}
+###
+# Main
+##
+
+# print help if no args
+if [[ "$#" -eq 0 ]]; then
+	usage
+fi
+
+# get the versions
+extract_component_data "$1"
+# output
+report "${ALL}"
+
+
+

clr-k8s-examples/kubeadm.yaml

@@ -1,12 +1,8 @@
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
-metadata:
-  name: init-config
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
-metadata:
-  name: kubelet-config
 cgroupDriver: systemd
 # Allowing for CPU pinning and isolation in case of guaranteed QoS class
 cpuManagerPolicy: static
@@ -19,8 +15,6 @@ kubeReserved:
 ---
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
-metadata:
-  name: cluster-config
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16

clr-k8s-examples/kubeadm/base/kustomization.yaml

@@ -1,2 +0,0 @@
-resources:
-- kubeadm.yaml

clr-k8s-examples/kubeadm/high-pod-count/cluster-config.json

@@ -1,3 +0,0 @@
-[
-  {"op": "add", "path": "/controllerManager", "value": {"extraArgs": {"node-cidr-mask-size": "20"}}}
-]

clr-k8s-examples/kubeadm/high-pod-count/kubelet-config.json

@@ -1,4 +0,0 @@
-[
-  {"op": "add", "path": "/maxOpenFiles", "value": 1048576},
-  {"op": "add", "path": "/maxPods", "value": 5000}
-]

clr-k8s-examples/kubeadm/high-pod-count/kustomization.yaml

@@ -1,17 +0,0 @@
-bases:
-- ../base/
-patchesJson6902:
-# increase limits in kubelet
-- target:
-    group: kubelet.config.k8s.io
-    version: v1beta1
-    kind: KubeletConfiguration
-    name: kubelet-config
-  path: kubelet-config.json
-# increase the address range per node
-- target:
-    group: kubeadm.k8s.io
-    version: v1beta1
-    kind: ClusterConfiguration
-    name: cluster-config
-  path: cluster-config.json

clr-k8s-examples/node-feature-discovery/overlays/v0.4.0/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - node-feature-discovery/nfd-daemonset-combined.yaml.template
+  - node-feature-discovery/nfd-worker-daemonset.yaml.template
+
+

clr-k8s-examples/reset_stack.sh

@@ -52,3 +52,4 @@ sudo systemctl is-enabled containerd && sudo systemctl restart containerd
 sudo systemctl restart kubelet
 
 reset_cluster
+sudo -E bash -c "rm -rf /var/lib/containerd/*"

clr-k8s-examples/setup_system.sh

@@ -5,15 +5,21 @@ set -o nounset
 
 # global vars
 CLRK8S_OS=${CLRK8S_OS:-""}
+CLR_VER=${CLRK8S_CLR_VER:-""}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
 
 # set no proxy
-ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12"
+ADD_NO_PROXY=".svc,10.244.0.0/16,10.96.0.0/12"
 ADD_NO_PROXY+=",$(hostname -I | sed 's/[[:space:]]/,/g')"
 : "${RUNNER:=crio}"
 
 # update os version
 function upate_os_version() {
+	if [[ -n "${CLR_VER}" ]];
+	then
+		sudo swupd repair -m "${CLR_VER}" --picky
+		return
+	fi
 	sudo swupd update
 }
 
@@ -145,7 +151,7 @@ function setup_proxy() {
 		sed_val=${ADD_NO_PROXY//\//\\/}
 		[ -f /etc/environment ] && sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
 		if [ -f /etc/profile.d/proxy.sh ]; then
-			sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+			sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/profile.d/proxy.sh
 		else
 			echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
 		fi

clr-k8s-examples/tests/deploy-svc-ing/test-ingress-kata.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
   name: php-apache-kata

clr-k8s-examples/tests/deploy-svc-ing/test-ingress-runc.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1
 kind: Ingress
 metadata:
   name: php-apache-runc

metrics/report/report_dockerfile/Dockerfile

@@ -13,7 +13,7 @@
 # We would have used the 'verse' base, that already has some of the docs processing
 # installed, but I could not figure out how to add in the extra bits we needed to
 # the lite tex version is uses.
-FROM rocker/tidyverse
+FROM rocker/tidyverse:latest
 
 # Version of the Dockerfile
 LABEL DOCKERFILE_VERSION="1.0"

metrics/report/report_dockerfile/collectd_scaling.R

@@ -30,15 +30,6 @@ cpustats=c()		# Statistics for cpu usage
 bootstats=c()		# Statistics for boot (launch) times
 inodestats=c()		# Statistics for inode usage
 
-# values for scaling the secondary y axes on some graphs
-mem_scale=1
-cpu_scale=1
-inode_scale=1
-ip_scale=1
-oct_scale=1
-drop_scale=1
-error_scale=1
-
 # iterate over every set of results (test run)
 for (currentdir in resultdirs) {
 	# For every results file we are interested in evaluating
@@ -140,8 +131,9 @@ for (currentdir in resultdirs) {
 				# filename has date on the end, so look for the right file name
 				freemem_pattern='^memory\\-free'
 				files=list.files(memory_dir, pattern=freemem_pattern)
-
+				# collectd csv plugin starts a new file for each day of data collected
-				mem_free_csv=paste(memory_dir, files[1], sep="/")
+				for(file in files) {
+					mem_free_csv=paste(memory_dir, file, sep="/")
 					node_mem_free_data=read.csv(mem_free_csv, header=TRUE, sep=",")
 					node_mem_free_data=cbind(node_mem_free_data,
 											 node=rep(n, length(node_mem_free_data$value)))
@@ -150,14 +142,15 @@ for (currentdir in resultdirs) {
 					node_mem_free_data$s_offset = node_mem_free_data$epoch - local_bootdata[1,]$epoch
 
 					mem_free_data=rbind(mem_free_data, node_mem_free_data)
+				}
 
 				# grab CPU data
 				cpu_dir=paste(localhost_dir, "aggregation-cpu-average", sep="/")
 				# filename has date on the end, so look for the right file name
 				percent_idle_pattern='^percent\\-idle'
 				files=list.files(cpu_dir, pattern=percent_idle_pattern)
-
+				for(file in files) {
-				cpu_idle_csv=paste(cpu_dir, files[1], sep="/")
+					cpu_idle_csv=paste(cpu_dir, file, sep="/")
 					node_cpu_idle_data=read.csv(cpu_idle_csv, header=TRUE, sep=",")
 					node_cpu_idle_data=cbind(node_cpu_idle_data,
 											 node=rep(n, length(node_cpu_idle_data$value)))
@@ -166,13 +159,15 @@ for (currentdir in resultdirs) {
 					node_cpu_idle_data$s_offset = node_cpu_idle_data$epoch - local_bootdata[1,]$epoch
 
 					cpu_idle_data=rbind(cpu_idle_data, node_cpu_idle_data)
+				}
 
 				# grab inode data
 				inode_dir=paste(localhost_dir, "df-root", sep="/")
 				# filename has date on the end, so look for the right file name
 				inode_free_pattern='^df_inodes\\-free'
 				files=list.files(inode_dir, pattern=inode_free_pattern)
-				inode_free_csv=paste(inode_dir, files[1], sep="/")
+				for(file in files) {
+					inode_free_csv=paste(inode_dir, file, sep="/")
 					node_inode_free_data=read.csv(inode_free_csv, header=TRUE, sep=",")
 					node_inode_free_data=cbind(node_inode_free_data,
 											   node=rep(n, length(node_inode_free_data$value)))
@@ -181,6 +176,7 @@ for (currentdir in resultdirs) {
 					node_inode_free_data$s_offset = node_inode_free_data$epoch - local_bootdata[1,]$epoch
 
 					inode_free_data=rbind(inode_free_data, node_inode_free_data)
+				}
 
 				# grab interface data
 				interface_dir_pattern='^interface\\-'
@@ -191,8 +187,9 @@ for (currentdir in resultdirs) {
 
 					# filename has date on the end, so look for the right file name
 					interface_packets_pattern='^if_packets'
-					files=list.files(interface_dir, pattern=interface_packets_pattern)
+					int_files=list.files(interface_dir, pattern=interface_packets_pattern)
-					interface_packets_csv=paste(interface_dir, files[1], sep="/")
+					for (int_file in int_files) {
+						interface_packets_csv=paste(interface_dir, int_file, sep="/")
 						node_interface_packets_data=read.csv(interface_packets_csv, header=TRUE, sep=",")
 						node_interface_packets_data=cbind(node_interface_packets_data,
 														  node=rep(n, length(node_interface_packets_data$epoch)))
@@ -205,11 +202,13 @@ for (currentdir in resultdirs) {
 						node_interface_packets_data$s_offset = node_interface_packets_data$epoch - local_bootdata[1,]$epoch
 
 						interface_packets_data=rbind(interface_packets_data, node_interface_packets_data)
+					}
 
 					# filename has date on the end, so look for the right file name
 					interface_octets_pattern='^if_octets'
-					files=list.files(interface_dir, pattern=interface_octets_pattern)
+					int_files=list.files(interface_dir, pattern=interface_octets_pattern)
-					interface_octets_csv=paste(interface_dir, files[1], sep="/")
+					for (int_file in int_files) {
+						interface_octets_csv=paste(interface_dir, int_file, sep="/")
 						node_interface_octets_data=read.csv(interface_octets_csv, header=TRUE, sep=",")
 						node_interface_octets_data=cbind(node_interface_octets_data,
 														 node=rep(n, length(node_interface_octets_data$epoch)))
@@ -222,11 +221,13 @@ for (currentdir in resultdirs) {
 						node_interface_octets_data$s_offset = node_interface_octets_data$epoch - local_bootdata[1,]$epoch
 
 						interface_octets_data=rbind(interface_octets_data, node_interface_octets_data)
+					}
 
 					# filename has date on the end, so look for the right file name
 					interface_dropped_pattern='^if_dropped'
-					files=list.files(interface_dir, pattern=interface_dropped_pattern)
+					int_files=list.files(interface_dir, pattern=interface_dropped_pattern)
-					interface_dropped_csv=paste(interface_dir, files[1], sep="/")
+					for (int_file in int_files) {
+						interface_dropped_csv=paste(interface_dir, int_file, sep="/")
 						node_interface_dropped_data=read.csv(interface_dropped_csv, header=TRUE, sep=",")
 						node_interface_dropped_data=cbind(node_interface_dropped_data,
 														  node=rep(n, length(node_interface_dropped_data$epoch)))
@@ -239,11 +240,13 @@ for (currentdir in resultdirs) {
 						node_interface_dropped_data$s_offset = node_interface_dropped_data$epoch - local_bootdata[1,]$epoch
 
 						interface_dropped_data=rbind(interface_dropped_data, node_interface_dropped_data)
+					}
 
 					# filename has date on the end, so look for the right file name
 					interface_errors_pattern='^if_errors'
-					files=list.files(interface_dir, pattern=interface_errors_pattern)
+					int_files=list.files(interface_dir, pattern=interface_errors_pattern)
-					interface_errors_csv=paste(interface_dir, files[1], sep="/")
+					for (int_file in int_files) {
+						interface_errors_csv=paste(interface_dir, int_file, sep="/")
 						node_interface_errors_data=read.csv(interface_errors_csv, header=TRUE, sep=",")
 						node_interface_errors_data=cbind(node_interface_errors_data,
 														 node=rep(n, length(node_interface_errors_data$epoch)))
@@ -257,6 +260,7 @@ for (currentdir in resultdirs) {
 
 						interface_errors_data=rbind(interface_errors_data, node_interface_errors_data)
 					}
+				}
 
 				# Do not use the master (non-schedulable) nodes to calculate
 				# launched pod metrics
@@ -264,43 +268,79 @@ for (currentdir in resultdirs) {
 					next
 				}
 
-				max_free_mem=max(node_mem_free_data$value)
+				# get the epoch time of first and last pod launch
-				min_free_mem=min(node_mem_free_data$value)
+				start_time=local_bootdata$epoch[1]
+				end_time=local_bootdata$epoch[length(local_bootdata$epoch)]
+
+				# get value closest to first pod launch
+				mem_start_index=Position(function(x) x > start_time, node_mem_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(mem_start_index)) {
+					mem_start_index = 1
+				} else if (mem_start_index > 1) {
+					mem_start_index = mem_start_index - 1
+				}
+				max_free_mem=node_mem_free_data$value[mem_start_index]
+
+				# get value closest to last pod launch
+				mem_end_index=Position(function(x) x > end_time, node_mem_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(mem_end_index)) {
+					mem_end_index = length(node_mem_free_data$epoch)
+				} else if (mem_end_index > 1) {
+					mem_end_index = mem_end_index - 1
+				}
+				min_free_mem=node_mem_free_data$value[mem_end_index]
+
 				memtotal = memtotal + (max_free_mem - min_free_mem)
-				max_idle_cpu=max(node_cpu_idle_data$value)
+
-				min_idle_cpu=min(node_cpu_idle_data$value)
+				# get value closest to first pod launch
+				cpu_start_index=Position(function(x) x > start_time, node_cpu_idle_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(cpu_start_index)) {
+					cpu_start_index = 1
+				} else if (cpu_start_index > 1) {
+					cpu_start_index = cpu_start_index - 1
+				}
+				max_idle_cpu=node_cpu_idle_data$value[cpu_start_index]
+
+				# get value closest to last pod launch
+				cpu_end_index=Position(function(x) x > end_time, node_cpu_idle_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(cpu_end_index)) {
+					cpu_end_index = length(node_cpu_idle_data$epoch)
+				} else if (cpu_end_index > 1) {
+					cpu_end_index = cpu_end_index - 1
+				}
+				min_idle_cpu=node_cpu_idle_data$value[cpu_end_index]
+
 				cputotal = cputotal + (max_idle_cpu - min_idle_cpu)
-				max_free_inode=max(node_inode_free_data$value)
+
-				min_free_inode=min(node_inode_free_data$value)
+				# get value closest to first pod launch
+				inode_start_index=Position(function(x) x > start_time, node_inode_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(inode_start_index)) {
+					inode_start_index = 1
+				} else if (inode_start_index > 1) {
+					inode_start_index = inode_start_index - 1
+				}
+				max_free_inode=node_inode_free_data$value[inode_start_index]
+
+				# get value closest to last pod launch
+				inode_end_index=Position(function(x) x > end_time, node_inode_free_data$epoch)
+				# take the reading previous to the index as long as a valid index
+				if (is.na(inode_end_index)) {
+					inode_end_index = length(node_cpu_idle_data$epoch)
+				} else if (inode_end_index > 1) {
+					inode_end_index = inode_end_index - 1
+				}
+				min_free_inode=node_inode_free_data$value[inode_end_index]
+
 				inodetotal = inodetotal + (max_free_inode - min_free_inode)
 			}
 
 			num_pods = local_bootdata$n_pods[length(local_bootdata$n_pods)]
 
-			# calculate scaling for secondary y axis
-			# the two y scales, in R, must be mathematically related
-			mem_scale = max(c(mem_scale,
-							  (max(mem_free_data$value) / (1024*1024*1024)) / num_pods))
-			cpu_scale = max(c(cpu_scale,
-							  max(cpu_idle_data$value) / num_pods))
-			inode_scale = max(c(inode_scale,
-								max(inode_free_data$value) / num_pods))
-			ip_scale = max(c(ip_scale,
-							 max(c(max(interface_packets_data$tx, na.rm=TRUE),
-								   max(interface_packets_data$rx, na.rm=TRUE))) / num_pods))
-			oct_scale = max(c(oct_scale,
-							  max(c(max(interface_octets_data$tx, na.rm=TRUE),
-									max(interface_octets_data$rx, na.rm=TRUE))) / num_pods))
-			# drops and scale are often 0, so providing 1 so we won't scale by infinity
-			drop_scale = max(c(drop_scale,
-							   max(c(1,
-									 max(interface_dropped_data$tx, na.rm=TRUE),
-									 max(interface_dropped_data$rx, na.rm=TRUE))) / num_pods))
-			error_scale = max(c(error_scale,
-								max(c(1,
-									  max(interface_errors_data$tx, na.rm=TRUE),
-									  max(interface_errors_data$rx, na.rm=TRUE))) / num_pods))
-
 			# We get data in b, but want the graphs in Gb.
 			memtotal = memtotal / (1024*1024*1024)
 			gb_per_pod = memtotal/num_pods
@@ -367,13 +407,14 @@ memfreedata$mem_free_gb = memfreedata$value/(1024*1024*1024)
 # And show the boot times in seconds, not ms
 podbootdata$launch_time_s = podbootdata$launch_time/1000.0
 
+
 ########### Output memory page ##############
 mem_stats_plot = suppressWarnings(ggtexttable(data.frame(memstats),
 	theme=ttheme(base_size=10),
 	rows=NULL
 	))
 
-#mem_sec_axis_scale=
+mem_scale = (max(memfreedata$value) / (1024*1024*1024)) / max(podbootdata$n_pods)
 mem_line_plot <- ggplot() +
 	geom_line(data=memfreedata,
 			  aes(s_offset, mem_free_gb, colour=interaction(testname, node),
@@ -411,6 +452,7 @@ cpu_stats_plot = suppressWarnings(ggtexttable(data.frame(cpustats),
 	rows=NULL
 	))
 
+cpu_scale = max(cpuidledata$value) / max(podbootdata$n_pods)
 cpu_line_plot <- ggplot() +
 	geom_line(data=cpuidledata,
 			  aes(x=s_offset, y=value, colour=interaction(testname, node),
@@ -472,6 +514,7 @@ inode_stats_plot = suppressWarnings(ggtexttable(data.frame(inodestats),
 	rows=NULL
 	))
 
+inode_scale = max(inodefreedata$value) / max(podbootdata$n_pods)
 inode_line_plot <- ggplot() +
 	geom_line(data=inodefreedata,
 			  aes(x=s_offset, y=value, colour=interaction(testname, node),
@@ -504,6 +547,8 @@ page4 = grid.arrange(
 cat("\n\n\\pagebreak\n")
 
 ########## Output interface page packets and octets ##############
+ip_scale = max(c(max(ifpacketdata$tx, na.rm=TRUE),
+				 max(ifpacketdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_packet_line_plot <- ggplot() +
 	geom_line(data=ifpacketdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -534,6 +579,8 @@ interface_packet_line_plot <- ggplot() +
 	ggtitle("interface packets") +
 	theme(axis.text.x=element_text(angle=90))
 
+oct_scale = max(c(max(ifoctetdata$tx, na.rm=TRUE),
+				  max(ifoctetdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_octet_line_plot <- ggplot() +
 	geom_line(data=ifoctetdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -564,7 +611,6 @@ interface_octet_line_plot <- ggplot() +
 	ggtitle("interface octets") +
 	theme(axis.text.x=element_text(angle=90))
 
-
 page5 = grid.arrange(
 	interface_packet_line_plot,
 	interface_octet_line_plot,
@@ -575,6 +621,10 @@ page5 = grid.arrange(
 cat("\n\n\\pagebreak\n")
 
 ########## Output interface page drops and errors ##############
+# drops are often 0, so providing 1 so we won't scale by infinity
+drop_scale = max(c(1,
+				   max(ifdropdata$tx, na.rm=TRUE),
+				   max(ifdropdata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_drop_line_plot <- ggplot() +
 	geom_line(data=ifdropdata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),
@@ -605,6 +655,10 @@ interface_drop_line_plot <- ggplot() +
 	ggtitle("interface drops") +
 	theme(axis.text.x=element_text(angle=90))
 
+# errors are often 0, so providing 1 so we won't scale by infinity
+error_scale = max(c(1,
+					max(iferrordata$tx, na.rm=TRUE),
+					max(iferrordata$rx, na.rm=TRUE))) / max(podbootdata$n_pods)
 interface_error_line_plot <- ggplot() +
 	geom_line(data=iferrordata,
 			  aes(x=s_offset, y=tx, colour=interaction(testname, node, name, "tx"),

metrics/scaling/k8s_parallel.sh

@@ -253,8 +253,8 @@ help()
 	usage=$(cat << EOF
 Usage: $0 [-h] [options]
    Description:
-	Launch a series of workloads and take memory metric measurements after
+	Launch a series of workloads in a parallel manner and take memory metric measurements
-	each launch.
+	after each launch.
    Options:
         -h,    Help page.
 EOF

metrics/scaling/k8s_scale.sh

@@ -380,8 +380,8 @@ help()
 	usage=$(cat << EOF
 Usage: $0 [-h] [options]
    Description:
-	Launch a series of workloads and take memory metric measurements after
+	Launch a series of workloads in a linear manner and take memory metric measurements
-	each launch.
+	after each launch.
    Options:
 		-h,    Help page.
 EOF