Add swupd update back

Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

clr-k8s-examples/0-canal/canal.yaml

@@ -159,6 +159,8 @@ spec:
               value: "info"
             - name: FELIX_HEALTHENABLED
               value: "true"
+            - name: FELIX_IGNORELOOSERPF
+              value: "true"
           securityContext:
             privileged: true
           resources:

clr-k8s-examples/README.md

@@ -50,14 +50,12 @@ master and also uses kubelet config via [`kubeadm.yaml`](kubeadm.yaml)
 to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
-There are two flavors of install -
-
-* `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
-* `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
+There are different flavors to install, run `./create_stack.sh help` to get
+more information.
 
 ```bash
-# default is 'all'
-./create_stack.sh [minimal|all]
+# default shows help
+./create_stack.sh <subcommand>
 ```
 
 ## Join Workers to the cluster

clr-k8s-examples/Vagrantfile

@@ -13,13 +13,21 @@ $disks = 2
 $disk_prefix = File.basename(File.dirname(__FILE__), "/")
 $disk_size = "10G"
 $box = "AntonioMeireles/ClearLinux"
-$loader = File.join(File.dirname(__FILE__), "OVMF.fd")
+File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
 $base_ip = IPAddr.new("192.52.100.10")
 $hosts = {}
 $proxy_ip_list = ""
 $driveletters = ('a'..'z').to_a
 $setup_fc = true ? (['true', '1'].include? ENV['SETUP_FC'].to_s) : false
+$runner = ENV.has_key?('RUNNER') ? ENV['RUNNER'].to_s : "containerd".to_s
+if !(["crio","containerd"].include? $runner)
+  abort("it's either crio or containerd. Cannot do anything else")
+end
+
+if not File.exists?($loader)
+  system('curl -O https://download.clearlinux.org/image/OVMF.fd')
+end
 
 # We need v 1.0.14 or above for this vagrantfile to work.
 unless Vagrant.has_plugin?("vagrant-guests-clearlinux")
@@ -39,6 +47,7 @@ Vagrant.configure("2") do |config|
   # Every Vagrant development environment requires a box. You can search for
   # boxes at https://vagrantcloud.com/search.
   config.vm.box = $box
+  config.vm.box_version = "30260"
 
   # Mount the current dir at home folder instead of default
   config.vm.synced_folder './', '/vagrant', disabled: true
@@ -61,6 +70,7 @@ Vagrant.configure("2") do |config|
       c.vm.provider :libvirt do |lv|
         lv.cpu_mode = "host-passthrough"
         lv.nested = true
+        lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
         (1..$disks).each do |d|
@@ -74,9 +84,17 @@ Vagrant.configure("2") do |config|
           c.proxy.no_proxy =  (ENV['no_proxy']+"#{proxy_ip_list}" || ENV['NO_PROXY']+"#{proxy_ip_list}" || "localhost,127.0.0.1,172.16.10.10#{proxy_ip_list}")
         end
       end
-      c.vm.provision "shell", privileged: false, path: "setup_system.sh"
+      c.vm.provision "shell", privileged: false, path: "setup_system.sh", env: {"RUNNER" => $runner}
       if $setup_fc
+        if $runner == "crio".to_s
         c.vm.provision "shell", privileged: false, path: "setup_kata_firecracker.sh"
+        else
+        # Wish we could use device mapper snapshotter with containerd, but it
+        # does not exist on any released containerd version. Failing for now
+        # when we use FC with containerd
+        abort("Cannot use containerd with FC for now.")
+        #c.vm.provision "shell", privileged: false, path: "containerd_devmapper_setup.sh"
+        end
       end
       # Include shells bundle to get bash completion and add kubectl's commands to vagrant's shell
       c.vm.provision "shell", privileged: false, inline: 'sudo -E swupd bundle-add shells; echo "source <(kubectl completion bash)" >> $HOME/.bashrc'

clr-k8s-examples/containerd_devmapper_setup.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+sudo rm -rf /var/lib/containerd/devmapper/data-disk.img
+sudo rm -rf /var/lib/containerd/devmapper/meta-disk.img
+sudo mkdir -p /var/lib/containerd/devmapper
+sudo truncate --size 10G /var/lib/containerd/devmapper/data-disk.img
+sudo truncate --size 10G /var/lib/containerd/devmapper/meta-disk.img
+
+sudo mkdir -p /etc/systemd/system
+
+cat<<EOT | sudo tee /etc/systemd/system/containerd-devmapper.service
+[Unit]
+Description=Setup containerd devmapper device
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=-/sbin/losetup /dev/loop20 /var/lib/containerd/devmapper/data-disk.img
+ExecStart=-/sbin/losetup /dev/loop21 /var/lib/containerd/devmapper/meta-disk.img
+
+[Install]
+WantedBy=local-fs.target
+EOT
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now containerd-devmapper
+
+# Time to setup the thin pool for consumption.
+# The table arguments are such.
+# start block in the virtual device
+# length of the segment (block device size in bytes / Sector size (512)
+# metadata device
+# block data device
+# data_block_size Currently set it 512 (128KB)
+# low_water_mark. Copied this from containerd snapshotter test setup
+# no. of feature arguments
+# Skip zeroing blocks for new volumes.
+sudo dmsetup create contd-thin-pool \
+  --table "0 2097152 thin-pool /dev/loop21 /dev/loop20 512 32768 1 skip_block_zeroing"
+
+sudo mkdir -p /etc/containerd/
+if [ -f /etc/containerd/config.toml ]
+then
+  sudo sed -i 's|^\(\[plugins\]\).*|\1\n  \[plugins.devmapper\]\n    pool_name = \"contd-thin-pool\"\n    base_image_size = \"512MB\"|' /etc/containerd/config.toml
+else
+  cat<<EOT | sudo tee /etc/containerd/config.toml
+[plugins]
+  [plugins.devmapper]
+    pool_name = "contd-thin-pool"
+    base_image_size = "512MB"
+EOT
+fi
+
+sudo systemctl restart containerd

clr-k8s-examples/create_stack.sh

@@ -8,12 +8,23 @@ CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 
 function print_usage_exit() {
-	echo $"Usage: $0 [minimal|all]"
-	exit 1
+	exit_code=${1:-0}
+	cat <<EOT
+Usage: $0 [subcommand]
+
+Subcommands:
+
+$(
+		for cmd in "${!command_handlers[@]}"; do
+			printf "\t%s:|\t%s\n" "${cmd}" "${command_help[${cmd}]:-Not-documented}"
+		done | sort | column -t -s "|"
+	)
+EOT
+	exit "${exit_code}"
 }
 
 function finish() {
-	cd $CUR_DIR
+	cd "${CUR_DIR}"
 }
 trap finish EXIT
 
@@ -22,10 +33,10 @@ function cluster_init() {
 	#to enable the RuntimeClass featuregate
 	sudo -E kubeadm init --config=./kubeadm.yaml
 
-	rm -rf $HOME/.kube
-	mkdir -p $HOME/.kube
-	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
-	sudo chown $(id -u):$(id -g) $HOME/.kube/config
+	rm -rf "${HOME}/.kube"
+	mkdir -p "${HOME}/.kube"
+	sudo cp -i /etc/kubernetes/admin.conf "${HOME}/.kube/config"
+	sudo chown "$(id -u):$(id -g)" "${HOME}/.kube/config"
 
 	# If this an interactive terminal then wait for user to join workers
 	if [ -t 0 ]; then
@@ -33,7 +44,7 @@ function cluster_init() {
 	fi
 
 	#Ensure single node k8s works
-	if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
+	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
 		kubectl taint nodes --all node-role.kubernetes.io/master-
 	fi
 }
@@ -109,20 +120,25 @@ function all() {
 	miscellaneous
 }
 
-cd $SCRIPT_DIR
-if [[ "$#" -eq 0 ]]; then
-	all
-	exit
-fi
+declare -A command_handlers
+command_handlers[init]=cluster_init
+command_handlers[cni]=cni
+command_handlers[minimal]=minimal
+command_handlers[all]=all
+command_handlers[help]=print_usage_exit
 
-case "$1" in
-minimal)
-	minimal
-	;;
-all)
-	all
-	;;
-*)
-	print_usage_exit
-	;;
-esac
+declare -A command_help
+command_help[init]="Only inits a cluster using kubeadm"
+command_help[cni]="Setup network for running cluster"
+command_help[minimal]="init + cni +  kata + metrics"
+command_help[all]="minimal + storage + monitoring + miscellaneous"
+command_help[help]="show this message"
+
+cd "${SCRIPT_DIR}"
+
+cmd_handler=${command_handlers[${1:-none}]:-unimplemented}
+if [ "${cmd_handler}" != "unimplemented" ]; then
+	"${cmd_handler}"
+else
+	print_usage_exit 1
+fi

clr-k8s-examples/kubeadm.yaml

@@ -1,7 +1,5 @@
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
-nodeRegistration:
-  criSocket: /var/run/crio/crio.sock
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration

clr-k8s-examples/reset_stack.sh

@@ -3,7 +3,10 @@
 set -o nounset
 
 #Cleanup
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster() {
+	sudo -E kubeadm reset -f
+}
+reset_cluster
 
 for ctr in $(sudo crictl ps --quiet); do
 	sudo crictl stop "$ctr"
@@ -15,9 +18,10 @@ for pod in $(sudo crictl pods --quiet); do
 done
 
 #Forcefull cleanup all artifacts
-#This is needed is things really go wrong
+#This is needed if things really go wrong
 sudo systemctl stop kubelet
-sudo systemctl stop crio
+systemctl is-active crio && sudo systemctl stop crio
+systemctl is-active containerd && sudo systemctl stop containerd
 sudo pkill -9 qemu
 sudo pkill -9 kata
 sudo pkill -9 kube
@@ -39,8 +43,11 @@ sudo -E bash -c "rm -r /var/run/kata-containers/*"
 sudo rm -rf /var/lib/rook
 
 sudo systemctl daemon-reload
-sudo systemctl enable kubelet crio
-sudo systemctl restart crio
+sudo systemctl is-active crio && sudo systemctl stop crio
+sudo systemctl is-active containerd && sudo systemctl stop containerd
+sudo systemctl is-enabled crio && sudo systemctl restart crio
+sudo systemctl is-enabled containerd && sudo systemctl restart containerd
+
 sudo systemctl restart kubelet
 
-sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
+reset_cluster

clr-k8s-examples/setup_system.sh

@@ -5,9 +5,10 @@ set -o nounset
 
 ADD_NO_PROXY="10.244.0.0/16,10.96.0.0/12"
 ADD_NO_PROXY+=",$(hostname -I | sed 's/[[:space:]]/,/g')"
+: ${RUNNER:="containerd"}
 
 #Install kubernetes and crio
-sudo -E swupd update
+sudo swupd update
 sudo -E swupd bundle-add cloud-native-basic storage-utils
 
 #Permanently disable swap
@@ -48,10 +49,8 @@ sudo systemctl daemon-reload
 # This will fail at this point, but puts it into a retry loop that
 # will therefore startup later once we have configured with kubeadm.
 echo "The following kubelet command may complain... it is not an error"
-sudo systemctl enable --now kubelet crio || true
+sudo systemctl enable --now kubelet $RUNNER || true
 
-sudo mkdir -p /usr/libexec/cni /opt/cni
-[ ! -e /opt/cni/bin/cni ] && sudo ln -s /usr/libexec/cni /opt/cni/bin
 #Ensure that the system is ready without requiring a reboot
 sudo swapoff -a
 sudo systemctl restart systemd-modules-load.service
@@ -68,7 +67,7 @@ if [[ ${http_proxy} ]] || [[ ${HTTP_PROXY} ]]; then
 		echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
 	fi
 
-	services=('crio' 'kubelet')
+	services=($RUNNER 'kubelet')
 	for s in "${services[@]}"; do
 		sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
 		cat <<EOF | sudo bash -c "cat > /etc/systemd/system/${s}.service.d/proxy.conf"
@@ -84,5 +83,5 @@ set -o nounset
 
 # We have potentially modified their env files, we need to restart the services.
 sudo systemctl daemon-reload
-sudo systemctl restart crio || true
+sudo systemctl restart $RUNNER || true
 sudo systemctl restart kubelet || true

clr-k8s-examples/tests/e2e/run_e2e.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Runs upstream k8s e2e tests against existing cloud native basic cluster
+# Requires cluster to already be up
+# To specify a parameter for --ginkgo.focus as described below, provide a focus as the first argument to this script
+#     https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#building-kubernetes-and-running-the-tests
+# One example would be Feature:Performance. The script will add square brackets for you
+# For other examples of values, see
+#     https://github.com/kubernetes/community/blob/master/contributors/devel/sig-testing/e2e-tests.md#kinds-of-tests
+
+set -o errexit
+set -o pipefail
+
+if [ ! -z $1 ]
+then
+    FOCUS=$1
+    echo Running e2e tests where spec matches $1
+else
+    echo Running all e2e tests, this will take a long time
+fi
+
+GO_INSTALLED=$(sudo swupd bundle-list | grep go-basic)
+if [ -z $GO_INSTALLED ]
+then
+    echo Installing go-basic bundle
+    sudo swupd bundle-add go-basic
+else
+    echo Skipping go-basic bundle installation
+fi
+
+if [ -z $GOPATH ] ; then GOPATH=$HOME/go; fi
+if [ -z $GOBIN ] ; then GOBIN=$HOME/go/bin; fi
+
+echo Getting kubetest
+go get -u k8s.io/test-infra/kubetest
+
+cd $GOPATH/src/k8s.io
+
+if [ -d kubernetes ]
+then
+    cd kubernetes
+    echo Checking status of existing k8s repo clone
+    git status kubernetes
+else
+    echo Cloning upstream k8s repo
+    git clone https://github.com/kubernetes/kubernetes.git
+    cd kubernetes
+fi
+
+PATH=$PATH:$GOBIN
+
+API_SERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"kubernetes\")].cluster.server}")
+CLIENT_VERSION=$(kubectl version --short | grep -E 'Client' | sed 's/Client Version: //')
+
+echo Running kubetest
+
+if [ -z $FOCUS ]
+then
+    echo sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER" --extract=$CLIENT_VERSION --provider=local 
+    sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER" --extract=$CLIENT_VERSION --provider=local
+else
+    echo sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER --ginkgo.focus=\[$FOCUS\]" --extract=$CLIENT_VERSION --provider=local
+    sudo -E kubetest --test --test_args="--kubeconfig=${HOME}/.kube/config --host=$API_SERVER --ginkgo.focus=\[$FOCUS\]" --extract=$CLIENT_VERSION --provider=local
+fi
+