Update CNIs to versions that work well with 1.20.X of Kubernetes

Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

clr-k8s-examples/0-canal/overlays/v3.18/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - canal/canal.yaml

clr-k8s-examples/0-canal/overlays/v3.22/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+

clr-k8s-examples/0-cilium/overlays/v1.9.13/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml

clr-k8s-examples/0-cilium/overlays/v1.9/values.yaml

@@ -0,0 +1,5 @@
+ipam:
+  mode: "cluster-pool"
+  operator:
+    clusterPoolIPv4PodCIDR: "10.244.0.0/16"
+    clusterPoolIPv4MaskSize: 24

clr-k8s-examples/0-flannel/overlays/v0.14.0-rc1/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/0-flannel/overlays/v0.16.3/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -50,9 +50,9 @@ delete_pair_ns() {
 case $CNI_COMMAND in
 ADD)
 	res=$(ipam)
-	ip=$(echo $res | jq -r '.ip4.ip')
+	ip=$(echo $res | jq -r '.ips[0].address')
 	add_pair_ns $ip
-	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	echo '{"cniVersion":"0.3.1"}' | jq -c --arg ip $ip '.ips[0].address = $ip' >&3
 	;;
 DEL)
 	set +o errexit

clr-k8s-examples/README.md

@@ -3,6 +3,11 @@
 ## Prerequisite
 This setup currently will work with Kubernetes 1.14 & above. Any version of Kubernetes before that might work, but is not guaranteed.
 
+## QUICK NOTE
+The version of Kubernetes* was bumped from 1.17.7 to 1.19.4 in Clear Linux* OS release 34090. The [guide](https://docs.01.org/clearlinux/latest/guides/clear/k8s-migration.html) and the Clear Linux OS bundle k8s-migration were created to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
+
+The new Clear Linux OS bundle k8s-migration was added in Clear Linux* OS release 34270. Please follow the guide for an upgrade.
+
 ## Sample multi-node vagrant setup
 
 To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
@@ -63,6 +68,28 @@ devicemapper storage. This should not be used for production.
 
 > NOTE: This step is done automatically if using vagrant.
 
+### For HA, setup the load balancer node
+
+Ideally, the load balancer node will be a separate node. However, one of the
+master nodes can also serve as the load balancer for the cluster. [HAProxy](https://www.haproxy.org/)
+is used in these instructions.
+
+```bash
+sudo swupd bundle-add haproxy
+sudo systemctl enable haproxy
+```
+
+Edit the master IP addresses and load balancer address and ports in [`haproxy.cfg.example`](haproxy.cfg.example)
+to match the IPs for the new cluster. If using a master node for the load balancer
+make sure that the `frontend bind` port is different than the Kubernetes API port, 6443.
+If using a separate machine for load balancing, the port can be 6443 if desired.
+
+```bash
+sudo mkdir -p /etc/haproxy
+sudo cp haproxy.cfg.example /etc/haproxy/haproxy.cfg
+sudo systemctl start haproxy
+```
+
 ## Bring up the master
 
 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -75,8 +102,9 @@ more information.
 
 > NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export
 the necessary environment variables if needed to be changed. By default it will use
-`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the 
-Vagrant.
+`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the
+Vagrant. If creating an HA cluster, make sure to specify `LOAD_BALANCER_IP` and
+`LOAD_BALANCER_PORT`.
 
 ```bash
 # default shows help
@@ -86,6 +114,13 @@ Vagrant.
 In order to enable running greater than 110 pods per node, set the environment
 variable `HIGH_POD_COUNT` to any non-empty value.
 
+If creating an HA cluster, join the other master nodes to the cluster.
+
+```bash
+kubeadm join <load-balancer-ip>:<load-balancer-port> --token <token> --discovery-token-ca-cert-hash <hash> \
+    --control-plane --certificate-key <certificate-key> --cri-socket=/run/crio/crio.sock
+```
+
 ## Join Workers to the cluster
 
 ```bash
@@ -94,6 +129,9 @@ kubeadm join <master-ip>:<master-port> --token <token> --discovery-token-ca-cert
 
 Note: Remember to append `--cri-socket=/run/crio/crio.sock` to the join command generated by the master.
 
+If creating an HA cluster, join the other worker nodes to the cluster. The same way,
+but replacing the `<master-ip>:<master-port>` with `<load-balancer-ip>:<load-balancer-port>`.
+
 On workers just use the join command that the master spits out. There nothing
 else you need to run on the worker. All the other Kubernetes customizations are pushed
 in from master via the values setup in the `kubeadm.yaml` file.

clr-k8s-examples/Vagrantfile

@@ -16,7 +16,7 @@ $box = "AntonioMeireles/ClearLinux"
 $box_ver = (ENV['CLEAR_VBOX_VER'])
 File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-$base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("10.10.100.10")
 $hosts = {}
 $proxy_ip_list = "192.168.121.0/24"
 $driveletters = ('a'..'z').to_a
@@ -76,6 +76,7 @@ Vagrant.configure("2") do |config|
         lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
+        lv.machine_virtual_size = 40
         (1..$disks).each do |d|
           lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end

clr-k8s-examples/create_stack.sh

@@ -14,11 +14,14 @@ SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 : ${MASTER_IP:=}
 : ${CERT_SANS:=}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
+LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""}
+LOAD_BALANCER_PORT="${LOAD_BALANCER_PORT:-6444}"
 
 # versions
-CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}"
-CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.6.4}"
-FLANNEL_VER="${CLRK8S_FLANNEL_VER:-960b3243b9a7faccdfe7b3c09097105e68030ea7}"
+CANAL_VER="${CLRK8S_CANAL_VER:-v3.22}"
+CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.9.13}"
+FLANNEL_VER="${CLRK8S_FLANNEL_VER:-v0.16.3}"
+CILIUM_VAL_OVERRIDE=""
 K8S_VER="${CLRK8S_K8S_VER:-}"
 KATA_VER="${CLRK8S_KATA_VER:-1.9.1-kernel-config}"
 ROOK_VER="${CLRK8S_ROOK_VER:-v1.2.6}"
@@ -90,7 +93,13 @@ function cluster_init() {
 		echo "/var/lib/etcd exists! skipping init."
 		return
 	fi
-	sudo -E kubeadm init --config=./kubeadm.yaml
+
+	if [[ -n "${LOAD_BALANCER_IP}" ]]; then
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\ncontrolPlaneEndpoint: ${LOAD_BALANCER_IP}:${LOAD_BALANCER_PORT}/g" ./kubeadm.yaml
+	fi
+	# upload-certs will automatically upload certificates that should be shared
+	# across control-plane nodes in HA clusters. It is harmless in non-HA cases.
+	sudo -E kubeadm init --upload-certs --config=./kubeadm.yaml
 
 	rm -rf "${HOME}/.kube"
 	mkdir -p "${HOME}/.kube"
@@ -136,7 +145,7 @@ function cni() {
 
 		# canal manifests are not kept in repo but in docs site so use curl
 		mkdir -p "${CANAL_DIR}/overlays/${CANAL_VER}/canal"
-		curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
+		curl -L -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
 		if [[ "$CANAL_VER" == "v3.3" ]]; then
 			curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/rbac.yaml" "$CANAL_URL/rbac.yaml"
 		fi
@@ -145,7 +154,7 @@ function cni() {
 		;;
 	flannel)
 		FLANNEL_VER=${1:-$FLANNEL_VER}
-		FLANNEL_URL="https://github.com/coreos/flannel"
+		FLANNEL_URL="https://github.com/flannel-io/flannel"
 		FLANNEL_DIR="0-flannel"
 
 		get_repo "${FLANNEL_URL}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
@@ -153,13 +162,17 @@ function cni() {
 		kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
 		;;
 	cilium)
+		local podsubnet=$(grep -Po 'podSubnet:\ \K[^*]*' ${SCRIPT_DIR}/kubeadm.yaml)
 		CILIUM_VER=${1:-$CILIUM_VER}
 		CILIUM_URL="https://github.com/cilium/cilium.git"
 		CILIUM_DIR="0-cilium"
 
 		get_repo "${CILIUM_URL}" "${CILIUM_DIR}/overlays/${CILIUM_VER}"
 		set_repo_version "${CILIUM_VER}" "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/"
-		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set global.containerRuntime.integration="$RUNNER" | kubectl apply -f -
+		if [ -f "${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml" ]; then
+			CILIUM_VAL_OVERRIDE="--values ${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml"
+		fi
+		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set containerRuntime.integration="$RUNNER" --set hubble.enabled=false --set ipam.operator.clusterPoolIPv4PodCIDR="${podsubnet}" | kubectl apply -f -
 		;;
 	*)
 		echo"Unknown cni $CNI"

clr-k8s-examples/haproxy.cfg.example

@@ -0,0 +1,34 @@
+global
+    log /dev/log    local0
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy-master.sock mode 660 level admin
+    stats timeout 30s
+    user haproxy
+    group haproxy
+    daemon
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
+    ssl-default-bind-options no-sslv3
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+    timeout tunnel  4h
+frontend kubernetes
+    bind 10.0.0.100:6444
+    option tcplog
+    mode tcp
+    default_backend kubernetes-master-nodes
+
+backend kubernetes-master-nodes
+    mode tcp
+    balance source
+    option tcp-check
+    server master-1 10.0.0.100:6443 check fall 3 rise 2
+    server master-2 10.0.0.101:6443 check fall 3 rise 2
+    server master-3 10.0.0.102:6443 check fall 3 rise 2

clr-k8s-examples/setup_system.sh

@@ -29,13 +29,10 @@ function add_os_deps() {
 
 # permanently disable swap
 function disable_swap() {
-	swapcount=$(sudo grep '^/dev/\([0-9a-z]*\).*' /proc/swaps | wc -l)
-
-	if [ "$swapcount" != "0" ]; then
-		sudo systemctl mask "$(sed -n -e 's#^/dev/\([0-9a-z]*\).*#dev-\1.swap#p' /proc/swaps)" 2>/dev/null
-	else
-		echo "Swap not enabled"
-	fi
+	# disable current swap
+	sudo swapoff -a
+	# permanently disable swap
+	sudo systemctl mask swap.target
 }
 
 # enable ip forwarding
@@ -137,7 +134,6 @@ function enable_kubelet_runner() {
 
 # ensure that the system is ready without requiring a reboot
 function ensure_system_ready() {
-	sudo swapoff -a
 	sudo systemctl restart systemd-modules-load.service
 }
 
@@ -192,12 +188,12 @@ fi
 
 echo "Init..."
 init
+echo "Disabling swap..."
+disable_swap
 echo "Setting OS Version..."
 upate_os_version
 echo "Adding OS Dependencies..."
 add_os_deps
-echo "Disabling swap..."
-disable_swap
 echo "Enabling IP Forwarding..."
 enable_ip_forwarding
 echo "Setting up modules to load..."
@@ -210,7 +206,7 @@ if [[ -n "${HIGH_POD_COUNT}" ]]; then
 fi
 echo "Reloading daemons..."
 daemon_reload
-echo "Enabling Kublet runner..."
+echo "Enabling kubelet runner..."
 enable_kubelet_runner
 echo "Ensuring system is ready..."
 ensure_system_ready