Update README to take not of k8s-migration package (#334)

Signed-off-by: Ganesh Maharaj Mahalingam <ganesh.mahalingam@intel.com>

clr-k8s-examples/9-multi-network/Dockerfile

@@ -1,20 +1,28 @@
 # Build multus plugin
-FROM golang:1.10 AS multus
+FROM busybox AS multus
-RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
+ARG MULTUS_VER=3.4.2
-WORKDIR /go/src/github.com/intel/multus-cni
+RUN wget -O multus.tgz https://github.com/intel/multus-cni/releases/download/v${MULTUS_VER}/multus-cni_${MULTUS_VER}_linux_amd64.tar.gz
-RUN ./build
+RUN tar xvzf multus.tgz --strip-components=1 -C /bin
 
 # Build sriov plugin
-FROM golang:1.10 AS sriov-cni
+FROM golang AS sriov-cni
-RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
+ARG SRIOV_CNI_VER=2.3
-WORKDIR /go/src/github.com/intel-corp/sriov-cni
+RUN wget -qO sriov-cni.tgz https://github.com/intel/sriov-cni/archive/v${SRIOV_CNI_VER}.tar.gz
-RUN ./build
+RUN mkdir -p sriov-cni && \
+    tar xzf sriov-cni.tgz --strip-components=1 -C sriov-cni && \
+    cd sriov-cni && \
+    make && \
+    cp build/sriov /bin
 
 # Build sriov device plugin
-FROM golang:1.10 AS sriov-dp
+FROM golang AS sriov-dp
-RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
+ARG SRIOV_DP_VER=3.2
-WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
+RUN wget -qO sriov-dp.tgz https://github.com/intel/sriov-network-device-plugin/archive/v${SRIOV_DP_VER}.tar.gz
-RUN make
+RUN mkdir -p sriov-dp && \
+    tar xzf sriov-dp.tgz --strip-components=1 -C sriov-dp && \
+    cd sriov-dp && \
+    make && \
+    cp build/sriovdp /bin
 
 # Build vfioveth plugin
 FROM busybox as vfioveth
@@ -25,9 +33,9 @@ RUN chmod +x /bin/vfioveth /bin/jq
 # Final image
 FROM centos/systemd
 WORKDIR /tmp/cni/bin
-COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
+COPY --from=multus /bin/multus-cni .
-COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=sriov-cni /bin/sriov .
 COPY --from=vfioveth /bin/vfioveth .
 COPY --from=vfioveth /bin/jq .
 WORKDIR /usr/bin
-COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .
+COPY --from=sriov-dp /bin/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -9,11 +9,10 @@ directories on the host with the necessary binaries and configuration files.
 
 ### Customization
 
-The device plugin will register the SR-IOV enabled devices on the host, specified as
+The device plugin will register the SR-IOV enabled devices on the host, specified with
-`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+`selectors` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
-file is provided, which enables SR-IOV for the above `rootDevices`
+file is provided, which enables SR-IOV for the above devices. More config options
-
+are listed [here](https://github.com/intel/sriov-network-device-plugin#configurations).
-> NOTE: This assumes homogenous nodes in the cluster
 
 ### Pre-req (SR-IOV only)
 

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -50,9 +50,9 @@ delete_pair_ns() {
 case $CNI_COMMAND in
 ADD)
 	res=$(ipam)
-	ip=$(echo $res | jq -r '.ip4.ip')
+	ip=$(echo $res | jq -r '.ips[0].address')
 	add_pair_ns $ip
-	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	echo '{"cniVersion":"0.3.1"}' | jq -c --arg ip $ip '.ips[0].address = $ip' >&3
 	;;
 DEL)
 	set +o errexit

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -2,25 +2,34 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
   name: network-attachment-definitions.k8s.cni.cncf.io
 spec:
-  # group name to use for REST API: /apis/<group>/<version>
   group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
   scope: Namespaced
   names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
     plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
     singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
     kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
     shortNames:
     - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -40,17 +49,43 @@ type: kubernetes.io/service-account-token
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: multus-pod-networks-lister
+  name: multus-pod-updater
 rules:
-- apiGroups: [""]
+  - apiGroups: ["k8s.cni.cncf.io"]
-  resources: ["pods"]
+    resources:
-  verbs: ["get"]
+      - '*'
-- apiGroups: [""]
+    verbs:
-  resources: ["pods/status"]
+      - '*'
-  verbs: ["update"]
+  - apiGroups:
-- apiGroups: ["k8s.cni.cncf.io"]
+      - ""
-  resources: ["*"]
+    resources:
-  verbs: ["get"]
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-updater
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -94,8 +129,9 @@ data:
     MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
     cat > /host/etc/cni/net.d/00-multus.conf <<EOF
     {
+      "cniVersion": "0.3.1",
       "name": "multus-cni-network",
-      "type": "multus",
+      "type": "multus-cni",
       "logFile": "/var/log/multus.log",
       "logLevel": "debug",
       "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
@@ -105,19 +141,6 @@ data:
     }
     EOF
 ---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus-rb
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus-pod-networks-lister
-subjects:
-- kind: ServiceAccount
-  name: multus-sa
-  namespace: kube-system
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -134,14 +157,14 @@ spec:
     spec:
       initContainers:
       - name: multus
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "bash", "-c" ]
         args:
-        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+        - cp /tmp/cni/bin/{multus-cni,sriov,vfioveth,jq} /host/opt/cni/bin/;
           /tmp/multus/install-multus-conf.sh;
           /tmp/multus/install-certs.sh;
           echo "Restarting crio kubelet";
-          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart crio;
           systemctl restart kubelet;
         volumeMounts:
         - name: usr-bin
@@ -160,7 +183,7 @@ spec:
           mountPath: /run/systemd
       containers:
       - name: sriovdp
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "sh", "-c" ]
         args:
         - /usr/bin/sriovdp --logtostderr -v 10;
@@ -209,4 +232,3 @@ spec:
           path: /var/lib/kubelet/device-plugins/
       hostNetwork: true
       hostPID: true
-

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -11,15 +11,15 @@ data:
       [
         {
           "resourceName": "sriov_netdevice",
-          "rootDevices": ["07:00.0"],
+          "selectors": {
-          "sriovMode": true,
+                  "drivers": ["i40evf", "iavf"]
-          "deviceType": "netdevice"
+          }
         },
         {
           "resourceName": "sriov_vfio",
-          "rootDevices": ["07:00.1"],
+          "selectors": {
-          "sriovMode": true,
+                  "drivers": ["vfio-pci"]
-          "deviceType": "vfio"
+          }
         }
       ]
     }

clr-k8s-examples/9-multi-network/test/sriov/2-pod-dpdk-ver.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1711
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:17.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1811
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:18.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1911
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:19.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-2002
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:20.02
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'

clr-k8s-examples/9-multi-network/test/sriov/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:bionic as ubuntu-build
+RUN apt-get update && \
+    apt-get -y install \
+        build-essential \
+        git \
+        libnuma-dev
+
+ARG DPDK_VER='master'
+ENV DPDK_DIR='/dpdk'
+ENV RTE_TARGET='x86_64-native-linuxapp-gcc'
+RUN git clone -b $DPDK_VER -q --depth 1 http://dpdk.org/git/dpdk-stable $DPDK_DIR 2>&1
+RUN cd ${DPDK_DIR} && \
+    sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \
+    sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \
+    make config T=x86_64-native-linuxapp-gcc && \
+    make -j $CPUS
+ENV PATH="$PATH:$DPDK_DIR/build/app/"

clr-k8s-examples/README.md

@@ -3,6 +3,11 @@
 ## Prerequisite
 This setup currently will work with Kubernetes 1.14 & above. Any version of Kubernetes before that might work, but is not guaranteed.
 
+## QUICK NOTE
+The version of Kubernetes* was bumped from 1.17.7 to 1.19.4 in Clear Linux* OS release 34090. The [guide](https://docs.01.org/clearlinux/latest/guides/clear/k8s-migration.html) and the Clear Linux OS bundle k8s-migration were created to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
+
+The new Clear Linux OS bundle k8s-migration was added in Clear Linux* OS release 34270. Please follow the guide for an upgrade.
+
 ## Sample multi-node vagrant setup
 
 To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
@@ -63,6 +68,28 @@ devicemapper storage. This should not be used for production.
 
 > NOTE: This step is done automatically if using vagrant.
 
+### For HA, setup the load balancer node
+
+Ideally, the load balancer node will be a separate node. However, one of the
+master nodes can also serve as the load balancer for the cluster. [HAProxy](https://www.haproxy.org/)
+is used in these instructions.
+
+```bash
+sudo swupd bundle-add haproxy
+sudo systemctl enable haproxy
+```
+
+Edit the master IP addresses and load balancer address and ports in [`haproxy.cfg.example`](haproxy.cfg.example)
+to match the IPs for the new cluster. If using a master node for the load balancer
+make sure that the `frontend bind` port is different than the Kubernetes API port, 6443.
+If using a separate machine for load balancing, the port can be 6443 if desired.
+
+```bash
+sudo mkdir -p /etc/haproxy
+sudo cp haproxy.cfg.example /etc/haproxy/haproxy.cfg
+sudo systemctl start haproxy
+```
+
 ## Bring up the master
 
 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -75,8 +102,9 @@ more information.
 
 > NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export
 the necessary environment variables if needed to be changed. By default it will use
-`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the 
+`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the
-Vagrant.
+Vagrant. If creating an HA cluster, make sure to specify `LOAD_BALANCER_IP` and
+`LOAD_BALANCER_PORT`.
 
 ```bash
 # default shows help
@@ -86,6 +114,13 @@ Vagrant.
 In order to enable running greater than 110 pods per node, set the environment
 variable `HIGH_POD_COUNT` to any non-empty value.
 
+If creating an HA cluster, join the other master nodes to the cluster.
+
+```bash
+kubeadm join <load-balancer-ip>:<load-balancer-port> --token <token> --discovery-token-ca-cert-hash <hash> \
+    --control-plane --certificate-key <certificate-key> --cri-socket=/run/crio/crio.sock
+```
+
 ## Join Workers to the cluster
 
 ```bash
@@ -94,6 +129,9 @@ kubeadm join <master-ip>:<master-port> --token <token> --discovery-token-ca-cert
 
 Note: Remember to append `--cri-socket=/run/crio/crio.sock` to the join command generated by the master.
 
+If creating an HA cluster, join the other worker nodes to the cluster. The same way,
+but replacing the `<master-ip>:<master-port>` with `<load-balancer-ip>:<load-balancer-port>`.
+
 On workers just use the join command that the master spits out. There nothing
 else you need to run on the worker. All the other Kubernetes customizations are pushed
 in from master via the values setup in the `kubeadm.yaml` file.

clr-k8s-examples/Vagrantfile

@@ -16,7 +16,7 @@ $box = "AntonioMeireles/ClearLinux"
 $box_ver = (ENV['CLEAR_VBOX_VER'])
 File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-$base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("10.10.100.10")
 $hosts = {}
 $proxy_ip_list = "192.168.121.0/24"
 $driveletters = ('a'..'z').to_a
@@ -76,6 +76,7 @@ Vagrant.configure("2") do |config|
         lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
+        lv.machine_virtual_size = 40
         (1..$disks).each do |d|
           lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end

clr-k8s-examples/create_stack.sh

@@ -14,6 +14,8 @@ SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 : ${MASTER_IP:=}
 : ${CERT_SANS:=}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
+LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""}
+LOAD_BALANCER_PORT="${LOAD_BALANCER_PORT:-6444}"
 
 # versions
 CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}"
@@ -90,7 +92,13 @@ function cluster_init() {
 		echo "/var/lib/etcd exists! skipping init."
 		return
 	fi
-	sudo -E kubeadm init --config=./kubeadm.yaml
+
+	if [[ -n "${LOAD_BALANCER_IP}" ]]; then
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\ncontrolPlaneEndpoint: ${LOAD_BALANCER_IP}:${LOAD_BALANCER_PORT}/g" ./kubeadm.yaml
+	fi
+	# upload-certs will automatically upload certificates that should be shared
+	# across control-plane nodes in HA clusters. It is harmless in non-HA cases.
+	sudo -E kubeadm init --upload-certs --config=./kubeadm.yaml
 
 	rm -rf "${HOME}/.kube"
 	mkdir -p "${HOME}/.kube"

clr-k8s-examples/haproxy.cfg.example

@@ -0,0 +1,34 @@
+global
+    log /dev/log    local0
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy-master.sock mode 660 level admin
+    stats timeout 30s
+    user haproxy
+    group haproxy
+    daemon
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
+    ssl-default-bind-options no-sslv3
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+    timeout tunnel  4h
+frontend kubernetes
+    bind 10.0.0.100:6444
+    option tcplog
+    mode tcp
+    default_backend kubernetes-master-nodes
+
+backend kubernetes-master-nodes
+    mode tcp
+    balance source
+    option tcp-check
+    server master-1 10.0.0.100:6443 check fall 3 rise 2
+    server master-2 10.0.0.101:6443 check fall 3 rise 2
+    server master-3 10.0.0.102:6443 check fall 3 rise 2

clr-k8s-examples/setup_system.sh

@@ -29,13 +29,10 @@ function add_os_deps() {
 
 # permanently disable swap
 function disable_swap() {
-	swapcount=$(sudo grep '^/dev/\([0-9a-z]*\).*' /proc/swaps | wc -l)
+	# disable current swap
-
+	sudo swapoff -a
-	if [ "$swapcount" != "0" ]; then
+	# permanently disable swap
-		sudo systemctl mask "$(sed -n -e 's#^/dev/\([0-9a-z]*\).*#dev-\1.swap#p' /proc/swaps)" 2>/dev/null
+	sudo systemctl mask swap.target
-	else
-		echo "Swap not enabled"
-	fi
 }
 
 # enable ip forwarding
@@ -137,7 +134,6 @@ function enable_kubelet_runner() {
 
 # ensure that the system is ready without requiring a reboot
 function ensure_system_ready() {
-	sudo swapoff -a
 	sudo systemctl restart systemd-modules-load.service
 }
 
@@ -192,12 +188,12 @@ fi
 
 echo "Init..."
 init
+echo "Disabling swap..."
+disable_swap
 echo "Setting OS Version..."
 upate_os_version
 echo "Adding OS Dependencies..."
 add_os_deps
-echo "Disabling swap..."
-disable_swap
 echo "Enabling IP Forwarding..."
 enable_ip_forwarding
 echo "Setting up modules to load..."
@@ -210,7 +206,7 @@ if [[ -n "${HIGH_POD_COUNT}" ]]; then
 fi
 echo "Reloading daemons..."
 daemon_reload
-echo "Enabling Kublet runner..."
+echo "Enabling kubelet runner..."
 enable_kubelet_runner
 echo "Ensuring system is ready..."
 ensure_system_ready