Fix create_stack for version 1.25

kubernetes 1.25 changed the key for the NoSchedule taint to
'control-plane'.

Fix the script to handle both pre and post version 1.25

Signed-off-by: Peter W. Morreale <pwmorreale@gmail.com>

clr-k8s-examples/0-canal/overlays/v3.18/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - canal/canal.yaml

clr-k8s-examples/0-canal/overlays/v3.22/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+

clr-k8s-examples/0-canal/overlays/v3.24/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - canal/canal.yaml
+
+

clr-k8s-examples/0-cilium/overlays/v1.9.13/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        cilium/cilium.yaml

clr-k8s-examples/0-cilium/overlays/v1.9/values.yaml

@@ -0,0 +1,5 @@
+ipam:
+  mode: "cluster-pool"
+  operator:
+    clusterPoolIPv4PodCIDR: "10.244.0.0/16"
+    clusterPoolIPv4MaskSize: 24

clr-k8s-examples/0-flannel/overlays/v0.14.0-rc1/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/0-flannel/overlays/v0.16.3/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+        - flannel/Documentation/kube-flannel.yml

clr-k8s-examples/1-core-metrics/overlays/v0.6.1/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - components.yaml
+patchesJson6902:
+  - target:
+      version: v1
+      kind: Deployment
+      name: metrics-server
+    path: patch_metricstls.yaml

clr-k8s-examples/1-core-metrics/overlays/v0.6.1/patch_metricstls.yaml

@@ -0,0 +1,3 @@
+- op: add
+  path: "/spec/template/spec/containers/0/args/-"
+  value: --kubelet-insecure-tls

clr-k8s-examples/2-dashboard/overlays/v2.6.1/dashboard-admin.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin-user
+  namespace: kubernetes-dashboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: admin-user
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: admin-user
+  namespace: kubernetes-dashboard

clr-k8s-examples/2-dashboard/overlays/v2.6.1/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - dashboard/aio/deploy/recommended.yaml
+  - dashboard-admin.yaml
+

clr-k8s-examples/3-efk/overlays/193692c92eb4667b8f4fb7d4cdf0462e229b5f13/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - instrumentation-addons/fluentd-elasticsearch/create-logging-namespace.yaml
+  - instrumentation-addons/fluentd-elasticsearch/es-service.yaml
+  - instrumentation-addons/fluentd-elasticsearch/es-statefulset.yaml
+  - instrumentation-addons/fluentd-elasticsearch/fluentd-es-configmap.yaml
+  - instrumentation-addons/fluentd-elasticsearch/fluentd-es-ds.yaml
+  - instrumentation-addons/fluentd-elasticsearch/kibana-deployment.yaml
+  - instrumentation-addons/fluentd-elasticsearch/kibana-service.yaml

clr-k8s-examples/4-kube-prometheus/overlays/v0.10.0/kustomization.yaml

@@ -0,0 +1,76 @@
+resources:
+  - kube-prometheus/manifests/kubeStateMetrics-serviceAccount.yaml
+  - kube-prometheus/manifests/blackboxExporter-clusterRole.yaml
+  - kube-prometheus/manifests/nodeExporter-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheus-prometheusRule.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml
+  - kube-prometheus/manifests/prometheus-serviceMonitor.yaml
+  - kube-prometheus/manifests/grafana-dashboardSources.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-prometheusRule.yaml
+  - kube-prometheus/manifests/kubePrometheus-prometheusRule.yaml
+  - kube-prometheus/manifests/prometheus-clusterRole.yaml
+  - kube-prometheus/manifests/blackboxExporter-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheus-roleSpecificNamespaces.yaml
+  - kube-prometheus/manifests/alertmanager-service.yaml
+  - kube-prometheus/manifests/prometheusAdapter-serviceMonitor.yaml
+  - kube-prometheus/manifests/nodeExporter-prometheusRule.yaml
+  - kube-prometheus/manifests/nodeExporter-service.yaml
+  - kube-prometheus/manifests/prometheus-roleConfig.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-clusterRole.yaml
+  - kube-prometheus/manifests/prometheusOperator-deployment.yaml
+  - kube-prometheus/manifests/prometheusOperator-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheusAdapter-deployment.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
+  - kube-prometheus/manifests/prometheusAdapter-configMap.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-prometheusRule.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-deployment.yaml
+  - kube-prometheus/manifests/blackboxExporter-configuration.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/blackboxExporter-serviceAccount.yaml
+  - kube-prometheus/manifests/grafana-dashboardDefinitions.yaml
+  - kube-prometheus/manifests/prometheusOperator-service.yaml
+  - kube-prometheus/manifests/grafana-service.yaml
+  - kube-prometheus/manifests/prometheus-prometheus.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml
+  - kube-prometheus/manifests/alertmanager-alertmanager.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml
+  - kube-prometheus/manifests/grafana-dashboardDatasources.yaml
+  - kube-prometheus/manifests/kubernetesControlPlane-serviceMonitorCoreDNS.yaml
+  - kube-prometheus/manifests/alertmanager-serviceMonitor.yaml
+  - kube-prometheus/manifests/grafana-deployment.yaml
+  - kube-prometheus/manifests/grafana-serviceAccount.yaml
+  - kube-prometheus/manifests/alertmanager-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml
+  - kube-prometheus/manifests/prometheusOperator-prometheusRule.yaml
+  - kube-prometheus/manifests/alertmanager-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/prometheus-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheus-service.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleServerResources.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheus-roleBindingConfig.yaml
+  - kube-prometheus/manifests/nodeExporter-daemonset.yaml
+  - kube-prometheus/manifests/prometheus-roleBindingSpecificNamespaces.yaml
+  - kube-prometheus/manifests/nodeExporter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheusOperator-serviceAccount.yaml
+  - kube-prometheus/manifests/prometheusOperator-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-serviceMonitor.yaml
+  - kube-prometheus/manifests/prometheusAdapter-roleBindingAuthReader.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRoleBindingDelegator.yaml
+  - kube-prometheus/manifests/prometheusAdapter-serviceAccount.yaml
+  - kube-prometheus/manifests/blackboxExporter-deployment.yaml
+  - kube-prometheus/manifests/alertmanager-prometheusRule.yaml
+  - kube-prometheus/manifests/prometheus-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/prometheusAdapter-clusterRole.yaml
+  - kube-prometheus/manifests/grafana-serviceMonitor.yaml
+  - kube-prometheus/manifests/nodeExporter-clusterRole.yaml
+  - kube-prometheus/manifests/prometheusAdapter-service.yaml
+  - kube-prometheus/manifests/prometheus-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/blackboxExporter-service.yaml
+  - kube-prometheus/manifests/nodeExporter-serviceMonitor.yaml
+  - kube-prometheus/manifests/blackboxExporter-clusterRoleBinding.yaml
+  - kube-prometheus/manifests/alertmanager-secret.yaml
+  - kube-prometheus/manifests/prometheusAdapter-apiService.yaml
+  - kube-prometheus/manifests/prometheusOperator-clusterRole.yaml
+  - kube-prometheus/manifests/grafana-config.yaml
+  - kube-prometheus/manifests/prometheusAdapter-podDisruptionBudget.yaml
+  - kube-prometheus/manifests/kubeStateMetrics-service.yaml

clr-k8s-examples/5-ingres-lb/overlays/controller-v1.3.0/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ingress-nginx/deploy/static/provider/baremetal/deploy.yaml

clr-k8s-examples/7-rook/overlays/v1.8.10/multinode/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  - probe_timeout.yaml

clr-k8s-examples/7-rook/overlays/v1.8.10/multinode/probe_timeout.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion:  ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  healthCheck:
+    startupProbe:
+      osd:
+        probe:
+           timeoutSeconds: 120
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 10
+           successThreshold: 1

clr-k8s-examples/7-rook/overlays/v1.8.10/standalone/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml
+  - probe_timeout.yaml

clr-k8s-examples/7-rook/overlays/v1.8.10/standalone/patch_cephcluster.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

clr-k8s-examples/7-rook/overlays/v1.8.10/standalone/probe_timeout.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion:  ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  healthCheck:
+    startupProbe:
+      mon:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1
+      mgr:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1
+      osd:
+        probe:
+           timeoutSeconds: 10
+           initialDelaySeconds: 100
+           periodSeconds: 10
+           failureThreshold: 12
+           successThreshold: 1

clr-k8s-examples/7-rook/overlays/v1.8.6/multinode/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml

clr-k8s-examples/7-rook/overlays/v1.8.6/standalone/kustomization.yaml

@@ -0,0 +1,8 @@
+resources:
+  - rook/deploy/examples/cluster.yaml
+  - rook/deploy/examples/csi/rbd/storageclass.yaml
+
+patchesStrategicMerge:
+  # patches rook to use 'directories' instead of partitions.
+  # comment out to use partitions
+  - patch_cephcluster.yaml

clr-k8s-examples/7-rook/overlays/v1.8.6/standalone/patch_cephcluster.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephCluster
+metadata:
+  name: rook-ceph
+  namespace: rook-ceph
+spec:
+  mon:
+    allowMultiplePerNode: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: replicapool
+  namespace: rook-ceph
+spec:
+  replicated:
+    requireSafeReplicaSize: false
+    size: 1

clr-k8s-examples/9-multi-network/Dockerfile

@@ -1,20 +1,28 @@
 # Build multus plugin
-FROM golang:1.10 AS multus
+FROM busybox AS multus
-RUN git clone -q --depth 1 https://github.com/intel/multus-cni.git /go/src/github.com/intel/multus-cni
+ARG MULTUS_VER=3.4.2
-WORKDIR /go/src/github.com/intel/multus-cni
+RUN wget -O multus.tgz https://github.com/intel/multus-cni/releases/download/v${MULTUS_VER}/multus-cni_${MULTUS_VER}_linux_amd64.tar.gz
-RUN ./build
+RUN tar xvzf multus.tgz --strip-components=1 -C /bin
 
 # Build sriov plugin
-FROM golang:1.10 AS sriov-cni
+FROM golang AS sriov-cni
-RUN git clone -q -b dev/k8s-deviceid-model https://github.com/Intel-Corp/sriov-cni.git /go/src/github.com/intel-corp/sriov-cni
+ARG SRIOV_CNI_VER=2.3
-WORKDIR /go/src/github.com/intel-corp/sriov-cni
+RUN wget -qO sriov-cni.tgz https://github.com/intel/sriov-cni/archive/v${SRIOV_CNI_VER}.tar.gz
-RUN ./build
+RUN mkdir -p sriov-cni && \
+    tar xzf sriov-cni.tgz --strip-components=1 -C sriov-cni && \
+    cd sriov-cni && \
+    make && \
+    cp build/sriov /bin
 
 # Build sriov device plugin
-FROM golang:1.10 AS sriov-dp
+FROM golang AS sriov-dp
-RUN git clone -q https://github.com/intel/sriov-network-device-plugin.git /go/src/github.com/intel/sriov-network-device-plugin
+ARG SRIOV_DP_VER=3.2
-WORKDIR /go/src/github.com/intel/sriov-network-device-plugin
+RUN wget -qO sriov-dp.tgz https://github.com/intel/sriov-network-device-plugin/archive/v${SRIOV_DP_VER}.tar.gz
-RUN make
+RUN mkdir -p sriov-dp && \
+    tar xzf sriov-dp.tgz --strip-components=1 -C sriov-dp && \
+    cd sriov-dp && \
+    make && \
+    cp build/sriovdp /bin
 
 # Build vfioveth plugin
 FROM busybox as vfioveth
@@ -25,9 +33,9 @@ RUN chmod +x /bin/vfioveth /bin/jq
 # Final image
 FROM centos/systemd
 WORKDIR /tmp/cni/bin
-COPY --from=multus /go/src/github.com/intel/multus-cni/bin/multus .
+COPY --from=multus /bin/multus-cni .
-COPY --from=sriov-cni /go/src/github.com/intel-corp/sriov-cni/bin/sriov .
+COPY --from=sriov-cni /bin/sriov .
 COPY --from=vfioveth /bin/vfioveth .
 COPY --from=vfioveth /bin/jq .
 WORKDIR /usr/bin
-COPY --from=sriov-dp /go/src/github.com/intel/sriov-network-device-plugin/build/sriovdp .
+COPY --from=sriov-dp /bin/sriovdp .

clr-k8s-examples/9-multi-network/README.md

@@ -9,11 +9,10 @@ directories on the host with the necessary binaries and configuration files.
 
 ### Customization
 
-The device plugin will register the SR-IOV enabled devices on the host, specified as
+The device plugin will register the SR-IOV enabled devices on the host, specified with
-`rootDevices` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
+`selectors` in [sriov-conf.yaml](sriov-conf.yaml). Helper [systemd unit](systemd/sriov.service)
-file is provided, which enables SR-IOV for the above `rootDevices`
+file is provided, which enables SR-IOV for the above devices. More config options
-
+are listed [here](https://github.com/intel/sriov-network-device-plugin#configurations).
-> NOTE: This assumes homogenous nodes in the cluster
 
 ### Pre-req (SR-IOV only)
 

clr-k8s-examples/9-multi-network/cni/vfioveth

@@ -50,9 +50,9 @@ delete_pair_ns() {
 case $CNI_COMMAND in
 ADD)
 	res=$(ipam)
-	ip=$(echo $res | jq -r '.ip4.ip')
+	ip=$(echo $res | jq -r '.ips[0].address')
 	add_pair_ns $ip
-	echo '{"cniVersion":"0.2.0"}' | jq -c --arg ip $ip '.ip4.ip = $ip' >&3
+	echo '{"cniVersion":"0.3.1"}' | jq -c --arg ip $ip '.ips[0].address = $ip' >&3
 	;;
 DEL)
 	set +o errexit

clr-k8s-examples/9-multi-network/multus-sriov-ds.yaml

@@ -2,25 +2,34 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
-  # name must match the spec fields below, and be in the form: <plural>.<group>
   name: network-attachment-definitions.k8s.cni.cncf.io
 spec:
-  # group name to use for REST API: /apis/<group>/<version>
   group: k8s.cni.cncf.io
-  # version name to use for REST API: /apis/<group>/<version>
-  version: v1
-  # either Namespaced or Cluster
   scope: Namespaced
   names:
-    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
     plural: network-attachment-definitions
-    # singular name to be used as an alias on the CLI and for display
     singular: network-attachment-definition
-    # kind is normally the CamelCased singular type. Your resource manifests use this.
     kind: NetworkAttachmentDefinition
-    # shortNames allow shorter string to match your resource on the CLI
     shortNames:
     - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -40,17 +49,43 @@ type: kubernetes.io/service-account-token
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: multus-pod-networks-lister
+  name: multus-pod-updater
 rules:
-- apiGroups: [""]
+  - apiGroups: ["k8s.cni.cncf.io"]
-  resources: ["pods"]
+    resources:
-  verbs: ["get"]
+      - '*'
-- apiGroups: [""]
+    verbs:
-  resources: ["pods/status"]
+      - '*'
-  verbs: ["update"]
+  - apiGroups:
-- apiGroups: ["k8s.cni.cncf.io"]
+      - ""
-  resources: ["*"]
+    resources:
-  verbs: ["get"]
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus-rb
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus-pod-updater
+subjects:
+- kind: ServiceAccount
+  name: multus-sa
+  namespace: kube-system
 ---
 kind: ConfigMap
 apiVersion: v1
@@ -94,8 +129,9 @@ data:
     MASTER_PLUGIN_JSON="$(cat /host/etc/cni/net.d/$MASTER_PLUGIN)"
     cat > /host/etc/cni/net.d/00-multus.conf <<EOF
     {
+      "cniVersion": "0.3.1",
       "name": "multus-cni-network",
-      "type": "multus",
+      "type": "multus-cni",
       "logFile": "/var/log/multus.log",
       "logLevel": "debug",
       "kubeconfig": "/etc/cni/net.d/multus-kubeconfig",
@@ -105,19 +141,6 @@ data:
     }
     EOF
 ---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus-rb
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus-pod-networks-lister
-subjects:
-- kind: ServiceAccount
-  name: multus-sa
-  namespace: kube-system
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -134,14 +157,14 @@ spec:
     spec:
       initContainers:
       - name: multus
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "bash", "-c" ]
         args:
-        - cp /tmp/cni/bin/{multus,sriov,vfioveth,jq} /host/opt/cni/bin/;
+        - cp /tmp/cni/bin/{multus-cni,sriov,vfioveth,jq} /host/opt/cni/bin/;
           /tmp/multus/install-multus-conf.sh;
           /tmp/multus/install-certs.sh;
           echo "Restarting crio kubelet";
-          systemctl restart crio; # Needed when crio manages ns lifecycle
+          systemctl restart crio;
           systemctl restart kubelet;
         volumeMounts:
         - name: usr-bin
@@ -160,7 +183,7 @@ spec:
           mountPath: /run/systemd
       containers:
       - name: sriovdp
-        image: krsna1729/multus-sriov:k8s-1.13
+        image: krsna1729/multus-sriov:3.4.2-2.3-3.2
         command: [ "sh", "-c" ]
         args:
         - /usr/bin/sriovdp --logtostderr -v 10;
@@ -209,4 +232,3 @@ spec:
           path: /var/lib/kubelet/device-plugins/
       hostNetwork: true
       hostPID: true
-

clr-k8s-examples/9-multi-network/sriov-conf.yaml

@@ -11,15 +11,15 @@ data:
       [
         {
           "resourceName": "sriov_netdevice",
-          "rootDevices": ["07:00.0"],
+          "selectors": {
-          "sriovMode": true,
+                  "drivers": ["i40evf", "iavf"]
-          "deviceType": "netdevice"
+          }
         },
         {
           "resourceName": "sriov_vfio",
-          "rootDevices": ["07:00.1"],
+          "selectors": {
-          "sriovMode": true,
+                  "drivers": ["vfio-pci"]
-          "deviceType": "vfio"
+          }
         }
       ]
     }

clr-k8s-examples/9-multi-network/test/sriov/2-pod-dpdk-ver.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1711
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:17.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1811
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:18.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-1911
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:19.11
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dpdk-2002
+  annotations:
+      k8s.v1.cni.cncf.io/networks: sriov-net-dpdk
+spec:
+  restartPolicy: Never
+  containers:
+  - name: dpdk
+    image: krsna1729/dpdk:20.02
+    stdin: true
+    tty: true
+    command: [ "/bin/bash", "-c"]
+    args:
+    - ls -l /dev/vfio;
+      testpmd --no-huge -m 2048 -- --stats-period=10 --nb-port=1 --port-topology=chained --auto-start --total-num-mbufs=2048 --forward-mode=macswap;
+    securityContext:
+      capabilities:
+        add:
+        - IPC_LOCK
+    resources:
+      limits:
+        intel.com/sriov_vfio: '1'

clr-k8s-examples/9-multi-network/test/sriov/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:bionic as ubuntu-build
+RUN apt-get update && \
+    apt-get -y install \
+        build-essential \
+        git \
+        libnuma-dev
+
+ARG DPDK_VER='master'
+ENV DPDK_DIR='/dpdk'
+ENV RTE_TARGET='x86_64-native-linuxapp-gcc'
+RUN git clone -b $DPDK_VER -q --depth 1 http://dpdk.org/git/dpdk-stable $DPDK_DIR 2>&1
+RUN cd ${DPDK_DIR} && \
+    sed -ri 's,(IGB_UIO=).*,\1n,' config/common_linux* && \
+    sed -ri 's,(KNI_KMOD=).*,\1n,' config/common_linux* && \
+    make config T=x86_64-native-linuxapp-gcc && \
+    make -j $CPUS
+ENV PATH="$PATH:$DPDK_DIR/build/app/"

clr-k8s-examples/README.md

@@ -3,6 +3,11 @@
 ## Prerequisite
 This setup currently will work with Kubernetes 1.14 & above. Any version of Kubernetes before that might work, but is not guaranteed.
 
+## QUICK NOTE
+The version of Kubernetes* was bumped from 1.17.7 to 1.19.4 in Clear Linux* OS release 34090. The [guide](https://docs.01.org/clearlinux/latest/guides/clear/k8s-migration.html) and the Clear Linux OS bundle k8s-migration were created to help facilitate migration of a cluster from 1.17.x to the latest 1.19.x .
+
+The new Clear Linux OS bundle k8s-migration was added in Clear Linux* OS release 34270. Please follow the guide for an upgrade.
+
 ## Sample multi-node vagrant setup
 
 To be able to test this tool, you can create a 3-node vagrant setup. In this tutorial, we will talk about using [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt), but you can use any hypervisor that you are familiar with.
@@ -63,6 +68,28 @@ devicemapper storage. This should not be used for production.
 
 > NOTE: This step is done automatically if using vagrant.
 
+### For HA, setup the load balancer node
+
+Ideally, the load balancer node will be a separate node. However, one of the
+master nodes can also serve as the load balancer for the cluster. [HAProxy](https://www.haproxy.org/)
+is used in these instructions.
+
+```bash
+sudo swupd bundle-add haproxy
+sudo systemctl enable haproxy
+```
+
+Edit the master IP addresses and load balancer address and ports in [`haproxy.cfg.example`](haproxy.cfg.example)
+to match the IPs for the new cluster. If using a master node for the load balancer
+make sure that the `frontend bind` port is different than the Kubernetes API port, 6443.
+If using a separate machine for load balancing, the port can be 6443 if desired.
+
+```bash
+sudo mkdir -p /etc/haproxy
+sudo cp haproxy.cfg.example /etc/haproxy/haproxy.cfg
+sudo systemctl start haproxy
+```
+
 ## Bring up the master
 
 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -75,8 +102,9 @@ more information.
 
 > NOTE: Before running [`create_stack.sh`](create_stack.sh) script, make sure to export
 the necessary environment variables if needed to be changed. By default it will use
-`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the 
+`CLRK8S_CNI` to be canal, and `CLRK8S_RUNNER` to be crio. Cilium is tested only in the
-Vagrant.
+Vagrant. If creating an HA cluster, make sure to specify `LOAD_BALANCER_IP` and
+`LOAD_BALANCER_PORT`.
 
 ```bash
 # default shows help
@@ -86,6 +114,13 @@ Vagrant.
 In order to enable running greater than 110 pods per node, set the environment
 variable `HIGH_POD_COUNT` to any non-empty value.
 
+If creating an HA cluster, join the other master nodes to the cluster.
+
+```bash
+kubeadm join <load-balancer-ip>:<load-balancer-port> --token <token> --discovery-token-ca-cert-hash <hash> \
+    --control-plane --certificate-key <certificate-key> --cri-socket=/run/crio/crio.sock
+```
+
 ## Join Workers to the cluster
 
 ```bash
@@ -94,6 +129,9 @@ kubeadm join <master-ip>:<master-port> --token <token> --discovery-token-ca-cert
 
 Note: Remember to append `--cri-socket=/run/crio/crio.sock` to the join command generated by the master.
 
+If creating an HA cluster, join the other worker nodes to the cluster. The same way,
+but replacing the `<master-ip>:<master-port>` with `<load-balancer-ip>:<load-balancer-port>`.
+
 On workers just use the join command that the master spits out. There nothing
 else you need to run on the worker. All the other Kubernetes customizations are pushed
 in from master via the values setup in the `kubeadm.yaml` file.

clr-k8s-examples/Vagrantfile

@@ -16,7 +16,7 @@ $box = "AntonioMeireles/ClearLinux"
 $box_ver = (ENV['CLEAR_VBOX_VER'])
 File.exists?("/usr/share/qemu/OVMF.fd") ? $loader = "/usr/share/qemu/OVMF.fd" : $loader = File.join(File.dirname(__FILE__), "OVMF.fd")
 $vm_name_prefix = "clr"
-$base_ip = IPAddr.new("192.52.100.10")
+$base_ip = IPAddr.new("10.10.100.10")
 $hosts = {}
 $proxy_ip_list = "192.168.121.0/24"
 $driveletters = ('a'..'z').to_a
@@ -76,6 +76,7 @@ Vagrant.configure("2") do |config|
         lv.loader = $loader
         lv.cpus = $cpus
         lv.memory = $memory
+        lv.machine_virtual_size = 40
         (1..$disks).each do |d|
           lv.storage :file, :device => "hd#{$driveletters[d]}", :path => "disk-#{$disk_prefix}-#{vm_name}-#{d}.disk", :size => $disk_size, :type => "raw"
         end

clr-k8s-examples/create_stack.sh

@@ -14,21 +14,24 @@ SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 : ${MASTER_IP:=}
 : ${CERT_SANS:=}
 HIGH_POD_COUNT=${HIGH_POD_COUNT:-""}
+LOAD_BALANCER_IP=${LOAD_BALANCER_IP:-""}
+LOAD_BALANCER_PORT="${LOAD_BALANCER_PORT:-6444}"
 
 # versions
-CANAL_VER="${CLRK8S_CANAL_VER:-v3.10}"
+CANAL_VER="${CLRK8S_CANAL_VER:-v3.24}"
-CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.6.4}"
+CILIUM_VER="${CLRK8S_CILIUM_VER:-v1.9.13}"
-FLANNEL_VER="${CLRK8S_FLANNEL_VER:-960b3243b9a7faccdfe7b3c09097105e68030ea7}"
+FLANNEL_VER="${CLRK8S_FLANNEL_VER:-v0.16.3}"
+CILIUM_VAL_OVERRIDE=""
 K8S_VER="${CLRK8S_K8S_VER:-}"
-KATA_VER="${CLRK8S_KATA_VER:-1.9.1-kernel-config}"
+KATA_VER="${CLRK8S_KATA_VER:-2.4.0}"
-ROOK_VER="${CLRK8S_ROOK_VER:-v1.2.6}"
+ROOK_VER="${CLRK8S_ROOK_VER:-v1.8.10}"
-METRICS_VER="${CLRK8S_METRICS_VER:-v0.3.6}"
+METRICS_VER="${CLRK8S_METRICS_VER:-v0.6.1}"
-DASHBOARD_VER="${CLRK8S_DASHBOARD_VER:-v2.0.0-beta2}"
+DASHBOARD_VER="${CLRK8S_DASHBOARD_VER:-v2.6.1}"
-INGRES_VER="${CLRK8S_INGRES_VER:-nginx-0.26.1}"
+INGRES_VER="${CLRK8S_INGRES_VER:-controller-v1.3.0}"
-EFK_VER="${CLRK8S_EFK_VER:-v1.15.1}"
+EFK_VER="${CLRK8S_EFK_VER:-193692c92eb4667b8f4fb7d4cdf0462e229b5f13}"
 METALLB_VER="${CLRK8S_METALLB_VER:-v0.8.3}"
 NPD_VER="${CLRK8S_NPD_VER:-v0.6.6}"
-PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-f458e85e5d7675f7bc253072e1b4c8892b51af0f}"
+PROMETHEUS_VER="${CLRK8S_PROMETHEUS_VER:-v0.10.0}"
 CNI=${CLRK8S_CNI:-"canal"}
 if [[ -z "${RUNNER+x}" ]]; then RUNNER="${CLRK8S_RUNNER:-"crio"}"; fi
 
@@ -90,7 +93,13 @@ function cluster_init() {
 		echo "/var/lib/etcd exists! skipping init."
 		return
 	fi
-	sudo -E kubeadm init --config=./kubeadm.yaml
+
+	if [[ -n "${LOAD_BALANCER_IP}" ]]; then
+		sed -i "s/ClusterConfiguration/ClusterConfiguration\ncontrolPlaneEndpoint: ${LOAD_BALANCER_IP}:${LOAD_BALANCER_PORT}/g" ./kubeadm.yaml
+	fi
+	# upload-certs will automatically upload certificates that should be shared
+	# across control-plane nodes in HA clusters. It is harmless in non-HA cases.
+	sudo -E kubeadm init --upload-certs --config=./kubeadm.yaml
 
 	rm -rf "${HOME}/.kube"
 	mkdir -p "${HOME}/.kube"
@@ -106,20 +115,30 @@ function cluster_init() {
 		fi
 	fi
 
-	#Ensure single node k8s works
+	#Ensure single node k8s works both pre and post v1.25
 	if [ "$(kubectl get nodes | wc -l)" -eq 2 ]; then
-		kubectl taint nodes --all node-role.kubernetes.io/master-
+		minor=$(kubeadm version -o short | cut -f 2 -d "." )
+		if [ $minor -ge "25" ]; then
+			kubectl taint nodes --all  node-role.kubernetes.io/control-plane-
+		elif [ $minor -eq "24" ]; then
+			kubectl taint nodes --all  node-role.kubernetes.io/control-plane-
+			kubectl taint nodes --all  node-role.kubernetes.io/master-
+		else
+			kubectl taint nodes --all  node-role.kubernetes.io/master-
+		fi
 		mode="standalone"
 	fi
 }
 
 function kata() {
 	KATA_VER=${1:-$KATA_VER}
-	KATA_URL="https://github.com/kata-containers/packaging.git"
+	KATA_URL="https://github.com/kata-containers/kata-containers.git"
 	KATA_DIR="8-kata"
 	get_repo "${KATA_URL}" "${KATA_DIR}/overlays/${KATA_VER}"
-	set_repo_version "${KATA_VER}" "${KATA_DIR}/overlays/${KATA_VER}/packaging"
+	set_repo_version "${KATA_VER}" "${KATA_DIR}/overlays/${KATA_VER}/kata-containers"
-	kubectl apply -k "${KATA_DIR}/overlays/${KATA_VER}"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/kata-rbac/base/kata-rbac.yaml"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
+	kubectl apply -f "${KATA_DIR}/overlays/${KATA_VER}/kata-containers/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml"
 
 }
 
@@ -128,7 +147,7 @@ function cni() {
 	canal)
 		# note version is not semver
 		CANAL_VER=${1:-$CANAL_VER}
-		CANAL_URL="https://docs.projectcalico.org/${CANAL_VER}/manifests"
+		CANAL_URL="https://projectcalico.docs.tigera.io/archive/${CANAL_VER}/manifests"
 		if [[ "$CANAL_VER" == "v3.3" ]]; then
 			CANAL_URL="https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/canal"
 		fi
@@ -136,7 +155,7 @@ function cni() {
 
 		# canal manifests are not kept in repo but in docs site so use curl
 		mkdir -p "${CANAL_DIR}/overlays/${CANAL_VER}/canal"
-		curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
+		curl -L -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/canal.yaml" "$CANAL_URL/canal.yaml"
 		if [[ "$CANAL_VER" == "v3.3" ]]; then
 			curl -o "${CANAL_DIR}/overlays/${CANAL_VER}/canal/rbac.yaml" "$CANAL_URL/rbac.yaml"
 		fi
@@ -145,7 +164,7 @@ function cni() {
 		;;
 	flannel)
 		FLANNEL_VER=${1:-$FLANNEL_VER}
-		FLANNEL_URL="https://github.com/coreos/flannel"
+		FLANNEL_URL="https://github.com/flannel-io/flannel"
 		FLANNEL_DIR="0-flannel"
 
 		get_repo "${FLANNEL_URL}" "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
@@ -153,13 +172,17 @@ function cni() {
 		kubectl apply -k "${FLANNEL_DIR}/overlays/${FLANNEL_VER}"
 		;;
 	cilium)
+		local podsubnet=$(grep -Po 'podSubnet:\ \K[^*]*' ${SCRIPT_DIR}/kubeadm.yaml)
 		CILIUM_VER=${1:-$CILIUM_VER}
 		CILIUM_URL="https://github.com/cilium/cilium.git"
 		CILIUM_DIR="0-cilium"
 
 		get_repo "${CILIUM_URL}" "${CILIUM_DIR}/overlays/${CILIUM_VER}"
 		set_repo_version "${CILIUM_VER}" "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/"
-		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set global.containerRuntime.integration="$RUNNER" | kubectl apply -f -
+		if [ -f "${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml" ]; then
+			CILIUM_VAL_OVERRIDE="--values ${CILIUM_DIR}/overlays/${CILIUM_VER}/values.yaml"
+		fi
+		helm template "${CILIUM_DIR}/overlays/${CILIUM_VER}/cilium/install/kubernetes/cilium" --namespace kube-system --set containerRuntime.integration="$RUNNER" --set hubble.enabled=false --set ipam.operator.clusterPoolIPv4PodCIDR="${podsubnet}" | kubectl apply -f -
 		;;
 	*)
 		echo"Unknown cni $CNI"
@@ -170,21 +193,19 @@ function cni() {
 
 function metrics() {
 	METRICS_VER="${1:-$METRICS_VER}"
-	METRICS_URL="https://github.com/kubernetes-sigs/metrics-server.git"
+	METRICS_URL="https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_VER}/components.yaml"
 	METRICS_DIR="1-core-metrics"
-	get_repo "${METRICS_URL}" "${METRICS_DIR}/overlays/${METRICS_VER}"
+
-	set_repo_version "${METRICS_VER}" "${METRICS_DIR}/overlays/${METRICS_VER}/metrics-server"
+	curl -L ${METRICS_URL} --output - >${METRICS_DIR}/overlays/${METRICS_VER}/components.yaml
-	kubectl apply -k "${METRICS_DIR}/overlays/${METRICS_VER}"
+	kubectl apply -k "${METRICS_DIR}/overlays/${METRICS_VER}/"
 
 }
 function wait_on_pvc() {
 	# create and destroy pvc until successful
 	while [[ $(kubectl get pvc test-pv-claim --no-headers | grep Bound -c) -ne 1 ]]; do
 		sleep 30
-		kubectl delete pvc test-pv-claim
-		create_pvc
-		sleep 10
 	done
+	kubectl delete pvc test-pv-claim
 }
 function create_pvc() {
 	kubectl apply -f - <<HERE
@@ -209,9 +230,19 @@ function storage() {
 	ROOK_URL="https://github.com/rook/rook.git"
 	ROOK_DIR=7-rook
 
+	# This function might be called standalone, so good to check the mode we are in.
+	if [ "$(kubectl get nodes --no-headers | wc -l)" -eq 1 ]; then
+		mode="standalone"
+	fi
+
 	# get and apply rook
 	get_repo "${ROOK_URL}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
 	set_repo_version "${ROOK_VER}" "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook"
+	kubectl apply -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/crds.yaml -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/common.yaml -f ${ROOK_DIR}/overlays/${ROOK_VER}/${mode}/rook/deploy/examples/operator.yaml
+	while [[ $(kubectl get po -n rook-ceph --field-selector=status.phase=Running | grep -e 'operator' -c) -lt 1 ]]; do
+		echo "Waiting on operator"
+		sleep 10
+	done
 	kubectl apply -k "${ROOK_DIR}/overlays/${ROOK_VER}/${mode}"
 	# wait for the rook OSDs to run which means rooks should be ready
 	while [[ $(kubectl get po --all-namespaces | grep -e 'osd.*Running.*' -c) -lt 1 ]]; do
@@ -233,13 +264,15 @@ function monitoring() {
 	PROMETHEUS_DIR="4-kube-prometheus"
 	get_repo "${PROMETHEUS_URL}" "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}"
 	set_repo_version "${PROMETHEUS_VER}" "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/kube-prometheus"
-	kubectl apply -k "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}"
+	kubectl apply --server-side -f "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/kube-prometheus/manifests/setup/"
 
-	while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com >/dev/null 2>&1) || $? -ne 0 ]]; do
+	while ! [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com) ]]; do
-		echo "Waiting for Prometheus CRDs"
+		echo "Waiting for prometheus crds"
-		sleep 2
+		sleep 10
 	done
 
+	kubectl apply -k "${PROMETHEUS_DIR}/overlays/${PROMETHEUS_VER}/"
+
 	#Expose the dashboards
 	#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
 	#kubectl --namespace monitoring port-forward svc/grafana 3000 &
@@ -253,6 +286,8 @@ function dashboard() {
 	get_repo "${DASHBOARD_URL}" "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}"
 	set_repo_version "${DASHBOARD_VER}" "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}/dashboard"
 	kubectl apply -k "${DASHBOARD_DIR}/overlays/${DASHBOARD_VER}"
+
+	echo 'Run "kubectl -n kubernetes-dashboard create token admin-user" to create an admin token'
 }
 
 function ingres() {
@@ -266,10 +301,10 @@ function ingres() {
 
 function efk() {
 	EFK_VER=${1:-$EFK_VER}
-	EFK_URL="https://github.com/kubernetes/kubernetes.git"
+	EFK_URL="https://github.com/kubernetes-sigs/instrumentation-addons.git"
 	EFK_DIR="3-efk"
 	get_repo "${EFK_URL}" "${EFK_DIR}/overlays/${EFK_VER}"
-	set_repo_version "${EFK_VER}" "${EFK_DIR}/overlays/${EFK_VER}/kubernetes"
+	set_repo_version "${EFK_VER}" "${EFK_DIR}/overlays/${EFK_VER}/instrumentation-addons"
 	kubectl apply -k "${EFK_DIR}/overlays/${EFK_VER}"
 
 }
@@ -330,7 +365,7 @@ function minimal() {
 
 function all() {
 	minimal
-	storage
+  storage
 	monitoring
 	miscellaneous
 }
@@ -369,6 +404,11 @@ command_handlers[monitoring]=monitoring
 command_handlers[metallb]=metallb
 command_handlers[npd]=npd
 command_handlers[nfd]=nfd
+command_handlers[kata]=kata
+command_handlers[metrics]=metrics
+command_handlers[dashboard]=dashboard
+command_handlers[efk]=efk
+command_handlers[ingres]=ingres
 
 declare -A command_help
 command_help[init]="Only inits a cluster using kubeadm"

clr-k8s-examples/haproxy.cfg.example

@@ -0,0 +1,34 @@
+global
+    log /dev/log    local0
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy-master.sock mode 660 level admin
+    stats timeout 30s
+    user haproxy
+    group haproxy
+    daemon
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
+    ssl-default-bind-options no-sslv3
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+    timeout tunnel  4h
+frontend kubernetes
+    bind 10.0.0.100:6444
+    option tcplog
+    mode tcp
+    default_backend kubernetes-master-nodes
+
+backend kubernetes-master-nodes
+    mode tcp
+    balance source
+    option tcp-check
+    server master-1 10.0.0.100:6443 check fall 3 rise 2
+    server master-2 10.0.0.101:6443 check fall 3 rise 2
+    server master-3 10.0.0.102:6443 check fall 3 rise 2

clr-k8s-examples/kubeadm.yaml

@@ -1,11 +1,9 @@
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 cgroupDriver: systemd
-# Allowing for CPU pinning and isolation in case of guaranteed QoS class
-cpuManagerPolicy: static
 systemReserved:
   cpu: 500m
   memory: 256M
@@ -13,7 +11,7 @@ kubeReserved:
   cpu: 500m
   memory: 256M
 ---
-apiVersion: kubeadm.k8s.io/v1beta2
+apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
   dnsDomain: cluster.local

clr-k8s-examples/setup_system.sh

@@ -29,13 +29,10 @@ function add_os_deps() {
 
 # permanently disable swap
 function disable_swap() {
-	swapcount=$(sudo grep '^/dev/\([0-9a-z]*\).*' /proc/swaps | wc -l)
+	# disable current swap
-
+	sudo swapoff -a
-	if [ "$swapcount" != "0" ]; then
+	# permanently disable swap
-		sudo systemctl mask "$(sed -n -e 's#^/dev/\([0-9a-z]*\).*#dev-\1.swap#p' /proc/swaps)" 2>/dev/null
+	sudo systemctl mask swap.target
-	else
-		echo "Swap not enabled"
-	fi
 }
 
 # enable ip forwarding
@@ -137,7 +134,6 @@ function enable_kubelet_runner() {
 
 # ensure that the system is ready without requiring a reboot
 function ensure_system_ready() {
-	sudo swapoff -a
 	sudo systemctl restart systemd-modules-load.service
 }
 
@@ -192,12 +188,12 @@ fi
 
 echo "Init..."
 init
+echo "Disabling swap..."
+disable_swap
 echo "Setting OS Version..."
 upate_os_version
 echo "Adding OS Dependencies..."
 add_os_deps
-echo "Disabling swap..."
-disable_swap
 echo "Enabling IP Forwarding..."
 enable_ip_forwarding
 echo "Setting up modules to load..."
@@ -210,7 +206,7 @@ if [[ -n "${HIGH_POD_COUNT}" ]]; then
 fi
 echo "Reloading daemons..."
 daemon_reload
-echo "Enabling Kublet runner..."
+echo "Enabling kubelet runner..."
 enable_kubelet_runner
 echo "Ensuring system is ready..."
 ensure_system_ready