Firecracker: Update README

Update README to call out how to try firecracker workloads.

Signed-off-by: Manohar Castelino <manohar.r.castelino@intel.com>

Makefile

@@ -5,5 +5,5 @@ TARGET ?= $(PREFIX)/share/
 all:
 
 install:
-        install -m 0755 -d $(DESTDIR)/$(TARGET)/clr-k8s-examples
+	install -m 0755 -d $(DESTDIR)/$(TARGET)/clr-k8s-examples
-        cp -r clr-k8s-examples/* $(DESTDIR)/$(TARGET)/clr-k8s-examples/
+	cp -r clr-k8s-examples/* $(DESTDIR)/$(TARGET)/clr-k8s-examples/

clr-k8s-examples/0-canal/canal.yaml

@@ -1,8 +1,8 @@
-# Canal Version v3.3.0
+# Canal Version v3.3.2
-# https://docs.projectcalico.org/v3.3/releases#v3.3.0
+# https://docs.projectcalico.org/v3.3/releases#v3.3.2
 # This manifest includes the following component versions:
-#   calico/node:v3.3.0
+#   calico/node:v3.3.2
-#   calico/cni:v3.3.0
+#   calico/cni:v3.3.2
 #   coreos/flannel:v0.9.1
 
 # This ConfigMap is used to configure a self-hosted Canal installation.
@@ -115,7 +115,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: quay.io/calico/node:v3.3.0
+          image: quay.io/calico/node:v3.3.2
           env:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
@@ -194,7 +194,7 @@ spec:
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: quay.io/calico/cni:v3.3.0
+          image: quay.io/calico/cni:v3.3.2
           command: ["/install-cni.sh"]
           env:
             # Name of the CNI config file to create.

clr-k8s-examples/8-kata/fire-runtimeClass.yaml

@@ -0,0 +1,6 @@
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1alpha1
+metadata:
+    name: fire
+spec:
+    runtimeHandler: fire

clr-k8s-examples/README.md

@@ -20,6 +20,19 @@ This script ensures the following
 * Customizes the system to ensure correct defaults are setup (IP Forwarding, Swap off,...)
 * Ensures all the dependencies are loaded on boot (kernel modules)
 
+> NOTE: This step is done automatically if using vagrant. 
+
+### Enabling experimental firecracker support
+
+> EXPERIMENTAL: Optionally run [`setup_firecracker.sh`](setup_firecracker.sh) to be
+able to use firecracker VMM with Kata.
+
+The firecracker setup switches the setup to use a sparse file backed loop device for
+devicemapper storage. This should not be used for production.
+
+> NOTE: This step is done automatically if using vagrant. 
+
+
 ## Bring up the master
 
 Run [`create_stack.sh`](create_stack.sh) on the master node. This sets up the
@@ -27,6 +40,15 @@ master and also uses kubelet config via [`kubeadm.yaml`](kubeadm.yaml)
 to propagate cluster wide kubelet configuration to all workers. Customize it if
 you need to setup other cluster wide properties.
 
+There are two flavors of install -
+- `minimal`: initialize cluster, add kata runtimeclass, install canal CNI and metrics server
+- `all`: minimal, install rook storage, prometheus, ELK, nginx-ingress, etc.,
+
+```bash
+# default is 'all'
+./create_stack.sh [minimal|all]
+```
+
 ## Join Workers to the cluster
 
 ```bash
@@ -53,6 +75,15 @@ An example is
 
 `kubectl apply -f tests/test-deploy-kata.yaml`
 
+### Running Kata Workloads with Firecracker
+
+> EXPERIMENTAL: If firecracker setup has been enabled, runtime class set to "fire" will launch the POD/Deployment
+with firecracker as the isolation mechanism for Kata.
+
+An example is
+
+`kubectl apply -f tests/test-deploy-fire.yaml`
+
 ## Making Kata the default runtime
 
 Today in `crio.conf` runc is the default runtime when a user does not specify

clr-k8s-examples/Vagrantfile

@@ -74,8 +74,8 @@ Vagrant.configure("2") do |config|
       end
       # Bad hack for the vagrant libvirt boxes. WIll be removed once they are fixed.
       c.vm.provision "shell", privileged: false, inline: "sudo usermod --password vagrant root"
-
       c.vm.provision "shell", privileged: false, path: "setup_system.sh"
+      c.vm.provision "shell", privileged: false, path: "setup_firecracker.sh"
     end
   end
 end

clr-k8s-examples/create_stack.sh

@@ -6,75 +6,126 @@ set -o nounset
 
 CUR_DIR=$(pwd)
 SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_DIR
 
-function finish {
+function print_usage_exit() {
-  cd $CUR_DIR
+	echo $"Usage: $0 [minimal|all]"
+	exit 1
+}
+
+function finish() {
+	cd $CUR_DIR
 }
 trap finish EXIT
 
-#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
+function cluster_init() {
-#to enable the RuntimeClass featuregate
+	#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
-sudo -E kubeadm init --config=./kubeadm.yaml
+	#to enable the RuntimeClass featuregate
+	sudo -E kubeadm init --config=./kubeadm.yaml
 
-# If this an interactive terminal then wait for user to join workers
+	# If this an interactive terminal then wait for user to join workers
-if [ -t 0 ]; then
+	if [ -t 0 ]; then
-read -p "Join other nodes. Press enter to continue"
+		read -p "Join other nodes. Press enter to continue"
+	fi
+
+	rm -rf $HOME/.kube
+	mkdir -p $HOME/.kube
+	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+	sudo chown $(id -u):$(id -g) $HOME/.kube/config
+
+	#Ensure single node k8s works
+	if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
+		kubectl taint nodes --all node-role.kubernetes.io/master-
+	fi
+}
+
+function runtimeclass_kata() {
+	#Add support for kata runtime
+	kubectl apply -f 8-kata/runtimeclass_crd.yaml
+	while [[ $(kubectl get crd runtimeclasses.node.k8s.io >/dev/null 2>&1) || $? -ne 0 ]]; do
+		echo "Waiting for runtime class CRD"
+		sleep 2
+	done
+	kubectl apply -f 8-kata/
+}
+
+function cni() {
+	kubectl apply -f 0-canal/rbac.yaml
+	kubectl apply -f 0-canal/canal.yaml
+}
+
+function metrics() {
+	kubectl apply -f 1-core-metrics/
+}
+
+function storage() {
+	#Start rook before any other component that requires storage
+	ROOK_URL=7-rook
+	kubectl apply -f ${ROOK_URL}/000-operator.yaml
+	while [[ $(kubectl get crd clusters.ceph.rook.io pools.ceph.rook.io >/dev/null 2>&1) || $? -ne 0 ]]; do
+		echo "Waiting for Rook CRDs"
+		sleep 2
+	done
+	kubectl apply -f ${ROOK_URL}/001-cluster.yaml
+	kubectl apply -f ${ROOK_URL}/002-storageclass.yaml
+}
+
+function monitoring() {
+	#Just to allow the CRD to be created. Ideally wait and then run second time
+	kubectl apply -f 4-kube-prometheus/
+	while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com >/dev/null 2>&1) || $? -ne 0 ]]; do
+		echo "Waiting for Prometheus CRDs"
+		sleep 2
+	done
+	kubectl apply -f 4-kube-prometheus/
+
+	#Expose the dashboards
+	#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
+	#kubectl --namespace monitoring port-forward svc/grafana 3000 &
+	#kubectl --namespace monitoring port-forward svc/alertmanager-main 9093 &
+}
+
+function miscellaneous() {
+	kubectl apply -f 2-dashboard/
+	kubectl apply -f 3-efk/
+	#Create an ingress load balancer
+	kubectl apply -f 5-ingres-lb/
+
+	#Create a bare metal load balancer.
+	#kubectl apply -f 6-metal-lb/metallb.yaml
+
+	#The config map should be properly modified to pick a range that can live
+	#on this subnet behind the same gateway (i.e. same L2 domain)
+	#kubectl apply -f 6-metal-lb/example-layer2-config.yaml
+}
+
+function minimal() {
+	cluster_init
+	runtimeclass_kata
+	cni
+	metrics
+}
+
+function all() {
+	minimal
+	storage
+	monitoring
+	miscellaneous
+}
+
+cd $SCRIPT_DIR
+if [[ "$#" -eq 0 ]]; then
+	all
+	exit
 fi
 
-rm -rf $HOME/.kube
+case "$1" in
-mkdir -p $HOME/.kube
+minimal)
-sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+	minimal
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
+	;;
-
+all)
-#Add support for kata runtime
+	all
-kubectl apply -f 8-kata/runtimeclass_crd.yaml
+	;;
-while [[ $(kubectl get crd runtimeclasses.node.k8s.io > /dev/null 2>&1) || $? -ne 0 ]];
+*)
-do echo "Waiting for runtime class CRD"; sleep 2; done
+	print_usage_exit
-kubectl apply -f 8-kata/kata-runtimeClass.yaml
+	;;
-
+esac
-kubectl apply -f 0-canal/rbac.yaml
-kubectl apply -f 0-canal/canal.yaml
-
-#Ensure single node k8s works
-if [ $(kubectl get nodes | wc -l) -eq 2 ]; then
-kubectl taint nodes --all node-role.kubernetes.io/master-
-fi
-
-#Start rook before any other component that requires storage
-ROOK_URL=7-rook
-kubectl apply -f ${ROOK_URL}/000-operator.yaml
-while [[ $(kubectl get crd clusters.ceph.rook.io pools.ceph.rook.io > /dev/null 2>&1) || $? -ne 0 ]];
-do echo "Waiting for Rook CRDs"; sleep 2; done
-kubectl apply -f ${ROOK_URL}/001-cluster.yaml
-kubectl apply -f ${ROOK_URL}/002-storageclass.yaml
-
-kubectl apply -f 1-core-metrics/
-
-kubectl apply -f 2-dashboard/
-
-kubectl apply -f 3-efk/
-
-#Just to allow the CRD to be created. Ideally wait and then run second time
-kubectl apply -f 4-kube-prometheus/
-while [[ $(kubectl get crd alertmanagers.monitoring.coreos.com prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com > /dev/null 2>&1) || $? -ne 0 ]];
-do echo "Waiting for Prometheus CRDs"; sleep 2; done
-kubectl apply -f 4-kube-prometheus/
-#Create an ingress load balancer
-kubectl apply -f 5-ingres-lb/
-
-
-#Create a bare metal load balancer.
-#kubectl apply -f 6-metal-lb/metallb.yaml
-
-#The config map should be properly modified to pick a range that can live
-#on this subnet behind the same gateway (i.e. same L2 domain)
-#kubectl apply -f 6-metal-lb/example-layer2-config.yaml
-
-
-#Expose the dashboards
-#kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090 &
-#kubectl --namespace monitoring port-forward svc/grafana 3000 &
-#kubectl --namespace monitoring port-forward svc/alertmanager-main 9093 &
-#kubectl proxy &
-

clr-k8s-examples/kubeadm.yaml

@@ -7,10 +7,18 @@ apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 featureGates:
   RuntimeClass: true
+# Kata does not work with static
+# https://github.com/kata-containers/runtime/issues/878
+# cpuManagerPolicy: static
+systemReserved:
+  cpu: 500m
+  memory: 256M
+kubeReserved:
+  cpu: 500m
+  memory: 256M
 ---
 apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration
-kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
   podSubnet: 10.244.0.0/16

clr-k8s-examples/reset_stack.sh

@@ -14,7 +14,6 @@ for pod in $(sudo crictl pods --quiet); do
 	sudo crictl rmp "$pod"
 done
 
-
 #Forcefull cleanup all artifacts
 #This is needed is things really go wrong
 sudo systemctl stop kubelet
@@ -45,4 +44,3 @@ sudo systemctl restart crio
 sudo systemctl restart kubelet
 
 sudo -E kubeadm reset -f --cri-socket="/var/run/crio/crio.sock"
-

clr-k8s-examples/setup_firecracker.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+sudo mkdir -p /etc/kata-containers
+
+# Setup a configuration to be used by firecracker
+cat <<EOT | sudo tee /etc/kata-containers/configuration_firecracker.toml
+[hypervisor.firecracker]
+path = "/usr/bin/firecracker"
+kernel = "/usr//share/kata-containers/vmlinux.container"
+image = "/usr//share/kata-containers/kata-containers.img"
+kernel_params = ""
+default_vcpus = 1
+default_memory = 4096
+default_maxvcpus = 0
+default_bridges = 1
+block_device_driver = "virtio-mmio"
+disable_block_device_use = false
+enable_debug = true
+use_vsock = true
+
+[shim.kata]
+path = "/usr//libexec/kata-containers/kata-shim"
+
+[agent.kata]
+
+[runtime]
+internetworking_model="tcfilter"
+EOT
+
+# Firecracker can only work with devicemapper
+# Setup a sparse disk to be used for devicemapper
+sudo rm -f /var/lib/crio/devicemapper/disk.img
+sudo mkdir -p /var/lib/crio/devicemapper
+sudo truncate /var/lib/crio/devicemapper/disk.img --size 10G
+
+# Ensure that this disk is loop mounted at each boot
+sudo mkdir -p /etc/systemd/system
+
+cat <<EOT | sudo tee /etc/systemd/system/devicemapper.service
+[Unit]
+Description=Setup CRIO devicemapper
+DefaultDependencies=no
+After=systemd-udev-settle.service
+Before=lvm2-activation-early.service
+Wants=systemd-udev-settle.service
+
+[Service]
+ExecStart=-/sbin/losetup /dev/loop8 /var/lib/crio/devicemapper/disk.img
+RemainAfterExit=true
+Type=oneshot
+
+[Install]
+WantedBy=local-fs.target
+EOT
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now devicemapper
+
+# For now till we address https://github.com/kubernetes-sigs/cri-o/issues/1991
+# use a shell script to expose firecracker through kata
+cat <<EOT | sudo tee /usr/bin/kata-runtime-fire
+#!/bin/bash
+
+/usr/bin/kata-runtime --kata-config /etc/kata-containers/configuration_firecracker.toml "\$@"
+EOT
+
+sudo chmod +x /usr/bin/kata-runtime-fire
+
+# Add firecracker as a second runtime
+# Also setup crio to use devicemapper
+
+sudo mkdir -p /etc/crio/
+sudo cp /usr/share/defaults/crio/crio.conf /etc/crio/crio.conf
+
+echo -e "\n[crio.runtime.runtimes.kata]\nruntime_path = \"/usr/bin/kata-runtime\"" | sudo tee -a /etc/crio/crio.conf
+echo -e "\n[crio.runtime.runtimes.fire]\nruntime_path = \"/usr/bin/kata-runtime-fire\"" | sudo tee -a /etc/crio/crio.conf
+
+sudo sed -i 's|\(\[crio\.runtime\]\)|\1\nmanage_network_ns_lifecycle = true|' /etc/crio/crio.conf
+
+sudo sed -i 's/storage_driver = \"overlay\"/storage_driver = \"devicemapper\"\
+storage_option = [\
+  \"dm.basesize=8G\",\
+  \"dm.directlvm_device=\/dev\/loop8\",\
+  \"dm.directlvm_device_force=true\",\
+  \"dm.fs=ext4\"\
+]/g' /etc/crio/crio.conf
+
+sudo systemctl restart crio || true

clr-k8s-examples/setup_system.sh

@@ -20,14 +20,14 @@ else
 fi
 
 sudo mkdir -p /etc/sysctl.d/
-cat <<EOT | sudo tee /etc/sysctl.d/60-k8s.conf
+cat <<EOT | sudo bash -c "cat > /etc/sysctl.d/60-k8s.conf"
 net.ipv4.ip_forward=1
 EOT
 sudo systemctl restart systemd-sysctl
 
 #Ensure the modules we need are preloaded
 sudo mkdir -p /etc/modules-load.d/
-cat <<EOT | sudo tee /etc/modules-load.d/k8s.conf
+cat <<EOT | sudo bash -c "cat > /etc/modules-load.d/k8s.conf"
 br_netfilter
 vhost_vsock
 overlay
@@ -35,7 +35,7 @@ EOT
 
 hostcount=$(grep '127.0.0.1 localhost' /etc/hosts | wc -l)
 if [ "$hostcount" == "0" ]; then
-	echo "127.0.0.1 localhost `hostname`" | sudo tee -a /etc/hosts
+	echo "127.0.0.1 localhost $(hostname)" | sudo bash -c "cat >> /etc/hosts"
 else
 	echo "/etc/hosts already configured"
 fi
@@ -43,32 +43,42 @@ fi
 sudo systemctl daemon-reload
 # This will fail at this point, but puts it into a retry loop that
 # will therefore startup later once we have configured with kubeadm.
+echo "The following kubelet command may complain... it is not an error"
 sudo systemctl enable --now kubelet crio || true
 
 sudo mkdir -p /usr/libexec/cni /opt/cni
-sudo ln -s /usr/libexec/cni /opt/cni/bin
+[ ! -e /opt/cni/bin/cni ] && sudo ln -s /usr/libexec/cni /opt/cni/bin
 #Ensure that the system is ready without requiring a reboot
 sudo swapoff -a
-sudo modprobe br_netfilter vhost_vsock overlay
+sudo systemctl restart systemd-modules-load.service
 
 set +o nounset
 if [[ ${http_proxy} ]] || [[ ${HTTP_PROXY} ]]; then
-  echo "Setting up proxy stuff...."
+	echo "Setting up proxy stuff...."
-  # Setup IP for users too
+	# Setup IP for users too
-  sed_val=${ADD_NO_PROXY//\//\\/}
+	sed_val=${ADD_NO_PROXY//\//\\/}
-  sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
+	[ -f /etc/environment ] && sudo sed -i "/no_proxy/I s/$/,${sed_val}/g" /etc/environment
-  sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+	if [ -f /etc/profile.d/proxy.sh ]; then
+		sudo sed -i "/no_proxy/I s/\"$/,${sed_val}\"/g" /etc/profile.d/proxy.sh
+	else
+		echo "Warning, failed to find /etc/profile.d/proxy.sh to edit no_proxy line"
+	fi
 
-  services=('crio' 'docker' 'kubelet')
+	services=('crio' 'kubelet')
-  for s in "${services[@]}"; do
+	for s in "${services[@]}"; do
-    sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
+		sudo mkdir -p "/etc/systemd/system/${s}.service.d/"
-    cat << EOF | sudo tee "/etc/systemd/system/${s}.service.d/proxy.conf"
+		cat <<EOF | sudo bash -c "cat > /etc/systemd/system/${s}.service.d/proxy.conf"
 [Service]
 Environment="HTTP_PROXY=${http_proxy}"
 Environment="HTTPS_PROXY=${https_proxy}"
 Environment="SOCKS_PROXY=${socks_proxy}"
 Environment="NO_PROXY=${no_proxy},${ADD_NO_PROXY}"
 EOF
-  done
+	done
 fi
 set -o nounset
+
+# We have potentially modified their env files, we need to restart the services.
+sudo systemctl daemon-reload
+sudo systemctl restart crio || true
+sudo systemctl restart kubelet || true

clr-k8s-examples/tests/test-cpumanager-kata.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-kata
+spec:
+  runtimeClassName: kata
+  restartPolicy: Never
+  containers:
+  - name: taskset
+    image: busybox
+    command: [ "taskset", "-p", "1" ]
+    resources:
+      limits:
+        cpu: 1
+        memory: 500Mi

clr-k8s-examples/tests/test-cpumanager-runc.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-cpumanager-runc
+spec:
+  restartPolicy: Never
+  containers:
+  - name: taskset
+    image: busybox
+    command: [ "taskset", "-p", "1" ]
+    resources:
+      limits:
+        cpu: 1
+        memory: 100Mi

clr-k8s-examples/tests/test-deploy-fire.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-fire
+  name: php-apache-fire
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: php-apache-fire
+  template:
+    metadata:
+      labels:
+        run: php-apache-fire
+    spec:
+      runtimeClassName: fire
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        imagePullPolicy: Always
+        name: php-apache
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1 
+kind: Service
+metadata:
+  name: php-apache-fire
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-fire
+  sessionAffinity: None
+  type: ClusterIP