diff --git a/metrics/README.md b/metrics/README.md index 7367026..c1b0353 100644 --- a/metrics/README.md +++ b/metrics/README.md @@ -13,6 +13,7 @@ is below: | Tool | Description | | ---- | ----------- | +| config | Configuration scripts | | lib | General library helper functions for forming and launching workloads, and storing results in a uniform manner to aid later analysis | | scaling | Tests to measure scaling, such as linear or parallel launching of pods | | report | Rmarkdown based report generator, used to produce a PDF comparison report of 1 or more sets of results | @@ -20,7 +21,7 @@ is below: ## Results storage and analysis -The tools generate JSON formaated results files via the `lib/json.bash` functions. The `metrics_json_save()` +The tools generate JSON formatted results files via the `lib/json.bash` functions. The `metrics_json_save()` function in that file has the ability to also `curl` or `socat` the JSON results to a database defined by environment variables (see the file source for details). This method has been used to store results in Elasticsearch and InfluxDB databases for instance, but should be adaptable to use with any REST API that accepts diff --git a/metrics/config/README.md b/metrics/config/README.md new file mode 100644 index 0000000..6f79096 --- /dev/null +++ b/metrics/config/README.md @@ -0,0 +1,5 @@ +## Node configuration +In order to push beyond the default maximum for pods per node in Kubernetes, +some additional configuration is necessary on each node in the cluster. For +large pod number (> 110) scaling tests, execute `node-config.sh` on each node +in your cluster before executing other scripts. diff --git a/metrics/config/node-config.sh b/metrics/config/node-config.sh new file mode 100755 index 0000000..f322c67 --- /dev/null +++ b/metrics/config/node-config.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -e + +# this is needed for the scaling scripts +sudo swupd bundle-add jq + +# increase max inotify watchers +cat < /etc/sysctl.conf" +fs.inotify.max_queued_events=1048576 +fs.inotify.max_user_watches=1048576 +fs.inotify.max_user_instances=1048576 +EOT +sudo sysctl -p + +sudo mkdir -p /etc/systemd/system/kubelet.service.d/ +cat < /etc/systemd/system/kubelet.service.d/limits.conf" +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=1048576 +TimeoutStartSec=0 +MemoryLimit=infinity +EOT + +sudo systemctl daemon-reload +sudo systemctl restart kubelet + +sudo mkdir -p /etc/systemd/system/containerd.service.d/ +cat < /etc/systemd/system/containerd.service.d/limits.conf" +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=1048576 +TimeoutStartSec=0 +MemoryLimit=infinity +EOT + +sudo systemctl daemon-reload +sudo systemctl restart containerd + +# increase limits in kubelet +sudo sed -i 's/^maxPods\:.*/maxPods\: 5000/' /var/lib/kubelet/config.yaml +sudo sed -i 's/^maxOpenFiles\:.*/maxOpenFiles\: 1048576/' /var/lib/kubelet/config.yaml + +sudo systemctl daemon-reload +sudo systemctl restart kubelet