WIP Add node problem detector (#160)

Signed-off-by: Justin Scott <justin.a.scott@intel.com>
2026-08-18 21:16:16 +00:00 · 2019-09-03 10:39:38 -07:00
parent 20c7eccf1f
commit 98f8e3eb5e
4 changed files with 128 additions and 6 deletions
@@ -31,12 +31,12 @@ function finish() {
 trap finish EXIT

 function cluster_init() {
-  if ! [ -z ${TOKEN} ]; then
-    sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml
-  fi
-  if ! [ -z ${MASTER_IP} ]; then
-    sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml
-  fi
+	if ! [ -z ${TOKEN} ]; then
+		sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml
+	fi
+	if ! [ -z ${MASTER_IP} ]; then
+		sed -i "/InitConfiguration/a localAPIEndpoint:\\n  advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml
+	fi
 	#This only works with kubernetes 1.12+. The kubeadm.yaml is setup
 	#to enable the RuntimeClass featuregate
 	if [[ -d /var/lib/etcd ]]; then
@@ -200,6 +200,15 @@ function metallb() {

 }

+function npd() {
+	NPD_VER=${1:-v0.6.6}
+	NPD_URL="https://github.com/kubernetes/node-problem-detector.git"
+	NPD_DIR="node-problem-detector"
+	get_repo "${NPD_URL}" "${NPD_DIR}/overlays/${NPD_VER}"
+	set_repo_version "${NPD_VER}" "${NPD_DIR}/overlays/${NPD_VER}/node-problem-detector"
+	kubectl apply -k "${NPD_DIR}/overlays/${NPD_VER}"
+}
+
 function miscellaneous() {

 	# dashboard
@@ -261,6 +270,7 @@ command_handlers[help]=print_usage_exit
 command_handlers[storage]=storage
 command_handlers[monitoring]=monitoring
 command_handlers[metallb]=metallb
+command_handlers[npd]=npd

 declare -A command_help
 command_help[init]="Only inits a cluster using kubeadm"
@@ -0,0 +1,17 @@
+resources:
+  - node-problem-detector/deployment/node-problem-detector-config.yaml
+  - node-problem-detector/deployment/node-problem-detector.yaml
+
+patchesJson6902:
+  # adds rule for mce
+  - target:
+      version: v1
+      kind: ConfigMap
+      name: node-problem-detector-config
+    path: patch_configmap_rules.yaml
+  # adds mce-monitor.json to configmap
+  - target:
+      version: v1
+      kind: ConfigMap
+      name: node-problem-detector-config
+    path: patch_configmap_mce-monitor.yaml
@@ -0,0 +1,22 @@
+# adds "mce" rule
+- op: add
+  path: /data/mce-monitor.json
+  value: |
+      {
+          "plugin": "journald",
+          "pluginConfig": {
+              "source": "mcelog"
+          },
+          "logPath": "/var/log/journal",
+          "lookback": "5m",
+          "bufferSize": 10,
+          "source": "mce-monitor",
+          "conditions": [],
+          "rules": [
+              {
+                  "type": "temporary",
+                  "reason": "Hardware Error",
+                  "pattern": "Hardware event.*"
+              }
+          ]
+      }
@@ -0,0 +1,73 @@
+# adds "mce" rule
+- op: add
+  path: /data/kernel-monitor.json
+  value: |
+    {
+        "plugin": "kmsg",
+        "logPath": "/dev/kmsg",
+        "lookback": "5m",
+        "bufferSize": 10,
+        "source": "kernel-monitor",
+        "conditions": [
+            {
+                "type": "KernelDeadlock",
+                "reason": "KernelHasNoDeadlock",
+                "message": "kernel has no deadlock"
+            },
+            {
+                "type": "ReadonlyFilesystem",
+                "reason": "FilesystemIsReadOnly",
+                "message": "Filesystem is read-only"
+            }
+        ],
+        "rules": [
+            {
+                "type": "temporary",
+                "reason": "Hardware Error",
+                "pattern": "mce:.*"
+            },
+            {
+                "type": "temporary",
+                "reason": "OOMKilling",
+                "pattern": "Kill process \\d+ (.+) score \\d+ or sacrifice child\\nKilled process \\d+ (.+) total-vm:\\d+kB, anon-rss:\\d+kB, file-rss:\\d+kB.*"
+            },
+            {
+                "type": "temporary",
+                "reason": "TaskHung",
+                "pattern": "task \\S+:\\w+ blocked for more than \\w+ seconds\\."
+            },
+            {
+                "type": "temporary",
+                "reason": "UnregisterNetDevice",
+                "pattern": "unregister_netdevice: waiting for \\w+ to become free. Usage count = \\d+"
+            },
+            {
+                "type": "temporary",
+                "reason": "KernelOops",
+                "pattern": "BUG: unable to handle kernel NULL pointer dereference at .*"
+            },
+            {
+                "type": "temporary",
+                "reason": "KernelOops",
+                "pattern": "divide error: 0000 \\[#\\d+\\] SMP"
+            },
+            {
+                "type": "permanent",
+                "condition": "KernelDeadlock",
+                "reason": "AUFSUmountHung",
+                "pattern": "task umount\\.aufs:\\w+ blocked for more than \\w+ seconds\\."
+            },
+            {
+                "type": "permanent",
+                "condition": "KernelDeadlock",
+                "reason": "DockerHung",
+                "pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\."
+            },
+            {
+                "type": "permanent",
+                "condition": "ReadonlyFilesystem",
+                "reason": "FilesystemIsReadOnly",
+                "pattern": "Remounting filesystem read-only"
+            }
+        ]
+    }