From 98f8e3eb5e70d460beb216613760960ceb71a86f Mon Sep 17 00:00:00 2001 From: Justin Scott Date: Tue, 3 Sep 2019 10:39:38 -0700 Subject: [PATCH] WIP Add node problem detector (#160) Signed-off-by: Justin Scott --- clr-k8s-examples/create_stack.sh | 22 ++++-- .../overlays/v0.6.6/kustomization.yaml | 17 +++++ .../v0.6.6/patch_configmap_mce-monitor.yaml | 22 ++++++ .../v0.6.6/patch_configmap_rules.yaml | 73 +++++++++++++++++++ 4 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 clr-k8s-examples/node-problem-detector/overlays/v0.6.6/kustomization.yaml create mode 100644 clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_mce-monitor.yaml create mode 100644 clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_rules.yaml diff --git a/clr-k8s-examples/create_stack.sh b/clr-k8s-examples/create_stack.sh index 238e53f..9b8f641 100755 --- a/clr-k8s-examples/create_stack.sh +++ b/clr-k8s-examples/create_stack.sh @@ -31,12 +31,12 @@ function finish() { trap finish EXIT function cluster_init() { - if ! [ -z ${TOKEN} ]; then - sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml - fi - if ! [ -z ${MASTER_IP} ]; then - sed -i "/InitConfiguration/a localAPIEndpoint:\\n advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml - fi + if ! [ -z ${TOKEN} ]; then + sed -i "/InitConfiguration/a bootstrapTokens:\\n- token: ${TOKEN}" ./kubeadm.yaml + fi + if ! [ -z ${MASTER_IP} ]; then + sed -i "/InitConfiguration/a localAPIEndpoint:\\n advertiseAddress: ${MASTER_IP}" ./kubeadm.yaml + fi #This only works with kubernetes 1.12+. The kubeadm.yaml is setup #to enable the RuntimeClass featuregate if [[ -d /var/lib/etcd ]]; then @@ -200,6 +200,15 @@ function metallb() { } +function npd() { + NPD_VER=${1:-v0.6.6} + NPD_URL="https://github.com/kubernetes/node-problem-detector.git" + NPD_DIR="node-problem-detector" + get_repo "${NPD_URL}" "${NPD_DIR}/overlays/${NPD_VER}" + set_repo_version "${NPD_VER}" "${NPD_DIR}/overlays/${NPD_VER}/node-problem-detector" + kubectl apply -k "${NPD_DIR}/overlays/${NPD_VER}" +} + function miscellaneous() { # dashboard @@ -261,6 +270,7 @@ command_handlers[help]=print_usage_exit command_handlers[storage]=storage command_handlers[monitoring]=monitoring command_handlers[metallb]=metallb +command_handlers[npd]=npd declare -A command_help command_help[init]="Only inits a cluster using kubeadm" diff --git a/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/kustomization.yaml b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/kustomization.yaml new file mode 100644 index 0000000..4ec0990 --- /dev/null +++ b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/kustomization.yaml @@ -0,0 +1,17 @@ +resources: + - node-problem-detector/deployment/node-problem-detector-config.yaml + - node-problem-detector/deployment/node-problem-detector.yaml + +patchesJson6902: + # adds rule for mce + - target: + version: v1 + kind: ConfigMap + name: node-problem-detector-config + path: patch_configmap_rules.yaml + # adds mce-monitor.json to configmap + - target: + version: v1 + kind: ConfigMap + name: node-problem-detector-config + path: patch_configmap_mce-monitor.yaml \ No newline at end of file diff --git a/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_mce-monitor.yaml b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_mce-monitor.yaml new file mode 100644 index 0000000..188d514 --- /dev/null +++ b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_mce-monitor.yaml @@ -0,0 +1,22 @@ +# adds "mce" rule +- op: add + path: /data/mce-monitor.json + value: | + { + "plugin": "journald", + "pluginConfig": { + "source": "mcelog" + }, + "logPath": "/var/log/journal", + "lookback": "5m", + "bufferSize": 10, + "source": "mce-monitor", + "conditions": [], + "rules": [ + { + "type": "temporary", + "reason": "Hardware Error", + "pattern": "Hardware event.*" + } + ] + } \ No newline at end of file diff --git a/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_rules.yaml b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_rules.yaml new file mode 100644 index 0000000..d13ab34 --- /dev/null +++ b/clr-k8s-examples/node-problem-detector/overlays/v0.6.6/patch_configmap_rules.yaml @@ -0,0 +1,73 @@ +# adds "mce" rule +- op: add + path: /data/kernel-monitor.json + value: | + { + "plugin": "kmsg", + "logPath": "/dev/kmsg", + "lookback": "5m", + "bufferSize": 10, + "source": "kernel-monitor", + "conditions": [ + { + "type": "KernelDeadlock", + "reason": "KernelHasNoDeadlock", + "message": "kernel has no deadlock" + }, + { + "type": "ReadonlyFilesystem", + "reason": "FilesystemIsReadOnly", + "message": "Filesystem is read-only" + } + ], + "rules": [ + { + "type": "temporary", + "reason": "Hardware Error", + "pattern": "mce:.*" + }, + { + "type": "temporary", + "reason": "OOMKilling", + "pattern": "Kill process \\d+ (.+) score \\d+ or sacrifice child\\nKilled process \\d+ (.+) total-vm:\\d+kB, anon-rss:\\d+kB, file-rss:\\d+kB.*" + }, + { + "type": "temporary", + "reason": "TaskHung", + "pattern": "task \\S+:\\w+ blocked for more than \\w+ seconds\\." + }, + { + "type": "temporary", + "reason": "UnregisterNetDevice", + "pattern": "unregister_netdevice: waiting for \\w+ to become free. Usage count = \\d+" + }, + { + "type": "temporary", + "reason": "KernelOops", + "pattern": "BUG: unable to handle kernel NULL pointer dereference at .*" + }, + { + "type": "temporary", + "reason": "KernelOops", + "pattern": "divide error: 0000 \\[#\\d+\\] SMP" + }, + { + "type": "permanent", + "condition": "KernelDeadlock", + "reason": "AUFSUmountHung", + "pattern": "task umount\\.aufs:\\w+ blocked for more than \\w+ seconds\\." + }, + { + "type": "permanent", + "condition": "KernelDeadlock", + "reason": "DockerHung", + "pattern": "task docker:\\w+ blocked for more than \\w+ seconds\\." + }, + { + "type": "permanent", + "condition": "ReadonlyFilesystem", + "reason": "FilesystemIsReadOnly", + "pattern": "Remounting filesystem read-only" + } + ] + }