75 lines
4.6 KiB
YAML
75 lines
4.6 KiB
YAML
# One-shot Job that seeds Hermes' built-in cron schedule on first install.
|
|
# Idempotent: skips job names that already exist.
|
|
#
|
|
# The agent's own cron jobs live in /opt/data/cron/jobs.json on the PVC and are
|
|
# NOT reconciled by ArgoCD (runtime state). Re-run this Job manually after a
|
|
# wipe to re-seed: kubectl job restart hermes-cron-seed -n platform-engineer
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: hermes-cron-seed
|
|
namespace: platform-engineer
|
|
labels:
|
|
app: hermes
|
|
spec:
|
|
backoffLimit: 4
|
|
ttlSecondsAfterFinished: 86400
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hermes
|
|
spec:
|
|
serviceAccountName: platform-engineer
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: seed
|
|
image: bitnami/kubectl:1.35
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
echo "Waiting for hermes pod to be Ready..."
|
|
kubectl -n platform-engineer wait --for=condition=Ready pod -l app=hermes --timeout=300s || true
|
|
|
|
POD=$(kubectl -n platform-engineer get pod -l app=hermes -o jsonpath='{.items[0].metadata.name}')
|
|
echo "Using pod: $POD"
|
|
|
|
exists() { kubectl -n platform-engineer exec "$POD" -- hermes cron list 2>/dev/null | grep -qi "name=$1\| $1 "; }
|
|
|
|
create() {
|
|
name="$1"; schedule="$2"; deliver="$3"; prompt="$4"
|
|
if exists "$name"; then
|
|
echo "cron job '$name' already exists — skipping"
|
|
else
|
|
echo "creating cron job '$name' ..."
|
|
kubectl -n platform-engineer exec "$POD" -- hermes cron create "$schedule" "$prompt" --name "$name" --deliver "$deliver"
|
|
fi
|
|
}
|
|
|
|
# ---- Watchdog checks (silent unless something is wrong) ----
|
|
create "cluster-health-check" "every 15m" "discord" \
|
|
"Run: kubectl get nodes; kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded; kubectl get events -A --field-selector type=Warning --since=20m. If everything is healthy and there are no Warning events, reply with exactly [SILENT]. Otherwise give a concise per-resource summary of what is wrong (node name, pod ns/name, phase, last event)."
|
|
|
|
create "pod-restart-loop" "every 10m" "discord" \
|
|
"Find pods in CrashLoopBackOff or ImagePullBackOff across all namespaces (kubectl get pods -A). For each, fetch kubectl logs (previous) and describe. If the cause is clearly transient (OOM kill, a one-off config parse error that will retry cleanly, a missing Secret the controller will recreate), attempt ONE safe remediation: kubectl rollout restart of the owning Deployment/StatefulSet/DaemonSet, OR delete the single stuck pod. Report what you did in one line per resource. If the cause is not clearly transient (bad image, missing config, auth failure), do NOT act — post the log excerpt and the proposed command and wait for Roger. If no such pods exist, reply [SILENT]."
|
|
|
|
create "pvc-pressure" "every 30m" "discord" \
|
|
"Check cluster storage health: kubectl get pv,pvc -A; kubectl top nodes. Alert if any PVC is Pending/Lost or any node filesystem usage is over 85%. If all healthy, reply [SILENT]."
|
|
|
|
create "argocd-sync-health" "every 1h" "discord" \
|
|
"Run: kubectl get applications -n argocd -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status. If every app is Synced and Healthy, reply [SILENT]. Otherwise list the OutOfSync/Degraded apps with their status. Do NOT hand-edit resources to fix them (Argo will revert) — just report."
|
|
|
|
create "cert-expiry" "0 9 * * *" "discord" \
|
|
"List all cert-manager Certificate resources (kubectl get certificates -A). For each, check notAfter. Alert on any certificate expiring in under 21 days. If none, reply [SILENT]."
|
|
|
|
create "node-resource-drift" "every 30m" "discord" \
|
|
"Run kubectl top nodes. If any node CPU or memory usage is over 90%, or any node is NotReady, report it with the numbers. Otherwise reply [SILENT]."
|
|
|
|
# ---- Daily report (always delivered) ----
|
|
create "daily-cluster-report" "0 8 * * *" "discord" \
|
|
"Produce a daily cluster report for Roger: (1) node count + Ready/NotReady; (2) top 5 pods by CPU and by memory across all namespaces (kubectl top pods -A --sort-by); (3) count of pods not Running; (4) ArgoCD apps OutOfSync or Degraded; (5) any certificates expiring within 30 days; (6) any recent Warning events (last 24h). Keep it under 1800 chars. Always deliver (no [SILENT])."
|
|
|
|
echo "Done. Listing all cron jobs:"
|
|
kubectl -n platform-engineer exec "$POD" -- hermes cron list
|