feat(infra): production host bootstrap and bare-metal Stalwart scaffolding

Host provisioning for the single-server production target: SSH + firewall hardening (nftables allowlist), k3s node registration, bare-metal Stalwart install with systemd units and TLS cert-sync from the cluster secret, and Restic encrypted backup/restore (primary + DR) with timer units. Host-specific secrets live in config.env (gitignored); config.env.example is the template. Also gitignores MemPalace per-project files.
2026-06-07 00:19:48 +02:00
parent 5ed3d2bc5f
commit 3831c85285
18 changed files with 1432 additions and 0 deletions
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+#
+# Dezky host backup — Restic to a Hetzner Storage Box (primary), then a
+# dedup-aware `restic copy` to a second Storage Box in Helsinki (DR).
+#
+# Runs as root (must read stalwart- and root-owned data). HOME is pointed at
+# /opt/dezky-backup so ssh uses the dedicated backup key + config (Storage Box
+# is SSH/SFTP on port 23). Triggered daily by dezky-backup.timer.
+#
+# Requires restic >= 0.14 (for `copy --from-repo`).
+
+set -euo pipefail
+
+RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
+info()  { echo -e "${BLUE}[INFO]${NC}  $*"; }
+ok()    { echo -e "${GREEN}[OK]${NC}    $*"; }
+warn()  { echo -e "${YELLOW}[WARN]${NC}  $*"; }
+error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
+
+BACKUP_HOME="/opt/dezky-backup"
+ENV_FILE="${ENV_FILE:-$BACKUP_HOME/restic.env}"
+
+if [[ ! -f "$ENV_FILE" ]]; then
+    error "Missing $ENV_FILE — run restic/install.sh first."
+    exit 1
+fi
+# shellcheck disable=SC1090
+source "$ENV_FILE"
+
+: "${RESTIC_PASSWORD:?RESTIC_PASSWORD required}"
+: "${BACKUP_PRIMARY_REPO:?BACKUP_PRIMARY_REPO required}"
+: "${BACKUP_PATHS:?BACKUP_PATHS required}"
+: "${BACKUP_RETENTION:=--keep-daily 7 --keep-weekly 4 --keep-monthly 6}"
+
+# ssh (spawned by restic) reads $HOME/.ssh/config — wildcard for *.your-storagebox.de
+export HOME="$BACKUP_HOME"
+export RESTIC_PASSWORD
+# For `copy`: both repos share the same password.
+export RESTIC_FROM_PASSWORD="$RESTIC_PASSWORD"
+
+# Optional dead-man's-switch (e.g. healthchecks.io). Pinged /start, success, /fail.
+HC="${BACKUP_HEALTHCHECK_URL:-}"
+ping_hc() { [[ -n "$HC" ]] && curl -fsS -m 10 --retry 3 "${HC}${1:-}" >/dev/null 2>&1 || true; }
+fail() { error "$1"; ping_hc "/fail"; exit 1; }
+
+ping_hc "/start"
+
+# Exclude obvious churn/noise from the PVC tree
+EXCLUDES=(--exclude-caches
+          --exclude '*/lost+found'
+          --exclude '*.tmp')
+
+# ── 1) Back up to the primary Storage Box ──────────────────────────────────
+info "Backing up to primary: $BACKUP_PRIMARY_REPO"
+# shellcheck disable=SC2086
+restic -r "$BACKUP_PRIMARY_REPO" backup $BACKUP_PATHS \
+    "${EXCLUDES[@]}" \
+    --tag dezky --tag host \
+    --host dezky-node1 \
+    || fail "Primary backup failed."
+ok "Primary backup done."
+
+# ── 2) Retention on primary ────────────────────────────────────────────────
+info "Applying retention on primary..."
+# shellcheck disable=SC2086
+restic -r "$BACKUP_PRIMARY_REPO" forget $BACKUP_RETENTION --prune \
+    || warn "Primary forget/prune reported an issue (backup itself is safe)."
+
+# ── 3) Light integrity check on primary ────────────────────────────────────
+restic -r "$BACKUP_PRIMARY_REPO" check || warn "restic check flagged the primary repo — investigate."
+
+# ── 4) Mirror to the Helsinki DR box (dedup-aware copy) ─────────────────────
+if [[ -n "${BACKUP_DR_REPO:-}" ]]; then
+    info "Copying snapshots to DR: $BACKUP_DR_REPO"
+    restic -r "$BACKUP_DR_REPO" copy --from-repo "$BACKUP_PRIMARY_REPO" \
+        || fail "DR copy failed."
+    # shellcheck disable=SC2086
+    restic -r "$BACKUP_DR_REPO" forget $BACKUP_RETENTION --prune \
+        || warn "DR forget/prune reported an issue."
+    ok "DR mirror done."
+else
+    warn "BACKUP_DR_REPO not set — skipping off-site mirror (set it for real DR)."
+fi
+
+ok "Backup cycle complete."
+ping_hc ""   # success ping (bare URL)