feat(infra): production host bootstrap and bare-metal Stalwart scaffolding
Host provisioning for the single-server production target: SSH + firewall hardening (nftables allowlist), k3s node registration, bare-metal Stalwart install with systemd units and TLS cert-sync from the cluster secret, and Restic encrypted backup/restore (primary + DR) with timer units. Host-specific secrets live in config.env (gitignored); config.env.example is the template. Also gitignores MemPalace per-project files.
This commit is contained in:
Ronni Baslund
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Dezky host backup — Restic to a Hetzner Storage Box (primary), then a
|
||||
# dedup-aware `restic copy` to a second Storage Box in Helsinki (DR).
|
||||
#
|
||||
# Runs as root (must read stalwart- and root-owned data). HOME is pointed at
|
||||
# /opt/dezky-backup so ssh uses the dedicated backup key + config (Storage Box
|
||||
# is SSH/SFTP on port 23). Triggered daily by dezky-backup.timer.
|
||||
#
|
||||
# Requires restic >= 0.14 (for `copy --from-repo`).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
|
||||
|
||||
BACKUP_HOME="/opt/dezky-backup"
|
||||
ENV_FILE="${ENV_FILE:-$BACKUP_HOME/restic.env}"
|
||||
|
||||
if [[ ! -f "$ENV_FILE" ]]; then
|
||||
error "Missing $ENV_FILE — run restic/install.sh first."
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
|
||||
: "${RESTIC_PASSWORD:?RESTIC_PASSWORD required}"
|
||||
: "${BACKUP_PRIMARY_REPO:?BACKUP_PRIMARY_REPO required}"
|
||||
: "${BACKUP_PATHS:?BACKUP_PATHS required}"
|
||||
: "${BACKUP_RETENTION:=--keep-daily 7 --keep-weekly 4 --keep-monthly 6}"
|
||||
|
||||
# ssh (spawned by restic) reads $HOME/.ssh/config — wildcard for *.your-storagebox.de
|
||||
export HOME="$BACKUP_HOME"
|
||||
export RESTIC_PASSWORD
|
||||
# For `copy`: both repos share the same password.
|
||||
export RESTIC_FROM_PASSWORD="$RESTIC_PASSWORD"
|
||||
|
||||
# Optional dead-man's-switch (e.g. healthchecks.io). Pinged /start, success, /fail.
|
||||
HC="${BACKUP_HEALTHCHECK_URL:-}"
|
||||
ping_hc() { [[ -n "$HC" ]] && curl -fsS -m 10 --retry 3 "${HC}${1:-}" >/dev/null 2>&1 || true; }
|
||||
fail() { error "$1"; ping_hc "/fail"; exit 1; }
|
||||
|
||||
ping_hc "/start"
|
||||
|
||||
# Exclude obvious churn/noise from the PVC tree
|
||||
EXCLUDES=(--exclude-caches
|
||||
--exclude '*/lost+found'
|
||||
--exclude '*.tmp')
|
||||
|
||||
# ── 1) Back up to the primary Storage Box ──────────────────────────────────
|
||||
info "Backing up to primary: $BACKUP_PRIMARY_REPO"
|
||||
# shellcheck disable=SC2086
|
||||
restic -r "$BACKUP_PRIMARY_REPO" backup $BACKUP_PATHS \
|
||||
"${EXCLUDES[@]}" \
|
||||
--tag dezky --tag host \
|
||||
--host dezky-node1 \
|
||||
|| fail "Primary backup failed."
|
||||
ok "Primary backup done."
|
||||
|
||||
# ── 2) Retention on primary ────────────────────────────────────────────────
|
||||
info "Applying retention on primary..."
|
||||
# shellcheck disable=SC2086
|
||||
restic -r "$BACKUP_PRIMARY_REPO" forget $BACKUP_RETENTION --prune \
|
||||
|| warn "Primary forget/prune reported an issue (backup itself is safe)."
|
||||
|
||||
# ── 3) Light integrity check on primary ────────────────────────────────────
|
||||
restic -r "$BACKUP_PRIMARY_REPO" check || warn "restic check flagged the primary repo — investigate."
|
||||
|
||||
# ── 4) Mirror to the Helsinki DR box (dedup-aware copy) ─────────────────────
|
||||
if [[ -n "${BACKUP_DR_REPO:-}" ]]; then
|
||||
info "Copying snapshots to DR: $BACKUP_DR_REPO"
|
||||
restic -r "$BACKUP_DR_REPO" copy --from-repo "$BACKUP_PRIMARY_REPO" \
|
||||
|| fail "DR copy failed."
|
||||
# shellcheck disable=SC2086
|
||||
restic -r "$BACKUP_DR_REPO" forget $BACKUP_RETENTION --prune \
|
||||
|| warn "DR forget/prune reported an issue."
|
||||
ok "DR mirror done."
|
||||
else
|
||||
warn "BACKUP_DR_REPO not set — skipping off-site mirror (set it for real DR)."
|
||||
fi
|
||||
|
||||
ok "Backup cycle complete."
|
||||
ping_hc "" # success ping (bare URL)
|
||||
@@ -0,0 +1,13 @@
|
||||
# Dezky nightly backup (Restic → Storage Box primary + Helsinki DR).
|
||||
[Unit]
|
||||
Description=Dezky host backup (Restic)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/opt/dezky-backup/backup.sh
|
||||
# Backups are I/O heavy but should never starve mail/k3s
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=6
|
||||
@@ -0,0 +1,12 @@
|
||||
# Nightly at 03:20 UTC, with a randomized delay so it doesn't hammer the
|
||||
# Storage Box at the same second every night. Catches up if the box was off.
|
||||
[Unit]
|
||||
Description=Run the Dezky host backup nightly
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 03:20:00
|
||||
RandomizedDelaySec=20min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Install Dezky host backups: Restic + a dedicated backup SSH key/config for the
|
||||
# Hetzner Storage Box(es), the env file, the backup/restore scripts, and the
|
||||
# nightly systemd timer. Idempotent.
|
||||
#
|
||||
# sudo ./install.sh
|
||||
#
|
||||
# Storage Box uses SSH/SFTP on PORT 23 with key auth. After this runs, you must
|
||||
# upload the printed public key to BOTH Storage Boxes, then re-run to init the
|
||||
# repos (the box must trust the key before `restic init` can connect).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
HOST_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CONFIG_FILE="${CONFIG_FILE:-$HOST_DIR/config.env}"
|
||||
BACKUP_HOME="/opt/dezky-backup"
|
||||
SSH_DIR="$BACKUP_HOME/.ssh"
|
||||
KEY="$SSH_DIR/id_ed25519"
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then error "Run as root."; exit 1; fi
|
||||
if [[ ! -f "$CONFIG_FILE" ]]; then error "Missing $CONFIG_FILE"; exit 1; fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
|
||||
: "${RESTIC_PASSWORD:?RESTIC_PASSWORD required (and STORE IT OFFLINE — losing it loses the backups)}"
|
||||
: "${BACKUP_PRIMARY_REPO:?BACKUP_PRIMARY_REPO required}"
|
||||
: "${BACKUP_PATHS:?BACKUP_PATHS required}"
|
||||
: "${BACKUP_RETENTION:=--keep-daily 7 --keep-weekly 4 --keep-monthly 6}"
|
||||
|
||||
# ── 1) Packages ────────────────────────────────────────────────────────────
|
||||
info "Installing restic + openssh client..."
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq restic curl openssh-client >/dev/null
|
||||
ok "restic $(restic version | awk '{print $2}') installed."
|
||||
|
||||
# ── 2) Backup home + SSH key/config ────────────────────────────────────────
|
||||
info "Setting up $BACKUP_HOME ..."
|
||||
install -d -m 0700 "$BACKUP_HOME" "$SSH_DIR"
|
||||
if [[ ! -f "$KEY" ]]; then
|
||||
ssh-keygen -t ed25519 -N "" -C "dezky-backup@node1" -f "$KEY" >/dev/null
|
||||
ok "Generated backup SSH key."
|
||||
fi
|
||||
# Single wildcard config covers BOTH Storage Boxes (same domain, port 23, key).
|
||||
cat > "$SSH_DIR/config" <<EOF
|
||||
Host *.your-storagebox.de
|
||||
Port 23
|
||||
IdentityFile $KEY
|
||||
IdentitiesOnly yes
|
||||
StrictHostKeyChecking accept-new
|
||||
UserKnownHostsFile $SSH_DIR/known_hosts
|
||||
EOF
|
||||
chmod 0600 "$SSH_DIR/config" "$KEY"
|
||||
chmod 0644 "$KEY.pub"
|
||||
|
||||
# ── 3) restic.env (secrets; generated, not in git) ─────────────────────────
|
||||
umask 077
|
||||
cat > "$BACKUP_HOME/restic.env" <<EOF
|
||||
# Generated by restic/install.sh from config.env — DO NOT commit.
|
||||
RESTIC_PASSWORD=${RESTIC_PASSWORD}
|
||||
BACKUP_PRIMARY_REPO=${BACKUP_PRIMARY_REPO}
|
||||
BACKUP_DR_REPO=${BACKUP_DR_REPO:-}
|
||||
BACKUP_PATHS=${BACKUP_PATHS}
|
||||
BACKUP_RETENTION=${BACKUP_RETENTION}
|
||||
BACKUP_HEALTHCHECK_URL=${BACKUP_HEALTHCHECK_URL:-}
|
||||
EOF
|
||||
chmod 0600 "$BACKUP_HOME/restic.env"
|
||||
ok "Wrote restic.env."
|
||||
|
||||
# ── 4) Scripts + systemd units ─────────────────────────────────────────────
|
||||
install -m 0750 "$SCRIPT_DIR/backup.sh" "$BACKUP_HOME/backup.sh"
|
||||
install -m 0750 "$SCRIPT_DIR/restore.sh" "$BACKUP_HOME/restore.sh"
|
||||
install -m 0644 "$SCRIPT_DIR/dezky-backup.service" /etc/systemd/system/dezky-backup.service
|
||||
install -m 0644 "$SCRIPT_DIR/dezky-backup.timer" /etc/systemd/system/dezky-backup.timer
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now dezky-backup.timer
|
||||
ok "Nightly timer enabled."
|
||||
|
||||
# ── 5) Try to init the repos (only works once the key is on the box) ───────
|
||||
export HOME="$BACKUP_HOME" RESTIC_PASSWORD
|
||||
init_repo() {
|
||||
local repo="$1" label="$2"
|
||||
[[ -z "$repo" ]] && return 0
|
||||
if restic -r "$repo" cat config >/dev/null 2>&1; then
|
||||
ok "$label repo already initialized."
|
||||
elif restic -r "$repo" init >/dev/null 2>&1; then
|
||||
ok "$label repo initialized."
|
||||
else
|
||||
warn "$label repo not reachable/authorized yet — upload the key, then re-run."
|
||||
fi
|
||||
}
|
||||
init_repo "$BACKUP_PRIMARY_REPO" "Primary"
|
||||
init_repo "${BACKUP_DR_REPO:-}" "DR"
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ Backup install complete ║"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
warn "Upload this PUBLIC key to BOTH Storage Boxes, then re-run install.sh:"
|
||||
echo ""
|
||||
cat "$KEY.pub"
|
||||
echo ""
|
||||
info " ssh-copy-id -p 23 -i $KEY.pub <primary-user>@<primary-host>.your-storagebox.de"
|
||||
info " ssh-copy-id -p 23 -i $KEY.pub <dr-user>@<dr-host>.your-storagebox.de"
|
||||
info "Then test: sudo $BACKUP_HOME/backup.sh (or wait for 03:20 UTC)"
|
||||
info "Drill restore: sudo $BACKUP_HOME/restore.sh restore latest /tmp/restore-test"
|
||||
warn "STORE RESTIC_PASSWORD OFFLINE. Without it, the encrypted backups are unrecoverable."
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Dezky restore helper. A backup you've never restored is a backup you don't
|
||||
# have — run a drill periodically. This wraps the common restic restore flows.
|
||||
#
|
||||
# sudo ./restore.sh snapshots # list snapshots (primary)
|
||||
# sudo ./restore.sh snapshots --dr # list from the DR box
|
||||
# sudo ./restore.sh restore <snapshot-id> <target-dir> [--dr]
|
||||
# sudo ./restore.sh restore latest /tmp/restore-test # safe drill target
|
||||
#
|
||||
# Restores go to an arbitrary target dir (NOT in place) so you can inspect first.
|
||||
# For Stalwart, stop the service, swap /opt/stalwart/data, then start it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
|
||||
|
||||
BACKUP_HOME="/opt/dezky-backup"
|
||||
ENV_FILE="${ENV_FILE:-$BACKUP_HOME/restic.env}"
|
||||
[[ -f "$ENV_FILE" ]] || { error "Missing $ENV_FILE"; exit 1; }
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
export HOME="$BACKUP_HOME"
|
||||
export RESTIC_PASSWORD
|
||||
|
||||
pick_repo() {
|
||||
if [[ "${*: -1}" == "--dr" ]]; then
|
||||
[[ -n "${BACKUP_DR_REPO:-}" ]] || { error "BACKUP_DR_REPO not set"; exit 1; }
|
||||
echo "$BACKUP_DR_REPO"
|
||||
else
|
||||
echo "$BACKUP_PRIMARY_REPO"
|
||||
fi
|
||||
}
|
||||
|
||||
cmd="${1:-}"; shift || true
|
||||
case "$cmd" in
|
||||
snapshots)
|
||||
repo="$(pick_repo "$@")"
|
||||
info "Snapshots in $repo:"
|
||||
restic -r "$repo" snapshots --tag dezky
|
||||
;;
|
||||
restore)
|
||||
snap="${1:?snapshot id (or 'latest')}"; target="${2:?target dir}"
|
||||
repo="$(pick_repo "$@")"
|
||||
mkdir -p "$target"
|
||||
info "Restoring $snap from $repo → $target"
|
||||
restic -r "$repo" restore "$snap" --target "$target"
|
||||
ok "Restored. Inspect $target before putting anything back in place."
|
||||
;;
|
||||
*)
|
||||
error "Usage: $0 {snapshots|restore} ... (see header)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user