dezky/infrastructure/production/host/bootstrap.sh

#!/usr/bin/env bash
#
# Dezky production host bootstrap — OS hardening for the AX41 k3s node.
#
# Run ONCE on a fresh Debian 12 (bookworm) install, as root, e.g.:
#   scp -r infrastructure/production/host root@<server>:/opt/dezky-host
#   ssh root@<server> 'cd /opt/dezky-host && cp config.env.example config.env && nano config.env'
#   ssh root@<server> 'cd /opt/dezky-host && ./bootstrap.sh'
#
# Order matters: we create your admin user + install your SSH key BEFORE
# disabling root/password login, so you can't lock yourself out. The script
# is idempotent — safe to re-run.
#
# What it does NOT do: install k3s, Stalwart, or backups. Those are separate
# steps in this host/ layer (added next). This is OS baseline + firewall only.

set -euo pipefail

RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
info()  { echo -e "${BLUE}[INFO]${NC}  $*"; }
ok()    { echo -e "${GREEN}[OK]${NC}    $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC}  $*"; }
error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_FILE="$SCRIPT_DIR/config.env"

echo ""
echo "╔══════════════════════════════════════════════════════════════╗"
echo "║          Dezky Production Host Bootstrap (Debian 12)         ║"
echo "╚══════════════════════════════════════════════════════════════╝"
echo ""

# ── Preflight ──────────────────────────────────────────────────────────────
if [[ $EUID -ne 0 ]]; then
    error "Run as root (you'll create the unprivileged admin user from here)."
    exit 1
fi

if [[ ! -f "$CONFIG_FILE" ]]; then
    error "Missing $CONFIG_FILE — copy config.env.example and fill it in."
    exit 1
fi
# shellcheck disable=SC1090
source "$CONFIG_FILE"

: "${ADMIN_USER:?ADMIN_USER required}"
: "${ADMIN_SSH_PUBKEY:?ADMIN_SSH_PUBKEY required — without it you would lock yourself out}"
: "${MGMT_ALLOW_V4:?MGMT_ALLOW_V4 required}"
: "${SERVER_HOSTNAME:?SERVER_HOSTNAME required}"
: "${SSH_PORT:=22}"

if [[ "$ADMIN_SSH_PUBKEY" != ssh-* ]]; then
    error "ADMIN_SSH_PUBKEY doesn't look like a public key (should start with 'ssh-')."
    exit 1
fi

# ── Step 1: base packages + system upgrade ─────────────────────────────────
info "Step 1: Updating system and installing base packages..."
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get upgrade -y -qq
apt-get install -y -qq \
    nftables fail2ban unattended-upgrades apt-listchanges \
    curl ca-certificates gnupg htop tmux vim chrony \
    open-iscsi nfs-common \
    >/dev/null
# Longhorn requires a running iscsid on every node; nfs-common is needed for
# RWX volumes / NFS backup targets.
systemctl enable --now iscsid >/dev/null 2>&1 || true
ok "Base packages installed (incl. Longhorn prereqs: open-iscsi, nfs-common)."

# ── Step 2: hostname + timezone + time sync ────────────────────────────────
info "Step 2: Hostname, timezone (UTC), time sync..."
hostnamectl set-hostname "$SERVER_HOSTNAME"
timedatectl set-timezone UTC
systemctl enable --now chrony >/dev/null 2>&1 || true
# Ensure the FQDN resolves locally
if ! grep -q "$SERVER_HOSTNAME" /etc/hosts; then
    echo "127.0.1.1 ${SERVER_HOSTNAME} ${SERVER_HOSTNAME%%.*}" >> /etc/hosts
fi
ok "Hostname set to $SERVER_HOSTNAME (UTC)."

# ── Step 3: admin user + SSH key (BEFORE locking SSH) ──────────────────────
info "Step 3: Admin user '$ADMIN_USER' + SSH key..."
if ! id -u "$ADMIN_USER" >/dev/null 2>&1; then
    adduser --disabled-password --gecos "" "$ADMIN_USER"
fi
usermod -aG sudo "$ADMIN_USER"
install -d -m 0700 -o "$ADMIN_USER" -g "$ADMIN_USER" "/home/$ADMIN_USER/.ssh"
AUTH_KEYS="/home/$ADMIN_USER/.ssh/authorized_keys"
touch "$AUTH_KEYS"
grep -qxF "$ADMIN_SSH_PUBKEY" "$AUTH_KEYS" || echo "$ADMIN_SSH_PUBKEY" >> "$AUTH_KEYS"
chmod 0600 "$AUTH_KEYS"
chown "$ADMIN_USER:$ADMIN_USER" "$AUTH_KEYS"
# Passworded sudo (member of sudo group). Set a password manually later if you
# want interactive sudo: `passwd $ADMIN_USER`. Key-only login still works.
ok "Admin user ready with your SSH key."

# ── Step 4: SSH hardening (drop-in) ────────────────────────────────────────
info "Step 4: Hardening SSH..."
SSHD_DROPIN="/etc/ssh/sshd_config.d/99-dezky.conf"
cat > "$SSHD_DROPIN" <<EOF
# Managed by Dezky bootstrap.sh
Port ${SSH_PORT}
PermitRootLogin no
PasswordAuthentication no
KbdInteractiveAuthentication no
ChallengeResponseAuthentication no
PubkeyAuthentication yes
PermitEmptyPasswords no
X11Forwarding no
MaxAuthTries 3
LoginGraceTime 30
AllowUsers ${ADMIN_USER}
EOF
if sshd -t; then
    systemctl reload ssh 2>/dev/null || systemctl reload sshd 2>/dev/null || true
    ok "SSH hardened: key-only, no root, AllowUsers=${ADMIN_USER}, port ${SSH_PORT}."
else
    error "sshd config test FAILED — removing drop-in, leaving SSH as-is."
    rm -f "$SSHD_DROPIN"
    exit 1
fi

# ── Step 5: kernel sysctl for k3s + sane limits ────────────────────────────
info "Step 5: sysctl + kernel modules for k3s..."
modprobe br_netfilter 2>/dev/null || true
modprobe overlay 2>/dev/null || true
cat > /etc/modules-load.d/dezky-k3s.conf <<EOF
br_netfilter
overlay
EOF
cat > /etc/sysctl.d/99-dezky-k3s.conf <<EOF
# Routing/bridging required by k3s/flannel
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
# Many containers => raise inotify + file limits
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 524288
fs.file-max = 2097152
EOF
sysctl --system >/dev/null
ok "sysctl applied."

# ── Step 6: disable swap (kubelet best practice) ───────────────────────────
info "Step 6: Disabling swap (recommended for k3s nodes)..."
swapoff -a || true
# Comment any swap entries so it stays off across reboots
sed -i.bak -E 's@^([^#].*\sswap\s.*)$@# \1  # disabled by dezky bootstrap@' /etc/fstab || true
ok "Swap disabled."

# ── Step 7: fail2ban (ssh) ─────────────────────────────────────────────────
info "Step 7: fail2ban for SSH..."
cat > /etc/fail2ban/jail.d/dezky-sshd.local <<EOF
[sshd]
enabled  = true
port     = ${SSH_PORT}
backend  = systemd
maxretry = 4
findtime = 10m
bantime  = 1h
EOF
systemctl enable --now fail2ban >/dev/null 2>&1 || true
systemctl restart fail2ban >/dev/null 2>&1 || true
ok "fail2ban active on SSH."

# ── Step 8: unattended security upgrades ───────────────────────────────────
info "Step 8: Enabling unattended security upgrades..."
cat > /etc/apt/apt.conf.d/20auto-upgrades <<EOF
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
EOF
# Keep defaults for which origins (security). Auto-reboot OFF — you decide when.
ok "Unattended security upgrades enabled (auto-reboot left off)."

# ── Step 9: firewall (k3s-safe nftables) ───────────────────────────────────
info "Step 9: Applying k3s-safe nftables firewall..."
# Ensure distro nftables.service won't fight us: we run our own unit and never
# flush the global ruleset. Disable the stock service's auto-load of its conf.
systemctl disable --now nftables.service >/dev/null 2>&1 || true
CONFIG_FILE="$CONFIG_FILE" "$SCRIPT_DIR/firewall/firewall.sh"

echo ""
echo "╔══════════════════════════════════════════════════════════════╗"
echo "║                    Host bootstrap complete                   ║"
echo "╚══════════════════════════════════════════════════════════════╝"
warn "BEFORE you close this root session:"
warn "  1. Open a new terminal and run:  ssh -p ${SSH_PORT} ${ADMIN_USER}@${SERVER_PUBLIC_IPV4:-<server-ip>}"
warn "  2. Confirm you get in with your key."
warn "  3. Only then close this session. KVM/LARA is your fallback if not."
echo ""
info "Next host-layer steps (separate scripts, added next): k3s registration,"
info "Stalwart mail, Restic backups."