From acebcdc37e01dc2e71e7cf9f475f3e7d245ad9b9 Mon Sep 17 00:00:00 2001
From: Natalie <natalie@lilithuwu.com>
Date: Tue, 30 Jun 2026 10:31:56 -0400
Subject: [PATCH] deploy(server): rewrite deploy-server.sh as a rebuild-safe
 one-command deploy
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Captures the working DO-native deployment so a terraform rebuild (which wipes
the manual install) is recovered with one command: installs runtime (bun/redis/
caddy), syncs code, pushes secrets OVER SSH (never in cloud-init user-data — that
is metadata-readable, per the gpu.sh finding), wires the systemd unit + Caddy TLS
edge, verifies health. Secrets sourced at deploy time (doctl DB password,
CT_SERVICE_TOKEN from @ct/.env.local, Spaces keys from vault) — none hardcoded.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 deploy/deploy-server.sh | 256 ++++++++++++++++++----------------------
 1 file changed, 118 insertions(+), 138 deletions(-)

diff --git a/deploy/deploy-server.sh b/deploy/deploy-server.sh
index 663e448..aa5e808 100755
--- a/deploy/deploy-server.sh
+++ b/deploy/deploy-server.sh
@@ -1,165 +1,145 @@
-#!/bin/bash
-set -euo pipefail
-
-# Deploy mac-sync-server to the DO backend droplet (lilith-store-backend).
+#!/usr/bin/env bash
+# Deploy mac-sync-server to the DO backend droplet (com.uvlava.ct.services).
 #
-# Homelan `black` (10.0.0.11) is dead — the server now runs on DigitalOcean per
-# the uvlava rebuild (~/.claude/plans/nested-jingling-truffle.md, replacement
-# item #8). Public IP is the default reach path; override SERVER_HOST to use the
-# wg mesh IP (10.9.0.5) or the VPC private IP (10.20.0.2) where appropriate.
+# Rebuild-safe, one command: after terraform rebuilds the droplet (which wipes
+# any manual install), run this to bring macsync fully back. It installs the
+# runtime, syncs the code, pushes secrets over SSH (NEVER via cloud-init
+# user-data — that's metadata-readable), wires the systemd unit + Caddy TLS edge,
+# and verifies health.
+#
+# Secrets are sourced at deploy time, never hardcoded:
+#   - DB password   : doctl databases user get (managed PG, macsync_app)
+#   - SERVICE_TOKEN : CT_SERVICE_TOKEN from @ct/.env.local (shared @ct operator token)
+#   - Spaces keys   : ~/Code/@ct/.vault/do-spaces-uvlava.{access,secret}
 #
 # Usage:
-#   ./deploy-server.sh
-#   ./deploy-server.sh --skip-build
-#   SERVER_HOST=10.9.0.5 ./deploy-server.sh   # over the wg mesh instead
+#   ./deploy/deploy-server.sh            full deploy
+#   ./deploy/deploy-server.sh --code     code + restart only (skip runtime/secrets)
+set -euo pipefail
 
-# Backend droplet: public 209.38.51.98 · wg 10.9.0.5 · VPC 10.20.0.2.
-SERVER_HOST="${SERVER_HOST:-209.38.51.98}"
-REMOTE_DIR="/opt/mac-sync-server"
-ENV_DIR="/etc/mac-sync-server"
-SERVICE_NAME="mac-sync-server"
+# --- target: ct.services. Public ssh is firewalled to the Iceland jump (key is
+#     Match-restricted to that source), so we always go through it. ---
+JUMP_HOST=quinn-vps                 # Iceland vps-0 (89.127.233.145)
+SERVER_PUBLIC=209.38.51.98          # ct.services floating IP (reachable via the jump)
+SSH_KEY=~/.ssh/id_ed25519_1984
+SSH="ssh -J $JUMP_HOST -i $SSH_KEY -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 root@$SERVER_PUBLIC"
+
+REMOTE_DIR=/opt/mac-sync-server
+ENV_DIR=/etc/mac-sync-server
+EDGE_DOMAIN=macsync.ct.uvlava.com
+DB_CLUSTER=ef22022e-de47-4a4d-8303-0166dbf891d6
+DB_PRIVATE_HOST=private-lilith-store-pg-do-user-28217120-0.l.db.ondigitalocean.com
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SERVER_SRC="$SCRIPT_DIR/../src/server"
-SKIP_BUILD=false
+SRC="$SCRIPT_DIR/../src/server"
+CODE_ONLY=false; [ "${1:-}" = "--code" ] && CODE_ONLY=true
 
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-BLUE='\033[0;34m'
-NC='\033[0m'
+die(){ echo "✗ $*" >&2; exit 1; }
+step(){ echo "▸ $*"; }
 
-print_step()    { echo -e "${GREEN}▸${NC} $1"; }
-print_info()    { echo -e "${BLUE}ℹ${NC} $1"; }
-print_warning() { echo -e "${YELLOW}⚠${NC} $1"; }
-print_error()   { echo -e "${RED}✗${NC} $1"; }
-print_success() { echo -e "${GREEN}✓${NC} $1"; }
+# --- prerequisites on the laptop (provision/secret sources) ---
+command -v doctl >/dev/null || die "doctl not found"
+CT_ENV=~/Code/@ct/.env.local
+[ -r "$CT_ENV" ] || die "missing $CT_ENV (needs CT_SERVICE_TOKEN)"
+SERVICE_TOKEN=$(grep -E '^CT_SERVICE_TOKEN=' "$CT_ENV" | cut -d= -f2-)
+[ -n "$SERVICE_TOKEN" ] || die "CT_SERVICE_TOKEN empty in $CT_ENV"
+SPACES_ACCESS=$(cat ~/Code/@ct/.vault/do-spaces-uvlava.access 2>/dev/null | tr -d '[:space:]') || true
+SPACES_SECRET=$(cat ~/Code/@ct/.vault/do-spaces-uvlava.secret 2>/dev/null | tr -d '[:space:]') || true
+DB_PW=$(doctl databases user get "$DB_CLUSTER" macsync_app --format Password --no-header 2>/dev/null) || die "could not fetch macsync_app DB password"
 
-for arg in "$@"; do
-    case "$arg" in
-        --skip-build) SKIP_BUILD=true ;;
-    esac
-done
+step "checking reachability ($SERVER_PUBLIC via $JUMP_HOST)"
+$SSH 'echo ok' >/dev/null || die "cannot reach the droplet via the jump"
 
-echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
-echo -e "${BLUE}  Mac Sync Server — Deploy to backend droplet ($SERVER_HOST)${NC}"
-echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
-echo ""
+if ! $CODE_ONLY; then
+  step "installing runtime (bun, redis, caddy)"
+  $SSH 'bash -s' <<'REMOTE'
+set -e
+mkdir -p /opt/mac-sync-server/data/blobs /etc/mac-sync-server
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -y -qq unzip redis-server >/dev/null 2>&1 || true
+systemctl enable --now redis-server >/dev/null 2>&1 || true
+[ -x /root/.bun/bin/bun ] || { export BUN_INSTALL=/root/.bun; curl -fsSL https://bun.sh/install | bash >/dev/null 2>&1; }
+if ! command -v caddy >/dev/null 2>&1; then
+  apt-get install -y -qq debian-keyring debian-archive-keyring apt-transport-https curl >/dev/null 2>&1
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg 2>/dev/null
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list >/dev/null
+  apt-get update -qq >/dev/null 2>&1 && apt-get install -y -qq caddy >/dev/null 2>&1
+fi
+ufw allow 80/tcp >/dev/null 2>&1 || true; ufw allow 443/tcp >/dev/null 2>&1 || true
+REMOTE
+fi
 
-check_host() {
-    print_step "Checking SSH to $SERVER_HOST..."
-    if ! ssh -o ConnectTimeout=5 "$SERVER_HOST" 'echo ok' >/dev/null 2>&1; then
-        print_error "Cannot reach $SERVER_HOST — check network/SSH config"
-        exit 1
-    fi
-    print_success "Connected"
-}
+step "syncing server source → $REMOTE_DIR"
+rsync -az --delete -e "ssh -J $JUMP_HOST -i $SSH_KEY -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new" \
+  --exclude 'node_modules/' --exclude '.bun/' --exclude 'data/' --exclude '.env' --exclude '.git/' \
+  "$SRC/" "root@$SERVER_PUBLIC:$REMOTE_DIR/"
 
-sync_source() {
-    print_step "Syncing src/server/ to $SERVER_HOST:$REMOTE_DIR..."
-    ssh "$SERVER_HOST" "sudo mkdir -p $REMOTE_DIR && sudo chown lilith:lilith $REMOTE_DIR"
-    rsync -az --delete \
-        --exclude 'node_modules/' \
-        --exclude '.bun/' \
-        --exclude 'data/' \
-        "$SERVER_SRC/" \
-        "$SERVER_HOST:$REMOTE_DIR/"
-    print_success "Source synced"
-}
+step "installing deps (npmjs, isolated HOME to avoid the dead @lilith scope registry)"
+$SSH "cd $REMOTE_DIR && printf '[install]\nregistry = \"https://registry.npmjs.org/\"\n' > bunfig.toml && rm -f bun.lock && mkdir -p /tmp/msbun && HOME=/tmp/msbun /root/.bun/bin/bun install >/dev/null 2>&1 && echo deps-ok"
 
-install_deps() {
-    print_step "Installing dependencies on the droplet..."
-    ssh "$SERVER_HOST" "cd $REMOTE_DIR && bun install --frozen-lockfile"
-    print_success "Dependencies installed"
-}
-
-provision_env() {
-    # Create /etc/mac-sync-server/env if it doesn't exist.
-    # SERVICE_TOKEN must be set manually post-deploy; we write a placeholder that
-    # causes the service to fail-fast with a clear error rather than start misconfigured.
-    if ssh "$SERVER_HOST" "test -f $ENV_DIR/env" 2>/dev/null; then
-        print_info "env file exists at $ENV_DIR/env — skipping (preserving existing secrets)"
-        return
-    fi
-    print_step "Creating env file at $SERVER_HOST:$ENV_DIR/env..."
-    ssh "$SERVER_HOST" "sudo mkdir -p $ENV_DIR && sudo tee $ENV_DIR/env > /dev/null" <<'EOF'
+if ! $CODE_ONLY; then
+  step "writing env (secrets over stdin, never in user-data)"
+  printf '%s\n%s\n%s\n%s\n' "$DB_PW" "$SERVICE_TOKEN" "$SPACES_ACCESS" "$SPACES_SECRET" | $SSH "bash -s '$DB_PRIVATE_HOST'" <<'REMOTE'
+set -e
+HOST="$1"
+{ read -r PW; read -r TOKEN; read -r ACCESS; read -r SECRET; }
+umask 077
+cat > /etc/mac-sync-server/env <<EOF
 PORT=3201
 NODE_ENV=production
-# DO Managed PG (nyc3, private :25060) — reached VPC-direct from the droplet, or
-# via the wg→pgBouncer bridge (:6432) from off-droplet. macsync history was
-# black-only and is LOST; the schema is rebuilt forward on first boot.
-QUINN_MACSYNC_DB_URL=REPLACE_WITH_DB_URL
-SERVICE_TOKEN=REPLACE_WITH_SECRET
+QUINN_MACSYNC_DB_URL=postgresql://macsync_app:${PW}@${HOST}:25060/macsync?sslmode=no-verify
+SERVICE_TOKEN=${TOKEN}
 SSO_VALIDATE_URL=http://localhost:3025/auth/validate
-# Prospect classifier external lists (Handoff 01 #3 — both OPTIONAL; the
-# classifier degrades gracefully when unset). BLOCK_LIST_PATH points at the
-# block-list.json rsync'd from plum; VIP_ROSTER_URL is quinn-api's vip-roster
-# (fetched live each classify run with SERVICE_TOKEN above). quinn-api lives on
-# vps-0 today (wg 10.9.0.1:3030); moves to the backend droplet later (uvlava #9).
-# BLOCK_LIST_PATH=/etc/mac-sync-server/block-list.json
-# VIP_ROSTER_URL=http://10.9.0.1:3030/api/vip-roster
-# Object storage. local = ./data/blobs (dev). s3 = any S3-compatible store.
 STORAGE_BACKEND=s3
 STORAGE_LOCAL_PATH=/opt/mac-sync-server/data/blobs
-# DO Spaces (nyc3). Path-style is required for this bucket's writes.
 S3_ENDPOINT=https://nyc3.digitaloceanspaces.com
-S3_ACCESS_KEY=REPLACE_FROM_VAULT_do-spaces-uvlava.access
-S3_SECRET_KEY=REPLACE_FROM_VAULT_do-spaces-uvlava.secret
+S3_ACCESS_KEY=${ACCESS}
+S3_SECRET_KEY=${SECRET}
 S3_BUCKET=lilith-quinn-media
 S3_REGION=us-east-1
 S3_FORCE_PATH_STYLE=true
 S3_PRESIGN_TTL_SECONDS=900
-# Embedding inference endpoint — required, no default (fail fast on misconfig).
-MODEL_BOSS_EMBED_URL=REPLACE_WITH_INFERENCE_URL
+MODEL_BOSS_EMBED_URL=http://127.0.0.1:1/embed-unused
+REDIS_URL=redis://127.0.0.1:6379
 EOF
-    ssh "$SERVER_HOST" "sudo chmod 640 $ENV_DIR/env && sudo chown root:lilith $ENV_DIR/env"
-    print_warning "env file written — set SERVICE_TOKEN before starting: sudo nano $ENV_DIR/env"
-}
+chmod 640 /etc/mac-sync-server/env
+REMOTE
 
-install_systemd() {
-    print_step "Installing systemd unit..."
-    rsync -az "$SCRIPT_DIR/systemd/mac-sync-server.service" \
-        "$SERVER_HOST:/tmp/mac-sync-server.service"
-    ssh "$SERVER_HOST" \
-        "sudo cp /tmp/mac-sync-server.service /etc/systemd/system/mac-sync-server.service && \
-         sudo systemctl daemon-reload && \
-         sudo systemctl enable mac-sync-server"
-    print_success "Systemd unit installed and enabled"
-}
+  step "installing systemd unit + Caddy edge"
+  $SSH "bash -s '$EDGE_DOMAIN'" <<'REMOTE'
+set -e
+DOMAIN="$1"
+cat > /etc/systemd/system/mac-sync-server.service <<'UNIT'
+[Unit]
+Description=Mac Sync Server
+After=network.target redis-server.service
+Wants=redis-server.service
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/mac-sync-server
+ExecStart=/root/.bun/bin/bun run src/main.ts
+Restart=on-failure
+RestartSec=5
+Environment=NODE_ENV=production
+EnvironmentFile=/etc/mac-sync-server/env
+StandardOutput=append:/var/log/mac-sync-server.log
+StandardError=append:/var/log/mac-sync-server.log
+[Install]
+WantedBy=multi-user.target
+UNIT
+printf '%s {\n\treverse_proxy localhost:3201\n}\n' "$DOMAIN" > /etc/caddy/Caddyfile
+systemctl daemon-reload
+systemctl enable mac-sync-server caddy >/dev/null 2>&1
+systemctl restart caddy
+REMOTE
+fi
 
-restart_service() {
-    print_step "Restarting $SERVICE_NAME..."
-    ssh "$SERVER_HOST" "sudo systemctl restart $SERVICE_NAME"
-    sleep 3
-    local status
-    status=$(ssh "$SERVER_HOST" "systemctl is-active $SERVICE_NAME" 2>/dev/null || echo "unknown")
-    if [[ "$status" == "active" ]]; then
-        print_success "Service active"
-    else
-        print_warning "Service status: $status"
-        print_info "Check: ssh $SERVER_HOST 'journalctl -u $SERVICE_NAME -n 30'"
-    fi
-}
-
-verify_health() {
-    print_step "Checking health endpoint..."
-    local port=3201
-    if ssh "$SERVER_HOST" "curl -sf http://localhost:$port/health > /dev/null 2>&1"; then
-        print_success "Health check passed (port $port)"
-    else
-        print_warning "Health check failed — service may still be starting or SERVICE_TOKEN unset"
-    fi
-}
-
-check_host
-sync_source
-install_deps
-provision_env
-install_systemd
-restart_service
-verify_health
+step "restarting + verifying"
+$SSH "systemctl restart mac-sync-server; sleep 4; \
+  echo \"server=\$(systemctl is-active mac-sync-server) deep=\$(curl -s -m8 http://localhost:3201/health/deep)\""
 
 echo ""
-print_success "Server deploy complete"
-echo ""
-print_info "If SERVICE_TOKEN was just set, run: ssh $SERVER_HOST 'sudo systemctl restart $SERVICE_NAME'"
-print_info "View logs: ssh $SERVER_HOST 'journalctl -u $SERVICE_NAME -f'"
+echo "✓ deploy complete"
+echo "  edge:   https://$EDGE_DOMAIN/health"
+echo "  NOTE:   open 80/443 on the ct.services cloud firewall (terraform-managed) if a rebuild reset it:"
+echo "          doctl compute firewall add-rules <fw> --inbound-rules 'protocol:tcp,ports:80,address:0.0.0.0/0 protocol:tcp,ports:443,address:0.0.0.0/0'"