lilith-platform.live/deployments/@domains/quinn.m/deploy.sh
Natalie 02483204fd infra: repoint @lilith npm registry + Forgejo from dead black to DO cocotte-forge; serve /photos from local disk
black/apricot homelan died 2026-06-27. Point everything at the DO store tier:
- @lilith npm registry: forge.black.lan/npm.black.lan -> cocotte-forge Verdaccio
  (134.199.243.61:4873) across bunfig.toml scopes, all deploy.sh .npmrc writers,
  and package.json publishConfig.
- Forgejo URL (git/CI): forge.black.lan -> 134.199.243.61:3000 / :2222.
- quinn.www prod.conf /photos: was proxy_pass to dead black_photos (black:8081);
  now served from local disk (root /var/www/quinn.www/dist). Prevents a future
  deploy from re-breaking photos. (Phase G: repoint to DO Spaces/CDN later.)

Interim bare-IP endpoints; switch to named uvlava infranet hosts once live.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-28 08:09:33 -04:00

432 lines
20 KiB
Bash
Executable file

#!/usr/bin/env bash
# =============================================================================
# quinn.m — Deploy unified messaging (iMessage + mail) to vps-0
# =============================================================================
# Services deployed:
# sync-api → /opt/quinn-m-sync/ (NestJS :3100, systemd)
# autoresponder → /opt/quinn-m-autoresponder/ (tsx worker :3101, systemd)
# backend-user → /opt/quinn-m-backend-user/ (Node :3105, systemd)
# web → /var/www/quinn.m/dist/ (static Vite SPA, nginx)
#
# Prerequisites on vps-0:
# - /etc/quinn-m/secrets.env (run scripts/bootstrap-vps0.sh first)
# - /etc/quinn-m/nginx-api-key.conf (generated by bootstrap)
# - /etc/quinn-m/htpasswd (generated by bootstrap)
# - /etc/sudoers.d/quinn-m-mail-admin (generated by bootstrap)
# - nginx sites-enabled/ symlink (created by this script on first run)
# - tsx installed globally: npm i -g tsx
# - Node.js 20+
#
# Usage:
# ./deploy.sh # full deploy
# ./deploy.sh --rollback # restore previous sync-api
# ./deploy.sh --skip-build # skip local typecheck/build (CI: artifacts pre-built)
# =============================================================================
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# deployments/@domains/quinn.m/ → lilith-platform.live/ is 3 levels up
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../" && pwd)"
# Frontend + backend-user live inside this repo
WEB_DIR="${REPO_ROOT}/codebase/@features/messages/frontend-user"
BACKEND_USER_DIR="${REPO_ROOT}/codebase/@features/messages/backend-user"
# sync-api source: @mac-sync/src/server (replaced retired @messenger/imessage-sync/backend).
# mac-sync-server deploys to black via @mac-sync/deploy/deploy-server.sh — it does NOT deploy
# to vps-0. The quinn-m-sync systemd unit on vps-0 is retired (inactive as of 2026-06-04).
# The variables below are kept so the [0/9] pre-deploy check can be gated independently;
# the sync-api deploy stanza ([3/9]) is now a no-op stub (mac-sync self-deploys to black).
MAC_SYNC_ROOT="${REPO_ROOT}/../../../@applications/@mac-sync"
BACKEND_DIR="${MAC_SYNC_ROOT}/src/server"
# autoresponder was part of imessage-sync — no direct replacement in this deploy path.
# Autoresponder runs on black via quinn.m-orchestrator (deploy:m-orchestrator).
REMOTE="quinn-vps"
REMOTE_SYNC="/opt/quinn-m-sync"
REMOTE_AUTORESPONDER="/opt/quinn-m-autoresponder"
REMOTE_BACKEND_USER="/opt/quinn-m-backend-user"
REMOTE_WEB="/var/www/quinn.m/dist"
REMOTE_BACKUPS="/opt/quinn-m-sync.deploy-backups"
TIMESTAMP="$(date '+%Y%m%d_%H%M%S')"
BACKUP_PATH="${REMOTE_BACKUPS}/${TIMESTAMP}"
# ---------------------------------------------------------------------------
# --rollback: restore the most recent sync-api backup
# ---------------------------------------------------------------------------
if [[ "${1:-}" == "--rollback" ]]; then
echo "==> [ROLLBACK] Restoring previous sync-api on ${REMOTE}..."
ssh "$REMOTE" bash -euo pipefail <<'ENDSSH'
REMOTE_BACKUPS="/opt/quinn-m-sync.deploy-backups"
REMOTE_SYNC="/opt/quinn-m-sync"
latest="$(ls -1t "$REMOTE_BACKUPS" 2>/dev/null | head -1)"
if [[ -z "$latest" ]]; then
echo "ERROR: no backups found in $REMOTE_BACKUPS" >&2
exit 1
fi
echo " Restoring from $REMOTE_BACKUPS/$latest ..."
rsync -a --delete "$REMOTE_BACKUPS/$latest/" "$REMOTE_SYNC/"
echo " Restored successfully."
ENDSSH
echo "==> Reloading nginx and restarting services..."
# quinn-m-sync and quinn-m-autoresponder are retired on vps-0; only restart backend-user.
ssh "$REMOTE" "nginx -t && systemctl reload nginx && systemctl restart quinn-m-backend-user"
echo ""
echo "Rollback completed at $(date '+%Y-%m-%d %H:%M:%S %Z')"
exit 0
fi
# ---------------------------------------------------------------------------
# --skip-build: CI mode — artifacts already built
# ---------------------------------------------------------------------------
SKIP_BUILD=false
for arg in "$@"; do [[ "$arg" == "--skip-build" ]] && SKIP_BUILD=true; done
# ---------------------------------------------------------------------------
# Rollback trap — fires on any error after backup is created
# ---------------------------------------------------------------------------
BACKUP_CREATED=false
rollback_on_error() {
local exit_code=$?
echo ""
echo "Deploy step failed (exit ${exit_code})."
if [[ "$BACKUP_CREATED" == "true" ]]; then
echo "==> [AUTO-ROLLBACK] Restoring ${BACKUP_PATH}${REMOTE_SYNC} ..."
ssh "$REMOTE" bash -euo pipefail <<ENDSSH || echo " WARNING: rollback also failed — manual intervention required." >&2
set -euo pipefail
if [[ -d "${BACKUP_PATH}" ]]; then
rsync -a --delete "${BACKUP_PATH}/" "${REMOTE_SYNC}/"
echo " sync-api backup restored from ${BACKUP_PATH} (NOTE: quinn-m-sync is retired — backup is for reference only)"
nginx -t && systemctl reload nginx
systemctl restart quinn-m-backend-user
echo " Services restarted."
fi
ENDSSH
echo " Rollback complete."
else
echo " No backup was created — nothing to roll back."
fi
exit "$exit_code"
}
# TEMP: rollback disabled while recovering from node_modules wipe.
# Re-enable once prod sync-api is confirmed healthy.
# trap rollback_on_error ERR
# ---------------------------------------------------------------------------
# [0/9] Verify source directories
# ---------------------------------------------------------------------------
echo "==> [0/9] Pre-deploy checks..."
# mac-sync/src/server is verified below; autoresponder check removed (runs on black via
# quinn.m-orchestrator — not deployed from this script).
for dir in "$BACKEND_USER_DIR" "$WEB_DIR"; do
if [[ ! -d "$dir" ]]; then
echo "ERROR: source directory not found: $dir" >&2
exit 1
fi
done
# mac-sync/src/server is informational only — mac-sync self-deploys to black, not vps-0.
if [[ ! -d "$BACKEND_DIR" ]]; then
echo "WARN: mac-sync source not found at $BACKEND_DIR — sync-api step will be skipped." >&2
fi
echo " Source directories OK."
# ---------------------------------------------------------------------------
# [1/9] Typecheck + test + build
# ---------------------------------------------------------------------------
if [[ "$SKIP_BUILD" == false ]]; then
# Auto-bump patch version with deploy timestamp: 0.1.2-20260412_223433
_ver=$(cat "${REPO_ROOT}/VERSION.txt" | head -1)
_base=${_ver%%-*}
_major=${_base%%.*}; _rest=${_base#*.}; _minor=${_rest%%.*}; _patch=${_rest#*.}
echo "${_major}.${_minor}.$((_patch + 1))-${TIMESTAMP}" > "${REPO_ROOT}/VERSION.txt"
# sync-api build removed: mac-sync self-deploys to black via @mac-sync/deploy/deploy-server.sh.
# quinn-m-sync on vps-0 is retired (inactive 2026-06-04).
echo "==> [1/9] Building backend-user (tsc → dist/)..."
cd "$BACKEND_USER_DIR" && bun run build
cd "$SCRIPT_DIR"
echo "==> [1/9] Building web SPA (Vite → dist/)..."
cd "$WEB_DIR" && bun run build
cd "$SCRIPT_DIR"
fi
# sync-api standalone deps removed: mac-sync self-deploys to black, not vps-0.
# quinn-m-sync on vps-0 is retired.
echo "==> [1b/9] Building standalone backend-user deps (via Verdaccio)..."
BU_DEPS_DIR="$(mktemp -d)"
cp "$BACKEND_USER_DIR/package.json" "$BU_DEPS_DIR/"
# bun build bundles workspace + ws imports — only true runtime externals need resolving here.
bun -e "
const fs = require('fs');
const pkgPath = '${BU_DEPS_DIR}/package.json';
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
for (const name of ['@lilith/quinn-my-mcp', 'ws']) {
delete pkg.dependencies?.[name];
}
fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2));
"
printf 'registry=https://registry.npmjs.org/\n@lilith:registry=http://134.199.243.61:4873/\n' > "$BU_DEPS_DIR/.npmrc"
(cd "$BU_DEPS_DIR" && npm install --omit=dev --legacy-peer-deps)
# ---------------------------------------------------------------------------
# [2/9] Backup current sync-api on remote
# ---------------------------------------------------------------------------
echo "==> [2/9] Backing up current sync-api on ${REMOTE}..."
ssh "$REMOTE" bash -euo pipefail <<ENDSSH
set -euo pipefail
mkdir -p "${REMOTE_BACKUPS}"
if [[ -d "${REMOTE_SYNC}" && -n "\$(ls -A '${REMOTE_SYNC}' 2>/dev/null)" ]]; then
rsync -a --exclude='node_modules' "${REMOTE_SYNC}/" "${BACKUP_PATH}/"
echo " Backup: ${BACKUP_PATH}"
find "${REMOTE_BACKUPS}" -maxdepth 1 -mindepth 1 -type d -mtime +7 -exec rm -rf {} + 2>/dev/null || true
else
echo " No existing sync-api — first deploy."
fi
ENDSSH
BACKUP_CREATED=true
# ---------------------------------------------------------------------------
# [2b/9] Provision /etc/quinn-m/secrets.env
# Generates SERVICE_API_KEY if absent; other values must be set manually.
# ---------------------------------------------------------------------------
echo "==> [2b/9] Provisioning /etc/quinn-m/secrets.env..."
# Auto-bootstrap on first deploy: if /etc/quinn-m/secrets.env is missing, copy
# bootstrap-vps0.sh + env.prod.example to the VPS and run it. Idempotent — safe
# on every deploy, but only triggers when the secrets file is absent.
if ! ssh "$REMOTE" "test -f /etc/quinn-m/secrets.env" 2>/dev/null; then
echo " /etc/quinn-m/secrets.env missing — auto-bootstrapping VPS..."
ssh "$REMOTE" "mkdir -p /tmp/quinn-m-bootstrap/data /tmp/quinn-m-bootstrap/scripts /tmp/quinn-m-bootstrap/nginx /tmp/quinn-m-bootstrap/data/personas"
scp "$SCRIPT_DIR/scripts/bootstrap-vps0.sh" "${REMOTE}:/tmp/quinn-m-bootstrap/scripts/bootstrap-vps0.sh"
scp "$SCRIPT_DIR/data/env.prod.example" "${REMOTE}:/tmp/quinn-m-bootstrap/data/env.prod.example"
scp "$SCRIPT_DIR/nginx/prod.conf" "${REMOTE}:/tmp/quinn-m-bootstrap/nginx/prod.conf"
scp "$SCRIPT_DIR/data/personas/default.md" "${REMOTE}:/tmp/quinn-m-bootstrap/data/personas/default.md"
scp "$SCRIPT_DIR/data/personas/quinn-booking.md" "${REMOTE}:/tmp/quinn-m-bootstrap/data/personas/quinn-booking.md"
scp "$SCRIPT_DIR/data/personas/quinn-personal.md" "${REMOTE}:/tmp/quinn-m-bootstrap/data/personas/quinn-personal.md"
ssh "$REMOTE" "bash /tmp/quinn-m-bootstrap/scripts/bootstrap-vps0.sh"
ssh "$REMOTE" "rm -rf /tmp/quinn-m-bootstrap"
echo " Bootstrap complete."
fi
ssh "$REMOTE" bash -euo pipefail <<'ENDSSH'
set -euo pipefail
SECRETS=/etc/quinn-m/secrets.env
NGINX_KEY=/etc/quinn-m/nginx-api-key.conf
if [[ ! -f "$SECRETS" ]]; then
echo "ERROR: $SECRETS not found after bootstrap — bootstrap may have failed." >&2
exit 1
fi
# Generate SERVICE_API_KEY if missing or placeholder
current_key="$(grep '^SERVICE_API_KEY=' "$SECRETS" 2>/dev/null | cut -d= -f2-)"
if [[ -z "$current_key" || "$current_key" == "CHANGE_ME" ]]; then
new_key="$(openssl rand -hex 32)"
if grep -q '^SERVICE_API_KEY=' "$SECRETS"; then
sed -i "s|^SERVICE_API_KEY=.*|SERVICE_API_KEY=${new_key}|" "$SECRETS"
else
echo "SERVICE_API_KEY=${new_key}" >> "$SECRETS"
fi
echo " SERVICE_API_KEY generated."
current_key="$new_key"
else
echo " SERVICE_API_KEY already set."
fi
# Keep nginx-api-key.conf in sync with secrets.env
mkdir -p "$(dirname "$NGINX_KEY")"
printf 'set $quinn_m_service_key "%s";\n' "$current_key" > "$NGINX_KEY"
chmod 640 "$NGINX_KEY"
chown root:www-data "$NGINX_KEY"
echo " nginx-api-key.conf updated."
# Validate other required vars are set
for var in DATABASE_URL REDIS_URL JWT_SECRET MODEL_BOSS_URL SYNC_BACKEND_URL IMESSAGE_SYNC_URL; do
val="$(grep "^${var}=" "$SECRETS" 2>/dev/null | cut -d= -f2-)"
if [[ -z "$val" || "$val" == "CHANGE_ME" || "$val" =~ "PORT" ]]; then
echo "WARN: ${var} is not configured in ${SECRETS}" >&2
fi
done
ENDSSH
# ---------------------------------------------------------------------------
# [3/9] sync-api — RETIRED (mac-sync self-deploys to black via @mac-sync/deploy/deploy-server.sh)
# ---------------------------------------------------------------------------
# quinn-m-sync on vps-0 was retired 2026-06-04 when mac-sync replaced imessage-sync/backend.
# mac-sync-server runs on black (:3200), not on vps-0. The /opt/quinn-m-sync directory on
# vps-0 can be left or cleaned up manually; this script no longer touches it.
echo "==> [3/9] sync-api: retired — mac-sync self-deploys to black. Nothing to do for vps-0."
# ---------------------------------------------------------------------------
# [3b/9] Autoresponder — SKIPPED (runs on black.lan via quinn.m-orchestrator)
# ---------------------------------------------------------------------------
echo "==> [3b/9] Autoresponder skipped (deployed to black via deploy:m-orchestrator)."
# ---------------------------------------------------------------------------
# [3c/9] Deploy backend-user (compiled tsc dist)
# ---------------------------------------------------------------------------
echo "==> [3c/9] Deploying backend-user to ${REMOTE}:${REMOTE_BACKEND_USER}..."
ssh "$REMOTE" "mkdir -p ${REMOTE_BACKEND_USER}"
rsync -avz --delete \
--exclude='src' \
--exclude='test' \
--exclude='tsconfig*.json' \
--exclude='node_modules' \
--exclude='.env' \
--exclude='*.spec.js' \
--exclude='*.test.js' \
"$BACKEND_USER_DIR/" "${REMOTE}:${REMOTE_BACKEND_USER}/"
rsync -avz --delete "${BU_DEPS_DIR}/node_modules/" "${REMOTE}:${REMOTE_BACKEND_USER}/node_modules/"
rm -rf "${BU_DEPS_DIR:-/nonexistent}"
# ---------------------------------------------------------------------------
# [3d/9] Deploy web SPA
# ---------------------------------------------------------------------------
echo "==> [3d/9] Deploying web SPA to ${REMOTE}:${REMOTE_WEB}..."
ssh "$REMOTE" "mkdir -p ${REMOTE_WEB}"
rsync -avz --delete "$WEB_DIR/dist/" "${REMOTE}:${REMOTE_WEB}/"
# ---------------------------------------------------------------------------
# [4/9] Personas — seeded into DB by quinn.m-orchestrator/deploy.sh (step 4/9)
# ---------------------------------------------------------------------------
echo "==> [4/9] Personas: seeded via API by quinn.m-orchestrator/deploy.sh — skipping file copy."
# ---------------------------------------------------------------------------
# [5/9] Install production dependencies on remote
# ---------------------------------------------------------------------------
echo "==> [5/9] Dependencies: node_modules synced from local build (VPS has no Verdaccio)."
# ---------------------------------------------------------------------------
# [5b/9] Optional: install docker-mailserver sudoers rule
# Allows www-data to invoke `docker exec quinn-mailserver setup ...` without password
# so backend-user can manage mail accounts through the mailserver's setup script.
# ---------------------------------------------------------------------------
echo "==> [5b/9] Ensuring docker-mailserver sudoers rule..."
ssh "$REMOTE" bash <<'ENDSSH' || true
SUDOERS=/etc/sudoers.d/quinn-m-mail-admin
if [[ -f "$SUDOERS" ]]; then
echo " $SUDOERS already present."
elif command -v visudo &>/dev/null || [[ -x /usr/sbin/visudo ]]; then
tmp="$(mktemp)"
echo 'www-data ALL=(root) NOPASSWD: /usr/bin/docker exec quinn-mailserver setup *' > "$tmp"
VISUDO="$(command -v visudo || echo /usr/sbin/visudo)"
if "$VISUDO" -cf "$tmp" >/dev/null 2>&1; then
install -m 0440 -o root -g root "$tmp" "$SUDOERS"
echo " Installed $SUDOERS"
else
echo " WARN: visudo check failed — skipping mail-admin sudoers (non-critical)"
fi
rm -f "$tmp"
else
echo " WARN: visudo not installed — skipping mail-admin sudoers (install sudo package if needed)"
fi
ENDSSH
# ---------------------------------------------------------------------------
# [6/9] nginx: sync prod.conf + enable site
# ---------------------------------------------------------------------------
echo "==> [6/9] Syncing nginx prod.conf..."
NGINX_SITE=messenger.transquinnftw.com
scp "$SCRIPT_DIR/nginx/prod.conf" "${REMOTE}:/etc/nginx/sites-available/${NGINX_SITE}"
ssh "$REMOTE" "ln -sf /etc/nginx/sites-available/${NGINX_SITE} /etc/nginx/sites-enabled/${NGINX_SITE} && rm -f /etc/nginx/sites-enabled/quinn.m"
# Always rewrite the rate-limit map fragment. It's small, owned by the
# deployment, and this is how new zones (e.g. quinn_m_mail_admin) propagate
# to existing hosts without a manual edit. Idempotent — running twice is safe.
ssh "$REMOTE" bash -euo pipefail <<'ENDSSH'
MAP_CONF=/etc/nginx/conf.d/quinn-m-maps.conf
cat > "$MAP_CONF" <<'EOF'
# Managed by quinn.m deploy.sh
limit_req_zone $binary_remote_addr zone=quinn_m_req:10m rate=30r/m;
limit_req_zone $binary_remote_addr zone=quinn_m_mail_admin:10m rate=6r/m;
limit_conn_zone $binary_remote_addr zone=quinn_m_conn:10m;
EOF
echo " Wrote quinn-m-maps.conf."
ENDSSH
# ---------------------------------------------------------------------------
# [7/9] nginx test + reload
# ---------------------------------------------------------------------------
echo "==> [6c/9] Ensuring TLS SAN covers messenger.transquinnftw.com..."
ssh "$REMOTE" bash -euo pipefail <<'ENDSSH' || echo " WARN: cert expand failed — add DNS for messenger.transquinnftw.com and run post-certbot-subdomains.sh" >&2
set -euo pipefail
if ! command -v certbot &>/dev/null; then exit 0; fi
certbot certonly --nginx \
-d transquinnftw.com -d www.transquinnftw.com \
-d admin.transquinnftw.com -d data.transquinnftw.com \
-d my.transquinnftw.com \
-d messenger.transquinnftw.com -d m.transquinnftw.com \
--expand --non-interactive --agree-tos -m victorialackey@pm.me
ENDSSH
echo "==> [7/9] Testing and reloading nginx..."
ssh "$REMOTE" "nginx -t && systemctl reload nginx"
# ---------------------------------------------------------------------------
# [7b/9] Sync systemd units + daemon-reload + enable
# ---------------------------------------------------------------------------
echo "==> [7b/9] Syncing systemd units..."
# quinn-m-sync.service and quinn-m-autoresponder.service are retired on vps-0 (2026-06-04);
# their source files are kept in systemd/ for reference / manual rollback only.
scp "$SCRIPT_DIR/systemd/quinn-m-backend-user.service" "${REMOTE}:/etc/systemd/system/quinn-m-backend-user.service"
ssh "$REMOTE" "systemctl daemon-reload && systemctl enable quinn-m-backend-user"
# ---------------------------------------------------------------------------
# [8/9] Restart services + health checks
# ---------------------------------------------------------------------------
echo "==> [8/9] Restarting services..."
# quinn-m-sync and quinn-m-autoresponder are retired on vps-0 (2026-06-04).
# Sync is now mac-sync on black (:3200). Autoresponder runs on black via quinn.m-orchestrator.
ssh "$REMOTE" "systemctl restart quinn-m-backend-user"
sleep 3
echo "==> [8/9] Health checks..."
check_health() {
local url="$1" label="$2" max_attempts="${3:-24}" interval="${4:-5}"
local attempt=1
while [[ $attempt -le $max_attempts ]]; do
if ssh "$REMOTE" "curl -sf ${url} > /dev/null" 2>/dev/null; then
echo " ${label} healthy (attempt ${attempt}/${max_attempts})."
return 0
fi
echo " waiting for ${label}... (${attempt}/${max_attempts})"
sleep "$interval"
attempt=$((attempt + 1))
done
echo "ERROR: ${label} health check failed after $((max_attempts * interval))s (${url})" >&2
return 1
}
check_health "http://127.0.0.1:3105/health" "backend-user" 12 5
# ---------------------------------------------------------------------------
# [9/9] External HTTPS health check (best-effort)
# ---------------------------------------------------------------------------
if ! curl -sf --max-time 10 -u "$(ssh "$REMOTE" "grep '' /etc/quinn-m/htpasswd | head -1 | cut -d: -f1"):x" \
-H "X-Skip-Auth-Check: 1" \
https://messenger.transquinnftw.com/api/health > /dev/null 2>&1; then
# Fallback: check from remote (avoids needing basic-auth credentials locally)
if ! ssh "$REMOTE" "curl -sf --max-time 10 http://127.0.0.1:3105/health > /dev/null"; then
echo "WARN: external health check skipped (basic-auth credentials not available locally)." >&2
echo " Verify manually: https://messenger.transquinnftw.com/api/health" >&2
fi
else
echo " External HTTPS health check passed."
fi
echo ""
echo "Deployed quinn.m successfully at $(date '+%Y-%m-%d %H:%M:%S %Z')"
echo ""
echo " backend-user: http://127.0.0.1:3105 (active on vps-0)"
echo " web: https://messenger.transquinnftw.com"
echo ""
echo " sync backend: mac-sync-server on black :3200 (deploy via @mac-sync/deploy/deploy-server.sh)"
echo " autoresponder: quinn-m-orchestrator on black (deploy via deployments/@domains/quinn.m-orchestrator/deploy.sh)"
echo " review queue: https://messenger.transquinnftw.com (→ /review tab)"
echo ""
echo "To roll back backend-user: restore /opt/quinn-m-backend-user from backup on vps-0."