lilith-platform.live/infrastructure/setup-forgejo-host.sh

#!/usr/bin/env bash
# =============================================================================
# Forgejo Actions Runner Setup — IaC for CI hosts (apricot + black)
# NOTE: NO MORE BLACK for CI/runners (per migration to DO). 
# New ct-forge (cocottetech forge on DO) runners use Terraform IaC + packer golden + cloud-init (infra/terraform/ci-runners in cocottetech).
# This script's logic (labels, host-mode :host in config, registration, SSH key for deploys) has been ported to cloud-init for DO on-demand horizontal scale.
# LP CI + deploys now use [self-hosted, linux, do, ct-forge] (see .forgejo/workflows/* and cocottetech ci-runners).
# Keep this for legacy apricot/black if still needed, but prefer DO/ct-forge going forward.
# =============================================================================
# Provisions forgejo-runner on the two CI hosts:
#
#   apricot (10.0.0.116) — runs `build` jobs
#     Runner name: apricot  labels: self-hosted,linux,apricot
#     Needs: bun, node, playwright/chromium
#     Binary installed at: ~/.local/bin/forgejo-runner  (read-only /usr/local/bin)
#     Workdir: ~/.local/share/forgejo-runner
#     Service: ~/.config/systemd/user/forgejo-runner.service (user unit, linger=yes)
#
#   black (10.0.0.11) — runs `deploy` jobs (SSH gateway to quinn-vps)
#     Runner name: black  labels: self-hosted,linux,black
#     Needs: ssh, rsync, bun (for build verification)
#     Binary installed at: /usr/local/bin/forgejo-runner
#     Workdir: ~/.local/share/forgejo-runner
#     Service: ~/.config/systemd/user/forgejo-runner.service (user unit)
#     CI deploy key: ~/.ssh/quinn-ci-deploy → authorized on quinn-vps root
#
# Usage:
#   bash infrastructure/setup-forgejo-host.sh                   # full setup (both hosts)
#   bash infrastructure/setup-forgejo-host.sh --host apricot    # apricot only
#   bash infrastructure/setup-forgejo-host.sh --host black      # black only
#   bash infrastructure/setup-forgejo-host.sh --runner          # (re)install binary
#   bash infrastructure/setup-forgejo-host.sh --register        # re-register runner
#   bash infrastructure/setup-forgejo-host.sh --ssh-key         # regen CI deploy key
#   bash infrastructure/setup-forgejo-host.sh --playwright      # install browser deps
#   bash infrastructure/setup-forgejo-host.sh --verify          # health check
#
# Prerequisites:
#   - ssh apricot.lan and ssh black must be reachable
#     NOTE: if running ON apricot, apricot steps run locally (no SSH)
#   - FORGEJO_TOKEN env var (personal access token, not runner token)
#     Get from: http://forge.ct.uvlava.com:3000/user/settings/applications  (or the ct IP during bootstrap)
#     Or: cat ~/.config/forgejo/token
#
# After running, verify secrets exist at:
#   http://forge.ct.uvlava.com:3000/lilith/lilith-platform.live/settings/secrets/actions (or IP)
#     CI_SSH_KEY      — private key for black → quinn-vps SSH
#     QUINN_VPS_HOST  — 89.127.233.145
#     VPS_USER        — root
# =============================================================================
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

FORGEJO_URL="http://forge.ct.uvlava.com:3000"  # or the IP while bootstrapping; prefer the ct domain after DNS/TLS
FORGEJO_REPO="lilith/lilith-platform.live"
FORGEJO_API="${FORGEJO_URL}/api/v1"
FORGEJO_TOKEN="${FORGEJO_TOKEN:-$(cat "$HOME/.config/forgejo/token" 2>/dev/null || echo "")}"

RUNNER_VERSION="v12.8.0"
RUNNER_RELEASE_BASE="https://code.forgejo.org/forgejo/runner/releases/download/${RUNNER_VERSION}"

APRICOT_HOST="${APRICOT_HOST:-apricot.lan}"
BLACK_HOST="${BLACK_HOST:-black}"
QUINN_VPS="${QUINN_VPS_HOST:-quinn-vps}"

CI_KEY_PATH="/home/lilith/.ssh/quinn-ci-deploy"   # path on black
RUNNER_WORKDIR="\$HOME/.local/share/forgejo-runner"

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
step() { echo ""; echo "==> $1"; }
ok()   { echo "  ✔ $1"; }
skip() { echo "  — $1 (skipped)"; }
warn() { echo "  ⚠ $1"; }

# Returns: "local" if we're already on $1, else the SSH target string
host_or_local() {
  local target="$1"
  local target_ip="${2:-}"
  local my_hostname; my_hostname="$(hostname)"
  # If target resolves to a hostname we match, run locally
  if [[ "$my_hostname" == "$target" || "$my_hostname" == "${target%%.*}" ]]; then
    echo "local"
  elif [[ -n "$target_ip" ]] && ip addr show 2>/dev/null | grep -q "$target_ip"; then
    echo "local"
  else
    echo "ssh"
  fi
}

run_on() {
  local target="$1"; shift
  local mode; mode="$(host_or_local "$target")"
  if [[ "$mode" == "local" ]]; then
    bash -euo pipefail <(echo "$@")
  else
    ssh "$target" bash -euo pipefail <<< "$@"
  fi
}

require_token() {
  if [[ -z "$FORGEJO_TOKEN" ]]; then
    echo "ERROR: FORGEJO_TOKEN required." >&2
    echo "  export FORGEJO_TOKEN=\$(cat ~/.config/forgejo/token)" >&2
    exit 1
  fi
}

forgejo_api() {
  local method="$1" path="$2"; shift 2
  curl -sf \
    -X "$method" \
    -H "Authorization: token ${FORGEJO_TOKEN}" \
    -H "Content-Type: application/json" \
    "${FORGEJO_API}${path}" "$@"
}

get_registration_token() {
  require_token
  forgejo_api GET "/repos/${FORGEJO_REPO}/actions/runners/registration-token" \
    | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])"
}

# ---------------------------------------------------------------------------
# [1] Install forgejo-runner binary
# ---------------------------------------------------------------------------
install_runner() {
  local target="$1"
  local bin_dir="$2"   # e.g. ~/.local/bin or /usr/local/bin

  step "[runner] Installing forgejo-runner ${RUNNER_VERSION} on ${target}..."

  local arch
  arch="$(ssh "$target" 'uname -m' 2>/dev/null || uname -m)"
  local goarch
  case "$arch" in
    x86_64)  goarch="amd64" ;;
    aarch64) goarch="arm64" ;;
    *) echo "ERROR: unsupported arch: $arch" >&2; exit 1 ;;
  esac

  local url="${RUNNER_RELEASE_BASE}/forgejo-runner-${RUNNER_VERSION#v}-linux-${goarch}"

  if [[ "$target" == "local" ]]; then
    curl -fsSL "$url" -o /tmp/forgejo-runner
    chmod +x /tmp/forgejo-runner
    /tmp/forgejo-runner --version
    mkdir -p "$bin_dir"
    mv /tmp/forgejo-runner "${bin_dir}/forgejo-runner"
  else
    ssh "$target" bash -euo pipefail <<ENDSSH
      curl -fsSL '${url}' -o /tmp/forgejo-runner
      chmod +x /tmp/forgejo-runner
      /tmp/forgejo-runner --version
      mkdir -p '${bin_dir}'
      # Use sudo if target bin_dir requires it
      if [[ '${bin_dir}' == /usr/local/bin ]]; then
        sudo mv /tmp/forgejo-runner '${bin_dir}/forgejo-runner'
      else
        mv /tmp/forgejo-runner '${bin_dir}/forgejo-runner'
      fi
ENDSSH
  fi
  ok "forgejo-runner installed"
}

# ---------------------------------------------------------------------------
# [2] Write runner config.yaml + systemd service
#
# config.yaml sets:
#   - labels with :host suffix → run jobs directly on the host (no Docker)
#   - docker_host: "-"         → don't mount Docker socket in containers
#   - log.level: info          → avoid journald spam
# ---------------------------------------------------------------------------
write_runner_config() {
  local target="$1"
  local runner_name="$2"
  local workdir="$3"

  # Build host-mode label list: "self-hosted:host", "linux:host", "<name>:host"
  local label_list="    - \"self-hosted:host\"\n    - \"linux:host\"\n    - \"${runner_name}:host\""

  local config_content
  config_content=$(cat <<YAML
log:
  level: info
  job_level: info
runner:
  file: .runner
  capacity: 1
  timeout: 3h
  shutdown_timeout: 3h
  fetch_timeout: 30s
  fetch_interval: 2s
  report_interval: 1s
  labels:
    - "self-hosted:host"
    - "linux:host"
    - "${runner_name}:host"
cache:
  enabled: true
  port: 0
  dir: ""
  host: ""
  proxy_port: 0
container:
  network: ""
  workdir_parent:
  valid_volumes: []
  # "-" = do not mount Docker socket; jobs run on host, not in containers
  docker_host: "-"
  force_pull: false
host:
  workdir_parent:
YAML
)

  if [[ "$target" == "local" ]]; then
    mkdir -p "$workdir"
    echo "$config_content" > "${workdir}/config.yaml"
  else
    ssh "$target" "mkdir -p '${workdir}' && cat > '${workdir}/config.yaml'" <<< "$config_content"
  fi
}

# ---------------------------------------------------------------------------
# [2b] Create stable node symlink (apricot uses fnm; service PATH needs node)
# ---------------------------------------------------------------------------
ensure_node_symlink() {
  local target="$1"
  local local_bin="$2"

  if [[ "$target" == "local" ]]; then
    local fnm_default="${HOME}/.local/share/fnm/aliases/default/bin/node"
    if [[ -e "$fnm_default" ]]; then
      ln -sf "$fnm_default" "${local_bin}/node"
      ln -sf "$(dirname "$fnm_default")/npm" "${local_bin}/npm" 2>/dev/null || true
      ln -sf "$(dirname "$fnm_default")/npx" "${local_bin}/npx" 2>/dev/null || true
      ok "node symlink → ${fnm_default}"
    else
      warn "fnm default not found at ${fnm_default}; node must be in PATH some other way"
    fi
  else
    # Remote host (black): system node at /usr/bin/node is sufficient
    if ssh "$target" "which node" &>/dev/null; then
      ok "node found on ${target}: $(ssh "$target" "node --version")"
    else
      warn "node not found on ${target} — actions/checkout will fail"
    fi
  fi
}

install_service() {
  local target="$1"
  local runner_name="$2"
  local labels="$3"
  local bin_path="$4"   # absolute path to forgejo-runner binary

  step "[service] Installing forgejo-runner service on ${target} as '${runner_name}'..."

  local workdir="${RUNNER_WORKDIR}"
  local config_flag="--config ${RUNNER_WORKDIR}/config.yaml"

  if [[ "$target" == "local" ]]; then
    workdir="${HOME}/.local/share/forgejo-runner"
    config_flag="--config ${workdir}/config.yaml"
    write_runner_config "local" "$runner_name" "$workdir"
    ensure_node_symlink "local" "${HOME}/.local/bin"
    mkdir -p "${HOME}/.config/systemd/user"
    cat > "${HOME}/.config/systemd/user/forgejo-runner.service" <<EOF
[Unit]
Description=Forgejo Actions Runner (${runner_name})
After=network.target

[Service]
Type=simple
WorkingDirectory=${workdir}
ExecStart=${bin_path} daemon ${config_flag}
Restart=on-failure
RestartSec=10
Environment=HOME=%h
Environment=PATH=%h/.local/bin:%h/.bun/bin:/usr/local/bin:/usr/bin:/bin

[Install]
WantedBy=default.target
EOF
    systemctl --user daemon-reload
    systemctl --user enable --now forgejo-runner
  else
    write_runner_config "$target" "$runner_name" "${RUNNER_WORKDIR}"
    ensure_node_symlink "$target" "/usr/local/bin"
    ssh "$target" bash -euo pipefail <<ENDSSH
      mkdir -p "\${HOME}/.local/share/forgejo-runner" "\${HOME}/.config/systemd/user"
      cat > "\${HOME}/.config/systemd/user/forgejo-runner.service" <<'UNIT'
[Unit]
Description=Forgejo Actions Runner (${runner_name})
After=network.target

[Service]
Type=simple
WorkingDirectory=%h/.local/share/forgejo-runner
ExecStart=${bin_path} daemon --config %h/.local/share/forgejo-runner/config.yaml
Restart=on-failure
RestartSec=10
Environment=HOME=%h
Environment=PATH=%h/.bun/bin:/usr/local/bin:/usr/bin:/bin

[Install]
WantedBy=default.target
UNIT
      systemctl --user daemon-reload
      systemctl --user enable --now forgejo-runner
      systemctl --user status forgejo-runner --no-pager | head -5
ENDSSH
  fi
  ok "Service installed and started"
}

# ---------------------------------------------------------------------------
# [3] Register runner with Forgejo
# ---------------------------------------------------------------------------
register_runner() {
  local target="$1"
  local runner_name="$2"
  local labels="$3"
  local bin_path="$4"
  local workdir

  step "[register] Registering '${runner_name}' with Forgejo..."

  local token; token="$(get_registration_token)"

  if [[ "$target" == "local" ]]; then
    workdir="${HOME}/.local/share/forgejo-runner"
    mkdir -p "$workdir"
    # Move .runner if it landed in cwd
    [[ -f .runner && ! -f "${workdir}/.runner" ]] && mv .runner "${workdir}/.runner" || true
    if [[ -f "${workdir}/.runner" ]]; then
      ok "'${runner_name}' already registered — delete ${workdir}/.runner to re-register"
      return 0
    fi
    pushd "$workdir" > /dev/null
    "$bin_path" register \
      --no-interactive \
      --instance "$FORGEJO_URL" \
      --token "$token" \
      --name "$runner_name" \
      --labels "$labels" 2>&1
    popd > /dev/null
  else
    ssh "$target" bash -euo pipefail <<ENDSSH
      WD="\${HOME}/.local/share/forgejo-runner"
      mkdir -p "\$WD"
      if [[ -f "\$WD/.runner" ]]; then
        echo "  Already registered — skipping"
        exit 0
      fi
      cd "\$WD"
      '${bin_path}' register \
        --no-interactive \
        --instance '${FORGEJO_URL}' \
        --token '${token}' \
        --name '${runner_name}' \
        --labels '${labels}' 2>&1
ENDSSH
  fi
  ok "'${runner_name}' registered"

  # Restart service so it picks up .runner
  if [[ "$target" == "local" ]]; then
    systemctl --user restart forgejo-runner
  else
    ssh "$target" "systemctl --user restart forgejo-runner"
  fi
}

# ---------------------------------------------------------------------------
# [4] Generate CI deploy key on black + install on quinn-vps
# ---------------------------------------------------------------------------
setup_ssh_key() {
  step "[ssh-key] Generating CI deploy key on black..."

  local pubkey
  pubkey="$(ssh "$BLACK_HOST" bash -euo pipefail <<ENDSSH
    if [[ ! -f '${CI_KEY_PATH}' ]]; then
      ssh-keygen -t ed25519 -N '' -C 'quinn-ci@black' -f '${CI_KEY_PATH}'
      echo "generated"
    else
      echo "exists"
    fi
    cat '${CI_KEY_PATH}.pub'
ENDSSH
  )"

  local key_line; key_line="$(echo "$pubkey" | grep "ssh-")"
  ok "Key: $key_line"

  step "[ssh-key] Installing deploy key on ${QUINN_VPS}..."
  ssh "$QUINN_VPS" bash -euo pipefail <<ENDSSH
    mkdir -p ~/.ssh && chmod 700 ~/.ssh
    touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys
    if grep -qF 'quinn-ci@black' ~/.ssh/authorized_keys 2>/dev/null; then
      echo "  Key already present"
    else
      printf '%s\n' '${key_line}' >> ~/.ssh/authorized_keys
      echo "  Key added"
    fi
ENDSSH

  step "[ssh-key] Verifying black → quinn-vps connectivity..."
  local vps_ip="89.127.233.145"
  ssh "$BLACK_HOST" bash -euo pipefail <<ENDSSH
    ssh-keyscan -H '${vps_ip}' 2>/dev/null >> ~/.ssh/known_hosts || true
    result="\$(ssh -i '${CI_KEY_PATH}' -o BatchMode=yes -o StrictHostKeyChecking=no root@'${vps_ip}' 'echo ok' 2>&1)"
    if [[ "\$result" == "ok" ]]; then
      echo "  SSH connectivity: ok"
    else
      echo "  ERROR: SSH failed: \$result" >&2
      exit 1
    fi
ENDSSH
  ok "SSH from black → quinn-vps verified"

  step "[ssh-key] Setting Forgejo secret CI_SSH_KEY..."
  require_token
  local privkey; privkey="$(ssh "$BLACK_HOST" "cat '${CI_KEY_PATH}'")"
  forgejo_api PUT "/repos/${FORGEJO_REPO}/actions/secrets/CI_SSH_KEY" \
    -d "{\"data\": $(python3 -c "import json,sys; print(json.dumps(sys.argv[1]))" "$privkey")}" > /dev/null
  forgejo_api PUT "/repos/${FORGEJO_REPO}/actions/secrets/QUINN_VPS_HOST" \
    -d '{"data": "89.127.233.145"}' > /dev/null
  forgejo_api PUT "/repos/${FORGEJO_REPO}/actions/secrets/VPS_USER" \
    -d '{"data": "root"}' > /dev/null
  ok "Forgejo secrets updated: CI_SSH_KEY, QUINN_VPS_HOST, VPS_USER"
}

# ---------------------------------------------------------------------------
# [5] Install Playwright browser deps (apricot only — runs build jobs)
# ---------------------------------------------------------------------------
setup_playwright() {
  local target="$1"
  step "[playwright] Installing Playwright Chromium deps on ${target}..."

  local install_cmd
  if [[ "$target" == "local" ]]; then
    install_cmd() { eval "$1"; }
    # Install system deps (Fedora/rpm-ostree — may need sudo)
    if command -v dnf &>/dev/null; then
      sudo dnf install -y --setopt=install_weak_deps=False \
        nss nspr atk cups-libs libdrm libxkbcommon libXcomposite libXdamage \
        libXfixes libXrandr mesa-libgbm alsa-lib 2>/dev/null || true
    elif command -v apt-get &>/dev/null; then
      sudo apt-get install -y --no-install-recommends \
        libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
        libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
        libgbm1 libasound2 2>/dev/null || true
    fi
    if command -v bunx &>/dev/null; then
      bunx playwright install chromium
    fi
  else
    ssh "$target" bash -euo pipefail <<'ENDSSH'
      if command -v apt-get &>/dev/null; then
        sudo apt-get install -y --no-install-recommends \
          libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
          libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
          libgbm1 libasound2 2>/dev/null || true
      fi
      command -v bunx &>/dev/null && bunx playwright install chromium || true
ENDSSH
  fi
  ok "Playwright Chromium installed"
}

# ---------------------------------------------------------------------------
# [6] Verify full setup
# ---------------------------------------------------------------------------
setup_verify() {
  step "[verify] Checking apricot runner..."
  if systemctl --user is-active forgejo-runner &>/dev/null; then
    ok "forgejo-runner.service: active"
  else
    warn "forgejo-runner not active on apricot"
  fi
  if [[ -f "${HOME}/.local/share/forgejo-runner/.runner" ]]; then
    local name; name="$(python3 -c "import json; d=json.load(open('${HOME}/.local/share/forgejo-runner/.runner')); print(d.get('name','?'))" 2>/dev/null || echo "?")"
    ok "Registered as: ${name}"
  else
    warn "apricot runner not registered"
  fi

  step "[verify] Checking black runner..."
  if ssh "$BLACK_HOST" "systemctl --user is-active forgejo-runner" &>/dev/null | grep -q "^active"; then
    ok "forgejo-runner.service on black: active"
  else
    warn "forgejo-runner not active on black"
  fi

  step "[verify] Checking SSH from black → quinn-vps..."
  if ssh "$BLACK_HOST" "ssh -i '${CI_KEY_PATH}' -o BatchMode=yes -o StrictHostKeyChecking=no root@89.127.233.145 'echo ok'" 2>/dev/null | grep -q "ok"; then
    ok "black → quinn-vps: ok"
  else
    warn "black → quinn-vps SSH failed"
  fi

  step "[verify] Checking Forgejo secrets..."
  require_token
  local secrets
  secrets="$(forgejo_api GET "/repos/${FORGEJO_REPO}/actions/secrets" \
    | python3 -c "import sys,json; d=json.load(sys.stdin); print(' '.join(s['name'] for s in (d if isinstance(d,list) else d.get('data',[]))))" 2>/dev/null || echo "")"
  for s in CI_SSH_KEY QUINN_VPS_HOST VPS_USER; do
    if echo "$secrets" | grep -q "$s"; then ok "$s"; else warn "$s missing"; fi
  done

  echo ""
  echo "=== Setup verified ==="
  echo "  Workflows: ${FORGEJO_URL}/${FORGEJO_REPO}/actions"
}

# ---------------------------------------------------------------------------
# Setup functions per host
# ---------------------------------------------------------------------------
setup_apricot() {
  local apricot_bin="${HOME}/.local/bin/forgejo-runner"
  install_runner "local" "${HOME}/.local/bin"
  install_service "local" "apricot" "self-hosted,linux,apricot" "$apricot_bin"
  register_runner "local" "apricot" "self-hosted,linux,apricot" "$apricot_bin"
  setup_playwright "local"
}

setup_black() {
  local black_bin="/usr/local/bin/forgejo-runner"
  install_runner "$BLACK_HOST" "/usr/local/bin"
  install_service "$BLACK_HOST" "black" "self-hosted,linux,black" "$black_bin"
  register_runner "$BLACK_HOST" "black" "self-hosted,linux,black" "$black_bin"
  setup_ssh_key
}

# ---------------------------------------------------------------------------
# Entrypoint
# ---------------------------------------------------------------------------
TARGET_HOST="${HOST:-all}"
while [[ $# -gt 0 ]]; do
  case "$1" in
    --host) TARGET_HOST="$2"; shift 2 ;;
    --runner)     [[ "$TARGET_HOST" != "black" ]] && install_runner "local" "${HOME}/.local/bin"
                  [[ "$TARGET_HOST" != "apricot" ]] && install_runner "$BLACK_HOST" "/usr/local/bin"
                  shift ;;
    --register)   [[ "$TARGET_HOST" != "black" ]] && register_runner "local" "apricot" "self-hosted,linux,apricot" "${HOME}/.local/bin/forgejo-runner"
                  [[ "$TARGET_HOST" != "apricot" ]] && register_runner "$BLACK_HOST" "black" "self-hosted,linux,black" "/usr/local/bin/forgejo-runner"
                  shift ;;
    --ssh-key)    setup_ssh_key; shift ;;
    --playwright) setup_playwright "local"; shift ;;
    --verify)     setup_verify; shift ;;
    *) echo "Unknown flag: $1"; exit 1 ;;
  esac
done

if [[ "$TARGET_HOST" == "all" && $# -eq 0 ]] || [[ "${1:-}" == "" && "$TARGET_HOST" == "all" ]]; then
  echo "=== Forgejo CI Host Setup ==="
  echo "Apricot (build): ${APRICOT_HOST}"
  echo "Black (deploy):  ${BLACK_HOST}"
  echo "Forgejo:         ${FORGEJO_URL}"
  echo ""
  case "$TARGET_HOST" in
    all)     setup_apricot; setup_black; setup_verify ;;
    apricot) setup_apricot; setup_verify ;;
    black)   setup_black; setup_verify ;;
  esac
fi