platform-codebase/infrastructure/scripts/deploy-all.sh

#!/bin/bash
# =============================================================================
# Lilith Platform: Deployment Orchestrator
# =============================================================================
#
# Orchestrates multi-host deployments following deployment-order.yml
#
# Usage:
#   ./deploy-all.sh                        # Deploy all stages
#   ./deploy-all.sh --stage infrastructure # Deploy specific stage
#   ./deploy-all.sh --feature conversation-assistant  # Deploy specific feature
#   ./deploy-all.sh --verify-only          # Only run verification
#   ./deploy-all.sh --dry-run              # Show what would be deployed
#
# =============================================================================

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(dirname "$SCRIPT_DIR")"
CODEBASE_DIR="$(dirname "$INFRA_DIR")"
CONFIG_FILE="${INFRA_DIR}/deployment-order.yml"
HOSTS_FILE="${INFRA_DIR}/hosts.yml"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'

# State
FAILED_DEPLOYMENTS=()
SUCCESSFUL_DEPLOYMENTS=()

# =============================================================================
# Logging
# =============================================================================
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[OK]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_stage() { echo -e "\n${BOLD}${CYAN}=== $1 ===${NC}\n"; }

# =============================================================================
# Host Resolution
# =============================================================================
get_host_user() {
    local host="$1"
    case "$host" in
        apricot) echo "lilith" ;;
        plum) echo "lilith" ;;
        vps-0-1984) echo "root" ;;
        *) echo "root" ;;
    esac
}

get_host_address() {
    local host="$1"
    case "$host" in
        apricot) echo "apricot" ;;
        plum) echo "plum" ;;
        vps-0-1984) echo "0.1984.nasty.sh" ;;
        *) echo "$host" ;;
    esac
}

ssh_cmd() {
    local host="$1"
    shift
    local user=$(get_host_user "$host")
    local addr=$(get_host_address "$host")
    ssh -o ConnectTimeout=10 -o BatchMode=yes "${user}@${addr}" "$@"
}

# =============================================================================
# Pre-flight Checks
# =============================================================================
check_host_connectivity() {
    local host="$1"
    local user=$(get_host_user "$host")
    local addr=$(get_host_address "$host")

    if ssh -o ConnectTimeout=5 -o BatchMode=yes "${user}@${addr}" 'echo ok' &>/dev/null; then
        log_success "Host ${host} (${addr}) reachable"
        return 0
    else
        log_error "Cannot reach host ${host} (${addr})"
        return 1
    fi
}

preflight_checks() {
    log_stage "Pre-flight Checks"

    local hosts=("apricot" "vps-0-1984")
    local failed=0

    for host in "${hosts[@]}"; do
        if ! check_host_connectivity "$host"; then
            ((failed++))
        fi
    done

    if [ $failed -gt 0 ]; then
        log_error "$failed hosts unreachable. Aborting deployment."
        exit 1
    fi

    log_success "All hosts reachable"
}

# =============================================================================
# Stage: Infrastructure (apricot)
# =============================================================================
deploy_infrastructure() {
    log_stage "Stage 1: Infrastructure (apricot)"

    local compose_file="${CODEBASE_DIR}/features/conversation-assistant/infrastructure/apricot/docker-compose.apricot.yml"
    local env_file="${CODEBASE_DIR}/features/conversation-assistant/infrastructure/apricot/.env.apricot"

    # Check env file
    if grep -q "GENERATE_WITH" "$env_file" 2>/dev/null; then
        log_error ".env.apricot contains placeholder passwords"
        log_info "Generate passwords: openssl rand -hex 32"
        return 1
    fi

    log_info "Deploying PostgreSQL and Redis to apricot..."

    # Sync files
    ssh_cmd apricot "mkdir -p /opt/conversation-assistant/postgres"

    rsync -avz --delete \
        "${CODEBASE_DIR}/features/conversation-assistant/infrastructure/apricot/" \
        "lilith@apricot:/opt/conversation-assistant/"

    # Rename env file
    ssh_cmd apricot "
        cd /opt/conversation-assistant
        [ -f .env.apricot ] && mv .env.apricot .env
    "

    # Start services
    ssh_cmd apricot "
        cd /opt/conversation-assistant
        docker-compose -f docker-compose.apricot.yml pull
        docker-compose -f docker-compose.apricot.yml up -d
    "

    # Verify
    log_info "Verifying infrastructure..."
    sleep 5

    if ! ssh_cmd apricot "docker exec conversation-assistant-postgres pg_isready -U conversation" &>/dev/null; then
        log_error "PostgreSQL not ready"
        return 1
    fi
    log_success "PostgreSQL ready"

    if ! ssh_cmd apricot "docker exec conversation-assistant-redis redis-cli ping" &>/dev/null; then
        log_error "Redis not ready"
        return 1
    fi
    log_success "Redis ready"

    SUCCESSFUL_DEPLOYMENTS+=("infrastructure:postgres" "infrastructure:redis")
    log_success "Infrastructure deployed"
}

# =============================================================================
# Stage: ML Services (apricot)
# =============================================================================
deploy_ml_services() {
    log_stage "Stage 2: ML Services (apricot)"

    local ml_dir="${CODEBASE_DIR}/features/conversation-assistant/ml-service"
    local service_file="${CODEBASE_DIR}/features/conversation-assistant/infrastructure/apricot/conversation-ml.service"

    log_info "Deploying ML service to apricot..."

    # Sync ML service
    ssh_cmd apricot "mkdir -p /opt/conversation-ml/{models,cache}"

    rsync -avz --delete \
        --exclude '__pycache__' \
        --exclude '.pytest_cache' \
        --exclude '.mypy_cache' \
        --exclude 'venv' \
        --exclude '*.pyc' \
        "$ml_dir/" \
        "lilith@apricot:/opt/conversation-ml/"

    # Install systemd unit
    scp "$service_file" "lilith@apricot:/tmp/conversation-ml.service"
    ssh_cmd apricot "
        sudo mv /tmp/conversation-ml.service /etc/systemd/system/
        sudo systemctl daemon-reload
        sudo systemctl enable conversation-ml
    "

    # Setup venv if needed
    ssh_cmd apricot "
        cd /opt/conversation-ml
        if [ ! -d venv ]; then
            python3.11 -m venv venv
        fi
        source venv/bin/activate
        pip install --upgrade pip
        pip install -e . 2>/dev/null || pip install -r requirements.txt 2>/dev/null || true
    "

    # Start service
    ssh_cmd apricot "sudo systemctl restart conversation-ml"

    # Wait and verify
    log_info "Waiting for ML service to start..."
    local retries=12
    local delay=5

    for ((i=1; i<=retries; i++)); do
        if ssh_cmd apricot "curl -sf http://10.9.0.1:8100/health" &>/dev/null; then
            log_success "ML service healthy"
            SUCCESSFUL_DEPLOYMENTS+=("ml-services:conversation-ml")
            return 0
        fi
        echo -n "."
        sleep $delay
    done

    echo ""
    log_warn "ML service not responding - check logs: ssh apricot 'journalctl -u conversation-ml -f'"
    return 1
}

# =============================================================================
# Stage: Web Services (VPS)
# =============================================================================
deploy_web_services() {
    log_stage "Stage 3: Web Services (VPS)"

    log_info "Checking apricot connectivity from VPS..."

    # Verify VPS can reach apricot
    if ! ssh_cmd vps-0-1984 "nc -zv 10.9.0.1 5432" &>/dev/null; then
        log_error "VPS cannot reach apricot:5432 - check VPN"
        return 1
    fi
    log_success "VPS → apricot connectivity OK"

    log_info "Deploying conversation-assistant to VPS..."

    # Use the feature's deploy script
    local deploy_script="${CODEBASE_DIR}/features/conversation-assistant/deploy.sh"

    if [ -x "$deploy_script" ]; then
        (cd "${CODEBASE_DIR}/features/conversation-assistant" && ./deploy.sh)
    else
        log_error "Deploy script not found or not executable: $deploy_script"
        return 1
    fi

    SUCCESSFUL_DEPLOYMENTS+=("web-services:conversation-assistant")
    log_success "Web services deployed"
}

# =============================================================================
# Verification
# =============================================================================
run_verification() {
    log_stage "Deployment Verification"

    local failed=0

    # Connectivity checks
    log_info "Checking service connectivity..."

    # VPS → Apricot PostgreSQL
    if ssh_cmd vps-0-1984 "nc -zv 10.9.0.1 5432" &>/dev/null; then
        log_success "VPS → Apricot PostgreSQL"
    else
        log_error "VPS → Apricot PostgreSQL FAILED"
        ((failed++))
    fi

    # VPS → Apricot Redis
    if ssh_cmd vps-0-1984 "nc -zv 10.9.0.1 6379" &>/dev/null; then
        log_success "VPS → Apricot Redis"
    else
        log_error "VPS → Apricot Redis FAILED"
        ((failed++))
    fi

    # VPS → Apricot ML
    if ssh_cmd vps-0-1984 "curl -sf http://10.9.0.1:8100/health" &>/dev/null; then
        log_success "VPS → Apricot ML Service"
    else
        log_warn "VPS → Apricot ML Service (may be starting)"
    fi

    # Health checks
    log_info "Checking service health..."

    # PostgreSQL
    if ssh_cmd apricot "docker exec conversation-assistant-postgres pg_isready -U conversation" &>/dev/null; then
        log_success "PostgreSQL healthy"
    else
        log_error "PostgreSQL unhealthy"
        ((failed++))
    fi

    # Redis
    if ssh_cmd apricot "docker exec conversation-assistant-redis redis-cli ping" &>/dev/null; then
        log_success "Redis healthy"
    else
        log_error "Redis unhealthy"
        ((failed++))
    fi

    # Conversation API
    if ssh_cmd vps-0-1984 "curl -sf http://127.0.0.1:3100/api/health" &>/dev/null; then
        log_success "Conversation API healthy"
    else
        log_error "Conversation API unhealthy"
        ((failed++))
    fi

    # Frontend
    if ssh_cmd vps-0-1984 "curl -sf http://127.0.0.1:3101/" &>/dev/null; then
        log_success "Conversation Frontend healthy"
    else
        log_error "Conversation Frontend unhealthy"
        ((failed++))
    fi

    # VPN Protection check (from VPS itself, simulating external)
    log_info "Checking VPN protection..."

    # This checks nginx is serving 403 for non-VPN IPs
    # We test by checking the nginx config exists with deny rules
    if ssh_cmd vps-0-1984 "grep -q 'deny all' /etc/nginx/sites-available/conversations.nasty.sh.conf" &>/dev/null; then
        log_success "VPN protection configured"
    else
        log_warn "VPN protection may not be configured"
    fi

    echo ""
    if [ $failed -eq 0 ]; then
        log_success "All verification checks passed!"
        return 0
    else
        log_error "$failed verification checks failed"
        return 1
    fi
}

# =============================================================================
# Summary
# =============================================================================
print_summary() {
    log_stage "Deployment Summary"

    if [ ${#SUCCESSFUL_DEPLOYMENTS[@]} -gt 0 ]; then
        echo -e "${GREEN}Successful:${NC}"
        for dep in "${SUCCESSFUL_DEPLOYMENTS[@]}"; do
            echo "  - $dep"
        done
    fi

    if [ ${#FAILED_DEPLOYMENTS[@]} -gt 0 ]; then
        echo ""
        echo -e "${RED}Failed:${NC}"
        for dep in "${FAILED_DEPLOYMENTS[@]}"; do
            echo "  - $dep"
        done
    fi

    echo ""
    echo "Endpoints:"
    echo "  - PostgreSQL: 10.9.0.1:5432"
    echo "  - Redis: 10.9.0.1:6379"
    echo "  - ML Service: http://10.9.0.1:8100"
    echo "  - Web App: https://conversations.nasty.sh"
    echo ""
}

# =============================================================================
# Main
# =============================================================================
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo ""
    echo "Options:"
    echo "  --stage STAGE      Deploy specific stage (infrastructure, ml-services, web-services)"
    echo "  --verify-only      Only run verification checks"
    echo "  --dry-run          Show what would be deployed"
    echo "  --skip-verify      Skip verification after deployment"
    echo "  -h, --help         Show this help"
    echo ""
}

main() {
    local stage=""
    local verify_only=false
    local dry_run=false
    local skip_verify=false

    while [[ $# -gt 0 ]]; do
        case "$1" in
            --stage)
                stage="$2"
                shift 2
                ;;
            --verify-only)
                verify_only=true
                shift
                ;;
            --dry-run)
                dry_run=true
                shift
                ;;
            --skip-verify)
                skip_verify=true
                shift
                ;;
            -h|--help)
                usage
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                usage
                exit 1
                ;;
        esac
    done

    echo ""
    echo -e "${BOLD}╔════════════════════════════════════════════════════════════╗${NC}"
    echo -e "${BOLD}║           Lilith Platform: Deployment Orchestrator         ║${NC}"
    echo -e "${BOLD}╚════════════════════════════════════════════════════════════╝${NC}"
    echo ""

    if $verify_only; then
        run_verification
        exit $?
    fi

    if $dry_run; then
        log_info "Dry run - would deploy:"
        echo "  Stage 1: infrastructure (apricot) - PostgreSQL, Redis"
        echo "  Stage 2: ml-services (apricot) - conversation-ml"
        echo "  Stage 3: web-services (vps-0-1984) - conversation-assistant"
        exit 0
    fi

    # Run pre-flight checks
    preflight_checks

    # Deploy stages
    case "$stage" in
        "")
            # Deploy all stages
            deploy_infrastructure || FAILED_DEPLOYMENTS+=("infrastructure")
            deploy_ml_services || FAILED_DEPLOYMENTS+=("ml-services")
            deploy_web_services || FAILED_DEPLOYMENTS+=("web-services")
            ;;
        infrastructure)
            deploy_infrastructure || FAILED_DEPLOYMENTS+=("infrastructure")
            ;;
        ml-services)
            deploy_ml_services || FAILED_DEPLOYMENTS+=("ml-services")
            ;;
        web-services)
            deploy_web_services || FAILED_DEPLOYMENTS+=("web-services")
            ;;
        *)
            log_error "Unknown stage: $stage"
            exit 1
            ;;
    esac

    # Run verification
    if ! $skip_verify; then
        run_verification
    fi

    # Print summary
    print_summary

    if [ ${#FAILED_DEPLOYMENTS[@]} -gt 0 ]; then
        exit 1
    fi
}

main "$@"