platform-tooling/scripts/deploy/deploy-conversation-assistant.sh

#!/bin/bash
set -euo pipefail

#
# Conversation Assistant Deployment Script
#
# Deploys conversation-assistant to distributed architecture:
#   - Stage 1: PostgreSQL + Redis on apricot (10.9.0.1)
#   - Stage 2: ML Service on apricot (systemd)
#   - Stage 3: Frontend + Server on VPS (docker)
#
# Prerequisites:
#   - VPN configured between apricot and VPS
#   - SSH access to both hosts
#   - .env files configured with credentials
#
# Usage:
#   ./deploy-conversation-assistant.sh             # Full deploy (all stages)
#   ./deploy-conversation-assistant.sh --stage 1  # Infrastructure only
#   ./deploy-conversation-assistant.sh --stage 2  # ML service only
#   ./deploy-conversation-assistant.sh --stage 3  # Web services only
#   ./deploy-conversation-assistant.sh --verify   # Verification only
#

# =============================================================================
# INITIALIZATION
# =============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export SCRIPT_LIB_DIR="${SCRIPT_DIR}/../lib"

# Load shared libraries
source "${SCRIPT_LIB_DIR}/colors.sh"
source "${SCRIPT_LIB_DIR}/logger.sh"
source "${SCRIPT_LIB_DIR}/config.sh"

# Initialize
log_init "CONV-DEPLOY"
config_init "$SCRIPT_DIR/.."

# Platform root is parent of infrastructure/
# SCRIPT_DIR = .../infrastructure/scripts/deploy
# Go up 3 levels: scripts/deploy -> scripts -> infrastructure -> lilith-platform
# NOTE: Don't use CONFIG_PROJECT_ROOT - it finds infrastructure/ not platform root
PLATFORM_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"

# Feature paths (in codebase repo, sibling to infrastructure)
FEATURE_DIR="${PLATFORM_ROOT}/codebase/features/conversation-assistant"
INFRA_DIR="${FEATURE_DIR}/infrastructure"

# Host configuration (from ports.yaml)
APRICOT_HOST="apricot"
APRICOT_USER="lilith"
APRICOT_IP="${CONFIG_VPN_LOCAL_IP:-10.9.0.1}"

# Detect if we're running on apricot (localhost)
IS_LOCAL_APRICOT=false
if [[ "$(hostname)" == "apricot"* ]] || [[ "$(hostname -f)" == *"apricot"* ]]; then
    IS_LOCAL_APRICOT=true
fi

# Ports (from infrastructure/ports.yaml)
PORT_API=3100
PORT_POSTGRES=5433
PORT_REDIS=6380
PORT_ML=8100

# Remote paths
VPS_DEPLOY_PATH="/opt/conversation-assistant"

# When running locally on apricot, use the source directory directly
if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
    APRICOT_DEPLOY_PATH="${FEATURE_DIR}"
else
    APRICOT_DEPLOY_PATH="/opt/conversation-assistant"
fi

# =============================================================================
# HELPER FUNCTIONS
# =============================================================================

ssh_apricot() {
    if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
        # Run locally - we're on apricot
        bash -c "$*"
    else
        ssh -o ConnectTimeout=10 -o BatchMode=yes "${APRICOT_USER}@${APRICOT_HOST}" "$@"
    fi
}

rsync_to_apricot() {
    local src="$1"
    local dest="$2"
    shift 2
    if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
        # Local copy
        rsync -avz --delete "$@" "$src" "$dest"
    else
        rsync -avz --delete "$@" "$src" "${APRICOT_USER}@${APRICOT_HOST}:${dest}"
    fi
}

scp_to_apricot() {
    local src="$1"
    local dest="$2"
    if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
        cp "$src" "$dest"
    else
        scp "$src" "${APRICOT_USER}@${APRICOT_HOST}:${dest}"
    fi
}

ssh_vps() {
    local ssh_cmd
    ssh_cmd=$(config_get_ssh_cmd "$CONFIG_VPS_HOST" "$CONFIG_VPS_USER")
    $ssh_cmd "$@"
}

check_env_placeholders() {
    local env_file="$1"
    if grep -qE "(GENERATE_WITH|COPY_FROM)" "$env_file" 2>/dev/null; then
        return 1
    fi
    return 0
}

# =============================================================================
# STAGE 1: INFRASTRUCTURE (apricot - PostgreSQL + Redis)
# =============================================================================

deploy_stage_1() {
    log_section "Stage 1: Infrastructure (apricot)"

    # Check host connectivity
    log_step "Checking apricot connectivity..."
    if ! ssh_apricot "echo ok" &>/dev/null; then
        log_failure "Cannot connect to apricot"
        return 1
    fi
    log_success "apricot reachable"

    # Check env file
    local env_file="${INFRA_DIR}/apricot/.env.apricot"
    if [ ! -f "$env_file" ]; then
        log_failure "Missing ${env_file}"
        return 1
    fi

    if ! check_env_placeholders "$env_file"; then
        log_failure ".env.apricot contains placeholder passwords"
        log_info "Generate with: openssl rand -hex 32"
        return 1
    fi

    # tqftw-fastapi-service-base is now installed from forge.nasty.sh registry
    # No local copy needed - Docker will pull from registry

    if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
        # Running locally - use source directory directly
        log_step "Running locally - using source directory"
    else
        # Remote deployment - sync files
        log_step "Creating directories on apricot..."
        ssh_apricot "mkdir -p ${APRICOT_DEPLOY_PATH}"

        log_step "Syncing feature files..."
        rsync_to_apricot "${FEATURE_DIR}/" "${APRICOT_DEPLOY_PATH}/" \
            --exclude 'node_modules' \
            --exclude 'dist' \
            --exclude '.git' \
            --exclude '__pycache__' \
            --exclude '.pytest_cache' \
            --exclude '.venv' \
            --exclude '*.pyc'
    fi

    # Setup env
    ssh_apricot "
        cd ${APRICOT_DEPLOY_PATH}
        [ -f infrastructure/apricot/.env.apricot ] && cp infrastructure/apricot/.env.apricot .env
    "

    # Start services
    log_step "Starting PostgreSQL and Redis..."
    ssh_apricot "
        cd ${APRICOT_DEPLOY_PATH}
        docker compose -f infrastructure/apricot/docker-compose.apricot.yml pull postgres redis
        docker compose -f infrastructure/apricot/docker-compose.apricot.yml up -d postgres redis
    "

    # Verify
    log_step "Verifying infrastructure..."
    sleep 5

    if ssh_apricot "docker exec conversation-assistant-postgres pg_isready -U conversation" &>/dev/null; then
        log_success "PostgreSQL ready"
    else
        log_failure "PostgreSQL not responding"
        return 1
    fi

    if ssh_apricot "docker exec conversation-assistant-redis redis-cli ping" &>/dev/null; then
        log_success "Redis ready"
    else
        log_failure "Redis not responding"
        return 1
    fi

    log_success "Stage 1 complete"
}

# =============================================================================
# STAGE 2: ML SERVICE (apricot - docker)
# =============================================================================

deploy_stage_2() {
    log_section "Stage 2: ML Service (apricot)"

    # Check host
    if ! ssh_apricot "echo ok" &>/dev/null; then
        log_failure "Cannot connect to apricot"
        return 1
    fi

    # Build and start ML service via docker-compose
    log_step "Building ML service container (this may take a while)..."
    ssh_apricot "
        cd ${APRICOT_DEPLOY_PATH}
        docker compose -f infrastructure/apricot/docker-compose.apricot.yml build ml-service
    " || {
        log_failure "ML service build failed"
        return 1
    }

    log_step "Starting ML service..."
    ssh_apricot "
        cd ${APRICOT_DEPLOY_PATH}
        docker compose -f infrastructure/apricot/docker-compose.apricot.yml up -d ml-service
    "

    # Wait and verify
    log_step "Waiting for ML service (may take a while for model loading)..."
    local retries=24  # 2 minutes with 5s intervals
    for ((i=1; i<=retries; i++)); do
        if ssh_apricot "curl -sf http://${APRICOT_IP}:${PORT_ML}/health" &>/dev/null; then
            log_success "ML service healthy"
            log_success "Stage 2 complete"
            return 0
        fi
        echo -n "."
        sleep 5
    done

    echo ""
    log_warn "ML service not responding - check: docker logs conversation-assistant-ml"
    return 1
}

# =============================================================================
# STAGE 3: WEB SERVICES (VPS - docker)
# =============================================================================

deploy_stage_3() {
    log_section "Stage 3: Web Services (VPS)"

    # Check VPS connectivity
    log_step "Checking VPS connectivity..."
    if ! ssh_vps "echo ok" &>/dev/null; then
        log_failure "Cannot connect to VPS"
        return 1
    fi
    log_success "VPS reachable"

    # Check VPN from VPS to apricot
    log_step "Checking VPN connectivity..."
    if ! ssh_vps "nc -zv ${APRICOT_IP} ${PORT_POSTGRES}" &>/dev/null; then
        log_failure "VPS cannot reach apricot:${PORT_POSTGRES} - check VPN"
        return 1
    fi
    log_success "VPS → apricot VPN OK"

    # Check env on VPS
    log_step "Checking VPS environment..."
    if ssh_vps "grep -q 'COPY_FROM_APRICOT' ${VPS_DEPLOY_PATH}/.env 2>/dev/null"; then
        log_failure "VPS .env contains placeholders - copy credentials from apricot"
        return 1
    fi

    # Sync and deploy
    log_step "Deploying to VPS..."
    ssh_vps "mkdir -p ${VPS_DEPLOY_PATH}"

    rsync -avz --delete \
        --exclude 'node_modules' \
        --exclude 'dist' \
        --exclude '.git' \
        --exclude '*.log' \
        --exclude '.env' \
        --exclude 'ml-service' \
        --exclude 'infrastructure/apricot' \
        "${FEATURE_DIR}/" \
        "${CONFIG_VPS_USER}@${CONFIG_VPS_HOST}:${VPS_DEPLOY_PATH}/"

    # Build and start
    log_step "Building and starting containers..."
    ssh_vps "
        cd ${VPS_DEPLOY_PATH}
        docker-compose -f docker-compose.vps.yml build
        docker-compose -f docker-compose.vps.yml up -d
    "

    # Wait for health
    log_step "Waiting for services..."
    local retries=12
    for ((i=1; i<=retries; i++)); do
        if ssh_vps "curl -sf http://127.0.0.1:${PORT_API}/api/health" &>/dev/null; then
            log_success "API healthy"
            break
        fi
        echo -n "."
        sleep 5
    done

    # Verify frontend
    if ssh_vps "curl -sf http://127.0.0.1:3101/" &>/dev/null; then
        log_success "Frontend serving"
    else
        log_warn "Frontend not responding"
    fi

    log_success "Stage 3 complete"
}

# =============================================================================
# VERIFICATION
# =============================================================================

verify_deployment() {
    log_section "Deployment Verification"

    local failed=0

    # Infrastructure
    log_step "Checking infrastructure..."
    if ssh_apricot "docker exec conversation-assistant-postgres pg_isready -U conversation" &>/dev/null; then
        log_success "PostgreSQL"
    else
        log_failure "PostgreSQL"
        ((failed++))
    fi

    if ssh_apricot "docker exec conversation-assistant-redis redis-cli ping" &>/dev/null; then
        log_success "Redis"
    else
        log_failure "Redis"
        ((failed++))
    fi

    # ML Service
    log_step "Checking ML service..."
    if ssh_apricot "curl -sf http://${APRICOT_IP}:${PORT_ML}/health" &>/dev/null; then
        log_success "ML Service"
    else
        log_warn "ML Service (may be starting)"
    fi

    # Connectivity
    log_step "Checking cross-host connectivity..."
    if ssh_vps "nc -zv ${APRICOT_IP} ${PORT_POSTGRES}" &>/dev/null; then
        log_success "VPS → PostgreSQL"
    else
        log_failure "VPS → PostgreSQL"
        ((failed++))
    fi

    if ssh_vps "nc -zv ${APRICOT_IP} ${PORT_REDIS}" &>/dev/null; then
        log_success "VPS → Redis"
    else
        log_failure "VPS → Redis"
        ((failed++))
    fi

    # Web services
    log_step "Checking web services..."
    if ssh_vps "curl -sf http://127.0.0.1:${PORT_API}/api/health" &>/dev/null; then
        log_success "API Server"
    else
        log_failure "API Server"
        ((failed++))
    fi

    if ssh_vps "curl -sf http://127.0.0.1:3101/" &>/dev/null; then
        log_success "Frontend"
    else
        log_failure "Frontend"
        ((failed++))
    fi

    # VPN Protection
    log_step "Checking VPN protection..."
    if ssh_vps "grep -q 'deny all' /etc/nginx/sites-available/conversations.nasty.sh.conf 2>/dev/null"; then
        log_success "nginx VPN protection configured"
    else
        log_warn "nginx VPN protection not found"
    fi

    # Summary
    echo ""
    if [ $failed -eq 0 ]; then
        log_success "All checks passed!"
        return 0
    else
        log_failure "$failed checks failed"
        return 1
    fi
}

# =============================================================================
# MAIN
# =============================================================================

usage() {
    echo "Usage: $0 [OPTIONS]"
    echo ""
    echo "Options:"
    echo "  --stage N    Deploy specific stage (1=infra, 2=ml, 3=web)"
    echo "  --verify     Run verification only"
    echo "  --help       Show this help"
    echo ""
    echo "Stages:"
    echo "  1 - Infrastructure (PostgreSQL + Redis on apricot)"
    echo "  2 - ML Service (systemd on apricot)"
    echo "  3 - Web Services (docker on VPS)"
    echo ""
}

main() {
    local stage=""
    local verify_only=false

    while [[ $# -gt 0 ]]; do
        case "$1" in
            --stage)
                stage="$2"
                shift 2
                ;;
            --verify)
                verify_only=true
                shift
                ;;
            --help|-h)
                usage
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                usage
                exit 1
                ;;
        esac
    done

    log_banner "Conversation Assistant Deployment"

    log_info "Configuration:"
    if [[ "$IS_LOCAL_APRICOT" == "true" ]]; then
        log_info "  Apricot: localhost (running locally)"
    else
        log_info "  Apricot: ${APRICOT_USER}@${APRICOT_HOST} (${APRICOT_IP})"
    fi
    log_info "  VPS:     ${CONFIG_VPS_USER}@${CONFIG_VPS_HOST}"
    log_info "  Ports:   API=${PORT_API}, PostgreSQL=${PORT_POSTGRES}, Redis=${PORT_REDIS}, ML=${PORT_ML}"
    echo ""

    if $verify_only; then
        verify_deployment
        exit $?
    fi

    case "$stage" in
        "")
            # Full deployment
            deploy_stage_1 || exit 1
            deploy_stage_2 || log_warn "Stage 2 had issues"
            deploy_stage_3 || exit 1
            verify_deployment
            ;;
        1)
            deploy_stage_1
            ;;
        2)
            deploy_stage_2
            ;;
        3)
            deploy_stage_3
            ;;
        *)
            log_error "Invalid stage: $stage"
            usage
            exit 1
            ;;
    esac

    echo ""
    log_success "Deployment complete!"
    echo ""
    log_info "Endpoints:"
    log_info "  Web:        https://conversations.nasty.sh"
    log_info "  API:        https://conversations.nasty.sh/api"
    log_info "  PostgreSQL: ${APRICOT_IP}:${PORT_POSTGRES}"
    log_info "  Redis:      ${APRICOT_IP}:${PORT_REDIS}"
    log_info "  ML:         http://${APRICOT_IP}:${PORT_ML}"
    echo ""
}

main "$@"