platform-codebase/infrastructure/scripts/verify-deployment.sh

#!/bin/bash
# =============================================================================
# Lilith Platform: Deployment Verification
# =============================================================================
#
# Comprehensive verification of deployed services
#
# Usage:
#   ./verify-deployment.sh              # Run all checks
#   ./verify-deployment.sh --quick      # Quick health checks only
#   ./verify-deployment.sh --verbose    # Detailed output
#   ./verify-deployment.sh --json       # JSON output for automation
#
# Exit codes:
#   0 - All checks passed
#   1 - Some checks failed
#   2 - Critical failure (infrastructure down)
#
# =============================================================================

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(dirname "$SCRIPT_DIR")"

# Hosts
APRICOT_IP="10.9.0.1"
VPS_HOST="0.1984.nasty.sh"
VPS_USER="root"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'

# State
CHECKS_PASSED=0
CHECKS_FAILED=0
CHECKS_WARNED=0
VERBOSE=false
JSON_OUTPUT=false
QUICK_MODE=false

# Results for JSON
declare -A RESULTS

# =============================================================================
# Logging
# =============================================================================
log_check() {
    local name="$1"
    local status="$2"
    local message="${3:-}"

    if $JSON_OUTPUT; then
        RESULTS["$name"]="$status"
        return
    fi

    case "$status" in
        pass)
            echo -e "  ${GREEN}✓${NC} $name"
            ((CHECKS_PASSED++))
            ;;
        fail)
            echo -e "  ${RED}✗${NC} $name${message:+ - $message}"
            ((CHECKS_FAILED++))
            ;;
        warn)
            echo -e "  ${YELLOW}!${NC} $name${message:+ - $message}"
            ((CHECKS_WARNED++))
            ;;
        skip)
            echo -e "  ${BLUE}-${NC} $name (skipped)"
            ;;
    esac
}

log_section() {
    if ! $JSON_OUTPUT; then
        echo ""
        echo -e "${BOLD}${CYAN}$1${NC}"
        echo -e "${CYAN}$(printf '%.0s─' {1..50})${NC}"
    fi
}

# =============================================================================
# Host Helpers
# =============================================================================
ssh_apricot() {
    ssh -o ConnectTimeout=5 -o BatchMode=yes "lilith@apricot" "$@" 2>/dev/null
}

ssh_vps() {
    ssh -o ConnectTimeout=5 -o BatchMode=yes "${VPS_USER}@${VPS_HOST}" "$@" 2>/dev/null
}

# =============================================================================
# Infrastructure Checks
# =============================================================================
check_infrastructure() {
    log_section "Infrastructure (apricot)"

    # Host reachability
    if ssh_apricot "echo ok" &>/dev/null; then
        log_check "apricot SSH" pass
    else
        log_check "apricot SSH" fail "Cannot connect"
        return 1
    fi

    # PostgreSQL container
    if ssh_apricot "docker ps --format '{{.Names}}' | grep -q conversation-assistant-postgres"; then
        log_check "PostgreSQL container running" pass
    else
        log_check "PostgreSQL container running" fail
        return 1
    fi

    # PostgreSQL health
    if ssh_apricot "docker exec conversation-assistant-postgres pg_isready -U conversation" &>/dev/null; then
        log_check "PostgreSQL accepting connections" pass
    else
        log_check "PostgreSQL accepting connections" fail
    fi

    # PostgreSQL listening on VPN
    if ssh_apricot "ss -tlnp | grep -q ':5432'" &>/dev/null; then
        log_check "PostgreSQL listening on 5432" pass
    else
        log_check "PostgreSQL listening on 5432" fail
    fi

    # Redis container
    if ssh_apricot "docker ps --format '{{.Names}}' | grep -q conversation-assistant-redis"; then
        log_check "Redis container running" pass
    else
        log_check "Redis container running" fail
    fi

    # Redis health
    if ssh_apricot "docker exec conversation-assistant-redis redis-cli ping" &>/dev/null; then
        log_check "Redis responding to ping" pass
    else
        log_check "Redis responding to ping" fail
    fi

    # Redis listening on VPN
    if ssh_apricot "ss -tlnp | grep -q ':6379'" &>/dev/null; then
        log_check "Redis listening on 6379" pass
    else
        log_check "Redis listening on 6379" fail
    fi
}

# =============================================================================
# ML Service Checks
# =============================================================================
check_ml_service() {
    log_section "ML Service (apricot)"

    # Systemd unit
    if ssh_apricot "systemctl is-active conversation-ml" &>/dev/null; then
        log_check "conversation-ml service active" pass
    else
        log_check "conversation-ml service active" fail
    fi

    # Health endpoint
    if ssh_apricot "curl -sf http://${APRICOT_IP}:8100/health" &>/dev/null; then
        log_check "ML service health endpoint" pass
    else
        log_check "ML service health endpoint" warn "May be starting up"
    fi

    # GPU availability (optional)
    if $VERBOSE; then
        if ssh_apricot "nvidia-smi" &>/dev/null; then
            log_check "GPU available" pass
        else
            log_check "GPU available" warn "nvidia-smi not found"
        fi
    fi
}

# =============================================================================
# Web Service Checks
# =============================================================================
check_web_services() {
    log_section "Web Services (VPS)"

    # Host reachability
    if ssh_vps "echo ok" &>/dev/null; then
        log_check "VPS SSH" pass
    else
        log_check "VPS SSH" fail "Cannot connect"
        return 1
    fi

    # Server container
    if ssh_vps "docker ps --format '{{.Names}}' | grep -q conversation-assistant-server"; then
        log_check "Server container running" pass
    else
        log_check "Server container running" fail
    fi

    # Frontend container
    if ssh_vps "docker ps --format '{{.Names}}' | grep -q conversation-assistant-frontend"; then
        log_check "Frontend container running" pass
    else
        log_check "Frontend container running" fail
    fi

    # Server health (internal)
    if ssh_vps "curl -sf http://127.0.0.1:3100/api/health" &>/dev/null; then
        log_check "Server health (localhost)" pass
    else
        log_check "Server health (localhost)" fail
    fi

    # Frontend health (internal)
    if ssh_vps "curl -sf http://127.0.0.1:3101/" &>/dev/null; then
        log_check "Frontend serving (localhost)" pass
    else
        log_check "Frontend serving (localhost)" fail
    fi

    # Nginx running
    if ssh_vps "systemctl is-active nginx" &>/dev/null; then
        log_check "nginx active" pass
    else
        log_check "nginx active" fail
    fi

    # Nginx config valid
    if ssh_vps "nginx -t" &>/dev/null; then
        log_check "nginx config valid" pass
    else
        log_check "nginx config valid" fail
    fi
}

# =============================================================================
# Connectivity Checks
# =============================================================================
check_connectivity() {
    log_section "Cross-Host Connectivity"

    # VPS → Apricot PostgreSQL
    if ssh_vps "nc -zv ${APRICOT_IP} 5432" &>/dev/null; then
        log_check "VPS → Apricot PostgreSQL" pass
    else
        log_check "VPS → Apricot PostgreSQL" fail "VPN issue?"
    fi

    # VPS → Apricot Redis
    if ssh_vps "nc -zv ${APRICOT_IP} 6379" &>/dev/null; then
        log_check "VPS → Apricot Redis" pass
    else
        log_check "VPS → Apricot Redis" fail "VPN issue?"
    fi

    # VPS → Apricot ML
    if ssh_vps "curl -sf http://${APRICOT_IP}:8100/health" &>/dev/null; then
        log_check "VPS → Apricot ML Service" pass
    else
        log_check "VPS → Apricot ML Service" warn "Service may be starting"
    fi

    # Database connection test (actual connection)
    if ssh_vps "docker exec conversation-assistant-server node -e \"
        const { Client } = require('pg');
        const c = new Client({host:'${APRICOT_IP}',port:5432,user:'conversation',database:'conversation_assistant'});
        c.connect().then(() => c.end()).catch(() => process.exit(1));
    \"" &>/dev/null; then
        log_check "Server → PostgreSQL connection" pass
    else
        log_check "Server → PostgreSQL connection" warn "Check credentials"
    fi
}

# =============================================================================
# VPN Protection Checks
# =============================================================================
check_vpn_protection() {
    log_section "VPN Protection"

    # Check nginx config has deny rules
    if ssh_vps "grep -q 'deny all' /etc/nginx/sites-available/conversations.nasty.sh.conf"; then
        log_check "nginx deny all configured" pass
    else
        log_check "nginx deny all configured" fail
    fi

    # Check allowed networks
    if ssh_vps "grep -q '10.9.0.0/24' /etc/nginx/sites-available/conversations.nasty.sh.conf"; then
        log_check "Tailscale network allowed" pass
    else
        log_check "Tailscale network allowed" fail
    fi

    if ssh_vps "grep -q '10.8.0.0/24' /etc/nginx/sites-available/conversations.nasty.sh.conf"; then
        log_check "Wireguard network allowed" pass
    else
        log_check "Wireguard network allowed" warn "Optional"
    fi

    # Test from VPN (we're on VPN if we can SSH to apricot)
    if curl -sf --connect-timeout 5 "https://conversations.nasty.sh/api/health" &>/dev/null; then
        log_check "HTTPS accessible from VPN" pass
    else
        log_check "HTTPS accessible from VPN" warn "Check VPN or SSL"
    fi
}

# =============================================================================
# Database Checks
# =============================================================================
check_database() {
    log_section "Database Health"

    # Tables exist
    local table_count
    table_count=$(ssh_apricot "docker exec conversation-assistant-postgres psql -U conversation -d conversation_assistant -t -c \"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public'\"" 2>/dev/null | tr -d ' ')

    if [ -n "$table_count" ] && [ "$table_count" -gt 0 ]; then
        log_check "Database tables exist ($table_count tables)" pass
    else
        log_check "Database tables exist" warn "Run migrations?"
    fi

    # Connection count
    if $VERBOSE; then
        local conn_count
        conn_count=$(ssh_apricot "docker exec conversation-assistant-postgres psql -U conversation -d conversation_assistant -t -c \"SELECT COUNT(*) FROM pg_stat_activity WHERE datname = 'conversation_assistant'\"" 2>/dev/null | tr -d ' ')
        log_check "Active connections: $conn_count" pass
    fi
}

# =============================================================================
# E2E Checks
# =============================================================================
check_e2e() {
    log_section "End-to-End Checks"

    # API returns valid JSON
    local health_response
    health_response=$(curl -sf --connect-timeout 5 "https://conversations.nasty.sh/api/health" 2>/dev/null || echo "")

    if echo "$health_response" | jq -e . &>/dev/null; then
        log_check "API returns valid JSON" pass
    else
        log_check "API returns valid JSON" fail
    fi

    # API has expected fields
    if echo "$health_response" | jq -e '.status' &>/dev/null; then
        log_check "API health has status field" pass
    else
        log_check "API health has status field" warn
    fi

    # Frontend serves HTML
    local frontend_response
    frontend_response=$(curl -sf --connect-timeout 5 "https://conversations.nasty.sh/" 2>/dev/null || echo "")

    if echo "$frontend_response" | grep -q '<html' &>/dev/null; then
        log_check "Frontend serves HTML" pass
    else
        log_check "Frontend serves HTML" fail
    fi
}

# =============================================================================
# Summary
# =============================================================================
print_summary() {
    if $JSON_OUTPUT; then
        echo "{"
        echo "  \"passed\": $CHECKS_PASSED,"
        echo "  \"failed\": $CHECKS_FAILED,"
        echo "  \"warned\": $CHECKS_WARNED,"
        echo "  \"results\": {"
        local first=true
        for key in "${!RESULTS[@]}"; do
            if $first; then
                first=false
            else
                echo ","
            fi
            echo -n "    \"$key\": \"${RESULTS[$key]}\""
        done
        echo ""
        echo "  }"
        echo "}"
        return
    fi

    echo ""
    echo -e "${BOLD}═══════════════════════════════════════════════════${NC}"
    echo -e "${BOLD}                   VERIFICATION SUMMARY             ${NC}"
    echo -e "${BOLD}═══════════════════════════════════════════════════${NC}"
    echo ""
    echo -e "  ${GREEN}Passed:${NC}  $CHECKS_PASSED"
    echo -e "  ${YELLOW}Warned:${NC}  $CHECKS_WARNED"
    echo -e "  ${RED}Failed:${NC}  $CHECKS_FAILED"
    echo ""

    if [ $CHECKS_FAILED -eq 0 ]; then
        echo -e "  ${GREEN}${BOLD}All critical checks passed!${NC}"
    else
        echo -e "  ${RED}${BOLD}Some checks failed - review above${NC}"
    fi
    echo ""
}

# =============================================================================
# Main
# =============================================================================
usage() {
    echo "Usage: $0 [OPTIONS]"
    echo ""
    echo "Options:"
    echo "  --quick      Quick checks only (health endpoints)"
    echo "  --verbose    Include detailed checks"
    echo "  --json       Output results as JSON"
    echo "  -h, --help   Show this help"
    echo ""
}

main() {
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --quick)
                QUICK_MODE=true
                shift
                ;;
            --verbose)
                VERBOSE=true
                shift
                ;;
            --json)
                JSON_OUTPUT=true
                shift
                ;;
            -h|--help)
                usage
                exit 0
                ;;
            *)
                echo "Unknown option: $1"
                usage
                exit 1
                ;;
        esac
    done

    if ! $JSON_OUTPUT; then
        echo ""
        echo -e "${BOLD}╔════════════════════════════════════════════════════════════╗${NC}"
        echo -e "${BOLD}║           Lilith Platform: Deployment Verification         ║${NC}"
        echo -e "${BOLD}╚════════════════════════════════════════════════════════════╝${NC}"
    fi

    # Run checks
    check_infrastructure || true
    check_ml_service || true
    check_web_services || true
    check_connectivity || true

    if ! $QUICK_MODE; then
        check_vpn_protection || true
        check_database || true
        check_e2e || true
    fi

    # Print summary
    print_summary

    # Exit code
    if [ $CHECKS_FAILED -gt 0 ]; then
        exit 1
    fi
    exit 0
}

main "$@"