b5fe73edd0
- Add PostgreSQL + Redis deployment stack - Add reconciliation framework for fleet management - Add VPS setup scripts (nginx, wireguard) - Add dev environment bootstrap scripts - Update service-registry and systemd configs 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
297 lines
9.6 KiB
YAML
297 lines
9.6 KiB
YAML
version: '3.8'
|
|
|
|
# =============================================================================
|
|
# DATABASE SERVICES: Production-ready database stack for Lilith Platform
|
|
# =============================================================================
|
|
#
|
|
# Host: apricot (10.9.0.1 on VPN)
|
|
# Purpose: Centralized database services for all environments
|
|
# Data Location: /mnt/bigdisk/_/lilith-platform/databases/
|
|
#
|
|
# Usage:
|
|
# docker-compose -f docker-compose.databases.yml --env-file .env.databases up -d
|
|
#
|
|
# Architecture:
|
|
# - PostgreSQL 16 + TimescaleDB: Primary data store with time-series optimization
|
|
# - Redis 7: Caching layer, sessions, rate limiting, job queues
|
|
# - Meilisearch: Full-text search engine for content discovery
|
|
#
|
|
# Network:
|
|
# - All services bind to 0.0.0.0 (accessible via VPN)
|
|
# - VPN subnet: 10.9.0.0/24
|
|
# - Apricot: 10.9.0.1, VPS: 10.9.0.2
|
|
#
|
|
# =============================================================================
|
|
|
|
services:
|
|
# =============================================================================
|
|
# PRIMARY DATABASE: PostgreSQL 16 with TimescaleDB Extension
|
|
# =============================================================================
|
|
# TimescaleDB provides time-series optimizations for analytics, metrics, logs
|
|
# All existing PostgreSQL functionality remains unchanged
|
|
postgres:
|
|
image: timescale/timescaledb:2.16.1-pg16
|
|
container_name: lilith-postgres-production
|
|
restart: unless-stopped
|
|
|
|
network_mode: host
|
|
|
|
environment:
|
|
# Database credentials
|
|
POSTGRES_USER: ${POSTGRES_USER:-postgres}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
POSTGRES_DB: ${POSTGRES_DB:-lilith_platform}
|
|
|
|
# Performance tuning - generous settings for dedicated database server
|
|
# shared_buffers: 25% of RAM (assuming 32GB+ available)
|
|
POSTGRES_SHARED_BUFFERS: ${POSTGRES_SHARED_BUFFERS:-8GB}
|
|
|
|
# work_mem: Per-operation memory for sorting/hashing
|
|
POSTGRES_WORK_MEM: ${POSTGRES_WORK_MEM:-256MB}
|
|
|
|
# effective_cache_size: Hint for query planner (50-75% of RAM)
|
|
POSTGRES_EFFECTIVE_CACHE_SIZE: ${POSTGRES_EFFECTIVE_CACHE_SIZE:-24GB}
|
|
|
|
# maintenance_work_mem: Memory for VACUUM, CREATE INDEX
|
|
POSTGRES_MAINTENANCE_WORK_MEM: ${POSTGRES_MAINTENANCE_WORK_MEM:-2GB}
|
|
|
|
# Connection pool settings
|
|
POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-200}
|
|
|
|
# WAL settings for performance
|
|
POSTGRES_WAL_BUFFERS: ${POSTGRES_WAL_BUFFERS:-16MB}
|
|
POSTGRES_CHECKPOINT_COMPLETION_TARGET: ${POSTGRES_CHECKPOINT_COMPLETION_TARGET:-0.9}
|
|
|
|
# Enable SSL for VPN connections
|
|
POSTGRES_HOST_AUTH_METHOD: ${POSTGRES_HOST_AUTH_METHOD:-scram-sha-256}
|
|
|
|
volumes:
|
|
# Data persistence on /mnt/bigdisk
|
|
- ${POSTGRES_DATA_DIR:-/mnt/bigdisk/_/lilith-platform/databases/postgresql}:/var/lib/postgresql/data
|
|
|
|
# Custom PostgreSQL configuration
|
|
- ./postgresql/postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
|
- ./postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro
|
|
|
|
# SSL certificates for VPN connections
|
|
- ${POSTGRES_SSL_CERT:-./postgresql/ssl/server.crt}:/var/lib/postgresql/server.crt:ro
|
|
- ${POSTGRES_SSL_KEY:-./postgresql/ssl/server.key}:/var/lib/postgresql/server.key:ro
|
|
|
|
# Initialization scripts
|
|
- ./postgresql/init.d:/docker-entrypoint-initdb.d:ro
|
|
|
|
command:
|
|
- postgres
|
|
- -c
|
|
- config_file=/etc/postgresql/postgresql.conf
|
|
- -c
|
|
- hba_file=/etc/postgresql/pg_hba.conf
|
|
- -c
|
|
- listen_addresses=0.0.0.0
|
|
- -c
|
|
- port=5432
|
|
- -c
|
|
- shared_buffers=${POSTGRES_SHARED_BUFFERS:-8GB}
|
|
- -c
|
|
- effective_cache_size=${POSTGRES_EFFECTIVE_CACHE_SIZE:-24GB}
|
|
- -c
|
|
- maintenance_work_mem=${POSTGRES_MAINTENANCE_WORK_MEM:-2GB}
|
|
- -c
|
|
- work_mem=${POSTGRES_WORK_MEM:-256MB}
|
|
- -c
|
|
- max_connections=${POSTGRES_MAX_CONNECTIONS:-200}
|
|
- -c
|
|
- wal_buffers=${POSTGRES_WAL_BUFFERS:-16MB}
|
|
- -c
|
|
- checkpoint_completion_target=${POSTGRES_CHECKPOINT_COMPLETION_TARGET:-0.9}
|
|
- -c
|
|
- random_page_cost=1.1
|
|
- -c
|
|
- effective_io_concurrency=200
|
|
- -c
|
|
- max_worker_processes=4
|
|
- -c
|
|
- max_parallel_workers_per_gather=2
|
|
- -c
|
|
- max_parallel_workers=4
|
|
- -c
|
|
- log_timezone=UTC
|
|
- -c
|
|
- timezone=UTC
|
|
|
|
healthcheck:
|
|
test: ['CMD-SHELL', 'pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-lilith_platform}']
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 30s
|
|
|
|
logging:
|
|
driver: "json-file"
|
|
options:
|
|
max-size: "100m"
|
|
max-file: "10"
|
|
labels: "service=postgres,env=production"
|
|
|
|
# =============================================================================
|
|
# CACHE & QUEUES: Redis 7
|
|
# =============================================================================
|
|
# Used for: Sessions, caching, rate limiting, BullMQ job queues, pub/sub
|
|
# Configured for generous memory usage as primary caching layer
|
|
redis:
|
|
image: redis:7.4-alpine
|
|
container_name: lilith-redis-production
|
|
restart: unless-stopped
|
|
|
|
network_mode: host
|
|
|
|
environment:
|
|
# Redis configuration via environment
|
|
REDIS_PASSWORD: ${REDIS_PASSWORD}
|
|
|
|
volumes:
|
|
# Data persistence with AOF (Append-Only File)
|
|
- ${REDIS_DATA_DIR:-/mnt/bigdisk/_/lilith-platform/databases/redis}:/data
|
|
|
|
# Custom Redis configuration
|
|
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
|
|
|
command:
|
|
- redis-server
|
|
- /usr/local/etc/redis/redis.conf
|
|
- --port
|
|
- "6379"
|
|
- --bind
|
|
- "0.0.0.0"
|
|
- --requirepass
|
|
- "${REDIS_PASSWORD}"
|
|
# Persistence: AOF enabled for durability
|
|
- --appendonly
|
|
- "yes"
|
|
- --appendfsync
|
|
- "everysec"
|
|
# Memory management
|
|
- --maxmemory
|
|
- "${REDIS_MAX_MEMORY:-4GB}"
|
|
- --maxmemory-policy
|
|
- "allkeys-lru"
|
|
# Performance tuning
|
|
- --tcp-backlog
|
|
- "511"
|
|
- --timeout
|
|
- "300"
|
|
- --tcp-keepalive
|
|
- "300"
|
|
# Disable RDB snapshots (AOF is sufficient)
|
|
- --save
|
|
- ""
|
|
# Logging
|
|
- --loglevel
|
|
- "${REDIS_LOG_LEVEL:-notice}"
|
|
|
|
healthcheck:
|
|
test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping']
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 5
|
|
start_period: 10s
|
|
|
|
logging:
|
|
driver: "json-file"
|
|
options:
|
|
max-size: "50m"
|
|
max-file: "5"
|
|
labels: "service=redis,env=production"
|
|
|
|
# =============================================================================
|
|
# SEARCH: Meilisearch
|
|
# =============================================================================
|
|
# Full-text search engine for profile discovery, content search
|
|
# Features: Typo tolerance, faceted filtering, geo search, sub-50ms responses
|
|
meilisearch:
|
|
image: getmeili/meilisearch:v1.12
|
|
container_name: lilith-meilisearch-production
|
|
restart: unless-stopped
|
|
|
|
network_mode: host
|
|
|
|
environment:
|
|
# Master key for API authentication
|
|
MEILI_MASTER_KEY: ${MEILI_MASTER_KEY}
|
|
|
|
# Environment (production/development)
|
|
MEILI_ENV: ${MEILI_ENV:-production}
|
|
|
|
# HTTP address binding
|
|
MEILI_HTTP_ADDR: ${MEILI_HTTP_ADDR:-0.0.0.0:7700}
|
|
|
|
# Database path
|
|
MEILI_DB_PATH: /meili_data
|
|
|
|
# Disable telemetry
|
|
MEILI_NO_ANALYTICS: ${MEILI_NO_ANALYTICS:-true}
|
|
|
|
# Performance settings
|
|
MEILI_MAX_INDEXING_MEMORY: ${MEILI_MAX_INDEXING_MEMORY:-2GB}
|
|
MEILI_MAX_INDEXING_THREADS: ${MEILI_MAX_INDEXING_THREADS:-2}
|
|
|
|
# Logging
|
|
MEILI_LOG_LEVEL: ${MEILI_LOG_LEVEL:-INFO}
|
|
|
|
volumes:
|
|
# Data persistence
|
|
- ${MEILI_DATA_DIR:-/mnt/bigdisk/_/lilith-platform/databases/meilisearch}:/meili_data
|
|
|
|
# Snapshots directory (for backups)
|
|
- ${MEILI_SNAPSHOT_DIR:-/mnt/bigdisk/_/lilith-platform/databases/meilisearch-snapshots}:/snapshots
|
|
|
|
healthcheck:
|
|
test: ['CMD', 'wget', '--no-verbose', '--tries=1', '--spider', 'http://localhost:7700/health']
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 5
|
|
start_period: 20s
|
|
|
|
logging:
|
|
driver: "json-file"
|
|
options:
|
|
max-size: "50m"
|
|
max-file: "5"
|
|
labels: "service=meilisearch,env=production"
|
|
|
|
# =============================================================================
|
|
# NOTES
|
|
# =============================================================================
|
|
#
|
|
# 1. Network Configuration:
|
|
# - Using host network mode for maximum performance and VPN accessibility
|
|
# - All services bind to 0.0.0.0 (accessible from VPN: 10.9.0.0/24)
|
|
# - No port mapping needed (services use standard ports directly)
|
|
#
|
|
# 2. Data Persistence:
|
|
# - All data stored on /mnt/bigdisk for large capacity
|
|
# - Volumes are bind mounts (not Docker volumes) for direct access
|
|
# - Backup strategy should target these directories
|
|
#
|
|
# 3. Security:
|
|
# - PostgreSQL: SSL-enabled, scram-sha-256 authentication
|
|
# - Redis: Password-protected (set REDIS_PASSWORD in .env.databases)
|
|
# - Meilisearch: Master key authentication (set MEILI_MASTER_KEY)
|
|
# - VPN-only access (no public internet exposure)
|
|
#
|
|
# 4. Performance:
|
|
# - PostgreSQL: Tuned for 32GB+ RAM server
|
|
# - Redis: 4GB max memory with LRU eviction
|
|
# - Meilisearch: 2GB indexing memory limit
|
|
#
|
|
# 5. Monitoring:
|
|
# - Health checks enabled for all services
|
|
# - Logs limited to prevent disk fill (100MB/50MB max per service)
|
|
# - Use `docker-compose logs -f <service>` to monitor
|
|
#
|
|
# 6. Backup Strategy:
|
|
# - PostgreSQL: pg_dump + WAL archiving (configure separately)
|
|
# - Redis: AOF persistence (automatic)
|
|
# - Meilisearch: Snapshots stored in separate directory
|
|
#
|
|
# =============================================================================
|