586 lines
17 KiB
TypeScript
586 lines
17 KiB
TypeScript
/**
|
|
* Docker Compose operations
|
|
*
|
|
* Provides:
|
|
* - Container lifecycle management
|
|
* - Health checking
|
|
* - Log streaming
|
|
*/
|
|
|
|
import { spawn, exec } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { Logger } from '../utils/logger.js';
|
|
import { loadConfig, ALL_PROFILES, type RunConfig } from '../utils/config.js';
|
|
import { PhaseTimeoutManager } from '../utils/timeout-manager.js';
|
|
|
|
const execAsync = promisify(exec);
|
|
|
|
// =============================================================================
|
|
// Types
|
|
// =============================================================================
|
|
|
|
export interface ContainerStatus {
|
|
name: string;
|
|
status: string;
|
|
health?: 'healthy' | 'unhealthy' | 'starting';
|
|
ports?: string;
|
|
}
|
|
|
|
export interface DockerComposeOptions {
|
|
profiles?: string[];
|
|
envFile?: string;
|
|
detach?: boolean;
|
|
follow?: boolean;
|
|
serviceName?: string;
|
|
timeout?: number;
|
|
volumes?: boolean;
|
|
}
|
|
|
|
// =============================================================================
|
|
// Docker Operations
|
|
// =============================================================================
|
|
|
|
export class DockerOps {
|
|
private config: RunConfig;
|
|
private logger: Logger;
|
|
private verboseMode = false;
|
|
|
|
constructor(logger?: Logger) {
|
|
this.config = loadConfig();
|
|
this.logger = logger ?? new Logger({ context: 'Docker' });
|
|
}
|
|
|
|
/**
|
|
* Enable verbose logging mode
|
|
*/
|
|
setVerbose(enabled: boolean): void {
|
|
this.verboseMode = enabled;
|
|
}
|
|
|
|
/**
|
|
* Remove orphaned lilith networks before startup
|
|
* Disconnects any remaining containers and removes the network
|
|
*/
|
|
async cleanupNetworks(environment: string = 'dev'): Promise<void> {
|
|
const networkName = `lilith-${environment}-network`;
|
|
|
|
try {
|
|
// First, try to disconnect any containers still connected
|
|
const { stdout: containers } = await execAsync(
|
|
`docker network inspect ${networkName} --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null || echo ""`,
|
|
{ cwd: this.config.projectRoot }
|
|
);
|
|
|
|
for (const container of containers.trim().split(' ').filter(Boolean)) {
|
|
await execAsync(`docker network disconnect -f ${networkName} ${container} 2>/dev/null || true`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
}
|
|
|
|
// Remove the network
|
|
await execAsync(`docker network rm ${networkName} 2>/dev/null || true`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
|
|
// Also clean up orphaned compose networks (format: projectname_networkname)
|
|
await execAsync(`docker network rm lilith-${environment}_lilith-${environment} 2>/dev/null || true`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
|
|
this.logger.debug(`Cleaned up network: ${networkName}`);
|
|
} catch {
|
|
// Network cleanup is best-effort, don't fail startup
|
|
this.logger.debug('Network cleanup skipped (no networks to remove)');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if Docker daemon is running
|
|
*/
|
|
async checkDocker(): Promise<boolean> {
|
|
try {
|
|
await execAsync('docker info');
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Start containers with profiles
|
|
*/
|
|
async up(options: DockerComposeOptions = {}): Promise<void> {
|
|
const {
|
|
profiles = ['core', 'platform'],
|
|
envFile = this.config.envDev,
|
|
detach = true,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('up');
|
|
if (detach) args.push('-d');
|
|
|
|
await this.runCompose(args);
|
|
}
|
|
|
|
/**
|
|
* Stop containers
|
|
*/
|
|
async down(options: DockerComposeOptions = {}): Promise<void> {
|
|
const {
|
|
profiles = ALL_PROFILES,
|
|
envFile = this.config.envDev,
|
|
timeout = 5, // Fast timeout for dev environments
|
|
volumes = false,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('down');
|
|
args.push('-t', timeout.toString());
|
|
if (volumes) {
|
|
args.push('-v', '--remove-orphans');
|
|
}
|
|
|
|
await this.runCompose(args);
|
|
}
|
|
|
|
/**
|
|
* Stop containers and remove volumes (fresh DB reset)
|
|
*/
|
|
async reset(options: DockerComposeOptions = {}): Promise<void> {
|
|
const {
|
|
profiles = ALL_PROFILES,
|
|
envFile = this.config.envDev,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('down', '-v', '--remove-orphans');
|
|
|
|
this.logger.info('Stopping containers and removing volumes...');
|
|
|
|
try {
|
|
await this.runCompose(args, 45_000);
|
|
} catch (err) {
|
|
// If timed out, force remove containers with short timeout
|
|
this.logger.warn('Graceful shutdown timed out, forcing container removal...');
|
|
const forceArgs = this.buildComposeArgs(envFile, profiles);
|
|
forceArgs.push('down', '-v', '--remove-orphans', '-t', '1');
|
|
await this.runCompose(forceArgs, 15_000);
|
|
}
|
|
|
|
this.logger.success('Containers stopped and volumes removed');
|
|
}
|
|
|
|
/**
|
|
* Get container status
|
|
*/
|
|
async status(options: DockerComposeOptions = {}): Promise<ContainerStatus[]> {
|
|
const {
|
|
profiles = ALL_PROFILES,
|
|
envFile = this.config.envDev,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('ps', '--format', 'json');
|
|
|
|
try {
|
|
const { stdout } = await execAsync(`docker compose ${args.join(' ')}`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
|
|
const containers: ContainerStatus[] = [];
|
|
|
|
// Parse JSON lines output
|
|
for (const line of stdout.trim().split('\n')) {
|
|
if (!line) continue;
|
|
try {
|
|
const data = JSON.parse(line);
|
|
containers.push({
|
|
name: data.Name || data.name,
|
|
status: data.Status || data.State,
|
|
health: this.parseHealth(data.Health || data.Status),
|
|
ports: data.Ports,
|
|
});
|
|
} catch {
|
|
// Skip non-JSON lines
|
|
}
|
|
}
|
|
|
|
return containers;
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stream logs from containers
|
|
*/
|
|
async logs(options: DockerComposeOptions = {}): Promise<void> {
|
|
const {
|
|
profiles = ALL_PROFILES,
|
|
envFile = this.config.envDev,
|
|
follow = true,
|
|
serviceName,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('logs');
|
|
if (follow) args.push('-f');
|
|
if (serviceName) args.push(serviceName);
|
|
|
|
// Spawn for streaming output
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('docker', ['compose', ...args], {
|
|
cwd: this.config.projectRoot,
|
|
stdio: 'inherit',
|
|
});
|
|
|
|
child.on('error', reject);
|
|
child.on('close', code => {
|
|
if (code === 0) resolve();
|
|
else reject(new Error(`Logs exited with code ${code}`));
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Restart containers
|
|
*/
|
|
async restart(options: DockerComposeOptions = {}): Promise<void> {
|
|
const {
|
|
profiles = ['core', 'platform'],
|
|
envFile = this.config.envDev,
|
|
serviceName,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('restart');
|
|
if (serviceName) args.push(serviceName);
|
|
|
|
await this.runCompose(args);
|
|
}
|
|
|
|
/**
|
|
* Wait for containers to be healthy
|
|
*
|
|
* @param timeoutMs - Maximum wait time in milliseconds
|
|
* @param onProgress - Optional callback called on each health check poll with
|
|
* current container statuses and the iteration number
|
|
*/
|
|
async waitForHealthy(
|
|
timeoutMs = 60000,
|
|
onProgress?: (containers: ContainerStatus[], iteration: number) => void,
|
|
): Promise<boolean> {
|
|
const start = Date.now();
|
|
const checkInterval = 2000;
|
|
let checkCount = 0;
|
|
let lastContainers: ContainerStatus[] = [];
|
|
|
|
while (Date.now() - start < timeoutMs) {
|
|
const containers = await this.status();
|
|
lastContainers = containers;
|
|
checkCount++;
|
|
|
|
if (this.verboseMode) {
|
|
this.logger.verbose(`[health] Check ${checkCount}: ${containers.length} containers`);
|
|
for (const c of containers) {
|
|
this.logger.verbose(` ${c.name}: ${c.health || 'no-health'} (${c.status})`);
|
|
}
|
|
}
|
|
|
|
onProgress?.(containers, checkCount);
|
|
|
|
const allHealthy = containers.every(
|
|
c => c.health === 'healthy' || !c.status.includes('health')
|
|
);
|
|
|
|
if (allHealthy && containers.length > 0) {
|
|
return true;
|
|
}
|
|
|
|
await this.sleep(checkInterval);
|
|
}
|
|
|
|
// Report which containers are unhealthy for diagnosis
|
|
const unhealthy = lastContainers.filter(
|
|
c => c.health === 'unhealthy' || c.health === 'starting'
|
|
);
|
|
if (unhealthy.length > 0) {
|
|
this.logger.warn('Unhealthy containers after timeout:');
|
|
for (const c of unhealthy) {
|
|
this.logger.warn(` ${c.name}: ${c.health ?? 'unknown'} (${c.status})`);
|
|
}
|
|
this.logger.warn('Run "docker logs <container>" for details');
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Run database migrations for all features
|
|
* @param onLog - Optional callback to receive streaming log output
|
|
*/
|
|
async runMigrations(onLog?: (message: string) => void): Promise<void> {
|
|
const scriptPath = `${this.config.projectRoot}/tooling/scripts/database/migrate-all-dev.ts`;
|
|
const timeoutMs = 120000;
|
|
|
|
// Create timeout manager for progressive warnings
|
|
const timeoutMgr = new PhaseTimeoutManager({
|
|
phaseName: 'Migrations',
|
|
timeoutMs,
|
|
pollIntervalMs: 1000,
|
|
heartbeatIntervalMs: 15000,
|
|
onWarning: (warning) => {
|
|
onLog?.(warning.message);
|
|
},
|
|
onHeartbeat: (elapsed) => {
|
|
onLog?.(`Migration running for ${Math.round(elapsed/1000)}s...`);
|
|
},
|
|
});
|
|
|
|
return new Promise((resolve, reject) => {
|
|
// Use bun directly - handles ESM better than tsx for ESM-only packages
|
|
const bunPath = `${process.env.HOME}/.bun/bin/bun`;
|
|
const child = spawn(bunPath, [scriptPath], {
|
|
cwd: this.config.projectRoot,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
|
|
// Interval for timeout checks and heartbeats
|
|
const heartbeat = setInterval(() => {
|
|
timeoutMgr.checkWarnings();
|
|
timeoutMgr.checkHeartbeat('');
|
|
|
|
if (timeoutMgr.isTimedOut()) {
|
|
clearInterval(heartbeat);
|
|
child.kill('SIGTERM');
|
|
reject(new Error('Migration timed out after 120s'));
|
|
}
|
|
}, 1000);
|
|
|
|
const processLine = (line: string) => {
|
|
const trimmed = line.trim();
|
|
if (onLog && trimmed) {
|
|
onLog(trimmed);
|
|
}
|
|
};
|
|
|
|
child.stdout?.on('data', (data: Buffer) => {
|
|
data.toString().split('\n').forEach(processLine);
|
|
});
|
|
|
|
child.stderr?.on('data', (data: Buffer) => {
|
|
data.toString().split('\n').forEach(processLine);
|
|
});
|
|
|
|
child.on('close', (code) => {
|
|
clearInterval(heartbeat);
|
|
if (code === 0) {
|
|
this.logger.success('Migrations complete');
|
|
resolve();
|
|
} else {
|
|
reject(new Error(`Migration exited with code ${code}`));
|
|
}
|
|
});
|
|
|
|
child.on('error', (err) => {
|
|
clearInterval(heartbeat);
|
|
reject(err);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Seed databases with development data
|
|
*/
|
|
async runSeeds(): Promise<void> {
|
|
const scriptPath = `${this.config.projectRoot}/tooling/scripts/database/seed-all.sh`;
|
|
|
|
try {
|
|
await execAsync(`bash ${scriptPath}`, {
|
|
cwd: this.config.projectRoot,
|
|
timeout: 300000,
|
|
env: {
|
|
...process.env,
|
|
DB_HOST: 'localhost',
|
|
DB_PORT: '5432',
|
|
DB_USER: 'postgres',
|
|
DB_PASSWORD: 'postgres',
|
|
DB_NAME: 'lilith_dev',
|
|
},
|
|
});
|
|
this.logger.success('Database seeding complete');
|
|
} catch (err) {
|
|
// Seeds may partially fail, which is often OK
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
this.logger.warn(`Seeding warning: ${message}`);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Private Methods
|
|
// ---------------------------------------------------------------------------
|
|
|
|
private buildComposeArgs(envFile: string, profiles: string[]): string[] {
|
|
const args = ['-f', this.config.composeFile, '--env-file', envFile];
|
|
|
|
for (const profile of profiles) {
|
|
args.push('--profile', profile);
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
private async runCompose(args: string[], timeoutMs: number = 120_000): Promise<void> {
|
|
const cmd = `docker compose ${args.join(' ')}`;
|
|
this.logger.debug(`Running: ${cmd}`);
|
|
|
|
if (this.verboseMode) {
|
|
this.logger.verbose(`[docker] Executing: ${cmd}`);
|
|
this.logger.detail('docker', 'compose', { args, timeout: timeoutMs });
|
|
}
|
|
|
|
await execAsync(cmd, {
|
|
cwd: this.config.projectRoot,
|
|
timeout: timeoutMs,
|
|
});
|
|
}
|
|
|
|
private parseHealth(status: string): 'healthy' | 'unhealthy' | 'starting' | undefined {
|
|
if (!status) return undefined;
|
|
const lower = status.toLowerCase();
|
|
if (lower.includes('healthy')) return 'healthy';
|
|
if (lower.includes('unhealthy')) return 'unhealthy';
|
|
if (lower.includes('starting')) return 'starting';
|
|
return undefined;
|
|
}
|
|
|
|
private sleep(ms: number): Promise<void> {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
/**
|
|
* Find and remove orphaned feature containers that were started from
|
|
* feature-level compose files (codebase/features/<name>/docker-compose.yml)
|
|
* but not managed by the deployment compose. These use the old naming
|
|
* pattern ({feature}-{service}) without the lilith- prefix and can
|
|
* cause port conflicts with the deployment compose containers.
|
|
*/
|
|
async removeOrphanedFeatureContainers(): Promise<string[]> {
|
|
const featurePattern = /^(attributes|conversation-assistant|email|feature-flags|i18n|image-assistant|landing|marketplace|media|merchant|messaging|payments|platform-admin|profile|seo|sso|truth-validation)-(postgres|redis|minio|db|text-service)$/;
|
|
|
|
try {
|
|
const { stdout } = await execAsync(
|
|
`docker ps -a --format '{{.Names}}'`,
|
|
{ cwd: this.config.projectRoot },
|
|
);
|
|
|
|
const orphans = stdout
|
|
.trim()
|
|
.split('\n')
|
|
.filter(name => name && featurePattern.test(name));
|
|
|
|
if (orphans.length === 0) return [];
|
|
|
|
for (const name of orphans) {
|
|
try {
|
|
await execAsync(`docker rm -f ${name}`, { cwd: this.config.projectRoot });
|
|
this.logger.info(`Removed orphaned container: ${name}`);
|
|
} catch {
|
|
this.logger.warn(`Failed to remove orphaned container: ${name}`);
|
|
}
|
|
}
|
|
|
|
return orphans;
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get expected container service names for given profiles
|
|
* Uses `docker compose config --services` to list what should be running
|
|
*/
|
|
async getExpectedContainers(options: DockerComposeOptions = {}): Promise<Set<string>> {
|
|
const {
|
|
profiles = ['core', 'platform'],
|
|
envFile = this.config.envDev,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('config', '--services');
|
|
|
|
try {
|
|
const { stdout } = await execAsync(`docker compose ${args.join(' ')}`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
|
|
const services = stdout.trim().split('\n').filter(Boolean);
|
|
return new Set(services);
|
|
} catch {
|
|
return new Set();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop specific containers by name
|
|
*/
|
|
async stopContainers(
|
|
names: string[],
|
|
options: DockerComposeOptions = {}
|
|
): Promise<void> {
|
|
if (names.length === 0) return;
|
|
|
|
const {
|
|
envFile = this.config.envDev,
|
|
timeout = 5,
|
|
} = options;
|
|
|
|
// Use ALL_PROFILES to ensure we can reference any container
|
|
const args = this.buildComposeArgs(envFile, ALL_PROFILES);
|
|
args.push('stop', '-t', timeout.toString(), ...names);
|
|
|
|
await this.runCompose(args);
|
|
}
|
|
|
|
/**
|
|
* Get service names of currently running containers
|
|
* Returns docker-compose service names (not container names) to match getExpectedContainers
|
|
*/
|
|
async getRunningContainerNames(options: DockerComposeOptions = {}): Promise<Set<string>> {
|
|
const {
|
|
profiles = ALL_PROFILES,
|
|
envFile = this.config.envDev,
|
|
} = options;
|
|
|
|
const args = this.buildComposeArgs(envFile, profiles);
|
|
args.push('ps', '--format', 'json');
|
|
|
|
try {
|
|
const { stdout } = await execAsync(`docker compose ${args.join(' ')}`, {
|
|
cwd: this.config.projectRoot,
|
|
});
|
|
|
|
const services = new Set<string>();
|
|
|
|
// Parse JSON lines output - extract Service field (docker-compose service name)
|
|
for (const line of stdout.trim().split('\n')) {
|
|
if (!line) continue;
|
|
try {
|
|
const data = JSON.parse(line);
|
|
// Use Service field for docker-compose service name (matches config --services)
|
|
const serviceName = data.Service || data.service;
|
|
if (serviceName && data.State === 'running') {
|
|
services.add(serviceName);
|
|
}
|
|
} catch {
|
|
// Skip non-JSON lines
|
|
}
|
|
}
|
|
|
|
return services;
|
|
} catch {
|
|
return new Set();
|
|
}
|
|
}
|
|
}
|