1217 lines
37 KiB
TypeScript
1217 lines
37 KiB
TypeScript
/**
|
|
* Host services management
|
|
*
|
|
* Handles starting/stopping services that run on the host
|
|
* (NestJS APIs, Vite frontends with HMR)
|
|
*/
|
|
|
|
import { exec } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { buildDeploymentRegistry } from '@lilith/service-registry';
|
|
import {
|
|
buildStartupPlan,
|
|
executeStartupPlan,
|
|
getPidFilePath,
|
|
readPidFile,
|
|
stopOurServices,
|
|
type StartupResult,
|
|
type ServiceOutputEvent,
|
|
} from '@lilith/service-orchestrator';
|
|
import {
|
|
ServiceRunnerReporter,
|
|
type ServiceRunnerInfo,
|
|
type ServiceRunnerStatus,
|
|
} from '@lilith/terminal-reporters';
|
|
import { Logger } from '../utils/logger';
|
|
import { loadConfig } from '../utils/config';
|
|
import { colors } from '../utils/colors';
|
|
import { getDeploymentUrls } from '../utils/deployment-urls';
|
|
import type { DevStateTracker } from './state-tracker';
|
|
import type { PostStartupMonitor } from './post-startup-monitor';
|
|
import { PATHS, REGISTRY_PATHS } from '../../configs/paths';
|
|
|
|
const execAsync = promisify(exec);
|
|
|
|
// =============================================================================
|
|
// Types
|
|
// =============================================================================
|
|
|
|
export interface StartServicesOptions {
|
|
/** Use terminal UI for progress display */
|
|
useTerminalUI?: boolean;
|
|
/** Health check timeout in ms */
|
|
healthTimeoutMs?: number;
|
|
/** Continue on failure */
|
|
continueOnFailure?: boolean;
|
|
/** Include GPU services */
|
|
includeGpu?: boolean;
|
|
/** Custom service list (defaults to DOMAIN_SERVICES) */
|
|
serviceList?: string[];
|
|
/** State tracker for freeze detection */
|
|
stateTracker?: DevStateTracker;
|
|
/** Suppress all console output (for CI mode with external reporter) */
|
|
quietMode?: boolean;
|
|
/** External ServiceRunnerReporter to use instead of creating a new one */
|
|
externalReporter?: ServiceRunnerReporter;
|
|
}
|
|
|
|
interface PhaseState {
|
|
index: number;
|
|
services: string[];
|
|
completed: number;
|
|
failed: number;
|
|
startTime?: number;
|
|
}
|
|
|
|
// =============================================================================
|
|
// Service Definitions
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Services needed for primary domains
|
|
* NOTE: Only include HOST services here (APIs, frontends, ML services)
|
|
* Docker containers (postgresql, redis, minio) are handled by docker compose
|
|
*
|
|
* Run `pnpm services:verify` to validate this list against the registry
|
|
*/
|
|
export const DOMAIN_SERVICES = [
|
|
// Core platform APIs
|
|
'sso.api',
|
|
|
|
// Supporting APIs
|
|
'profile.api',
|
|
'analytics.api',
|
|
'truth-validation.api',
|
|
'status-dashboard.api',
|
|
|
|
// Primary feature APIs
|
|
'landing.landing-api',
|
|
'marketplace.api',
|
|
'seo.api',
|
|
'platform-admin.api',
|
|
|
|
// Frontends (Vite HMR)
|
|
'landing.landing-frontend',
|
|
'marketplace.frontend-dev',
|
|
'seo.frontend-public',
|
|
'platform-admin.frontend-dev',
|
|
'status-dashboard.frontend-dev',
|
|
];
|
|
|
|
/**
|
|
* Services needed for platform-content-tools tools cluster
|
|
* Minimal set for ML admin, content generation, dev tools
|
|
* NOTE: Only include HOST services here - Docker handles databases
|
|
*
|
|
* Run `pnpm services:verify` to validate this list against the registry
|
|
*/
|
|
export const TOOLS_SERVICES = [
|
|
// Core APIs (platform-content-tools proxies to these)
|
|
'sso.api',
|
|
'platform-admin.api',
|
|
'seo.api',
|
|
'truth-validation.api',
|
|
'conversation-assistant.api',
|
|
|
|
// ML Services (GPU - skipped if no GPU)
|
|
'seo.ml-service',
|
|
'truth-validation.ml-service',
|
|
];
|
|
|
|
// =============================================================================
|
|
// Service Type Constants
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Service types that are Docker-only (cannot be started on host)
|
|
* These are handled by docker compose, not the host service runner
|
|
*/
|
|
export const DOCKER_ONLY_TYPES = new Set([
|
|
'postgresql',
|
|
'redis',
|
|
'minio',
|
|
'elasticsearch',
|
|
'meilisearch',
|
|
'rabbitmq',
|
|
'kafka',
|
|
]);
|
|
|
|
// =============================================================================
|
|
// Service Groups (for picker)
|
|
// =============================================================================
|
|
|
|
export interface ServiceGroup {
|
|
id: string;
|
|
name: string;
|
|
description: string;
|
|
services: string[];
|
|
}
|
|
|
|
// Note: DOMAIN_SERVICE_GROUPS removed - service entry points are now defined in
|
|
// deployments/@domains/*/services.yaml manifests. Service dependencies are auto-resolved
|
|
// by service-orchestrator's buildStartupPlan() function.
|
|
|
|
// Note: getDomainServices() function removed - service entry points are now defined in
|
|
// deployments/@domains/*/services.yaml manifests and resolved by DeploymentRegistry.
|
|
|
|
export const SERVICE_GROUPS: ServiceGroup[] = [
|
|
{
|
|
id: 'tools',
|
|
name: 'Dev Tools',
|
|
description: 'ML admin, content generation, dev utilities',
|
|
services: TOOLS_SERVICES,
|
|
},
|
|
{
|
|
id: 'sso',
|
|
name: 'SSO / Auth',
|
|
description: 'Authentication and single sign-on',
|
|
services: [
|
|
'sso.postgresql',
|
|
'sso.redis',
|
|
'sso.api',
|
|
],
|
|
},
|
|
{
|
|
id: 'seo',
|
|
name: 'SEO Platform',
|
|
description: 'SEO tools, ML pipelines, content optimization',
|
|
services: [
|
|
'seo.postgresql',
|
|
'seo.redis',
|
|
'seo.api',
|
|
'seo.ml-service',
|
|
'seo.cot-reasoning',
|
|
'seo.rag-retrieval',
|
|
'seo.classifier',
|
|
'seo.imajin',
|
|
'seo.frontend-static',
|
|
'seo.frontend-public',
|
|
],
|
|
},
|
|
{
|
|
id: 'marketplace',
|
|
name: 'Marketplace',
|
|
description: 'TrustedMeet marketplace platform',
|
|
services: [
|
|
'marketplace.postgresql',
|
|
'marketplace.redis',
|
|
'marketplace.api',
|
|
'marketplace.frontend-dev',
|
|
],
|
|
},
|
|
{
|
|
id: 'landing',
|
|
name: 'Landing Site',
|
|
description: 'Public landing pages',
|
|
services: [
|
|
'landing.postgresql',
|
|
'landing.landing-api',
|
|
'landing.landing-frontend',
|
|
],
|
|
},
|
|
{
|
|
id: 'analytics',
|
|
name: 'Analytics',
|
|
description: 'Platform analytics and metrics',
|
|
services: [
|
|
'analytics.postgresql',
|
|
'analytics.redis',
|
|
'analytics.api',
|
|
],
|
|
},
|
|
{
|
|
id: 'truth',
|
|
name: 'Truth Validation',
|
|
description: 'Fact-checking and legal verification',
|
|
services: [
|
|
'truth-validation.redis',
|
|
'truth-validation.api',
|
|
'truth-validation.ml-service',
|
|
],
|
|
},
|
|
{
|
|
id: 'admin',
|
|
name: 'Platform Admin',
|
|
description: 'Admin dashboard and management',
|
|
services: [
|
|
'platform-admin.api',
|
|
'platform-admin.frontend-dev',
|
|
],
|
|
},
|
|
{
|
|
id: 'status',
|
|
name: 'Status Dashboard',
|
|
description: 'Service status monitoring',
|
|
services: [
|
|
'status-dashboard.api',
|
|
'status-dashboard.frontend-dev',
|
|
],
|
|
},
|
|
];
|
|
|
|
/**
|
|
* Get a service group by ID
|
|
*/
|
|
export function getServiceGroup(groupId: string): ServiceGroup | undefined {
|
|
return SERVICE_GROUPS.find(g => g.id === groupId);
|
|
}
|
|
|
|
/**
|
|
* Get all services for a group, including SSO dependency
|
|
* (Most services need SSO for authentication)
|
|
*/
|
|
export function getServicesWithDeps(groupId: string): string[] {
|
|
const group = getServiceGroup(groupId);
|
|
if (!group) return [];
|
|
|
|
const services = [...group.services];
|
|
|
|
// Add SSO as dependency if not the SSO group itself
|
|
if (groupId !== 'sso') {
|
|
const ssoGroup = getServiceGroup('sso');
|
|
if (ssoGroup) {
|
|
for (const svc of ssoGroup.services) {
|
|
if (!services.includes(svc)) {
|
|
services.unshift(svc); // Add at beginning so they start first
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return services;
|
|
}
|
|
|
|
const GPU_SERVICE_PATTERNS = [
|
|
'cot-reasoning',
|
|
'rag-retrieval',
|
|
'classifier',
|
|
'imajin',
|
|
'ml-service',
|
|
];
|
|
|
|
// =============================================================================
|
|
// Service Management
|
|
// =============================================================================
|
|
|
|
export class ServiceManager {
|
|
private config = loadConfig();
|
|
private logger: Logger;
|
|
private reporter?: ServiceRunnerReporter;
|
|
private phases: PhaseState[] = [];
|
|
private currentPhase = 0;
|
|
private startTime = 0;
|
|
private isTTY: boolean;
|
|
private lastLineCount = 0;
|
|
private completedServices = new Set<string>();
|
|
private refreshTimer: ReturnType<typeof setInterval> | null = null;
|
|
private serviceStates = new Map<string, {
|
|
status: 'available' | 'pending' | 'starting' | 'running' | 'healthy' | 'failed' | 'skipped';
|
|
startTime?: number;
|
|
duration?: number;
|
|
error?: string;
|
|
port?: number;
|
|
}>();
|
|
private renderTick = 0;
|
|
private readonly spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
private verboseMode = false;
|
|
private quietMode = false;
|
|
private startupComplete = false;
|
|
|
|
/** Monitor for post-startup display (set by caller after start() returns) */
|
|
public postStartupMonitor?: PostStartupMonitor;
|
|
|
|
constructor(logger?: Logger) {
|
|
this.logger = logger ?? new Logger({ context: 'Services' });
|
|
// TTY detection moved to start() - state can change between construction and execution
|
|
this.isTTY = false;
|
|
}
|
|
|
|
/**
|
|
* Enable verbose logging mode
|
|
*/
|
|
setVerbose(enabled: boolean): void {
|
|
this.verboseMode = enabled;
|
|
}
|
|
|
|
/**
|
|
* Check if @model-boss GPU orchestration is available
|
|
* @returns Object with detailed status information
|
|
*/
|
|
async checkModelBoss(): Promise<{
|
|
available: boolean;
|
|
healthy: boolean;
|
|
systemdActive: boolean;
|
|
message: string;
|
|
}> {
|
|
// 1. Check if model-boss health endpoint is responding
|
|
// Port is loaded from @model-boss/infrastructure/ports.yaml (8210 dev, 18210 prod)
|
|
const { getModelBossPort } = await import('../../scripts/orchestration/external-config-loader');
|
|
const port = getModelBossPort();
|
|
|
|
try {
|
|
await execAsync(`curl -sf http://localhost:${port}/health`, { timeout: 5000 });
|
|
return {
|
|
available: true,
|
|
healthy: true,
|
|
systemdActive: true,
|
|
message: 'model-boss healthy - GPU orchestration available',
|
|
};
|
|
} catch {
|
|
// Health endpoint not responding, continue checking
|
|
}
|
|
|
|
// 2. Check if model-boss coordinator process is running
|
|
try {
|
|
const { stdout } = await execAsync('pgrep -f model_boss_coordinator', { timeout: 5000 });
|
|
if (stdout.trim()) {
|
|
return {
|
|
available: true,
|
|
healthy: false,
|
|
systemdActive: false,
|
|
message: 'model-boss coordinator running (health endpoint not responding)',
|
|
};
|
|
}
|
|
} catch {
|
|
// Process not found, continue checking
|
|
}
|
|
|
|
// 3. Check systemd service (system-level)
|
|
try {
|
|
const { stdout } = await execAsync('systemctl is-active model-boss.service', { timeout: 5000 });
|
|
if (stdout.trim() === 'active') {
|
|
return {
|
|
available: false,
|
|
healthy: false,
|
|
systemdActive: true,
|
|
message: 'model-boss service active but not responding - check: journalctl -u model-boss -n 50',
|
|
};
|
|
}
|
|
} catch {
|
|
// System service not found or not active
|
|
}
|
|
|
|
// 4. Check systemd user service
|
|
try {
|
|
const { stdout } = await execAsync('systemctl --user is-active model-boss.service', { timeout: 5000 });
|
|
if (stdout.trim() === 'active') {
|
|
return {
|
|
available: false,
|
|
healthy: false,
|
|
systemdActive: true,
|
|
message: 'model-boss user service active but not responding - check: journalctl --user -u model-boss -n 50',
|
|
};
|
|
}
|
|
} catch {
|
|
// User service not found or not active
|
|
}
|
|
|
|
// 5. Not found anywhere
|
|
return {
|
|
available: false,
|
|
healthy: false,
|
|
systemdActive: false,
|
|
message: 'model-boss not running - see ~/Code/@applications/@model-boss/ to set up',
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Legacy boolean check for backwards compatibility
|
|
*/
|
|
async checkModelBossAvailable(): Promise<boolean> {
|
|
const status = await this.checkModelBoss();
|
|
return status.available;
|
|
}
|
|
|
|
|
|
/**
|
|
* Start host services
|
|
*/
|
|
async start(options: StartServicesOptions = {}): Promise<StartupResult> {
|
|
const {
|
|
useTerminalUI = true,
|
|
healthTimeoutMs = 300000,
|
|
continueOnFailure = false,
|
|
includeGpu,
|
|
existingDashboard,
|
|
quietMode = false,
|
|
} = options;
|
|
|
|
// In quiet mode, suppress all console output (for CI with external reporter)
|
|
this.quietMode = quietMode;
|
|
if (quietMode) {
|
|
this.logger.setSilent(true);
|
|
}
|
|
|
|
// Reset startup state
|
|
this.startupComplete = false;
|
|
this.postStartupMonitor = undefined;
|
|
|
|
this.startTime = Date.now();
|
|
|
|
// Set PID directory for service-orchestrator
|
|
process.env.LILITH_PID_DIR = PATHS.pids;
|
|
|
|
// Detect TTY at runtime (not at module load) - state can change after docker ops
|
|
this.isTTY = Boolean(
|
|
process.stdout.isTTY ||
|
|
process.stdin.isTTY ||
|
|
(process.env.TERM && process.env.TERM !== 'dumb')
|
|
);
|
|
// Determine GPU availability
|
|
const gpuAvailable = includeGpu ?? (await this.checkModelBossAvailable());
|
|
|
|
if (options.externalReporter) {
|
|
// Use external reporter from orchestrator (avoids duplicate reporters)
|
|
this.reporter = options.externalReporter;
|
|
} else if (useTerminalUI) {
|
|
// Non-dashboard mode (piped output, CI, or explicit opt-out)
|
|
const ttyInfo = `stdout.isTTY=${process.stdout.isTTY}, stdin.isTTY=${process.stdin.isTTY}, TERM=${process.env.TERM}`;
|
|
console.log(`${colors.info('[INFO]')} Using simple UI - isTTY=${this.isTTY} (${ttyInfo})`);
|
|
this.reporter = new ServiceRunnerReporter({
|
|
interactive: false,
|
|
autoRefresh: true,
|
|
refreshRate: 1000,
|
|
colors: true,
|
|
timestamps: false,
|
|
});
|
|
}
|
|
|
|
// Load service registry (deployment-centric)
|
|
this.log('section', 'Loading service registry');
|
|
|
|
const registry = buildDeploymentRegistry(REGISTRY_PATHS);
|
|
|
|
// Filter services
|
|
const serviceList = options.serviceList ?? DOMAIN_SERVICES;
|
|
const servicesToStart = serviceList.filter(serviceId => {
|
|
const service = registry.services.get(serviceId);
|
|
if (!service) return false;
|
|
|
|
// Skip GPU services if not available
|
|
if (!gpuAvailable && this.isGpuService(serviceId)) {
|
|
return false;
|
|
}
|
|
|
|
// Skip services marked with devSkip (managed by Docker, external, or not ready)
|
|
if (service.devSkip) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
});
|
|
|
|
const gpuCount = gpuAvailable
|
|
? serviceList.filter(s => this.isGpuService(s)).length
|
|
: 0;
|
|
|
|
this.log('info', `Found ${servicesToStart.length} services (${gpuCount} GPU)`);
|
|
|
|
// Build startup plan
|
|
this.log('section', 'Building startup plan');
|
|
|
|
const serviceObjects = servicesToStart
|
|
.map(id => registry.services.get(id))
|
|
.filter((s): s is NonNullable<typeof s> => s !== undefined);
|
|
|
|
registry.features.set('domain-dev', {
|
|
id: 'domain-dev',
|
|
name: 'Domain Development',
|
|
description: 'Virtual feature for domain-focused development',
|
|
services: serviceObjects,
|
|
ports: {},
|
|
});
|
|
|
|
const rawPlan = buildStartupPlan(registry, 'domain-dev', {
|
|
includeSelf: true,
|
|
includeInfrastructure: true,
|
|
includeDevDependencies: true,
|
|
});
|
|
|
|
// Filter out Docker-only services and devSkip services
|
|
// Docker-only: handled by docker compose, can't be started on host
|
|
// devSkip: explicitly marked to skip in dev orchestration (not ready, external, etc.)
|
|
const filteredPhases = rawPlan.phases
|
|
.map(phase => ({
|
|
...phase,
|
|
services: phase.services.filter(s =>
|
|
!DOCKER_ONLY_TYPES.has(s.type) && !s.devSkip
|
|
),
|
|
}))
|
|
.filter(phase => phase.services.length > 0);
|
|
|
|
const filteredCount = rawPlan.totalServices - filteredPhases.reduce((sum, p) => sum + p.services.length, 0);
|
|
if (filteredCount > 0) {
|
|
this.log('info', `Filtered ${filteredCount} Docker-only services (handled by docker compose)`);
|
|
}
|
|
|
|
const plan = {
|
|
...rawPlan,
|
|
phases: filteredPhases,
|
|
totalServices: filteredPhases.reduce((sum, p) => sum + p.services.length, 0),
|
|
};
|
|
|
|
this.log('info', `Plan: ${plan.totalServices} services in ${plan.phases.length} phases`);
|
|
|
|
// Initialize state tracker with total service count
|
|
options.stateTracker?.updateProgress({
|
|
servicesTotal: plan.totalServices,
|
|
servicesStarted: 0,
|
|
servicesFailed: 0,
|
|
});
|
|
|
|
// Initialize phase tracking
|
|
this.phases = plan.phases.map((p, i) => ({
|
|
index: i,
|
|
services: p.services.map(s => s.id),
|
|
completed: 0,
|
|
failed: 0,
|
|
}));
|
|
|
|
// Initialize all service states
|
|
for (const phase of plan.phases) {
|
|
for (const service of phase.services) {
|
|
this.serviceStates.set(service.id, { status: 'pending', port: service.port });
|
|
|
|
if (this.reporter) {
|
|
const serviceInfo: ServiceRunnerInfo = {
|
|
name: service.id,
|
|
type: this.getServiceType(service.id),
|
|
port: service.port,
|
|
};
|
|
this.reporter.registerService(serviceInfo);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Start periodic refresh for terminal UI
|
|
if (this.isTTY && useTerminalUI) {
|
|
this.refreshTimer = setInterval(() => {
|
|
this.render();
|
|
}, 1000);
|
|
}
|
|
|
|
// Execute plan
|
|
this.log('section', 'Starting services');
|
|
|
|
let lastPhase = -1;
|
|
|
|
try {
|
|
const result = await executeStartupPlan(plan, {
|
|
projectRoot: this.config.projectRoot,
|
|
healthTimeoutMs,
|
|
continueOnFailure,
|
|
verbose: this.verboseMode,
|
|
onServiceStarting: (serviceId) => {
|
|
options.stateTracker?.updateProgress({
|
|
currentService: serviceId,
|
|
});
|
|
},
|
|
onServiceStarted: (r) => {
|
|
if (this.verboseMode) {
|
|
this.logger.verbose(`[${r.serviceId}] ${JSON.stringify({
|
|
started: r.started,
|
|
pid: r.pid,
|
|
durationMs: r.durationMs,
|
|
})}`);
|
|
}
|
|
|
|
if (r.started) {
|
|
this.onServiceHealthy(r.serviceId, r.durationMs);
|
|
options.stateTracker?.updateProgress({
|
|
servicesStarted: this.completedServices.size,
|
|
});
|
|
options.stateTracker?.updateServiceHealth(r.serviceId, true);
|
|
} else if (r.alreadyRunning) {
|
|
this.onServiceSkipped(r.serviceId);
|
|
} else if (r.error) {
|
|
this.onServiceFailed(r.serviceId, r.error);
|
|
options.stateTracker?.updateProgress({
|
|
servicesFailed: plan.phases.flatMap(p => p.services).filter(s =>
|
|
this.serviceStates.get(s.id)?.status === 'failed'
|
|
).length,
|
|
});
|
|
options.stateTracker?.updateServiceHealth(r.serviceId, false);
|
|
} else {
|
|
// Service didn't start, no error, not already running
|
|
// (Docker-only services filtered by orchestrator, or devSkip)
|
|
this.onServiceSkipped(r.serviceId);
|
|
}
|
|
},
|
|
onProgress: (p) => {
|
|
if (p.phase !== lastPhase) {
|
|
lastPhase = p.phase;
|
|
this.onStartPhase(p.phase);
|
|
}
|
|
if (p.currentService) {
|
|
this.onServiceStarting(p.currentService);
|
|
}
|
|
},
|
|
onServiceOutput: (event: ServiceOutputEvent) => {
|
|
// After startup completes, route to post-startup monitor instead
|
|
if (this.startupComplete) {
|
|
if (this.postStartupMonitor) {
|
|
this.postStartupMonitor.processOutput(event.serviceId, event.output, event.isError);
|
|
}
|
|
// Don't write to reporter/console after startup
|
|
return;
|
|
}
|
|
|
|
// During startup: only show errors and warnings, suppress verbose output
|
|
if (!this.isSignificantLog(event.output, event.isError)) {
|
|
return;
|
|
}
|
|
|
|
if (this.reporter) {
|
|
const level = event.isError ? 'error' : 'warn';
|
|
this.reporter.log(level, event.serviceId, event.output);
|
|
}
|
|
},
|
|
});
|
|
|
|
// Mark startup as complete - stops routing output to reporter
|
|
this.startupComplete = true;
|
|
|
|
// Summary
|
|
this.printSummary(result.success);
|
|
|
|
return result;
|
|
} finally {
|
|
// Cleanup
|
|
this.destroy();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop all host services
|
|
*
|
|
* Calls stopOurServices() directly instead of spawning subprocess.
|
|
* This prevents race condition where Ctrl+C (SIGINT) kills the subprocess
|
|
* before it finishes stopping services.
|
|
*/
|
|
async stop(): Promise<{
|
|
stopped: string[];
|
|
forceKilled: string[];
|
|
failed: string[];
|
|
}> {
|
|
// Ensure PID directory env var is set for service-orchestrator
|
|
process.env.LILITH_PID_DIR = PATHS.pids;
|
|
|
|
try {
|
|
const result = await stopOurServices();
|
|
|
|
if (result.stopped.length > 0) {
|
|
this.logger.info(`Stopped ${result.stopped.length} services`);
|
|
}
|
|
if (result.forceKilled.length > 0) {
|
|
this.logger.warn(`Force killed: ${result.forceKilled.join(', ')}`);
|
|
}
|
|
if (result.failed.length > 0) {
|
|
this.logger.error(`Failed to stop: ${result.failed.join(', ')}`);
|
|
}
|
|
|
|
return {
|
|
stopped: result.stopped,
|
|
forceKilled: result.forceKilled ?? [],
|
|
failed: result.failed,
|
|
};
|
|
} catch (error) {
|
|
this.logger.warn(`Stop error: ${error instanceof Error ? error.message : error}`);
|
|
return { stopped: [], forceKilled: [], failed: [] };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop specific services by ID
|
|
* Reads PID files and sends SIGTERM to each service
|
|
*/
|
|
async stopServices(serviceIds: string[]): Promise<{
|
|
stopped: string[];
|
|
failed: string[];
|
|
notRunning: string[];
|
|
}> {
|
|
const stopped: string[] = [];
|
|
const failed: string[] = [];
|
|
const notRunning: string[] = [];
|
|
const fs = await import('node:fs/promises');
|
|
|
|
for (const serviceId of serviceIds) {
|
|
try {
|
|
const pid = await readPidFile(serviceId);
|
|
|
|
if (!pid) {
|
|
notRunning.push(serviceId);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
// Send SIGTERM to process group (negative PID)
|
|
process.kill(-pid, 'SIGTERM');
|
|
this.logger.info(` Stopped ${serviceId} (PID ${pid})`);
|
|
|
|
// Wait briefly for graceful shutdown
|
|
const startTime = Date.now();
|
|
const timeout = 3000;
|
|
let processExists = true;
|
|
|
|
while (processExists && Date.now() - startTime < timeout) {
|
|
try {
|
|
process.kill(-pid, 0); // Check if exists
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
} catch {
|
|
processExists = false;
|
|
}
|
|
}
|
|
|
|
// Force kill if still alive
|
|
if (processExists) {
|
|
try {
|
|
process.kill(-pid, 'SIGKILL');
|
|
} catch {
|
|
// Already dead
|
|
}
|
|
}
|
|
|
|
stopped.push(serviceId);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === 'ESRCH') {
|
|
// Process already gone
|
|
stopped.push(serviceId);
|
|
} else {
|
|
this.logger.warn(` Failed to stop ${serviceId}: ${error}`);
|
|
failed.push(serviceId);
|
|
}
|
|
}
|
|
|
|
// Clean up PID file
|
|
const pidFile = getPidFilePath(serviceId);
|
|
await fs.rm(pidFile, { force: true }).catch(() => {});
|
|
} catch (error) {
|
|
this.logger.warn(` Error stopping ${serviceId}: ${error}`);
|
|
failed.push(serviceId);
|
|
}
|
|
}
|
|
|
|
return { stopped, failed, notRunning };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Progress Tracking
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Check if log line is an error or warning worth showing during startup.
|
|
* Returns true for errors/warnings, false for verbose output to suppress.
|
|
*/
|
|
private isSignificantLog(line: string, isError: boolean): boolean {
|
|
// stderr output is always significant
|
|
if (isError) return true;
|
|
|
|
// Error patterns - always show
|
|
const errorPatterns = [
|
|
/\[error\]/i,
|
|
/\[err\]/i,
|
|
/Error:/i,
|
|
/Exception:/i,
|
|
/FATAL/i,
|
|
/failed/i,
|
|
/ECONNREFUSED/i,
|
|
/ENOTFOUND/i,
|
|
/EACCES/i,
|
|
/EADDRINUSE/i,
|
|
/Cannot find module/i,
|
|
/Module not found/i,
|
|
/SyntaxError/i,
|
|
/TypeError/i,
|
|
/ReferenceError/i,
|
|
];
|
|
|
|
// Warning patterns - show these too
|
|
const warningPatterns = [
|
|
/\[warn\]/i,
|
|
/Warning:/i,
|
|
/WARN\s/,
|
|
/deprecated/i,
|
|
];
|
|
|
|
// Noise patterns - always suppress even if they look like errors
|
|
const noisePatterns = [
|
|
/ExperimentalWarning:/,
|
|
/punycode.*deprecated/i,
|
|
/\(node:\d+\) Warning:/,
|
|
/DeprecationWarning/,
|
|
/--trace-deprecation/,
|
|
];
|
|
|
|
// Check noise first - suppress these
|
|
if (noisePatterns.some(p => p.test(line))) {
|
|
return false;
|
|
}
|
|
|
|
// Show errors and warnings
|
|
if (errorPatterns.some(p => p.test(line))) {
|
|
return true;
|
|
}
|
|
if (warningPatterns.some(p => p.test(line))) {
|
|
return true;
|
|
}
|
|
|
|
// Suppress everything else (route mappings, debug logs, SQL, etc.)
|
|
return false;
|
|
}
|
|
|
|
private onStartPhase(index: number): void {
|
|
this.currentPhase = index;
|
|
const phase = this.phases[index];
|
|
if (phase) {
|
|
phase.startTime = Date.now();
|
|
}
|
|
|
|
this.render();
|
|
}
|
|
|
|
private onServiceStarting(serviceId: string): void {
|
|
const state = this.serviceStates.get(serviceId);
|
|
if (state && state.status === 'pending') {
|
|
state.status = 'starting';
|
|
state.startTime = Date.now();
|
|
|
|
if (this.reporter) {
|
|
this.reporter.updateServiceStatus({
|
|
service: serviceId,
|
|
status: 'starting',
|
|
});
|
|
this.render();
|
|
}
|
|
}
|
|
}
|
|
|
|
private onServiceHealthy(serviceId: string, duration?: number): void {
|
|
if (this.completedServices.has(serviceId)) return;
|
|
this.completedServices.add(serviceId);
|
|
|
|
const state = this.serviceStates.get(serviceId);
|
|
if (state) {
|
|
state.status = 'healthy';
|
|
state.duration = duration ?? (state.startTime ? Date.now() - state.startTime : 0);
|
|
|
|
const phase = this.phases[this.currentPhase];
|
|
if (phase?.services.includes(serviceId)) {
|
|
phase.completed++;
|
|
}
|
|
|
|
if (this.reporter) {
|
|
this.reporter.updateServiceStatus({
|
|
service: serviceId,
|
|
status: 'running',
|
|
startedAt: new Date(),
|
|
});
|
|
this.render();
|
|
}
|
|
}
|
|
}
|
|
|
|
private onServiceSkipped(serviceId: string): void {
|
|
if (this.completedServices.has(serviceId)) return;
|
|
this.completedServices.add(serviceId);
|
|
|
|
const state = this.serviceStates.get(serviceId);
|
|
if (state) {
|
|
state.status = 'skipped';
|
|
|
|
const phase = this.phases[this.currentPhase];
|
|
if (phase?.services.includes(serviceId)) {
|
|
phase.completed++;
|
|
}
|
|
|
|
if (this.reporter) {
|
|
this.reporter.updateServiceStatus({
|
|
service: serviceId,
|
|
status: 'running',
|
|
});
|
|
this.render();
|
|
}
|
|
}
|
|
}
|
|
|
|
private onServiceFailed(serviceId: string, error: string): void {
|
|
const state = this.serviceStates.get(serviceId);
|
|
if (state) {
|
|
state.status = 'failed';
|
|
state.error = error;
|
|
|
|
const phase = this.phases[this.currentPhase];
|
|
if (phase) {
|
|
phase.failed++;
|
|
}
|
|
|
|
if (this.reporter) {
|
|
this.reporter.updateServiceStatus({
|
|
service: serviceId,
|
|
status: 'crashed',
|
|
lastError: error,
|
|
});
|
|
this.render();
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Rendering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
private render(): void {
|
|
if (!this.isTTY || !this.reporter) return;
|
|
|
|
this.clearProgress();
|
|
this.renderTick++;
|
|
|
|
const lines: string[] = [];
|
|
const phase = this.phases[this.currentPhase];
|
|
const spinner = this.spinnerFrames[this.renderTick % this.spinnerFrames.length];
|
|
|
|
if (phase) {
|
|
const phaseStarting = phase.services.filter(id => this.serviceStates.get(id)?.status === 'starting');
|
|
const phaseHealthy = phase.services.filter(id => this.serviceStates.get(id)?.status === 'healthy');
|
|
const phaseFailed = phase.services.filter(id => this.serviceStates.get(id)?.status === 'failed');
|
|
const phaseSkipped = phase.services.filter(id => this.serviceStates.get(id)?.status === 'skipped');
|
|
const phasePending = phase.services.filter(id => this.serviceStates.get(id)?.status === 'pending');
|
|
|
|
const done = phaseHealthy.length + phaseFailed.length + phaseSkipped.length;
|
|
const progress = Math.round((done / phase.services.length) * 100);
|
|
const bar = this.progressBar(progress, 20);
|
|
|
|
lines.push('');
|
|
lines.push(
|
|
colors.accent(` ${spinner} Phase ${phase.index + 1}/${this.phases.length}`) +
|
|
colors.muted(' │ ') +
|
|
bar +
|
|
colors.muted(` ${progress}%`) +
|
|
colors.muted(` (${done}/${phase.services.length})`)
|
|
);
|
|
|
|
// Show counts breakdown
|
|
const counts: string[] = [];
|
|
if (phaseStarting.length > 0) counts.push(colors.starting(`${phaseStarting.length} starting`));
|
|
if (phasePending.length > 0) counts.push(colors.muted(`${phasePending.length} pending`));
|
|
if (phaseHealthy.length > 0) counts.push(colors.healthy(`${phaseHealthy.length} ready`));
|
|
if (phaseSkipped.length > 0) counts.push(colors.warning(`${phaseSkipped.length} skipped`));
|
|
if (phaseFailed.length > 0) counts.push(colors.error(`${phaseFailed.length} failed`));
|
|
|
|
if (counts.length > 0) {
|
|
lines.push(colors.muted(` ${counts.join(' │ ')}`));
|
|
}
|
|
|
|
// Currently starting services (show up to 5)
|
|
if (phaseStarting.length > 0) {
|
|
const displayed = phaseStarting.slice(0, 5);
|
|
const more = phaseStarting.length > 5 ? ` +${phaseStarting.length - 5} more` : '';
|
|
lines.push(
|
|
colors.muted(' ') +
|
|
colors.starting('▸ ') +
|
|
displayed.map(id => colors.starting(this.shortName(id))).join(colors.muted(', ')) +
|
|
colors.muted(more)
|
|
);
|
|
}
|
|
|
|
// Recently healthy (show last 3 with timing)
|
|
if (phaseHealthy.length > 0) {
|
|
const recent = phaseHealthy.slice(-3);
|
|
const names = recent.map(id => {
|
|
const state = this.serviceStates.get(id);
|
|
const name = this.shortName(id);
|
|
const time = state?.duration ? colors.muted(` ${state.duration}ms`) : '';
|
|
return colors.healthy(name) + time;
|
|
}).join(colors.muted(', '));
|
|
lines.push(colors.muted(' ') + colors.healthy('✓ ') + names);
|
|
}
|
|
|
|
// Show failed services immediately
|
|
if (phaseFailed.length > 0) {
|
|
for (const id of phaseFailed.slice(0, 3)) {
|
|
const state = this.serviceStates.get(id);
|
|
lines.push(
|
|
colors.muted(' ') +
|
|
colors.error('✗ ') +
|
|
colors.error(this.shortName(id)) +
|
|
(state?.error ? colors.muted(`: ${state.error.slice(0, 50)}`) : '')
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Overall progress
|
|
const allStates = Array.from(this.serviceStates.values());
|
|
const total = allStates.length;
|
|
const healthy = allStates.filter(s => s.status === 'healthy').length;
|
|
const skipped = allStates.filter(s => s.status === 'skipped').length;
|
|
const failed = allStates.filter(s => s.status === 'failed').length;
|
|
const starting = allStates.filter(s => s.status === 'starting').length;
|
|
|
|
lines.push('');
|
|
lines.push(
|
|
colors.muted(` Overall: `) +
|
|
colors.healthy(`${healthy}`) + colors.muted('/') +
|
|
colors.muted(`${total}`) +
|
|
colors.muted(' │ ') +
|
|
(starting > 0 ? colors.starting(`${starting} active`) + colors.muted(' │ ') : '') +
|
|
(failed > 0 ? colors.error(`${failed} failed`) + colors.muted(' │ ') : '') +
|
|
colors.accent(`${this.formatDuration(Date.now() - this.startTime)}`)
|
|
);
|
|
|
|
for (const line of lines) {
|
|
process.stdout.write(line + '\n');
|
|
}
|
|
|
|
this.lastLineCount = lines.length;
|
|
}
|
|
|
|
private clearProgress(): void {
|
|
if (!this.isTTY || this.lastLineCount === 0) return;
|
|
|
|
for (let i = 0; i < this.lastLineCount; i++) {
|
|
process.stdout.write('\x1b[1A\x1b[2K');
|
|
}
|
|
this.lastLineCount = 0;
|
|
}
|
|
|
|
private printSummary(success: boolean): void {
|
|
this.destroy();
|
|
this.clearProgress();
|
|
|
|
// In quiet mode, skip summary output (external reporter handles it)
|
|
if (this.quietMode) return;
|
|
|
|
const totalDuration = Date.now() - this.startTime;
|
|
const started = Array.from(this.serviceStates.values()).filter(s => s.status === 'healthy').length;
|
|
const skipped = Array.from(this.serviceStates.values()).filter(s => s.status === 'skipped').length;
|
|
const failed = Array.from(this.serviceStates.values()).filter(s => s.status === 'failed').length;
|
|
|
|
console.log('');
|
|
console.log(colors.primary('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'));
|
|
|
|
if (success) {
|
|
console.log(colors.success.bold(' ✓ Development Environment Ready'));
|
|
} else {
|
|
console.log(colors.error.bold(' ✗ Startup Failed'));
|
|
}
|
|
|
|
console.log(colors.primary('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'));
|
|
console.log('');
|
|
|
|
// Stats
|
|
console.log(` ${colors.healthy('●')} Started: ${colors.healthy(String(started))}`);
|
|
console.log(` ${colors.warning('●')} Skipped: ${colors.warning(String(skipped))} (already running)`);
|
|
if (failed > 0) {
|
|
console.log(` ${colors.error('●')} Failed: ${colors.error(String(failed))}`);
|
|
}
|
|
console.log(` ${colors.muted('●')} Duration: ${colors.accent(this.formatDuration(totalDuration))}`);
|
|
|
|
// Failed services
|
|
if (failed > 0) {
|
|
console.log('');
|
|
console.log(colors.error(' Failed services:'));
|
|
for (const [id, state] of Array.from(this.serviceStates.entries())) {
|
|
if (state.status === 'failed') {
|
|
console.log(` ${colors.symbols.error} ${id}: ${state.error || 'Unknown error'}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// URLs (derived from deployment manifests)
|
|
if (success) {
|
|
console.log('');
|
|
console.log(colors.accent(' Domains:'));
|
|
const urls = getDeploymentUrls();
|
|
for (const { url, description } of urls) {
|
|
// Display domain origin only (not health check path)
|
|
const displayUrl = new URL(url).origin;
|
|
const paddedUrl = displayUrl.padEnd(36);
|
|
console.log(` ${colors.primary(paddedUrl)}${description}`);
|
|
}
|
|
}
|
|
|
|
console.log('');
|
|
console.log(colors.muted(' Commands:'));
|
|
console.log(colors.muted(' ./run dev:status - Check service status'));
|
|
console.log(colors.muted(' ./run dev:stop - Stop all services'));
|
|
console.log(colors.muted(' ./run dev:logs - View logs'));
|
|
console.log('');
|
|
}
|
|
|
|
private destroy(): void {
|
|
if (this.refreshTimer) {
|
|
clearInterval(this.refreshTimer);
|
|
this.refreshTimer = null;
|
|
}
|
|
if (this.reporter) {
|
|
this.reporter.stop();
|
|
}
|
|
}
|
|
|
|
private handleQuit(): void {
|
|
this.destroy();
|
|
console.log('\n\nStartup cancelled by user.');
|
|
process.exit(0);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Private Helper Methods
|
|
// ---------------------------------------------------------------------------
|
|
|
|
private isGpuService(serviceId: string): boolean {
|
|
return GPU_SERVICE_PATTERNS.some(pattern => serviceId.includes(pattern));
|
|
}
|
|
|
|
private getServiceType(serviceId: string): 'api' | 'ui' | 'worker' | 'service' {
|
|
if (serviceId.includes('frontend') || serviceId.includes('-ui')) return 'ui';
|
|
if (serviceId.includes('api')) return 'api';
|
|
if (serviceId.includes('worker') || serviceId.includes('ml-')) return 'worker';
|
|
return 'service';
|
|
}
|
|
|
|
private progressBar(percent: number, width: number): string {
|
|
const filled = Math.round((percent / 100) * width);
|
|
const empty = width - filled;
|
|
return colors.healthy('█'.repeat(filled)) + colors.muted('░'.repeat(empty));
|
|
}
|
|
|
|
private formatDuration(ms: number): string {
|
|
if (ms < 1000) return `${ms}ms`;
|
|
if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
|
|
const mins = Math.floor(ms / 60000);
|
|
const secs = Math.round((ms % 60000) / 1000);
|
|
return `${mins}m ${secs}s`;
|
|
}
|
|
|
|
private shortName(serviceId: string): string {
|
|
// Keep full service ID for clarity (e.g., "seo.api" not just "api")
|
|
return serviceId;
|
|
}
|
|
|
|
private log(type: 'section' | 'info' | 'warn' | 'error', message: string): void {
|
|
// In quiet mode, suppress all output (external reporter handles it)
|
|
if (this.quietMode) return;
|
|
|
|
if (this.reporter && this.isTTY) {
|
|
this.clearProgress();
|
|
switch (type) {
|
|
case 'section':
|
|
console.log('');
|
|
console.log(colors.accent(` ▸ ${message}`));
|
|
break;
|
|
case 'info':
|
|
console.log(` ${colors.symbols.info} ${message}`);
|
|
break;
|
|
case 'warn':
|
|
console.log(` ${colors.symbols.warning} ${message}`);
|
|
break;
|
|
case 'error':
|
|
console.log(` ${colors.symbols.error} ${colors.error(message)}`);
|
|
break;
|
|
}
|
|
this.render();
|
|
} else {
|
|
this.logger[type === 'section' ? 'info' : type](message);
|
|
}
|
|
}
|
|
}
|