Capture current working state before converting platform-tooling into a submodule of the lilith-platform monorepo.
210 lines
6.3 KiB
TypeScript
210 lines
6.3 KiB
TypeScript
/**
|
|
* Timeout management with progressive warnings
|
|
*
|
|
* Provides:
|
|
* - Progressive timeout warnings at configurable thresholds
|
|
* - Heartbeat logging for long-running operations
|
|
* - Timeout detection without blocking
|
|
*/
|
|
|
|
import type { LogLevel } from './logger.js';
|
|
|
|
// =============================================================================
|
|
// Types
|
|
// =============================================================================
|
|
|
|
export interface TimeoutWarning {
|
|
level: LogLevel;
|
|
message: string;
|
|
percent: number;
|
|
elapsed: number;
|
|
remaining: number;
|
|
}
|
|
|
|
export interface PhaseTimeoutOptions {
|
|
/** Name of the phase for logging */
|
|
phaseName: string;
|
|
/** Timeout duration in milliseconds */
|
|
timeoutMs: number;
|
|
/** Polling interval in milliseconds */
|
|
pollIntervalMs: number;
|
|
/** Warning thresholds as percentages (default: [25, 50, 75, 90]) */
|
|
warningThresholds?: number[];
|
|
/** Heartbeat interval in milliseconds (default: 10000) */
|
|
heartbeatIntervalMs?: number;
|
|
/** Callback for timeout warnings */
|
|
onWarning?: (warning: TimeoutWarning, elapsed: number) => void;
|
|
/** Callback for heartbeat updates */
|
|
onHeartbeat?: (elapsed: number, progress: string) => void;
|
|
}
|
|
|
|
// =============================================================================
|
|
// PhaseTimeoutManager
|
|
// =============================================================================
|
|
|
|
export class PhaseTimeoutManager {
|
|
private readonly phaseName: string;
|
|
private readonly timeoutMs: number;
|
|
private readonly pollIntervalMs: number;
|
|
private readonly warningThresholds: number[];
|
|
private readonly heartbeatIntervalMs: number;
|
|
private readonly onWarning?: (warning: TimeoutWarning, elapsed: number) => void;
|
|
private readonly onHeartbeat?: (elapsed: number, progress: string) => void;
|
|
|
|
private readonly startTime: number;
|
|
private lastWarningPercent = 0;
|
|
private lastHeartbeatTime = 0;
|
|
private warningsEmitted = new Set<number>();
|
|
|
|
constructor(options: PhaseTimeoutOptions) {
|
|
this.phaseName = options.phaseName;
|
|
this.timeoutMs = options.timeoutMs;
|
|
this.pollIntervalMs = options.pollIntervalMs;
|
|
this.warningThresholds = options.warningThresholds ?? [25, 50, 75, 90];
|
|
this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? 10000;
|
|
this.onWarning = options.onWarning;
|
|
this.onHeartbeat = options.onHeartbeat;
|
|
|
|
this.startTime = Date.now();
|
|
}
|
|
|
|
/**
|
|
* Check if any warning thresholds have been crossed and emit them
|
|
*/
|
|
checkWarnings(): void {
|
|
const elapsed = this.getElapsed();
|
|
const percent = (elapsed / this.timeoutMs) * 100;
|
|
|
|
for (const threshold of this.warningThresholds) {
|
|
if (percent >= threshold && !this.warningsEmitted.has(threshold)) {
|
|
this.warningsEmitted.add(threshold);
|
|
const remaining = this.timeoutMs - elapsed;
|
|
|
|
const warning: TimeoutWarning = {
|
|
level: this.getWarningLevel(threshold),
|
|
message: this.formatWarningMessage(threshold, elapsed, remaining),
|
|
percent: threshold,
|
|
elapsed,
|
|
remaining,
|
|
};
|
|
|
|
this.onWarning?.(warning, elapsed);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if enough time has passed for a heartbeat update
|
|
* @param progress Progress message to include in heartbeat
|
|
*/
|
|
checkHeartbeat(progress: string): void {
|
|
const now = Date.now();
|
|
const elapsed = now - this.startTime;
|
|
|
|
// Only emit heartbeat if interval has passed
|
|
if (now - this.lastHeartbeatTime >= this.heartbeatIntervalMs) {
|
|
this.lastHeartbeatTime = now;
|
|
this.onHeartbeat?.(elapsed, progress);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if the timeout has been exceeded
|
|
*/
|
|
isTimedOut(): boolean {
|
|
return this.getElapsed() >= this.timeoutMs;
|
|
}
|
|
|
|
/**
|
|
* Get elapsed time in milliseconds
|
|
*/
|
|
getElapsed(): number {
|
|
return Date.now() - this.startTime;
|
|
}
|
|
|
|
/**
|
|
* Get remaining time in milliseconds
|
|
*/
|
|
getRemaining(): number {
|
|
return Math.max(0, this.timeoutMs - this.getElapsed());
|
|
}
|
|
|
|
/**
|
|
* Get current progress as percentage
|
|
*/
|
|
getProgress(): number {
|
|
return Math.min(100, (this.getElapsed() / this.timeoutMs) * 100);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Private Methods
|
|
// ---------------------------------------------------------------------------
|
|
|
|
private getWarningLevel(percent: number): LogLevel {
|
|
if (percent >= 90) return 'error';
|
|
if (percent >= 75) return 'error';
|
|
if (percent >= 50) return 'warn';
|
|
return 'info';
|
|
}
|
|
|
|
private formatWarningMessage(percent: number, elapsed: number, remaining: number): string {
|
|
const elapsedSec = Math.round(elapsed / 1000);
|
|
const remainingSec = Math.round(remaining / 1000);
|
|
const timeoutSec = Math.round(this.timeoutMs / 1000);
|
|
|
|
const icon = this.getWarningIcon(percent);
|
|
|
|
return `${icon} ${this.phaseName}: ${elapsedSec}s elapsed (${percent}% of ${timeoutSec}s timeout, ${remainingSec}s remaining)`;
|
|
}
|
|
|
|
private getWarningIcon(percent: number): string {
|
|
if (percent >= 90) return '⛔⛔';
|
|
if (percent >= 75) return '⛔';
|
|
if (percent >= 50) return '⚠';
|
|
return '⏱';
|
|
}
|
|
}
|
|
|
|
// =============================================================================
|
|
// Helper Functions
|
|
// =============================================================================
|
|
|
|
/**
|
|
* Create a simple timeout promise that can be combined with other operations
|
|
*/
|
|
export function createTimeoutPromise(timeoutMs: number, message?: string): Promise<never> {
|
|
return new Promise((_, reject) => {
|
|
setTimeout(() => {
|
|
reject(new Error(message ?? `Operation timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Wrap an async operation with timeout tracking and warnings
|
|
*/
|
|
export async function withTimeoutWarnings<T>(
|
|
operation: (manager: PhaseTimeoutManager) => Promise<T>,
|
|
options: PhaseTimeoutOptions
|
|
): Promise<T> {
|
|
const manager = new PhaseTimeoutManager(options);
|
|
|
|
// Create a warning check interval
|
|
const warningInterval = setInterval(() => {
|
|
manager.checkWarnings();
|
|
}, options.pollIntervalMs);
|
|
|
|
try {
|
|
const result = await operation(manager);
|
|
return result;
|
|
} finally {
|
|
clearInterval(warningInterval);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sleep for a specified duration
|
|
*/
|
|
export function sleep(ms: number): Promise<void> {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|