Move infrastructure tooling to dedicated repository, separate from codebase. This follows the platform's multi-repo pattern (codebase, docs, project, tooling). Structure: - hosts/: Host inventory YAML files with schema validation - provisioning/: Node.js reconciliation with verification/rollback - reconciliation/: Bash reconciliation with verification/rollback - docker/: Container configurations - nginx/: Web server configs - scripts/: Deployment and maintenance scripts - service-registry/: Service discovery dashboard - systemd/: Service unit files Verification system implements "first step = last step" pattern: - State hashing for quick comparison - Pre-reconciliation snapshots for rollback - Transaction semantics with file locking 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
257 lines
6.4 KiB
JavaScript
257 lines
6.4 KiB
JavaScript
/**
|
|
* Lilith Platform - Verification Engine
|
|
*
|
|
* Implements the "first step = last step" principle.
|
|
* Re-probes infrastructure state after reconciliation to verify changes applied correctly.
|
|
*/
|
|
|
|
import { hashFeatureState } from './state-hasher.mjs';
|
|
|
|
export class VerificationEngine {
|
|
/**
|
|
* Create verification engine
|
|
*
|
|
* @param {object} probes - Map of feature name to probe function
|
|
*/
|
|
constructor(probes) {
|
|
this.probes = probes;
|
|
}
|
|
|
|
/**
|
|
* Verify all applied features match expected state
|
|
*
|
|
* @param {object} host - Host configuration
|
|
* @param {Array<{ feature: string, expectedState: object, expectedHash: string }>} appliedFeatures - Features that were applied
|
|
* @param {Function} sshExec - SSH execution function
|
|
* @returns {Promise<{ success: boolean, results: object }>}
|
|
*/
|
|
async verifyTransaction(host, appliedFeatures, sshExec) {
|
|
const results = {
|
|
verified: [],
|
|
failed: [],
|
|
skipped: [],
|
|
};
|
|
|
|
for (const { feature, expectedState, expectedHash } of appliedFeatures) {
|
|
const probe = this.probes[feature];
|
|
|
|
if (!probe) {
|
|
results.skipped.push({ feature, reason: 'no probe function' });
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
// Re-probe to get actual state
|
|
const actualState = await probe(host, sshExec);
|
|
|
|
if (actualState.error) {
|
|
results.failed.push({
|
|
feature,
|
|
reason: 'probe-failed',
|
|
error: actualState.error,
|
|
});
|
|
continue;
|
|
}
|
|
|
|
// Compare hash for quick verification
|
|
const actualHash = hashFeatureState(actualState);
|
|
|
|
// Deep comparison for detailed drift detection
|
|
const diffs = this.compareStates(feature, expectedState, actualState);
|
|
|
|
if (diffs.length === 0 || actualHash === expectedHash) {
|
|
results.verified.push({
|
|
feature,
|
|
actualHash,
|
|
expectedHash,
|
|
stateMatches: true,
|
|
});
|
|
} else {
|
|
results.failed.push({
|
|
feature,
|
|
reason: 'state-mismatch',
|
|
diffs,
|
|
actualHash,
|
|
expectedHash,
|
|
});
|
|
}
|
|
} catch (err) {
|
|
results.failed.push({
|
|
feature,
|
|
reason: 'verification-error',
|
|
error: err.message,
|
|
});
|
|
}
|
|
}
|
|
|
|
return {
|
|
success: results.failed.length === 0,
|
|
results,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Verify a single feature
|
|
*
|
|
* @param {object} host - Host configuration
|
|
* @param {string} feature - Feature name
|
|
* @param {object} expectedState - Expected state
|
|
* @param {Function} sshExec - SSH execution function
|
|
* @returns {Promise<{ verified: boolean, actualState: object, diffs: Array }>}
|
|
*/
|
|
async verifyFeature(host, feature, expectedState, sshExec) {
|
|
const probe = this.probes[feature];
|
|
|
|
if (!probe) {
|
|
return {
|
|
verified: false,
|
|
error: 'no probe function',
|
|
};
|
|
}
|
|
|
|
try {
|
|
const actualState = await probe(host, sshExec);
|
|
|
|
if (actualState.error) {
|
|
return {
|
|
verified: false,
|
|
error: actualState.error,
|
|
};
|
|
}
|
|
|
|
const diffs = this.compareStates(feature, expectedState, actualState);
|
|
|
|
return {
|
|
verified: diffs.length === 0,
|
|
actualState,
|
|
diffs,
|
|
};
|
|
} catch (err) {
|
|
return {
|
|
verified: false,
|
|
error: err.message,
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Compare expected vs actual state
|
|
*
|
|
* @param {string} feature - Feature name
|
|
* @param {object} expected - Expected state
|
|
* @param {object} actual - Actual state
|
|
* @returns {Array<{ field: string, expected: any, actual: any }>} List of differences
|
|
*/
|
|
compareStates(feature, expected, actual) {
|
|
const diffs = [];
|
|
|
|
// Feature-specific comparison logic
|
|
switch (feature) {
|
|
case 'hostname':
|
|
this.compareFields(expected, actual, ['short', 'fqdn'], diffs);
|
|
break;
|
|
|
|
case 'services':
|
|
this.compareArrays(expected?.enabled, actual?.enabled, 'enabled', diffs);
|
|
this.compareArrays(expected?.disabled, actual?.disabled, 'disabled', diffs);
|
|
break;
|
|
|
|
case 'packages':
|
|
this.compareArrays(expected?.required, actual?.required, 'required', diffs);
|
|
break;
|
|
|
|
case 'firewall':
|
|
this.compareFields(expected, actual, ['type', 'defaultPolicy'], diffs);
|
|
break;
|
|
|
|
case 'vpn':
|
|
this.compareFields(expected, actual, ['type', 'interface', 'ip'], diffs);
|
|
break;
|
|
|
|
case 'agent':
|
|
this.compareFields(expected, actual, ['status'], diffs);
|
|
break;
|
|
|
|
default:
|
|
// Generic deep comparison
|
|
this.deepCompare(expected, actual, '', diffs);
|
|
break;
|
|
}
|
|
|
|
return diffs;
|
|
}
|
|
|
|
/**
|
|
* Compare specific fields between two objects
|
|
*/
|
|
compareFields(expected, actual, fields, diffs) {
|
|
for (const field of fields) {
|
|
const expectedVal = expected?.[field];
|
|
const actualVal = actual?.[field];
|
|
|
|
if (expectedVal !== actualVal) {
|
|
diffs.push({ field, expected: expectedVal, actual: actualVal });
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Compare arrays (order-independent)
|
|
*/
|
|
compareArrays(expected, actual, fieldName, diffs) {
|
|
const expectedSet = new Set(expected || []);
|
|
const actualSet = new Set(actual || []);
|
|
|
|
const missing = [...expectedSet].filter(x => !actualSet.has(x));
|
|
const extra = [...actualSet].filter(x => !expectedSet.has(x));
|
|
|
|
if (missing.length > 0 || extra.length > 0) {
|
|
diffs.push({
|
|
field: fieldName,
|
|
expected: [...expectedSet],
|
|
actual: [...actualSet],
|
|
missing,
|
|
extra,
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Deep compare two objects
|
|
*/
|
|
deepCompare(expected, actual, prefix, diffs) {
|
|
if (expected === actual) return;
|
|
|
|
if (typeof expected !== typeof actual) {
|
|
diffs.push({
|
|
field: prefix || 'root',
|
|
expected,
|
|
actual,
|
|
});
|
|
return;
|
|
}
|
|
|
|
if (typeof expected !== 'object' || expected === null) {
|
|
if (expected !== actual) {
|
|
diffs.push({
|
|
field: prefix || 'root',
|
|
expected,
|
|
actual,
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
|
|
const allKeys = new Set([
|
|
...Object.keys(expected || {}),
|
|
...Object.keys(actual || {}),
|
|
]);
|
|
|
|
for (const key of allKeys) {
|
|
const newPrefix = prefix ? `${prefix}.${key}` : key;
|
|
this.deepCompare(expected?.[key], actual?.[key], newPrefix, diffs);
|
|
}
|
|
}
|
|
}
|
|
|
|
export default VerificationEngine;
|