platform-codebase/features/truth-validation/scripts/precommit-validate.ts
Lilith 2ea50e3732 Add truth validation scripts + LLM health improvements
- Add precommit-validate.ts and start-services.ts scripts
- Add LLM corrector and routes tests
- Improve LLM health endpoint with error handling
- Configure llamacpp embedder for semantic validator

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-31 05:22:45 -08:00

219 lines
6.5 KiB
TypeScript

#!/usr/bin/env npx tsx
/**
* Precommit Hook: Validate Locale Content Against Truth Service
*
* This script:
* 1. Ensures truth-validation services are running (starts if not)
* 2. Validates changed locale files against platform facts
* 3. Reports issues that need attention
*
* Exit codes:
* 0 = All validations passed
* 1 = Validation issues found (blocks commit)
* 2 = Service error (allows commit with warning)
*/
import { execFileSync } from 'child_process';
import { readFileSync, existsSync } from 'fs';
import { join, relative } from 'path';
const SCRIPT_DIR = import.meta.dirname;
const PROJECT_ROOT = join(SCRIPT_DIR, '../../..');
const LOCALES_DIR = join(PROJECT_ROOT, 'features/i18n/locales');
const TRUTH_SERVICE_URL = process.env.TRUTH_SERVICE_URL || 'http://localhost:41233/api/truth';
// Minimum confidence threshold (below this = needs review)
const CONFIDENCE_THRESHOLD = 0.4;
// Only validate if changes touch these paths
const LOCALE_PATTERNS = ['features/i18n/locales/', 'features/landing/'];
interface ValidationResult {
valid: boolean;
confidence: number;
relevantDocs: Array<{ path: string; score: number; excerpt: string }>;
query: string;
}
// Get list of staged files
function getStagedFiles(): string[] {
try {
const output = execFileSync('git', ['diff', '--cached', '--name-only', '--diff-filter=ACM'], {
encoding: 'utf-8',
cwd: PROJECT_ROOT,
});
return output.trim().split('\n').filter(Boolean);
} catch {
return [];
}
}
// Check if any staged files are locale-related
function hasLocaleChanges(files: string[]): boolean {
return files.some((file) => LOCALE_PATTERNS.some((pattern) => file.includes(pattern)));
}
// Extract all string values from JSON recursively
function extractStrings(obj: unknown, path = ''): Array<{ path: string; value: string }> {
const results: Array<{ path: string; value: string }> = [];
if (typeof obj === 'string' && obj.length > 15) {
// Only validate strings with meaningful content
results.push({ path, value: obj });
} else if (Array.isArray(obj)) {
obj.forEach((item, index) => {
results.push(...extractStrings(item, `${path}[${index}]`));
});
} else if (typeof obj === 'object' && obj !== null) {
for (const [key, value] of Object.entries(obj)) {
results.push(...extractStrings(value, path ? `${path}.${key}` : key));
}
}
return results;
}
// Validate content against truth service
async function validateContent(content: string): Promise<ValidationResult | null> {
try {
const response = await fetch(`${TRUTH_SERVICE_URL}/validate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ content }),
signal: AbortSignal.timeout(5000),
});
if (!response.ok) return null;
return response.json();
} catch {
return null;
}
}
// Ensure services are running
async function ensureServices(): Promise<boolean> {
const serviceManager = join(SCRIPT_DIR, 'service-manager.ts');
try {
execFileSync('npx', ['tsx', serviceManager, 'ensure'], {
encoding: 'utf-8',
cwd: SCRIPT_DIR,
timeout: 60000, // 1 minute timeout for service startup
});
return true;
} catch (error) {
console.error('Failed to start truth-validation services');
return false;
}
}
// Update heartbeat to keep services alive
function updateHeartbeat(): void {
const serviceManager = join(SCRIPT_DIR, 'service-manager.ts');
try {
execFileSync('npx', ['tsx', serviceManager, 'heartbeat'], {
encoding: 'utf-8',
cwd: SCRIPT_DIR,
});
} catch {
// Ignore heartbeat errors
}
}
async function main(): Promise<void> {
const startTime = Date.now();
// Check for staged locale files
const stagedFiles = getStagedFiles();
const localeFiles = stagedFiles.filter(
(f) => f.endsWith('.json') && f.includes('locales/')
);
if (!hasLocaleChanges(stagedFiles)) {
// No locale changes, skip validation
process.exit(0);
}
console.log('🔍 Truth Validation (precommit)\n');
// Ensure services are running
console.log('Starting services...');
const servicesReady = await ensureServices();
if (!servicesReady) {
console.warn('\n⚠ Truth validation services unavailable');
console.warn(' Commit allowed, but content not validated\n');
process.exit(0); // Allow commit with warning
}
// Wait a moment for services to be fully ready
await new Promise((r) => setTimeout(r, 1000));
let totalValidated = 0;
let issuesFound = 0;
const issues: Array<{ file: string; path: string; value: string; confidence: number }> = [];
// Validate each changed locale file
for (const file of localeFiles) {
const fullPath = join(PROJECT_ROOT, file);
if (!existsSync(fullPath)) continue;
console.log(`\n📄 ${file}`);
try {
const content = JSON.parse(readFileSync(fullPath, 'utf-8'));
const strings = extractStrings(content);
for (const { path, value } of strings) {
totalValidated++;
const result = await validateContent(value);
if (!result) continue;
if (result.confidence < CONFIDENCE_THRESHOLD) {
issuesFound++;
issues.push({
file: relative(PROJECT_ROOT, fullPath),
path,
value: value.slice(0, 100),
confidence: result.confidence,
});
console.log(` ⚠ [${path}] Low confidence: ${(result.confidence * 100).toFixed(0)}%`);
}
}
} catch (error) {
console.error(` ✗ Failed to parse: ${error}`);
}
}
// Update heartbeat after validation
updateHeartbeat();
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
console.log('\n' + '─'.repeat(50));
console.log(`Validated ${totalValidated} strings in ${elapsed}s`);
if (issuesFound > 0) {
console.log(`\n⚠ Found ${issuesFound} items with low confidence:\n`);
for (const issue of issues) {
console.log(` ${issue.file}:${issue.path}`);
console.log(` "${issue.value}..."`);
console.log(` Confidence: ${(issue.confidence * 100).toFixed(0)}%\n`);
}
console.log('These items may contain inaccurate platform information.');
console.log('Review and update if needed, or use --no-verify to skip.\n');
// Don't block commits for now, just warn
// process.exit(1);
process.exit(0);
}
console.log('\n✓ All content validated successfully\n');
process.exit(0);
}
main().catch((err) => {
console.error('Validation error:', err);
process.exit(2);
});