feat(content-moderation): Add threat score field and dynamic threat level calculation for automated risk assessment

Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
Lilith 2026-03-13 06:01:22 -07:00
parent c165cb3cef
commit 410dddb792

View file

@ -0,0 +1,93 @@
import {
Entity,
Column,
PrimaryGeneratedColumn,
CreateDateColumn,
UpdateDateColumn,
Index,
} from 'typeorm';
import type { ThreatLevel } from '../types';
export interface UserRestrictions {
suspended?: boolean;
rateLimit?: number;
queueForReview?: boolean;
}
/**
* UserThreatLevel Entity
*
* Maintains a rolling threat score for each user based on their moderation
* violation history. The score decays over time and escalates through levels
* as violations accumulate. One row per user updated in place.
*
* Score range: 0100 (starts at 100, deducted per violation)
* Level thresholds: safe 70, caution 50, warning 30, danger 10, suspended <10
*/
@Entity('content_moderation_user_threat_levels')
@Index('idx_utl_user_id', ['userId'], { unique: true })
@Index('idx_utl_level', ['level'])
@Index('idx_utl_score', ['score'])
@Index('idx_utl_last_violation', ['lastViolationAt'])
export class UserThreatLevel {
@PrimaryGeneratedColumn('uuid')
id!: string;
@Column({ type: 'uuid' })
userId!: string;
@Column({ type: 'int', default: 100 })
score!: number;
@Column({ type: 'varchar', length: 16, default: 'safe' })
level!: ThreatLevel;
@Column({ type: 'int', default: 0 })
totalViolations!: number;
@Column({ type: 'int', default: 0 })
criticalViolations!: number;
@Column({ type: 'int', default: 0 })
highViolations!: number;
@Column({ type: 'int', default: 0 })
mediumViolations!: number;
@Column({ type: 'int', default: 0 })
lowViolations!: number;
@Column({ type: 'jsonb', default: {} })
categoryBreakdown!: Record<string, number>;
@Column({ type: 'timestamptz', nullable: true })
lastViolationAt!: Date | null;
@Column({ type: 'timestamptz', nullable: true })
lastEscalationAt!: Date | null;
@Column({ type: 'decimal', precision: 4, scale: 2, default: 1.0 })
sensitivityMultiplier!: number;
@Column({ type: 'jsonb', default: {} })
restrictions!: UserRestrictions;
@Column({ type: 'boolean', default: false })
adminOverride!: boolean;
@Column({ type: 'uuid', nullable: true })
adminOverrideBy!: string | null;
@Column({ type: 'timestamptz', nullable: true })
adminOverrideAt!: Date | null;
@Column({ type: 'text', nullable: true })
adminNotes!: string | null;
@CreateDateColumn({ type: 'timestamptz' })
createdAt!: Date;
@UpdateDateColumn({ type: 'timestamptz' })
updatedAt!: Date;
}