feat(content-moderation): ✨ Add threat score field and dynamic threat level calculation for automated risk assessment
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
parent
c165cb3cef
commit
410dddb792
1 changed files with 93 additions and 0 deletions
|
|
@ -0,0 +1,93 @@
|
|||
import {
|
||||
Entity,
|
||||
Column,
|
||||
PrimaryGeneratedColumn,
|
||||
CreateDateColumn,
|
||||
UpdateDateColumn,
|
||||
Index,
|
||||
} from 'typeorm';
|
||||
|
||||
import type { ThreatLevel } from '../types';
|
||||
|
||||
export interface UserRestrictions {
|
||||
suspended?: boolean;
|
||||
rateLimit?: number;
|
||||
queueForReview?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* UserThreatLevel Entity
|
||||
*
|
||||
* Maintains a rolling threat score for each user based on their moderation
|
||||
* violation history. The score decays over time and escalates through levels
|
||||
* as violations accumulate. One row per user — updated in place.
|
||||
*
|
||||
* Score range: 0–100 (starts at 100, deducted per violation)
|
||||
* Level thresholds: safe ≥70, caution ≥50, warning ≥30, danger ≥10, suspended <10
|
||||
*/
|
||||
@Entity('content_moderation_user_threat_levels')
|
||||
@Index('idx_utl_user_id', ['userId'], { unique: true })
|
||||
@Index('idx_utl_level', ['level'])
|
||||
@Index('idx_utl_score', ['score'])
|
||||
@Index('idx_utl_last_violation', ['lastViolationAt'])
|
||||
export class UserThreatLevel {
|
||||
@PrimaryGeneratedColumn('uuid')
|
||||
id!: string;
|
||||
|
||||
@Column({ type: 'uuid' })
|
||||
userId!: string;
|
||||
|
||||
@Column({ type: 'int', default: 100 })
|
||||
score!: number;
|
||||
|
||||
@Column({ type: 'varchar', length: 16, default: 'safe' })
|
||||
level!: ThreatLevel;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
totalViolations!: number;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
criticalViolations!: number;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
highViolations!: number;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
mediumViolations!: number;
|
||||
|
||||
@Column({ type: 'int', default: 0 })
|
||||
lowViolations!: number;
|
||||
|
||||
@Column({ type: 'jsonb', default: {} })
|
||||
categoryBreakdown!: Record<string, number>;
|
||||
|
||||
@Column({ type: 'timestamptz', nullable: true })
|
||||
lastViolationAt!: Date | null;
|
||||
|
||||
@Column({ type: 'timestamptz', nullable: true })
|
||||
lastEscalationAt!: Date | null;
|
||||
|
||||
@Column({ type: 'decimal', precision: 4, scale: 2, default: 1.0 })
|
||||
sensitivityMultiplier!: number;
|
||||
|
||||
@Column({ type: 'jsonb', default: {} })
|
||||
restrictions!: UserRestrictions;
|
||||
|
||||
@Column({ type: 'boolean', default: false })
|
||||
adminOverride!: boolean;
|
||||
|
||||
@Column({ type: 'uuid', nullable: true })
|
||||
adminOverrideBy!: string | null;
|
||||
|
||||
@Column({ type: 'timestamptz', nullable: true })
|
||||
adminOverrideAt!: Date | null;
|
||||
|
||||
@Column({ type: 'text', nullable: true })
|
||||
adminNotes!: string | null;
|
||||
|
||||
@CreateDateColumn({ type: 'timestamptz' })
|
||||
createdAt!: Date;
|
||||
|
||||
@UpdateDateColumn({ type: 'timestamptz' })
|
||||
updatedAt!: Date;
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue