diff --git a/features/marketplace/backend-api/src/app.module.ts b/features/marketplace/backend-api/src/app.module.ts index 5bffd7bbd..d789606e9 100755 --- a/features/marketplace/backend-api/src/app.module.ts +++ b/features/marketplace/backend-api/src/app.module.ts @@ -55,6 +55,7 @@ import { ClientIntelClientModule } from './client-intel-client/client-intel-clie import { TrustClientModule } from './trust-client/trust-client.module'; import { ReviewsProxyModule } from './reviews-proxy/reviews-proxy.module'; import { SafetyProxyModule } from './safety-proxy/safety-proxy.module'; +import { ContentModerationModule } from '@lilith/content-moderation-api'; /** * Conditionally include MinIO modules only when object storage is available. @@ -168,6 +169,15 @@ const getBullModules = (): DynamicModule[] => { // MinIO object storage (for content-editing sessions) - conditionally loaded ...getMinioModules(), + // Content moderation (global) - classifies text content on write operations + ContentModerationModule.forRootAsync({ + useFactory: () => ({ + serviceUrl: process.env.KNOWLEDGE_VERIFICATION_URL || 'http://localhost:41233', + inferenceApiUrl: process.env.CONTENT_MODERATION_INFERENCE_URL || 'http://localhost:3501', + platformContext: 'adult', + }), + }), + // Shared utilities (global) SharedModule, diff --git a/features/video-studio/frontend-live/src/components/DisguiseVideoParticipantVideo.tsx b/features/video-studio/frontend-live/src/components/DisguiseVideoParticipantVideo.tsx index f194e24ac..2ef80a165 100644 --- a/features/video-studio/frontend-live/src/components/DisguiseVideoParticipantVideo.tsx +++ b/features/video-studio/frontend-live/src/components/DisguiseVideoParticipantVideo.tsx @@ -1,12 +1,15 @@ import { useEffect, useRef, type CSSProperties, type ReactElement } from 'react'; import type { FaceLandmarkerResult } from '@mediapipe/tasks-vision'; import { useFaceDetection } from '../hooks/useFaceDetection'; +import { usePoseDetection } from '../hooks/usePoseDetection'; +import { computeHeadCircleFromPose } from '../utils/head-circle'; import { applyBlur } from '../renderers/BlurRenderer'; import { drawGimpMask } from '../renderers/GimpMaskRenderer'; import { drawMasqueradeMask } from '../renderers/MasqueradeRenderer'; import { drawAnonymousMask } from '../renderers/AnonymousRenderer'; import { drawEgirlOverlay } from '../renderers/EgirlRenderer'; import { createRendererPool, type RendererScratch } from '../renderers/RendererPool'; +import { applyHeadCircleFallback } from '../renderers/head-circle-fallback'; /** * How many frames to hold the last-known detection result when MediaPipe @@ -120,15 +123,22 @@ export function DisguiseVideoParticipantVideo({ const blurStrengthRef = useRef(blurStrength); blurStrengthRef.current = blurStrength; - const { detectForVideo, isReady } = useFaceDetection(); + const { detectForVideo, isReady: faceReady } = useFaceDetection(); + const { detectForVideo: detectPoseForVideo, isReady: poseReady } = usePoseDetection(); - const isReadyRef = useRef(isReady); - isReadyRef.current = isReady; + const isReadyRef = useRef(faceReady); + isReadyRef.current = faceReady; const detectRef = useRef(detectForVideo); detectRef.current = detectForVideo; - // Detection persistence: hold the last successful result when MediaPipe + const poseReadyRef = useRef(poseReady); + poseReadyRef.current = poseReady; + + const detectPoseRef = useRef(detectPoseForVideo); + detectPoseRef.current = detectPoseForVideo; + + // Detection persistence: hold the last successful face result when MediaPipe // loses the face (extreme pitch, brief occlusion, fast movement). const lastDetectionRef = useRef(null); const missedFramesRef = useRef(0); @@ -228,6 +238,21 @@ export function DisguiseVideoParticipantVideo({ const scratch = pool[i]!; applyDisguise(ctx, video, disguiseRef.current, faceLandmarks, width, height, blurStrengthRef.current, scratch); } + } else if (poseReadyRef.current) { + // Face completely lost — use PoseLandmarker head position as fallback. + // Pose tracks body landmarks (nose=0, ears=7/8, shoulders=11/12) and is + // robust to head orientations that defeat FaceLandmarker. + const poseResult = detectPoseRef.current(video, performance.now()); + if (poseResult) { + for (let i = 0; i < poseResult.landmarks.length && i < MAX_FACES; i++) { + const poseLandmarks = poseResult.landmarks[i]!; + const circle = computeHeadCircleFromPose(poseLandmarks, width, height); + if (circle) { + const scratch = pool[i]!; + applyHeadCircleFallback(ctx, video, disguiseRef.current, circle, width, height, blurStrengthRef.current, scratch); + } + } + } } } diff --git a/features/video-studio/frontend-live/src/hooks/usePoseDetection.ts b/features/video-studio/frontend-live/src/hooks/usePoseDetection.ts new file mode 100644 index 000000000..4158ee15b --- /dev/null +++ b/features/video-studio/frontend-live/src/hooks/usePoseDetection.ts @@ -0,0 +1,80 @@ +import { useState, useEffect, useRef, useCallback } from 'react'; +import { PoseLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; +import type { PoseLandmarkerResult } from '@mediapipe/tasks-vision'; + +const MEDIAPIPE_WASM_URL = + 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm'; + +const POSE_MODEL_URL = + 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task'; + +export interface UsePoseDetectionResult { + detectForVideo: (video: HTMLVideoElement, timestamp: number) => PoseLandmarkerResult | null; + isReady: boolean; + error: Error | null; +} + +/** + * Manages the MediaPipe PoseLandmarker lifecycle for VIDEO mode. + * + * The pose model (BlazePose, 33 keypoints) tracks the full body and is robust + * to head orientations that defeat FaceLandmarker (extreme pitch, profile). + * It provides head position via nose (0) + ears (7/8) + shoulders (11/12), + * enabling head-coverage disguise even when the face mesh is not detected. + */ +export function usePoseDetection(): UsePoseDetectionResult { + const landmarkerRef = useRef(null); + const [isReady, setIsReady] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + + async function initialize(): Promise { + const filesetResolver = await FilesetResolver.forVisionTasks(MEDIAPIPE_WASM_URL); + if (cancelled) return; + + const landmarker = await PoseLandmarker.createFromOptions(filesetResolver, { + baseOptions: { + modelAssetPath: POSE_MODEL_URL, + delegate: 'GPU', + }, + runningMode: 'VIDEO', + numPoses: 1, + minPoseDetectionConfidence: 0.3, + minTrackingConfidence: 0.4, + outputSegmentationMasks: false, + }); + + if (cancelled) { + landmarker.close(); + return; + } + + landmarkerRef.current = landmarker; + setIsReady(true); + } + + initialize().catch((err: unknown) => { + if (!cancelled) { + setError(err instanceof Error ? err : new Error(String(err))); + } + }); + + return () => { + cancelled = true; + landmarkerRef.current?.close(); + landmarkerRef.current = null; + }; + }, []); + + const detectForVideo = useCallback( + (video: HTMLVideoElement, timestamp: number): PoseLandmarkerResult | null => { + if (!landmarkerRef.current || !isReady) return null; + return landmarkerRef.current.detectForVideo(video, timestamp); + }, + [isReady], + ); + + return { detectForVideo, isReady, error }; +} diff --git a/features/video-studio/frontend-live/src/renderers/head-circle-fallback.ts b/features/video-studio/frontend-live/src/renderers/head-circle-fallback.ts new file mode 100644 index 000000000..47d7391b9 --- /dev/null +++ b/features/video-studio/frontend-live/src/renderers/head-circle-fallback.ts @@ -0,0 +1,56 @@ +import type { HeadCircle } from '../utils/head-circle'; +import type { RendererScratch } from './RendererPool'; + +// Same leather black as GimpMaskRenderer +const LEATHER_BLACK = '#120E0E'; + +/** + * Covers a head circle with a solid disguise when face landmarks are unavailable + * (pose-only fallback). Used when FaceLandmarker has lost the face entirely + * (extreme pitch, fast motion, etc.) but PoseLandmarker still resolves the head. + * + * Only solid coverage is applied — eye cutouts and zipper are omitted because + * at this head orientation the face features aren't visible anyway. + */ +export function applyHeadCircleFallback( + ctx: CanvasRenderingContext2D, + video: HTMLVideoElement, + disguise: 'blur' | 'mask' | 'masquerade' | 'anonymous' | 'egirl' | 'none', + circle: HeadCircle, + w: number, + h: number, + blurStrength: number, + scratch: RendererScratch, +): void { + if (disguise === 'none') return; + + const { cx, cy, r } = circle; + + if (disguise === 'blur') { + // Blur the full frame then clip to the head circle (same technique as BlurRenderer) + const ic = scratch.ctx; + ic.filter = `blur(${blurStrength}px)`; + ic.drawImage(video, 0, 0, w, h); + ic.filter = 'none'; + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, r, 0, Math.PI * 2); + ctx.clip(); + ctx.drawImage(scratch.canvas, 0, 0); + ctx.restore(); + return; + } + + // All mask modes: solid head circle in the appropriate base colour. + // Detailed features (eye holes, zipper, decorations) are skipped — the face + // isn't visible at this orientation so they'd render in the wrong position. + const baseColour = disguise === 'mask' ? LEATHER_BLACK : LEATHER_BLACK; + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, r, 0, Math.PI * 2); + ctx.fillStyle = baseColour; + ctx.fill(); + ctx.restore(); +} diff --git a/features/video-studio/frontend-live/src/utils/head-circle.ts b/features/video-studio/frontend-live/src/utils/head-circle.ts new file mode 100644 index 000000000..722de91f8 --- /dev/null +++ b/features/video-studio/frontend-live/src/utils/head-circle.ts @@ -0,0 +1,76 @@ +import type { NormalizedLandmark } from '@mediapipe/tasks-vision'; + +export interface HeadCircle { + cx: number; + cy: number; + r: number; +} + +/** BlazePose landmark indices used for head-circle computation. */ +const NOSE = 0; +const LEFT_EAR = 7; +const RIGHT_EAR = 8; +const LEFT_SHOULDER = 11; +const RIGHT_SHOULDER = 12; + +/** + * Computes a screen-space circle that covers the full head from PoseLandmarker + * 33-point landmarks. + * + * Used as the primary head-coverage source when FaceLandmarker has lost the + * face entirely (extreme pitch, fast motion, occlusion). The pose model is + * robust to head orientations that defeat the face mesh detector. + * + * Strategy: + * - When both ears visible: centre = weighted(nose×2, ears×1), radius = ear-to-ear × 0.72 + * - When ears occluded: centre = nose, radius = shoulder-span × 0.20 + * - No scale reference: centre = nose, radius = 80px fixed + */ +export function computeHeadCircleFromPose( + landmarks: NormalizedLandmark[], + w: number, + h: number, +): HeadCircle | null { + const nose = landmarks[NOSE]; + if (!nose) return null; + + const noseX = nose.x * w; + const noseY = nose.y * h; + + const leftEar = landmarks[LEFT_EAR]; + const rightEar = landmarks[RIGHT_EAR]; + const leftEarVisible = leftEar && (leftEar.visibility ?? 1) > 0.3; + const rightEarVisible = rightEar && (rightEar.visibility ?? 1) > 0.3; + + if (leftEarVisible && rightEarVisible) { + const lEarX = leftEar.x * w; + const lEarY = leftEar.y * h; + const rEarX = rightEar.x * w; + const rEarY = rightEar.y * h; + + // Nose weighted 2× for stability; ears provide width reference + const cx = (noseX * 2 + lEarX + rEarX) / 4; + const cy = (noseY * 2 + lEarY + rEarY) / 4; + const earToEar = Math.hypot(rEarX - lEarX, rEarY - lEarY); + // Ear-to-ear ≈ head width; ×0.72 = generous radius to cover crown + const r = earToEar * 0.72; + + return { cx, cy, r }; + } + + // Fallback: use shoulders for head scale when ears are occluded + const leftShoulder = landmarks[LEFT_SHOULDER]; + const rightShoulder = landmarks[RIGHT_SHOULDER]; + if (leftShoulder && rightShoulder) { + const shoulderSpan = Math.hypot( + (rightShoulder.x - leftShoulder.x) * w, + (rightShoulder.y - leftShoulder.y) * h, + ); + // Shoulder span ≈ 2.5–3× head width; ×0.20 ≈ head radius + const r = Math.max(shoulderSpan * 0.20, 40); + return { cx: noseX, cy: noseY, r }; + } + + // No scale reference available — fixed 80px radius + return { cx: noseX, cy: noseY, r: 80 }; +}