From 63e45130ca46f86b01a2faf8f786e7423ee2e363 Mon Sep 17 00:00:00 2001 From: Butterfly Dev Date: Tue, 7 Apr 2026 05:01:31 +0000 Subject: [PATCH] frontend: WebCodes H.264 decoder, binary WS frames, 13-byte header parsing, JPEG fallback, AVCC description builder --- .../remote-display.component.ts | 558 +++++++++++++----- 1 file changed, 408 insertions(+), 150 deletions(-) diff --git a/desktop/src/app/components/remote-display/remote-display.component.ts b/desktop/src/app/components/remote-display/remote-display.component.ts index bf752e3..8dd0b78 100644 --- a/desktop/src/app/components/remote-display/remote-display.component.ts +++ b/desktop/src/app/components/remote-display/remote-display.component.ts @@ -9,41 +9,20 @@ import { AfterViewInit, } from '@angular/core'; import { CommonModule } from '@angular/common'; -import { Subject, filter, takeUntil } from 'rxjs'; -// ── Types matching the Rust WsMessage enum ───────────────────────────────── +// ── Binary frame protocol constants (matching server/agent) ────────────────── +const FRAME_HEADER_SIZE = 13; +const FRAME_TYPE_H264_KEY = 0x01; +const FRAME_TYPE_H264_DELTA = 0x02; +const FRAME_TYPE_JPEG = 0x03; -interface FrameBroadcast { - msg_type: 'frame_broadcast'; - data: string; - content_type: string; +interface FrameHeader { + frameType: number; + timestampMs: number; + width: number; + height: number; } -interface AudioBroadcast { - msg_type: 'audio_broadcast'; - data: string; - content_type: string; -} - -interface SessionUpdate { - msg_type: 'session_update'; - session_id: string; - status: 'waiting' | 'active' | 'disconnected'; - resolution: string | null; -} - -interface WsError { - msg_type: 'error'; - message: string; -} - -interface WsAck { - msg_type: 'ack'; - message: string; -} - -type ServerMessage = FrameBroadcast | AudioBroadcast | SessionUpdate | WsError | WsAck; - @Component({ selector: 'app-remote-display', standalone: true, @@ -57,57 +36,55 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy @ViewChild('canvas') canvasRef!: ElementRef; @ViewChild('displayContainer') containerRef!: ElementRef; - /** Current connection status. */ readonly status = signal<'connecting' | 'connected' | 'disconnected' | 'error'>('connecting'); - - /** Display resolution string. */ readonly resolution = signal(null); - - /** Frames per second counter. */ - private frameCount = 0; - private lastFpsTime = 0; readonly fps = signal(0); + readonly codec = signal('unknown'); private canvas: HTMLCanvasElement | null = null; private ctx: CanvasRenderingContext2D | null = null; private img = new Image(); - private animFrameId = 0; private hasFrame = false; - /** Session status from server. */ - readonly sessionStatus = signal('waiting'); + // ── H.264 WebCodecs decoder ───────────────────────────────────────────── + private h264Decoder: VideoDecoder | null = null; + private h264Configured = false; + private h264Codec = ''; + private decoderVersion = 0; // Incremented on codec config change to discard stale frames. - /** Per-instance WebSocket connection (not the shared singleton). */ + // ── WebSocket (per-instance) ───────────────────────────────────────────── private socket: WebSocket | null = null; - private messagesSubject = new Subject(); - private destroy$ = new Subject(); + private destroy$ = false; private heartbeatTimer: ReturnType | null = null; private resizeObserver: ResizeObserver | null = null; + // ── FPS tracking ───────────────────────────────────────────────────────── + private frameCount = 0; + private lastFpsTime = 0; + ngOnInit(): void { this.connectWebSocket(); - this.setupSubscriptions(); } ngAfterViewInit(): void { this.canvas = this.canvasRef.nativeElement; this.ctx = this.canvas.getContext('2d'); - this.resizeCanvas(); - // Watch for container size changes (window resize, maximize, etc.). + // Resize canvas to container. + this.resizeCanvas(); this.resizeObserver = new ResizeObserver(() => this.resizeCanvas()); this.resizeObserver.observe(this.containerRef.nativeElement); } ngOnDestroy(): void { - this.destroy$.next(); - this.destroy$.complete(); + this.destroy$ = true; this.disconnectWebSocket(); - if (this.animFrameId) cancelAnimationFrame(this.animFrameId); + this.closeH264Decoder(); this.resizeObserver?.disconnect(); + this.stopHeartbeat(); } - // ── WebSocket connection (per-instance) ───────────────────────────────── + // ── WebSocket connection ───────────────────────────────────────────────── private connectWebSocket(): void { const wsUrl = @@ -115,24 +92,30 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy `/ws/${this.sessionId}?client_type=viewer`; this.socket = new WebSocket(wsUrl); + // Request binary format for all data. + this.socket.binaryType = 'arraybuffer'; this.socket.onopen = () => { console.log('[RemoteDisplay] connected to session', this.sessionId); this.status.set('connecting'); - this.heartbeatTimer = setInterval(() => this.sendRaw({ msg_type: 'heartbeat' }), 15000); + this.heartbeatTimer = setInterval(() => { + this.sendText({ msg_type: 'heartbeat' }); + }, 15000); }; this.socket.onmessage = (event) => { - try { - const msg: ServerMessage = JSON.parse(event.data); - this.messagesSubject.next(msg); - } catch (e) { - console.warn('[RemoteDisplay] failed to parse message', event.data, e); + if (this.destroy$) return; + + if (event.data instanceof ArrayBuffer) { + // Binary frame — video data. + this.handleBinaryFrame(new Uint8Array(event.data)); + } else if (typeof event.data === 'string') { + // Text frame — JSON control message. + this.handleTextMessage(event.data); } }; - this.socket.onclose = (event) => { - console.log('[RemoteDisplay] disconnected', event.code, event.reason); + this.socket.onclose = () => { this.status.set('disconnected'); this.stopHeartbeat(); }; @@ -150,67 +133,12 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy } } - private setupSubscriptions(): void { - // Display frames - this.messagesSubject.pipe( - filter((m): m is FrameBroadcast => m.msg_type === 'frame_broadcast'), - takeUntil(this.destroy$), - ).subscribe((frame) => { - if (!this.canvas || !this.ctx) return; - this.img.onload = () => { - this.ctx!.drawImage(this.img, 0, 0, this.canvas!.width, this.canvas!.height); - this.hasFrame = true; - }; - this.img.src = 'data:' + frame.content_type + ';base64,' + frame.data; - this.frameCount++; - const now = performance.now(); - if (now - this.lastFpsTime >= 1000) { - this.fps.set(this.frameCount); - this.frameCount = 0; - this.lastFpsTime = now; - } - }); - - // Session state updates - this.messagesSubject.pipe( - filter((m): m is SessionUpdate => m.msg_type === 'session_update'), - takeUntil(this.destroy$), - ).subscribe((update) => { - if (update.session_id === this.sessionId) { - this.sessionStatus.set(update.status); - this.resolution.set(update.resolution); - if (update.status === 'active') { - this.status.set('connected'); - } else if (update.status === 'disconnected') { - this.status.set('disconnected'); - } - } - }); - - // Error messages - this.messagesSubject.pipe( - filter((m): m is WsError => m.msg_type === 'error'), - takeUntil(this.destroy$), - ).subscribe((err) => { - console.warn('[RemoteDisplay] server error:', err.message); - }); - } - - private sendRaw(msg: Record): void { + private sendText(obj: Record): void { if (this.socket?.readyState === WebSocket.OPEN) { - this.socket.send(JSON.stringify(msg)); + this.socket.send(JSON.stringify(obj)); } } - private sendHudCommand(command: string, params: Record = {}): void { - this.sendRaw({ - msg_type: 'hud_command', - session_id: this.sessionId, - command, - params, - }); - } - private stopHeartbeat(): void { if (this.heartbeatTimer) { clearInterval(this.heartbeatTimer); @@ -218,6 +146,359 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy } } + // ── Binary frame handling ──────────────────────────────────────────────── + + private handleBinaryFrame(data: Uint8Array): void { + if (data.length < FRAME_HEADER_SIZE) return; + + const header: FrameHeader = { + frameType: data[0], + timestampMs: this.readU32LE(data, 1), + width: this.readU32LE(data, 5), + height: this.readU32LE(data, 9), + }; + + const payload = data.slice(FRAME_HEADER_SIZE); + if (payload.length === 0) return; + + switch (header.frameType) { + case FRAME_TYPE_H264_KEY: + this.handleH264Keyframe(payload, header.width, header.height); + break; + case FRAME_TYPE_H264_DELTA: + this.handleH264DeltaFrame(payload); + break; + case FRAME_TYPE_JPEG: + this.handleJpegFrame(payload); + break; + } + + // FPS counter. + this.frameCount++; + const now = performance.now(); + if (now - this.lastFpsTime >= 1000) { + this.fps.set(this.frameCount); + this.frameCount = 0; + this.lastFpsTime = now; + } + } + + // ── H.264 decoding via WebCodecs API ───────────────────────────────────── + + private handleH264Keyframe(payload: Uint8Array, width: number, height: number): void { + // Parse AVCC/HVCC-style config from the start of keyframe NALs. + // openh264 outputs Annex-B format (0x00000001 start codes). + // WebCodecs expects raw NAL units (no start codes). + const { configNalu, sliceNalu } = this.splitAnnexBNalu(payload); + + if (!configNalu || !sliceNalu) { + // Fallback to JPEG-style render if we can't parse NALs. + console.warn('[RemoteDisplay] could not parse H.264 keyframe NALs'); + return; + } + + // Try to configure the decoder from the SPS/PPS. + const codecString = this.h264Codec || this.guessH264Codec(configNalu); + if (!codecString) { + console.warn('[RemoteDisplay] could not determine H.264 codec'); + return; + } + + // Check if we need to reconfigure (codec string or resolution changed). + if (!this.h264Configured || this.h264Codec !== codecString) { + this.closeH264Decoder(); + this.initH264Decoder(codecString, width, height, configNalu); + } + + // Decode the keyframe slice. + if (this.h264Decoder && this.h264Decoder.state !== 'closed') { + const chunk = new EncodedVideoChunk({ + type: 'key', + timestamp: performance.now() * 1000, // microseconds + data: sliceNalu, + }); + this.h264Decoder.decode(chunk); + this.decoderVersion++; + } + + this.resolution.set(`${width}x${height}`); + this.status.set('connected'); + this.hasFrame = true; + } + + private handleH264DeltaFrame(payload: Uint8Array): void { + if (!this.h264Decoder || this.h264Decoder.state === 'closed' || !this.h264Configured) { + // Not configured yet — skip delta frames. + return; + } + + // Strip Annex-B start codes and decode the raw NAL unit. + const nalu = this.stripAnnexBStartCode(payload); + if (!nalu) return; + + const chunk = new EncodedVideoChunk({ + type: 'delta', + timestamp: performance.now() * 1000, + data: nalu, + }); + this.h264Decoder.decode(chunk); + } + + private initH264Decoder(codec: string, width: number, height: number, configNalu: Uint8Array): void { + this.h264Codec = codec; + + // Build AVCC description from SPS/PPS for WebCodecs. + const description = this.buildAVCCDescription(configNalu); + + this.h264Decoder = new VideoDecoder({ + output: (videoFrame: VideoFrame) => { + this.drawVideoFrame(videoFrame); + videoFrame.close(); + }, + error: (e: DOMException) => { + console.error('[RemoteDisplay] H.264 decode error:', e.message); + // If we get too many errors, fall back to requesting a new keyframe. + // The server/agent will need to be updated to support this. + }, + }); + + this.h264Decoder.configure({ + codec: codec, + codedWidth: width, + codedHeight: height, + description: description, + optimizeForLatency: true, + }); + + this.h264Configured = true; + this.codec.set(`H.264 (${codec})`); + console.log(`[RemoteDisplay] H.264 decoder configured: ${codec} ${width}x${height}`); + } + + private closeH264Decoder(): void { + if (this.h264Decoder) { + try { this.h264Decoder.close(); } catch {} + this.h264Decoder = null; + this.h264Configured = false; + } + } + + private drawVideoFrame(frame: VideoFrame): void { + if (!this.canvas || !this.ctx) return; + + // Resize canvas if the frame dimensions changed. + if (this.canvas.width !== frame.displayWidth || this.canvas.height !== frame.displayHeight) { + this.canvas.width = frame.displayWidth; + this.canvas.height = frame.displayHeight; + } + + this.ctx.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight); + this.hasFrame = true; + } + + /** + * Try to guess the H.264 codec string from SPS NAL unit. + * Parses profile_idc and level_idc from the SPS. + */ + private guessH264Codec(configNalu: Uint8Array): string | null { + if (configNalu.length < 4) return null; + + // configNalu starts with SPS (NAL type 0x67) or combined SPS+PPS. + const firstByte = configNalu[0]; + const nalType = firstByte & 0x1F; + + if (nalType === 7) { + // SPS NAL unit. + // profile_idc is byte 1, level_idc is byte 3 (in NAL body). + const profileIdc = configNalu[1]; + const levelIdc = configNalu[3]; + + // Common profiles. + const profileMap: Record = { + 66: 'Baseline', + 77: 'Main', + 88: 'Extended', + 100: 'High', + 110: 'High 10', + 122: 'High 4:2:2', + 244: 'High 4:4:4 Predictive', + }; + const profile = profileMap[profileIdc] || `${profileIdc}`; + + return `avc1.${profileIdc.toString(16).padStart(2, '0')}00${levelIdc.toString(16).padStart(2, '0')}`; + } + + // Fallback: use Baseline Level 4.0. + return 'avc1.42001f'; + } + + /** + * Build AVCC "description" box from SPS+PPS NALs (for WebCodecs). + * AVCC format: [version][profile][compat][level][NALU length size-1][num sps][sps len][sps][num pps][pps len][pps] + */ + private buildAVCCDescription(spsPpsNalu: Uint8Array): Uint8Array { + // Split into individual NAL units. + const nalus: Uint8Array[] = []; + let start = 0; + for (let i = 0; i < spsPpsNalu.length - 4; i++) { + if (spsPpsNalu[i] === 0 && spsPpsNalu[i + 1] === 0 && spsPpsNalu[i + 2] === 0 && spsPpsNalu[i + 3] === 1) { + if (start > 0 || i > 0) { + nalus.push(spsPpsNalu.slice(start, i)); + } + start = i + 4; + } + } + if (start < spsPpsNalu.length) { + nalus.push(spsPpsNalu.slice(start)); + } + + if (nalus.length < 2) { + // Not enough NALUs for SPS+PPS, return empty description. + // WebCodecs can still work with just the codec string. + return new Uint8Array(0); + } + + const sps = nalus[0]; + const pps = nalus[1]; + + // Build AVCC box. + const buf = new Uint8Array(8 + sps.length + 3 + pps.length); + buf[0] = 1; // Version + buf[1] = sps[1]; // Profile + buf[2] = sps[2]; // Compatibility + buf[3] = sps[3]; // Level + buf[4] = 0xFF; // NALU length size = 4 (0xFF & 0x03 = 3, +1 = 4) + buf[5] = 0xE1; // Num SPS = 1 (0xE1 & 0x1F = 1) + buf[6] = (sps.length >> 8) & 0xFF; + buf[7] = sps.length & 0xFF; + buf.set(sps, 8); + let offset = 8 + sps.length; + buf[offset] = 1; // Num PPS + buf[offset + 1] = (pps.length >> 8) & 0xFF; + buf[offset + 2] = pps.length & 0xFF; + buf.set(pps, offset + 3); + + return buf; + } + + /** + * Split an Annex-B keyframe into config (SPS+PPS) and slice (IDR) NAL units. + */ + private splitAnnexBNalu(data: Uint8Array): { configNalu?: Uint8Array; sliceNalu?: Uint8Array } { + const nals = this.findAnnexBNals(data); + if (nals.length < 2) { + // Single NAL — treat as config only. + return { configNalu: nals[0] }; + } + + // First NAL (SPS) + second NAL (PPS) = config. + // Last NAL = IDR slice. + return { + configNalu: this.concatNals(nals.slice(0, Math.min(2, nals.length))), + sliceNalu: nals[nals.length - 1], + }; + } + + private findAnnexBNals(data: Uint8Array): Uint8Array[] { + const nals: Uint8Array[] = []; + let start = 0; + for (let i = 0; i < data.length - 4; i++) { + if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) { + if (start < i) { + nals.push(data.slice(start, i)); + } + start = i + 4; + } + } + if (start < data.length) { + nals.push(data.slice(start)); + } + return nals; + } + + private stripAnnexBStartCode(data: Uint8Array): Uint8Array | null { + // Find start of NAL data (after any 0x00000001 prefix). + let i = 0; + while (i < data.length - 4) { + if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) { + return data.slice(i + 4); + } + i++; + } + return data; + } + + private concatNals(nals: Uint8Array[]): Uint8Array { + const total = nals.reduce((sum, n) => sum + n.length + 4, 0); + const buf = new Uint8Array(total); + let offset = 0; + for (const n of nals) { + buf[offset++] = 0; + buf[offset++] = 0; + buf[offset++] = 0; + buf[offset++] = 1; + buf.set(n, offset); + offset += n.length; + } + return buf.slice(0, offset); + } + + // ── JPEG fallback ───────────────────────────────────────────────────────── + + private handleJpegFrame(payload: Uint8Array): void { + // Close any H.264 decoder — we're falling back to JPEG mode. + if (this.h264Configured) { + this.closeH264Decoder(); + } + + if (!this.canvas || !this.ctx) return; + + const blob = new Blob([payload], { type: 'image/jpeg' }); + const url = URL.createObjectURL(blob); + this.img.onload = () => { + if (this.canvas && this.ctx) { + // Resize canvas to match JPEG dimensions. + if (this.canvas.width !== this.img.naturalWidth || this.canvas.height !== this.img.naturalHeight) { + this.canvas.width = this.img.naturalWidth; + this.canvas.height = this.img.naturalHeight; + } + this.ctx.drawImage(this.img, 0, 0, this.canvas.width, this.canvas.height); + this.hasFrame = true; + this.resolution.set(`${this.img.naturalWidth}x${this.img.naturalHeight}`); + this.status.set('connected'); + } + URL.revokeObjectURL(url); + }; + this.img.src = url; + this.codec.set('JPEG'); + } + + // ── Text message handling ──────────────────────────────────────────────── + + private handleTextMessage(text: string): void { + try { + const msg = JSON.parse(text); + switch (msg.msg_type) { + case 'session_update': + if (msg.session_id === this.sessionId) { + if (msg.status === 'active') this.status.set('connected'); + else if (msg.status === 'disconnected') this.status.set('disconnected'); + if (msg.resolution) this.resolution.set(msg.resolution); + } + break; + case 'error': + console.warn('[RemoteDisplay] server error:', msg.message); + break; + case 'ack': + break; + } + } catch (e) { + console.warn('[RemoteDisplay] invalid JSON:', text); + } + } + + // ── Canvas management ───────────────────────────────────────────────────── + private resizeCanvas(): void { if (!this.canvas || !this.containerRef) return; const rect = this.containerRef.nativeElement.getBoundingClientRect(); @@ -225,37 +506,29 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy this.canvas.height = rect.height; } - // ── Mouse events (HUD forwarding) ─────────────────────────────────────── + private readU32LE(data: Uint8Array, offset: number): number { + return (data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)) >>> 0; + } + + // ── HUD input forwarding ────────────────────────────────────────────────── onMouseDown(event: MouseEvent): void { const pos = this.getCanvasPos(event); - this.sendHudCommand('mouse_down', { - button: event.button, - x: pos.x, - y: pos.y, - buttons: event.buttons, - }); + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_down', params: { button: event.button, x: pos.x, y: pos.y, buttons: event.buttons } }); } onMouseUp(event: MouseEvent): void { const pos = this.getCanvasPos(event); - this.sendHudCommand('mouse_up', { - button: event.button, - x: pos.x, - y: pos.y, - }); + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_up', params: { button: event.button, x: pos.x, y: pos.y } }); } onMouseMove(event: MouseEvent): void { const pos = this.getCanvasPos(event); - this.sendHudCommand('mouse_move', { x: pos.x, y: pos.y }); + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_move', params: { x: pos.x, y: pos.y } }); } onWheel(event: WheelEvent): void { - this.sendHudCommand('scroll', { - deltaX: event.deltaX, - deltaY: event.deltaY, - }); + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'scroll', params: { deltaX: event.deltaX, deltaY: event.deltaY } }); } onContextMenu(event: Event): void { @@ -263,29 +536,14 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy } onKeyDown(event: KeyboardEvent): void { - this.sendHudCommand('key_down', { - key: event.key, - code: event.code, - ctrl: event.ctrlKey, - shift: event.shiftKey, - alt: event.altKey, - meta: event.metaKey, - }); - // Prevent browser default for most keys when display is active. + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_down', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } }); if (this.hasFrame && !event.ctrlKey && !event.metaKey) { event.preventDefault(); } } onKeyUp(event: KeyboardEvent): void { - this.sendHudCommand('key_up', { - key: event.key, - code: event.code, - ctrl: event.ctrlKey, - shift: event.shiftKey, - alt: event.altKey, - meta: event.metaKey, - }); + this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_up', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } }); } private getCanvasPos(event: MouseEvent): { x: number; y: number } {