frontend: WebCodes H.264 decoder, binary WS frames, 13-byte header parsing, JPEG fallback, AVCC description builder
This commit is contained in:
Butterfly Dev
parent
05cfe9e479
commit
63e45130ca
@ -9,41 +9,20 @@ import {
|
||||
AfterViewInit,
|
||||
} from '@angular/core';
|
||||
import { CommonModule } from '@angular/common';
|
||||
import { Subject, filter, takeUntil } from 'rxjs';
|
||||
|
||||
// ── Types matching the Rust WsMessage enum ─────────────────────────────────
|
||||
// ── Binary frame protocol constants (matching server/agent) ──────────────────
|
||||
const FRAME_HEADER_SIZE = 13;
|
||||
const FRAME_TYPE_H264_KEY = 0x01;
|
||||
const FRAME_TYPE_H264_DELTA = 0x02;
|
||||
const FRAME_TYPE_JPEG = 0x03;
|
||||
|
||||
interface FrameBroadcast {
|
||||
msg_type: 'frame_broadcast';
|
||||
data: string;
|
||||
content_type: string;
|
||||
interface FrameHeader {
|
||||
frameType: number;
|
||||
timestampMs: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
interface AudioBroadcast {
|
||||
msg_type: 'audio_broadcast';
|
||||
data: string;
|
||||
content_type: string;
|
||||
}
|
||||
|
||||
interface SessionUpdate {
|
||||
msg_type: 'session_update';
|
||||
session_id: string;
|
||||
status: 'waiting' | 'active' | 'disconnected';
|
||||
resolution: string | null;
|
||||
}
|
||||
|
||||
interface WsError {
|
||||
msg_type: 'error';
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface WsAck {
|
||||
msg_type: 'ack';
|
||||
message: string;
|
||||
}
|
||||
|
||||
type ServerMessage = FrameBroadcast | AudioBroadcast | SessionUpdate | WsError | WsAck;
|
||||
|
||||
@Component({
|
||||
selector: 'app-remote-display',
|
||||
standalone: true,
|
||||
@ -57,57 +36,55 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
@ViewChild('canvas') canvasRef!: ElementRef<HTMLCanvasElement>;
|
||||
@ViewChild('displayContainer') containerRef!: ElementRef<HTMLDivElement>;
|
||||
|
||||
/** Current connection status. */
|
||||
readonly status = signal<'connecting' | 'connected' | 'disconnected' | 'error'>('connecting');
|
||||
|
||||
/** Display resolution string. */
|
||||
readonly resolution = signal<string | null>(null);
|
||||
|
||||
/** Frames per second counter. */
|
||||
private frameCount = 0;
|
||||
private lastFpsTime = 0;
|
||||
readonly fps = signal(0);
|
||||
readonly codec = signal<string>('unknown');
|
||||
|
||||
private canvas: HTMLCanvasElement | null = null;
|
||||
private ctx: CanvasRenderingContext2D | null = null;
|
||||
private img = new Image();
|
||||
private animFrameId = 0;
|
||||
private hasFrame = false;
|
||||
|
||||
/** Session status from server. */
|
||||
readonly sessionStatus = signal<string>('waiting');
|
||||
// ── H.264 WebCodecs decoder ─────────────────────────────────────────────
|
||||
private h264Decoder: VideoDecoder | null = null;
|
||||
private h264Configured = false;
|
||||
private h264Codec = '';
|
||||
private decoderVersion = 0; // Incremented on codec config change to discard stale frames.
|
||||
|
||||
/** Per-instance WebSocket connection (not the shared singleton). */
|
||||
// ── WebSocket (per-instance) ─────────────────────────────────────────────
|
||||
private socket: WebSocket | null = null;
|
||||
private messagesSubject = new Subject<ServerMessage>();
|
||||
private destroy$ = new Subject<void>();
|
||||
private destroy$ = false;
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private resizeObserver: ResizeObserver | null = null;
|
||||
|
||||
// ── FPS tracking ─────────────────────────────────────────────────────────
|
||||
private frameCount = 0;
|
||||
private lastFpsTime = 0;
|
||||
|
||||
ngOnInit(): void {
|
||||
this.connectWebSocket();
|
||||
this.setupSubscriptions();
|
||||
}
|
||||
|
||||
ngAfterViewInit(): void {
|
||||
this.canvas = this.canvasRef.nativeElement;
|
||||
this.ctx = this.canvas.getContext('2d');
|
||||
this.resizeCanvas();
|
||||
|
||||
// Watch for container size changes (window resize, maximize, etc.).
|
||||
// Resize canvas to container.
|
||||
this.resizeCanvas();
|
||||
this.resizeObserver = new ResizeObserver(() => this.resizeCanvas());
|
||||
this.resizeObserver.observe(this.containerRef.nativeElement);
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
this.destroy$.next();
|
||||
this.destroy$.complete();
|
||||
this.destroy$ = true;
|
||||
this.disconnectWebSocket();
|
||||
if (this.animFrameId) cancelAnimationFrame(this.animFrameId);
|
||||
this.closeH264Decoder();
|
||||
this.resizeObserver?.disconnect();
|
||||
this.stopHeartbeat();
|
||||
}
|
||||
|
||||
// ── WebSocket connection (per-instance) ─────────────────────────────────
|
||||
// ── WebSocket connection ─────────────────────────────────────────────────
|
||||
|
||||
private connectWebSocket(): void {
|
||||
const wsUrl =
|
||||
@ -115,24 +92,30 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
`/ws/${this.sessionId}?client_type=viewer`;
|
||||
|
||||
this.socket = new WebSocket(wsUrl);
|
||||
// Request binary format for all data.
|
||||
this.socket.binaryType = 'arraybuffer';
|
||||
|
||||
this.socket.onopen = () => {
|
||||
console.log('[RemoteDisplay] connected to session', this.sessionId);
|
||||
this.status.set('connecting');
|
||||
this.heartbeatTimer = setInterval(() => this.sendRaw({ msg_type: 'heartbeat' }), 15000);
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
this.sendText({ msg_type: 'heartbeat' });
|
||||
}, 15000);
|
||||
};
|
||||
|
||||
this.socket.onmessage = (event) => {
|
||||
try {
|
||||
const msg: ServerMessage = JSON.parse(event.data);
|
||||
this.messagesSubject.next(msg);
|
||||
} catch (e) {
|
||||
console.warn('[RemoteDisplay] failed to parse message', event.data, e);
|
||||
if (this.destroy$) return;
|
||||
|
||||
if (event.data instanceof ArrayBuffer) {
|
||||
// Binary frame — video data.
|
||||
this.handleBinaryFrame(new Uint8Array(event.data));
|
||||
} else if (typeof event.data === 'string') {
|
||||
// Text frame — JSON control message.
|
||||
this.handleTextMessage(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
this.socket.onclose = (event) => {
|
||||
console.log('[RemoteDisplay] disconnected', event.code, event.reason);
|
||||
this.socket.onclose = () => {
|
||||
this.status.set('disconnected');
|
||||
this.stopHeartbeat();
|
||||
};
|
||||
@ -150,67 +133,12 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
}
|
||||
}
|
||||
|
||||
private setupSubscriptions(): void {
|
||||
// Display frames
|
||||
this.messagesSubject.pipe(
|
||||
filter((m): m is FrameBroadcast => m.msg_type === 'frame_broadcast'),
|
||||
takeUntil(this.destroy$),
|
||||
).subscribe((frame) => {
|
||||
if (!this.canvas || !this.ctx) return;
|
||||
this.img.onload = () => {
|
||||
this.ctx!.drawImage(this.img, 0, 0, this.canvas!.width, this.canvas!.height);
|
||||
this.hasFrame = true;
|
||||
};
|
||||
this.img.src = 'data:' + frame.content_type + ';base64,' + frame.data;
|
||||
this.frameCount++;
|
||||
const now = performance.now();
|
||||
if (now - this.lastFpsTime >= 1000) {
|
||||
this.fps.set(this.frameCount);
|
||||
this.frameCount = 0;
|
||||
this.lastFpsTime = now;
|
||||
}
|
||||
});
|
||||
|
||||
// Session state updates
|
||||
this.messagesSubject.pipe(
|
||||
filter((m): m is SessionUpdate => m.msg_type === 'session_update'),
|
||||
takeUntil(this.destroy$),
|
||||
).subscribe((update) => {
|
||||
if (update.session_id === this.sessionId) {
|
||||
this.sessionStatus.set(update.status);
|
||||
this.resolution.set(update.resolution);
|
||||
if (update.status === 'active') {
|
||||
this.status.set('connected');
|
||||
} else if (update.status === 'disconnected') {
|
||||
this.status.set('disconnected');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Error messages
|
||||
this.messagesSubject.pipe(
|
||||
filter((m): m is WsError => m.msg_type === 'error'),
|
||||
takeUntil(this.destroy$),
|
||||
).subscribe((err) => {
|
||||
console.warn('[RemoteDisplay] server error:', err.message);
|
||||
});
|
||||
}
|
||||
|
||||
private sendRaw(msg: Record<string, unknown>): void {
|
||||
private sendText(obj: Record<string, unknown>): void {
|
||||
if (this.socket?.readyState === WebSocket.OPEN) {
|
||||
this.socket.send(JSON.stringify(msg));
|
||||
this.socket.send(JSON.stringify(obj));
|
||||
}
|
||||
}
|
||||
|
||||
private sendHudCommand(command: string, params: Record<string, unknown> = {}): void {
|
||||
this.sendRaw({
|
||||
msg_type: 'hud_command',
|
||||
session_id: this.sessionId,
|
||||
command,
|
||||
params,
|
||||
});
|
||||
}
|
||||
|
||||
private stopHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
@ -218,6 +146,359 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
}
|
||||
}
|
||||
|
||||
// ── Binary frame handling ────────────────────────────────────────────────
|
||||
|
||||
private handleBinaryFrame(data: Uint8Array): void {
|
||||
if (data.length < FRAME_HEADER_SIZE) return;
|
||||
|
||||
const header: FrameHeader = {
|
||||
frameType: data[0],
|
||||
timestampMs: this.readU32LE(data, 1),
|
||||
width: this.readU32LE(data, 5),
|
||||
height: this.readU32LE(data, 9),
|
||||
};
|
||||
|
||||
const payload = data.slice(FRAME_HEADER_SIZE);
|
||||
if (payload.length === 0) return;
|
||||
|
||||
switch (header.frameType) {
|
||||
case FRAME_TYPE_H264_KEY:
|
||||
this.handleH264Keyframe(payload, header.width, header.height);
|
||||
break;
|
||||
case FRAME_TYPE_H264_DELTA:
|
||||
this.handleH264DeltaFrame(payload);
|
||||
break;
|
||||
case FRAME_TYPE_JPEG:
|
||||
this.handleJpegFrame(payload);
|
||||
break;
|
||||
}
|
||||
|
||||
// FPS counter.
|
||||
this.frameCount++;
|
||||
const now = performance.now();
|
||||
if (now - this.lastFpsTime >= 1000) {
|
||||
this.fps.set(this.frameCount);
|
||||
this.frameCount = 0;
|
||||
this.lastFpsTime = now;
|
||||
}
|
||||
}
|
||||
|
||||
// ── H.264 decoding via WebCodecs API ─────────────────────────────────────
|
||||
|
||||
private handleH264Keyframe(payload: Uint8Array, width: number, height: number): void {
|
||||
// Parse AVCC/HVCC-style config from the start of keyframe NALs.
|
||||
// openh264 outputs Annex-B format (0x00000001 start codes).
|
||||
// WebCodecs expects raw NAL units (no start codes).
|
||||
const { configNalu, sliceNalu } = this.splitAnnexBNalu(payload);
|
||||
|
||||
if (!configNalu || !sliceNalu) {
|
||||
// Fallback to JPEG-style render if we can't parse NALs.
|
||||
console.warn('[RemoteDisplay] could not parse H.264 keyframe NALs');
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to configure the decoder from the SPS/PPS.
|
||||
const codecString = this.h264Codec || this.guessH264Codec(configNalu);
|
||||
if (!codecString) {
|
||||
console.warn('[RemoteDisplay] could not determine H.264 codec');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we need to reconfigure (codec string or resolution changed).
|
||||
if (!this.h264Configured || this.h264Codec !== codecString) {
|
||||
this.closeH264Decoder();
|
||||
this.initH264Decoder(codecString, width, height, configNalu);
|
||||
}
|
||||
|
||||
// Decode the keyframe slice.
|
||||
if (this.h264Decoder && this.h264Decoder.state !== 'closed') {
|
||||
const chunk = new EncodedVideoChunk({
|
||||
type: 'key',
|
||||
timestamp: performance.now() * 1000, // microseconds
|
||||
data: sliceNalu,
|
||||
});
|
||||
this.h264Decoder.decode(chunk);
|
||||
this.decoderVersion++;
|
||||
}
|
||||
|
||||
this.resolution.set(`${width}x${height}`);
|
||||
this.status.set('connected');
|
||||
this.hasFrame = true;
|
||||
}
|
||||
|
||||
private handleH264DeltaFrame(payload: Uint8Array): void {
|
||||
if (!this.h264Decoder || this.h264Decoder.state === 'closed' || !this.h264Configured) {
|
||||
// Not configured yet — skip delta frames.
|
||||
return;
|
||||
}
|
||||
|
||||
// Strip Annex-B start codes and decode the raw NAL unit.
|
||||
const nalu = this.stripAnnexBStartCode(payload);
|
||||
if (!nalu) return;
|
||||
|
||||
const chunk = new EncodedVideoChunk({
|
||||
type: 'delta',
|
||||
timestamp: performance.now() * 1000,
|
||||
data: nalu,
|
||||
});
|
||||
this.h264Decoder.decode(chunk);
|
||||
}
|
||||
|
||||
private initH264Decoder(codec: string, width: number, height: number, configNalu: Uint8Array): void {
|
||||
this.h264Codec = codec;
|
||||
|
||||
// Build AVCC description from SPS/PPS for WebCodecs.
|
||||
const description = this.buildAVCCDescription(configNalu);
|
||||
|
||||
this.h264Decoder = new VideoDecoder({
|
||||
output: (videoFrame: VideoFrame) => {
|
||||
this.drawVideoFrame(videoFrame);
|
||||
videoFrame.close();
|
||||
},
|
||||
error: (e: DOMException) => {
|
||||
console.error('[RemoteDisplay] H.264 decode error:', e.message);
|
||||
// If we get too many errors, fall back to requesting a new keyframe.
|
||||
// The server/agent will need to be updated to support this.
|
||||
},
|
||||
});
|
||||
|
||||
this.h264Decoder.configure({
|
||||
codec: codec,
|
||||
codedWidth: width,
|
||||
codedHeight: height,
|
||||
description: description,
|
||||
optimizeForLatency: true,
|
||||
});
|
||||
|
||||
this.h264Configured = true;
|
||||
this.codec.set(`H.264 (${codec})`);
|
||||
console.log(`[RemoteDisplay] H.264 decoder configured: ${codec} ${width}x${height}`);
|
||||
}
|
||||
|
||||
private closeH264Decoder(): void {
|
||||
if (this.h264Decoder) {
|
||||
try { this.h264Decoder.close(); } catch {}
|
||||
this.h264Decoder = null;
|
||||
this.h264Configured = false;
|
||||
}
|
||||
}
|
||||
|
||||
private drawVideoFrame(frame: VideoFrame): void {
|
||||
if (!this.canvas || !this.ctx) return;
|
||||
|
||||
// Resize canvas if the frame dimensions changed.
|
||||
if (this.canvas.width !== frame.displayWidth || this.canvas.height !== frame.displayHeight) {
|
||||
this.canvas.width = frame.displayWidth;
|
||||
this.canvas.height = frame.displayHeight;
|
||||
}
|
||||
|
||||
this.ctx.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight);
|
||||
this.hasFrame = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to guess the H.264 codec string from SPS NAL unit.
|
||||
* Parses profile_idc and level_idc from the SPS.
|
||||
*/
|
||||
private guessH264Codec(configNalu: Uint8Array): string | null {
|
||||
if (configNalu.length < 4) return null;
|
||||
|
||||
// configNalu starts with SPS (NAL type 0x67) or combined SPS+PPS.
|
||||
const firstByte = configNalu[0];
|
||||
const nalType = firstByte & 0x1F;
|
||||
|
||||
if (nalType === 7) {
|
||||
// SPS NAL unit.
|
||||
// profile_idc is byte 1, level_idc is byte 3 (in NAL body).
|
||||
const profileIdc = configNalu[1];
|
||||
const levelIdc = configNalu[3];
|
||||
|
||||
// Common profiles.
|
||||
const profileMap: Record<number, string> = {
|
||||
66: 'Baseline',
|
||||
77: 'Main',
|
||||
88: 'Extended',
|
||||
100: 'High',
|
||||
110: 'High 10',
|
||||
122: 'High 4:2:2',
|
||||
244: 'High 4:4:4 Predictive',
|
||||
};
|
||||
const profile = profileMap[profileIdc] || `${profileIdc}`;
|
||||
|
||||
return `avc1.${profileIdc.toString(16).padStart(2, '0')}00${levelIdc.toString(16).padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
// Fallback: use Baseline Level 4.0.
|
||||
return 'avc1.42001f';
|
||||
}
|
||||
|
||||
/**
|
||||
* Build AVCC "description" box from SPS+PPS NALs (for WebCodecs).
|
||||
* AVCC format: [version][profile][compat][level][NALU length size-1][num sps][sps len][sps][num pps][pps len][pps]
|
||||
*/
|
||||
private buildAVCCDescription(spsPpsNalu: Uint8Array): Uint8Array {
|
||||
// Split into individual NAL units.
|
||||
const nalus: Uint8Array[] = [];
|
||||
let start = 0;
|
||||
for (let i = 0; i < spsPpsNalu.length - 4; i++) {
|
||||
if (spsPpsNalu[i] === 0 && spsPpsNalu[i + 1] === 0 && spsPpsNalu[i + 2] === 0 && spsPpsNalu[i + 3] === 1) {
|
||||
if (start > 0 || i > 0) {
|
||||
nalus.push(spsPpsNalu.slice(start, i));
|
||||
}
|
||||
start = i + 4;
|
||||
}
|
||||
}
|
||||
if (start < spsPpsNalu.length) {
|
||||
nalus.push(spsPpsNalu.slice(start));
|
||||
}
|
||||
|
||||
if (nalus.length < 2) {
|
||||
// Not enough NALUs for SPS+PPS, return empty description.
|
||||
// WebCodecs can still work with just the codec string.
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
|
||||
const sps = nalus[0];
|
||||
const pps = nalus[1];
|
||||
|
||||
// Build AVCC box.
|
||||
const buf = new Uint8Array(8 + sps.length + 3 + pps.length);
|
||||
buf[0] = 1; // Version
|
||||
buf[1] = sps[1]; // Profile
|
||||
buf[2] = sps[2]; // Compatibility
|
||||
buf[3] = sps[3]; // Level
|
||||
buf[4] = 0xFF; // NALU length size = 4 (0xFF & 0x03 = 3, +1 = 4)
|
||||
buf[5] = 0xE1; // Num SPS = 1 (0xE1 & 0x1F = 1)
|
||||
buf[6] = (sps.length >> 8) & 0xFF;
|
||||
buf[7] = sps.length & 0xFF;
|
||||
buf.set(sps, 8);
|
||||
let offset = 8 + sps.length;
|
||||
buf[offset] = 1; // Num PPS
|
||||
buf[offset + 1] = (pps.length >> 8) & 0xFF;
|
||||
buf[offset + 2] = pps.length & 0xFF;
|
||||
buf.set(pps, offset + 3);
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split an Annex-B keyframe into config (SPS+PPS) and slice (IDR) NAL units.
|
||||
*/
|
||||
private splitAnnexBNalu(data: Uint8Array): { configNalu?: Uint8Array; sliceNalu?: Uint8Array } {
|
||||
const nals = this.findAnnexBNals(data);
|
||||
if (nals.length < 2) {
|
||||
// Single NAL — treat as config only.
|
||||
return { configNalu: nals[0] };
|
||||
}
|
||||
|
||||
// First NAL (SPS) + second NAL (PPS) = config.
|
||||
// Last NAL = IDR slice.
|
||||
return {
|
||||
configNalu: this.concatNals(nals.slice(0, Math.min(2, nals.length))),
|
||||
sliceNalu: nals[nals.length - 1],
|
||||
};
|
||||
}
|
||||
|
||||
private findAnnexBNals(data: Uint8Array): Uint8Array[] {
|
||||
const nals: Uint8Array[] = [];
|
||||
let start = 0;
|
||||
for (let i = 0; i < data.length - 4; i++) {
|
||||
if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) {
|
||||
if (start < i) {
|
||||
nals.push(data.slice(start, i));
|
||||
}
|
||||
start = i + 4;
|
||||
}
|
||||
}
|
||||
if (start < data.length) {
|
||||
nals.push(data.slice(start));
|
||||
}
|
||||
return nals;
|
||||
}
|
||||
|
||||
private stripAnnexBStartCode(data: Uint8Array): Uint8Array | null {
|
||||
// Find start of NAL data (after any 0x00000001 prefix).
|
||||
let i = 0;
|
||||
while (i < data.length - 4) {
|
||||
if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) {
|
||||
return data.slice(i + 4);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private concatNals(nals: Uint8Array[]): Uint8Array {
|
||||
const total = nals.reduce((sum, n) => sum + n.length + 4, 0);
|
||||
const buf = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const n of nals) {
|
||||
buf[offset++] = 0;
|
||||
buf[offset++] = 0;
|
||||
buf[offset++] = 0;
|
||||
buf[offset++] = 1;
|
||||
buf.set(n, offset);
|
||||
offset += n.length;
|
||||
}
|
||||
return buf.slice(0, offset);
|
||||
}
|
||||
|
||||
// ── JPEG fallback ─────────────────────────────────────────────────────────
|
||||
|
||||
private handleJpegFrame(payload: Uint8Array): void {
|
||||
// Close any H.264 decoder — we're falling back to JPEG mode.
|
||||
if (this.h264Configured) {
|
||||
this.closeH264Decoder();
|
||||
}
|
||||
|
||||
if (!this.canvas || !this.ctx) return;
|
||||
|
||||
const blob = new Blob([payload], { type: 'image/jpeg' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
this.img.onload = () => {
|
||||
if (this.canvas && this.ctx) {
|
||||
// Resize canvas to match JPEG dimensions.
|
||||
if (this.canvas.width !== this.img.naturalWidth || this.canvas.height !== this.img.naturalHeight) {
|
||||
this.canvas.width = this.img.naturalWidth;
|
||||
this.canvas.height = this.img.naturalHeight;
|
||||
}
|
||||
this.ctx.drawImage(this.img, 0, 0, this.canvas.width, this.canvas.height);
|
||||
this.hasFrame = true;
|
||||
this.resolution.set(`${this.img.naturalWidth}x${this.img.naturalHeight}`);
|
||||
this.status.set('connected');
|
||||
}
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
this.img.src = url;
|
||||
this.codec.set('JPEG');
|
||||
}
|
||||
|
||||
// ── Text message handling ────────────────────────────────────────────────
|
||||
|
||||
private handleTextMessage(text: string): void {
|
||||
try {
|
||||
const msg = JSON.parse(text);
|
||||
switch (msg.msg_type) {
|
||||
case 'session_update':
|
||||
if (msg.session_id === this.sessionId) {
|
||||
if (msg.status === 'active') this.status.set('connected');
|
||||
else if (msg.status === 'disconnected') this.status.set('disconnected');
|
||||
if (msg.resolution) this.resolution.set(msg.resolution);
|
||||
}
|
||||
break;
|
||||
case 'error':
|
||||
console.warn('[RemoteDisplay] server error:', msg.message);
|
||||
break;
|
||||
case 'ack':
|
||||
break;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('[RemoteDisplay] invalid JSON:', text);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Canvas management ─────────────────────────────────────────────────────
|
||||
|
||||
private resizeCanvas(): void {
|
||||
if (!this.canvas || !this.containerRef) return;
|
||||
const rect = this.containerRef.nativeElement.getBoundingClientRect();
|
||||
@ -225,37 +506,29 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
this.canvas.height = rect.height;
|
||||
}
|
||||
|
||||
// ── Mouse events (HUD forwarding) ───────────────────────────────────────
|
||||
private readU32LE(data: Uint8Array, offset: number): number {
|
||||
return (data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
// ── HUD input forwarding ──────────────────────────────────────────────────
|
||||
|
||||
onMouseDown(event: MouseEvent): void {
|
||||
const pos = this.getCanvasPos(event);
|
||||
this.sendHudCommand('mouse_down', {
|
||||
button: event.button,
|
||||
x: pos.x,
|
||||
y: pos.y,
|
||||
buttons: event.buttons,
|
||||
});
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_down', params: { button: event.button, x: pos.x, y: pos.y, buttons: event.buttons } });
|
||||
}
|
||||
|
||||
onMouseUp(event: MouseEvent): void {
|
||||
const pos = this.getCanvasPos(event);
|
||||
this.sendHudCommand('mouse_up', {
|
||||
button: event.button,
|
||||
x: pos.x,
|
||||
y: pos.y,
|
||||
});
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_up', params: { button: event.button, x: pos.x, y: pos.y } });
|
||||
}
|
||||
|
||||
onMouseMove(event: MouseEvent): void {
|
||||
const pos = this.getCanvasPos(event);
|
||||
this.sendHudCommand('mouse_move', { x: pos.x, y: pos.y });
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_move', params: { x: pos.x, y: pos.y } });
|
||||
}
|
||||
|
||||
onWheel(event: WheelEvent): void {
|
||||
this.sendHudCommand('scroll', {
|
||||
deltaX: event.deltaX,
|
||||
deltaY: event.deltaY,
|
||||
});
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'scroll', params: { deltaX: event.deltaX, deltaY: event.deltaY } });
|
||||
}
|
||||
|
||||
onContextMenu(event: Event): void {
|
||||
@ -263,29 +536,14 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
|
||||
}
|
||||
|
||||
onKeyDown(event: KeyboardEvent): void {
|
||||
this.sendHudCommand('key_down', {
|
||||
key: event.key,
|
||||
code: event.code,
|
||||
ctrl: event.ctrlKey,
|
||||
shift: event.shiftKey,
|
||||
alt: event.altKey,
|
||||
meta: event.metaKey,
|
||||
});
|
||||
// Prevent browser default for most keys when display is active.
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_down', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } });
|
||||
if (this.hasFrame && !event.ctrlKey && !event.metaKey) {
|
||||
event.preventDefault();
|
||||
}
|
||||
}
|
||||
|
||||
onKeyUp(event: KeyboardEvent): void {
|
||||
this.sendHudCommand('key_up', {
|
||||
key: event.key,
|
||||
code: event.code,
|
||||
ctrl: event.ctrlKey,
|
||||
shift: event.shiftKey,
|
||||
alt: event.altKey,
|
||||
meta: event.metaKey,
|
||||
});
|
||||
this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_up', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } });
|
||||
}
|
||||
|
||||
private getCanvasPos(event: MouseEvent): { x: number; y: number } {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user