frontend: WebCodes H.264 decoder, binary WS frames, 13-byte header parsing, JPEG fallback, AVCC description builder

desktop/src/app/components/remote-display/remote-display.component.ts

@@ -9,41 +9,20 @@ import {
   AfterViewInit,
 } from '@angular/core';
 import { CommonModule } from '@angular/common';
-import { Subject, filter, takeUntil } from 'rxjs';
 
-// ── Types matching the Rust WsMessage enum ─────────────────────────────────
+// ── Binary frame protocol constants (matching server/agent) ──────────────────
+const FRAME_HEADER_SIZE = 13;
+const FRAME_TYPE_H264_KEY = 0x01;
+const FRAME_TYPE_H264_DELTA = 0x02;
+const FRAME_TYPE_JPEG = 0x03;
 
-interface FrameBroadcast {
+interface FrameHeader {
-  msg_type: 'frame_broadcast';
+  frameType: number;
-  data: string;
+  timestampMs: number;
-  content_type: string;
+  width: number;
+  height: number;
 }
 
-interface AudioBroadcast {
-  msg_type: 'audio_broadcast';
-  data: string;
-  content_type: string;
-}
-
-interface SessionUpdate {
-  msg_type: 'session_update';
-  session_id: string;
-  status: 'waiting' | 'active' | 'disconnected';
-  resolution: string | null;
-}
-
-interface WsError {
-  msg_type: 'error';
-  message: string;
-}
-
-interface WsAck {
-  msg_type: 'ack';
-  message: string;
-}
-
-type ServerMessage = FrameBroadcast | AudioBroadcast | SessionUpdate | WsError | WsAck;
-
 @Component({
   selector: 'app-remote-display',
   standalone: true,
@@ -57,57 +36,55 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
   @ViewChild('canvas') canvasRef!: ElementRef<HTMLCanvasElement>;
   @ViewChild('displayContainer') containerRef!: ElementRef<HTMLDivElement>;
 
-  /** Current connection status. */
   readonly status = signal<'connecting' | 'connected' | 'disconnected' | 'error'>('connecting');
-
-  /** Display resolution string. */
   readonly resolution = signal<string | null>(null);
-
-  /** Frames per second counter. */
-  private frameCount = 0;
-  private lastFpsTime = 0;
   readonly fps = signal(0);
+  readonly codec = signal<string>('unknown');
 
   private canvas: HTMLCanvasElement | null = null;
   private ctx: CanvasRenderingContext2D | null = null;
   private img = new Image();
-  private animFrameId = 0;
   private hasFrame = false;
 
-  /** Session status from server. */
+  // ── H.264 WebCodecs decoder ─────────────────────────────────────────────
-  readonly sessionStatus = signal<string>('waiting');
+  private h264Decoder: VideoDecoder | null = null;
+  private h264Configured = false;
+  private h264Codec = '';
+  private decoderVersion = 0; // Incremented on codec config change to discard stale frames.
 
-  /** Per-instance WebSocket connection (not the shared singleton). */
+  // ── WebSocket (per-instance) ─────────────────────────────────────────────
   private socket: WebSocket | null = null;
-  private messagesSubject = new Subject<ServerMessage>();
+  private destroy$ = false;
-  private destroy$ = new Subject<void>();
   private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
   private resizeObserver: ResizeObserver | null = null;
 
+  // ── FPS tracking ─────────────────────────────────────────────────────────
+  private frameCount = 0;
+  private lastFpsTime = 0;
+
   ngOnInit(): void {
     this.connectWebSocket();
-    this.setupSubscriptions();
   }
 
   ngAfterViewInit(): void {
     this.canvas = this.canvasRef.nativeElement;
     this.ctx = this.canvas.getContext('2d');
-    this.resizeCanvas();
 
-    // Watch for container size changes (window resize, maximize, etc.).
+    // Resize canvas to container.
+    this.resizeCanvas();
     this.resizeObserver = new ResizeObserver(() => this.resizeCanvas());
     this.resizeObserver.observe(this.containerRef.nativeElement);
   }
 
   ngOnDestroy(): void {
-    this.destroy$.next();
+    this.destroy$ = true;
-    this.destroy$.complete();
     this.disconnectWebSocket();
-    if (this.animFrameId) cancelAnimationFrame(this.animFrameId);
+    this.closeH264Decoder();
     this.resizeObserver?.disconnect();
+    this.stopHeartbeat();
   }
 
-  // ── WebSocket connection (per-instance) ─────────────────────────────────
+  // ── WebSocket connection ─────────────────────────────────────────────────
 
   private connectWebSocket(): void {
     const wsUrl =
@@ -115,24 +92,30 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
       `/ws/${this.sessionId}?client_type=viewer`;
 
     this.socket = new WebSocket(wsUrl);
+    // Request binary format for all data.
+    this.socket.binaryType = 'arraybuffer';
 
     this.socket.onopen = () => {
       console.log('[RemoteDisplay] connected to session', this.sessionId);
       this.status.set('connecting');
-      this.heartbeatTimer = setInterval(() => this.sendRaw({ msg_type: 'heartbeat' }), 15000);
+      this.heartbeatTimer = setInterval(() => {
+        this.sendText({ msg_type: 'heartbeat' });
+      }, 15000);
     };
 
     this.socket.onmessage = (event) => {
-      try {
+      if (this.destroy$) return;
-        const msg: ServerMessage = JSON.parse(event.data);
+
-        this.messagesSubject.next(msg);
+      if (event.data instanceof ArrayBuffer) {
-      } catch (e) {
+        // Binary frame — video data.
-        console.warn('[RemoteDisplay] failed to parse message', event.data, e);
+        this.handleBinaryFrame(new Uint8Array(event.data));
+      } else if (typeof event.data === 'string') {
+        // Text frame — JSON control message.
+        this.handleTextMessage(event.data);
       }
     };
 
-    this.socket.onclose = (event) => {
+    this.socket.onclose = () => {
-      console.log('[RemoteDisplay] disconnected', event.code, event.reason);
       this.status.set('disconnected');
       this.stopHeartbeat();
     };
@@ -150,67 +133,12 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
     }
   }
 
-  private setupSubscriptions(): void {
+  private sendText(obj: Record<string, unknown>): void {
-    // Display frames
-    this.messagesSubject.pipe(
-      filter((m): m is FrameBroadcast => m.msg_type === 'frame_broadcast'),
-      takeUntil(this.destroy$),
-    ).subscribe((frame) => {
-      if (!this.canvas || !this.ctx) return;
-      this.img.onload = () => {
-        this.ctx!.drawImage(this.img, 0, 0, this.canvas!.width, this.canvas!.height);
-        this.hasFrame = true;
-      };
-      this.img.src = 'data:' + frame.content_type + ';base64,' + frame.data;
-      this.frameCount++;
-      const now = performance.now();
-      if (now - this.lastFpsTime >= 1000) {
-        this.fps.set(this.frameCount);
-        this.frameCount = 0;
-        this.lastFpsTime = now;
-      }
-    });
-
-    // Session state updates
-    this.messagesSubject.pipe(
-      filter((m): m is SessionUpdate => m.msg_type === 'session_update'),
-      takeUntil(this.destroy$),
-    ).subscribe((update) => {
-      if (update.session_id === this.sessionId) {
-        this.sessionStatus.set(update.status);
-        this.resolution.set(update.resolution);
-        if (update.status === 'active') {
-          this.status.set('connected');
-        } else if (update.status === 'disconnected') {
-          this.status.set('disconnected');
-        }
-      }
-    });
-
-    // Error messages
-    this.messagesSubject.pipe(
-      filter((m): m is WsError => m.msg_type === 'error'),
-      takeUntil(this.destroy$),
-    ).subscribe((err) => {
-      console.warn('[RemoteDisplay] server error:', err.message);
-    });
-  }
-
-  private sendRaw(msg: Record<string, unknown>): void {
     if (this.socket?.readyState === WebSocket.OPEN) {
-      this.socket.send(JSON.stringify(msg));
+      this.socket.send(JSON.stringify(obj));
     }
   }
 
-  private sendHudCommand(command: string, params: Record<string, unknown> = {}): void {
-    this.sendRaw({
-      msg_type: 'hud_command',
-      session_id: this.sessionId,
-      command,
-      params,
-    });
-  }
-
   private stopHeartbeat(): void {
     if (this.heartbeatTimer) {
       clearInterval(this.heartbeatTimer);
@@ -218,6 +146,359 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
     }
   }
 
+  // ── Binary frame handling ────────────────────────────────────────────────
+
+  private handleBinaryFrame(data: Uint8Array): void {
+    if (data.length < FRAME_HEADER_SIZE) return;
+
+    const header: FrameHeader = {
+      frameType: data[0],
+      timestampMs: this.readU32LE(data, 1),
+      width: this.readU32LE(data, 5),
+      height: this.readU32LE(data, 9),
+    };
+
+    const payload = data.slice(FRAME_HEADER_SIZE);
+    if (payload.length === 0) return;
+
+    switch (header.frameType) {
+      case FRAME_TYPE_H264_KEY:
+        this.handleH264Keyframe(payload, header.width, header.height);
+        break;
+      case FRAME_TYPE_H264_DELTA:
+        this.handleH264DeltaFrame(payload);
+        break;
+      case FRAME_TYPE_JPEG:
+        this.handleJpegFrame(payload);
+        break;
+    }
+
+    // FPS counter.
+    this.frameCount++;
+    const now = performance.now();
+    if (now - this.lastFpsTime >= 1000) {
+      this.fps.set(this.frameCount);
+      this.frameCount = 0;
+      this.lastFpsTime = now;
+    }
+  }
+
+  // ── H.264 decoding via WebCodecs API ─────────────────────────────────────
+
+  private handleH264Keyframe(payload: Uint8Array, width: number, height: number): void {
+    // Parse AVCC/HVCC-style config from the start of keyframe NALs.
+    // openh264 outputs Annex-B format (0x00000001 start codes).
+    // WebCodecs expects raw NAL units (no start codes).
+    const { configNalu, sliceNalu } = this.splitAnnexBNalu(payload);
+
+    if (!configNalu || !sliceNalu) {
+      // Fallback to JPEG-style render if we can't parse NALs.
+      console.warn('[RemoteDisplay] could not parse H.264 keyframe NALs');
+      return;
+    }
+
+    // Try to configure the decoder from the SPS/PPS.
+    const codecString = this.h264Codec || this.guessH264Codec(configNalu);
+    if (!codecString) {
+      console.warn('[RemoteDisplay] could not determine H.264 codec');
+      return;
+    }
+
+    // Check if we need to reconfigure (codec string or resolution changed).
+    if (!this.h264Configured || this.h264Codec !== codecString) {
+      this.closeH264Decoder();
+      this.initH264Decoder(codecString, width, height, configNalu);
+    }
+
+    // Decode the keyframe slice.
+    if (this.h264Decoder && this.h264Decoder.state !== 'closed') {
+      const chunk = new EncodedVideoChunk({
+        type: 'key',
+        timestamp: performance.now() * 1000, // microseconds
+        data: sliceNalu,
+      });
+      this.h264Decoder.decode(chunk);
+      this.decoderVersion++;
+    }
+
+    this.resolution.set(`${width}x${height}`);
+    this.status.set('connected');
+    this.hasFrame = true;
+  }
+
+  private handleH264DeltaFrame(payload: Uint8Array): void {
+    if (!this.h264Decoder || this.h264Decoder.state === 'closed' || !this.h264Configured) {
+      // Not configured yet — skip delta frames.
+      return;
+    }
+
+    // Strip Annex-B start codes and decode the raw NAL unit.
+    const nalu = this.stripAnnexBStartCode(payload);
+    if (!nalu) return;
+
+    const chunk = new EncodedVideoChunk({
+      type: 'delta',
+      timestamp: performance.now() * 1000,
+      data: nalu,
+    });
+    this.h264Decoder.decode(chunk);
+  }
+
+  private initH264Decoder(codec: string, width: number, height: number, configNalu: Uint8Array): void {
+    this.h264Codec = codec;
+
+    // Build AVCC description from SPS/PPS for WebCodecs.
+    const description = this.buildAVCCDescription(configNalu);
+
+    this.h264Decoder = new VideoDecoder({
+      output: (videoFrame: VideoFrame) => {
+        this.drawVideoFrame(videoFrame);
+        videoFrame.close();
+      },
+      error: (e: DOMException) => {
+        console.error('[RemoteDisplay] H.264 decode error:', e.message);
+        // If we get too many errors, fall back to requesting a new keyframe.
+        // The server/agent will need to be updated to support this.
+      },
+    });
+
+    this.h264Decoder.configure({
+      codec: codec,
+      codedWidth: width,
+      codedHeight: height,
+      description: description,
+      optimizeForLatency: true,
+    });
+
+    this.h264Configured = true;
+    this.codec.set(`H.264 (${codec})`);
+    console.log(`[RemoteDisplay] H.264 decoder configured: ${codec} ${width}x${height}`);
+  }
+
+  private closeH264Decoder(): void {
+    if (this.h264Decoder) {
+      try { this.h264Decoder.close(); } catch {}
+      this.h264Decoder = null;
+      this.h264Configured = false;
+    }
+  }
+
+  private drawVideoFrame(frame: VideoFrame): void {
+    if (!this.canvas || !this.ctx) return;
+
+    // Resize canvas if the frame dimensions changed.
+    if (this.canvas.width !== frame.displayWidth || this.canvas.height !== frame.displayHeight) {
+      this.canvas.width = frame.displayWidth;
+      this.canvas.height = frame.displayHeight;
+    }
+
+    this.ctx.drawImage(frame, 0, 0, frame.displayWidth, frame.displayHeight);
+    this.hasFrame = true;
+  }
+
+  /**
+   * Try to guess the H.264 codec string from SPS NAL unit.
+   * Parses profile_idc and level_idc from the SPS.
+   */
+  private guessH264Codec(configNalu: Uint8Array): string | null {
+    if (configNalu.length < 4) return null;
+
+    // configNalu starts with SPS (NAL type 0x67) or combined SPS+PPS.
+    const firstByte = configNalu[0];
+    const nalType = firstByte & 0x1F;
+
+    if (nalType === 7) {
+      // SPS NAL unit.
+      // profile_idc is byte 1, level_idc is byte 3 (in NAL body).
+      const profileIdc = configNalu[1];
+      const levelIdc = configNalu[3];
+
+      // Common profiles.
+      const profileMap: Record<number, string> = {
+        66: 'Baseline',
+        77: 'Main',
+        88: 'Extended',
+        100: 'High',
+        110: 'High 10',
+        122: 'High 4:2:2',
+        244: 'High 4:4:4 Predictive',
+      };
+      const profile = profileMap[profileIdc] || `${profileIdc}`;
+
+      return `avc1.${profileIdc.toString(16).padStart(2, '0')}00${levelIdc.toString(16).padStart(2, '0')}`;
+    }
+
+    // Fallback: use Baseline Level 4.0.
+    return 'avc1.42001f';
+  }
+
+  /**
+   * Build AVCC "description" box from SPS+PPS NALs (for WebCodecs).
+   * AVCC format: [version][profile][compat][level][NALU length size-1][num sps][sps len][sps][num pps][pps len][pps]
+   */
+  private buildAVCCDescription(spsPpsNalu: Uint8Array): Uint8Array {
+    // Split into individual NAL units.
+    const nalus: Uint8Array[] = [];
+    let start = 0;
+    for (let i = 0; i < spsPpsNalu.length - 4; i++) {
+      if (spsPpsNalu[i] === 0 && spsPpsNalu[i + 1] === 0 && spsPpsNalu[i + 2] === 0 && spsPpsNalu[i + 3] === 1) {
+        if (start > 0 || i > 0) {
+          nalus.push(spsPpsNalu.slice(start, i));
+        }
+        start = i + 4;
+      }
+    }
+    if (start < spsPpsNalu.length) {
+      nalus.push(spsPpsNalu.slice(start));
+    }
+
+    if (nalus.length < 2) {
+      // Not enough NALUs for SPS+PPS, return empty description.
+      // WebCodecs can still work with just the codec string.
+      return new Uint8Array(0);
+    }
+
+    const sps = nalus[0];
+    const pps = nalus[1];
+
+    // Build AVCC box.
+    const buf = new Uint8Array(8 + sps.length + 3 + pps.length);
+    buf[0] = 1; // Version
+    buf[1] = sps[1]; // Profile
+    buf[2] = sps[2]; // Compatibility
+    buf[3] = sps[3]; // Level
+    buf[4] = 0xFF; // NALU length size = 4 (0xFF & 0x03 = 3, +1 = 4)
+    buf[5] = 0xE1; // Num SPS = 1 (0xE1 & 0x1F = 1)
+    buf[6] = (sps.length >> 8) & 0xFF;
+    buf[7] = sps.length & 0xFF;
+    buf.set(sps, 8);
+    let offset = 8 + sps.length;
+    buf[offset] = 1; // Num PPS
+    buf[offset + 1] = (pps.length >> 8) & 0xFF;
+    buf[offset + 2] = pps.length & 0xFF;
+    buf.set(pps, offset + 3);
+
+    return buf;
+  }
+
+  /**
+   * Split an Annex-B keyframe into config (SPS+PPS) and slice (IDR) NAL units.
+   */
+  private splitAnnexBNalu(data: Uint8Array): { configNalu?: Uint8Array; sliceNalu?: Uint8Array } {
+    const nals = this.findAnnexBNals(data);
+    if (nals.length < 2) {
+      // Single NAL — treat as config only.
+      return { configNalu: nals[0] };
+    }
+
+    // First NAL (SPS) + second NAL (PPS) = config.
+    // Last NAL = IDR slice.
+    return {
+      configNalu: this.concatNals(nals.slice(0, Math.min(2, nals.length))),
+      sliceNalu: nals[nals.length - 1],
+    };
+  }
+
+  private findAnnexBNals(data: Uint8Array): Uint8Array[] {
+    const nals: Uint8Array[] = [];
+    let start = 0;
+    for (let i = 0; i < data.length - 4; i++) {
+      if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) {
+        if (start < i) {
+          nals.push(data.slice(start, i));
+        }
+        start = i + 4;
+      }
+    }
+    if (start < data.length) {
+      nals.push(data.slice(start));
+    }
+    return nals;
+  }
+
+  private stripAnnexBStartCode(data: Uint8Array): Uint8Array | null {
+    // Find start of NAL data (after any 0x00000001 prefix).
+    let i = 0;
+    while (i < data.length - 4) {
+      if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0 && data[i + 3] === 1) {
+        return data.slice(i + 4);
+      }
+      i++;
+    }
+    return data;
+  }
+
+  private concatNals(nals: Uint8Array[]): Uint8Array {
+    const total = nals.reduce((sum, n) => sum + n.length + 4, 0);
+    const buf = new Uint8Array(total);
+    let offset = 0;
+    for (const n of nals) {
+      buf[offset++] = 0;
+      buf[offset++] = 0;
+      buf[offset++] = 0;
+      buf[offset++] = 1;
+      buf.set(n, offset);
+      offset += n.length;
+    }
+    return buf.slice(0, offset);
+  }
+
+  // ── JPEG fallback ─────────────────────────────────────────────────────────
+
+  private handleJpegFrame(payload: Uint8Array): void {
+    // Close any H.264 decoder — we're falling back to JPEG mode.
+    if (this.h264Configured) {
+      this.closeH264Decoder();
+    }
+
+    if (!this.canvas || !this.ctx) return;
+
+    const blob = new Blob([payload], { type: 'image/jpeg' });
+    const url = URL.createObjectURL(blob);
+    this.img.onload = () => {
+      if (this.canvas && this.ctx) {
+        // Resize canvas to match JPEG dimensions.
+        if (this.canvas.width !== this.img.naturalWidth || this.canvas.height !== this.img.naturalHeight) {
+          this.canvas.width = this.img.naturalWidth;
+          this.canvas.height = this.img.naturalHeight;
+        }
+        this.ctx.drawImage(this.img, 0, 0, this.canvas.width, this.canvas.height);
+        this.hasFrame = true;
+        this.resolution.set(`${this.img.naturalWidth}x${this.img.naturalHeight}`);
+        this.status.set('connected');
+      }
+      URL.revokeObjectURL(url);
+    };
+    this.img.src = url;
+    this.codec.set('JPEG');
+  }
+
+  // ── Text message handling ────────────────────────────────────────────────
+
+  private handleTextMessage(text: string): void {
+    try {
+      const msg = JSON.parse(text);
+      switch (msg.msg_type) {
+        case 'session_update':
+          if (msg.session_id === this.sessionId) {
+            if (msg.status === 'active') this.status.set('connected');
+            else if (msg.status === 'disconnected') this.status.set('disconnected');
+            if (msg.resolution) this.resolution.set(msg.resolution);
+          }
+          break;
+        case 'error':
+          console.warn('[RemoteDisplay] server error:', msg.message);
+          break;
+        case 'ack':
+          break;
+      }
+    } catch (e) {
+      console.warn('[RemoteDisplay] invalid JSON:', text);
+    }
+  }
+
+  // ── Canvas management ─────────────────────────────────────────────────────
+
   private resizeCanvas(): void {
     if (!this.canvas || !this.containerRef) return;
     const rect = this.containerRef.nativeElement.getBoundingClientRect();
@@ -225,37 +506,29 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
     this.canvas.height = rect.height;
   }
 
-  // ── Mouse events (HUD forwarding) ───────────────────────────────────────
+  private readU32LE(data: Uint8Array, offset: number): number {
+    return (data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)) >>> 0;
+  }
+
+  // ── HUD input forwarding ──────────────────────────────────────────────────
 
   onMouseDown(event: MouseEvent): void {
     const pos = this.getCanvasPos(event);
-    this.sendHudCommand('mouse_down', {
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_down', params: { button: event.button, x: pos.x, y: pos.y, buttons: event.buttons } });
-      button: event.button,
-      x: pos.x,
-      y: pos.y,
-      buttons: event.buttons,
-    });
   }
 
   onMouseUp(event: MouseEvent): void {
     const pos = this.getCanvasPos(event);
-    this.sendHudCommand('mouse_up', {
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_up', params: { button: event.button, x: pos.x, y: pos.y } });
-      button: event.button,
-      x: pos.x,
-      y: pos.y,
-    });
   }
 
   onMouseMove(event: MouseEvent): void {
     const pos = this.getCanvasPos(event);
-    this.sendHudCommand('mouse_move', { x: pos.x, y: pos.y });
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'mouse_move', params: { x: pos.x, y: pos.y } });
   }
 
   onWheel(event: WheelEvent): void {
-    this.sendHudCommand('scroll', {
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'scroll', params: { deltaX: event.deltaX, deltaY: event.deltaY } });
-      deltaX: event.deltaX,
-      deltaY: event.deltaY,
-    });
   }
 
   onContextMenu(event: Event): void {
@@ -263,29 +536,14 @@ export class RemoteDisplayComponent implements OnInit, AfterViewInit, OnDestroy
   }
 
   onKeyDown(event: KeyboardEvent): void {
-    this.sendHudCommand('key_down', {
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_down', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } });
-      key: event.key,
-      code: event.code,
-      ctrl: event.ctrlKey,
-      shift: event.shiftKey,
-      alt: event.altKey,
-      meta: event.metaKey,
-    });
-    // Prevent browser default for most keys when display is active.
     if (this.hasFrame && !event.ctrlKey && !event.metaKey) {
       event.preventDefault();
     }
   }
 
   onKeyUp(event: KeyboardEvent): void {
-    this.sendHudCommand('key_up', {
+    this.sendText({ msg_type: 'hud_command', session_id: this.sessionId, command: 'key_up', params: { key: event.key, code: event.code, ctrl: event.ctrlKey, shift: event.shiftKey, alt: event.altKey, meta: event.metaKey } });
-      key: event.key,
-      code: event.code,
-      ctrl: event.ctrlKey,
-      shift: event.shiftKey,
-      alt: event.altKey,
-      meta: event.metaKey,
-    });
   }
 
   private getCanvasPos(event: MouseEvent): { x: number; y: number } {