/** * RustDesk Standalone Web Client - Video Decoder * * Handles video decoding using ogvjs (VP8/VP9/AV1 WebAssembly decoders) * and rendering to canvas using yuv-canvas (WebGL YUV→RGB conversion). */ const RDVideo = (() => { let _canvas = null; let _ctx = null; let _yuvCanvas = null; let _decoders = {}; // Codec-specific decoders let _currentCodec = null; // Currently active codec let _displayWidth = 0; let _displayHeight = 0; let _lastRenderTime = 0; let _frameCount = 0; let _fpsCounter = 0; let _fpsTime = 0; let _currentFps = 0; let _ogvSupport = null; // Supported codecs const CODEC = { VP8: 'vp8', VP9: 'vp9', AV1: 'av1', H264: 'h264', H265: 'h265' }; /** * Initialize video subsystem * @param {HTMLCanvasElement} canvas - The canvas to render to */ function init(canvas) { _canvas = canvas; _ctx = canvas.getContext('2d'); _yuvCanvas = new YUVCanvas({ canvas: canvas, webgl: true }); _fpsTime = performance.now(); console.log('[RDVideo] Initialized with canvas:', canvas.width, 'x', canvas.height); return true; } /** * Initialize a video decoder for the given codec */ function initDecoder(codec) { if (_decoders[codec]) return _decoders[codec]; console.log('[RDVideo] Initializing decoder for codec:', codec); try { switch (codec) { case CODEC.VP9: _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-vp9-wasm'); _currentCodec = codec; break; case CODEC.VP8: _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-vp8-wasm'); _currentCodec = codec; break; case CODEC.AV1: _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-av1-wasm'); _currentCodec = codec; break; default: console.warn('[RDVideo] Unsupported codec:', codec); return null; } return _decoders[codec]; } catch (e) { console.error('[RDVideo] Failed to init decoder for', codec, ':', e); return null; } } /** * Create an ogvjs video decoder */ function createOgvVideoDecoder(moduleName) { // ogvjs uses a specific API for creating decoders // We need to use the OGVDecoderVideoVPx or similar classes if (typeof OGVDecoderVideoVP9 !== 'undefined') { const decoderClass = { 'ogv-decoder-video-vp9-wasm': OGVDecoderVideoVP9, 'ogv-decoder-video-vp8-wasm': OGVDecoderVideoVP8, 'ogv-decoder-video-av1-wasm': OGVDecoderVideoAV1, }[moduleName]; if (decoderClass) { const decoder = new decoderClass({ type: moduleName, }); // Process any loaded frames decoder.onvideo = function(frame) { processDecodedFrame(frame); }; return decoder; } } // Fallback: create a frame processor that processes raw video data console.warn('[RDVideo] ogvjs decoder classes not available, using fallback'); return { codec: moduleName, processFrame: processFallbackFrame }; } /** * Process a decoded video frame and render it */ function processDecodedFrame(frame) { if (!frame || !frame.format) return; const width = frame.format.width; const height = frame.format.height; if (width !== _displayWidth || height !== _displayHeight) { _displayWidth = width; _displayHeight = height; _canvas.width = width; _canvas.height = height; console.log('[RDVideo] Display size changed:', width, 'x', height); } // ogvjs provides frames in YUV420 format // YUV data: frame.buf (planar Y, U, V) if (frame.pixels) { // RGBA data available directly const imageData = new ImageData( new Uint8ClampedArray(frame.pixels), width, height ); _ctx.putImageData(imageData, 0, 0); } else if (frame.frames && frame.frames.length > 0 && frame.frames[0].buf) { // YUV data - use yuv-canvas for WebGL rendering const yuvFrame = frame.frames[0]; renderYUV(yuvFrame.buf, width, height); } updateFps(); } /** * Process a fallback frame (raw RGB/YUV data) */ function processFallbackFrame(data, width, height) { if (!data || !data.byteLength) return; // Try to render as RGBA if (_canvas.width !== width || _canvas.height !== height) { _displayWidth = width; _displayHeight = height; _canvas.width = width; _canvas.height = height; } const imageData = new ImageData( new Uint8ClampedArray(data.buffer || data), width, height ); _ctx.putImageData(imageData, 0, 0); updateFps(); } /** * Render YUV data to canvas using WebGL */ function renderYUV(yuvData, width, height) { if (!_yuvCanvas) return; try { // YUV420 layout: Y plane (width*height), U plane (width*height/4), V plane (width*height/4) const ySize = width * height; const uvSize = ySize / 4; const yPlane = new Uint8Array(yuvData.buffer || yuvData, 0, ySize); const uPlane = new Uint8Array(yuvData.buffer || yuvData, ySize, uvSize); const vPlane = new Uint8Array(yuvData.buffer || yuvData, ySize + uvSize, uvSize); _yuvCanvas.drawFrame({ format: { width: width, height: height, chromaWidth: width / 2, chromaHeight: height / 2, chromaPitch: width / 2, lumaPitch: width, cropLeft: 0, cropTop: 0, cropWidth: width, cropHeight: height, bitDepth: 8 }, planes: [ { data: yPlane, stride: width }, { data: uPlane, stride: width / 2 }, { data: vPlane, stride: width / 2 } ], time: { low: Date.now() % 0x100000000, high: 0 } }); } catch (e) { console.error('[RDVideo] YUV render failed:', e); // Fallback to 2D canvas const ySize = width * height; const yPlane = new Uint8Array(yuvData.buffer || yuvData, 0, ySize); renderYUVToRGBA(yPlane, width, height); } } /** * Fallback YUV to RGBA conversion using 2D canvas */ function renderYUVToRGBA(yPlane, width, height) { const imageData = _ctx.createImageData(width, height); const data = imageData.data; for (let i = 0; i < width * height; i++) { const y = yPlane[i]; const idx = i * 4; // Simple grayscale (no U/V in this fallback) data[idx] = y; data[idx + 1] = y; data[idx + 2] = y; data[idx + 3] = 255; } _ctx.putImageData(imageData, 0, 0); } /** * Handle incoming VideoFrame from the session * @param {Object} videoFrame - Decoded protobuf VideoFrame */ function handleVideoFrame(videoFrame) { if (!videoFrame) return; const display = videoFrame.display || 0; // Check for VP9 frames if (videoFrame.vp9s && videoFrame.vp9s.frames && videoFrame.vp9s.frames.length > 0) { const codec = CODEC.VP9; if (_currentCodec !== codec) initDecoder(codec); processEncodedFrames(videoFrame.vp9s.frames, codec); return; } // Check for VP8 frames if (videoFrame.vp8s && videoFrame.vp8s.frames && videoFrame.vp8s.frames.length > 0) { const codec = CODEC.VP8; if (_currentCodec !== codec) initDecoder(codec); processEncodedFrames(videoFrame.vp8s.frames, codec); return; } // Check for AV1 frames if (videoFrame.av1s && videoFrame.av1s.frames && videoFrame.av1s.frames.length > 0) { const codec = CODEC.AV1; if (_currentCodec !== codec) initDecoder(codec); processEncodedFrames(videoFrame.av1s.frames, codec); return; } // Check for raw RGB if (videoFrame.rgb) { // RGB data comes as the raw protobuf payload after the RGB message // In practice, the raw data is appended after the VideoFrame message console.log('[RDVideo] RGB frame received (not fully supported)'); return; } // Check for raw YUV if (videoFrame.yuv) { // Similar to RGB, YUV data is appended console.log('[RDVideo] YUV frame received (not fully supported)'); return; } // H264/H265 not supported in browser without WASM decoders if (videoFrame.h264s) { console.log('[RDVideo] H.264 frame received (not supported in browser)'); } if (videoFrame.h265s) { console.log('[RDVideo] H.265 frame received (not supported in browser)'); } } /** * Process encoded video frames through the decoder */ function processEncodedFrames(frames, codec) { const decoder = _decoders[codec]; if (!decoder) { console.warn('[RDVideo] No decoder for codec:', codec); return; } for (const frame of frames) { if (!frame.data || frame.data.byteLength === 0) continue; try { if (decoder.processFrame) { // Custom fallback decoder // Try to determine frame dimensions from data // VP9 bitstream: first bytes contain frame header // For now, just log _frameCount++; } else if (decoder instanceof OGVDecoderVideoVP9 || decoder instanceof OGVDecoderVideoVP8 || decoder instanceof OGVDecoderVideoAV1) { // ogvjs native decoder decoder.processHeader(frame.data); // Check if decoder has loaded header info for dimensions } } catch (e) { console.error('[RDVideo] Frame decode error:', e); } } } /** * Process raw RGBA data (from fallback/secondary path) * @param {Uint8Array} rgba - RGBA pixel data * @param {number} width - Frame width * @param {number} height - Frame height */ function processRGBA(rgba, width, height) { if (_canvas.width !== width || _canvas.height !== height) { _canvas.width = width; _canvas.height = height; _displayWidth = width; _displayHeight = height; } const imageData = new ImageData( new Uint8ClampedArray(rgba.buffer || rgba), width, height ); _ctx.putImageData(imageData, 0, 0); updateFps(); } /** * Update FPS counter */ function updateFps() { _frameCount++; _fpsCounter++; const now = performance.now(); if (now - _fpsTime >= 1000) { _currentFps = _fpsCounter; _fpsCounter = 0; _fpsTime = now; } } /** * Get current FPS */ function getFps() { return _currentFps; } /** * Resize the canvas */ function resize(width, height) { if (_canvas) { _canvas.width = width; _canvas.height = height; } } /** * Get current display dimensions */ function getDisplaySize() { return { width: _displayWidth, height: _displayHeight }; } /** * Take a screenshot of the current canvas */ function screenshot() { if (!_canvas) return null; return _canvas.toDataURL('image/png'); } return { CODEC, init, handleVideoFrame, processRGBA, resize, getDisplaySize, getFps, screenshot, get canvas() { return _canvas; } }; })();