projects/shelled/rustdesk-web-client/js/video.js

/**
 * RustDesk Standalone Web Client - Video Decoder
 * 
 * Handles video decoding using ogvjs (VP8/VP9/AV1 WebAssembly decoders)
 * and rendering to canvas using yuv-canvas (WebGL YUV→RGB conversion).
 */

const RDVideo = (() => {
    let _canvas = null;
    let _ctx = null;
    let _yuvCanvas = null;
    let _decoders = {};        // Codec-specific decoders
    let _currentCodec = null;  // Currently active codec
    let _displayWidth = 0;
    let _displayHeight = 0;
    let _lastRenderTime = 0;
    let _frameCount = 0;
    let _fpsCounter = 0;
    let _fpsTime = 0;
    let _currentFps = 0;
    let _ogvSupport = null;

    // Supported codecs
    const CODEC = {
        VP8: 'vp8',
        VP9: 'vp9',
        AV1: 'av1',
        H264: 'h264',
        H265: 'h265'
    };

    /**
     * Initialize video subsystem
     * @param {HTMLCanvasElement} canvas - The canvas to render to
     */
    function init(canvas) {
        _canvas = canvas;
        _ctx = canvas.getContext('2d');
        _yuvCanvas = new YUVCanvas({
            canvas: canvas,
            webgl: true
        });
        _fpsTime = performance.now();

        console.log('[RDVideo] Initialized with canvas:', canvas.width, 'x', canvas.height);
        return true;
    }

    /**
     * Initialize a video decoder for the given codec
     */
    function initDecoder(codec) {
        if (_decoders[codec]) return _decoders[codec];

        console.log('[RDVideo] Initializing decoder for codec:', codec);

        try {
            switch (codec) {
                case CODEC.VP9:
                    _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-vp9-wasm');
                    _currentCodec = codec;
                    break;
                case CODEC.VP8:
                    _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-vp8-wasm');
                    _currentCodec = codec;
                    break;
                case CODEC.AV1:
                    _decoders[codec] = createOgvVideoDecoder('ogv-decoder-video-av1-wasm');
                    _currentCodec = codec;
                    break;
                default:
                    console.warn('[RDVideo] Unsupported codec:', codec);
                    return null;
            }
            return _decoders[codec];
        } catch (e) {
            console.error('[RDVideo] Failed to init decoder for', codec, ':', e);
            return null;
        }
    }

    /**
     * Create an ogvjs video decoder
     */
    function createOgvVideoDecoder(moduleName) {
        // ogvjs uses a specific API for creating decoders
        // We need to use the OGVDecoderVideoVPx or similar classes
        if (typeof OGVDecoderVideoVP9 !== 'undefined') {
            const decoderClass = {
                'ogv-decoder-video-vp9-wasm': OGVDecoderVideoVP9,
                'ogv-decoder-video-vp8-wasm': OGVDecoderVideoVP8,
                'ogv-decoder-video-av1-wasm': OGVDecoderVideoAV1,
            }[moduleName];

            if (decoderClass) {
                const decoder = new decoderClass({
                    type: moduleName,
                });
                // Process any loaded frames
                decoder.onvideo = function(frame) {
                    processDecodedFrame(frame);
                };
                return decoder;
            }
        }

        // Fallback: create a frame processor that processes raw video data
        console.warn('[RDVideo] ogvjs decoder classes not available, using fallback');
        return {
            codec: moduleName,
            processFrame: processFallbackFrame
        };
    }

    /**
     * Process a decoded video frame and render it
     */
    function processDecodedFrame(frame) {
        if (!frame || !frame.format) return;

        const width = frame.format.width;
        const height = frame.format.height;

        if (width !== _displayWidth || height !== _displayHeight) {
            _displayWidth = width;
            _displayHeight = height;
            _canvas.width = width;
            _canvas.height = height;
            console.log('[RDVideo] Display size changed:', width, 'x', height);
        }

        // ogvjs provides frames in YUV420 format
        // YUV data: frame.buf (planar Y, U, V)
        if (frame.pixels) {
            // RGBA data available directly
            const imageData = new ImageData(
                new Uint8ClampedArray(frame.pixels),
                width,
                height
            );
            _ctx.putImageData(imageData, 0, 0);
        } else if (frame.frames && frame.frames.length > 0 && frame.frames[0].buf) {
            // YUV data - use yuv-canvas for WebGL rendering
            const yuvFrame = frame.frames[0];
            renderYUV(yuvFrame.buf, width, height);
        }

        updateFps();
    }

    /**
     * Process a fallback frame (raw RGB/YUV data)
     */
    function processFallbackFrame(data, width, height) {
        if (!data || !data.byteLength) return;

        // Try to render as RGBA
        if (_canvas.width !== width || _canvas.height !== height) {
            _displayWidth = width;
            _displayHeight = height;
            _canvas.width = width;
            _canvas.height = height;
        }

        const imageData = new ImageData(
            new Uint8ClampedArray(data.buffer || data),
            width,
            height
        );
        _ctx.putImageData(imageData, 0, 0);
        updateFps();
    }

    /**
     * Render YUV data to canvas using WebGL
     */
    function renderYUV(yuvData, width, height) {
        if (!_yuvCanvas) return;

        try {
            // YUV420 layout: Y plane (width*height), U plane (width*height/4), V plane (width*height/4)
            const ySize = width * height;
            const uvSize = ySize / 4;

            const yPlane = new Uint8Array(yuvData.buffer || yuvData, 0, ySize);
            const uPlane = new Uint8Array(yuvData.buffer || yuvData, ySize, uvSize);
            const vPlane = new Uint8Array(yuvData.buffer || yuvData, ySize + uvSize, uvSize);

            _yuvCanvas.drawFrame({
                format: {
                    width: width,
                    height: height,
                    chromaWidth: width / 2,
                    chromaHeight: height / 2,
                    chromaPitch: width / 2,
                    lumaPitch: width,
                    cropLeft: 0,
                    cropTop: 0,
                    cropWidth: width,
                    cropHeight: height,
                    bitDepth: 8
                },
                planes: [
                    { data: yPlane, stride: width },
                    { data: uPlane, stride: width / 2 },
                    { data: vPlane, stride: width / 2 }
                ],
                time: { low: Date.now() % 0x100000000, high: 0 }
            });
        } catch (e) {
            console.error('[RDVideo] YUV render failed:', e);
            // Fallback to 2D canvas
            const ySize = width * height;
            const yPlane = new Uint8Array(yuvData.buffer || yuvData, 0, ySize);
            renderYUVToRGBA(yPlane, width, height);
        }
    }

    /**
     * Fallback YUV to RGBA conversion using 2D canvas
     */
    function renderYUVToRGBA(yPlane, width, height) {
        const imageData = _ctx.createImageData(width, height);
        const data = imageData.data;
        
        for (let i = 0; i < width * height; i++) {
            const y = yPlane[i];
            const idx = i * 4;
            // Simple grayscale (no U/V in this fallback)
            data[idx] = y;
            data[idx + 1] = y;
            data[idx + 2] = y;
            data[idx + 3] = 255;
        }
        
        _ctx.putImageData(imageData, 0, 0);
    }

    /**
     * Handle incoming VideoFrame from the session
     * @param {Object} videoFrame - Decoded protobuf VideoFrame
     */
    function handleVideoFrame(videoFrame) {
        if (!videoFrame) return;

        const display = videoFrame.display || 0;

        // Check for VP9 frames
        if (videoFrame.vp9s && videoFrame.vp9s.frames && videoFrame.vp9s.frames.length > 0) {
            const codec = CODEC.VP9;
            if (_currentCodec !== codec) initDecoder(codec);
            processEncodedFrames(videoFrame.vp9s.frames, codec);
            return;
        }

        // Check for VP8 frames
        if (videoFrame.vp8s && videoFrame.vp8s.frames && videoFrame.vp8s.frames.length > 0) {
            const codec = CODEC.VP8;
            if (_currentCodec !== codec) initDecoder(codec);
            processEncodedFrames(videoFrame.vp8s.frames, codec);
            return;
        }

        // Check for AV1 frames
        if (videoFrame.av1s && videoFrame.av1s.frames && videoFrame.av1s.frames.length > 0) {
            const codec = CODEC.AV1;
            if (_currentCodec !== codec) initDecoder(codec);
            processEncodedFrames(videoFrame.av1s.frames, codec);
            return;
        }

        // Check for raw RGB
        if (videoFrame.rgb) {
            // RGB data comes as the raw protobuf payload after the RGB message
            // In practice, the raw data is appended after the VideoFrame message
            console.log('[RDVideo] RGB frame received (not fully supported)');
            return;
        }

        // Check for raw YUV
        if (videoFrame.yuv) {
            // Similar to RGB, YUV data is appended
            console.log('[RDVideo] YUV frame received (not fully supported)');
            return;
        }

        // H264/H265 not supported in browser without WASM decoders
        if (videoFrame.h264s) {
            console.log('[RDVideo] H.264 frame received (not supported in browser)');
        }
        if (videoFrame.h265s) {
            console.log('[RDVideo] H.265 frame received (not supported in browser)');
        }
    }

    /**
     * Process encoded video frames through the decoder
     */
    function processEncodedFrames(frames, codec) {
        const decoder = _decoders[codec];
        if (!decoder) {
            console.warn('[RDVideo] No decoder for codec:', codec);
            return;
        }

        for (const frame of frames) {
            if (!frame.data || frame.data.byteLength === 0) continue;

            try {
                if (decoder.processFrame) {
                    // Custom fallback decoder
                    // Try to determine frame dimensions from data
                    // VP9 bitstream: first bytes contain frame header
                    // For now, just log
                    _frameCount++;
                } else if (decoder instanceof OGVDecoderVideoVP9 || 
                           decoder instanceof OGVDecoderVideoVP8 ||
                           decoder instanceof OGVDecoderVideoAV1) {
                    // ogvjs native decoder
                    decoder.processHeader(frame.data);
                    // Check if decoder has loaded header info for dimensions
                }
            } catch (e) {
                console.error('[RDVideo] Frame decode error:', e);
            }
        }
    }

    /**
     * Process raw RGBA data (from fallback/secondary path)
     * @param {Uint8Array} rgba - RGBA pixel data
     * @param {number} width - Frame width
     * @param {number} height - Frame height
     */
    function processRGBA(rgba, width, height) {
        if (_canvas.width !== width || _canvas.height !== height) {
            _canvas.width = width;
            _canvas.height = height;
            _displayWidth = width;
            _displayHeight = height;
        }

        const imageData = new ImageData(
            new Uint8ClampedArray(rgba.buffer || rgba),
            width,
            height
        );
        _ctx.putImageData(imageData, 0, 0);
        updateFps();
    }

    /**
     * Update FPS counter
     */
    function updateFps() {
        _frameCount++;
        _fpsCounter++;
        const now = performance.now();
        if (now - _fpsTime >= 1000) {
            _currentFps = _fpsCounter;
            _fpsCounter = 0;
            _fpsTime = now;
        }
    }

    /**
     * Get current FPS
     */
    function getFps() {
        return _currentFps;
    }

    /**
     * Resize the canvas
     */
    function resize(width, height) {
        if (_canvas) {
            _canvas.width = width;
            _canvas.height = height;
        }
    }

    /**
     * Get current display dimensions
     */
    function getDisplaySize() {
        return { width: _displayWidth, height: _displayHeight };
    }

    /**
     * Take a screenshot of the current canvas
     */
    function screenshot() {
        if (!_canvas) return null;
        return _canvas.toDataURL('image/png');
    }

    return {
        CODEC,
        init,
        handleVideoFrame,
        processRGBA,
        resize,
        getDisplaySize,
        getFps,
        screenshot,
        get canvas() { return _canvas; }
    };
})();