go2rtc/www/video-renderer.js

/**
 * VideoRenderer — Cascading VideoFrame renderer: WebGPU → WebGL2 → Canvas 2D.
 *
 * Each tier uses its own canvas to avoid context-type locking.
 * Automatically initializes on first frame and downgrades on errors.
 *
 * Usage:
 *   const renderer = new VideoRenderer(container, {cascade: 'webgpu,webgl,2d'});
 *   // in VideoDecoder output callback:
 *   renderer.draw(frame);
 *   frame.close();
 *   // cleanup:
 *   renderer.destroy();
 */

const TIER_WEBGPU = 0;
const TIER_WEBGL2 = 1;
const TIER_CANVAS2D = 2;
const TIER_NAMES = ['WebGPU', 'WebGL2', 'Canvas2D'];
const TIER_MAP = {webgpu: TIER_WEBGPU, webgl: TIER_WEBGL2, '2d': TIER_CANVAS2D};

const WGSL_VERTEX = `
struct Out { @builtin(position) pos: vec4f, @location(0) uv: vec2f }
@vertex fn main(@builtin(vertex_index) i: u32) -> Out {
    var p = array<vec2f,6>(
        vec2f( 1, 1), vec2f( 1,-1), vec2f(-1,-1),
        vec2f( 1, 1), vec2f(-1,-1), vec2f(-1, 1));
    var u = array<vec2f,6>(
        vec2f(1,0), vec2f(1,1), vec2f(0,1),
        vec2f(1,0), vec2f(0,1), vec2f(0,0));
    return Out(vec4f(p[i],0,1), u[i]);
}`;

const WGSL_FRAGMENT = `
@group(0) @binding(0) var s: sampler;
@group(0) @binding(1) var t: texture_external;
@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f {
    return textureSampleBaseClampToEdge(t, s, uv);
}`;

class WebGPUTier {
    constructor(canvas) {
        this.canvas = canvas;
        this.device = null;
        this.ctx = null;
        this.pipeline = null;
        this.sampler = null;
        this.format = null;
    }

    async init() {
        if (!navigator.gpu) { VideoRenderer.log('WebGPU: API not available'); return false; }
        try {
            VideoRenderer.log('WebGPU: requesting adapter...');
            const adapter = await navigator.gpu.requestAdapter();
            if (!adapter) { VideoRenderer.log('WebGPU: no adapter available'); return false; }
            const info = adapter.info || {};
            VideoRenderer.log('WebGPU: adapter:', info.vendor || '?', info.architecture || '?', info.description || '');

            this.device = await adapter.requestDevice();
            this.format = navigator.gpu.getPreferredCanvasFormat();

            this.ctx = this.canvas.getContext('webgpu');
            if (!this.ctx) {
                VideoRenderer.log('WebGPU: getContext("webgpu") returned null');
                this.device.destroy(); this.device = null; return false;
            }
            this.ctx.configure({device: this.device, format: this.format, alphaMode: 'opaque'});

            this.pipeline = this.device.createRenderPipeline({
                layout: 'auto',
                vertex: {
                    module: this.device.createShaderModule({code: WGSL_VERTEX}),
                    entryPoint: 'main',
                },
                fragment: {
                    module: this.device.createShaderModule({code: WGSL_FRAGMENT}),
                    entryPoint: 'main',
                    targets: [{format: this.format}],
                },
                primitive: {topology: 'triangle-list'},
            });

            this.sampler = this.device.createSampler({magFilter: 'linear', minFilter: 'linear'});
            VideoRenderer.log('WebGPU: initialized, format:', this.format);
            return true;
        } catch (e) {
            VideoRenderer.log('WebGPU: init failed:', e.message || e);
            this.device = null; this.ctx = null;
            return false;
        }
    }

    draw(frame, w, h) {
        if (this.canvas.width !== w || this.canvas.height !== h) {
            this.canvas.width = w; this.canvas.height = h;
        }
        const bind = this.device.createBindGroup({
            layout: this.pipeline.getBindGroupLayout(0),
            entries: [
                {binding: 0, resource: this.sampler},
                {binding: 1, resource: this.device.importExternalTexture({source: frame})},
            ],
        });
        const enc = this.device.createCommandEncoder();
        const pass = enc.beginRenderPass({colorAttachments: [{
            view: this.ctx.getCurrentTexture().createView(),
            loadOp: 'clear', storeOp: 'store',
        }]});
        pass.setPipeline(this.pipeline);
        pass.setBindGroup(0, bind);
        pass.draw(6);
        pass.end();
        this.device.queue.submit([enc.finish()]);
    }

    destroy() {
        try { if (this.device) this.device.destroy(); } catch {}
        this.device = null; this.ctx = null; this.pipeline = null; this.sampler = null;
    }
}

const GLSL_VERTEX = `#version 300 es
out vec2 vUV;
void main() {
    float x = float(gl_VertexID & 1) * 2.0;
    float y = float(gl_VertexID & 2);
    vUV = vec2(x, 1.0 - y);
    gl_Position = vec4(vUV * 2.0 - 1.0, 0.0, 1.0);
    vUV.y = 1.0 - vUV.y;
}`;

const GLSL_FRAGMENT = `#version 300 es
precision mediump float;
in vec2 vUV;
uniform sampler2D uTex;
out vec4 c;
void main() { c = texture(uTex, vUV); }`;

class WebGL2Tier {
    constructor(canvas) {
        this.canvas = canvas;
        this.gl = null;
        this.program = null;
        this.texture = null;
        this.lastW = 0;
        this.lastH = 0;
    }

    init() {
        try {
            this.gl = this.canvas.getContext('webgl2', {
                alpha: false, desynchronized: true, antialias: false,
                powerPreference: 'high-performance',
            });
        } catch (e) { VideoRenderer.log('WebGL2: getContext threw:', e.message || e); this.gl = null; }
        if (!this.gl) { VideoRenderer.log('WebGL2: not available'); return false; }

        const gl = this.gl;

        const vs = gl.createShader(gl.VERTEX_SHADER);
        gl.shaderSource(vs, GLSL_VERTEX);
        gl.compileShader(vs);
        if (!gl.getShaderParameter(vs, gl.COMPILE_STATUS)) {
            VideoRenderer.log('WebGL2: vertex shader error:', gl.getShaderInfoLog(vs));
            gl.deleteShader(vs); this.gl = null; return false;
        }

        const fs = gl.createShader(gl.FRAGMENT_SHADER);
        gl.shaderSource(fs, GLSL_FRAGMENT);
        gl.compileShader(fs);
        if (!gl.getShaderParameter(fs, gl.COMPILE_STATUS)) {
            VideoRenderer.log('WebGL2: fragment shader error:', gl.getShaderInfoLog(fs));
            gl.deleteShader(vs); gl.deleteShader(fs); this.gl = null; return false;
        }

        this.program = gl.createProgram();
        gl.attachShader(this.program, vs);
        gl.attachShader(this.program, fs);
        gl.linkProgram(this.program);
        gl.deleteShader(vs);
        gl.deleteShader(fs);

        if (!gl.getProgramParameter(this.program, gl.LINK_STATUS)) {
            VideoRenderer.log('WebGL2: program link error:', gl.getProgramInfoLog(this.program));
            gl.deleteProgram(this.program);
            this.gl = null; this.program = null; return false;
        }

        gl.useProgram(this.program);
        this.texture = gl.createTexture();
        gl.activeTexture(gl.TEXTURE0);
        gl.bindTexture(gl.TEXTURE_2D, this.texture);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
        gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
        gl.uniform1i(gl.getUniformLocation(this.program, 'uTex'), 0);

        const dbg = gl.getExtension('WEBGL_debug_renderer_info');
        const gpuName = dbg ? gl.getParameter(dbg.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER);
        VideoRenderer.log('WebGL2: initialized, GPU:', gpuName);
        return true;
    }

    draw(frame, w, h) {
        const gl = this.gl;
        if (w !== this.lastW || h !== this.lastH) {
            this.canvas.width = w; this.canvas.height = h;
            this.lastW = w; this.lastH = h;
            gl.viewport(0, 0, w, h);
        }
        gl.useProgram(this.program);
        gl.bindTexture(gl.TEXTURE_2D, this.texture);
        gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
        gl.drawArrays(gl.TRIANGLES, 0, 6);
    }

    destroy() {
        try {
            if (this.gl) {
                if (this.texture) this.gl.deleteTexture(this.texture);
                if (this.program) this.gl.deleteProgram(this.program);
                const ext = this.gl.getExtension('WEBGL_lose_context');
                if (ext) ext.loseContext();
            }
        } catch {}
        this.gl = null; this.program = null; this.texture = null;
    }
}

class Canvas2DTier {
    constructor(canvas) {
        this.canvas = canvas;
        this.ctx = null;
    }

    init() { return true; }

    draw(frame, w, h) {
        if (this.canvas.width !== w) this.canvas.width = w;
        if (this.canvas.height !== h) this.canvas.height = h;
        if (!this.ctx) this.ctx = this.canvas.getContext('2d');
        if (this.ctx) this.ctx.drawImage(frame, 0, 0, w, h);
    }

    destroy() { this.ctx = null; }
}

export class VideoRenderer {
    static log(msg, ...args) {
        console.debug('[WebCodecs]', msg, ...args);
    }

    /**
     * @param {HTMLElement} container — element to insert canvases into
     * @param {object} [options]
     * @param {string} [options.cascade='webgpu,webgl,2d'] — comma-separated tier names
     * @param {string} [options.canvasStyle] — CSS for created canvases
     */
    constructor(container, options = {}) {
        const cascade = options.cascade || 'webgpu,webgl,2d';
        const canvasStyle = options.canvasStyle || 'display:block;max-width:100%;max-height:100%';

        this._container = container;
        this._allowedTiers = cascade.split(',').map(s => TIER_MAP[s.trim()]).filter(t => t !== undefined);
        this._tier = -1; // not initialized
        this._initPromise = null;
        this._destroyed = false;

        const makeCanvas = () => {
            const c = document.createElement('canvas');
            c.style.cssText = canvasStyle;
            return c;
        };

        // Each tier gets its own canvas
        this._tiers = {
            [TIER_WEBGPU]: new WebGPUTier(makeCanvas()),
            [TIER_WEBGL2]: new WebGL2Tier(makeCanvas()),
            [TIER_CANVAS2D]: new Canvas2DTier(makeCanvas()),
        };

        this._activeCanvas = null;

        // WebGL2 context loss/restore
        const gl2Canvas = this._tiers[TIER_WEBGL2].canvas;
        gl2Canvas.addEventListener('webglcontextlost', (e) => {
            e.preventDefault();
            if (this._tier === TIER_WEBGL2) {
                this._tiers[TIER_WEBGL2].destroy();
                this._tier = this._nextAllowed(TIER_CANVAS2D);
                this._swapCanvas(this._tier);
                VideoRenderer.log('WebGL2 context lost, fallback to', TIER_NAMES[this._tier]);
            }
        });
        gl2Canvas.addEventListener('webglcontextrestored', () => {
            if (this._tier > TIER_WEBGL2 && this._allowedTiers.includes(TIER_WEBGL2)) {
                if (this._tiers[TIER_WEBGL2].init()) {
                    this._tier = TIER_WEBGL2;
                    this._swapCanvas(TIER_WEBGL2);
                    VideoRenderer.log('WebGL2 context restored');
                }
            }
        });

        VideoRenderer.log('cascade:', this._allowedTiers.map(t => TIER_NAMES[t]).join(' → '));
    }

    /** @returns {string} current tier name or 'none' */
    get currentTier() {
        return this._tier >= 0 ? TIER_NAMES[this._tier] : 'none';
    }

    /** @returns {HTMLCanvasElement|null} currently active canvas */
    get canvas() {
        return this._activeCanvas;
    }

    /**
     * Draw a VideoFrame. Does NOT close the frame — caller is responsible.
     * @param {VideoFrame} frame
     */
    draw(frame) {
        if (this._destroyed) return;
        const w = frame.displayWidth, h = frame.displayHeight;
        if (w === 0 || h === 0) return;

        // Already initialized — fast path
        if (this._tier >= 0) {
            try {
                this._tiers[this._tier].draw(frame, w, h);
            } catch (e) {
                this._downgrade(e);
                try {
                    this._tiers[this._tier].draw(frame, w, h);
                } catch (e2) {
                    this._tier = TIER_CANVAS2D;
                    this._swapCanvas(TIER_CANVAS2D);
                    VideoRenderer.log('renderer error, fallback to Canvas2D:', e2.message || e2);
                    this._tiers[TIER_CANVAS2D].draw(frame, w, h);
                }
            }
            return;
        }

        // Async init in progress — use Canvas2D temporarily
        if (this._initPromise) {
            this._tiers[TIER_CANVAS2D].draw(frame, w, h);
            if (!this._activeCanvas) this._swapCanvas(TIER_CANVAS2D);
            return;
        }

        // First frame — initialize
        VideoRenderer.log('first frame, resolution:', w + 'x' + h, '— initializing...');
        const first = this._allowedTiers[0] ?? TIER_CANVAS2D;

        if (first === TIER_WEBGPU && navigator.gpu) {
            this._initPromise = this._tiers[TIER_WEBGPU].init().then(ok => {
                if (this._destroyed) return;
                if (ok) {
                    this._tier = TIER_WEBGPU;
                    this._swapCanvas(TIER_WEBGPU);
                } else {
                    this._initSync(TIER_WEBGL2);
                }
                VideoRenderer.log('renderer ready:', TIER_NAMES[this._tier]);
                this._initPromise = null;
            });
            // Render first frames with Canvas2D while WebGPU inits
            this._tiers[TIER_CANVAS2D].draw(frame, w, h);
            this._swapCanvas(TIER_CANVAS2D);
        } else {
            this._initSync(first === TIER_WEBGPU ? TIER_WEBGL2 : first);
            VideoRenderer.log('renderer ready:', TIER_NAMES[this._tier]);
            this._tiers[this._tier].draw(frame, w, h);
            this._swapCanvas(this._tier);
        }
    }

    destroy() {
        this._destroyed = true;
        for (const tier of Object.values(this._tiers)) {
            tier.destroy();
            if (tier.canvas.parentElement) tier.canvas.remove();
        }
        this._activeCanvas = null;
        this._tier = -1;
    }

    _nextAllowed(minTier) {
        for (const t of this._allowedTiers) { if (t >= minTier) return t; }
        return TIER_CANVAS2D;
    }

    _initSync(startTier) {
        const tryGL = this._allowedTiers.includes(TIER_WEBGL2);
        if (startTier <= TIER_WEBGL2 && tryGL && this._tiers[TIER_WEBGL2].init()) {
            this._tier = TIER_WEBGL2;
        } else {
            this._tier = this._nextAllowed(TIER_CANVAS2D);
        }
    }

    _downgrade(error) {
        const oldTier = this._tier;
        this._tier = this._nextAllowed(oldTier + 1);
        if (this._tier === TIER_WEBGL2 && !this._tiers[TIER_WEBGL2].gl && !this._tiers[TIER_WEBGL2].init()) {
            this._tier = this._nextAllowed(TIER_CANVAS2D);
        }
        this._swapCanvas(this._tier);
        VideoRenderer.log('renderer error, downgrade:', TIER_NAMES[oldTier], '→', TIER_NAMES[this._tier], error.message || error);
    }

    _swapCanvas(tier) {
        const target = this._tiers[tier]?.canvas;
        if (!target || target === this._activeCanvas) return;

        if (this._activeCanvas) {
            this._activeCanvas.style.display = 'none';
        }

        if (target.parentElement !== this._container) {
            // Insert before the first hidden canvas or at the end
            const ref = this._activeCanvas || null;
            this._container.insertBefore(target, ref);
        }
        target.style.display = '';
        this._activeCanvas = target;
    }
}