// Audio pipeline using WebCodecs AudioDecoder. // // Wire format (from libs/hbb_common/protos/message.proto): // Misc{audio_format: {sample_rate, channels}} sent once at session start // Message{audio_frame: {data: }} periodic // // We feed each opus frame into AudioDecoder, get back an AudioData with PCM // samples, copy into an AudioBufferSourceNode, and schedule it on a single // AudioContext "playhead" timeline so consecutive packets play seamlessly. import { hbb } from "../proto/generated.js"; export class AudioPipeline { private decoder: AudioDecoder | null = null; private ctx: AudioContext | null = null; private playhead = 0; // next absolute scheduling time, in ctx seconds private timestamp = 0; // monotonic counter for EncodedAudioChunk private muted = false; private gain: GainNode | null = null; /** Configure on the first AudioFormat message. Throws if WebCodecs unavailable. */ configure(format: hbb.IAudioFormat): void { if (typeof AudioDecoder === "undefined") { throw new Error("WebCodecs AudioDecoder unavailable. Open via http://localhost or https:// — secure-context only."); } const sampleRate = format.sample_rate || 48000; const channels = format.channels || 2; if (this.decoder) this.decoder.close(); this.ctx = new AudioContext({ sampleRate, latencyHint: "interactive" }); this.gain = this.ctx.createGain(); this.gain.connect(this.ctx.destination); this.playhead = this.ctx.currentTime + 0.05; // 50ms initial buffer this.timestamp = 0; this.decoder = new AudioDecoder({ output: (data) => this.onAudioData(data), error: (e) => console.error("[rustdesk-web] AudioDecoder error:", e), }); this.decoder.configure({ codec: "opus", sampleRate, numberOfChannels: channels, }); } /** Feed one opus packet (raw bytes from AudioFrame.data). */ pushFrame(data: Uint8Array): void { if (!this.decoder || data.length === 0 || this.muted) return; try { const chunk = new EncodedAudioChunk({ type: "key", // opus is all-key per packet timestamp: this.timestamp, data, }); this.timestamp += 20_000; // ~20ms per opus frame in microseconds this.decoder.decode(chunk); } catch (e) { console.error("[rustdesk-web] audio decode failed:", e); } } /** Resume the AudioContext after a user gesture (some browsers gate * audio playback to the first interaction). */ async resume(): Promise { if (this.ctx && this.ctx.state === "suspended") { await this.ctx.resume(); } } setMuted(muted: boolean): void { this.muted = muted; if (this.gain && this.ctx) { this.gain.gain.setValueAtTime(muted ? 0 : 1, this.ctx.currentTime); } } isMuted(): boolean { return this.muted; } close(): void { if (this.decoder) { try { this.decoder.close(); } catch { /* ignore */ } this.decoder = null; } if (this.ctx) { try { this.ctx.close(); } catch { /* ignore */ } this.ctx = null; } } private onAudioData(data: AudioData): void { if (!this.ctx || !this.gain) { data.close(); return; } try { const numChannels = data.numberOfChannels; const numFrames = data.numberOfFrames; const sampleRate = data.sampleRate; const buffer = this.ctx.createBuffer(numChannels, numFrames, sampleRate); // WebCodecs opus output is typically `f32` (interleaved), not // `f32-planar`. Detect the format and deinterleave when needed. const format = data.format || "f32"; if (format.endsWith("-planar")) { // Each plane is one channel — copy directly. for (let ch = 0; ch < numChannels; ch++) { data.copyTo(buffer.getChannelData(ch), { planeIndex: ch }); } } else { // Interleaved: copy all samples, then deinterleave into the // AudioBuffer's planar channels. const interleaved = new Float32Array(numChannels * numFrames); data.copyTo(interleaved, { planeIndex: 0 }); for (let ch = 0; ch < numChannels; ch++) { const dest = buffer.getChannelData(ch); for (let i = 0; i < numFrames; i++) { dest[i] = interleaved[i * numChannels + ch]!; } } } const src = this.ctx.createBufferSource(); src.buffer = buffer; src.connect(this.gain); // Schedule on a sliding playhead so consecutive packets are gap-less. // If we've fallen behind real time (network hiccup / decoder stall), // re-anchor to "now + 50ms" to avoid scheduling a runaway pile-up. const now = this.ctx.currentTime; if (this.playhead < now) { this.playhead = now + 0.05; } src.start(this.playhead); this.playhead += numFrames / sampleRate; } catch (e) { console.error("[rustdesk-web] audio render failed:", e); } finally { data.close(); } } }