rustdesk-server/web_client/src/decode/audio.ts

// Audio pipeline using WebCodecs AudioDecoder.
//
// Wire format (from libs/hbb_common/protos/message.proto):
//   Misc{audio_format: {sample_rate, channels}}    sent once at session start
//   Message{audio_frame: {data: <opus frame>}}     periodic
//
// We feed each opus frame into AudioDecoder, get back an AudioData with PCM
// samples, copy into an AudioBufferSourceNode, and schedule it on a single
// AudioContext "playhead" timeline so consecutive packets play seamlessly.

import { hbb } from "../proto/generated.js";

export class AudioPipeline {
  private decoder: AudioDecoder | null = null;
  private ctx: AudioContext | null = null;
  private playhead = 0;          // next absolute scheduling time, in ctx seconds
  private timestamp = 0;         // monotonic counter for EncodedAudioChunk
  private muted = false;
  private gain: GainNode | null = null;

  /** Configure on the first AudioFormat message. Throws if WebCodecs unavailable. */
  configure(format: hbb.IAudioFormat): void {
    if (typeof AudioDecoder === "undefined") {
      throw new Error("WebCodecs AudioDecoder unavailable. Open via http://localhost or https:// — secure-context only.");
    }
    const sampleRate = format.sample_rate || 48000;
    const channels = format.channels || 2;
    if (this.decoder) this.decoder.close();
    this.ctx = new AudioContext({ sampleRate, latencyHint: "interactive" });
    this.gain = this.ctx.createGain();
    this.gain.connect(this.ctx.destination);
    this.playhead = this.ctx.currentTime + 0.05; // 50ms initial buffer
    this.timestamp = 0;

    this.decoder = new AudioDecoder({
      output: (data) => this.onAudioData(data),
      error: (e) => console.error("[rustdesk-web] AudioDecoder error:", e),
    });
    this.decoder.configure({
      codec: "opus",
      sampleRate,
      numberOfChannels: channels,
    });
  }

  /** Feed one opus packet (raw bytes from AudioFrame.data). */
  pushFrame(data: Uint8Array): void {
    if (!this.decoder || data.length === 0 || this.muted) return;
    try {
      const chunk = new EncodedAudioChunk({
        type: "key",                    // opus is all-key per packet
        timestamp: this.timestamp,
        data,
      });
      this.timestamp += 20_000;          // ~20ms per opus frame in microseconds
      this.decoder.decode(chunk);
    } catch (e) {
      console.error("[rustdesk-web] audio decode failed:", e);
    }
  }

  /** Resume the AudioContext after a user gesture (some browsers gate
   *  audio playback to the first interaction). */
  async resume(): Promise<void> {
    if (this.ctx && this.ctx.state === "suspended") {
      await this.ctx.resume();
    }
  }

  setMuted(muted: boolean): void {
    this.muted = muted;
    if (this.gain && this.ctx) {
      this.gain.gain.setValueAtTime(muted ? 0 : 1, this.ctx.currentTime);
    }
  }

  isMuted(): boolean {
    return this.muted;
  }

  close(): void {
    if (this.decoder) {
      try { this.decoder.close(); } catch { /* ignore */ }
      this.decoder = null;
    }
    if (this.ctx) {
      try { this.ctx.close(); } catch { /* ignore */ }
      this.ctx = null;
    }
  }

  private onAudioData(data: AudioData): void {
    if (!this.ctx || !this.gain) {
      data.close();
      return;
    }
    try {
      const numChannels = data.numberOfChannels;
      const numFrames = data.numberOfFrames;
      const sampleRate = data.sampleRate;
      const buffer = this.ctx.createBuffer(numChannels, numFrames, sampleRate);

      // WebCodecs opus output is typically `f32` (interleaved), not
      // `f32-planar`. Detect the format and deinterleave when needed.
      const format = data.format || "f32";
      if (format.endsWith("-planar")) {
        // Each plane is one channel — copy directly.
        for (let ch = 0; ch < numChannels; ch++) {
          data.copyTo(buffer.getChannelData(ch), { planeIndex: ch });
        }
      } else {
        // Interleaved: copy all samples, then deinterleave into the
        // AudioBuffer's planar channels.
        const interleaved = new Float32Array(numChannels * numFrames);
        data.copyTo(interleaved, { planeIndex: 0 });
        for (let ch = 0; ch < numChannels; ch++) {
          const dest = buffer.getChannelData(ch);
          for (let i = 0; i < numFrames; i++) {
            dest[i] = interleaved[i * numChannels + ch]!;
          }
        }
      }

      const src = this.ctx.createBufferSource();
      src.buffer = buffer;
      src.connect(this.gain);

      // Schedule on a sliding playhead so consecutive packets are gap-less.
      // If we've fallen behind real time (network hiccup / decoder stall),
      // re-anchor to "now + 50ms" to avoid scheduling a runaway pile-up.
      const now = this.ctx.currentTime;
      if (this.playhead < now) {
        this.playhead = now + 0.05;
      }
      src.start(this.playhead);
      this.playhead += numFrames / sampleRate;
    } catch (e) {
      console.error("[rustdesk-web] audio render failed:", e);
    } finally {
      data.close();
    }
  }
}