Duet

// Re-time bursty TTS audio to wall clock, then hand each frame to the peer.
// PCM16 at the session's rate; one frame every frameMs.
function pacedPipe({ sampleRate = 16_000, frameMs = 100, tailSilenceMs = 1200, send }) {
  const frameSamples = Math.round((sampleRate * frameMs) / 1000);
  let pending = new Int16Array(0);
  let tailFrames = 0;

  const encode = (samples) => {
    const bytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
    return btoa(String.fromCharCode(...bytes));
  };

  const timer = setInterval(() => {
    if (pending.length >= frameSamples) {
      send(encode(pending.subarray(0, frameSamples)));
      pending = pending.slice(frameSamples);
      return;
    }
    if (tailFrames > 0 && pending.length > 0) {
      // Flush the last partial frame, padded, BEFORE the silence. Holding it
      // sends the tail of this utterance after the silence that ended it —
      // where the peer hears it as the opening of the next turn.
      const last = new Int16Array(frameSamples);
      last.set(pending, 0);
      pending = new Int16Array(0);
      send(encode(last));
      return;
    }
    if (tailFrames > 0) {
      tailFrames -= 1;
      send(encode(new Int16Array(frameSamples)));   // the silence that ends a turn
    }
    // Otherwise send nothing: an underrun mid-utterance must not end one early.
  }, frameMs);

  return {
    push(base64Delta) {
      const bytes = Uint8Array.from(atob(base64Delta), (c) => c.charCodeAt(0));
      const grown = new Int16Array(pending.length + bytes.byteLength / 2);
      grown.set(pending, 0);
      grown.set(new Int16Array(bytes.buffer), pending.length);
      pending = grown;
      tailFrames = 0;
    },
    endOfUtterance() {
      tailFrames = Math.ceil(tailSilenceMs / frameMs);
    },
    stop() {
      clearInterval(timer);   // or the timer outlives the duet
    },
  };
}

// A's audio becomes B's microphone, and back.
const seats = { a: socketA, b: socketB };
const pipes = {
  a: pacedPipe({ send: (audio) => seats.b.send(JSON.stringify({ type: "input_audio_buffer.append", audio })) }),
  b: pacedPipe({ send: (audio) => seats.a.send(JSON.stringify({ type: "input_audio_buffer.append", audio })) }),
};

for (const [side, socket] of Object.entries(seats)) {
  socket.addEventListener("message", (message) => {
    const event = JSON.parse(message.data);
    if (event.type === "response.output_audio.delta") pipes[side].push(event.delta);
    if (event.type === "response.output_audio.done") pipes[side].endOfUtterance();
  });
}

// Somebody has to speak first, or both sides wait forever.
seats.a.send(JSON.stringify({ type: "response.create" }));

// Stop both pipes when the duet ends. Nothing else clears these timers.
const endDuet = () => { pipes.a.stop(); pipes.b.stop(); };

wss://api.eesi.ai/v1/realtime?model=nur-realtime-v1&pair_id=run-4711&source=duet