Build a voice agent in a browser

// server: POST /api/realtime-ticket
export async function POST(request) {
  const viewer = await authenticate(request); // your own session check
  if (!viewer) return new Response("Unauthorized", { status: 401 });

  const response = await fetch("https://api.eesi.ai/v1/realtime/ticket", {
    method: "POST",
    headers: { Authorization: `Bearer ${process.env.EESI_API_KEY}` },
  });

  if (!response.ok) {
    return new Response("Could not mint a ticket", { status: 502 });
  }

  // { ticket: "rt_...", expires_in: 30 }
  return Response.json(await response.json());
}

const WIRE_RATE = 16000;

const WORKLET = URL.createObjectURL(
  new Blob(
    [
      `class PcmCapture extends AudioWorkletProcessor {
         process(inputs) {
           const channel = inputs[0] && inputs[0][0];
           if (channel && channel.length) {
             // Copy: the engine reuses the input buffer between quanta.
             const frame = new Float32Array(channel.length);
             frame.set(channel);
             this.port.postMessage(frame, [frame.buffer]);
           }
           return true;
         }
       }
       registerProcessor("pcm-capture", PcmCapture);`,
    ],
    { type: "text/javascript" },
  ),
);

function resample(samples, from, to) {
  if (from === to) return samples;
  const ratio = from / to;
  const out = new Float32Array(Math.floor(samples.length / ratio));
  for (let i = 0; i < out.length; i += 1) {
    const at = i * ratio;
    const low = Math.floor(at);
    const high = Math.min(low + 1, samples.length - 1);
    out[i] = samples[low] + (samples[high] - samples[low]) * (at - low);
  }
  return out;
}

function toBase64Pcm16(samples) {
  const pcm = new Int16Array(samples.length);
  for (let i = 0; i < samples.length; i += 1) {
    const clamped = Math.max(-1, Math.min(1, samples[i]));
    pcm[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
  }
  const bytes = new Uint8Array(pcm.buffer);
  let binary = "";
  for (let i = 0; i < bytes.length; i += 1) binary += String.fromCharCode(bytes[i]);
  return btoa(binary);
}

async function startMicrophone(onChunk) {
  const stream = await navigator.mediaDevices.getUserMedia({
    audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
  });
  const context = new AudioContext({ sampleRate: WIRE_RATE });
  await context.audioWorklet.addModule(WORKLET);

  // ~100 ms of audio per message, measured at whatever rate we actually got.
  const chunkSamples = Math.round(context.sampleRate / 10);
  let pending = [];
  let pendingSamples = 0;

  const worklet = new AudioWorkletNode(context, "pcm-capture");
  worklet.port.onmessage = (event) => {
    pending.push(event.data);
    pendingSamples += event.data.length;
    if (pendingSamples < chunkSamples) return;

    const merged = new Float32Array(pendingSamples);
    let offset = 0;
    for (const part of pending) {
      merged.set(part, offset);
      offset += part.length;
    }
    pending = [];
    pendingSamples = 0;
    onChunk(toBase64Pcm16(resample(merged, context.sampleRate, WIRE_RATE)));
  };

  context.createMediaStreamSource(stream).connect(worklet);
  return () => {
    for (const track of stream.getTracks()) track.stop();
    context.close();
  };
}

function createPlayer() {
  let context = null;
  let playhead = 0;
  const scheduled = new Set();

  return {
    // Call from a click handler: the autoplay policy requires a user gesture.
    init() {
      context ??= new AudioContext({ sampleRate: WIRE_RATE });
      return context.resume();
    },

    enqueue(base64) {
      if (!context) return;
      const binary = atob(base64);
      const bytes = new Uint8Array(binary.length);
      for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i);
      const pcm = new Int16Array(bytes.buffer);

      const buffer = context.createBuffer(1, pcm.length, WIRE_RATE);
      const channel = buffer.getChannelData(0);
      for (let i = 0; i < pcm.length; i += 1) channel[i] = pcm[i] / 0x8000;

      const source = context.createBufferSource();
      source.buffer = buffer;
      source.connect(context.destination);
      // A small lead the first time, so the first chunk is not already late.
      const startAt = Math.max(playhead, context.currentTime + 0.06);
      source.start(startAt);
      playhead = startAt + buffer.duration;
      scheduled.add(source);
      source.onended = () => scheduled.delete(source);
    },

    // Barge-in: drop what is queued rather than draining it.
    flush() {
      for (const source of scheduled) {
        try {
          source.stop();
        } catch {
          // already finished
        }
      }
      scheduled.clear();
      playhead = context ? context.currentTime : 0;
    },
  };
}

const MODEL = "nur-realtime-v1";
const player = createPlayer();

const state = {
  socket: null,
  stopMicrophone: null,
  sessionId: null,
  resumeToken: null,
  established: false,
  attempt: 0,
  intentional: false,
  limit: null,
  userTurn: "",
  assistantTurn: "",
};

async function connect({ resume = false } = {}) {
  const { ticket } = await fetch("/api/realtime-ticket", { method: "POST" })
    .then((response) => response.json());

  const url = new URL("wss://api.eesi.ai/v1/realtime");
  url.searchParams.set("model", MODEL);
  url.searchParams.set("source", "api");
  url.searchParams.set("token", ticket);
  if (resume && state.sessionId && state.resumeToken) {
    url.searchParams.set("resume", state.sessionId);
    url.searchParams.set("resume_token", state.resumeToken);
  }

  const socket = new WebSocket(url);
  state.socket = socket;

  socket.onopen = async () => {
    socket.send(
      JSON.stringify({
        type: "session.update",
        session: {
          type: "realtime",
          instructions: "You are warm and brief. Two sentences at most.",
          // The voice goes here and nowhere else.
          audio: { output: { voice: "ev_1a2b3c4d" } },
        },
      }),
    );

    // Capture starts once and outlives every reconnect, so the frames must
    // go through `state.socket` — not through the `socket` this closure
    // captured. See the warning below.
    state.stopMicrophone ??= await startMicrophone((chunk) => {
      if (state.socket?.readyState !== WebSocket.OPEN) return;
      state.socket.send(
        JSON.stringify({ type: "input_audio_buffer.append", audio: chunk }),
      );
    });
  };

  socket.onmessage = (message) => handleEvent(JSON.parse(message.data));
  socket.onclose = (event) => handleClose(event);
}

function handleEvent(event) {
  switch (event.type) {
    case "session.created": {
      state.established = true;
      state.attempt = 0;
      // Read the rates the gateway accepts rather than assuming them. An
      // absent eesi_protocol_version means a gateway predating versioning —
      // treat it as version 0, never as an empty string.
      const rates = event.capabilities?.audio_input_rates ?? [WIRE_RATE];
      if (!rates.includes(WIRE_RATE)) {
        console.warn(`This gateway accepts ${rates.join(", ")} Hz, not ${WIRE_RATE}.`);
      }
      break;
    }

    case "eesi.session": {
      if (event.limit) {
        // A limit reached mid-call. Resumption is withdrawn, so drop the
        // handle: redialling would be refused for the same reason.
        state.limit = event.limit;
        state.sessionId = null;
        state.resumeToken = null;
      } else {
        state.sessionId = event.session_id;
        state.resumeToken = event.resume_token;
      }
      break;
    }

    case "input_audio_buffer.speech_started":
      // Barge-in. Stop playing NOW — do not drain the buffer.
      player.flush();
      state.assistantTurn = "";
      break;

    case "conversation.item.input_audio_transcription.delta":
      // Cumulative: this carries the whole utterance so far and REPLACES the
      // open entry. Appending gives "hello hello there hello there world".
      state.userTurn = event.delta;
      render();
      break;

    case "conversation.item.input_audio_transcription.completed":
      commitTurn("user", event.transcript || state.userTurn);
      state.userTurn = "";
      break;

    case "response.output_audio.delta":
      player.enqueue(event.delta);
      break;

    case "response.output_audio_transcript.done":
      // One per SEGMENT of the reply, not one per turn. Keeping only the last
      // truncates every multi-segment answer, so join them.
      state.assistantTurn = [state.assistantTurn, event.transcript]
        .filter(Boolean)
        .join(" ");
      render();
      break;

    case "response.done": {
      // Both fields are nested under `response`, and either one alone means
      // barge-in: some interrupted turns come back `incomplete`, or with no
      // status at all, and only `status_details.reason` says what happened.
      const outcome = event.response ?? {};
      const interrupted =
        outcome.status === "cancelled" ||
        outcome.status_details?.reason === "turn_detected";
      commitTurn("assistant", state.assistantTurn, interrupted);
      state.assistantTurn = "";
      break;
    }

    case "error":
      // Nested under `error`, and the gateway spells the code `code` while
      // OpenAI spells it `type`.
      console.error(event.error?.code ?? event.error?.type, event.error?.message);
      break;
  }
}

const FATAL = new Set([1002, 1003, 1008]);
const MAX_ATTEMPTS = 6;

function backoffMs(attempt) {
  // Equal jitter: half the window fixed so a redeploy is not hammered while it
  // comes back, half random so every dropped session does not return in step.
  const window = Math.min(8000, 1000 * 2 ** attempt);
  return Math.round(window / 2 + Math.random() * (window / 2));
}

function handleClose(event) {
  if (state.intentional || !state.established) return;

  if (event.code === 1008) {
    // A limit or a rejection. If a limit frame arrived first, it says which.
    show(state.limit?.message ?? event.reason ?? "The session was refused.");
    return;
  }
  if (event.code === 1011) {
    show("The realtime service is unavailable. Try again in a few minutes.");
    return;
  }
  if (event.code === 1000 || FATAL.has(event.code) || state.attempt >= MAX_ATTEMPTS) {
    show("The conversation ended.");
    return;
  }

  const delay = backoffMs(state.attempt);
  state.attempt += 1;
  // A fresh ticket per redial: the old one is spent, and 30 seconds is
  // shorter than the shortest backoff anyway.
  setTimeout(() => connect({ resume: true }), delay);
}

document.querySelector("#talk").addEventListener("click", async () => {
  await player.init();
  await connect();
});

document.querySelector("#hang-up").addEventListener("click", () => {
  state.intentional = true;
  state.stopMicrophone?.();
  state.socket?.close(1000, "hang up");
  player.flush();
});