Building a Production Voice AI Agent: From Whisper to TTS

Voice AI has gone from novelty to expectation. Here's how to build one that feels fast enough to be useful.

The Pipeline

User speaks -> Whisper (STT) -> LLM -> TTS -> User hears

Each step adds latency. A good voice agent targets under 1.5s end-to-end.

Speech-to-Text with Whisper

import openai, sounddevice as sd, numpy as np
import scipy.io.wavfile as wav
import tempfile, os

client = openai.OpenAI()

def record_audio(duration: int = 5, sample_rate: int = 16000) -> np.ndarray:
    audio = sd.rec(int(duration * sample_rate), samplerate=sample_rate, channels=1)
    sd.wait()
    return audio

def transcribe(audio: np.ndarray, sample_rate: int = 16000) -> str:
    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
        wav.write(f.name, sample_rate, audio)
        with open(f.name, "rb") as audio_file:
            transcript = client.audio.transcriptions.create(
                model="whisper-1",
                file=audio_file,
                language="en"
            )
        os.unlink(f.name)
    return transcript.text

Streaming LLM Response

Don't wait for the full response before starting TTS. Stream in sentence chunks:

import re

def stream_sentences(prompt: str):
    stream = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}],
        stream=True
    )

    buffer = ""
    for chunk in stream:
        delta = chunk.choices[0].delta.content or ""
        buffer += delta

        sentences = re.split(r'(?<=[.!?])\s+', buffer)
        for sentence in sentences[:-1]:
            if sentence.strip():
                yield sentence.strip()
        buffer = sentences[-1]

    if buffer.strip():
        yield buffer.strip()

Text-to-Speech

import pygame, io

def speak(text: str, voice: str = "nova") -> None:
    response = client.audio.speech.create(
        model="tts-1",
        voice=voice,
        input=text,
        speed=1.1  # Slightly faster feels more natural
    )

    audio_data = io.BytesIO(response.content)
    pygame.mixer.init()
    pygame.mixer.music.load(audio_data)
    pygame.mixer.music.play()
    while pygame.mixer.music.get_busy():
        pygame.time.wait(100)

Latency Optimization: Parallel TTS

Start speaking the first sentence while the LLM generates the rest:

import threading, queue

def voice_pipeline(user_input: str):
    tts_queue = queue.Queue()

    def tts_worker():
        while True:
            text = tts_queue.get()
            if text is None:
                break
            speak(text)
            tts_queue.task_done()

    worker = threading.Thread(target=tts_worker)
    worker.start()

    for sentence in stream_sentences(user_input):
        print(f"AI: {sentence}")
        tts_queue.put(sentence)

    tts_queue.put(None)
    worker.join()

Voice Activity Detection

Don't use fixed-duration recordings. Detect when the user stops speaking:

import webrtcvad

def record_with_vad(sample_rate: int = 16000, max_silence_ms: int = 1000) -> bytes:
    vad = webrtcvad.Vad(aggressiveness=2)
    chunk_ms = 30
    chunk_size = int(sample_rate * chunk_ms / 1000)

    frames = []
    silence_frames = 0
    max_silence = max_silence_ms // chunk_ms
    speaking = False

    with sd.RawInputStream(samplerate=sample_rate, channels=1, dtype='int16') as stream:
        while True:
            data, _ = stream.read(chunk_size)
            is_speech = vad.is_speech(bytes(data), sample_rate)

            if is_speech:
                speaking = True
                silence_frames = 0
                frames.append(bytes(data))
            elif speaking:
                silence_frames += 1
                frames.append(bytes(data))
                if silence_frames >= max_silence:
                    break

    return b"".join(frames)

Latency Benchmarks

Typical latency breakdown:

• Whisper transcription: 300-500ms
• LLM first token: 200-400ms
• TTS first chunk: 200-400ms
• Total to first audio: ~700-1300ms

With streaming, users hear the first word within ~1s. That's the threshold where it feels responsive.

Production Considerations

• Use tts-1 not tts-1-hd -- 2x faster with minimal quality loss for voice chat
• Cache common responses: greetings, confirmations, error messages
• Monitor for hallucinated audio -- TTS will speak whatever the LLM outputs
• Add a soft "thinking" sound while waiting to reduce perceived latency