Developer docs / AI Fabric tutorials

Voice chat with an LLM

  • GPU
  • Web endpoint
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

voice_chat.py
import bladekit

app = bladekit.app("voice-chat")

# The maintained Coqui fork ("coqui-tts") supports Python 3.12, so pin the
# image interpreter and deploy from a Python 3.12 client venv (AI Fabric matches
# the client and image Python). Whisper needs ffmpeg; Coqui needs espeak-ng.
image = (
    bladekit.Image.debian_slim(python_version="3.12")
    .apt_install("ffmpeg", "espeak-ng")
    .pip_install(
        "torch", "torchaudio", "openai-whisper",
        # coqui-tts imports private transformers helpers that moved in newer
        # releases, so pin a version both Qwen and Coqui are happy with.
        "transformers==4.46.2", "accelerate", "coqui-tts",
    )
    # Small helper Coqui's model manager imports but does not declare.
    .pip_install("click")
)

_models = None


def _load_models():
    """Load all three models once per replica and keep them warm."""
    global _models
    import whisper
    from transformers import pipeline
    from TTS.api import TTS as TTSModel

    if _models is None:
        asr = whisper.load_model("base")
        llm = pipeline(
            "text-generation", model="Qwen/Qwen2.5-0.5B-Instruct",
            torch_dtype="auto", device=0,
        )
        tts = TTSModel("tts_models/en/ljspeech/tacotron2-DDC").to("cuda")
        _models = (asr, llm, tts)
    return _models


# min_replicas=1: loading three models takes longer than the cold-start wake
# window, so keep one warm replica instead of scaling to zero.
@app.web_endpoint(methods=["POST"], min_replicas=1, max_replicas=2)
@app.function(gpu="RTX-A4500", image=image, timeout=300)
def voice_chat(item: dict):
    """Voice in, voice out: Whisper STT -> Qwen LLM -> Coqui TTS.

    POST {"audio_b64": "<base64 WAV/MP3>"}  -> spoken reply as WAV bytes
    POST {"text": "..."}                    -> plain TTS of that text (handy
                                               to generate a test input WAV)
    """
    import base64, tempfile

    asr, llm, tts = _load_models()

    def speak(text: str) -> bytes:
        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
            out_path = f.name
        tts.tts_to_file(text=text, file_path=out_path)
        wav = open(out_path, "rb").read()
        print(f"[tts] {len(wav)} wav bytes")
        return wav

    # Bootstrap mode: synthesize arbitrary text (no STT/LLM involved).
    if "text" in item:
        return speak(item["text"])

    # 1. Speech -> text
    audio_bytes = base64.b64decode(item["audio_b64"])
    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
        f.write(audio_bytes)
        tmp = f.name
    question = asr.transcribe(tmp)["text"].strip()
    print(f"[stt] transcript: {question}")

    # 2. Text -> reply (chat format so the instruct model answers
    #    instead of just continuing the sentence)
    messages = [{"role": "user", "content": question}]
    reply = llm(messages, max_new_tokens=80)[0]["generated_text"][-1]["content"].strip()
    print(f"[llm] reply: {reply}")

    # 3. Reply -> speech
    return speak(reply)

Run it

  1. The image pins Python 3.12 (Coqui/Whisper dependency matrix) and AI Fabric requires the client Python to MATCH: uv venv -p 3.12 .venv312 && uv pip install --python .venv312/bin/python bladekit, then .venv312/bin/blade deploy voice_chat.py.
  2. One replica hosts the full pipeline: Whisper (STT) + Qwen2.5-0.5B (LLM) + Coqui VITS (TTS) all fit a single A4500 with room to spare. min_replicas=1 keeps it warm.
  3. Bootstrap a test question with the endpoint itself (text mode skips STT/LLM): curl -X POST <app-url>/voice_chat -H 'Authorization: Bearer <FABRIC_TOKEN>' -d '{"text": "What is the capital of France?"}' -o question.wav.
  4. Full round-trip, voice in and voice out: curl -X POST <app-url>/voice_chat -d '{"audio_b64": "'$(base64 -i question.wav)'"}' -H 'Authorization: Bearer <FABRIC_TOKEN>' -o reply.wav. Whisper transcribes, Qwen answers, Coqui speaks the reply.
  5. Swap any stage independently: a bigger Whisper model for accuracy, a bigger LLM for smarter replies, or an XTTS voice for cloning.

Concepts used: Cold start, Endpoint, Replica, GPU

Contact the team

Loading the form…