Developer docs / AI Fabric tutorials
Voice chat with an LLM
- GPU
- Web endpoint
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("voice-chat")
# The maintained Coqui fork ("coqui-tts") supports Python 3.12, so pin the
# image interpreter and deploy from a Python 3.12 client venv (AI Fabric matches
# the client and image Python). Whisper needs ffmpeg; Coqui needs espeak-ng.
image = (
bladekit.Image.debian_slim(python_version="3.12")
.apt_install("ffmpeg", "espeak-ng")
.pip_install(
"torch", "torchaudio", "openai-whisper",
# coqui-tts imports private transformers helpers that moved in newer
# releases, so pin a version both Qwen and Coqui are happy with.
"transformers==4.46.2", "accelerate", "coqui-tts",
)
# Small helper Coqui's model manager imports but does not declare.
.pip_install("click")
)
_models = None
def _load_models():
"""Load all three models once per replica and keep them warm."""
global _models
import whisper
from transformers import pipeline
from TTS.api import TTS as TTSModel
if _models is None:
asr = whisper.load_model("base")
llm = pipeline(
"text-generation", model="Qwen/Qwen2.5-0.5B-Instruct",
torch_dtype="auto", device=0,
)
tts = TTSModel("tts_models/en/ljspeech/tacotron2-DDC").to("cuda")
_models = (asr, llm, tts)
return _models
# min_replicas=1: loading three models takes longer than the cold-start wake
# window, so keep one warm replica instead of scaling to zero.
@app.web_endpoint(methods=["POST"], min_replicas=1, max_replicas=2)
@app.function(gpu="RTX-A4500", image=image, timeout=300)
def voice_chat(item: dict):
"""Voice in, voice out: Whisper STT -> Qwen LLM -> Coqui TTS.
POST {"audio_b64": "<base64 WAV/MP3>"} -> spoken reply as WAV bytes
POST {"text": "..."} -> plain TTS of that text (handy
to generate a test input WAV)
"""
import base64, tempfile
asr, llm, tts = _load_models()
def speak(text: str) -> bytes:
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
out_path = f.name
tts.tts_to_file(text=text, file_path=out_path)
wav = open(out_path, "rb").read()
print(f"[tts] {len(wav)} wav bytes")
return wav
# Bootstrap mode: synthesize arbitrary text (no STT/LLM involved).
if "text" in item:
return speak(item["text"])
# 1. Speech -> text
audio_bytes = base64.b64decode(item["audio_b64"])
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
f.write(audio_bytes)
tmp = f.name
question = asr.transcribe(tmp)["text"].strip()
print(f"[stt] transcript: {question}")
# 2. Text -> reply (chat format so the instruct model answers
# instead of just continuing the sentence)
messages = [{"role": "user", "content": question}]
reply = llm(messages, max_new_tokens=80)[0]["generated_text"][-1]["content"].strip()
print(f"[llm] reply: {reply}")
# 3. Reply -> speech
return speak(reply)Run it
- The image pins Python 3.12 (Coqui/Whisper dependency matrix) and AI Fabric requires the client Python to MATCH:
uv venv -p 3.12 .venv312 && uv pip install --python .venv312/bin/python bladekit, then.venv312/bin/blade deploy voice_chat.py. - One replica hosts the full pipeline: Whisper (STT) + Qwen2.5-0.5B (LLM) + Coqui VITS (TTS) all fit a single A4500 with room to spare.
min_replicas=1keeps it warm. - Bootstrap a test question with the endpoint itself (text mode skips STT/LLM):
curl -X POST <app-url>/voice_chat -H 'Authorization: Bearer <FABRIC_TOKEN>' -d '{"text": "What is the capital of France?"}' -o question.wav. - Full round-trip, voice in and voice out:
curl -X POST <app-url>/voice_chat -d '{"audio_b64": "'$(base64 -i question.wav)'"}' -H 'Authorization: Bearer <FABRIC_TOKEN>' -o reply.wav. Whisper transcribes, Qwen answers, Coqui speaks the reply. - Swap any stage independently: a bigger Whisper model for accuracy, a bigger LLM for smarter replies, or an XTTS voice for cloning.
Concepts used: Cold start, Endpoint, Replica, GPU