Developer docs / AI Fabric tutorials
Streaming speech-to-text with Kyutai STT
- GPU
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
# Streaming (real-time) speech-to-text with Kyutai STT.
#
# Kyutai STT is a *streaming* ASR model built on the moshi delayed-streams
# architecture: it consumes audio frame-by-frame (80 ms Mimi codec frames) and
# emits text tokens as it listens, so partial transcripts appear WHILE the
# audio is still playing, unlike batch Whisper which needs the whole clip first.
#
# This example feeds a ~10 s speech clip through the model one frame at a time
# and prints each new word the instant the decoder emits it, proving the
# incremental behaviour, then prints the final full transcript.
import bladekit
app = bladekit.app("kyutai-stt-streaming")
# fabric/torch already has CUDA + a recent torch; we add the moshi runtime (the
# Kyutai STT stack), sphn for audio I/O, julius for resampling, and datasets so
# the script can fetch its own short sample clip (self-contained, no upload).
image = (
bladekit.Image.from_catalog("fabric/torch")
.apt_install("ffmpeg")
.pip_install("moshi", "sphn", "julius", "datasets", "soundfile")
)
@app.function(image=image, timeout=300)
def sample_audio() -> bytes:
"""Fetch one short public English clip and return it as WAV bytes, so the
tutorial is self-contained, with no audio file to supply."""
import io
import numpy as np
import soundfile as sf
from datasets import Audio, load_dataset
ds = load_dataset(
"hf-internal-testing/librispeech_asr_dummy", "clean", split="validation"
)
# decode=False hands us the raw FLAC bytes (avoids datasets' torchcodec
# backend); soundfile decodes them to float32 samples.
ds = ds.cast_column("audio", Audio(decode=False))
arrays, sr = [], 16000
for i in range(3): # concatenate a few utterances -> ~10 s of speech
arr, sr = sf.read(io.BytesIO(ds[i]["audio"]["bytes"]), dtype="float32")
arrays.append(arr)
audio = np.concatenate(arrays)
buf = io.BytesIO()
sf.write(buf, audio, sr, format="WAV")
return buf.getvalue()
@app.function(gpu="RTX-A4500", image=image, timeout=900)
def transcribe_streaming(wav_bytes: bytes) -> dict:
"""Stream the audio through Kyutai STT frame-by-frame, emitting partial
transcripts as tokens arrive. Returns the full transcript plus a few
growing snapshots that prove the output is incremental."""
import itertools
import math
import os
import tempfile
# Run in eager mode: moshi wraps its transformer in torch.compile, which
# would need a C compiler (triton/inductor) that the slim image doesn't ship.
# Disabling dynamo falls back to eager: a touch slower, but no compiler needed.
os.environ["TORCH_COMPILE_DISABLE"] = "1"
os.environ["TORCHDYNAMO_DISABLE"] = "1"
import julius
import moshi.models
import sphn
import torch
device = "cuda"
hf_repo = "kyutai/stt-1b-en_fr" # 1B streaming STT, English + French
# Load the Mimi audio codec + the STT language model from the Hub.
info = moshi.models.loaders.CheckpointInfo.from_hf_repo(hf_repo)
mimi = info.get_mimi(device=device)
tokenizer = info.get_text_tokenizer()
lm = info.get_moshi(device=device, dtype=torch.bfloat16)
lm_gen = moshi.models.LMGen(lm, temp=0, temp_text=0)
# Streaming hyper-params baked into the checkpoint: a silence prefix to warm
# the model, and an audio delay (the model reads a few seconds ahead before
# committing text; that's the "delayed streams" trick).
silence_prefix_s = info.stt_config.get("audio_silence_prefix_seconds", 1.0)
audio_delay_s = info.stt_config.get("audio_delay_seconds", 5.0)
padding_token_id = info.raw_config.get("text_padding_token_id", 3)
# Read the clip and resample to the codec's sample rate.
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
f.write(wav_bytes)
path = f.name
audio, in_sr = sphn.read(path)
audio = torch.from_numpy(audio).to(device)
audio = julius.resample_frac(audio, in_sr, mimi.sample_rate)
if audio.shape[-1] % mimi.frame_size != 0:
pad = mimi.frame_size - audio.shape[-1] % mimi.frame_size
audio = torch.nn.functional.pad(audio, (0, pad))
n_prefix = math.ceil(silence_prefix_s * mimi.frame_rate)
n_suffix = math.ceil(audio_delay_s * mimi.frame_rate)
silence = torch.zeros(
(1, 1, mimi.frame_size), dtype=torch.float32, device=device
)
frames = itertools.chain(
itertools.repeat(silence, n_prefix),
torch.split(audio[None], mimi.frame_size, dim=-1),
itertools.repeat(silence, n_suffix),
)
pieces: list[str] = []
snapshots: list[str] = [] # growing partial transcripts, captured to prove
# the output is emitted incrementally rather than all at the end.
# mimi.streaming + lm_gen.streaming keep the recurrent state across frames;
# this is what makes it *streaming* rather than re-reading the whole clip.
with mimi.streaming(1), lm_gen.streaming(1):
for frame in frames:
if frame.shape[-1] != mimi.frame_size:
continue
audio_tokens = mimi.encode(frame)
text_tokens = lm_gen.step(audio_tokens)
if text_tokens is None:
continue
tok = text_tokens[0, 0, 0].item()
if tok in (0, padding_token_id):
continue
piece = tokenizer.id_to_piece(tok).replace("▁", " ")
pieces.append(piece)
# Emit the partial transcript the moment a new word arrives.
partial = "".join(pieces).strip()
print(f"[partial] {partial}", flush=True)
if len(pieces) % 12 == 0: # snapshot every ~12 emitted tokens
snapshots.append(partial)
return {"transcript": "".join(pieces).strip(), "snapshots": snapshots}
@app.local_entrypoint()
def main():
wav = sample_audio.remote()
print(f"Fetched {len(wav)} bytes of sample audio\n")
result = transcribe_streaming.remote(wav)
# The snapshots show the transcript GROWING as frames stream in, proof the
# model emits text incrementally, not in one shot at the end.
print("Incremental partial transcripts (streaming):")
for i, snap in enumerate(result["snapshots"], 1):
print(f" [{i:>2}] {snap}")
print("\nFull transcript:\n", result["transcript"])Run it
- Use Python 3.12 locally. It must match the catalog image (
fabric/torchis built on Python 3.12), otherwise the run fails withPYTHON_VERSION_MISMATCH. - Run
blade run kyutai_stt_streaming.py. No audio file needed:sample_audiopulls a few short LibriSpeech clips and soundfile stitches them into ~10 s of WAV. To transcribe your own audio, read a file and pass its bytes:transcribe_streaming.remote(open('speech.wav','rb').read()). sample_audiodecodes withAudio(decode=False)+ soundfile on purpose: datasets 5.x needstorchcodecfor its built-in audio decoder, which the image doesn't ship, and pulling raw bytes and decoding yourself sidesteps that.- The model wraps its transformer in
torch.compile; the slim image has no C compiler, sotorch.compile(triton/inductor) would crash with "Failed to find C compiler". SettingTORCH_COMPILE_DISABLE=1before importing torch falls back to eager: no compiler needed, only a touch slower. (Alternatively, addbuild-essentialto the image and keep compilation on.) - Kyutai STT streams incrementally:
mimi.streaming(1)+lm_gen.streaming(1)keep recurrent state across 80 ms frames, andlm_gen.step()emits text tokens as audio arrives. The checkpoint'saudio_delay_seconds(~5 s of look-ahead) is the "delayed streams" mechanism, hence the trailing silence frames that flush the tail. - Tested live on an RTX A4500: the run prints 8 growing partial transcripts ("Mr. Quilter is the apostle of the…" → full sentence) followed by the exact reference transcript, proving the output is incremental, not one-shot.
Concepts used: Run, GPU, Cold start