Developer docs / AI Fabric tutorials

Streaming speech-to-text with Kyutai STT

  • GPU
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

kyutai_stt_streaming.py
# Streaming (real-time) speech-to-text with Kyutai STT.
#
# Kyutai STT is a *streaming* ASR model built on the moshi delayed-streams
# architecture: it consumes audio frame-by-frame (80 ms Mimi codec frames) and
# emits text tokens as it listens, so partial transcripts appear WHILE the
# audio is still playing, unlike batch Whisper which needs the whole clip first.
#
# This example feeds a ~10 s speech clip through the model one frame at a time
# and prints each new word the instant the decoder emits it, proving the
# incremental behaviour, then prints the final full transcript.
import bladekit

app = bladekit.app("kyutai-stt-streaming")

# fabric/torch already has CUDA + a recent torch; we add the moshi runtime (the
# Kyutai STT stack), sphn for audio I/O, julius for resampling, and datasets so
# the script can fetch its own short sample clip (self-contained, no upload).
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .apt_install("ffmpeg")
    .pip_install("moshi", "sphn", "julius", "datasets", "soundfile")
)


@app.function(image=image, timeout=300)
def sample_audio() -> bytes:
    """Fetch one short public English clip and return it as WAV bytes, so the
    tutorial is self-contained, with no audio file to supply."""
    import io

    import numpy as np
    import soundfile as sf
    from datasets import Audio, load_dataset

    ds = load_dataset(
        "hf-internal-testing/librispeech_asr_dummy", "clean", split="validation"
    )
    # decode=False hands us the raw FLAC bytes (avoids datasets' torchcodec
    # backend); soundfile decodes them to float32 samples.
    ds = ds.cast_column("audio", Audio(decode=False))
    arrays, sr = [], 16000
    for i in range(3):  # concatenate a few utterances -> ~10 s of speech
        arr, sr = sf.read(io.BytesIO(ds[i]["audio"]["bytes"]), dtype="float32")
        arrays.append(arr)
    audio = np.concatenate(arrays)
    buf = io.BytesIO()
    sf.write(buf, audio, sr, format="WAV")
    return buf.getvalue()


@app.function(gpu="RTX-A4500", image=image, timeout=900)
def transcribe_streaming(wav_bytes: bytes) -> dict:
    """Stream the audio through Kyutai STT frame-by-frame, emitting partial
    transcripts as tokens arrive. Returns the full transcript plus a few
    growing snapshots that prove the output is incremental."""
    import itertools
    import math
    import os
    import tempfile

    # Run in eager mode: moshi wraps its transformer in torch.compile, which
    # would need a C compiler (triton/inductor) that the slim image doesn't ship.
    # Disabling dynamo falls back to eager: a touch slower, but no compiler needed.
    os.environ["TORCH_COMPILE_DISABLE"] = "1"
    os.environ["TORCHDYNAMO_DISABLE"] = "1"

    import julius
    import moshi.models
    import sphn
    import torch

    device = "cuda"
    hf_repo = "kyutai/stt-1b-en_fr"  # 1B streaming STT, English + French

    # Load the Mimi audio codec + the STT language model from the Hub.
    info = moshi.models.loaders.CheckpointInfo.from_hf_repo(hf_repo)
    mimi = info.get_mimi(device=device)
    tokenizer = info.get_text_tokenizer()
    lm = info.get_moshi(device=device, dtype=torch.bfloat16)
    lm_gen = moshi.models.LMGen(lm, temp=0, temp_text=0)

    # Streaming hyper-params baked into the checkpoint: a silence prefix to warm
    # the model, and an audio delay (the model reads a few seconds ahead before
    # committing text; that's the "delayed streams" trick).
    silence_prefix_s = info.stt_config.get("audio_silence_prefix_seconds", 1.0)
    audio_delay_s = info.stt_config.get("audio_delay_seconds", 5.0)
    padding_token_id = info.raw_config.get("text_padding_token_id", 3)

    # Read the clip and resample to the codec's sample rate.
    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
        f.write(wav_bytes)
        path = f.name
    audio, in_sr = sphn.read(path)
    audio = torch.from_numpy(audio).to(device)
    audio = julius.resample_frac(audio, in_sr, mimi.sample_rate)
    if audio.shape[-1] % mimi.frame_size != 0:
        pad = mimi.frame_size - audio.shape[-1] % mimi.frame_size
        audio = torch.nn.functional.pad(audio, (0, pad))

    n_prefix = math.ceil(silence_prefix_s * mimi.frame_rate)
    n_suffix = math.ceil(audio_delay_s * mimi.frame_rate)
    silence = torch.zeros(
        (1, 1, mimi.frame_size), dtype=torch.float32, device=device
    )
    frames = itertools.chain(
        itertools.repeat(silence, n_prefix),
        torch.split(audio[None], mimi.frame_size, dim=-1),
        itertools.repeat(silence, n_suffix),
    )

    pieces: list[str] = []
    snapshots: list[str] = []  # growing partial transcripts, captured to prove
    # the output is emitted incrementally rather than all at the end.
    # mimi.streaming + lm_gen.streaming keep the recurrent state across frames;
    # this is what makes it *streaming* rather than re-reading the whole clip.
    with mimi.streaming(1), lm_gen.streaming(1):
        for frame in frames:
            if frame.shape[-1] != mimi.frame_size:
                continue
            audio_tokens = mimi.encode(frame)
            text_tokens = lm_gen.step(audio_tokens)
            if text_tokens is None:
                continue
            tok = text_tokens[0, 0, 0].item()
            if tok in (0, padding_token_id):
                continue
            piece = tokenizer.id_to_piece(tok).replace("▁", " ")
            pieces.append(piece)
            # Emit the partial transcript the moment a new word arrives.
            partial = "".join(pieces).strip()
            print(f"[partial] {partial}", flush=True)
            if len(pieces) % 12 == 0:  # snapshot every ~12 emitted tokens
                snapshots.append(partial)

    return {"transcript": "".join(pieces).strip(), "snapshots": snapshots}


@app.local_entrypoint()
def main():
    wav = sample_audio.remote()
    print(f"Fetched {len(wav)} bytes of sample audio\n")
    result = transcribe_streaming.remote(wav)

    # The snapshots show the transcript GROWING as frames stream in, proof the
    # model emits text incrementally, not in one shot at the end.
    print("Incremental partial transcripts (streaming):")
    for i, snap in enumerate(result["snapshots"], 1):
        print(f"  [{i:>2}] {snap}")

    print("\nFull transcript:\n", result["transcript"])

Run it

  1. Use Python 3.12 locally. It must match the catalog image (fabric/torch is built on Python 3.12), otherwise the run fails with PYTHON_VERSION_MISMATCH.
  2. Run blade run kyutai_stt_streaming.py. No audio file needed: sample_audio pulls a few short LibriSpeech clips and soundfile stitches them into ~10 s of WAV. To transcribe your own audio, read a file and pass its bytes: transcribe_streaming.remote(open('speech.wav','rb').read()).
  3. sample_audio decodes with Audio(decode=False) + soundfile on purpose: datasets 5.x needs torchcodec for its built-in audio decoder, which the image doesn't ship, and pulling raw bytes and decoding yourself sidesteps that.
  4. The model wraps its transformer in torch.compile; the slim image has no C compiler, so torch.compile (triton/inductor) would crash with "Failed to find C compiler". Setting TORCH_COMPILE_DISABLE=1 before importing torch falls back to eager: no compiler needed, only a touch slower. (Alternatively, add build-essential to the image and keep compilation on.)
  5. Kyutai STT streams incrementally: mimi.streaming(1) + lm_gen.streaming(1) keep recurrent state across 80 ms frames, and lm_gen.step() emits text tokens as audio arrives. The checkpoint's audio_delay_seconds (~5 s of look-ahead) is the "delayed streams" mechanism, hence the trailing silence frames that flush the tail.
  6. Tested live on an RTX A4500: the run prints 8 growing partial transcripts ("Mr. Quilter is the apostle of the…" → full sentence) followed by the exact reference transcript, proving the output is incremental, not one-shot.

Concepts used: Run, GPU, Cold start

Contact the team

Loading the form…