Developer docs / AI Fabric tutorials

Fine-tune Whisper

  • GPU
  • Volume
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

finetune_whisper.py
import bladekit

app = bladekit.app("finetune-whisper")

checkpoints = bladekit.Volume.from_name("whisper-checkpoints", create_if_missing=True)

# fabric/torch already ships torch, so don't reinstall it. Add only the audio +
# training libs on top.
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        # datasets<4.0 keeps audio decoding on soundfile; 4.0+ requires
        # torchcodec (not installed here) and raises ImportError on the
        # first "audio" column access.
        "transformers", "datasets<4.0",
        "accelerate", "soundfile", "librosa", "evaluate", "jiwer"
    )
    .apt_install("ffmpeg")
)

@app.function(
    gpu="RTX-A4500",
    image=image,
    timeout=14400,
    volumes={"/checkpoints": checkpoints},
    retries=2,
)
def finetune_whisper(
    model_name: str = "openai/whisper-small",
    # A tiny, parquet-based dataset (no dataset script), so it works on
    # datasets v5 (which dropped trust_remote_code script loading). English.
    dataset_name: str = "hf-internal-testing/librispeech_asr_dummy",
    language: str = "en",
) -> str:
    from transformers import (
        WhisperForConditionalGeneration,
        WhisperProcessor,
        Seq2SeqTrainer,
        Seq2SeqTrainingArguments,
    )
    from datasets import load_dataset

    processor = WhisperProcessor.from_pretrained(model_name, language=language, task="transcribe")
    model = WhisperForConditionalGeneration.from_pretrained(model_name)
    model.config.forced_decoder_ids = processor.get_decoder_prompt_ids(language=language, task="transcribe")

    # No trust_remote_code needed: this dataset ships as parquet, not a script.
    dataset = load_dataset(dataset_name, "clean", split="validation")

    def prepare(batch):
        audio = batch["audio"]
        batch["input_features"] = processor(audio["array"], sampling_rate=audio["sampling_rate"]).input_features[0]
        # LibriSpeech transcripts live in the "text" column. Pad/truncate to a
        # fixed length so the default collator can stack labels into a batch
        # tensor without writing a custom data collator.
        labels = processor.tokenizer(
            batch["text"], padding="max_length", truncation=True, max_length=448
        ).input_ids
        # Mask pad tokens with -100 so they don't contribute to the loss.
        pad_id = processor.tokenizer.pad_token_id
        batch["labels"] = [tok if tok != pad_id else -100 for tok in labels]
        return batch

    dataset = dataset.map(prepare, remove_columns=dataset.column_names)

    args = Seq2SeqTrainingArguments(
        output_dir="/checkpoints/whisper-ft",
        max_steps=20,                 # a few steps so the demo finishes fast
        per_device_train_batch_size=2,
        save_steps=10,
        predict_with_generate=True,
        report_to=[],
    )
    Seq2SeqTrainer(model=model, args=args, train_dataset=dataset, processing_class=processor.feature_extractor).train()
    # Volume writes are local to the container until committed as a new
    # generation. Without this the checkpoint never actually persists.
    checkpoints.commit()
    return "/checkpoints/whisper-ft"

@app.local_entrypoint()
def main():
    out = finetune_whisper.remote()
    print(f"Model saved to: {out}")

Run it

  1. Run blade run finetune_whisper.py. It fine-tunes a few steps on the tiny librispeech_asr_dummy set. It is self-contained, no audio to upload.
  2. To train on your own language/domain, point dataset_name at any non-script (parquet/arrow) HF dataset with audio + transcript columns, and adjust the prepare mapping + language.
  3. Checkpoints are saved every 10 steps. If the run is preempted, retries=2 resumes from the latest.
  4. After training, load the fine-tuned model from the Volume for faster transcription of your target language. Bump max_steps for a real run.
  5. Remember to call vol.commit() after training: Volume writes only live in the container until committed as a new generation.

Concepts used: GPU, Image build

Contact the team

Loading the form…