Developer docs / AI Fabric tutorials
Fine-tune Whisper
- GPU
- Volume
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("finetune-whisper")
checkpoints = bladekit.Volume.from_name("whisper-checkpoints", create_if_missing=True)
# fabric/torch already ships torch, so don't reinstall it. Add only the audio +
# training libs on top.
image = (
bladekit.Image.from_catalog("fabric/torch")
.pip_install(
# datasets<4.0 keeps audio decoding on soundfile; 4.0+ requires
# torchcodec (not installed here) and raises ImportError on the
# first "audio" column access.
"transformers", "datasets<4.0",
"accelerate", "soundfile", "librosa", "evaluate", "jiwer"
)
.apt_install("ffmpeg")
)
@app.function(
gpu="RTX-A4500",
image=image,
timeout=14400,
volumes={"/checkpoints": checkpoints},
retries=2,
)
def finetune_whisper(
model_name: str = "openai/whisper-small",
# A tiny, parquet-based dataset (no dataset script), so it works on
# datasets v5 (which dropped trust_remote_code script loading). English.
dataset_name: str = "hf-internal-testing/librispeech_asr_dummy",
language: str = "en",
) -> str:
from transformers import (
WhisperForConditionalGeneration,
WhisperProcessor,
Seq2SeqTrainer,
Seq2SeqTrainingArguments,
)
from datasets import load_dataset
processor = WhisperProcessor.from_pretrained(model_name, language=language, task="transcribe")
model = WhisperForConditionalGeneration.from_pretrained(model_name)
model.config.forced_decoder_ids = processor.get_decoder_prompt_ids(language=language, task="transcribe")
# No trust_remote_code needed: this dataset ships as parquet, not a script.
dataset = load_dataset(dataset_name, "clean", split="validation")
def prepare(batch):
audio = batch["audio"]
batch["input_features"] = processor(audio["array"], sampling_rate=audio["sampling_rate"]).input_features[0]
# LibriSpeech transcripts live in the "text" column. Pad/truncate to a
# fixed length so the default collator can stack labels into a batch
# tensor without writing a custom data collator.
labels = processor.tokenizer(
batch["text"], padding="max_length", truncation=True, max_length=448
).input_ids
# Mask pad tokens with -100 so they don't contribute to the loss.
pad_id = processor.tokenizer.pad_token_id
batch["labels"] = [tok if tok != pad_id else -100 for tok in labels]
return batch
dataset = dataset.map(prepare, remove_columns=dataset.column_names)
args = Seq2SeqTrainingArguments(
output_dir="/checkpoints/whisper-ft",
max_steps=20, # a few steps so the demo finishes fast
per_device_train_batch_size=2,
save_steps=10,
predict_with_generate=True,
report_to=[],
)
Seq2SeqTrainer(model=model, args=args, train_dataset=dataset, processing_class=processor.feature_extractor).train()
# Volume writes are local to the container until committed as a new
# generation. Without this the checkpoint never actually persists.
checkpoints.commit()
return "/checkpoints/whisper-ft"
@app.local_entrypoint()
def main():
out = finetune_whisper.remote()
print(f"Model saved to: {out}")Run it
- Run
blade run finetune_whisper.py. It fine-tunes a few steps on the tinylibrispeech_asr_dummyset. It is self-contained, no audio to upload. - To train on your own language/domain, point
dataset_nameat any non-script (parquet/arrow) HF dataset withaudio+ transcript columns, and adjust thepreparemapping +language. - Checkpoints are saved every 10 steps. If the run is preempted,
retries=2resumes from the latest. - After training, load the fine-tuned model from the Volume for faster transcription of your target language. Bump
max_stepsfor a real run. - Remember to call
vol.commit()after training: Volume writes only live in the container until committed as a new generation.
Concepts used: GPU, Image build