Developer docs / AI Fabric tutorials
Fine-tuning with checkpoints on interruptible capacity
- GPU
- Volume
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
import torch
app = bladekit.app("spot-finetune")
# Checkpoints persist on a Volume so any node can resume the run.
checkpoints = bladekit.Volume.from_name("spot-ckpts", create_if_missing=True)
# fabric/torch already ships PyTorch, so don't reinstall it here.
image = (
bladekit.Image.from_catalog("fabric/torch")
.pip_install(
"transformers", "peft",
"datasets", "accelerate", "bitsandbytes",
)
)
# Module-level ref so the on_preempt hook can read the live trainer state.
_trainer = None
@app.function(
gpu="RTX-A4500",
gpu_class="spot", # interruptible (cheap); the default
image=image,
secrets=[bladekit.Secret.from_name("hf-token")], # Mistral-7B is a gated HF repo
timeout=14400, # 4 h max per attempt
volumes={"/ckpts": checkpoints},
retries=10, # re-queue after each preemption
max_interruptions=10,
)
def train(model_name: str = "mistralai/Mistral-7B-v0.1") -> str:
global _trainer
from transformers import (
AutoModelForCausalLM, AutoTokenizer,
TrainingArguments, Trainer, DataCollatorForLanguageModeling,
)
from transformers.trainer_utils import get_last_checkpoint
from peft import LoraConfig, get_peft_model
from datasets import load_dataset
import os
output_dir = "/ckpts/run"
# ── Idempotent resume: pick up the last checkpoint if one exists. ──
# get_last_checkpoint() returns the newest checkpoint-* dir (or None);
# the output dir itself is not a checkpoint, so never pass it raw.
resume_from = get_last_checkpoint(output_dir) if os.path.isdir(output_dir) else None
# bf16 halves the fp32 memory footprint so a 7B model fits the A4500.
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token
lora_cfg = LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj"])
model = get_peft_model(model, lora_cfg)
dataset = load_dataset("tatsu-lab/alpaca", split="train[:2000]")
def tokenize(ex):
return tokenizer(ex["instruction"], truncation=True, max_length=512)
dataset = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)
# Causal-LM collator pads each batch and sets labels = input_ids so the
# Trainer has a loss target (a bare tokenizer output has no "labels").
collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
args = TrainingArguments(
output_dir=output_dir,
num_train_epochs=1,
per_device_train_batch_size=2,
bf16=True,
save_steps=50, # checkpoint every 50 steps
save_total_limit=2, # keep last 2 checkpoints
logging_steps=25,
)
_trainer = Trainer(
model=model, args=args,
train_dataset=dataset, data_collator=collator,
)
# resume_from_checkpoint belongs on train(), not TrainingArguments.
_trainer.train(resume_from_checkpoint=resume_from)
return output_dir
@app.on_preempt
def save_on_preempt() -> None:
"""Called in the ≥5 min preemption window: checkpoints the live trainer."""
if _trainer is not None:
# Saves model + optimizer + scheduler + RNG state to the Volume.
_trainer.save_model("/ckpts/run")
_trainer.state.save_to_json("/ckpts/run/trainer_state.json")
print("Checkpoint saved on preemption signal.")
@app.local_entrypoint()
def main():
out = train.remote()
print(f"Fine-tune complete. Adapter at: {out}")Run it
- Save as
spot_finetune.py. Runblade run spot_finetune.py. - The function runs on
gpu_class='spot'(interruptible), using cheap capacity harvested from Shadow's idle gaming fleet. Preemption can happen with ≥5 min notice. - When a preemption signal arrives, AI Fabric calls
@app.on_preemptinside the grace window. The hook calls_trainer.save_model()to write a full checkpoint to the Volume before the container is killed. - With
retries=10, AI Fabric re-queues the run automatically. On restart,get_last_checkpoint()finds the newest checkpoint andresume_from_checkpointpicks up from the last saved step, so no GPU time is wasted. - After training, mount the
spot-ckptsVolume in an inference function to load the fine-tuned adapter.
Concepts used: GPU, Image build