Developer docs / AI Fabric tutorials

Fine-tuning with checkpoints on interruptible capacity

  • GPU
  • Volume
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

spot_finetune.py
import bladekit
import torch

app = bladekit.app("spot-finetune")

# Checkpoints persist on a Volume so any node can resume the run.
checkpoints = bladekit.Volume.from_name("spot-ckpts", create_if_missing=True)

# fabric/torch already ships PyTorch, so don't reinstall it here.
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        "transformers", "peft",
        "datasets", "accelerate", "bitsandbytes",
    )
)

# Module-level ref so the on_preempt hook can read the live trainer state.
_trainer = None

@app.function(
    gpu="RTX-A4500",
    gpu_class="spot",          # interruptible (cheap); the default
    image=image,
    secrets=[bladekit.Secret.from_name("hf-token")],  # Mistral-7B is a gated HF repo
    timeout=14400,             # 4 h max per attempt
    volumes={"/ckpts": checkpoints},
    retries=10,                # re-queue after each preemption
    max_interruptions=10,
)
def train(model_name: str = "mistralai/Mistral-7B-v0.1") -> str:
    global _trainer
    from transformers import (
        AutoModelForCausalLM, AutoTokenizer,
        TrainingArguments, Trainer, DataCollatorForLanguageModeling,
    )
    from transformers.trainer_utils import get_last_checkpoint
    from peft import LoraConfig, get_peft_model
    from datasets import load_dataset
    import os

    output_dir = "/ckpts/run"

    # ── Idempotent resume: pick up the last checkpoint if one exists. ──
    # get_last_checkpoint() returns the newest checkpoint-* dir (or None);
    # the output dir itself is not a checkpoint, so never pass it raw.
    resume_from = get_last_checkpoint(output_dir) if os.path.isdir(output_dir) else None

    # bf16 halves the fp32 memory footprint so a 7B model fits the A4500.
    model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    tokenizer.pad_token = tokenizer.eos_token

    lora_cfg = LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj"])
    model = get_peft_model(model, lora_cfg)

    dataset = load_dataset("tatsu-lab/alpaca", split="train[:2000]")

    def tokenize(ex):
        return tokenizer(ex["instruction"], truncation=True, max_length=512)

    dataset = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)
    # Causal-LM collator pads each batch and sets labels = input_ids so the
    # Trainer has a loss target (a bare tokenizer output has no "labels").
    collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)

    args = TrainingArguments(
        output_dir=output_dir,
        num_train_epochs=1,
        per_device_train_batch_size=2,
        bf16=True,
        save_steps=50,                 # checkpoint every 50 steps
        save_total_limit=2,            # keep last 2 checkpoints
        logging_steps=25,
    )
    _trainer = Trainer(
        model=model, args=args,
        train_dataset=dataset, data_collator=collator,
    )
    # resume_from_checkpoint belongs on train(), not TrainingArguments.
    _trainer.train(resume_from_checkpoint=resume_from)
    return output_dir

@app.on_preempt
def save_on_preempt() -> None:
    """Called in the ≥5 min preemption window: checkpoints the live trainer."""
    if _trainer is not None:
        # Saves model + optimizer + scheduler + RNG state to the Volume.
        _trainer.save_model("/ckpts/run")
        _trainer.state.save_to_json("/ckpts/run/trainer_state.json")
        print("Checkpoint saved on preemption signal.")

@app.local_entrypoint()
def main():
    out = train.remote()
    print(f"Fine-tune complete. Adapter at: {out}")

Run it

  1. Save as spot_finetune.py. Run blade run spot_finetune.py.
  2. The function runs on gpu_class='spot' (interruptible), using cheap capacity harvested from Shadow's idle gaming fleet. Preemption can happen with ≥5 min notice.
  3. When a preemption signal arrives, AI Fabric calls @app.on_preempt inside the grace window. The hook calls _trainer.save_model() to write a full checkpoint to the Volume before the container is killed.
  4. With retries=10, AI Fabric re-queues the run automatically. On restart, get_last_checkpoint() finds the newest checkpoint and resume_from_checkpoint picks up from the last saved step, so no GPU time is wasted.
  5. After training, mount the spot-ckpts Volume in an inference function to load the fine-tuned adapter.

Concepts used: GPU, Image build

Contact the team

Loading the form…