Developer docs / AI Fabric tutorials

LoRA fine-tuning

  • GPU
  • Volume
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

lora_finetune.py
import bladekit

app = bladekit.app("lora-finetune")

checkpoints = bladekit.Volume.from_name("lora-checkpoints", create_if_missing=True)

# fabric/torch already ships torch, so don't reinstall it (a pip torch upgrade
# breaks compiled ops). Add only the training libs on top.
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        "transformers", "peft",
        "datasets", "accelerate"
    )
)

@app.function(
    gpu="RTX-A4500",
    image=image,
    timeout=7200,
    volumes={"/checkpoints": checkpoints},
    retries=2,        # resume from last checkpoint if preempted
)
def train(
    model_name: str = "Qwen/Qwen2.5-0.5B",
    dataset_name: str = "yahma/alpaca-cleaned",
    output_dir: str = "/checkpoints/run1",
) -> str:
    import torch
    from transformers import (
        AutoModelForCausalLM, AutoTokenizer,
        TrainingArguments, Trainer, DataCollatorForLanguageModeling,
    )
    from peft import LoraConfig, get_peft_model
    from datasets import load_dataset

    # Load a small base model and apply LoRA adapters.
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
    lora_config = LoraConfig(r=8, lora_alpha=16, target_modules=["q_proj", "v_proj"])
    model = get_peft_model(model, lora_config)

    # Tokenise the dataset into input_ids/labels. Without this the Trainer drops
    # the text columns and has nothing to train on. Keep it small + fast.
    dataset = load_dataset(dataset_name, split="train[:100]")

    def tokenize(ex):
        prompt = (ex.get("instruction", "") + "\n" + ex.get("input", "")).strip()
        text = prompt + "\n" + ex.get("output", "")
        out = tokenizer(text, truncation=True, max_length=256, padding="max_length")
        out["labels"] = out["input_ids"].copy()
        return out

    dataset = dataset.map(tokenize, remove_columns=dataset.column_names)

    args = TrainingArguments(
        output_dir=output_dir,
        max_steps=20,                 # tiny run so it finishes fast
        per_device_train_batch_size=2,
        save_steps=10,                # checkpoint every 10 steps
        logging_steps=5,
        report_to=[],
    )
    collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
    Trainer(model=model, args=args, train_dataset=dataset,
            data_collator=collator).train()
    model.save_pretrained(output_dir + "/final")

    # Local writes under the mount are container-local until committed. Without
    # this the adapter above is lost the moment the container exits.
    checkpoints.commit()
    return output_dir + "/final"

@app.local_entrypoint()
def main():
    result = train.remote()
    print(f"Saved adapter to: {result}")

Run it

  1. Save as lora_finetune.py. Run blade run lora_finetune.py.
  2. The dataset is tokenised into input_ids/labels (with remove_columns) so the Trainer has tensors to learn from, and max_steps=20 keeps the demo fast.
  3. Checkpoints are saved to the Volume every 10 steps. If the run is preempted, the next retry resumes automatically.
  4. After training, the LoRA adapter lives in the lora-checkpoints Volume. Bump max_steps, swap in a bigger base model, or point at your own dataset to scale up.

Concepts used: GPU, Image build

Contact the team

Loading the form…