Developer docs / AI Fabric tutorials

Flux LoRA fine-tuning on a 20 GB GPU

  • GPU
  • Volume
  • Secret
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

app.py
import bladekit

app = bladekit.app("flux-lora-a4500")

data_vol = bladekit.Volume.from_name("flux-lora-data", create_if_missing=True)
out_vol = bladekit.Volume.from_name("flux-lora-output", create_if_missing=True)
hf_cache = bladekit.Volume.from_name("hf-cache", create_if_missing=True)

# fabric/torch already ships torch, so don't reinstall it. bitsandbytes provides the
# NF4 quantiser and the 8-bit optimiser; sentencepiece + protobuf are needed by
# the T5 tokenizer.
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        "diffusers>=0.31.0", "transformers", "accelerate", "peft",
        "bitsandbytes>=0.45.0", "Pillow", "sentencepiece", "protobuf",
        "hf_transfer",
    )
    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1", "HF_HOME": "/hf-cache"})
)


@app.function(
    gpu="RTX-A4500",       # 20 GB card, ~19.6 GiB usable (the whole point here)
    image=image,
    timeout=7200,
    memory=32768,          # host RAM: the bf16 shards are read before quantising
    volumes={"/data": data_vol, "/output": out_vol, "/hf-cache": hf_cache},
    secrets=[bladekit.Secret.from_name("hf-token")],  # FLUX.1-dev is a gated repo
)
def train_flux_lora(
    base_model: str = "black-forest-labs/FLUX.1-dev",
    instance_prompt: str = "a photo of sks style",
    resolution: int = 512,          # 1024 also fits (10.79 GiB peak), ~5x slower
    num_train_steps: int = 1000,
    lora_rank: int = 16,
    learning_rate: float = 2e-4,
    guidance_scale: float = 3.5,   # MUST match what you generate with
) -> str:
    import gc, glob, os
    import numpy as np
    import torch

    def vram(tag):
        torch.cuda.synchronize()
        print(f"[vram] {tag:<28} alloc={torch.cuda.memory_allocated()/1024**3:5.2f}"
              f"  peak={torch.cuda.max_memory_allocated()/1024**3:5.2f} GiB", flush=True)

    # The cuDNN SDPA backward kernel crashes with native CUDA errors on this
    # torch build, so force the flash/math attention paths for training.
    torch.backends.cuda.enable_cudnn_sdp(False)
    device, dtype = torch.device("cuda"), torch.bfloat16

    # Your images go on the /data Volume. Anything else is a placeholder.
    from PIL import Image as PILImage, ImageDraw
    paths = sorted(p for ext in ("jpg", "jpeg", "png", "webp")
                   for p in glob.glob(f"/data/**/*.{ext}", recursive=True))
    if paths:
        print(f"using {len(paths)} image(s) from /data", flush=True)
        images = [PILImage.open(p).convert("RGB").resize((resolution, resolution))
                  for p in paths]
    else:
        # No dataset yet: a deliberately unmistakable concept (magenta disc on
        # white) so you can SEE whether the adapter took, without a real dataset.
        print("no images in /data, synthesising 8 magenta discs", flush=True)
        images = []
        for i in range(8):
            img = PILImage.new("RGB", (resolution, resolution), "white")
            pad = 60 + i * 8
            ImageDraw.Draw(img).ellipse(
                [pad, pad, resolution - pad, resolution - pad], fill=(255, 0, 200))
            images.append(img)

    # ── PASS 1: cache prompt embeddings + latents, then free everything ─────
    # transformer=None loads ONLY the text encoders and the VAE. T5-XXL alone is
    # ~9.5 GB; it must never be resident at the same time as the transformer.
    from diffusers import FluxPipeline
    pipe = FluxPipeline.from_pretrained(
        base_model, transformer=None, torch_dtype=dtype,
        token=os.environ["HF_TOKEN"],
    ).to(device)
    vram("text encoders + VAE")

    with torch.no_grad():
        prompt_embeds, pooled_embeds, text_ids = pipe.encode_prompt(
            prompt=instance_prompt, prompt_2=None, device=device,
            max_sequence_length=256,
        )
        latents = []
        for img in images:
            px = (torch.from_numpy(np.asarray(img, dtype=np.uint8).copy())
                  .permute(2, 0, 1).unsqueeze(0).to(device, dtype))
            px = px / 127.5 - 1.0                      # -> [-1, 1]
            lat = pipe.vae.encode(px).latent_dist.sample()
            lat = (lat - pipe.vae.config.shift_factor) * pipe.vae.config.scaling_factor
            latents.append(lat.cpu())
    cached = torch.cat(latents, dim=0)
    prompt_embeds, pooled_embeds, text_ids = (
        prompt_embeds.cpu(), pooled_embeds.cpu(), text_ids.cpu())
    vram("after encoding")

    # This del is load-bearing: it is what makes the fit work.
    del pipe, latents
    gc.collect(); torch.cuda.empty_cache(); torch.cuda.reset_peak_memory_stats()
    vram("encoders freed")

    # ── PASS 2: NF4 transformer + LoRA on top (QLoRA) ──────────────────────
    from diffusers import BitsAndBytesConfig, FluxTransformer2DModel
    transformer = FluxTransformer2DModel.from_pretrained(
        base_model, subfolder="transformer",
        quantization_config=BitsAndBytesConfig(
            load_in_4bit=True, bnb_4bit_quant_type="nf4",
            bnb_4bit_compute_dtype=dtype),
        torch_dtype=dtype, token=os.environ["HF_TOKEN"],
    )
    vram("NF4 transformer")           # ~6.3 GiB instead of ~24 GiB in bf16

    transformer.enable_gradient_checkpointing()

    from peft import LoraConfig, get_peft_model
    # Attention projections alone (to_q/k/v/out) are not enough capacity: adding
    # the MLPs took the training loss from ~0.014 to ~0.004 over the same 1000
    # steps, at 65.4M trainable params instead of 18.7M (131 MB adapter, +0.6 GiB).
    transformer = get_peft_model(transformer, LoraConfig(
        r=lora_rank, lora_alpha=lora_rank,
        target_modules=[
            "to_q", "to_k", "to_v", "to_out.0",
            "add_q_proj", "add_k_proj", "add_v_proj", "to_add_out",
            "ff.net.0.proj", "ff.net.2",
            "ff_context.net.0.proj", "ff_context.net.2",
            "proj_mlp", "proj_out",
        ],
        lora_dropout=0.0, bias="none",
    ))
    # PEFT creates adapters in fp32; bf16 matches the compute dtype.
    for p in transformer.parameters():
        if p.requires_grad:
            p.data = p.data.to(dtype)
    print(f"trainable LoRA params: "
          f"{sum(p.numel() for p in transformer.parameters() if p.requires_grad)/1e6:.1f}M",
          flush=True)

    import bitsandbytes as bnb
    optimizer = bnb.optim.AdamW8bit(
        [p for p in transformer.parameters() if p.requires_grad], lr=learning_rate)

    prompt_embeds = prompt_embeds.to(device)
    pooled_embeds = pooled_embeds.to(device)
    text_ids = text_ids.to(device)
    transformer.train()

    losses = []
    # Loss per sigma band (0-0.2 ... 0.8-1.0); see the note inside the loop.
    band_sum, band_n = [0.0] * 5, [0] * 5
    for step in range(num_train_steps):
        lat = cached[step % cached.shape[0]].unsqueeze(0).to(device, dtype)
        b, c, h, w = lat.shape
        # Pack latents (B, C, H, W) -> (B, H/2*W/2, C*4) as Flux expects.
        packed = (lat.view(b, c, h // 2, 2, w // 2, 2)
                  .permute(0, 2, 4, 1, 3, 5)
                  .reshape(b, (h // 2) * (w // 2), c * 4))
        img_ids = torch.zeros(h // 2, w // 2, 3, device=device, dtype=dtype)
        img_ids[..., 1] = torch.arange(h // 2, device=device)[:, None]
        img_ids[..., 2] = torch.arange(w // 2, device=device)[None, :]
        img_ids = img_ids.reshape(-1, 3)
        if text_ids.ndim == 3:            # older diffusers use batched ids
            img_ids = img_ids.unsqueeze(0)

        # Flow-matching objective: predict (noise - clean) at a random sigma.
        noise = torch.randn_like(packed)
        # THE trap, and the whole reason this recipe works. Sampling sigma only in
        # a comfortable middle band (e.g. 0.1..0.9) trains NOTHING at sigma > 0.9,
        # including sigma = 1.0, the pure noise every sampling run starts from, and
        # the band where composition and dominant colour are committed. The loop
        # still converges and the adapter still measurably shifts a forward pass,
        # yet generated images do not change at all. Cover the full range, like the
        # reference trainer (uniform over the scheduler grid, UNSHIFTED, since the
        # resolution-dependent shift belongs to inference, not to training).
        t = torch.rand(b, device=device, dtype=dtype).clamp(1e-3, 1.0 - 1e-3)
        noisy = (1 - t[:, None, None]) * packed + t[:, None, None] * noise
        # FLUX.1-dev is guidance-distilled: the guidance value is an INPUT
        # embedding, not a sampler trick. Train at the value you will generate
        # at. A mismatch measurably weakens the adapter (see the notes).
        guidance = None
        if getattr(transformer.config, "guidance_embeds", False):
            guidance = torch.full((b,), guidance_scale, device=device, dtype=dtype)

        pred = transformer(
            hidden_states=noisy, timestep=t, guidance=guidance,
            pooled_projections=pooled_embeds, encoder_hidden_states=prompt_embeds,
            txt_ids=text_ids, img_ids=img_ids, return_dict=False,
        )[0]
        loss = torch.nn.functional.mse_loss(pred.float(), (noise - packed).float())
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        losses.append(loss.item())
        # The scalar mean loss CANNOT tell you whether the high-sigma band is
        # learning: bands differ in intrinsic loss scale by orders of magnitude, so
        # the average is dominated by the easy low-sigma end. Watch the top band.
        band = min(int(t.item() * 5), 4)
        band_sum[band] += loss.item()
        band_n[band] += 1
        if step == 0:
            vram("after first step")
        if step % 25 == 0 or step == num_train_steps - 1:
            print(f"step {step + 1}/{num_train_steps} loss={loss.item():.4f}", flush=True)

    vram("end of training")
    print(f"mean loss: first 25 = {sum(losses[:25])/len(losses[:25]):.4f} | "
          f"last 25 = {sum(losses[-25:])/len(losses[-25:]):.4f}", flush=True)
    for i in range(5):
        mean = band_sum[i] / band_n[i] if band_n[i] else float("nan")
        print(f"  sigma {i*0.2:.1f}-{(i+1)*0.2:.1f}: loss {mean:.4f} (n={band_n[i]})",
              flush=True)

    out = "/output/flux-lora-a4500"
    transformer.save_pretrained(out)
    out_vol.commit()
    return out


@app.local_entrypoint()
def main():
    print(f"LoRA adapter saved to: {train_flux_lora.remote()}")

Run it

  1. Put your training images on the flux-lora-data Volume and add your HF token (blade secret set hf-token HF_TOKEN=...), since FLUX.1-dev is a gated repo. With an empty Volume the snippet synthesises magenta discs, which is deliberate: the concept is unmistakable, so you can tell whether the adapter took without needing a real dataset.
  2. Why it fits where the bf16 recipe does not: the transformer is loaded in NF4 (6.3 GiB resident instead of ~24 GiB) and LoRA trains on top of it: QLoRA, exactly as on an LLM. Measured on an A4500 at 1024×1024: 10.8 GiB peak in pass 1, 8.5 GiB in pass 2, on a card with 19.6 GiB usable.
  3. What makes it fit is the ordering, not the quantiser. Pass 1 loads the pipeline with transformer=None, caches the prompt embeddings and the VAE latents, then frees the encoders; only then is the transformer loaded. T5-XXL (~9.5 GB) and the transformer are never resident together. Keep them both and you OOM before the first step.
  4. The other platform fix: torch.backends.cuda.enable_cudnn_sdp(False) before training. The cuDNN SDPA backward kernel crashes with native CUDA errors on the current torch build; the flash/math attention paths are fine.
  5. Match the guidance value between training and generation. FLUX.1-dev is guidance-distilled (the value is an input embedding, not a sampler trick), so the adapter learns deltas conditioned on it. Training at 1.0 and generating at 3.5 cut the measured effect ~140× versus a matched pair.
  6. The failure mode to watch for: a loop that converges well and still changes nothing. Sampling sigma in a comfortable middle band trains no gradient step at sigma > 0.9, and every sampling run starts at sigma = 1.0, where composition and dominant colour are decided. In that state the training loss fell 8×, the adapter was provably applied (838 injected modules, a 31.7% forward-pass shift), and generated images were pixel-for-pixel uninteresting: 0.0% of the target concept, across four training runs. Covering the full sigma range is a one-line change and the entire difference.
  7. Because of that, do not trust the scalar mean loss. Bands differ in intrinsic loss scale by orders of magnitude, so the average is dominated by the easy low-sigma end and stays silent about the band that decides your image. The snippet prints loss per sigma band for this reason. The top band is the one to watch.
  8. Tested end to end on an A4500, same seed and same prompt embeddings, adapter off then on: the base model produced an unrelated photograph, the adapter produced the trained concept. Magenta-pixel fraction 0.0% → 71.9%, mean metric −1.87 → +157.45. 1000 steps at 512×512 in 29 minutes.
  9. Sample at the resolution you trained at. Token count, RoPE range and the inference-time timestep shift all depend on it, so a 512-trained adapter used at 1024 loses much of its effect. That is an easy way to conclude a working adapter is broken.
  10. Load it at inference the same way: encode the prompt, free the encoders, load the NF4 transformer, then PeftModel.from_pretrained(pipe.transformer, "/output/flux-lora-a4500"). Generation peaks at 8.8 GiB, so it fits on the same card that trained it.

Concepts used: GPU, Image build, Volume, Secret

Contact the team

Loading the form…