Developer docs / AI Fabric tutorials
Flux LoRA fine-tuning on a 20 GB GPU
- GPU
- Volume
- Secret
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("flux-lora-a4500")
data_vol = bladekit.Volume.from_name("flux-lora-data", create_if_missing=True)
out_vol = bladekit.Volume.from_name("flux-lora-output", create_if_missing=True)
hf_cache = bladekit.Volume.from_name("hf-cache", create_if_missing=True)
# fabric/torch already ships torch, so don't reinstall it. bitsandbytes provides the
# NF4 quantiser and the 8-bit optimiser; sentencepiece + protobuf are needed by
# the T5 tokenizer.
image = (
bladekit.Image.from_catalog("fabric/torch")
.pip_install(
"diffusers>=0.31.0", "transformers", "accelerate", "peft",
"bitsandbytes>=0.45.0", "Pillow", "sentencepiece", "protobuf",
"hf_transfer",
)
.env({"HF_HUB_ENABLE_HF_TRANSFER": "1", "HF_HOME": "/hf-cache"})
)
@app.function(
gpu="RTX-A4500", # 20 GB card, ~19.6 GiB usable (the whole point here)
image=image,
timeout=7200,
memory=32768, # host RAM: the bf16 shards are read before quantising
volumes={"/data": data_vol, "/output": out_vol, "/hf-cache": hf_cache},
secrets=[bladekit.Secret.from_name("hf-token")], # FLUX.1-dev is a gated repo
)
def train_flux_lora(
base_model: str = "black-forest-labs/FLUX.1-dev",
instance_prompt: str = "a photo of sks style",
resolution: int = 512, # 1024 also fits (10.79 GiB peak), ~5x slower
num_train_steps: int = 1000,
lora_rank: int = 16,
learning_rate: float = 2e-4,
guidance_scale: float = 3.5, # MUST match what you generate with
) -> str:
import gc, glob, os
import numpy as np
import torch
def vram(tag):
torch.cuda.synchronize()
print(f"[vram] {tag:<28} alloc={torch.cuda.memory_allocated()/1024**3:5.2f}"
f" peak={torch.cuda.max_memory_allocated()/1024**3:5.2f} GiB", flush=True)
# The cuDNN SDPA backward kernel crashes with native CUDA errors on this
# torch build, so force the flash/math attention paths for training.
torch.backends.cuda.enable_cudnn_sdp(False)
device, dtype = torch.device("cuda"), torch.bfloat16
# Your images go on the /data Volume. Anything else is a placeholder.
from PIL import Image as PILImage, ImageDraw
paths = sorted(p for ext in ("jpg", "jpeg", "png", "webp")
for p in glob.glob(f"/data/**/*.{ext}", recursive=True))
if paths:
print(f"using {len(paths)} image(s) from /data", flush=True)
images = [PILImage.open(p).convert("RGB").resize((resolution, resolution))
for p in paths]
else:
# No dataset yet: a deliberately unmistakable concept (magenta disc on
# white) so you can SEE whether the adapter took, without a real dataset.
print("no images in /data, synthesising 8 magenta discs", flush=True)
images = []
for i in range(8):
img = PILImage.new("RGB", (resolution, resolution), "white")
pad = 60 + i * 8
ImageDraw.Draw(img).ellipse(
[pad, pad, resolution - pad, resolution - pad], fill=(255, 0, 200))
images.append(img)
# ── PASS 1: cache prompt embeddings + latents, then free everything ─────
# transformer=None loads ONLY the text encoders and the VAE. T5-XXL alone is
# ~9.5 GB; it must never be resident at the same time as the transformer.
from diffusers import FluxPipeline
pipe = FluxPipeline.from_pretrained(
base_model, transformer=None, torch_dtype=dtype,
token=os.environ["HF_TOKEN"],
).to(device)
vram("text encoders + VAE")
with torch.no_grad():
prompt_embeds, pooled_embeds, text_ids = pipe.encode_prompt(
prompt=instance_prompt, prompt_2=None, device=device,
max_sequence_length=256,
)
latents = []
for img in images:
px = (torch.from_numpy(np.asarray(img, dtype=np.uint8).copy())
.permute(2, 0, 1).unsqueeze(0).to(device, dtype))
px = px / 127.5 - 1.0 # -> [-1, 1]
lat = pipe.vae.encode(px).latent_dist.sample()
lat = (lat - pipe.vae.config.shift_factor) * pipe.vae.config.scaling_factor
latents.append(lat.cpu())
cached = torch.cat(latents, dim=0)
prompt_embeds, pooled_embeds, text_ids = (
prompt_embeds.cpu(), pooled_embeds.cpu(), text_ids.cpu())
vram("after encoding")
# This del is load-bearing: it is what makes the fit work.
del pipe, latents
gc.collect(); torch.cuda.empty_cache(); torch.cuda.reset_peak_memory_stats()
vram("encoders freed")
# ── PASS 2: NF4 transformer + LoRA on top (QLoRA) ──────────────────────
from diffusers import BitsAndBytesConfig, FluxTransformer2DModel
transformer = FluxTransformer2DModel.from_pretrained(
base_model, subfolder="transformer",
quantization_config=BitsAndBytesConfig(
load_in_4bit=True, bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=dtype),
torch_dtype=dtype, token=os.environ["HF_TOKEN"],
)
vram("NF4 transformer") # ~6.3 GiB instead of ~24 GiB in bf16
transformer.enable_gradient_checkpointing()
from peft import LoraConfig, get_peft_model
# Attention projections alone (to_q/k/v/out) are not enough capacity: adding
# the MLPs took the training loss from ~0.014 to ~0.004 over the same 1000
# steps, at 65.4M trainable params instead of 18.7M (131 MB adapter, +0.6 GiB).
transformer = get_peft_model(transformer, LoraConfig(
r=lora_rank, lora_alpha=lora_rank,
target_modules=[
"to_q", "to_k", "to_v", "to_out.0",
"add_q_proj", "add_k_proj", "add_v_proj", "to_add_out",
"ff.net.0.proj", "ff.net.2",
"ff_context.net.0.proj", "ff_context.net.2",
"proj_mlp", "proj_out",
],
lora_dropout=0.0, bias="none",
))
# PEFT creates adapters in fp32; bf16 matches the compute dtype.
for p in transformer.parameters():
if p.requires_grad:
p.data = p.data.to(dtype)
print(f"trainable LoRA params: "
f"{sum(p.numel() for p in transformer.parameters() if p.requires_grad)/1e6:.1f}M",
flush=True)
import bitsandbytes as bnb
optimizer = bnb.optim.AdamW8bit(
[p for p in transformer.parameters() if p.requires_grad], lr=learning_rate)
prompt_embeds = prompt_embeds.to(device)
pooled_embeds = pooled_embeds.to(device)
text_ids = text_ids.to(device)
transformer.train()
losses = []
# Loss per sigma band (0-0.2 ... 0.8-1.0); see the note inside the loop.
band_sum, band_n = [0.0] * 5, [0] * 5
for step in range(num_train_steps):
lat = cached[step % cached.shape[0]].unsqueeze(0).to(device, dtype)
b, c, h, w = lat.shape
# Pack latents (B, C, H, W) -> (B, H/2*W/2, C*4) as Flux expects.
packed = (lat.view(b, c, h // 2, 2, w // 2, 2)
.permute(0, 2, 4, 1, 3, 5)
.reshape(b, (h // 2) * (w // 2), c * 4))
img_ids = torch.zeros(h // 2, w // 2, 3, device=device, dtype=dtype)
img_ids[..., 1] = torch.arange(h // 2, device=device)[:, None]
img_ids[..., 2] = torch.arange(w // 2, device=device)[None, :]
img_ids = img_ids.reshape(-1, 3)
if text_ids.ndim == 3: # older diffusers use batched ids
img_ids = img_ids.unsqueeze(0)
# Flow-matching objective: predict (noise - clean) at a random sigma.
noise = torch.randn_like(packed)
# THE trap, and the whole reason this recipe works. Sampling sigma only in
# a comfortable middle band (e.g. 0.1..0.9) trains NOTHING at sigma > 0.9,
# including sigma = 1.0, the pure noise every sampling run starts from, and
# the band where composition and dominant colour are committed. The loop
# still converges and the adapter still measurably shifts a forward pass,
# yet generated images do not change at all. Cover the full range, like the
# reference trainer (uniform over the scheduler grid, UNSHIFTED, since the
# resolution-dependent shift belongs to inference, not to training).
t = torch.rand(b, device=device, dtype=dtype).clamp(1e-3, 1.0 - 1e-3)
noisy = (1 - t[:, None, None]) * packed + t[:, None, None] * noise
# FLUX.1-dev is guidance-distilled: the guidance value is an INPUT
# embedding, not a sampler trick. Train at the value you will generate
# at. A mismatch measurably weakens the adapter (see the notes).
guidance = None
if getattr(transformer.config, "guidance_embeds", False):
guidance = torch.full((b,), guidance_scale, device=device, dtype=dtype)
pred = transformer(
hidden_states=noisy, timestep=t, guidance=guidance,
pooled_projections=pooled_embeds, encoder_hidden_states=prompt_embeds,
txt_ids=text_ids, img_ids=img_ids, return_dict=False,
)[0]
loss = torch.nn.functional.mse_loss(pred.float(), (noise - packed).float())
loss.backward()
optimizer.step()
optimizer.zero_grad()
losses.append(loss.item())
# The scalar mean loss CANNOT tell you whether the high-sigma band is
# learning: bands differ in intrinsic loss scale by orders of magnitude, so
# the average is dominated by the easy low-sigma end. Watch the top band.
band = min(int(t.item() * 5), 4)
band_sum[band] += loss.item()
band_n[band] += 1
if step == 0:
vram("after first step")
if step % 25 == 0 or step == num_train_steps - 1:
print(f"step {step + 1}/{num_train_steps} loss={loss.item():.4f}", flush=True)
vram("end of training")
print(f"mean loss: first 25 = {sum(losses[:25])/len(losses[:25]):.4f} | "
f"last 25 = {sum(losses[-25:])/len(losses[-25:]):.4f}", flush=True)
for i in range(5):
mean = band_sum[i] / band_n[i] if band_n[i] else float("nan")
print(f" sigma {i*0.2:.1f}-{(i+1)*0.2:.1f}: loss {mean:.4f} (n={band_n[i]})",
flush=True)
out = "/output/flux-lora-a4500"
transformer.save_pretrained(out)
out_vol.commit()
return out
@app.local_entrypoint()
def main():
print(f"LoRA adapter saved to: {train_flux_lora.remote()}")Run it
- Put your training images on the
flux-lora-dataVolume and add your HF token (blade secret set hf-token HF_TOKEN=...), since FLUX.1-dev is a gated repo. With an empty Volume the snippet synthesises magenta discs, which is deliberate: the concept is unmistakable, so you can tell whether the adapter took without needing a real dataset. - Why it fits where the bf16 recipe does not: the transformer is loaded in NF4 (6.3 GiB resident instead of ~24 GiB) and LoRA trains on top of it: QLoRA, exactly as on an LLM. Measured on an A4500 at 1024×1024: 10.8 GiB peak in pass 1, 8.5 GiB in pass 2, on a card with 19.6 GiB usable.
- What makes it fit is the ordering, not the quantiser. Pass 1 loads the pipeline with
transformer=None, caches the prompt embeddings and the VAE latents, then frees the encoders; only then is the transformer loaded. T5-XXL (~9.5 GB) and the transformer are never resident together. Keep them both and you OOM before the first step. - The other platform fix:
torch.backends.cuda.enable_cudnn_sdp(False)before training. The cuDNN SDPA backward kernel crashes with native CUDA errors on the current torch build; the flash/math attention paths are fine. - Match the guidance value between training and generation. FLUX.1-dev is guidance-distilled (the value is an input embedding, not a sampler trick), so the adapter learns deltas conditioned on it. Training at 1.0 and generating at 3.5 cut the measured effect ~140× versus a matched pair.
- The failure mode to watch for: a loop that converges well and still changes nothing. Sampling sigma in a comfortable middle band trains no gradient step at sigma > 0.9, and every sampling run starts at sigma = 1.0, where composition and dominant colour are decided. In that state the training loss fell 8×, the adapter was provably applied (838 injected modules, a 31.7% forward-pass shift), and generated images were pixel-for-pixel uninteresting: 0.0% of the target concept, across four training runs. Covering the full sigma range is a one-line change and the entire difference.
- Because of that, do not trust the scalar mean loss. Bands differ in intrinsic loss scale by orders of magnitude, so the average is dominated by the easy low-sigma end and stays silent about the band that decides your image. The snippet prints loss per sigma band for this reason. The top band is the one to watch.
- Tested end to end on an A4500, same seed and same prompt embeddings, adapter off then on: the base model produced an unrelated photograph, the adapter produced the trained concept. Magenta-pixel fraction 0.0% → 71.9%, mean metric −1.87 → +157.45. 1000 steps at 512×512 in 29 minutes.
- Sample at the resolution you trained at. Token count, RoPE range and the inference-time timestep shift all depend on it, so a 512-trained adapter used at 1024 loses much of its effect. That is an easy way to conclude a working adapter is broken.
- Load it at inference the same way: encode the prompt, free the encoders, load the NF4 transformer, then
PeftModel.from_pretrained(pipe.transformer, "/output/flux-lora-a4500"). Generation peaks at 8.8 GiB, so it fits on the same card that trained it.
Concepts used: GPU, Image build, Volume, Secret