Developer docs / AI Fabric tutorials
Fast Flux with torch.compile
- GPU
- Volume
- Class
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
import io
app = bladekit.app("flux-compile")
hf_cache = bladekit.Volume.from_name("hf-cache", create_if_missing=True)
image = (
bladekit.Image.from_catalog("fabric/torch")
# torch.compile JIT-builds Triton kernels at runtime, so the container
# needs a C compiler or every request fails with an InductorError.
.apt_install("build-essential")
.pip_install(
"diffusers", "transformers", "accelerate",
"safetensors", "sentencepiece", "torchao", "hf_transfer",
)
.env({"HF_HOME": "/hf-cache", "HF_HUB_ENABLE_HF_TRANSFER": "1"})
)
MODEL = "black-forest-labs/FLUX.1-schnell"
@app.cls(
gpu="RTX-A4500",
image=image,
timeout=1800,
# The bf16 checkpoint (~24 GB) is loaded and quantized in host RAM
# before moving to the GPU; the 16 GB default gets OOM-killed.
memory=49152,
volumes={"/hf-cache": hf_cache},
# FLUX.1-schnell is a gated HF repo, so inject a token via a Secret.
secrets=[bladekit.Secret.from_name("hf-token")],
)
class FluxCompiled:
def __init__(self):
import torch
from diffusers import FluxPipeline, FluxTransformer2DModel, TorchAoConfig
from torchao.quantization import Int8WeightOnlyConfig, quantize_
# bf16 FLUX (~24 GB transformer alone) exceeds a 20 GB card, and CPU
# offload is incompatible with torch.compile. int8 weight-only
# quantization keeps the compiled transformer fully resident (~12 GB).
transformer = FluxTransformer2DModel.from_pretrained(
MODEL,
subfolder="transformer",
quantization_config=TorchAoConfig(Int8WeightOnlyConfig()),
torch_dtype=torch.bfloat16,
)
self.pipe = FluxPipeline.from_pretrained(
MODEL, transformer=transformer, torch_dtype=torch.bfloat16
)
# Quantize the 4.7B T5 text encoder too so the whole pipeline fits.
quantize_(self.pipe.text_encoder_2, Int8WeightOnlyConfig())
self.pipe.to("cuda")
self.pipe.vae.enable_slicing()
self.pipe.vae.enable_tiling()
# Compile the transformer once. The first request pays the compile
# cost (~10 min on an A4500), later requests reuse the kernels.
# fullgraph=True traces the whole forward pass as one graph.
self.pipe.transformer = torch.compile(
self.pipe.transformer, fullgraph=True
)
# min_replicas=1 keeps one replica warm: model load + compile take
# minutes, far beyond the gateway's cold-start wake window.
@app.web_endpoint(methods=["POST"], min_replicas=1)
def generate(self, item: dict) -> bytes:
import time
prompt = item.get("prompt", "a photon shooting through a prism")
t0 = time.time()
result = self.pipe(
prompt,
num_inference_steps=4,
guidance_scale=0.0, # schnell is guidance-distilled
height=768,
width=768,
)
print(f"[flux-compile] generated in {time.time() - t0:.1f}s")
buf = io.BytesIO()
result.images[0].save(buf, format="PNG")
return buf.getvalue()Run it
- Deploy:
blade deploy flux_compile.py. - int8 weight-only quantization (torchao) is what makes compile possible here: bf16 FLUX (~24 GB transformer) exceeds the A4500, and CPU offload is incompatible with
torch.compile. Quantized, the whole pipeline sits resident in ~12 GB. build-essentialin the image matters:torch.compileJIT-builds Triton kernels at runtime and fails with anInductorErrorwithout a C compiler.- The FIRST request pays the inductor compile of the 12B transformer (~10 min on the A4500). Call with a generous timeout (
curl --max-time 1200) or poll. Warm requests reuse the kernels (measured 1.7× faster end-to-end). - Caveat from live verification: on the A4500 a warm call still spends several minutes re-specializing step 1. The compile win is modest on this card; on larger GPUs (unquantized, resident) the speedup is much sharper.
Concepts used: Cold start, Replica, GPU, Image build