Developer docs / AI Fabric tutorials
Reduce cold starts with Volume cache
- GPU
- Volume
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("llm-cold-start")
# Weights persist across deploys in this Volume.
weights_vol = bladekit.Volume.from_name("model-weights-cache", create_if_missing=True)
MODEL = "Qwen/Qwen2.5-1.5B-Instruct"
# Full snapshot lives at this path on the Volume after the first download.
LOCAL_DIR = "/weights/Qwen2.5-1.5B-Instruct"
# fabric/torch already ships torch, so add only what we need on top.
image = (
bladekit.Image.from_catalog("fabric/torch")
.pip_install("transformers", "huggingface_hub", "accelerate")
)
@app.function(image=image, volumes={"/weights": weights_vol}, timeout=1800)
def download_weights() -> None:
from huggingface_hub import snapshot_download
# Pull the complete repo into a plain directory on the Volume.
snapshot_download(MODEL, local_dir=LOCAL_DIR)
# Commit is required: writes only become visible to *other* containers
# once the Volume is committed; without this the cache never persists.
weights_vol.commit()
print(f"Downloaded {MODEL} to {LOCAL_DIR}")
_model = None
@app.function(gpu="RTX-A4500", image=image, timeout=600, volumes={"/weights": weights_vol})
def generate(prompt: str) -> str:
global _model
import os, torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from huggingface_hub import snapshot_download
if _model is None:
# If the snapshot isn't on the Volume yet, fetch it once. After that,
# every cold start loads from local disk, with no network round-trip.
if not os.path.isdir(LOCAL_DIR):
snapshot_download(MODEL, local_dir=LOCAL_DIR)
weights_vol.commit()
tok = AutoTokenizer.from_pretrained(LOCAL_DIR)
model = AutoModelForCausalLM.from_pretrained(
LOCAL_DIR, torch_dtype=torch.float16, device_map="cuda"
)
_model = (tok, model)
tok, model = _model
messages = [{"role": "user", "content": prompt}]
chat = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tok(chat, return_tensors="pt").to("cuda")
out = model.generate(**inputs, max_new_tokens=128, do_sample=False)
n_in = inputs["input_ids"].shape[1]
return tok.decode(out[0][n_in:], skip_special_tokens=True).strip()
@app.local_entrypoint()
def main():
# Download once into the Volume, then serve from local disk.
download_weights.remote()
print(generate.remote("Summarize what a cold start is in one sentence."))Run it
- Run
blade run llm_cold_start.pyonce.download_weightssnapshots the full model repo into themodel-weights-cacheVolume at/weights, then callsweights_vol.commit()so the write is visible to other containers. generateloads the model from the local snapshot directory (from_pretrained(LOCAL_DIR)), so every subsequent cold start skips the HuggingFace download. Skipping the download alone doesn't make the whole cold start N× faster (GPU model load still dominates for a small model like this), but it removes the network round-trip and its variance entirely.- The Volume persists between deploys so you only pay for the download once. The serve function also self-heals: if the snapshot is missing it fetches it once and commits before continuing.
- The commit step is required, not optional: without it, writes stay local to the container that made them and every future cold start re-downloads from HuggingFace.
- Use
blade volume ls model-weights-cacheto inspect what's cached.
Concepts used: Cold start, Image build, GPU