Developer docs / AI Fabric tutorials
Multimodal RAG over PDF (VLM embeddings)
- GPU
- Volume
- Web endpoint
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("rag-multimodal")
index_vol = bladekit.Volume.from_name("colpali-index", create_if_missing=True)
# colpali-engine 0.3.6 pulls transformers 4.46.x, the version whose model layout
# matches the colpali-v1.2 adapter checkpoint (later transformers renamed the
# PaliGemma submodules, which silently drops the LoRA weights). We pin 0.3.6
# rather than 0.3.5 because 0.3.5 caps numpy<2, which has no Python 3.13 wheel and
# fails to build from source (no compiler in the image); 0.3.6 leaves numpy
# unpinned so a prebuilt numpy 2.x wheel resolves. fabric/torch already ships
# torch, so we don't reinstall torch on top of it.
#
# peft is capped to <0.12 (the range colpali-engine's own extras use): newer peft
# reaches for torch.distributed.tensor.DTensor, which the base torch build does
# not expose, and crashes while injecting the adapter.
image = (
bladekit.Image.from_catalog("fabric/torch")
.pip_install(
"colpali-engine==0.3.6", "peft==0.11.1", "Pillow", "faiss-cpu"
)
)
# A ColPali model embeds each PAGE IMAGE into a multi-vector representation; we
# mean-pool to a single vector per page so a flat FAISS index suffices for a demo.
MODEL = "vidore/colpali-v1.2"
def _make_sample_pages():
"""Synthesise a couple of 'document page' images in-code (no PDF needed)."""
from PIL import Image as PILImage, ImageDraw
pages_text = [
"Section 1: AI Fabric is a serverless GPU platform.",
"Section 3: Refunds are available within 30 days of purchase.",
]
pages = []
for txt in pages_text:
img = PILImage.new("RGB", (640, 880), "white")
d = ImageDraw.Draw(img)
d.text((40, 60), txt, fill="black")
pages.append(img)
return pages
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def build_index() -> int:
import numpy as np, faiss, pickle, torch
from colpali_engine.models import ColPali, ColPaliProcessor
model = ColPali.from_pretrained(MODEL, torch_dtype=torch.bfloat16).to("cuda").eval()
processor = ColPaliProcessor.from_pretrained(MODEL)
pages = _make_sample_pages()
inputs = processor.process_images(pages).to("cuda")
with torch.no_grad():
emb = model(**inputs) # (n_pages, seq, dim)
vecs = emb.float().mean(dim=1).cpu().numpy() # mean-pool -> (n_pages, dim)
faiss.normalize_L2(vecs)
index = faiss.IndexFlatIP(vecs.shape[1])
index.add(vecs)
faiss.write_index(index, "/index/colpali.faiss")
with open("/index/pages.pkl", "wb") as f:
pickle.dump({"n_pages": len(pages)}, f)
# Publish the writes as a new volume generation so the `ask` endpoint (a
# separate container) can read them; without commit() they stay container-local.
index_vol.commit()
return len(pages)
# Model + index are loaded once per container and cached in a module global so
# warm invocations skip the (slow) model load.
_state = {}
def _search(question: str) -> dict:
import faiss, torch
from colpali_engine.models import ColPali, ColPaliProcessor
if "model" not in _state:
_state["model"] = ColPali.from_pretrained(MODEL, torch_dtype=torch.bfloat16).to("cuda").eval()
_state["proc"] = ColPaliProcessor.from_pretrained(MODEL)
_state["index"] = faiss.read_index("/index/colpali.faiss")
q_inputs = _state["proc"].process_queries([question]).to("cuda")
with torch.no_grad():
q_emb = _state["model"](**q_inputs).float().mean(dim=1).cpu().numpy()
faiss.normalize_L2(q_emb)
_, pages = _state["index"].search(q_emb, k=2)
return {"question": question, "top_pages": pages[0].tolist()}
# Plain function for the local-run demo: build_index writes the index, then this
# queries it. (The HTTP endpoint below serves the same logic to real clients.)
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def query(question: str) -> dict:
return _search(question)
@app.web_endpoint(methods=["POST"], min_replicas=0, max_replicas=2)
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def ask(item: dict) -> dict:
return _search(item.get("question", ""))
@app.local_entrypoint()
def main():
# Self-contained: build the index from synthetic page images, then query it.
n = build_index.remote()
print(f"Indexed {n} pages.")
out = query.remote("What is the refund policy?")
print("Top matching pages:", out["top_pages"])Run it
- Run
blade run rag_multimodal.py. It synthesises a couple of document-page images in-code, embeds them with ColPali on a GPU, commits the index to thecolpali-indexVolume, then runs a query. You'll seeIndexed 2 pages.followed byTop matching pages: [1, 0](the refund page ranks first for a refund question). - Deploy the Q&A endpoint to serve queries over HTTP:
blade deploy rag_multimodal.py. - Query:
curl -X POST <URL>/ask -H 'Authorization: Bearer <FABRIC_TOKEN>' -d '{"question": "What is the refund policy?"}'. Endpoints are private by default, so pass your AI Fabric token (or deploy withpublic=Truefor an unauthenticated endpoint). The endpoint scales to zero, so the first request cold-starts the model (retry once if it 5xxs while waking). - ColPali embeds whole page IMAGES, so it understands tables and figures that plain text extraction misses. The stack is pinned to
colpali-engine==0.3.6(transformers 4.46.x) because newer transformers rename the PaliGemma submodules and silently drop the adapter's LoRA weights. Swap_make_sample_pages()forpdf2image.convert_from_path(...)(addpoppler-utils) to index real PDFs.
Concepts used: Endpoint, Replica, Scale-to-zero, GPU