Developer docs / AI Fabric tutorials

Multimodal RAG over PDF (VLM embeddings)

  • GPU
  • Volume
  • Web endpoint
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

rag_multimodal.py
import bladekit

app = bladekit.app("rag-multimodal")

index_vol = bladekit.Volume.from_name("colpali-index", create_if_missing=True)

# colpali-engine 0.3.6 pulls transformers 4.46.x, the version whose model layout
# matches the colpali-v1.2 adapter checkpoint (later transformers renamed the
# PaliGemma submodules, which silently drops the LoRA weights). We pin 0.3.6
# rather than 0.3.5 because 0.3.5 caps numpy<2, which has no Python 3.13 wheel and
# fails to build from source (no compiler in the image); 0.3.6 leaves numpy
# unpinned so a prebuilt numpy 2.x wheel resolves. fabric/torch already ships
# torch, so we don't reinstall torch on top of it.
#
# peft is capped to <0.12 (the range colpali-engine's own extras use): newer peft
# reaches for torch.distributed.tensor.DTensor, which the base torch build does
# not expose, and crashes while injecting the adapter.
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        "colpali-engine==0.3.6", "peft==0.11.1", "Pillow", "faiss-cpu"
    )
)

# A ColPali model embeds each PAGE IMAGE into a multi-vector representation; we
# mean-pool to a single vector per page so a flat FAISS index suffices for a demo.
MODEL = "vidore/colpali-v1.2"

def _make_sample_pages():
    """Synthesise a couple of 'document page' images in-code (no PDF needed)."""
    from PIL import Image as PILImage, ImageDraw
    pages_text = [
        "Section 1: AI Fabric is a serverless GPU platform.",
        "Section 3: Refunds are available within 30 days of purchase.",
    ]
    pages = []
    for txt in pages_text:
        img = PILImage.new("RGB", (640, 880), "white")
        d = ImageDraw.Draw(img)
        d.text((40, 60), txt, fill="black")
        pages.append(img)
    return pages

@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def build_index() -> int:
    import numpy as np, faiss, pickle, torch
    from colpali_engine.models import ColPali, ColPaliProcessor

    model = ColPali.from_pretrained(MODEL, torch_dtype=torch.bfloat16).to("cuda").eval()
    processor = ColPaliProcessor.from_pretrained(MODEL)

    pages = _make_sample_pages()
    inputs = processor.process_images(pages).to("cuda")
    with torch.no_grad():
        emb = model(**inputs)              # (n_pages, seq, dim)
    vecs = emb.float().mean(dim=1).cpu().numpy()   # mean-pool -> (n_pages, dim)

    faiss.normalize_L2(vecs)
    index = faiss.IndexFlatIP(vecs.shape[1])
    index.add(vecs)
    faiss.write_index(index, "/index/colpali.faiss")
    with open("/index/pages.pkl", "wb") as f:
        pickle.dump({"n_pages": len(pages)}, f)
    # Publish the writes as a new volume generation so the `ask` endpoint (a
    # separate container) can read them; without commit() they stay container-local.
    index_vol.commit()
    return len(pages)

# Model + index are loaded once per container and cached in a module global so
# warm invocations skip the (slow) model load.
_state = {}

def _search(question: str) -> dict:
    import faiss, torch
    from colpali_engine.models import ColPali, ColPaliProcessor

    if "model" not in _state:
        _state["model"] = ColPali.from_pretrained(MODEL, torch_dtype=torch.bfloat16).to("cuda").eval()
        _state["proc"] = ColPaliProcessor.from_pretrained(MODEL)
        _state["index"] = faiss.read_index("/index/colpali.faiss")

    q_inputs = _state["proc"].process_queries([question]).to("cuda")
    with torch.no_grad():
        q_emb = _state["model"](**q_inputs).float().mean(dim=1).cpu().numpy()
    faiss.normalize_L2(q_emb)
    _, pages = _state["index"].search(q_emb, k=2)
    return {"question": question, "top_pages": pages[0].tolist()}

# Plain function for the local-run demo: build_index writes the index, then this
# queries it. (The HTTP endpoint below serves the same logic to real clients.)
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def query(question: str) -> dict:
    return _search(question)

@app.web_endpoint(methods=["POST"], min_replicas=0, max_replicas=2)
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def ask(item: dict) -> dict:
    return _search(item.get("question", ""))

@app.local_entrypoint()
def main():
    # Self-contained: build the index from synthetic page images, then query it.
    n = build_index.remote()
    print(f"Indexed {n} pages.")
    out = query.remote("What is the refund policy?")
    print("Top matching pages:", out["top_pages"])

Run it

  1. Run blade run rag_multimodal.py. It synthesises a couple of document-page images in-code, embeds them with ColPali on a GPU, commits the index to the colpali-index Volume, then runs a query. You'll see Indexed 2 pages. followed by Top matching pages: [1, 0] (the refund page ranks first for a refund question).
  2. Deploy the Q&A endpoint to serve queries over HTTP: blade deploy rag_multimodal.py.
  3. Query: curl -X POST <URL>/ask -H 'Authorization: Bearer <FABRIC_TOKEN>' -d '{"question": "What is the refund policy?"}'. Endpoints are private by default, so pass your AI Fabric token (or deploy with public=True for an unauthenticated endpoint). The endpoint scales to zero, so the first request cold-starts the model (retry once if it 5xxs while waking).
  4. ColPali embeds whole page IMAGES, so it understands tables and figures that plain text extraction misses. The stack is pinned to colpali-engine==0.3.6 (transformers 4.46.x) because newer transformers rename the PaliGemma submodules and silently drop the adapter's LoRA weights. Swap _make_sample_pages() for pdf2image.convert_from_path(...) (add poppler-utils) to index real PDFs.

Concepts used: Endpoint, Replica, Scale-to-zero, GPU

Contact the team

Loading the form…