Developer docs / AI Fabric tutorials

RAG over PDF documents

  • GPU
  • Web endpoint
  • Volume
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

rag_pdf.py
import bladekit

app = bladekit.app("rag-pdf")

index_vol = bladekit.Volume.from_name("rag-index", create_if_missing=True)

# fabric/torch already ships torch, so add only the RAG libs (incl. fpdf2 so the
# entrypoint can synthesise a sample PDF with no external file).
image = (
    bladekit.Image.from_catalog("fabric/torch")
    .pip_install(
        "faiss-cpu", "sentence-transformers",
        "pypdf", "fpdf2", "numpy"
    )
)

# ── Step 1: Build the index from raw PDF bytes, persist to the Volume ──
@app.function(gpu="RTX-A4500", image=image, timeout=1800, volumes={"/index": index_vol})
def build_index(pdf_bytes: bytes) -> int:
    import io, faiss, numpy as np, pickle
    from sentence_transformers import SentenceTransformer
    from pypdf import PdfReader

    model = SentenceTransformer("all-MiniLM-L6-v2", device="cuda")
    reader = PdfReader(io.BytesIO(pdf_bytes))
    texts, sources = [], []
    for i, page in enumerate(reader.pages):
        chunk = (page.extract_text() or "").strip()
        if chunk:
            texts.append(chunk)
            sources.append(f"page{i+1}")

    vecs = model.encode(texts, batch_size=64, show_progress_bar=False)
    index = faiss.IndexFlatL2(vecs.shape[1])
    index.add(vecs.astype(np.float32))
    faiss.write_index(index, "/index/faiss.index")
    with open("/index/metadata.pkl", "wb") as f:
        pickle.dump({"texts": texts, "sources": sources}, f)
    # Writes are local to this container until committed; without this the
    # deployed `ask` endpoint (a different container, maybe a different node)
    # would keep serving whatever generation it last saw.
    index_vol.commit()
    return len(texts)

# ── Step 2: Serve Q&A (loads the index from the Volume) ──
_state = {}

@app.web_endpoint(methods=["POST"], min_replicas=0, max_replicas=2)
@app.function(gpu="RTX-A4500", image=image, timeout=120, volumes={"/index": index_vol})
@app.concurrent(max_inputs=8)
def ask(item: dict) -> dict:
    import faiss, numpy as np, pickle
    from sentence_transformers import SentenceTransformer
    from transformers import AutoModelForSeq2SeqLM, AutoTokenizer

    if "embed" not in _state:
        _state["embed"] = SentenceTransformer("all-MiniLM-L6-v2", device="cuda")
        _state["idx"] = faiss.read_index("/index/faiss.index")
        with open("/index/metadata.pkl", "rb") as f:
            _state["meta"] = pickle.load(f)
        # flan-t5 is seq2seq, so load it directly rather than through the
        # pipeline() task registry (task names there churn across
        # transformers releases; the model classes are stable).
        _state["tok"] = AutoTokenizer.from_pretrained("google/flan-t5-base")
        _state["qa"] = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-base").to("cuda")

    question = item.get("question", "")
    q_vec = _state["embed"].encode([question]).astype(np.float32)
    _, idxs = _state["idx"].search(q_vec, k=3)
    meta = _state["meta"]
    context = " ".join(meta["texts"][i] for i in idxs[0])
    prompt = f"Answer based on context: {context}\n\nQuestion: {question}"
    input_ids = _state["tok"](prompt, return_tensors="pt").input_ids.to("cuda")
    output_ids = _state["qa"].generate(input_ids, max_new_tokens=64)
    answer = _state["tok"].decode(output_ids[0], skip_special_tokens=True)
    return {"answer": answer, "sources": [meta["sources"][i] for i in idxs[0]]}

@app.local_entrypoint()
def main():
    # Self-contained: synthesise a small PDF in-code (no file to supply), build
    # the index on the GPU, then ask a question against it.
    from fpdf import FPDF

    pdf = FPDF()
    pdf.add_page()
    pdf.set_font("Helvetica", size=12)
    paragraphs = [
        "AI Fabric is a serverless GPU platform. Functions scale to zero when idle.",
        "The refund policy: customers may request a full refund within 30 days.",
        "Volumes persist data across runs and are mounted into containers.",
    ]
    for p in paragraphs:
        pdf.multi_cell(0, 8, p)
        pdf.ln(2)
    pdf_bytes = bytes(pdf.output())

    n = build_index.remote(pdf_bytes)
    print(f"Indexed {n} chunks.")
    answer = ask.remote({"question": "What is the refund policy?"})
    print("Answer:", answer["answer"])

Run it

  1. Run blade run rag_pdf.py. The entrypoint generates a small sample PDF in-code, builds the FAISS index on a GPU, writes it to the rag-index Volume, then asks a question against it.
  2. Deploy the Q&A endpoint to serve queries over HTTP: blade deploy rag_pdf.py.
  3. Query it: curl -X POST <URL> -H 'Authorization: Bearer <FABRIC_TOKEN>' -d '{"question": "What is the refund policy?"}'. Endpoints are private by default, so pass your AI Fabric token (or deploy with public=True for an unauthenticated endpoint).
  4. The index is persisted in the rag-index Volume. To index your own PDFs, read their bytes and call build_index.remote(pdf_bytes); rebuild only when your documents change.

Concepts used: Endpoint, Replica, Scale-to-zero

Contact the team

Loading the form…