Developer docs / AI Fabric tutorials

GRPO + veRL math reasoning (RL training)

  • GPU
  • Volume
  • Image DSL

Take it for a spin

The whole program in one file, ready to run.

grpo_verl_maths.py
import bladekit

app = bladekit.app("grpo-verl-maths")

checkpoints = bladekit.Volume.from_name("grpo-checkpoints", create_if_missing=True)

# Prebuilt flash-attn wheel matching torch 2.6 (pulled in by vllm 0.8.5) on
# Python 3.12, which avoids a multi-hour source build.
FLASH_ATTN_WHEEL = (
    "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/"
    "v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE"
    "-cp312-cp312-linux_x86_64.whl"
)

image = (
    bladekit.Image.debian_slim(python_version="3.12")
    .pip_install("vllm==0.8.5")  # brings torch 2.6.0 (CUDA 12.4)
    .pip_install(FLASH_ATTN_WHEEL)
    # setuptools<81: verl 0.4.x imports pkg_resources, removed in setuptools 81+
    .pip_install("verl==0.4.1", "transformers==4.51.3", "datasets", "accelerate", "setuptools<81")
    .env(
        {
            "HF_HOME": "/checkpoints/hf-cache",   # cache models on the Volume
            "NCCL_P2P_DISABLE": "1",              # A4500s have no NVLink
            "VLLM_USE_V1": "1",
            "VLLM_USE_FLASHINFER_SAMPLER": "0",   # required on sm_86 GPUs
        }
    )
)


@app.function(
    gpu="RTX-A4500:8",
    image=image,
    timeout=7200,
    memory=98304,  # 96 GiB: the 16 GiB default is too small for 8 Ray workers
    volumes={"/checkpoints": checkpoints},
)
def train_grpo(
    model_name: str = "Qwen/Qwen2.5-0.5B-Instruct",
    num_train_samples: int = 64,   # bump to the full 7.5k set for a real run
    num_val_samples: int = 32,
    total_training_steps: int = 2,  # smoke-test length; raise for real training
) -> str:
    import os
    import time

    # Bounded HF Hub timeouts: fail fast and retry instead of hanging the run.
    os.environ["HF_HUB_DOWNLOAD_TIMEOUT"] = "60"
    os.environ["HF_HUB_ETAG_TIMEOUT"] = "60"
    # The container's /dev/shm is small (64MB): keep NCCL off SHM transport.
    os.environ["NCCL_SHM_DISABLE"] = "1"
    os.environ["TOKENIZERS_PARALLELISM"] = "true"

    import pandas as pd
    from datasets import load_dataset
    from huggingface_hub import snapshot_download

    def with_retries(fn, what: str, attempts: int = 4):
        """The Hub occasionally times out; retry instead of failing the run."""
        for attempt in range(attempts):
            try:
                return fn()
            except Exception as exc:
                if attempt == attempts - 1:
                    raise
                print(f"[retry] {what} failed ({exc!r}), retrying in 20s", flush=True)
                time.sleep(20)

    # -- 1. Pre-download the model so training never hits the network mid-run
    with_retries(lambda: snapshot_download(model_name), f"download {model_name}")

    # -- 2. Prepare GSM8K in verl's parquet format ---------------------------
    data_dir = "/checkpoints/data/gsm8k"
    os.makedirs(data_dir, exist_ok=True)
    ds = with_retries(lambda: load_dataset("openai/gsm8k", "main"), "load gsm8k")
    instruction = 'Let\'s think step by step and output the final answer after "####".'

    def rows(split: str, n: int):
        out = []
        for idx, ex in enumerate(ds[split].select(range(n))):
            answer = ex["answer"].split("####")[1].strip().replace(",", "")
            out.append(
                {
                    "data_source": "openai/gsm8k",
                    "prompt": [{"role": "user", "content": f'{ex["question"]} {instruction}'}],
                    "ability": "math",
                    # verl's built-in rule reward grades against this answer
                    "reward_model": {"style": "rule", "ground_truth": answer},
                    "extra_info": {"split": split, "index": idx},
                }
            )
        return out

    pd.DataFrame(rows("train", num_train_samples)).to_parquet(f"{data_dir}/train.parquet")
    pd.DataFrame(rows("test", num_val_samples)).to_parquet(f"{data_dir}/val.parquet")

    # -- 3. Launch GRPO via verl's PPO trainer, in-process --------------------
    # Initialize Ray ourselves with a small CPU count: with the default (one
    # worker per core) the prestarted workers wedge against the container's
    # 64MB /dev/shm and ray.init never returns. verl reuses our instance.
    import ray

    ray.init(num_cpus=16, num_gpus=8)

    overrides = [
        "algorithm.adv_estimator=grpo",
        f"data.train_files={data_dir}/train.parquet",
        f"data.val_files={data_dir}/val.parquet",
        "data.train_batch_size=8",
        "data.max_prompt_length=1024",
        "data.max_response_length=256",
        "data.filter_overlong_prompts=True",
        f"actor_rollout_ref.model.path={model_name}",
        "actor_rollout_ref.model.use_remove_padding=True",
        "actor_rollout_ref.actor.optim.lr=1e-6",
        "actor_rollout_ref.actor.ppo_mini_batch_size=8",
        "actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1",
        "actor_rollout_ref.actor.use_kl_loss=True",
        "actor_rollout_ref.actor.kl_loss_coef=0.001",
        "actor_rollout_ref.actor.kl_loss_type=low_var_kl",
        "actor_rollout_ref.actor.entropy_coeff=0",
        # rollouts run on vLLM; eager mode avoids CUDA-graph crashes on sm_86
        "actor_rollout_ref.rollout.name=vllm",
        "actor_rollout_ref.rollout.tensor_model_parallel_size=1",
        "actor_rollout_ref.rollout.gpu_memory_utilization=0.4",
        "actor_rollout_ref.rollout.enforce_eager=True",
        "actor_rollout_ref.rollout.n=4",  # GRPO group size
        "actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2",
        "actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2",
        "algorithm.use_kl_in_reward=False",
        "trainer.critic_warmup=0",
        "trainer.logger=[console]",
        "trainer.project_name=grpo-gsm8k",
        "trainer.experiment_name=qwen05b-gsm8k",
        "trainer.n_gpus_per_node=8",
        "trainer.nnodes=1",
        "trainer.save_freq=-1",  # set e.g. 10 to checkpoint to the Volume
        "trainer.test_freq=-1",
        "trainer.val_before_train=False",
        "trainer.total_epochs=1",
        f"trainer.total_training_steps={total_training_steps}",
        "trainer.default_local_dir=/checkpoints/grpo-run",
    ]

    import verl.trainer
    from hydra import compose, initialize_config_dir
    from verl.trainer.main_ppo import run_ppo

    cfg_dir = os.path.join(os.path.dirname(verl.trainer.__file__), "config")
    with initialize_config_dir(config_dir=cfg_dir, version_base=None):
        config = compose(config_name="ppo_trainer", overrides=overrides)
    run_ppo(config)
    return "/checkpoints/grpo-run"


@app.local_entrypoint()
def main():
    out = train_grpo.remote()
    print(f"GRPO training complete. Checkpoint at: {out}")

Run it

  1. Save as grpo_verl_maths.py. The image pins Python 3.12 and AI Fabric requires the client Python to MATCH: uv venv -p 3.12 .venv312 && uv pip install --python .venv312/bin/python bladekit, then .venv312/bin/blade run grpo_verl_maths.py.
  2. gpu="RTX-A4500:8": veRL's hybrid engine colocates FSDP training and vLLM rollout generation on the same 8 GPUs of the node (ray.init(num_cpus=16, num_gpus=8); veRL reuses the in-container Ray instance).
  3. GSM8K is downloaded (with retries, since HF can be flaky) and converted to veRL's parquet format inside the container before training starts.
  4. Each GRPO step logs the metrics that matter: critic/rewards/mean (how often sampled answers score), actor/pg_loss, actor/ppo_kl, response lengths and rollout timings.
  5. total_training_steps=2 is smoke-test length (~18 min end-to-end, dominated by model download + engine start). Raise it and swap in a bigger base model for real training.

Concepts used: GPU, Image build

Contact the team

Loading the form…