Developer docs / AI Fabric tutorials
GRPO + veRL math reasoning (RL training)
- GPU
- Volume
- Image DSL
Take it for a spin
The whole program in one file, ready to run.
import bladekit
app = bladekit.app("grpo-verl-maths")
checkpoints = bladekit.Volume.from_name("grpo-checkpoints", create_if_missing=True)
# Prebuilt flash-attn wheel matching torch 2.6 (pulled in by vllm 0.8.5) on
# Python 3.12, which avoids a multi-hour source build.
FLASH_ATTN_WHEEL = (
"flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/"
"v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE"
"-cp312-cp312-linux_x86_64.whl"
)
image = (
bladekit.Image.debian_slim(python_version="3.12")
.pip_install("vllm==0.8.5") # brings torch 2.6.0 (CUDA 12.4)
.pip_install(FLASH_ATTN_WHEEL)
# setuptools<81: verl 0.4.x imports pkg_resources, removed in setuptools 81+
.pip_install("verl==0.4.1", "transformers==4.51.3", "datasets", "accelerate", "setuptools<81")
.env(
{
"HF_HOME": "/checkpoints/hf-cache", # cache models on the Volume
"NCCL_P2P_DISABLE": "1", # A4500s have no NVLink
"VLLM_USE_V1": "1",
"VLLM_USE_FLASHINFER_SAMPLER": "0", # required on sm_86 GPUs
}
)
)
@app.function(
gpu="RTX-A4500:8",
image=image,
timeout=7200,
memory=98304, # 96 GiB: the 16 GiB default is too small for 8 Ray workers
volumes={"/checkpoints": checkpoints},
)
def train_grpo(
model_name: str = "Qwen/Qwen2.5-0.5B-Instruct",
num_train_samples: int = 64, # bump to the full 7.5k set for a real run
num_val_samples: int = 32,
total_training_steps: int = 2, # smoke-test length; raise for real training
) -> str:
import os
import time
# Bounded HF Hub timeouts: fail fast and retry instead of hanging the run.
os.environ["HF_HUB_DOWNLOAD_TIMEOUT"] = "60"
os.environ["HF_HUB_ETAG_TIMEOUT"] = "60"
# The container's /dev/shm is small (64MB): keep NCCL off SHM transport.
os.environ["NCCL_SHM_DISABLE"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "true"
import pandas as pd
from datasets import load_dataset
from huggingface_hub import snapshot_download
def with_retries(fn, what: str, attempts: int = 4):
"""The Hub occasionally times out; retry instead of failing the run."""
for attempt in range(attempts):
try:
return fn()
except Exception as exc:
if attempt == attempts - 1:
raise
print(f"[retry] {what} failed ({exc!r}), retrying in 20s", flush=True)
time.sleep(20)
# -- 1. Pre-download the model so training never hits the network mid-run
with_retries(lambda: snapshot_download(model_name), f"download {model_name}")
# -- 2. Prepare GSM8K in verl's parquet format ---------------------------
data_dir = "/checkpoints/data/gsm8k"
os.makedirs(data_dir, exist_ok=True)
ds = with_retries(lambda: load_dataset("openai/gsm8k", "main"), "load gsm8k")
instruction = 'Let\'s think step by step and output the final answer after "####".'
def rows(split: str, n: int):
out = []
for idx, ex in enumerate(ds[split].select(range(n))):
answer = ex["answer"].split("####")[1].strip().replace(",", "")
out.append(
{
"data_source": "openai/gsm8k",
"prompt": [{"role": "user", "content": f'{ex["question"]} {instruction}'}],
"ability": "math",
# verl's built-in rule reward grades against this answer
"reward_model": {"style": "rule", "ground_truth": answer},
"extra_info": {"split": split, "index": idx},
}
)
return out
pd.DataFrame(rows("train", num_train_samples)).to_parquet(f"{data_dir}/train.parquet")
pd.DataFrame(rows("test", num_val_samples)).to_parquet(f"{data_dir}/val.parquet")
# -- 3. Launch GRPO via verl's PPO trainer, in-process --------------------
# Initialize Ray ourselves with a small CPU count: with the default (one
# worker per core) the prestarted workers wedge against the container's
# 64MB /dev/shm and ray.init never returns. verl reuses our instance.
import ray
ray.init(num_cpus=16, num_gpus=8)
overrides = [
"algorithm.adv_estimator=grpo",
f"data.train_files={data_dir}/train.parquet",
f"data.val_files={data_dir}/val.parquet",
"data.train_batch_size=8",
"data.max_prompt_length=1024",
"data.max_response_length=256",
"data.filter_overlong_prompts=True",
f"actor_rollout_ref.model.path={model_name}",
"actor_rollout_ref.model.use_remove_padding=True",
"actor_rollout_ref.actor.optim.lr=1e-6",
"actor_rollout_ref.actor.ppo_mini_batch_size=8",
"actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1",
"actor_rollout_ref.actor.use_kl_loss=True",
"actor_rollout_ref.actor.kl_loss_coef=0.001",
"actor_rollout_ref.actor.kl_loss_type=low_var_kl",
"actor_rollout_ref.actor.entropy_coeff=0",
# rollouts run on vLLM; eager mode avoids CUDA-graph crashes on sm_86
"actor_rollout_ref.rollout.name=vllm",
"actor_rollout_ref.rollout.tensor_model_parallel_size=1",
"actor_rollout_ref.rollout.gpu_memory_utilization=0.4",
"actor_rollout_ref.rollout.enforce_eager=True",
"actor_rollout_ref.rollout.n=4", # GRPO group size
"actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2",
"actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2",
"algorithm.use_kl_in_reward=False",
"trainer.critic_warmup=0",
"trainer.logger=[console]",
"trainer.project_name=grpo-gsm8k",
"trainer.experiment_name=qwen05b-gsm8k",
"trainer.n_gpus_per_node=8",
"trainer.nnodes=1",
"trainer.save_freq=-1", # set e.g. 10 to checkpoint to the Volume
"trainer.test_freq=-1",
"trainer.val_before_train=False",
"trainer.total_epochs=1",
f"trainer.total_training_steps={total_training_steps}",
"trainer.default_local_dir=/checkpoints/grpo-run",
]
import verl.trainer
from hydra import compose, initialize_config_dir
from verl.trainer.main_ppo import run_ppo
cfg_dir = os.path.join(os.path.dirname(verl.trainer.__file__), "config")
with initialize_config_dir(config_dir=cfg_dir, version_base=None):
config = compose(config_name="ppo_trainer", overrides=overrides)
run_ppo(config)
return "/checkpoints/grpo-run"
@app.local_entrypoint()
def main():
out = train_grpo.remote()
print(f"GRPO training complete. Checkpoint at: {out}")Run it
- Save as
grpo_verl_maths.py. The image pins Python 3.12 and AI Fabric requires the client Python to MATCH:uv venv -p 3.12 .venv312 && uv pip install --python .venv312/bin/python bladekit, then.venv312/bin/blade run grpo_verl_maths.py. gpu="RTX-A4500:8": veRL's hybrid engine colocates FSDP training and vLLM rollout generation on the same 8 GPUs of the node (ray.init(num_cpus=16, num_gpus=8); veRL reuses the in-container Ray instance).- GSM8K is downloaded (with retries, since HF can be flaky) and converted to veRL's parquet format inside the container before training starts.
- Each GRPO step logs the metrics that matter:
critic/rewards/mean(how often sampled answers score),actor/pg_loss,actor/ppo_kl, response lengths and rollout timings. total_training_steps=2is smoke-test length (~18 min end-to-end, dominated by model download + engine start). Raise it and swap in a bigger base model for real training.
Concepts used: GPU, Image build