{ "generator": "gaudi-cookbook catalog", "repository": "https://github.com/thediymaker/gaudi-cookbook", "recipes": [ { "id": "MiniMaxAI/MiniMax-M2.7", "title": "MiniMax-M2.7", "provider": "MiniMax", "description": "229B MoE (sigmoid routing, 256 experts) reasoning and agent model, official FP8 checkpoint, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "229B", "active_parameters": "10B", "precision": "fp8", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "laguna-moe-sigmoid-routing", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 7/8 at temperature 0 (one looped in its reasoning to max_tokens) and 4/4 with 4 in flight", "10 prompts with the checkpoint's sampling defaults (temperature 1.0, top_p 0.95, top_k 40): 10/10 finished; at temperature 0: 7/10", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Needles, 3 depths each, cold: 3/3 at 32K and 3/3 at 120K tokens", "Single stream 22.9 to 25.4 tokens/s, 4 streams 87.6 to 95.1 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Leave VLLM_ROW_PARALLEL_CHUNKS unset: on v0.26.0 any value above 1 stops vLLM at startup (AttributeError in register_hpu_lora_layers), and with 1024-token chunks a 16384-token chunking threshold would never be reached anyway.", "Licence: non-commercial (MiniMax M2.7 licence); commercial use needs written authorization from MiniMax.", "v0.26.0 cannot load this checkpoint: its minimax_m2 model calls FusedMoE.make_expert_params_mapping, which vLLM 568afb3a no longer has.", "Loading is CPU-bound: every FP8 tensor is rescaled on the CPU, and vllm-gaudi sizes its load thread pool from the host's core count. Run without a CPU limit, or cap the pool at the quota." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/MiniMaxAI/MiniMax-M2.7/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/MiniMaxAI/MiniMax-M2.7/recipe.yaml", "page": "MiniMaxAI/MiniMax-M2.7/", "guide": "docs/recipes/minimax-m2-7/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_WEIGHT_SHARING=0\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.30\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve MiniMaxAI/MiniMax-M2.7 \\\n --dtype auto \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.94 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m2 \\\n --reasoning-parser minimax_m2", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_WEIGHT_SHARING=0\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.30\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve MiniMaxAI/MiniMax-M2.7 \\\n --dtype auto \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.94 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m2 \\\n --reasoning-parser minimax_m2\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n exec vllm serve MiniMaxAI/MiniMax-M2.7 \\\n --served-model-name minimax-m2-7 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --dtype auto \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.94 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m2 \\\n --reasoning-parser minimax_m2\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPU_WEIGHT_SHARING\n value: \"0\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.30\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"MiniMaxAI/MiniMax-M2.7\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model MiniMaxAI/MiniMax-M2.7 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "Qwen/Qwen-AgentWorld-35B-A3B", "title": "Qwen-AgentWorld-35B-A3B", "provider": "Qwen", "description": "35B hybrid MoE (Gated DeltaNet + attention, 3B active) agent model, BF16, native 256K context on two Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "35B", "active_parameters": "3B", "precision": "bf16", "cards": { "total": 2, "tp": 2, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "hybrid-decode-pad", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 16/16 with 16 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 26.3 tokens/s, 16 streams 321.4 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching is off, as verified. On the stock release prefix-cache hits on Gated DeltaNet layers resume from the wrong state; gdn-prefix-cache makes them correct on other Qwen3.5-family recipes but has not been run on this model.", "On easy arithmetic the model often answers itself instead of calling an offered calculator tool; this is model behaviour, not parsing." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen-AgentWorld-35B-A3B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen-AgentWorld-35B-A3B/recipe.yaml", "page": "Qwen/Qwen-AgentWorld-35B-A3B/", "guide": "docs/recipes/qwen-agentworld-35b-a3b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen-AgentWorld-35B-A3B \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen-AgentWorld-35B-A3B \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve Qwen/Qwen-AgentWorld-35B-A3B \\\n --served-model-name qwen-agentworld-35b-a3b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"2\"\n limits:\n habana.ai/gaudi: \"2\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen-AgentWorld-35B-A3B\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen-AgentWorld-35B-A3B \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "title": "Qwen3-235B-A22B-Instruct-2507-FP8", "provider": "Qwen", "description": "235B MoE (22B active) instruct model, FP8 block-quantized checkpoint, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "235B", "active_parameters": "22B", "precision": "fp8", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "fp8-moe-gather", "rank-exception-fatal" ], "verified": [ "Greedy answers identical to the previous image on the functional set (6/6)", "top_logprobs soak 400/400 clean with gather-logprobs-int32 and logprobs-decode-guard", "Tool calls, plain and streamed, on the first replicas under live traffic: no 5xx", "Thinking twin, same image and recipe: 90-minute soak at 12 in flight (prompts p50 1.2K, p99 35K, shared prefixes), 888 done, 0 errors, 0 restarts, HBM peak 90.1 GB per card" ], "notes": [ "Without --max-cudagraph-capture-size every prefill shape captures an HPU graph: the 90-minute soak reached 97.5 GB of 98.3 GB per card on v0.26.0 without the cap.", "The capture cap costs about 1.2 s on an 8K-token prefill (TTFT 3.75 s vs 2.56 s uncapped); 1K and 32K prompts are unchanged.", "Loading is CPU-bound (FP8 rescale per tensor on Gaudi 2); a restart takes about 26 minutes." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8/recipe.yaml", "page": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8/", "guide": "docs/recipes/qwen3-235b-a22b-2507-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_HPU_FORCE_CHANNEL_FP8=1\nexport RUNTIME_SCALE_PATCHING=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=2048\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 16384 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --enable-chunked-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_HPU_FORCE_CHANNEL_FP8=1\nexport RUNTIME_SCALE_PATCHING=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=2048\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 16384 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --enable-chunked-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n exec vllm serve Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \\\n --served-model-name qwen3-235b-a22b-instruct-2507-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 16384 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --enable-chunked-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_HPU_MOE_GATHER\n value: \"1\"\n - name: VLLM_HPU_MOE_GATHER_RATIO\n value: \"0.25\"\n - name: VLLM_HPU_MOE_GATHER_VERIFY\n value: \"0\"\n - name: VLLM_HPU_FORCE_CHANNEL_FP8\n value: \"1\"\n - name: RUNTIME_SCALE_PATCHING\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.15\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"2048\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"8192\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-235B-A22B-Thinking-2507-FP8", "title": "Qwen3-235B-A22B-Thinking-2507-FP8", "provider": "Qwen", "description": "235B MoE (22B active) reasoning model, FP8 block-quantized checkpoint, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "235B", "active_parameters": "22B", "precision": "fp8", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "fp8-moe-gather", "rank-exception-fatal" ], "verified": [ "Reasoning split clean (no in content); non-streamed replies carry the reasoning field; streamed tool calls parse", "90-minute soak at 12 in flight (prompts p50 1.2K, p99 35K, shared prefixes): 888 done, 0 errors, 0 restarts, HBM peak 90.1 GB per card", "--max-num-batched-tokens 32768 vs 16384 at 8 seqs (A/B on an earlier image): equal throughput, 4 x 20K-token prompts TTFT 3.5-5.5 s vs 6.4 s, no OOM" ], "notes": [ "The soak left 8.2 GB of HBM headroom per card; if this recipe ever OOMs, lower --max-num-batched-tokens to 16384 first.", "Without --max-cudagraph-capture-size every prefill shape captures an HPU graph and HBM grows until PT_DEVMEM allocation failures.", "Loading is CPU-bound (FP8 rescale per tensor on Gaudi 2); a restart takes about 26 minutes." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-235B-A22B-Thinking-2507-FP8/recipe.yaml", "page": "Qwen/Qwen3-235B-A22B-Thinking-2507-FP8/", "guide": "docs/recipes/qwen3-235b-a22b-2507-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_HPU_FORCE_CHANNEL_FP8=1\nexport VLLM_HPU_CONVERT_TO_FP8UZ=1\nexport PT_HPU_WEIGHT_SHARING=0\nexport RUNTIME_SCALE_PATCHING=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3-235B-A22B-Thinking-2507-FP8 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 32768 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_HPU_FORCE_CHANNEL_FP8=1\nexport VLLM_HPU_CONVERT_TO_FP8UZ=1\nexport PT_HPU_WEIGHT_SHARING=0\nexport RUNTIME_SCALE_PATCHING=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3-235B-A22B-Thinking-2507-FP8 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 32768 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 fp8-moe-gather || exit 1\n exec vllm serve Qwen/Qwen3-235B-A22B-Thinking-2507-FP8 \\\n --served-model-name qwen3-235b-a22b-thinking-2507-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.45 \\\n --block-size 128 \\\n --max-num-batched-tokens 32768 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_HPU_MOE_GATHER\n value: \"1\"\n - name: VLLM_HPU_MOE_GATHER_RATIO\n value: \"0.25\"\n - name: VLLM_HPU_MOE_GATHER_VERIFY\n value: \"0\"\n - name: VLLM_HPU_FORCE_CHANNEL_FP8\n value: \"1\"\n - name: VLLM_HPU_CONVERT_TO_FP8UZ\n value: \"1\"\n - name: PT_HPU_WEIGHT_SHARING\n value: \"0\"\n - name: RUNTIME_SCALE_PATCHING\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.15\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"8192\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-235B-A22B-Thinking-2507-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-235B-A22B-Thinking-2507-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "title": "Qwen3-30B-A3B-Instruct-2507", "provider": "Qwen", "description": "30B MoE (3B active) instruct model, non-thinking, served at 128K context on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "30.5B", "active_parameters": "3.3B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling" ], "verified": [ "v0.26.0 vs the previous image, same flags: tool calls, 84K-token needle, 400-request top_logprobs soak (0 corrupt) all pass", "Decode 108.7 tok/s single stream, 179 tok/s aggregate at 2 in flight (before --max-num-seqs 8)", "prompt_logprobs: 68/68 sizes x prompt_logprobs values; greedy parity 10/10 bit-identical with the fix", "Prefix-cache isolation with prompt-order: 48/48, 0 errors", "Thinking twin (same architecture, image and flags): 8 in flight with about 105K-token prompts mixed in, 32/32 with 5 preemptions, 0 restarts" ], "notes": [ "KV pool is 221,312 tokens at these settings (1.69 x 128K).", "Keep --max-num-batched-tokens at 2048: on this architecture a padded prefill tile near 2^29 elements (8192-token chunk at 32K+ context) returns garbage or kills the engine.", "Penalties (repetition, presence, frequency) are not applied under async scheduling on v0.26.0." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-30B-A3B-Instruct-2507/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-30B-A3B-Instruct-2507/recipe.yaml", "page": "Qwen/Qwen3-30B-A3B-Instruct-2507/", "guide": "docs/recipes/qwen3-30b-a3b-2507/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-30B-A3B-Instruct-2507 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-30B-A3B-Instruct-2507 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve Qwen/Qwen3-30B-A3B-Instruct-2507 \\\n --served-model-name qwen3-30b-a3b-instruct-2507 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.5\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"32768\"\n - name: OMP_NUM_THREADS\n value: \"1\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-30B-A3B-Instruct-2507\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-30B-A3B-Instruct-2507 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-30B-A3B-Thinking-2507", "title": "Qwen3-30B-A3B-Thinking-2507", "provider": "Qwen", "description": "30B MoE (3B active) reasoning model, always thinks, served at 128K context on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "30.5B", "active_parameters": "3.3B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling" ], "verified": [ "Production-shaped load (about 19K-token prompts, natural reasoning output): 12/12 at 2, 16/16 at 4, 24/24 at 8 in flight, 0 failures", "Long-context arm (1 in 8 prompts at about 105K tokens) at 8 in flight: 32/32 with 5 preemptions, 0 restarts", "Without async-discard-counter the same arm killed the engine on the first preemption" ], "notes": [ "Give test requests a real max_tokens: a short budget is spent inside the reasoning and content comes back empty.", "KV pool is 221,312 tokens at these settings (1.69 x 128K).", "Keep --max-num-batched-tokens at 2048: on this architecture a padded prefill tile near 2^29 elements (8192-token chunk at 32K+ context) returns garbage or kills the engine." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-30B-A3B-Thinking-2507/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-30B-A3B-Thinking-2507/recipe.yaml", "page": "Qwen/Qwen3-30B-A3B-Thinking-2507/", "guide": "docs/recipes/qwen3-30b-a3b-2507/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-30B-A3B-Thinking-2507 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-30B-A3B-Thinking-2507 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve Qwen/Qwen3-30B-A3B-Thinking-2507 \\\n --served-model-name qwen3-30b-a3b-thinking-2507 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prompt-tokens-details \\\n --reasoning-parser deepseek_r1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.5\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"32768\"\n - name: OMP_NUM_THREADS\n value: \"1\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-30B-A3B-Thinking-2507\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-30B-A3B-Thinking-2507 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "title": "Qwen3-Coder-30B-A3B-Instruct", "provider": "Qwen", "description": "30B MoE (3B active) coding model with tool calling, served at 128K context on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "30.5B", "active_parameters": "3.3B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling" ], "verified": [ "Long-prompt ladder 38,656 to 130,900 tokens, greedy and default sampling with tools: all needles found", "38-minute soak, 8 coding/tool workers plus a 2K-131K ladder: 90 requests, needles 37/37, tool calls 22/22 with valid arguments, 0 markup leaks", "Streamed tool calls 15/15 clean; bare output (no ) parsed correctly by qwen3_coder", "Against the previous image at temperature 0: 6/6 coding answers and 6/6 tool calls byte-identical", "top_logprobs=20 under load: 20/20 well-formed", "75 minutes, 303 requests: 0 restarts, 0 errors, 0 preemptions; HBM 88-91 GiB of 96" ], "notes": [ "--max-num-batched-tokens 2048 is load-bearing: at 8192 a prompt of 38,657+ tokens returns '!' (greedy) or kills the engine (sampling).", "Do not try 4096 without re-testing the top of the window: 4096 x 131,072 is exactly 2^29 elements.", "Penalties (repetition, presence, frequency) are not applied under async scheduling on v0.26.0." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-Coder-30B-A3B-Instruct/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-Coder-30B-A3B-Instruct/recipe.yaml", "page": "Qwen/Qwen3-Coder-30B-A3B-Instruct/", "guide": "docs/recipes/qwen3-coder-30b-a3b-instruct/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-Coder-30B-A3B-Instruct \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=32768\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve Qwen/Qwen3-Coder-30B-A3B-Instruct \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve Qwen/Qwen3-Coder-30B-A3B-Instruct \\\n --served-model-name qwen3-coder-30b-a3b-instruct \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.5\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"32768\"\n - name: OMP_NUM_THREADS\n value: \"1\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-Coder-30B-A3B-Instruct\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-Coder-30B-A3B-Instruct \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-Coder-Next", "title": "Qwen3-Coder-Next", "provider": "Qwen", "description": "80B hybrid MoE (Gated DeltaNet + attention, 3B active) coding model with tool calling, BF16, native 256K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "80B", "active_parameters": "3B", "precision": "bf16", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 8/8 with 8 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Prefix-cache hit vs cold, 8 greedy prompts of 6K to 126K tokens: identical text and bit-identical logprobs 8/8; cached tokens a multiple of 384 on every hit", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 12.7 tokens/s, 8 streams 92.4 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages (384-token blocks here) and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "No VLLM_*_BUCKET_* variables: in compile mode torch.compile owns the shapes, and the lazy-mode bucket settings only cost throughput.", "Test with 2 or more concurrent decodes: the batch-collapse bug that v0.26.0 fixes for this model only shows at batch 2 and above.", "Decode reads every BF16 expert at each step: 12.7 tokens/s for one stream, 11.6 per stream at 8 in flight (256 tokens, end to end)." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-Coder-Next/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-Coder-Next/recipe.yaml", "page": "Qwen/Qwen3-Coder-Next/", "guide": "docs/recipes/qwen3-coder-next/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3-Coder-Next \\\n --tensor-parallel-size 4 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3-Coder-Next \\\n --tensor-parallel-size 4 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n exec vllm serve Qwen/Qwen3-Coder-Next \\\n --served-model-name qwen3-coder-next \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-Coder-Next\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-Coder-Next \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-Embedding-4B", "title": "Qwen3-Embedding-4B", "provider": "Qwen", "description": "4B instruction-aware text embedding model (2560 dimensions, Matryoshka truncation) on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "embedding" ], "architecture": "dense", "parameters": "4B", "active_parameters": "4B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 32768, "features": [], "required_fixes": [], "verified": [ "Retrieval nDCG@10 with the documented query instruction: SciFact 78.1, NFCorpus 40.9, ArguAna 73.5, CodeSearchNet python 95.9, QMSum 51.2", "Multilingual Wikipedia retrieval nDCG@10: de 89.7, hi 86.8, pt 87.5; STSBenchmark Spearman 93.6", "At 256 dimensions: SciFact 73.9, ArguAna 70.3, STSBenchmark 93.6", "Stock image, no fixes: the checkpoint is plain Qwen3ForCausalLM and is mapped to Qwen3ForEmbedding at load" ], "notes": [ "Queries need the prefix 'Instruct: \\nQuery:'; documents go in bare. vLLM does not apply sentence-transformers prompts.", "Cost per input is flat at about 50 ms whether inputs are batched or concurrent: pooling prefill runs one sequence at a time on this image. Scale out with replicas." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-Embedding-4B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-Embedding-4B/recipe.yaml", "page": "Qwen/Qwen3-Embedding-4B/", "guide": "docs/recipes/qwen3-embedding-4b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=64\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nvllm serve Qwen/Qwen3-Embedding-4B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'", "docker": "# 1. Write the launch script (the Shell tab).\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=64\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nvllm serve Qwen/Qwen3-Embedding-4B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n exec vllm serve Qwen/Qwen3-Embedding-4B \\\n --served-model-name qwen3-embedding-4b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'\n env:\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"64\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"32768\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/embeddings \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-Embedding-4B\", \"input\": [\"What is a Gaudi accelerator?\"]}'" } }, { "id": "Qwen/Qwen3-Embedding-8B", "title": "Qwen3-Embedding-8B", "provider": "Qwen", "description": "8B instruction-aware text embedding model (4096 dimensions, Matryoshka truncation) on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "embedding" ], "architecture": "dense", "parameters": "8B", "active_parameters": "8B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 32768, "features": [], "required_fixes": [], "verified": [ "Retrieval nDCG@10 with the documented query instruction: SciFact 78.4, NFCorpus 41.3, ArguAna 74.3, CodeSearchNet python 96.2, QMSum 54.6", "Multilingual Wikipedia retrieval nDCG@10: de 91.3, hi 88.0, pt 90.0; STSBenchmark Spearman 93.5", "At 256 dimensions: SciFact 76.1, ArguAna 72.5, STSBenchmark 93.5", "Stock image, no fixes: the checkpoint is plain Qwen3ForCausalLM and is mapped to Qwen3ForEmbedding at load" ], "notes": [ "Queries need the prefix 'Instruct: \\nQuery:'; documents go in bare. vLLM does not apply sentence-transformers prompts.", "Cost per input is flat at about 50 ms whether inputs are batched or concurrent: pooling prefill runs one sequence at a time on this image. Scale out with replicas.", "Cold start is about 2 minutes." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-Embedding-8B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-Embedding-8B/recipe.yaml", "page": "Qwen/Qwen3-Embedding-8B/", "guide": "docs/recipes/qwen3-embedding-8b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=64\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nvllm serve Qwen/Qwen3-Embedding-8B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'", "docker": "# 1. Write the launch script (the Shell tab).\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=64\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nvllm serve Qwen/Qwen3-Embedding-8B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n exec vllm serve Qwen/Qwen3-Embedding-8B \\\n --served-model-name qwen3-embedding-8b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --convert embed \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --max-num-batched-tokens 32768 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --hf-overrides '{\"is_matryoshka\": true}'\n env:\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"64\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"32768\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/embeddings \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-Embedding-8B\", \"input\": [\"What is a Gaudi accelerator?\"]}'" } }, { "id": "Qwen/Qwen3-VL-32B-Instruct-FP8", "title": "Qwen3-VL-32B-Instruct-FP8", "provider": "Qwen", "description": "32B dense vision-language model (images and video) in FP8, 128K context on two Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "32B", "active_parameters": "32B", "precision": "fp8", "cards": { "total": 2, "tp": 2, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "mm-merge-idxcopy", "mm-encoder-budget" ], "verified": [ "Text, OCR of an arbitrary number, three shapes, text-only after image 3/3, 16 x 2048 px images in one request, 16 mixed requests at 8 concurrent", "7 x 4096 px images (114,719 tokens) in one request: peak 87,580 / 91,345 MiB with mm-encoder-budget (two pods), 95,055 MiB without (one run)", "Ten novel prompt lengths: device memory flat at capture size 128; 44,619 to 90,637 MiB at the default (same pod)", "hermes tool call, top_logprobs and prompt_logprobs, needle at about 120K tokens" ], "notes": [ "Video is enabled (two per prompt) but was not part of the checks." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-VL-32B-Instruct-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-VL-32B-Instruct-FP8/recipe.yaml", "page": "Qwen/Qwen3-VL-32B-Instruct-FP8/", "guide": "docs/recipes/qwen3-vl-32b-instruct-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_HPU_MM_ENCODER_MAX_PATCHES=65536\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-encoder-budget || exit 1\n\nvllm serve Qwen/Qwen3-VL-32B-Instruct-FP8 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.55 \\\n --max-num-batched-tokens 1024 \\\n --max-cudagraph-capture-size 128 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":2}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_HPU_MM_ENCODER_MAX_PATCHES=65536\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-encoder-budget || exit 1\n\nvllm serve Qwen/Qwen3-VL-32B-Instruct-FP8 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.55 \\\n --max-num-batched-tokens 1024 \\\n --max-cudagraph-capture-size 128 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":2}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-encoder-budget || exit 1\n exec vllm serve Qwen/Qwen3-VL-32B-Instruct-FP8 \\\n --served-model-name qwen3-vl-32b-instruct-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.55 \\\n --max-num-batched-tokens 1024 \\\n --max-cudagraph-capture-size 128 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":2}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.5\"\n - name: VLLM_HPU_MM_ENCODER_MAX_PATCHES\n value: \"65536\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"2\"\n limits:\n habana.ai/gaudi: \"2\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-VL-32B-Instruct-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3-VL-32B-Instruct-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3-VL-Embedding-8B", "title": "Qwen3-VL-Embedding-8B", "provider": "Qwen", "description": "8B multimodal embedding model (text, images, screenshots; 4096 dimensions, Matryoshka truncation) on one Gaudi 2 card in BF16.", "status": "verified", "tasks": [ "embedding", "multimodal" ], "architecture": "dense", "parameters": "8B", "active_parameters": "8B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 32768, "features": [], "required_fixes": [ "mm-merge-idxcopy", "mm-pooling-inputs" ], "verified": [ "Stock image without mm-pooling-inputs: EngineDeadError on the first request", "Served vectors vs a CPU FP32 transformers reference: text 0.9961 / 0.9935, image 0.9885 / 0.9915 cosine", "COCO val2017 39769 vs captions: matching 0.69, unrelated 0.13; four different images mean pairwise cosine 0.169", "Flickr30k 1k test, first 300 images: caption to image R@1 81.5, image to caption R@1 96.0", "SciFact nDCG@10 with the instruction as the system message: 78.9" ], "notes": [ "Send inputs as chat messages: system = the instruction (queries) or \"Represent the user's input.\" (documents), user = text and/or images.", "Plain-string input gets no chat wrapping and a different last token; do not mix it with messages in one index." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3-VL-Embedding-8B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3-VL-Embedding-8B/recipe.yaml", "page": "Qwen/Qwen3-VL-Embedding-8B/", "guide": "docs/recipes/qwen3-vl-embedding-8b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-pooling-inputs || exit 1\n\nvllm serve Qwen/Qwen3-VL-Embedding-8B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --chat-template \"$COOKBOOK/recipes/Qwen/Qwen3-VL-Embedding-8B/embed.jinja\" \\\n --limit-mm-per-prompt '{\"image\":32,\"video\":0}' \\\n --hf-overrides '{\"is_matryoshka\": true, \"matryoshka_dimensions\": [64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096]}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-pooling-inputs || exit 1\n\nvllm serve Qwen/Qwen3-VL-Embedding-8B \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --chat-template \"$COOKBOOK/recipes/Qwen/Qwen3-VL-Embedding-8B/embed.jinja\" \\\n --limit-mm-per-prompt '{\"image\":32,\"video\":0}' \\\n --hf-overrides '{\"is_matryoshka\": true, \"matryoshka_dimensions\": [64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096]}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 mm-pooling-inputs || exit 1\n exec vllm serve Qwen/Qwen3-VL-Embedding-8B \\\n --served-model-name qwen3-vl-embedding-8b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --runner pooling \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.80 \\\n --no-enable-prefix-caching \\\n --no-enable-chunked-prefill \\\n --chat-template \"$COOKBOOK/recipes/Qwen/Qwen3-VL-Embedding-8B/embed.jinja\" \\\n --limit-mm-per-prompt '{\"image\":32,\"video\":0}' \\\n --hf-overrides '{\"is_matryoshka\": true, \"matryoshka_dimensions\": [64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096]}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/embeddings \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3-VL-Embedding-8B\", \"input\": [\"What is a Gaudi accelerator?\"]}'" } }, { "id": "Qwen/Qwen3.5-122B-A10B-FP8", "title": "Qwen3.5-122B-A10B-FP8", "provider": "Qwen", "description": "122B hybrid MoE (Gated DeltaNet + attention, 10B active) reasoning model, FP8, native 256K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "122B", "active_parameters": "10B", "precision": "fp8", "cards": { "total": 4, "tp": 4, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "fp8-moe-gather", "rope-cpu-cache", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 16/16 with 16 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Prefix-cache hit vs cold, 8 greedy prompts of 6K to 126K tokens: identical text and bit-identical logprobs 8/8; cached tokens a multiple of 1,152 on every hit", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 43.8 tokens/s, 16 streams 315.6 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages (1152-token blocks here) and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "VLLM_HPU_MOE_GATHER_RATIO is per card: with expert parallelism each card holds 64 experts, so 1.0 keeps the gather path up to 8 decode tokens. Above that the stock MoE op runs.", "Loading under a CPU quota is slow: vllm-gaudi sizes its load thread pool from the host's core count, and the per-tensor FP8 rescale is then CPU-throttled. Run without a CPU limit, or cap the pool at the quota." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.5-122B-A10B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.5-122B-A10B-FP8/recipe.yaml", "page": "Qwen/Qwen3.5-122B-A10B-FP8/", "guide": "docs/recipes/qwen3-5-122b-a10b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=1.0\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3.5-122B-A10B-FP8 \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --language-model-only", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=1.0\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache fp8-moe-gather || exit 1\n\nvllm serve Qwen/Qwen3.5-122B-A10B-FP8 \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --language-model-only\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache fp8-moe-gather || exit 1\n exec vllm serve Qwen/Qwen3.5-122B-A10B-FP8 \\\n --served-model-name qwen3-5-122b-a10b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --language-model-only\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_HPU_MOE_GATHER\n value: \"1\"\n - name: VLLM_HPU_MOE_GATHER_RATIO\n value: \"1.0\"\n - name: VLLM_HPU_MOE_GATHER_VERIFY\n value: \"0\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"262144\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"16384\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.5-122B-A10B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.5-122B-A10B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "Qwen/Qwen3.5-27B-FP8", "title": "Qwen3.5-27B-FP8", "provider": "Qwen", "description": "27B dense hybrid (Gated DeltaNet + attention) reasoning model with vision, FP8, native 256K context on one Gaudi 2 card.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "27B", "active_parameters": "27B", "precision": "fp8", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 32/32 with 32 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Vision: 16 images in one prompt, 16/16 colours named in order", "Prefix-cache hit vs cold, 8 greedy prompts of 6K to 126K tokens: identical text and bit-identical logprobs 8/8; cached tokens a multiple of 896 on every hit", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 47.1 tokens/s, 32 streams 575.3 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages (896-token blocks here) and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "Prompts above about 131K tokens can stop the engine with PT_DEVMEM OOM at --max-num-batched-tokens 3584 or more; keep 1792." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.5-27B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.5-27B-FP8/recipe.yaml", "page": "Qwen/Qwen3.5-27B-FP8/", "guide": "docs/recipes/qwen3-5-27b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=32\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3.5-27B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=32\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3.5-27B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n exec vllm serve Qwen/Qwen3.5-27B-FP8 \\\n --served-model-name qwen3-5-27b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"32\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"262144\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"16384\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.5-27B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.5-27B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 32" } }, { "id": "Qwen/Qwen3.5-397B-A17B-FP8", "title": "Qwen3.5-397B-A17B-FP8", "provider": "Qwen", "description": "397B MoE (17B active) hybrid (Gated DeltaNet + attention) reasoning model, FP8, native 256K context, text only, on one 8-card Gaudi 2 node.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "397B", "active_parameters": "17B", "precision": "fp8", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-inputs", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "moe-explicit-overload", "rank-exception-fatal", "vllm-long-prefill-share" ], "verified": [ "Greedy parity, 16 prompts of 5K to 31K tokens, cache hit vs cold: the 12 requests that hit all matched cold; cold vs cold 16/16 (the one other split had 0 cached tokens, batch composition)", "Cross-request isolation, 12 rounds: 0 leaks", "Needles at 128K, 200K and 250K, three depths: 9/9 cold, 9/9 from the cache", "16 contexts of about 129K tokens resident at once, 1,024 decoded each: 16/16 correct, 0 errors, 0 preemptions, KV peak 96.9%", "GSM8K first 50, thinking off, greedy: 50/50", "Tools (auto with thinking on and off, required, named, streaming, multi-turn), json_schema 9/9, reasoning separation: all pass", "top_logprobs 20, prompt_logprobs, seeded sampling, logit_bias, bad_words (token level), penalties: served, engine healthy", "Minimum free HBM on any card across all gates: 5.48 GiB, 0 allocation failures" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. With EP on eight cards the hybrid KV block is 640 tokens; 3,392 blocks hold 16 sequences of 131,072 tokens at once.", "With the full KV pool the cards had 0.07-0.25 GiB free under 250K prompts, and one rank failed a 2 GiB attention allocation on the v0.26.0 build: keep the pool pinned.", "Do not raise --max-num-batched-tokens: a 4096-token chunk against a 256K context already needs a 2 GiB attention buffer per card." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.5-397B-A17B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.5-397B-A17B-FP8/recipe.yaml", "page": "Qwen/Qwen3.5-397B-A17B-FP8/", "guide": "docs/recipes/qwen3-5-397b-a17b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.2\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gdn-prefix-cache vllm-long-prefill-share || exit 1\n\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --num-gpu-blocks-override 3392 \\\n --gpu-memory-utilization 0.85 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.2\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gdn-prefix-cache vllm-long-prefill-share || exit 1\n\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --num-gpu-blocks-override 3392 \\\n --gpu-memory-utilization 0.85 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gdn-prefix-cache vllm-long-prefill-share || exit 1\n exec vllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n --served-model-name qwen3-5-397b-a17b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --num-gpu-blocks-override 3392 \\\n --gpu-memory-utilization 0.85 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_HPU_MOE_GATHER\n value: \"1\"\n - name: VLLM_HPU_MOE_GATHER_RATIO\n value: \"0.2\"\n - name: VLLM_HPU_MOE_GATHER_VERIFY\n value: \"0\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"32768\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.5-397B-A17B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.5-397B-A17B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "Qwen/Qwen3.6-27B-FP8", "title": "Qwen3.6-27B-FP8", "provider": "Qwen", "description": "27B dense hybrid (Gated DeltaNet + attention) reasoning model with vision, FP8, native 256K context on one Gaudi 2 card.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "27B", "active_parameters": "27B", "precision": "fp8", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache", "structured-outputs-auto-whitespace", "qwen3-strict-think-end" ], "verified": [ "Greedy parity, 32 prompts of 5K to 31K tokens, cache hit vs cold: 31/32 identical, the other forked at an exact tie (both top-2 logprobs -0.6932); cold vs cold 32/32", "Cold greedy outputs, caching on vs off: 27/32 identical, inside the noise floor of changing the prefill chunk (caching off at 1,792 vs 896 batched tokens: 20/32)", "Cross-request isolation, 24 rounds: 0 leaks; images served from the cache, 8 cases: hit equals cold 8/8", "Requests sharing a prefix in the same step, 24: 24/24 correct; 22/24 word for word as when sent one at a time (batch-composition wording, one of them without a hit)", "Needles at 65K, 130K and 250K, three depths: 9/9 cold, 9/9 from the cache, hit equals cold 9/9", "16 images per prompt: 16/16 colours named in order (thinking off), a 17th rejected with a clean 400", "Forced tool calls (tool_choice required): whitespace runaway 0/32 with disable_any_whitespace", "Tool reliability, 3 cases x 40 sampled at 0.6: 119/120 with strict think-end", "prompt_logprobs: aligned, in vocabulary and consistent with decode logprobs (plp check PASS)", "Agent replay, 4 sessions 8K to 64K: wall 171 s vs 513 s with caching off, later-turn TTFT p50 6.7 vs 24.2 s; 0 preemptions, 0 restarts" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages of the cache and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "With prefix caching on, the hybrid KV block is 896 tokens and prefill chunks end on block boundaries, so a 1,792-token batch is two blocks. Prompts shorter than one block never hit.", "Qwen3.5-27B-FP8 has the same geometry and runs this recipe unchanged (drop VLLM_QWEN3_STRICT_THINK_END and its fix: it scores 120/120 without)." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.6-27B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.6-27B-FP8/recipe.yaml", "page": "Qwen/Qwen3.6-27B-FP8/", "guide": "docs/recipes/qwen3-6-27b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_QWEN3_STRICT_THINK_END=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=32\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache qwen3-strict-think-end || exit 1\n\nvllm serve Qwen/Qwen3.6-27B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --structured-outputs-config '{\"disable_any_whitespace\": true}' \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_QWEN3_STRICT_THINK_END=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=32\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache qwen3-strict-think-end || exit 1\n\nvllm serve Qwen/Qwen3.6-27B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --structured-outputs-config '{\"disable_any_whitespace\": true}' \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache qwen3-strict-think-end || exit 1\n exec vllm serve Qwen/Qwen3.6-27B-FP8 \\\n --served-model-name qwen3-6-27b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 1792 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --structured-outputs-config '{\"disable_any_whitespace\": true}' \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_QWEN3_STRICT_THINK_END\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"32\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"262144\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"16384\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.6-27B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.6-27B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 32" } }, { "id": "Qwen/Qwen3.6-35B-A3B-FP8", "title": "Qwen3.6-35B-A3B-FP8", "provider": "Qwen", "description": "35B MoE (3B active) hybrid (Gated DeltaNet + attention) reasoning model with vision, FP8, native 256K context on one Gaudi 2 card.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "moe", "parameters": "35B", "active_parameters": "3B", "precision": "fp8", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "fp8-moe-gather", "rope-cpu-cache", "qwen3-strict-think-end" ], "verified": [ "Greedy parity, 32 prompts of 5K to 31K tokens, cache hit vs cold: 32/32 identical with bit-identical logprobs; cold vs cold 32/32", "Cross-request isolation, 24 rounds: 0 leaks; images served from the cache, 8 cases: hit equals cold 8/8", "Needles at 65K, 130K and 250K, three depths: 9/9 cold, 9/9 from the cache, hit equals cold 9/9", "16 images per prompt: 16/16 colours named in order (thinking off), a 17th rejected with a clean 400", "Requests sharing a prefix in the same step, 24: 24/24 correct; 11/24 worded differently from sequential runs, including 2 requests that had no cache hit. With prefix caching off the same test words 16/24 differently: batch composition on this MoE, not the cache", "Forced tool calls (tool_choice required): whitespace runaway 0/32", "Tool reliability, 3 cases x 40 sampled: 120/120 with strict think-end", "prompt_logprobs: aligned, in vocabulary and consistent with decode logprobs (plp check PASS)", "Agent replay, 4 sessions 8K to 64K: wall 130.5 s vs 607.5 s with caching off, later-turn TTFT p50 3.7 vs 18.3 s, 78% of prompt tokens served from the cache; 0 preemptions", "Functional set (arithmetic, factual, code, multi-turn, streaming, tool calls): 10/10 (earlier run, prefix caching off)", "Decode 74.5 tokens/s single stream, 204.8 tokens/s per card at 8 in flight, with fp8-moe-gather (earlier run, prefix caching off)", "Gathered-expert MoE vs the stock op: no element beyond 2 FP8 ULPs" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages of the cache and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "With prefix caching on, the hybrid KV block is 1,152 tokens and prefill chunks end on block boundaries, so a 1,280-token batch carries one block of prompt. Prompts shorter than one block never hit.", "--max-num-seqs and VLLM_HPU_MOE_GATHER_RATIO move together: 9+ decode tokens fall back to the stock MoE op at about 19 tok/s per stream (16 sequences would need a ratio of 0.5, not measured).", "Loading under a CPU quota can take 78 minutes: vllm-gaudi sizes its load thread pool from the host's core count, and the per-tensor FP8 rescale is then CPU-throttled. Run without a CPU limit, or cap the pool at the quota." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.6-35B-A3B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.6-35B-A3B-FP8/recipe.yaml", "page": "Qwen/Qwen3.6-35B-A3B-FP8/", "guide": "docs/recipes/qwen3-6-35b-a3b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_QWEN3_STRICT_THINK_END=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache fp8-moe-gather qwen3-strict-think-end || exit 1\n\nvllm serve Qwen/Qwen3.6-35B-A3B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 1280 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_HPU_MOE_GATHER=1\nexport VLLM_HPU_MOE_GATHER_RATIO=0.25\nexport VLLM_HPU_MOE_GATHER_VERIFY=0\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_QWEN3_STRICT_THINK_END=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache fp8-moe-gather qwen3-strict-think-end || exit 1\n\nvllm serve Qwen/Qwen3.6-35B-A3B-FP8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 1280 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gdn-prefix-cache fp8-moe-gather qwen3-strict-think-end || exit 1\n exec vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \\\n --served-model-name qwen3-6-35b-a3b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.75 \\\n --block-size 128 \\\n --max-num-batched-tokens 1280 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_HPU_MOE_GATHER\n value: \"1\"\n - name: VLLM_HPU_MOE_GATHER_RATIO\n value: \"0.25\"\n - name: VLLM_HPU_MOE_GATHER_VERIFY\n value: \"0\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_QWEN3_STRICT_THINK_END\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"262144\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"16384\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.6-35B-A3B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.6-35B-A3B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "Qwen/Qwen3.8-27B", "title": "Qwen3.8-27B", "provider": "Qwen", "description": "27B dense hybrid (Gated DeltaNet + attention) reasoning model with vision, BF16, native 256K context on one Gaudi 2 card, 4 sequences.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "27B", "active_parameters": "27B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gdn-prefix-cache", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 4/4 with 4 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Vision: 16 images in one prompt, 16/16 colours named in order", "Prefix-cache hit vs cold, 8 greedy prompts of 6K to 126K tokens: identical text and bit-identical logprobs 8/8; cached tokens a multiple of 896 on every hit", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 36.4 tokens/s, 4 streams 117.3 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching needs gdn-prefix-cache: on the stock release, prefix-cache hits on Gated DeltaNet layers resume from the wrong state. The fix stores GDN state checkpoints in the KV pages (896-token blocks here) and keeps the compact GDN slots (VLLM_COMPACT_GDN=1).", "TP 1 is deliberate: TP 2 and TP 4 stop the engine at 4 concurrent requests (the GDN chunked-prefill kernel, not memory). Scale with replicas.", "Only xhigh, medium and low are legal reasoning_effort values for this template; high returns HTTP 400. The top-level reasoning_effort field is ignored." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/Qwen/Qwen3.8-27B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/Qwen/Qwen3.8-27B/recipe.yaml", "page": "Qwen/Qwen3.8-27B/", "guide": "docs/recipes/qwen3-8-27b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3.8-27B \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.60 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"medium\"}' \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=262144\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=16384\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n\nvllm serve Qwen/Qwen3.8-27B \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.60 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"medium\"}' \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gdn-prefix-cache || exit 1\n exec vllm serve Qwen/Qwen3.8-27B \\\n --served-model-name qwen3-8-27b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.60 \\\n --block-size 128 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --reasoning-parser qwen3 \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"medium\"}' \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --limit-mm-per-prompt '{\"image\":16,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"262144\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"16384\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"Qwen/Qwen3.8-27B\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model Qwen/Qwen3.8-27B \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", "title": "MiMo-V2.6-Flash-RL", "provider": "Xiaomi", "description": "309B MoE (15B active) with 128-token sliding-window attention and sinks, served at 512K context on one 8-card Gaudi 2 node through a port.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "309B", "active_parameters": "15B", "precision": "fp8", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 524288, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-inputs", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "logprobs-decode-guard", "swa-decode-window", "rank-exception-fatal", "vllm-long-prefill-share" ], "verified": [ "install.sh: commit check, v0.26.0 profile + vllm-long-prefill-share, both runner patches, plugin install applied", "device_ops.py on one card: windowed attention with sinks 0.0036 worst relative vs fp32; 12/12 sampled experts re-encode to FP8 exactly", "decode 33.8 / 33.2 / 25.6 tok/s per stream at 1 / 4 / 16 streams (floor 20)", "tool calls 6/6, including tool_choice required and a call after a 32.9K-token document", "needles at 128K / 256K / 500K x depths 0.1 / 0.5 / 0.9: 3/3 each; four 500K-token prompts at once 4/4 (932 s)", "window-sized KV vs the stock layout, greedy: chats, 12K and 30K chunked prompts, a prefix hit, 2 of 4 batch rows identical" ], "notes": [ "Needs the port image (ports/mimo-v2-6-flash). Its install.sh applies the v0.26.0 profile and vllm-long-prefill-share before the port's runner patches, so the launch's fix step only confirms them.", "TP8 only: window-sized KV needs the global and sliding layers to hold the same KV heads per rank.", "A start loads in about 3 minutes from a warm page cache, then warms up every shape for 40 to 50 minutes." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/XiaomiMiMo/MiMo-V2.6-Flash-RL/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/XiaomiMiMo/MiMo-V2.6-Flash-RL/recipe.yaml", "page": "XiaomiMiMo/MiMo-V2.6-Flash-RL/", "guide": "docs/recipes/mimo-v2-6-flash-rl/", "commands": { "shell": "export CHECKPOINT_DIR=/data/mimo-v2.6-flash-rl-hpu\nexport PT_HPU_LAZY_MODE=0\nexport VLLM_T_COMPILE_DYNAMIC_SHAPES=false\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=false\nexport MIMO_SWA_KV_GROUPS=1\nexport MIMO_GA_TILE=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=65536\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\npython3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Flash-RL \"$CHECKPOINT_DIR\" --revision 5711b268169967567844e1e560e8a3966da959b1 || exit 1\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Flash-RL \\\n --tensor-parallel-size 8 \\\n --max-model-len 524288 \\\n --max-num-seqs 16 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo", "docker": "# 0. Build the port image from a stock image (ports/mimo-v2-6-flash/README.md).\ndocker build -f gaudi-cookbook/ports/mimo-v2-6-flash/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport CHECKPOINT_DIR=/data/mimo-v2.6-flash-rl-hpu\nexport PT_HPU_LAZY_MODE=0\nexport VLLM_T_COMPILE_DYNAMIC_SHAPES=false\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=false\nexport MIMO_SWA_KV_GROUPS=1\nexport MIMO_GA_TILE=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=65536\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\npython3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Flash-RL \"$CHECKPOINT_DIR\" --revision 5711b268169967567844e1e560e8a3966da959b1 || exit 1\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Flash-RL \\\n --tensor-parallel-size 8 \\\n --max-model-len 524288 \\\n --max-num-seqs 16 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/mimo-v2-6-flash/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n python3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Flash-RL \"$CHECKPOINT_DIR\" --revision 5711b268169967567844e1e560e8a3966da959b1 || exit 1\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n exec vllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Flash-RL \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --max-model-len 524288 \\\n --max-num-seqs 16 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: CHECKPOINT_DIR\n value: \"/data/mimo-v2.6-flash-rl-hpu\"\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: VLLM_T_COMPILE_DYNAMIC_SHAPES\n value: \"false\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"false\"\n - name: MIMO_SWA_KV_GROUPS\n value: \"1\"\n - name: MIMO_GA_TILE\n value: \"1\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"65536\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"XiaomiMiMo/MiMo-V2.6-Flash-RL\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model XiaomiMiMo/MiMo-V2.6-Flash-RL \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "title": "MiMo-V2.6-Pro-RL", "provider": "Xiaomi", "description": "1.02T MoE (42B active), 128-token sliding windows with sinks, MXFP4 experts on the MXFP4 decode kernels, 256K context on one 8-card node.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "1.02T", "active_parameters": "42B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-inputs", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "logprobs-decode-guard", "swa-decode-window", "rank-exception-fatal", "vllm-long-prefill-share" ], "verified": [ "install.sh: profile, runner patches, plugin, kernel build --variant glm --pt2; library gate 3,253 GUIDs and both op gates pass", "mxk_ops.py on one card: 28/28 decode cases within the parity bound, kernel worst rel 1.71e-3 vs 2.3e-3 to 4.8e-3 for the stock op; compiled = eager", "decode 40.3 / 27.5 / 15.1 tok/s per stream at 1 / 4 / 8 streams (floor 12)", "tool calls 6/6, including tool_choice required and a call after a 32.9K-token document", "needles at 240K x 3 depths 3/3; four 240K-token prompts at once 4/4 (899 s); 128K 2/3 (one refusal: the model called the needle inserted text)", "window-sized KV vs the stock layout: not completed (pod preempted); identical on single requests on Flash with the same port code" ], "notes": [ "Needs the port image (ports/mimo-v2-6-flash, which serves both MiMo-V2.6 models). Its install.sh applies the v0.26.0 profile and vllm-long-prefill-share before the port's runner patches, and builds the MXFP4 MoE kernels (glm variant with the torch.compile op library).", "TP8 only: window-sized KV needs the global and sliding layers to hold the same KV heads per rank.", "--gpu-memory-utilization 0.8: in development runs 0.9 ran out of memory at a short prefill and 0.85 failed on a prefix-cached prefill." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/XiaomiMiMo/MiMo-V2.6-Pro-RL/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/XiaomiMiMo/MiMo-V2.6-Pro-RL/recipe.yaml", "page": "XiaomiMiMo/MiMo-V2.6-Pro-RL/", "guide": "docs/recipes/mimo-v2-6-pro-rl/", "commands": { "shell": "export CHECKPOINT_DIR=/data/mimo-v2.6-pro-rl-hpu\nexport PT_HPU_LAZY_MODE=0\nexport VLLM_T_COMPILE_DYNAMIC_SHAPES=false\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=false\nexport MIMO_EXPERT_DTYPE=mxfp4\nexport MIMO_MXK=1\nexport MIMO_SWA_KV_GROUPS=1\nexport MIMO_GA_TILE=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=65536\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\npython3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Pro-RL \"$CHECKPOINT_DIR\" --revision 73875d00b30a89ef8cc353a0b60b0e9f9561952d || exit 1\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Pro-RL \\\n --tensor-parallel-size 8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.8 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo", "docker": "# 0. Build the port image from a stock image (ports/mimo-v2-6-flash/README.md).\ndocker build -f gaudi-cookbook/ports/mimo-v2-6-flash/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport CHECKPOINT_DIR=/data/mimo-v2.6-pro-rl-hpu\nexport PT_HPU_LAZY_MODE=0\nexport VLLM_T_COMPILE_DYNAMIC_SHAPES=false\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=false\nexport MIMO_EXPERT_DTYPE=mxfp4\nexport MIMO_MXK=1\nexport MIMO_SWA_KV_GROUPS=1\nexport MIMO_GA_TILE=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=65536\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\npython3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Pro-RL \"$CHECKPOINT_DIR\" --revision 73875d00b30a89ef8cc353a0b60b0e9f9561952d || exit 1\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Pro-RL \\\n --tensor-parallel-size 8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.8 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/mimo-v2-6-flash/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n python3 \"$COOKBOOK/ports/mimo-v2-6-flash/convert/stage_checkpoint.py\" XiaomiMiMo/MiMo-V2.6-Pro-RL \"$CHECKPOINT_DIR\" --revision 73875d00b30a89ef8cc353a0b60b0e9f9561952d || exit 1\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 vllm-long-prefill-share || exit 1\n exec vllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name XiaomiMiMo/MiMo-V2.6-Pro-RL \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --long-prefill-token-threshold 2048 \\\n --max-num-batched-tokens 8192 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.8 \\\n --enable-prompt-tokens-details \\\n --trust-remote-code \\\n --generation-config vllm \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}' \\\n --chat-template \"$COOKBOOK/ports/mimo-v2-6-flash/serving/chat_template.jinja\" \\\n --reasoning-parser mimo \\\n --enable-auto-tool-choice \\\n --tool-call-parser mimo\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: CHECKPOINT_DIR\n value: \"/data/mimo-v2.6-pro-rl-hpu\"\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: VLLM_T_COMPILE_DYNAMIC_SHAPES\n value: \"false\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"false\"\n - name: MIMO_EXPERT_DTYPE\n value: \"mxfp4\"\n - name: MIMO_MXK\n value: \"1\"\n - name: MIMO_SWA_KV_GROUPS\n value: \"1\"\n - name: MIMO_GA_TILE\n value: \"1\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"65536\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"XiaomiMiMo/MiMo-V2.6-Pro-RL\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model XiaomiMiMo/MiMo-V2.6-Pro-RL \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "amd/GLM-5.2-MXFP4", "title": "GLM-5.2 (MXFP4)", "provider": "Z.ai", "description": "744B MoE (about 41B active) with DeepSeek sparse attention, AMD's MXFP4 release repacked byte for byte and served at 128K context with MTP on one 8-card Gaudi 2 node, through the glm-5-mxfp4 port.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "744B", "active_parameters": "41B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning", "spec_decoding" ], "required_fixes": [ "structured-output-rows", "so-row-owners", "gather-logprobs-int32", "logprobs-decode-guard", "seeded-sampling", "rank-exception-fatal", "vllm-long-prefill-share", "glm47-streaming-text-markers" ], "verified": [ "Needles, 3 codes at 10/50/90% depth, two fillers, 16K to 126K tokens: 30/30", "GSM8K first 50, reasoning low, greedy: 48/50", "Tools (required, named, auto, none, JSON schema; streaming and not; greedy and sampled): 32/32", "MTP acceptance at default sampling: 0.858 over 1,651 drafts", "Logprobs, 40 sampled requests with top_logprobs 5 under MTP: 40/40 complete and finite", "Stress, 4 decodes while 64K and 100K prompts prefill: 4/4 finished, needles 3/3, largest decode gap 0.9 s", "Decode 67.3 tok/s at 1 stream, 112.3 aggregate at 4 (~2K-token prompt, greedy); TTFT 0.49 / 1.65 s at 2K / 8K" ], "notes": [ "Needs the glm-5-mxfp4 port image: docker build -f ports/glm-5-mxfp4/Dockerfile from the stock releases/v0.30.0 image. install.sh applies the fixes before the port's patches, so the launch's fix step finds every fix applied.", "vllm-gaudi cannot load AMD's Quark per-expert layout: repack amd/GLM-5.2-MXFP4 once with ports/glm-5-mxfp4/convert (CPU only). MIT licence.", "Speculative decoding turns async scheduling off." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/amd/GLM-5.2-MXFP4/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/amd/GLM-5.2-MXFP4/recipe.yaml", "page": "amd/GLM-5.2-MXFP4/", "guide": "docs/recipes/glm-5-2-mxfp4/", "commands": { "shell": "export CHECKPOINT_DIR=/models/GLM-5.2-MXFP4-fused\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_HPU_FSDPA_SOFTMAX_MODE=None\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport GLM_DSA_SPARSE=1\nexport GLM_MXK=1\nexport GLM_MXK_PREFILL=1\nexport GLM_MOE_TP=2\nexport GLM_ROUTER_FAST=1\nexport GLM_EARLY_DRAFT=1\nexport GLM_SPEC_DEVICE_REJECT=1\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/GLM-5.2-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --override-generation-config '{\"temperature\": 0.8}'", "docker": "# 0. Build the port image from a stock image (ports/glm-5-mxfp4/README.md).\ndocker build -f gaudi-cookbook/ports/glm-5-mxfp4/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport CHECKPOINT_DIR=/models/GLM-5.2-MXFP4-fused\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_HPU_FSDPA_SOFTMAX_MODE=None\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport GLM_DSA_SPARSE=1\nexport GLM_MXK=1\nexport GLM_MXK_PREFILL=1\nexport GLM_MOE_TP=2\nexport GLM_ROUTER_FAST=1\nexport GLM_EARLY_DRAFT=1\nexport GLM_SPEC_DEVICE_REJECT=1\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/GLM-5.2-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --override-generation-config '{\"temperature\": 0.8}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/glm-5-mxfp4/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n exec vllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/GLM-5.2-MXFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --override-generation-config '{\"temperature\": 0.8}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: CHECKPOINT_DIR\n value: \"/models/GLM-5.2-MXFP4-fused\"\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_BUCKETING_FROM_FILE\n value: \"/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: PT_HPU_QKV_SLICE_SEQ_LEN_THLD\n value: \"128\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_HPU_FSDPA_SOFTMAX_MODE\n value: \"None\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"32768\"\n - name: GLM_DSA_SPARSE\n value: \"1\"\n - name: GLM_MXK\n value: \"1\"\n - name: GLM_MXK_PREFILL\n value: \"1\"\n - name: GLM_MOE_TP\n value: \"2\"\n - name: GLM_ROUTER_FAST\n value: \"1\"\n - name: GLM_EARLY_DRAFT\n value: \"1\"\n - name: GLM_SPEC_DEVICE_REJECT\n value: \"1\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"amd/GLM-5.2-MXFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model amd/GLM-5.2-MXFP4 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "amd/Kimi-K2.7-Code-MXFP4", "title": "Kimi-K2.7-Code (MXFP4)", "provider": "Moonshot AI", "description": "1T-parameter MoE coding model (32B active, MLA, 384 experts) served text-only from AMD's MXFP4 release at 128K context on one 8-card Gaudi 2 node, through the kimi-k2-7 port, with the MXFP4 decode kernels behind one switch.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "1T", "active_parameters": "32B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "gather-logprobs-int32", "logprobs-decode-guard", "seeded-sampling", "rank-exception-fatal" ], "verified": [ "kernel parity, kimi build on real layer-3 experts, T 1 to 16: PARITY PASS, SIG_DEVICE PASS", "golden answers 12/12, KIMI_MXK=0 and KIMI_MXK=1", "tool call through kimi_k2, streaming and non-streaming: pass", "needles 9/9 at 32K, 64K and 120K tokens", "top_logprobs=20 soak, 200 requests with CJK and emoji output: 200/200 clean", "teacher-forced NLL 1.2888 with the kernels vs 1.2878 stock, top-1 agreement 0.955 over 8,198 positions", "decode at a 16K cached prefix, per stream: 44.5 / 34.5 / 23.4 / 14.8 tok/s at 1 / 2 / 4 / 8 streams (stock op 17.9 / 12.9 / 9.8 / 7.5)" ], "notes": [ "Needs the kimi-k2-7 port image: docker build -f ports/kimi-k2-7/Dockerfile from the stock releases/v0.30.0 image, and AMD's checkpoint repacked once with ports/kimi-k2-7/convert/repack_quark_mxfp4.py (CPU only, byte copies of the experts).", "Modified MIT licence: a product or service with more than 100 million monthly active users or more than 20 million US dollars of monthly revenue must display \"Kimi K2.7 Code\" prominently in its user interface." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/amd/Kimi-K2.7-Code-MXFP4/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/amd/Kimi-K2.7-Code-MXFP4/recipe.yaml", "page": "amd/Kimi-K2.7-Code-MXFP4/", "guide": "docs/recipes/kimi-k2-7-code/", "commands": { "shell": "export KIMI_MXFP4_DIR=/models/Kimi-K2.7-Code-MXFP4\nexport KIMI_MXK=1\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=1\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=false\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=1\nexport VLLM_PROMPT_QUERY_BUCKET_MIN=512\nexport VLLM_PROMPT_QUERY_BUCKET_STEP=512\nexport VLLM_PROMPT_QUERY_BUCKET_MAX=2048\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=256\nexport VLLM_PROMPT_CTX_BUCKET_MAX=1024\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=256\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=512\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve \"$KIMI_MXFP4_DIR\" \\\n --served-model-name amd/Kimi-K2.7-Code-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --enable-chunked-prefill \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --language-model-only \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2 \\\n --reasoning-parser kimi_k2 \\\n --trust-remote-code", "docker": "# 0. Build the port image from a stock image (ports/kimi-k2-7/README.md).\ndocker build -f gaudi-cookbook/ports/kimi-k2-7/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport KIMI_MXFP4_DIR=/models/Kimi-K2.7-Code-MXFP4\nexport KIMI_MXK=1\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=1\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=false\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=1\nexport VLLM_PROMPT_QUERY_BUCKET_MIN=512\nexport VLLM_PROMPT_QUERY_BUCKET_STEP=512\nexport VLLM_PROMPT_QUERY_BUCKET_MAX=2048\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=256\nexport VLLM_PROMPT_CTX_BUCKET_MAX=1024\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=256\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=512\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=8192\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve \"$KIMI_MXFP4_DIR\" \\\n --served-model-name amd/Kimi-K2.7-Code-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --enable-chunked-prefill \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --language-model-only \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2 \\\n --reasoning-parser kimi_k2 \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/kimi-k2-7/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n exec vllm serve \"$KIMI_MXFP4_DIR\" \\\n --served-model-name amd/Kimi-K2.7-Code-MXFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --enable-chunked-prefill \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --gpu-memory-utilization 0.9 \\\n --language-model-only \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2 \\\n --reasoning-parser kimi_k2 \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: KIMI_MXFP4_DIR\n value: \"/models/Kimi-K2.7-Code-MXFP4\"\n - name: KIMI_MXK\n value: \"1\"\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"1\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"false\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"1\"\n - name: VLLM_PROMPT_QUERY_BUCKET_MIN\n value: \"512\"\n - name: VLLM_PROMPT_QUERY_BUCKET_STEP\n value: \"512\"\n - name: VLLM_PROMPT_QUERY_BUCKET_MAX\n value: \"2048\"\n - name: VLLM_PROMPT_CTX_BUCKET_MIN\n value: \"0\"\n - name: VLLM_PROMPT_CTX_BUCKET_STEP\n value: \"256\"\n - name: VLLM_PROMPT_CTX_BUCKET_MAX\n value: \"1024\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"256\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"512\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"8192\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: PT_HPU_QKV_SLICE_SEQ_LEN_THLD\n value: \"128\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"amd/Kimi-K2.7-Code-MXFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model amd/Kimi-K2.7-Code-MXFP4 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "amd/MiniMax-M3-MXFP4", "title": "MiniMax-M3 (FP8 experts)", "provider": "MiniMax", "description": "427B / 23B-active MoE served text-only on one 8-card Gaudi 2 node at 128K context, from AMD's MXFP4 checkpoint re-encoded exactly to per-channel FP8 experts.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "427B", "active_parameters": "23B", "precision": "fp8", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rank-exception-fatal" ], "verified": [ "Built and served as written: install.sh on the stock v0.26.0 image, the v0.26.0 fix profile, this command, one 8-card node; run_gates.sh ALL PASS", "Golden prompts 5/5 (arithmetic, code, facts, a get_weather tool call); greedy loops 0/10", "Needles 3/3 at 16K, 32K, 64K, 100K and 125K prompt tokens", "prompt_logprobs over 11,901 tokens aligned, 0 bad entries; top_logprobs 16 concurrent requests, 0 errors", "Streamed and non-streamed reasoning split cleanly (no in content); streamed tool call with name and arguments", "Decode 25.0 tok/s at one stream, 21.9 per stream at 8 streams (1.6K-token prompts)", "Converter re-run: one MoE layer reproduced 768/768 tensors byte for byte; shared experts 9/9; validate.py inventory and samples pass" ], "notes": [ "Serve at the model's default temperature 1.0: at 0.6 and below, hard prompts can reason without end (finish_reason length, empty content).", "Weight load: about 10 minutes from idle shared storage with --enable-ep-weight-filter, 38 minutes measured on a busy store; without the filter every card reads all 411 GiB.", "Licence: MiniMax Community License (non-commercial grant; commercial use needs 'Built with MiniMax M3' and a notice to MiniMax)." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/amd/MiniMax-M3-MXFP4/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/amd/MiniMax-M3-MXFP4/recipe.yaml", "page": "amd/MiniMax-M3-MXFP4/", "guide": "docs/recipes/minimax-m3/", "commands": { "shell": "export CHECKPOINT_DIR=/models/MiniMax-M3-FP8\nexport PT_HPU_LAZY_MODE=0\nexport LOG_LEVEL_PT_BRIDGE=4\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_SCALE_ADJUSTMENT=false\nexport VLLM_M3_FP8_MOE=native\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/MiniMax-M3-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --enable-ep-weight-filter \\\n --max-model-len 131072 \\\n --max-num-seqs 32 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --enable-prompt-tokens-details \\\n --limit-mm-per-prompt '{\"image\": 0, \"video\": 0}' \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3_py \\\n --trust-remote-code", "docker": "# 0. Build the port image from a stock image (ports/minimax-m3/README.md).\ndocker build -f gaudi-cookbook/ports/minimax-m3/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport CHECKPOINT_DIR=/models/MiniMax-M3-FP8\nexport PT_HPU_LAZY_MODE=0\nexport LOG_LEVEL_PT_BRIDGE=4\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_SCALE_ADJUSTMENT=false\nexport VLLM_M3_FP8_MOE=native\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/MiniMax-M3-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --enable-ep-weight-filter \\\n --max-model-len 131072 \\\n --max-num-seqs 32 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --enable-prompt-tokens-details \\\n --limit-mm-per-prompt '{\"image\": 0, \"video\": 0}' \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3_py \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/minimax-m3/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name amd/MiniMax-M3-MXFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --enable-ep-weight-filter \\\n --max-model-len 131072 \\\n --max-num-seqs 32 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --enable-prompt-tokens-details \\\n --limit-mm-per-prompt '{\"image\": 0, \"video\": 0}' \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser minimax_m3_py \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: CHECKPOINT_DIR\n value: \"/models/MiniMax-M3-FP8\"\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: LOG_LEVEL_PT_BRIDGE\n value: \"4\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_SCALE_ADJUSTMENT\n value: \"false\"\n - name: VLLM_M3_FP8_MOE\n value: \"native\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"amd/MiniMax-M3-MXFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model amd/MiniMax-M3-MXFP4 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 32" } }, { "id": "amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4", "title": "Muse Spark Open (MXFP4)", "provider": "Meta", "description": "816B MoE (42B active) with a latent MoE and per-expert output norm, the AMD Quark MXFP4 checkpoint as published, 256K tokens per request on one 8-card Gaudi 2 node, text only, through the muse-spark-open port.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "816B", "active_parameters": "42B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "async-inputs", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "topk-subset-sampler", "swa-decode-window", "rank-exception-fatal", "vllm-long-prefill-share" ], "verified": [ "kernel switch off: build gates 8/8; CPU tests 33 passed, patch chain on a clean release copy passed", "kernel switch off: perplexity vs the CPU reference, 5 passages, 17,408 tokens: worst ratio +0.27%, top-1 0.980, KL 0.0013", "kernel switch off: decode vs prefill 8 x 512 tokens: mean abs dlogprob 0.0058 (gate 0.05)", "kernel switch off: needles 32K and 258K x 3 depths 6/6 in content, 32K control clean; 2K 2/3 (one refusal, code in reasoning)", "kernel switch off: tool_choice auto / required / streamed pass; reasoning_effort low / high pass; greedy repeat 16/16", "kernel switch off: decode 22.7 tok/s at 1 stream, 43.3 aggregate at 4 (256 greedy tokens)", "kernel switch on: perplexity vs the CPU reference, same 5 passages: worst ratio +0.22%, top-1 0.983, KL 0.0011", "kernel switch on: decode vs prefill 8 x 512 tokens: mean abs dlogprob 0.0025 (gate 0.05)", "kernel switch on: needles 2K, 32K and 258K x 3 depths 9/9 found, 9/9 answered, 32K control clean", "kernel switch on: tool_choice auto / required / streamed pass; reasoning_effort low / high pass; greedy repeat 16/16", "kernel switch on: decode 40.8 tok/s at 1 stream, 102.8 aggregate at 4 (256 greedy tokens); peak HBM 82.9 of 94.6 GiB" ], "notes": [ "Needs the muse-spark-open port image: docker build -f ports/muse-spark-open/Dockerfile from the stock releases/v0.30.0 image. install.sh applies the v0.30.0 fix profile before the port's runner patches, so the launch's fix step finds the profile applied and adds the two opt-in fixes.", "Status supported: the gates above ran with MUSE_SPARK_MXFP4_KERNELS=0; the run of this exact command (switch on, the default) was not complete when the recipe was written.", "Text only: the checkpoint's vision and audio towers are not loaded.", "Muse Model License: products using the model must show \"Built with Muse Spark Open\"." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4/recipe.yaml", "page": "amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4/", "guide": "docs/recipes/muse-spark-open/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_RECIPE_CACHE_CONFIG=/data/muse-spark-recipe-cache,False,32768\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=65536\nexport VLLM_FP32_SOFTMAX=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_HPU_PROMPT_LOGPROBS_ROWS=256\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/muse-spark-open/serving/buckets-256k-seqs4.txt\nexport VLLM_LONG_PREFILL_SHARE=0.25\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport MUSE_SPARK_MOE_TP_SPLIT=4\nexport MUSE_SPARK_SWA_KV=1\nexport MUSE_SPARK_LONG_ATTN=1\nexport MUSE_SPARK_DECODE_GRAPH_MAX_BLOCKS=5408\nexport MUSE_SPARK_DECODE_LAYER_GROUP=62\nexport MUSE_SPARK_MXFP4_KERNELS=1\nexport MUSE_SPARK_MEMLOG=1\n\nmkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n topk-subset-sampler vllm-long-prefill-share || exit 1\n\nvllm serve amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --language-model-only \\\n --reasoning-parser muse \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse \\\n --chat-template-content-format openai \\\n --num-gpu-blocks-override 10752 \\\n --long-prefill-token-threshold 1024", "docker": "# 0. Build the port image from a stock image (ports/muse-spark-open/README.md).\ndocker build -f gaudi-cookbook/ports/muse-spark-open/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_RECIPE_CACHE_CONFIG=/data/muse-spark-recipe-cache,False,32768\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=65536\nexport VLLM_FP32_SOFTMAX=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_HPU_PROMPT_LOGPROBS_ROWS=256\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/muse-spark-open/serving/buckets-256k-seqs4.txt\nexport VLLM_LONG_PREFILL_SHARE=0.25\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport MUSE_SPARK_MOE_TP_SPLIT=4\nexport MUSE_SPARK_SWA_KV=1\nexport MUSE_SPARK_LONG_ATTN=1\nexport MUSE_SPARK_DECODE_GRAPH_MAX_BLOCKS=5408\nexport MUSE_SPARK_DECODE_LAYER_GROUP=62\nexport MUSE_SPARK_MXFP4_KERNELS=1\nexport MUSE_SPARK_MEMLOG=1\n\nmkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n topk-subset-sampler vllm-long-prefill-share || exit 1\n\nvllm serve amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --language-model-only \\\n --reasoning-parser muse \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse \\\n --chat-template-content-format openai \\\n --num-gpu-blocks-override 10752 \\\n --long-prefill-token-threshold 1024\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/muse-spark-open/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n mkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n topk-subset-sampler vllm-long-prefill-share || exit 1\n exec vllm serve amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4 \\\n --served-model-name muse-spark-open \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 4 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --language-model-only \\\n --reasoning-parser muse \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse \\\n --chat-template-content-format openai \\\n --num-gpu-blocks-override 10752 \\\n --long-prefill-token-threshold 1024\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPU_RECIPE_CACHE_CONFIG\n value: \"/data/muse-spark-recipe-cache,False,32768\"\n - name: VLLM_CONTIGUOUS_PA\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_ENGINE_READY_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"65536\"\n - name: VLLM_FP32_SOFTMAX\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_HPU_PROMPT_LOGPROBS_ROWS\n value: \"256\"\n - name: VLLM_BUCKETING_FROM_FILE\n value: \"/opt/gaudi-cookbook/ports/muse-spark-open/serving/buckets-256k-seqs4.txt\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.25\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"32768\"\n - name: MUSE_SPARK_MOE_TP_SPLIT\n value: \"4\"\n - name: MUSE_SPARK_SWA_KV\n value: \"1\"\n - name: MUSE_SPARK_LONG_ATTN\n value: \"1\"\n - name: MUSE_SPARK_DECODE_GRAPH_MAX_BLOCKS\n value: \"5408\"\n - name: MUSE_SPARK_DECODE_LAYER_GROUP\n value: \"62\"\n - name: MUSE_SPARK_MXFP4_KERNELS\n value: \"1\"\n - name: MUSE_SPARK_MEMLOG\n value: \"1\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model amd/Muse-Spark-Open-816B-A42B-Quark-MXFP4 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "title": "DeepSeek-V4.1-Flash-1017", "provider": "DeepSeek", "description": "40-layer MoE (384 routed experts, top-6) with compressed sparse attention, hyper-connections and Engram tables, served at 128K context on one 8-card Gaudi 2 node through the deepseek-v4-1-flash port, from a lossless re-layout of the public checkpoint.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "749B", "active_parameters": "17B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rank-exception-fatal" ], "verified": [ "install.sh on the stock releases/v0.30.0 image: commits, plugin, kernel-env patch, kernel build and image gate PASS", "converter re-run on one MoE layer and all dense tensors: sha256 equal to the served checkpoint, 1266/1266", "perplexity vs the fp32 CPU oracle (1536 tokens): 1.6180 vs 1.6239", "decode vs prefill teacher-forced: mean |dlogprob| 0.016, 10/756 off argmax, max gap 0.69 nats", "needles at 13.4K and 45.3K tokens: 2/2", "isolation, shared cached prefix, repeats and aborts: own code 48/48, 0 foreign hits", "structured output and tool calls (required and auto) under load: 60/60", "decode, kernels on, idle server: 37.6 tok/s at 1 stream, 108.1 tok/s aggregate at 4", "every gate also passes with DSV41_MXK=0; kernels off vs on, same pod: 20.7 vs 37.6 tok/s at 1 stream", "kernel parity, one card (run_parity.sh --variant dsv41): PARITY PASS, SIG_DEVICE PASS" ], "notes": [ "Needs the deepseek-v4-1-flash port image (docker build -f ports/deepseek-v4-1-flash/Dockerfile from the stock releases/v0.30.0 image) and the converted checkpoint: run ports/deepseek-v4-1-flash/convert/ on the public deepseek-ai/DeepSeek-V4.1-Flash-1017 (MIT). The original deepseek-ai/DeepSeek-V4.1-Flash converts with the same script.", "The plugin refuses to start unless prompt-order, resumed-prefill-logits, gather-logprobs-int32 and prompt-logprobs are applied.", "Text only. Async scheduling must stay off (Engram hashing reads every sampled token on the host).", "Run ports/deepseek-v4-1-flash/tests/hpu/run_gates.sh after the warm-up (/tmp/dsv41-warmup.done) before serving users." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/deepseek-ai/DeepSeek-V4.1-Flash-1017/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/deepseek-ai/DeepSeek-V4.1-Flash-1017/recipe.yaml", "page": "deepseek-ai/DeepSeek-V4.1-Flash-1017/", "guide": "docs/recipes/deepseek-v4-1-flash-1017/", "commands": { "shell": "export DSV41_CHECKPOINT=/models/DeepSeek-V4.1-Flash-1017-hpu\nexport HF_HUB_OFFLINE=1\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_RECIPE_CACHE_CONFIG=/data/dsv41-recipe-cache,False,32768\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=65536\nexport DSV41_LAYER_GRAPHS=1\nexport DSV41_LAYER_GROUP=10\nexport DSV41_DET_TOPK=1\nexport DSV41_TOPK_SELECT=thresh\nexport DSV41_PREFILL_ATTN=1\nexport DSV41_INDEX_BF16_DOT=1\nexport DSV41_REPLAY_KEEP=140\nexport DSV41_PREFILL_HEADROOM_GIB=6\nexport DSV41_MHC_SLICED=1\nexport DSV41_HC_SINKHORN=kernel\nexport DSV41_MHC_TPC_DOT=1\nexport DSV41_FQ_FASTBITS=1\nexport DSV41_LEAN_OPS=1\nexport DSV41_DECODE_HOIST=1\nexport DSV41_ROPE_FUSED=1\nexport DSV41_INDEX_DECODE=1\nexport DSV41_WOA_LINEAR=1\nexport DSV41_MXK=1\nexport DSV41_WARMUP=1\nexport DSV41_WARMUP_MAX_SEQS=8\nexport DSV41_WARMUP_MAX_S=2400\n\npython3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/check_checkpoint.py\" \"$DSV41_CHECKPOINT\" || exit 1\nmkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\npython3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/warmup.py\" > /tmp/dsv41-warmup.log 2>&1 &\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve \"$DSV41_CHECKPOINT\" \\\n --served-model-name deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 4096 \\\n --num-gpu-blocks-override 2304 \\\n --gpu-memory-utilization 0.90 \\\n --block-size 128 \\\n --enable-prefix-caching \\\n --no-async-scheduling \\\n --enforce-eager \\\n --tokenizer-mode deepseek_v41 \\\n --reasoning-parser deepseek_v41 \\\n --enable-auto-tool-choice \\\n --tool-call-parser deepseek_v41 \\\n --exclude-tools-when-tool-choice-none \\\n --language-model-only", "docker": "# 0. Build the port image from a stock image (ports/deepseek-v4-1-flash/README.md).\ndocker build -f gaudi-cookbook/ports/deepseek-v4-1-flash/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport DSV41_CHECKPOINT=/models/DeepSeek-V4.1-Flash-1017-hpu\nexport HF_HUB_OFFLINE=1\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPU_RECIPE_CACHE_CONFIG=/data/dsv41-recipe-cache,False,32768\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=65536\nexport DSV41_LAYER_GRAPHS=1\nexport DSV41_LAYER_GROUP=10\nexport DSV41_DET_TOPK=1\nexport DSV41_TOPK_SELECT=thresh\nexport DSV41_PREFILL_ATTN=1\nexport DSV41_INDEX_BF16_DOT=1\nexport DSV41_REPLAY_KEEP=140\nexport DSV41_PREFILL_HEADROOM_GIB=6\nexport DSV41_MHC_SLICED=1\nexport DSV41_HC_SINKHORN=kernel\nexport DSV41_MHC_TPC_DOT=1\nexport DSV41_FQ_FASTBITS=1\nexport DSV41_LEAN_OPS=1\nexport DSV41_DECODE_HOIST=1\nexport DSV41_ROPE_FUSED=1\nexport DSV41_INDEX_DECODE=1\nexport DSV41_WOA_LINEAR=1\nexport DSV41_MXK=1\nexport DSV41_WARMUP=1\nexport DSV41_WARMUP_MAX_SEQS=8\nexport DSV41_WARMUP_MAX_S=2400\n\npython3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/check_checkpoint.py\" \"$DSV41_CHECKPOINT\" || exit 1\nmkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\npython3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/warmup.py\" > /tmp/dsv41-warmup.log 2>&1 &\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve \"$DSV41_CHECKPOINT\" \\\n --served-model-name deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 4096 \\\n --num-gpu-blocks-override 2304 \\\n --gpu-memory-utilization 0.90 \\\n --block-size 128 \\\n --enable-prefix-caching \\\n --no-async-scheduling \\\n --enforce-eager \\\n --tokenizer-mode deepseek_v41 \\\n --reasoning-parser deepseek_v41 \\\n --enable-auto-tool-choice \\\n --tool-call-parser deepseek_v41 \\\n --exclude-tools-when-tool-choice-none \\\n --language-model-only\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/deepseek-v4-1-flash/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n python3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/check_checkpoint.py\" \"$DSV41_CHECKPOINT\" || exit 1\n mkdir -p \"${PT_HPU_RECIPE_CACHE_CONFIG%%,*}\"\n python3 \"$COOKBOOK/ports/deepseek-v4-1-flash/serving/warmup.py\" > /tmp/dsv41-warmup.log 2>&1 &\n # Fixes: the v0.30.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n exec vllm serve \"$DSV41_CHECKPOINT\" \\\n --served-model-name deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 4096 \\\n --num-gpu-blocks-override 2304 \\\n --gpu-memory-utilization 0.90 \\\n --block-size 128 \\\n --enable-prefix-caching \\\n --no-async-scheduling \\\n --enforce-eager \\\n --tokenizer-mode deepseek_v41 \\\n --reasoning-parser deepseek_v41 \\\n --enable-auto-tool-choice \\\n --tool-call-parser deepseek_v41 \\\n --exclude-tools-when-tool-choice-none \\\n --language-model-only\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: DSV41_CHECKPOINT\n value: \"/models/DeepSeek-V4.1-Flash-1017-hpu\"\n - name: HF_HUB_OFFLINE\n value: \"1\"\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPU_RECIPE_CACHE_CONFIG\n value: \"/data/dsv41-recipe-cache,False,32768\"\n - name: VLLM_CONTIGUOUS_PA\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_ENGINE_READY_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"65536\"\n - name: DSV41_LAYER_GRAPHS\n value: \"1\"\n - name: DSV41_LAYER_GROUP\n value: \"10\"\n - name: DSV41_DET_TOPK\n value: \"1\"\n - name: DSV41_TOPK_SELECT\n value: \"thresh\"\n - name: DSV41_PREFILL_ATTN\n value: \"1\"\n - name: DSV41_INDEX_BF16_DOT\n value: \"1\"\n - name: DSV41_REPLAY_KEEP\n value: \"140\"\n - name: DSV41_PREFILL_HEADROOM_GIB\n value: \"6\"\n - name: DSV41_MHC_SLICED\n value: \"1\"\n - name: DSV41_HC_SINKHORN\n value: \"kernel\"\n - name: DSV41_MHC_TPC_DOT\n value: \"1\"\n - name: DSV41_FQ_FASTBITS\n value: \"1\"\n - name: DSV41_LEAN_OPS\n value: \"1\"\n - name: DSV41_DECODE_HOIST\n value: \"1\"\n - name: DSV41_ROPE_FUSED\n value: \"1\"\n - name: DSV41_INDEX_DECODE\n value: \"1\"\n - name: DSV41_WOA_LINEAR\n value: \"1\"\n - name: DSV41_MXK\n value: \"1\"\n - name: DSV41_WARMUP\n value: \"1\"\n - name: DSV41_WARMUP_MAX_SEQS\n value: \"8\"\n - name: DSV41_WARMUP_MAX_S\n value: \"2400\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"deepseek-ai/DeepSeek-V4.1-Flash-1017\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "google/gemma-4-31B-it", "title": "Gemma 4 31B IT", "provider": "Google", "description": "31B dense multimodal (image + text) instruct model, BF16, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "31B", "active_parameters": "31B", "precision": "bf16", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "mm-merge-idxcopy", "rank-exception-fatal", "gemma4-required-tool-grammar", "gemma4-buffered-tool-args", "gemma4-bos-tokenizer" ], "verified": [ "Tool reliability, 3 cases x 40 sampled: 120/120; streaming tool_choice required, named and auto all correct", "Streamed tool-argument replay (escaped quotes, regex backslashes): 10/10 with gemma4-buffered-tool-args, 5/10 stock", "Multi-image requests with top_logprobs=20, 16 concurrent x 3 rounds: 96/96, 0 restarts", "Teacher-forced top-1 agreement with a reference run, 24 prompts, 11,754 tokens: 0.9924, mean NLL 0.05", "top_logprobs soak (unique single-token prompts, 4/8/16 concurrent): 0/1000 corrupt with the fix, 52/200 and 75/200 stock", "Raw completions with the BOS tokenizer match explicit- prompts byte for byte; chat prompt tokens unchanged (5/5, image prompt 298 tokens)", "Seeded traffic at saturation with client aborts, three replicas: 8,650 requests, 0 errors, 0 restarts with seeded-sampling; sequential same-seed repeats 12/12" ], "notes": [ "The top_logprobs checks ran with one more logprobs patch (an extra host sync) that is not part of the fixes pack.", "Thinking is off by default; send chat_template_kwargs {\"enable_thinking\": true} to turn it on.", "Penalties (repetition, presence, frequency, including a --override-generation-config repetition_penalty) are not applied under async scheduling on v0.26.0.", "Rare PT_DEVMEM allocation failures in graph replay under heavy load are not root-caused; vLLM exits 0 after them, so read the previous container log." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/google/gemma-4-31B-it/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/google/gemma-4-31B-it/recipe.yaml", "page": "google/gemma-4-31B-it/", "guide": "docs/recipes/gemma-4-31b-it/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=4096\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args || exit 1\n# Separate step, not run by apply_all.py: a tokenizer copy that adds \npython3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-31B-it /tmp/gemma4-bos-tokenizer || exit 1\n\nvllm serve google/gemma-4-31B-it \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.78 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":64,\"video\":0}' \\\n --trust-remote-code \\\n --tokenizer /tmp/gemma4-bos-tokenizer", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=8\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=4096\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args || exit 1\n# Separate step, not run by apply_all.py: a tokenizer copy that adds \npython3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-31B-it /tmp/gemma4-bos-tokenizer || exit 1\n\nvllm serve google/gemma-4-31B-it \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.78 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":64,\"video\":0}' \\\n --trust-remote-code \\\n --tokenizer /tmp/gemma4-bos-tokenizer\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args || exit 1\n # Separate step, not run by apply_all.py: a tokenizer copy that adds \n python3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-31B-it /tmp/gemma4-bos-tokenizer || exit 1\n exec vllm serve google/gemma-4-31B-it \\\n --served-model-name gemma-4-31b-it \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.78 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":64,\"video\":0}' \\\n --trust-remote-code \\\n --tokenizer /tmp/gemma4-bos-tokenizer\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.15\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"8\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"4096\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"google/gemma-4-31B-it\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model google/gemma-4-31B-it \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "google/gemma-4-E2B-it", "title": "Gemma 4 E2B IT", "provider": "Google", "description": "Small Gemma 4 instruct model (2.3B effective, 20 KV-shared layers) with KV-sharing fast prefill, BF16, text only, on one Gaudi 2 card.", "status": "verified", "tasks": [ "text" ], "architecture": "dense", "parameters": "5.1B", "active_parameters": "2.3B", "precision": "bf16", "cards": { "total": 1, "tp": 1, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 32768, "features": [ "tool_calling" ], "required_fixes": [ "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "gather-logprobs-int32", "logprobs-decode-guard", "seeded-sampling", "gemma4-required-tool-grammar", "gemma4-buffered-tool-args", "gemma4-fast-prefill", "gemma4-bos-tokenizer" ], "verified": [ "Fast prefill vs stock, offline, 64 wikitext prompts x 1,024 tokens, greedy: first token 64/64, 32-token continuations identical 64/64 (stock vs stock bit-identical)", "Served, 64 concurrent code-generation prompts (about 600 tokens in, 640 out), two rounds per mode: every output starts the same in both modes; whole outputs 58-61/64 identical between modes, 60/64 between two stock rounds", "Tool reliability, 3 cases x 40 sampled: 120/120", "Served first-token median 0.76-0.77 s with fast prefill, 1.00-1.01 s without (same card, 64 concurrent, warmed)" ], "notes": [ "Prompt batch cap (VLLM_PROMPT_BS_BUCKET_MAX) stays at the default 1: several prompts per prefill forward change the bf16 rounding of the prompt K/V.", "Prefix caching is off: a hit takes the full 35-layer path, and the checks ran without it.", "vLLM rejects prompt_logprobs while --kv-sharing-fast-prefill is on." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/google/gemma-4-E2B-it/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/google/gemma-4-E2B-it/recipe.yaml", "page": "google/gemma-4-E2B-it/", "guide": "docs/recipes/gemma-4-e2b-it/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\nunset VLLM_SKIP_WARMUP\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args gemma4-fast-prefill || exit 1\n# Separate step, not run by apply_all.py: a tokenizer copy that adds \npython3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-E2B-it /tmp/gemma4-bos-tokenizer || exit 1\n\nvllm serve google/gemma-4-E2B-it \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 8192 \\\n --no-enable-prefix-caching \\\n --kv-sharing-fast-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":0,\"audio\":0,\"video\":0}' \\\n --tokenizer /tmp/gemma4-bos-tokenizer", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\nunset VLLM_SKIP_WARMUP\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args gemma4-fast-prefill || exit 1\n# Separate step, not run by apply_all.py: a tokenizer copy that adds \npython3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-E2B-it /tmp/gemma4-bos-tokenizer || exit 1\n\nvllm serve google/gemma-4-E2B-it \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 8192 \\\n --no-enable-prefix-caching \\\n --kv-sharing-fast-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":0,\"audio\":0,\"video\":0}' \\\n --tokenizer /tmp/gemma4-bos-tokenizer\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n unset VLLM_SKIP_WARMUP\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n gemma4-required-tool-grammar gemma4-buffered-tool-args gemma4-fast-prefill || exit 1\n # Separate step, not run by apply_all.py: a tokenizer copy that adds \n python3 \"$COOKBOOK/fixes/gemma4-bos-tokenizer/apply.py\" google/gemma-4-E2B-it /tmp/gemma4-bos-tokenizer || exit 1\n exec vllm serve google/gemma-4-E2B-it \\\n --served-model-name gemma-4-e2b-it \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --max-model-len 32768 \\\n --max-num-seqs 64 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 8192 \\\n --no-enable-prefix-caching \\\n --kv-sharing-fast-prefill \\\n --enable-auto-tool-choice \\\n --tool-call-parser gemma4 \\\n --limit-mm-per-prompt '{\"image\":0,\"audio\":0,\"video\":0}' \\\n --tokenizer /tmp/gemma4-bos-tokenizer\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n resources:\n requests:\n habana.ai/gaudi: \"1\"\n limits:\n habana.ai/gaudi: \"1\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"google/gemma-4-E2B-it\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model google/gemma-4-E2B-it \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 64" } }, { "id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "title": "Llama 4 Scout 17B-16E Instruct", "provider": "Meta", "description": "109B MoE (17B active, 16 experts, top-1) multimodal instruct model, BF16, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "moe", "parameters": "109B", "active_parameters": "17B", "precision": "bf16", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "moe-router-weight-on-input", "rank-exception-fatal", "llama4-tools-lenient" ], "verified": [ "Next-token KL against a float32 CPU reference (59 positions, prompts of 37, 7.6K and 9.7K tokens): 0.003 / 0.006 / 0.007 with the router fix, 0.029 / 0.354 / 0.098 without", "Coding-agent tool scenarios straight at the server: 23/24 with the tools template, parser and router fix; 0/16 with the stock template and parser and no router fix", "Tool reliability 120/120 sampled at 0.6; with seeded requests 120/120", "Prefix-cache hits vs cold: 29/32, equal to the noise floor of shifting the chunk boundary by one token (29/32 with no cache)", "Image prefix-cache gate (same image new text, new image same text, swapped order, 4 images, bursts): 114 checks, 0 wrong reuse, 0 wrong answers", "Needles at 43K, 65K and 129.5K tokens: 9/9 cold, 9/9 from cache", "Greedy top-1 agreement with the previous image 0.9964 over 11,188 tokens; 24/24 greedy texts identical to 512 tokens; vision 6/6", "15-minute seeded storm (48 clients, 75% seeded, streaming, aborts): 1,991 requests, 0 errors, 0 restarts" ], "notes": [ "Fix paths in serve_args are relative to a checkout of this repository; point them at wherever the fixes pack lives in the container.", "With prefix caching on, vllm-gaudi turns off contiguous PA: single-stream decode is 6-11% slower than with caching off; 4+ streams are equal or faster.", "Penalties (repetition, presence, frequency) are not applied under async scheduling on v0.26.0." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/meta-llama/Llama-4-Scout-17B-16E-Instruct/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/meta-llama/Llama-4-Scout-17B-16E-Instruct/recipe.yaml", "page": "meta-llama/Llama-4-Scout-17B-16E-Instruct/", "guide": "docs/recipes/llama-4-scout-17b-16e-instruct/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_ALLOW_LONG_MAX_MODEL_LEN=1\nexport VLLM_HPU_MOE_WEIGHT_ON_INPUT=1\nexport LLAMA4_TOOL_CALL_POLICY=independent\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n moe-router-weight-on-input llama4-tools-lenient || exit 1\n\nvllm serve meta-llama/Llama-4-Scout-17B-16E-Instruct \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --chat-template \"$COOKBOOK/fixes/llama4-tools-lenient/llama4-tools.jinja\" \\\n --default-chat-template-kwargs '{\"fmt\": \"tag\", \"sys_mode\": \"meta\", \"defs\": \"meta\"}' \\\n --tool-parser-plugin \"$COOKBOOK/fixes/llama4-tools-lenient/llama4_tools_lenient.py\" \\\n --tool-call-parser llama4_lenient \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport PT_HPUGRAPH_DISABLE_TENSOR_CACHE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_GRAPH_PROMPT_RATIO=0.5\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_ALLOW_LONG_MAX_MODEL_LEN=1\nexport VLLM_HPU_MOE_WEIGHT_ON_INPUT=1\nexport LLAMA4_TOOL_CALL_POLICY=independent\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n moe-router-weight-on-input llama4-tools-lenient || exit 1\n\nvllm serve meta-llama/Llama-4-Scout-17B-16E-Instruct \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --chat-template \"$COOKBOOK/fixes/llama4-tools-lenient/llama4-tools.jinja\" \\\n --default-chat-template-kwargs '{\"fmt\": \"tag\", \"sys_mode\": \"meta\", \"defs\": \"meta\"}' \\\n --tool-parser-plugin \"$COOKBOOK/fixes/llama4-tools-lenient/llama4_tools_lenient.py\" \\\n --tool-call-parser llama4_lenient \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 \\\n moe-router-weight-on-input llama4-tools-lenient || exit 1\n exec vllm serve meta-llama/Llama-4-Scout-17B-16E-Instruct \\\n --served-model-name llama-4-scout-17b-16e-instruct \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 2048 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --chat-template \"$COOKBOOK/fixes/llama4-tools-lenient/llama4-tools.jinja\" \\\n --default-chat-template-kwargs '{\"fmt\": \"tag\", \"sys_mode\": \"meta\", \"defs\": \"meta\"}' \\\n --tool-parser-plugin \"$COOKBOOK/fixes/llama4-tools-lenient/llama4_tools_lenient.py\" \\\n --tool-call-parser llama4_lenient \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: PT_HPUGRAPH_DISABLE_TENSOR_CACHE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.5\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_ALLOW_LONG_MAX_MODEL_LEN\n value: \"1\"\n - name: VLLM_HPU_MOE_WEIGHT_ON_INPUT\n value: \"1\"\n - name: LLAMA4_TOOL_CALL_POLICY\n value: \"independent\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"meta-llama/Llama-4-Scout-17B-16E-Instruct\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model meta-llama/Llama-4-Scout-17B-16E-Instruct \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "meta-models/Muse-Glimmer-30B", "title": "Muse Glimmer 30B", "provider": "Meta", "description": "29.6B dense vision-language agent model (3:1 sliding/global attention, image input), BF16, 128K context on two Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "dense", "parameters": "29.6B", "active_parameters": "29.6B", "precision": "bf16", "cards": { "total": 2, "tp": 2, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "async-inputs", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "topk-subset-sampler", "mm-merge-idxcopy", "rank-exception-fatal" ], "verified": [ "Port image path (install.sh on the stock image, then the v0.30.0 profile plus topk-subset-sampler): 11/11 device gates in tests/hpu/run_gates.sh", "32 concurrent requests 32/32 correct; stock v0.30.0 stops the engine on the first two concurrent requests", "Needles at 8K, 32K (5% and 95% depth), 120K and 127K prompt tokens: 5/5", "Vision: shape count and OCR of a rendered image, 8 image sizes 8/8, text-only request after images correct", "tool_choice required 24/24 (8 greedy, 16 sampled at the generation defaults), named call with schema-valid arguments, auto still answers in prose; streaming required call has id, name and arguments", "logprobs 5 on completions 3/3 clean with greedy top-1 at every position; chat top_logprobs; seeded requests 128/128 at 64 concurrent, same-seed repeats identical", "KV cache 1,571,584 tokens (11.99 sequences of 131,072)" ], "notes": [ "The port adds three model-only patches to the stock image: the attention output keeps its 3D shape (batch > 1 stops the engine without it), the sliding layers read only their window in chunked prefill (without it a 120K prompt returned garbage and corrupted later requests), and tool_choice required/named are enforced with a grammar.", "Sampling defaults from generation_config.json are temperature 1.0, top_p 0.95, top_k 64.", "response_format json_schema is not enforced on this model: the grammar waits for a reasoning end that a plain answer never sends.", "tool_choice none can leak ATEM tool markup into content when the model calls a tool anyway." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/meta-models/Muse-Glimmer-30B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/meta-models/Muse-Glimmer-30B/recipe.yaml", "page": "meta-models/Muse-Glimmer-30B/", "guide": "docs/recipes/muse-glimmer-30b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=192\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=6144\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 topk-subset-sampler || exit 1\n\nvllm serve meta-models/Muse-Glimmer-30B \\\n --api-server-count 4 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 192 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --limit-mm-per-prompt '{\"image\":8,\"video\":0}'", "docker": "# 0. Build the port image from a stock image (ports/muse-glimmer-30b/README.md).\ndocker build -f gaudi-cookbook/ports/muse-glimmer-30b/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_GRAPH_RESERVED_MEM=0.3\nexport VLLM_GRAPH_PROMPT_RATIO=0.15\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=192\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=4096\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=131072\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=6144\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 topk-subset-sampler || exit 1\n\nvllm serve meta-models/Muse-Glimmer-30B \\\n --api-server-count 4 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 192 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --limit-mm-per-prompt '{\"image\":8,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/muse-glimmer-30b/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 topk-subset-sampler || exit 1\n exec vllm serve meta-models/Muse-Glimmer-30B \\\n --served-model-name muse-glimmer-30b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --api-server-count 4 \\\n --tensor-parallel-size 2 \\\n --max-model-len 131072 \\\n --max-num-seqs 192 \\\n --gpu-memory-utilization 0.85 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --limit-mm-per-prompt '{\"image\":8,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.3\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.15\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"192\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"4096\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"131072\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"6144\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n resources:\n requests:\n habana.ai/gaudi: \"2\"\n limits:\n habana.ai/gaudi: \"2\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"meta-models/Muse-Glimmer-30B\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model meta-models/Muse-Glimmer-30B \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 192" } }, { "id": "mistralai/Devstral-2-123B-Instruct-2512", "title": "Devstral-2-123B-Instruct-2512", "provider": "Mistral AI", "description": "123B dense agentic coding model, official FP8 checkpoint, 128K context with tool calling on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "dense", "parameters": "123B", "active_parameters": "123B", "precision": "fp8", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 4/4 with 4 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Needles, 3 depths each, cold: 3/3 at 32K and 3/3 at 120K tokens", "Single stream 22.1 tokens/s, 4 streams 85.7 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Chunked prefill does not interleave a short request with a long prefill: the 4096-token budget goes to the long prompt's chunk every step, so a short request waits for it.", "Licence: Modified MIT with a revenue limit (companies above USD 20 million monthly revenue need a commercial licence from Mistral AI)." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/mistralai/Devstral-2-123B-Instruct-2512/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/mistralai/Devstral-2-123B-Instruct-2512/recipe.yaml", "page": "mistralai/Devstral-2-123B-Instruct-2512/", "guide": "docs/recipes/devstral-2-123b-instruct-2512/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.40\nexport VLLM_GRAPH_PROMPT_RATIO=0.30\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=False\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=32\nexport VLLM_PROMPT_CTX_BUCKET_MAX=992\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=2048\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=4096\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=4096\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve mistralai/Devstral-2-123B-Instruct-2512 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_GRAPH_RESERVED_MEM=0.40\nexport VLLM_GRAPH_PROMPT_RATIO=0.30\nexport VLLM_GRAPH_PROMPT_STRATEGY=max_bs\nexport VLLM_GRAPH_DECODE_STRATEGY=max_bs\nexport VLLM_EXPONENTIAL_BUCKETING=False\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=32\nexport VLLM_PROMPT_CTX_BUCKET_MAX=992\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=2048\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=4096\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=4096\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve mistralai/Devstral-2-123B-Instruct-2512 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve mistralai/Devstral-2-123B-Instruct-2512 \\\n --served-model-name devstral-2-123b-instruct-2512 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --gpu-memory-utilization 0.92 \\\n --block-size 128 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.40\"\n - name: VLLM_GRAPH_PROMPT_RATIO\n value: \"0.30\"\n - name: VLLM_GRAPH_PROMPT_STRATEGY\n value: \"max_bs\"\n - name: VLLM_GRAPH_DECODE_STRATEGY\n value: \"max_bs\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"False\"\n - name: VLLM_PROMPT_CTX_BUCKET_MIN\n value: \"0\"\n - name: VLLM_PROMPT_CTX_BUCKET_STEP\n value: \"32\"\n - name: VLLM_PROMPT_CTX_BUCKET_MAX\n value: \"992\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"2048\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"4096\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"4096\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"mistralai/Devstral-2-123B-Instruct-2512\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model mistralai/Devstral-2-123B-Instruct-2512 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "openai/gpt-oss-120b", "title": "gpt-oss-120b", "provider": "OpenAI", "description": "117B MoE (5.1B active) reasoning model in native MXFP4, harmony format, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "117B", "active_parameters": "5.1B", "precision": "mxfp4", "cards": { "total": 4, "tp": 4, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "sink-once", "swa-decode-window", "rank-exception-fatal", "gptoss-harmony-parse-recovery" ], "verified": [ "16 in flight, about 4.1K-token prompts and 1.7K-token outputs: 96/96; two about 107K-token prompts 2/2 (TTFT 78-99 s)", "Request options: structured output, named and required tool calls through harmony, min_tokens, min_p, logit_bias all take effect", "Exact sinks and sliding window: long-context needles 9/9 with sink-once and swa-decode-window, 5/9 without", "Model test suite with the sink fixes 17/17 twice (14/17 and 15/17 without); coding-agent tier 5/5 twice (2/5 and 3/5 without)", "Gates at 32 seqs: harmony tools 8/8 per case, json_schema 6/6, two about 112K-token needles found, GSM8K 49/50" ], "notes": [ "openai_harmony downloads its vocab at start and does not create its cache directory: mkdir -p \"$TIKTOKEN_RS_CACHE_DIR\" first, on storage that survives restarts.", "HPU_MXFP4_NATIVE=0 dequantizes to BF16 and lands on a fused MoE kernel larger than the Gaudi 2 TPC instruction cache (device fault).", "The cookbook MXFP4 MoE decode kernel (kernels/mxfp4-moe) is 2-3x faster per stream at 4-16 in flight and makes two cards practical; this recipe is the stock path." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/openai/gpt-oss-120b/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/openai/gpt-oss-120b/recipe.yaml", "page": "openai/gpt-oss-120b/", "guide": "docs/recipes/gpt-oss-120b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport HPU_MXFP4_NATIVE=1\nexport HPU_FUSED_MOE=1\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport TIKTOKEN_RS_CACHE_DIR=/data/tiktoken-rs-cache\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nmkdir -p \"$TIKTOKEN_RS_CACHE_DIR\"\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gptoss-harmony-parse-recovery || exit 1\n\nvllm serve openai/gpt-oss-120b \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser openai \\\n --reasoning-parser openai_gptoss \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport HPU_MXFP4_NATIVE=1\nexport HPU_FUSED_MOE=1\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport TIKTOKEN_RS_CACHE_DIR=/data/tiktoken-rs-cache\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=True\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=1\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\nmkdir -p \"$TIKTOKEN_RS_CACHE_DIR\"\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gptoss-harmony-parse-recovery || exit 1\n\nvllm serve openai/gpt-oss-120b \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser openai \\\n --reasoning-parser openai_gptoss \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n mkdir -p \"$TIKTOKEN_RS_CACHE_DIR\"\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 gptoss-harmony-parse-recovery || exit 1\n exec vllm serve openai/gpt-oss-120b \\\n --served-model-name gpt-oss-120b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser openai \\\n --reasoning-parser openai_gptoss \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: HPU_MXFP4_NATIVE\n value: \"1\"\n - name: HPU_FUSED_MOE\n value: \"1\"\n - name: PT_HPU_QKV_SLICE_SEQ_LEN_THLD\n value: \"128\"\n - name: TIKTOKEN_RS_CACHE_DIR\n value: \"/data/tiktoken-rs-cache\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"True\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"openai/gpt-oss-120b\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model openai/gpt-oss-120b \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "ornith-ai/Ornith-1.0-35B", "title": "Ornith-1.0-35B", "provider": "Ornith", "description": "35B hybrid MoE (Gated DeltaNet + attention, 3B active) agentic coding model with vision, BF16, native 256K context on two Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "moe", "parameters": "35B", "active_parameters": "3B", "precision": "bf16", "cards": { "total": 2, "tp": 2, "ep": false }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-discard-counter", "hybrid-decode-pad", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "rope-cpu-cache", "rank-exception-fatal" ], "verified": [ "Known-answer prompts 8/8 at temperature 0 and 16/16 with 16 in flight", "Tool calls: 5/5 valid calls sampled at 0.6, streamed call assembled, tool result used in the answer", "top_logprobs 20, prompt_logprobs, JSON schema and logit_bias all take effect; server healthy after each", "Vision: 16 images in one prompt, 16/16 colours named in order", "Needles, 3 depths each, cold: 3/3 at 65K, 131K and 244K tokens", "Single stream 23.2 tokens/s, 16 streams 298.2 tokens/s aggregate (256 tokens, end to end)" ], "notes": [ "Prefix caching is off, as verified. On the stock release prefix-cache hits on Gated DeltaNet layers resume from the wrong state; gdn-prefix-cache makes them correct on other Qwen3.5-family recipes but has not been run on this model." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/ornith-ai/Ornith-1.0-35B/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/ornith-ai/Ornith-1.0-35B/recipe.yaml", "page": "ornith-ai/Ornith-1.0-35B/", "guide": "docs/recipes/ornith-1-0-35b/", "commands": { "shell": "export PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve ornith-ai/Ornith-1.0-35B \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --limit-mm-per-prompt '{\"image\": 16}' \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --trust-remote-code", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=0\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_USE_NAIVE_MAMBA_CACHE_SHARING=1\nexport VLLM_COMPACT_GDN=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.26.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n\nvllm serve ornith-ai/Ornith-1.0-35B \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --limit-mm-per-prompt '{\"image\": 16}' \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --trust-remote-code\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 || exit 1\n exec vllm serve ornith-ai/Ornith-1.0-35B \\\n --served-model-name ornith-1-0-35b \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --limit-mm-per-prompt '{\"image\": 16}' \\\n --reasoning-parser qwen3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_xml \\\n --trust-remote-code\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"0\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_USE_NAIVE_MAMBA_CACHE_SHARING\n value: \"1\"\n - name: VLLM_COMPACT_GDN\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"2\"\n limits:\n habana.ai/gaudi: \"2\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"ornith-ai/Ornith-1.0-35B\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model ornith-ai/Ornith-1.0-35B \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "poolside/Laguna-S-2.1", "title": "Laguna-S-2.1", "provider": "poolside", "description": "118B MoE (8B active) coding and agent model with reasoning, BF16, 128K context on four Gaudi 2 cards.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "118B", "active_parameters": "8B", "precision": "bf16", "cards": { "total": 4, "tp": 4, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "logits-empty-guard", "gather-logprobs-int32", "logprobs-decode-guard", "seeded-sampling", "laguna-moe-sigmoid-routing", "rank-exception-fatal" ], "verified": [ "Exact-tracking probes (17x20, 23x17, primes 50-70, sum of squares, letter count) x3: all correct with the routing fix; prose stays fluent but these collapse without it", "Tool calls 4/4 non-streaming and 4/4 streaming; reasoning separated", "Needle at 128K tokens: found (117 s cold)", "LiveBench coding at temperature 1.0, 32K budget: LCB generation 85.9; coding completion 52.0 / 56.0 / 50.0 over three runs (the previous image scored 56.0 the same day)", "Sampler check at temperature 1.0 (400 first-token draws x 4 prompts): sampled frequency within 0.01-0.02 of the model's probability" ], "notes": [ "TP4, not TP8: at TP8 (one KV head per rank) the output was corrupt in lazy and compile mode.", "Stay at 128K with 2048-token chunks: chunk x context beyond about 2048 x 130K trips the Synapse device watchdog ('No progress'), which kills the engine.", "Sync scheduling (--no-async-scheduling) with TP 4 needs logits-empty-guard; async-inputs (in the profile) then has nothing to do.", "rope-cpu-cache is not needed: the RoPE fix is upstream in v0.30.0, and the v0.30.0 profile leaves it out.", "Do not use step-1 decode block buckets with VLLM_SKIP_WARMUP: every new context size compiles a graph (seconds to 40 s) and real sessions keep hitting new ones." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/poolside/Laguna-S-2.1/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/poolside/Laguna-S-2.1/recipe.yaml", "page": "poolside/Laguna-S-2.1/", "guide": "docs/recipes/laguna-s-2-1/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=False\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=2048\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=16\nexport VLLM_PROMPT_CTX_BUCKET_MAX=1008\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=4\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=5120\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_READY_TIMEOUT_S=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_RPC_TIMEOUT=7200000\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve poolside/Laguna-S-2.1 \\\n --tensor-parallel-size 4 \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gpu-memory-utilization 0.90 \\\n --trust-remote-code \\\n --no-async-scheduling \\\n --reasoning-parser poolside_v1 \\\n --tool-call-parser poolside_v1 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"enable_thinking\": true}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_EXPONENTIAL_BUCKETING=False\nexport VLLM_PROMPT_BS_BUCKET_MIN=1\nexport VLLM_PROMPT_BS_BUCKET_STEP=1\nexport VLLM_PROMPT_BS_BUCKET_MAX=4\nexport VLLM_PROMPT_SEQ_BUCKET_MIN=128\nexport VLLM_PROMPT_SEQ_BUCKET_STEP=1024\nexport VLLM_PROMPT_SEQ_BUCKET_MAX=2048\nexport VLLM_PROMPT_CTX_BUCKET_MIN=0\nexport VLLM_PROMPT_CTX_BUCKET_STEP=16\nexport VLLM_PROMPT_CTX_BUCKET_MAX=1008\nexport VLLM_DECODE_BS_BUCKET_MIN=1\nexport VLLM_DECODE_BS_BUCKET_STEP=4\nexport VLLM_DECODE_BS_BUCKET_MAX=16\nexport VLLM_DECODE_BLOCK_BUCKET_MIN=128\nexport VLLM_DECODE_BLOCK_BUCKET_STEP=1024\nexport VLLM_DECODE_BLOCK_BUCKET_MAX=5120\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport OMP_NUM_THREADS=1\nexport VLLM_ENGINE_READY_TIMEOUT_S=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_RPC_TIMEOUT=7200000\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve poolside/Laguna-S-2.1 \\\n --tensor-parallel-size 4 \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gpu-memory-utilization 0.90 \\\n --trust-remote-code \\\n --no-async-scheduling \\\n --reasoning-parser poolside_v1 \\\n --tool-call-parser poolside_v1 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"enable_thinking\": true}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n exec vllm serve poolside/Laguna-S-2.1 \\\n --served-model-name laguna-s-2-1 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 4 \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 16 \\\n --block-size 128 \\\n --max-num-batched-tokens 2048 \\\n --enable-chunked-prefill \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --gpu-memory-utilization 0.90 \\\n --trust-remote-code \\\n --no-async-scheduling \\\n --reasoning-parser poolside_v1 \\\n --tool-call-parser poolside_v1 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"enable_thinking\": true}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_EXPONENTIAL_BUCKETING\n value: \"False\"\n - name: VLLM_PROMPT_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_STEP\n value: \"1\"\n - name: VLLM_PROMPT_BS_BUCKET_MAX\n value: \"4\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MIN\n value: \"128\"\n - name: VLLM_PROMPT_SEQ_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_PROMPT_SEQ_BUCKET_MAX\n value: \"2048\"\n - name: VLLM_PROMPT_CTX_BUCKET_MIN\n value: \"0\"\n - name: VLLM_PROMPT_CTX_BUCKET_STEP\n value: \"16\"\n - name: VLLM_PROMPT_CTX_BUCKET_MAX\n value: \"1008\"\n - name: VLLM_DECODE_BS_BUCKET_MIN\n value: \"1\"\n - name: VLLM_DECODE_BS_BUCKET_STEP\n value: \"4\"\n - name: VLLM_DECODE_BS_BUCKET_MAX\n value: \"16\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MIN\n value: \"128\"\n - name: VLLM_DECODE_BLOCK_BUCKET_STEP\n value: \"1024\"\n - name: VLLM_DECODE_BLOCK_BUCKET_MAX\n value: \"5120\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: OMP_NUM_THREADS\n value: \"1\"\n - name: VLLM_ENGINE_READY_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n resources:\n requests:\n habana.ai/gaudi: \"4\"\n limits:\n habana.ai/gaudi: \"4\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"poolside/Laguna-S-2.1\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model poolside/Laguna-S-2.1 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "reflection/Beam-1.0-501B-A23B-FP8", "title": "Beam-1.0-501B-A23B-FP8", "provider": "Reflection", "description": "501B MoE (23B active) reasoning and coding model, compressed-tensors FP8, 256K context on one 8-card Gaudi 2 node, through the beam-1-0 port.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "501B", "active_parameters": "23B", "precision": "fp8", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "structured-output-rows", "so-row-owners", "async-inputs", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "swa-decode-window", "moe-explicit-overload", "rank-exception-fatal" ], "verified": [ "Build gates 8/8 and CPU tests 103 passed in the port image built by install.sh from the stock releases/v0.30.0 image", "Smoke, 20 chat prompts at temperature 0: 20/20 correct, finish stop 20/20, reasoning 20/20, no markup", "Perplexity vs the per-token FP8 CPU reference: -0.09% / +0.23% / +0.04% on 3 passages; top-1 and top-20 KL within the gates", "Decode vs prefill, 8 x 512 greedy tokens: mean |dlogprob| 0.032 (gate 0.05)", "Needles 12/12 at 16K, 64K, 129K and 255K x 3 depths; no-needle control clean", "Copy test 64/64 at 129,938 and 254,938 tokens", "Tools 11/11 (auto, required, named, none, streaming and not, parallel calls, json_schema, effort none)", "Isolation: 8 concurrent requests, own code 8/8, foreign content 0", "Stress: 15 streaming decodes next to a 250K-token prompt, 15/15 complete, server healthy" ], "notes": [ "Needs the beam-1-0 port image: docker build -f ports/beam-1-0/Dockerfile from the stock releases/v0.30.0 image. install.sh applies the v0.30.0 fix profile before the port's runner patches, so the launch's fix step finds every fix applied.", "The KV pool holds about 776K tokens (only 9 of 52 layers keep full-length KV): about 3x a 256K request.", "Do not set --chat-template-content-format openai: the Beam template takes string content, and every chat request then fails with HTTP 400.", "Outputs are not bit-identical across batch compositions or cache hits (per-token FP8 activations turn kernel-shape differences into flips at near-ties)." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/reflection/Beam-1.0-501B-A23B-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/reflection/Beam-1.0-501B-A23B-FP8/recipe.yaml", "page": "reflection/Beam-1.0-501B-A23B-FP8/", "guide": "docs/recipes/beam-1-0-501b-a23b-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_FP32_SOFTMAX=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/beam-1-0/serving/tp8-seqs16-256k.txt\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT=1\nexport VLLM_BEAM_STREAM_TOOL_ARGS=1\nexport BEAM_EXACT_ROUNDING=0\nexport BEAM_PREFILL_INFLIGHT=1\nexport BEAM_FSDPA_WS_GIB=2\nexport BEAM_FSDPA_Q_TILE=256\nexport BEAM_GRAMMAR_DEVICE_MASK=1\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve reflection/Beam-1.0-501B-A23B-FP8 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --load-format beam \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --reasoning-parser beam \\\n --enable-auto-tool-choice \\\n --tool-call-parser beam", "docker": "# 0. Build the port image from a stock image (ports/beam-1-0/README.md).\ndocker build -f gaudi-cookbook/ports/beam-1-0/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_FP32_SOFTMAX=true\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/beam-1-0/serving/tp8-seqs16-256k.txt\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT=1\nexport VLLM_BEAM_STREAM_TOOL_ARGS=1\nexport BEAM_EXACT_ROUNDING=0\nexport BEAM_PREFILL_INFLIGHT=1\nexport BEAM_FSDPA_WS_GIB=2\nexport BEAM_FSDPA_Q_TILE=256\nexport BEAM_GRAMMAR_DEVICE_MASK=1\n\n# Fixes: the v0.30.0 profile, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n\nvllm serve reflection/Beam-1.0-501B-A23B-FP8 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --load-format beam \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --reasoning-parser beam \\\n --enable-auto-tool-choice \\\n --tool-call-parser beam\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/beam-1-0/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 || exit 1\n exec vllm serve reflection/Beam-1.0-501B-A23B-FP8 \\\n --served-model-name beam-1-0-501b-a23b-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --load-format beam \\\n --dtype bfloat16 \\\n --block-size 128 \\\n --max-model-len 262144 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 4096 \\\n --gpu-memory-utilization 0.90 \\\n --max-cudagraph-capture-size 4 \\\n --enable-prefix-caching \\\n --enable-prompt-tokens-details \\\n --async-scheduling \\\n --reasoning-parser beam \\\n --enable-auto-tool-choice \\\n --tool-call-parser beam\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_CONTIGUOUS_PA\n value: \"false\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_FP32_SOFTMAX\n value: \"true\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: VLLM_BUCKETING_FROM_FILE\n value: \"/opt/gaudi-cookbook/ports/beam-1-0/serving/tp8-seqs16-256k.txt\"\n - name: VLLM_ENGINE_READY_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT\n value: \"1\"\n - name: VLLM_BEAM_STREAM_TOOL_ARGS\n value: \"1\"\n - name: BEAM_EXACT_ROUNDING\n value: \"0\"\n - name: BEAM_PREFILL_INFLIGHT\n value: \"1\"\n - name: BEAM_FSDPA_WS_GIB\n value: \"2\"\n - name: BEAM_FSDPA_Q_TILE\n value: \"256\"\n - name: BEAM_GRAMMAR_DEVICE_MASK\n value: \"1\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"reflection/Beam-1.0-501B-A23B-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model reflection/Beam-1.0-501B-A23B-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "thinkingmachines/Inkling-Small", "title": "Inkling-Small", "provider": "Thinking Machines", "description": "276B-A12B MoE with relative-position attention and short convs, text only, BF16 at 256K context on one 8-card Gaudi 2 node (port).", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "276B", "active_parameters": "12B", "precision": "bf16", "cards": { "total": 8, "tp": 8, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "prompt-order", "sampling-rows-refresh", "structured-output-rows", "so-row-owners", "bad-words-index", "logitsprocs-rows", "async-inputs", "resumed-prefill-logits", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "rank-exception-fatal", "vllm-long-prefill-share" ], "verified": [ "Preflight in the image (plugin, parsers, KV layout, CPU tests): 21 passed", "Smoke (arithmetic, tool call, JSON mode, tool_choice required), greedy repeats, reasoning split, streamed reasoning and tool call: pass", "Needles at both ends of 3K and 6K prompts 4/4; at 8K, 32K and 131K x 3 depths 9/9; at 200K and 250K x 3 depths 6/6", "200 logprobs requests at concurrency 8: 200/200 clean; 16 concurrent 4K needles 16/16; a 264,111-token prompt gets HTTP 400 in 0.4 s", "Prefix-cache hit vs cold, 14 prompts: every hit recalled its code, 9 to 11 of 14 byte-identical, every split at a near-tie (the gate was changed from byte-identical after its first run)", "Three 100K prompts resident at once: 0 preemptions; a 250K prompt while 16 streams decode: needle found, streams 28.3 tok/s before, 34.7 during" ], "notes": [ "Text only: the vision and audio towers and the MTP draft head are not built; their tensors are skipped at load.", "Cold start is dominated by reading 532 GB of weights: 35 to 75 minutes from network storage.", "Prompts longer than --max-model-len get HTTP 400 at admission; the server refuses to start if the KV pool cannot hold one max-length request." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/thinkingmachines/Inkling-Small/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/thinkingmachines/Inkling-Small/recipe.yaml", "page": "thinkingmachines/Inkling-Small/", "guide": "docs/recipes/inkling-small/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_USE_V2_MODEL_RUNNER=0\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport INKLING_PREFILL_BIAS_BUDGET=805306368\nexport INKLING_DECODE_WINDOW_PLUS1=1\nexport INKLING_MOE_SINGLE_REDUCE=1\nexport INKLING_LOCAL_WINDOW_PREFILL=1\nexport INKLING_STEP_HOIST=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=5400\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 vllm-long-prefill-share || exit 1\n\nvllm serve thinkingmachines/Inkling-Small \\\n --tensor-parallel-size 8 \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --num-gpu-blocks-override 2560 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --max-cudagraph-capture-size 2048 \\\n --tokenizer-mode hf \\\n --gpu-memory-utilization 0.75 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser inkling_py \\\n --reasoning-parser inkling_py", "docker": "# 0. Build the port image from a stock image (ports/inkling-small/README.md).\ndocker build -f gaudi-cookbook/ports/inkling-small/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_USE_V2_MODEL_RUNNER=0\nexport VLLM_CONTIGUOUS_PA=false\nexport VLLM_GRAPH_RESERVED_MEM=0.4\nexport INKLING_PREFILL_BIAS_BUDGET=805306368\nexport INKLING_DECODE_WINDOW_PLUS1=1\nexport INKLING_MOE_SINGLE_REDUCE=1\nexport INKLING_LOCAL_WINDOW_PREFILL=1\nexport INKLING_STEP_HOIST=1\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=5400\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 vllm-long-prefill-share || exit 1\n\nvllm serve thinkingmachines/Inkling-Small \\\n --tensor-parallel-size 8 \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --num-gpu-blocks-override 2560 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --max-cudagraph-capture-size 2048 \\\n --tokenizer-mode hf \\\n --gpu-memory-utilization 0.75 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser inkling_py \\\n --reasoning-parser inkling_py\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/inkling-small/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 vllm-long-prefill-share || exit 1\n exec vllm serve thinkingmachines/Inkling-Small \\\n --served-model-name inkling-small \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --num-gpu-blocks-override 2560 \\\n --max-num-seqs 16 \\\n --max-num-batched-tokens 2048 \\\n --block-size 128 \\\n --max-cudagraph-capture-size 2048 \\\n --tokenizer-mode hf \\\n --gpu-memory-utilization 0.75 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --enable-auto-tool-choice \\\n --tool-call-parser inkling_py \\\n --reasoning-parser inkling_py\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_USE_V2_MODEL_RUNNER\n value: \"0\"\n - name: VLLM_CONTIGUOUS_PA\n value: \"false\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.4\"\n - name: INKLING_PREFILL_BIAS_BUDGET\n value: \"805306368\"\n - name: INKLING_DECODE_WINDOW_PLUS1\n value: \"1\"\n - name: INKLING_MOE_SINGLE_REDUCE\n value: \"1\"\n - name: INKLING_LOCAL_WINDOW_PREFILL\n value: \"1\"\n - name: INKLING_STEP_HOIST\n value: \"1\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"32768\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"5400\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"thinkingmachines/Inkling-Small\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model thinkingmachines/Inkling-Small \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 16" } }, { "id": "zai-org/GLM-4.5V-FP8", "title": "GLM-4.5V (FP8)", "provider": "Z.ai", "description": "106B MoE vision-language model (12B active) in FP8, 32K context on two Gaudi 2 cards.", "status": "verified", "tasks": [ "multimodal" ], "architecture": "moe", "parameters": "106B", "active_parameters": "12B", "precision": "fp8", "cards": { "total": 2, "tp": 2, "ep": false }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 32768, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "mm-merge-idxcopy", "glm4-moe-3d-input", "glm45v-box-tokens" ], "verified": [ "Stock profile only: ValueError in Glm4MoE.forward on the first request", "Box markers in short answers: 4/4 with glm4-moe-3d-input only, 0/4 with both fixes, streamed and not", "Text, reasoning split, OCR of an arbitrary number, three shapes, text-only after image 3/3, 4 x 2048 px images, 16 mixed requests at 8 concurrent", "glm45 tool call, top_logprobs and prompt_logprobs, needle at about 28K tokens" ], "notes": [ "32K served: a 64K prefill hung the device on an older release and was not re-tested on this image.", "Thinking is on by default; chat_template_kwargs {\"enable_thinking\": false} turns it off." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/zai-org/GLM-4.5V-FP8/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/zai-org/GLM-4.5V-FP8/recipe.yaml", "page": "zai-org/GLM-4.5V-FP8/", "guide": "docs/recipes/glm-4-5v-fp8/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n glm4-moe-3d-input glm45v-box-tokens || exit 1\n\nvllm serve zai-org/GLM-4.5V-FP8 \\\n --tensor-parallel-size 2 \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser glm45 \\\n --reasoning-parser glm45 \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'", "docker": "# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport PT_HPU_ENABLE_LAZY_COLLECTIVES=true\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_MULTIMODAL_BUCKETS=None\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=3600\nexport VLLM_RPC_TIMEOUT=3600000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n glm4-moe-3d-input glm45v-box-tokens || exit 1\n\nvllm serve zai-org/GLM-4.5V-FP8 \\\n --tensor-parallel-size 2 \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser glm45 \\\n --reasoning-parser glm45 \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# Assumes this repository is copied into the image at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n glm4-moe-3d-input glm45v-box-tokens || exit 1\n exec vllm serve zai-org/GLM-4.5V-FP8 \\\n --served-model-name glm-4-5v-fp8 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 2 \\\n --max-model-len 32768 \\\n --max-num-seqs 8 \\\n --gpu-memory-utilization 0.9 \\\n --max-num-batched-tokens 4096 \\\n --enable-chunked-prefill \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser glm45 \\\n --reasoning-parser glm45 \\\n --limit-mm-per-prompt '{\"image\":4,\"video\":0}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: PT_HPU_ENABLE_LAZY_COLLECTIVES\n value: \"true\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_MULTIMODAL_BUCKETS\n value: \"None\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"3600\"\n - name: VLLM_RPC_TIMEOUT\n value: \"3600000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n resources:\n requests:\n habana.ai/gaudi: \"2\"\n limits:\n habana.ai/gaudi: \"2\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"zai-org/GLM-4.5V-FP8\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model zai-org/GLM-4.5V-FP8 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 8" } }, { "id": "zai-org/GLM-5.3-BF16", "title": "GLM-5.3 (MXFP4)", "provider": "Z.ai", "description": "744B MoE (about 41B active) with DeepSeek sparse attention, quantized to MXFP4 experts from the BF16 release and served at 128K context with MTP on one 8-card Gaudi 2 node, through the glm-5-mxfp4 port.", "status": "verified", "tasks": [ "text" ], "architecture": "moe", "parameters": "744B", "active_parameters": "41B", "precision": "mxfp4", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "releases/v0.30.0", "vllm_gaudi_commit": "058f107", "vllm_commit": "ced6857a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 131072, "features": [ "tool_calling", "reasoning", "spec_decoding" ], "required_fixes": [ "structured-output-rows", "so-row-owners", "gather-logprobs-int32", "logprobs-decode-guard", "seeded-sampling", "rank-exception-fatal", "vllm-long-prefill-share", "glm47-streaming-text-markers" ], "verified": [ "Needles, 3 codes at 10/50/90% depth, two fillers, 16K to 126K tokens: 30/30", "GSM8K first 50, reasoning low, greedy: 49/50", "Tools (required, named, auto, none, JSON schema; streaming and not; greedy and sampled): 31/32 on the first run, 32/32 on three runs after a restart; the failing case (required with no tool needed, sampled) returned two calls in 4 of 40 repeats", "MTP acceptance at default sampling: 0.816 over 1,465 drafts", "Logprobs, 40 sampled requests with top_logprobs 5 under MTP: 40/40 complete and finite", "Stress, 4 decodes while 64K and 100K prompts prefill: 4/4 finished, needles 3/3, largest decode gap 1.7 s", "Greedy with GLM_MXK_PREFILL=0 and with the decode levers off: 16/16 token-identical", "Decode 68.6 tok/s at 1 stream, 116.1 aggregate at 4 (~2K-token prompt, greedy); TTFT 0.52 / 1.76 s at 2K / 8K" ], "notes": [ "Needs the glm-5-mxfp4 port image: docker build -f ports/glm-5-mxfp4/Dockerfile from the stock releases/v0.30.0 image. install.sh applies the fixes before the port's patches, so the launch's fix step finds every fix applied.", "There is no public MXFP4 GLM-5.3: convert zai-org/GLM-5.3-BF16 once with ports/glm-5-mxfp4/convert (CPU only). The result is a derivative under the GLM-5.3 License.", "Speculative decoding turns async scheduling off." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/zai-org/GLM-5.3-BF16/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/zai-org/GLM-5.3-BF16/recipe.yaml", "page": "zai-org/GLM-5.3-BF16/", "guide": "docs/recipes/glm-5-3-mxfp4/", "commands": { "shell": "export CHECKPOINT_DIR=/models/GLM-5.3-MXFP4\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_HPU_FSDPA_SOFTMAX_MODE=None\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport GLM_DSA_SPARSE=1\nexport GLM_MXK=1\nexport GLM_MXK_PREFILL=1\nexport GLM_MOE_TP=2\nexport GLM_MOE_PLACEMENT=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/glm53-ep4-placement.json\nexport GLM_ROUTER_FAST=1\nexport GLM_EARLY_DRAFT=1\nexport GLM_SPEC_DEVICE_REJECT=1\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name zai-org/GLM-5.3-BF16 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\", \"clear_thinking\": true}' \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}'", "docker": "# 0. Build the port image from a stock image (ports/glm-5-mxfp4/README.md).\ndocker build -f gaudi-cookbook/ports/glm-5-mxfp4/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport CHECKPOINT_DIR=/models/GLM-5.3-MXFP4\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_BUCKETING_FROM_FILE=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\nexport VLLM_GRAPH_RESERVED_MEM=0.2\nexport VLLM_MOE_CHUNK=256\nexport VLLM_MOE_TOKEN_BOUNDARY=2048\nexport PT_HPU_QKV_SLICE_SEQ_LEN_THLD=128\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=3600\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_HPU_FSDPA_SOFTMAX_MODE=None\nexport VLLM_LONG_PREFILL_SHARE=0.5\nexport VLLM_LONG_PREFILL_SHARE_ASYNC=1\nexport VLLM_LONG_PREFILL_MIN_TOKENS=32768\nexport GLM_DSA_SPARSE=1\nexport GLM_MXK=1\nexport GLM_MXK_PREFILL=1\nexport GLM_MOE_TP=2\nexport GLM_MOE_PLACEMENT=/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/glm53-ep4-placement.json\nexport GLM_ROUTER_FAST=1\nexport GLM_EARLY_DRAFT=1\nexport GLM_SPEC_DEVICE_REJECT=1\n\n# Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n\nvllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name zai-org/GLM-5.3-BF16 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\", \"clear_thinking\": true}' \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/glm-5-mxfp4/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.30.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.30.0 \\\n vllm-long-prefill-share glm47-streaming-text-markers || exit 1\n exec vllm serve \"$CHECKPOINT_DIR\" \\\n --served-model-name zai-org/GLM-5.3-BF16 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 131072 \\\n --max-num-seqs 4 \\\n --block-size 128 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --long-prefill-token-threshold 4096 \\\n --gpu-memory-utilization 0.86 \\\n --max-cudagraph-capture-size 2048 \\\n --trust-remote-code \\\n --enable-prompt-tokens-details \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\", \"clear_thinking\": true}' \\\n --override-generation-config '{\"temperature\": 1.0, \"top_p\": 0.95}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: CHECKPOINT_DIR\n value: \"/models/GLM-5.3-MXFP4\"\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_BUCKETING_FROM_FILE\n value: \"/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/tp8-seqs4-128k.txt\"\n - name: VLLM_GRAPH_RESERVED_MEM\n value: \"0.2\"\n - name: VLLM_MOE_CHUNK\n value: \"256\"\n - name: VLLM_MOE_TOKEN_BOUNDARY\n value: \"2048\"\n - name: PT_HPU_QKV_SLICE_SEQ_LEN_THLD\n value: \"128\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"3600\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_HPU_FSDPA_SOFTMAX_MODE\n value: \"None\"\n - name: VLLM_LONG_PREFILL_SHARE\n value: \"0.5\"\n - name: VLLM_LONG_PREFILL_SHARE_ASYNC\n value: \"1\"\n - name: VLLM_LONG_PREFILL_MIN_TOKENS\n value: \"32768\"\n - name: GLM_DSA_SPARSE\n value: \"1\"\n - name: GLM_MXK\n value: \"1\"\n - name: GLM_MXK_PREFILL\n value: \"1\"\n - name: GLM_MOE_TP\n value: \"2\"\n - name: GLM_MOE_PLACEMENT\n value: \"/opt/gaudi-cookbook/ports/glm-5-mxfp4/serving/glm53-ep4-placement.json\"\n - name: GLM_ROUTER_FAST\n value: \"1\"\n - name: GLM_EARLY_DRAFT\n value: \"1\"\n - name: GLM_SPEC_DEVICE_REJECT\n value: \"1\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"zai-org/GLM-5.3-BF16\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model zai-org/GLM-5.3-BF16 \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 4" } }, { "id": "zai-org/GLM-5.3-Flash", "title": "GLM-5.3-Flash", "provider": "Z.ai", "description": "320B MoE (18B active) with KDA linear attention, NoPE MLA, mHC and image input, block FP8, 256K context on eight Gaudi 2 cards through the glm-5-3-flash port.", "status": "verified", "tasks": [ "text", "multimodal" ], "architecture": "moe", "parameters": "320B", "active_parameters": "18B", "precision": "fp8", "cards": { "total": 8, "tp": 8, "ep": true }, "image": { "vllm_gaudi": "v0.26.0", "vllm_gaudi_commit": "b059f9c", "vllm_commit": "568afb3a", "gaudi_sw": "1.24.1", "placeholder": "" }, "max_model_len": 262144, "features": [ "tool_calling", "reasoning" ], "required_fixes": [ "sampling-rows-refresh", "structured-output-rows", "bad-words-index", "logitsprocs-rows", "async-inputs", "async-discard-counter", "resumed-prefill-logits", "hybrid-decode-pad", "gather-logprobs-int32", "prompt-logprobs", "logprobs-decode-guard", "seeded-sampling", "moe-explicit-overload", "mm-merge-idxcopy", "rank-exception-fatal", "xgrammar-generation-eos-stop", "glm47-streaming-text-markers" ], "verified": [ "Golden: 12 greedy prompts token-identical to the reference outputs", "Needles: 3/3 at 16K, 63K, 127K and 239K tokens", "Tool calls 8/8: single, parallel and no call, streamed and not, multi-turn, a call after 32K tokens", "Images 8/8: colours, quadrants, counting, text before and after; a 5-image prompt rejected with 400", "Single-stream decode 43.5 tokens/s at 2K and 38.6 at 128K; warm TTFT 2.6 / 10.3 / 23.8 s at 16K / 64K / 128K", "mHC Sinkhorn kernel bit-identical to vLLM's stock loop at 1 to 1,024 rows", "Switches off: the same server gates pass and the golden prompts stay token-identical" ], "notes": [ "Needs the glm-5-3-flash port image: model plugin, four source patches and the mHC Sinkhorn kernel (ports/glm-5-3-flash/).", "Prefix caching must stay off: the KDA layer implements only the non-caching state contract and refuses to start with it.", "KV is pinned (--num-gpu-blocks-override 1280, 640-token blocks, 819,200 tokens): autosizing over-commits and fails.", "Video is not served; images only, at most 4 per prompt." ], "recipe_yaml": "https://github.com/thediymaker/gaudi-cookbook/blob/main/recipes/zai-org/GLM-5.3-Flash/recipe.yaml", "raw_yaml": "https://raw.githubusercontent.com/thediymaker/gaudi-cookbook/main/recipes/zai-org/GLM-5.3-Flash/recipe.yaml", "page": "zai-org/GLM-5.3-Flash/", "guide": "docs/recipes/glm-5-3-flash/", "commands": { "shell": "export PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\nexport GLM5_NEXT_VISION=1\nexport GLM5_NEXT_IMAGE_MAX_TOKENS=2048\nexport GLM5_NEXT_KDA_SCAN=1\nexport GLM5_NEXT_DECODE_LAYER_GROUP=45\nexport GLM5_NEXT_MHC_SINKHORN=kernel\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 glm47-streaming-text-markers || exit 1\n\nvllm serve zai-org/GLM-5.3-Flash \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --num-gpu-blocks-override 1280 \\\n --max-num-batched-tokens 8192 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.82 \\\n --no-enable-prefix-caching \\\n --trust-remote-code \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --chat-template \"$COOKBOOK/ports/glm-5-3-flash/serving/chat_template.jinja\" \\\n --limit-mm-per-prompt '{\"image\": 4}'", "docker": "# 0. Build the port image from a stock image (ports/glm-5-3-flash/README.md).\ndocker build -f gaudi-cookbook/ports/glm-5-3-flash/Dockerfile \\\n --build-arg BASE= -t gaudi-cookbook\n\n# 1. Write the launch script (the Shell tab) next to a checkout of this repository.\ncat > launch.sh <<'EOF'\nexport PT_HPU_LAZY_MODE=1\nexport VLLM_SKIP_WARMUP=true\nexport VLLM_ENGINE_ITERATION_TIMEOUT_S=7200\nexport VLLM_RPC_TIMEOUT=7200000\nexport VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800\nexport GLM5_NEXT_VISION=1\nexport GLM5_NEXT_IMAGE_MAX_TOKENS=2048\nexport GLM5_NEXT_KDA_SCAN=1\nexport GLM5_NEXT_DECODE_LAYER_GROUP=45\nexport GLM5_NEXT_MHC_SINKHORN=kernel\n\n# Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n# Stops with a non-zero exit code if the image does not match.\npython3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 glm47-streaming-text-markers || exit 1\n\nvllm serve zai-org/GLM-5.3-Flash \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --num-gpu-blocks-override 1280 \\\n --max-num-batched-tokens 8192 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.82 \\\n --no-enable-prefix-caching \\\n --trust-remote-code \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --chat-template \"$COOKBOOK/ports/glm-5-3-flash/serving/chat_template.jinja\" \\\n --limit-mm-per-prompt '{\"image\": 4}'\nEOF\n\n# 2. Run it in a vllm-gaudi container.\ndocker run -it --rm --runtime=habana \\\n -e HABANA_VISIBLE_DEVICES=all \\\n -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\\n -e HF_TOKEN \\\n --cap-add=sys_nice --ipc=host --net=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v \"$PWD/gaudi-cookbook:/cookbook:ro\" -e COOKBOOK=/cookbook \\\n -v \"$PWD/launch.sh:/launch.sh:ro\" \\\n \\\n bash /launch.sh", "kubernetes": "# Container for kubernetes/manifests/vllm-deployment.yaml: keep the template's probes, volumes,\n# hugepages and drain settings, and size CPU and memory per card (docs/kubernetes/pod-spec.md).\n# The port image (ports/glm-5-3-flash/Dockerfile) carries this repository at /opt/gaudi-cookbook.\ncontainers:\n - name: vllm\n image: \n command: [\"/bin/bash\", \"-c\"]\n args:\n - |\n set -euo pipefail\n # Fixes: the v0.26.0 profile plus this recipe's opt-in fixes, in the pack's order.\n # Stops with a non-zero exit code if the image does not match.\n python3 \"$COOKBOOK/fixes/apply_all.py\" --profile v0.26.0 glm47-streaming-text-markers || exit 1\n exec vllm serve zai-org/GLM-5.3-Flash \\\n --served-model-name glm-5-3-flash \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --shutdown-timeout 120 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --dtype bfloat16 \\\n --max-model-len 262144 \\\n --max-num-seqs 32 \\\n --num-gpu-blocks-override 1280 \\\n --max-num-batched-tokens 8192 \\\n --max-cudagraph-capture-size 2048 \\\n --enable-chunked-prefill \\\n --gpu-memory-utilization 0.82 \\\n --no-enable-prefix-caching \\\n --trust-remote-code \\\n --reasoning-parser glm47 \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --default-chat-template-kwargs '{\"reasoning_effort\": \"high\"}' \\\n --chat-template \"$COOKBOOK/ports/glm-5-3-flash/serving/chat_template.jinja\" \\\n --limit-mm-per-prompt '{\"image\": 4}'\n env:\n - name: COOKBOOK\n value: /opt/gaudi-cookbook\n - name: PT_HPU_LAZY_MODE\n value: \"1\"\n - name: VLLM_SKIP_WARMUP\n value: \"true\"\n - name: VLLM_ENGINE_ITERATION_TIMEOUT_S\n value: \"7200\"\n - name: VLLM_RPC_TIMEOUT\n value: \"7200000\"\n - name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS\n value: \"1800\"\n - name: GLM5_NEXT_VISION\n value: \"1\"\n - name: GLM5_NEXT_IMAGE_MAX_TOKENS\n value: \"2048\"\n - name: GLM5_NEXT_KDA_SCAN\n value: \"1\"\n - name: GLM5_NEXT_DECODE_LAYER_GROUP\n value: \"45\"\n - name: GLM5_NEXT_MHC_SINKHORN\n value: \"kernel\"\n resources:\n requests:\n habana.ai/gaudi: \"8\"\n limits:\n habana.ai/gaudi: \"8\"\n ports:\n - name: http\n containerPort: 8000" }, "clients": { "curl": "curl -s http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\": \"zai-org/GLM-5.3-Flash\", \"messages\": [{\"role\": \"user\", \"content\": \"Say hello in one sentence.\"}], \"max_tokens\": 512}'", "bench": "vllm bench serve --model zai-org/GLM-5.3-Flash \\\n --dataset-name random --random-input-len 1024 --random-output-len 256 \\\n --num-prompts 64 --max-concurrency 32" } } ] }