Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/dstack/_internal/core/backends/kubernetes/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -625,7 +625,8 @@ def get_nvidia_gpu_from_node_labels(
gpu_product, _, mig_profile = gpu_product.partition(NVIDIA_MIG_PRODUCT_SEPARATOR)
if mig_resource is not None:
mig_profile = mig_resource.removeprefix(NVIDIA_MIG_RESOURCE_PREFIX)
gpu_product = gpu_product.replace("RTX-", "RTX")
# Join "RTX" with the model, e.g. "RTX-4090" -> "RTX4090", "RTX-PRO-6000" -> "RTXPRO6000"
gpu_product = gpu_product.replace("RTX-PRO-", "RTXPRO").replace("RTX-", "RTX")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change also makes the RTX PRO 5000 detectable, but the 48GB card gets the wrong memory:

get_nvidia_gpu_from_node_labels({"nvidia.com/gpu.product": "NVIDIA-RTX-PRO-5000-Blackwell"})
# Gpu(name='RTXPRO5000', memory_mib=73728), i.e. 72GB instead of 48GB

So a run that needs more than 48GB could be scheduled on it. gpuhunt has two RTXPRO5000 entries, 48GB and 72GB, and NVIDIA_GPU_NAME_TO_GPU_INFO keeps the last one. Only the 72GB card has its memory in the product name (NVIDIA RTX PRO 5000 72GB Blackwell).

Could you add this next to the A100 check below, plus test cases for both cards?

    # RTX PRO 5000 may come in two variants
    if gpu_name == "RTXPRO5000":
        gpu_memory = (72 if "72GB" in gpu_product else 48) * 1024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, confirmed: the driver's supported-GPU list names the cards "NVIDIA RTX PRO 5000 Blackwell" and "NVIDIA RTX PRO 5000 72GB Blackwell" (same device ID), so only the 72GB label carries the size. Added your check next to the A100 one, with test cases for both labels (cd1c911).

for gpu_name in NVIDIA_GPU_NAMES:
if gpu_name.lower() in gpu_product.lower().split("-"):
break
Expand All @@ -636,6 +637,9 @@ def get_nvidia_gpu_from_node_labels(
# A100 may come in two variants
if "40GB" in gpu_product:
gpu_memory = 40 * 1024
# RTX PRO 5000 may come in two variants
if gpu_name == "RTXPRO5000":
gpu_memory = (72 if "72GB" in gpu_product else 48) * 1024
if mig_profile:
mig_memory = _get_mig_memory_mib(mig_profile)
if mig_memory is None:
Expand Down
33 changes: 33 additions & 0 deletions src/tests/_internal/core/backends/kubernetes/test_resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,39 @@ def test_returns_correct_memory_for_different_A100(self):
{"nvidia.com/gpu.product": "A100-SXM4-80GB"}
) == Gpu(vendor=AcceleratorVendor.NVIDIA, name="A100", memory_mib=80 * 1024)

@pytest.mark.parametrize(
["product", "expected_name", "expected_memory_gb"],
[
# The label set by GPU Feature Discovery v0.20.1
pytest.param(
"NVIDIA-RTX-PRO-6000-Blackwell-Max-Q-Workstation-Edition",
"RTXPRO6000",
96,
id="rtx-pro-6000-max-q",
),
pytest.param(
"NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition",
"RTXPRO6000",
96,
id="rtx-pro-6000-server",
),
# The two RTX PRO 5000 variants share the name; only the 72GB one has its memory
# in the product name
pytest.param("NVIDIA-RTX-PRO-5000-Blackwell", "RTXPRO5000", 48, id="rtx-pro-5000"),
pytest.param(
"NVIDIA-RTX-PRO-5000-72GB-Blackwell", "RTXPRO5000", 72, id="rtx-pro-5000-72gb"
),
pytest.param("NVIDIA-RTX-PRO-4500-Blackwell", "RTXPRO4500", 32, id="rtx-pro-4500"),
pytest.param("NVIDIA-GeForce-RTX-4090", "RTX4090", 24, id="rtx-4090"),
],
)
def test_returns_rtx_gpu(self, product: str, expected_name: str, expected_memory_gb: int):
assert get_nvidia_gpu_from_node_labels({"nvidia.com/gpu.product": product}) == Gpu(
vendor=AcceleratorVendor.NVIDIA,
name=expected_name,
memory_mib=expected_memory_gb * 1024,
)

@pytest.mark.parametrize(
["product", "expected_name", "expected_memory_gb"],
[
Expand Down
Loading