Skip to content

Commit a4bdeb7

Browse files
peterschmidt85Andrey Cheptsov
andauthored
[Nebius] Add opt-in capacity advisor (#4430)
* Use Nebius Capacity Advisor for GPU offers * Clarify Nebius helper contracts * Keep Nebius reserved capacity eligible for provisioning * Allow sharing Nebius capacity advice across backends * Make Nebius capacity advisor opt-in --------- Co-authored-by: Andrey Cheptsov <andrey.cheptsov@github.com>
1 parent 6d70e58 commit a4bdeb7

7 files changed

Lines changed: 787 additions & 20 deletions

File tree

‎mkdocs/docs/concepts/backends.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -869,6 +869,15 @@ projects:
869869
</div>
870870

871871

872+
??? info "Capacity advisor"
873+
By default, dstack does not check GPU availability or quota. To show these statuses in
874+
offers and skip unavailable offers or those with no quota, enable
875+
[Nebius Capacity Advisor](https://docs.nebius.com/compute/virtual-machines/capacity-advisor)
876+
with `capacity_advisor: true`.
877+
878+
Fetching availability and quota adds roughly 1–3 seconds to getting offers.
879+
Both are cached for 10 minutes.
880+
872881
### Crusoe
873882

874883
Log into your [Crusoe](https://console.crusoecloud.com/) console and create an API key

‎src/dstack/_internal/core/backends/nebius/backend.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
from typing import Optional
2+
13
from dstack._internal.core.backends.base.backend import Backend
24
from dstack._internal.core.backends.nebius.compute import NebiusCompute
35
from dstack._internal.core.backends.nebius.models import NebiusConfig
@@ -8,9 +10,9 @@ class NebiusBackend(Backend):
810
TYPE = BackendType.NEBIUS
911
COMPUTE_CLASS = NebiusCompute
1012

11-
def __init__(self, config: NebiusConfig):
13+
def __init__(self, config: NebiusConfig, compute: Optional[NebiusCompute] = None):
1214
self.config = config
13-
self._compute = NebiusCompute(self.config)
15+
self._compute = compute if compute is not None else NebiusCompute(self.config)
1416

1517
def compute(self) -> NebiusCompute:
1618
return self._compute

‎src/dstack/_internal/core/backends/nebius/compute.py‎

Lines changed: 164 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,27 @@
22
import random
33
import shlex
44
import time
5+
from collections import defaultdict
6+
from collections.abc import Iterable
57
from dataclasses import dataclass
68
from decimal import ROUND_CEILING, Decimal
79
from functools import cached_property
810
from typing import List, Optional
911

12+
from cachetools import TTLCache, cachedmethod
13+
from cachetools.keys import hashkey
1014
from grpc import StatusCode
1115
from nebius.aio.service_error import RequestError
1216
from nebius.api.nebius.billing.v1 import PricingPolicy, PricingPolicyStatus
17+
from nebius.api.nebius.capacity.v1 import ResourceAdvice, ResourceAdviceStatus
1318
from nebius.api.nebius.common.v1 import ResourceMetadata
1419
from nebius.api.nebius.compute.v1 import Disk, DiskStatus, Instance, InstanceStatus
1520
from nebius.sdk import SDK
1621

1722
from dstack._internal.core.backends.base.backend import Compute
1823
from dstack._internal.core.backends.base.compute import (
1924
PROVIDER_REF_TAG_KEY,
25+
ComputeTTLCache,
2026
ComputeWithAllOffersCached,
2127
ComputeWithCreateInstanceSupport,
2228
ComputeWithInstanceVolumesSupport,
@@ -40,6 +46,7 @@
4046
from dstack._internal.core.errors import (
4147
BackendError,
4248
ComputeError,
49+
NoCapacityError,
4350
NotYetTerminated,
4451
ProvisioningError,
4552
)
@@ -76,6 +83,8 @@
7683
min=Memory.parse("40GB"), # min for the ubuntu24.04-cuda13.0 image
7784
max=Memory.parse("8192GB"), # max for the NETWORK_SSD disk type
7885
)
86+
RESOURCE_ADVICE_CACHE_TTL = 600
87+
"""Seconds to cache tenant advice across offer refreshes; matches AWS's quota cache."""
7988
WAIT_FOR_DISK_TIMEOUT = 20
8089
WAIT_FOR_PRICING_POLICY_TIMEOUT = 10
8190
WAIT_FOR_INSTANCE_TIMEOUT = 30
@@ -122,10 +131,17 @@ class NebiusCompute(
122131
ComputeWithResourceListingSupport,
123132
Compute,
124133
):
125-
def __init__(self, config: NebiusConfig):
134+
def __init__(
135+
self, config: NebiusConfig, resource_advice_cache: Optional[ComputeTTLCache] = None
136+
):
126137
super().__init__()
127138
self.config = config
128139
self._subnet_id_cache: dict[str, str] = {}
140+
if resource_advice_cache is None:
141+
resource_advice_cache = ComputeTTLCache(
142+
cache=TTLCache(maxsize=1, ttl=RESOURCE_ADVICE_CACHE_TTL)
143+
)
144+
self._resource_advice_cache = resource_advice_cache
129145

130146
@cached_property
131147
def _sdk(self) -> SDK:
@@ -139,12 +155,17 @@ def close(self) -> None:
139155
del self._sdk
140156
resources.close_sdk(sdk)
141157

158+
@cached_property
159+
def _tenant_id(self) -> str:
160+
return resources.get_tenant_id(self._sdk)
161+
142162
@cached_property
143163
def _region_to_project_id(self) -> dict[str, str]:
144164
return resources.get_region_to_project_id_map(
145165
self._sdk,
146166
configured_regions=self.config.regions,
147167
configured_project_ids=self.config.projects,
168+
tenant_id=self._tenant_id,
148169
)
149170

150171
def _get_subnet_id(self, region: str) -> str:
@@ -162,14 +183,63 @@ def get_all_offers_with_availability(
162183
locations=list(self._region_to_project_id),
163184
extra_filter=_supported_instances,
164185
)
186+
# Capacity Advisor batches GPU advice; prices stay in the offline catalog.
187+
advice_by_offer: dict[tuple[str, str, bool], dict[str, InstanceAvailability]] = (
188+
defaultdict(dict)
189+
)
190+
if self.config.capacity_advisor and any(offer.instance.resources.gpus for offer in offers):
191+
try:
192+
advice = self._get_resource_advice()
193+
except Exception:
194+
# Missing permissions or a Capacity Advisor outage must not hide catalog offers.
195+
logger.warning("Failed to get Nebius capacity advice", exc_info=True)
196+
else:
197+
for item in advice:
198+
instance = item.spec.compute_instance
199+
if instance is None:
200+
continue
201+
name = f"{instance.platform} {instance.preset.name}"
202+
# AUTO reservation policy can use reserved capacity before the shared pool.
203+
# Advice is tenant-wide; project reservation limits remain unchecked, so
204+
# reserved capacity keeps offers eligible as UNKNOWN, not AVAILABLE.
205+
reserved = _get_capacity_availability(item.status.reserved)
206+
if reserved == InstanceAvailability.AVAILABLE:
207+
reserved = InstanceAvailability.UNKNOWN
208+
on_demand = _get_best_availability(
209+
(_get_capacity_availability(item.status.on_demand), reserved)
210+
)
211+
for spot, availability in (
212+
(False, on_demand),
213+
(True, _get_capacity_availability(item.status.preemptible)),
214+
):
215+
key = (item.spec.region, name, spot)
216+
advice_by_offer[key][item.spec.fabric] = availability
165217
return [
166-
offer.with_availability(
167-
availability=InstanceAvailability.UNKNOWN,
168-
price=_get_price_adjusted_for_pricing_policy(offer),
218+
_apply_resource_advice(
219+
offer,
220+
advice_by_offer.get(
221+
(offer.region, offer.instance.name, offer.instance.resources.spot), {}
222+
),
169223
)
170224
for offer in offers
171225
]
172226

227+
def _get_resource_advice_key(self) -> tuple:
228+
return hashkey(
229+
self._tenant_id,
230+
self.config.creds.service_account_id,
231+
self.config.creds.public_key_id,
232+
)
233+
234+
@cachedmethod(
235+
cache=lambda self: self._resource_advice_cache.cache,
236+
key=_get_resource_advice_key,
237+
lock=lambda self: self._resource_advice_cache.lock,
238+
)
239+
def _get_resource_advice(self) -> list[ResourceAdvice]:
240+
"""Return advice for offer collection; fetch on cache miss and propagate API errors."""
241+
return resources.list_resource_advice(self._sdk, self._tenant_id)
242+
173243
def get_offer_disk_size_range(
174244
self, offer: InstanceOfferWithAvailability
175245
) -> Optional[Range[Memory]]:
@@ -394,14 +464,14 @@ def create_placement_group(
394464
master_instance_offer_backend_data = validate_extra_ignore(
395465
NebiusOfferBackendData, master_instance_offer.backend_data
396466
)
397-
fabrics = list(master_instance_offer_backend_data.fabrics)
398-
if self.config.fabrics is not None:
399-
fabrics = [f for f in fabrics if f in self.config.fabrics]
467+
fabrics = self._get_eligible_fabrics(master_instance_offer_backend_data)
468+
if master_instance_offer_backend_data.fabrics and not fabrics:
469+
raise NoCapacityError("No eligible InfiniBand fabrics for the instance offer")
400470
placement_group_backend_data = NebiusPlacementGroupBackendData(cluster=None)
401471
# Only create a Nebius cluster if the instance supports it.
402472
# For other instances, return dummy PlacementGroupProvisioningData.
403473
if fabrics:
404-
fabric = random.choice(fabrics)
474+
fabric = random.choice(list(fabrics))
405475
op = resources.create_cluster(
406476
self._sdk,
407477
name=placement_group.name,
@@ -442,9 +512,24 @@ def is_suitable_placement_group(
442512
)
443513
return (
444514
placement_group_backend_data.cluster is None
445-
or placement_group_backend_data.cluster.fabric in instance_offer_backend_data.fabrics
515+
or placement_group_backend_data.cluster.fabric
516+
in self._get_eligible_fabrics(instance_offer_backend_data)
446517
)
447518

519+
def _get_eligible_fabrics(self, data: NebiusOfferBackendData) -> set[str]:
520+
"""Return compatible, allowed cluster fabrics, applying capacity advice when enabled."""
521+
return {
522+
fabric
523+
for fabric in data.fabrics
524+
if (self.config.fabrics is None or fabric in self.config.fabrics)
525+
and (
526+
not self.config.capacity_advisor
527+
or data.fabric_availability.get(
528+
fabric, InstanceAvailability.UNKNOWN
529+
).is_available()
530+
)
531+
}
532+
448533

449534
class NebiusInstanceBackendData(CoreModel):
450535
boot_disk_id: str
@@ -478,6 +563,76 @@ def load(cls, raw: Optional[str]) -> "NebiusPlacementGroupBackendData":
478563
return validate_json_extra_ignore(cls, raw)
479564

480565

566+
def _get_capacity_availability(
567+
status: ResourceAdviceStatus.Availability,
568+
) -> InstanceAvailability:
569+
"""Convert fresh Nebius advice to offer status; stale or unknown data returns UNKNOWN."""
570+
if status.data_state != ResourceAdviceStatus.Availability.DataState.DATA_STATE_FRESH:
571+
return InstanceAvailability.UNKNOWN
572+
level = status.availability_level
573+
levels = ResourceAdviceStatus.Availability.AvailabilityLevel
574+
# Nebius defines LIMIT_REACHED as a customer quota or capacity-block limit.
575+
# `limit` is the maximum quota, not the remaining quota; zero `available` alone
576+
# does not mean quota exhaustion.
577+
if level == levels.AVAILABILITY_LEVEL_LIMIT_REACHED:
578+
return InstanceAvailability.NO_QUOTA
579+
if level not in (
580+
levels.AVAILABILITY_LEVEL_HIGH,
581+
levels.AVAILABILITY_LEVEL_MEDIUM,
582+
levels.AVAILABILITY_LEVEL_LOW,
583+
):
584+
return InstanceAvailability.UNKNOWN
585+
if status.available > 0:
586+
return InstanceAvailability.AVAILABLE
587+
return InstanceAvailability.NOT_AVAILABLE
588+
589+
590+
def _apply_resource_advice(
591+
offer: InstanceOffer,
592+
advice_by_fabric: dict[str, InstanceAvailability],
593+
) -> InstanceOfferWithAvailability:
594+
"""Set offer status across compatible fabrics; keep per-fabric advice for cluster placement."""
595+
availability = InstanceAvailability.UNKNOWN
596+
backend_data = offer.backend_data
597+
if offer.instance.resources.gpus:
598+
data = validate_extra_ignore(NebiusOfferBackendData, offer.backend_data)
599+
if data.fabrics:
600+
fabrics = data.fabrics
601+
fabric_availability = {
602+
f: advice_by_fabric.get(f, InstanceAvailability.UNKNOWN) for f in fabrics
603+
}
604+
availability = _get_best_availability(fabric_availability.values())
605+
backend_data = {
606+
**offer.backend_data,
607+
"fabric_availability": {
608+
fabric: status.value for fabric, status in fabric_availability.items()
609+
},
610+
}
611+
else:
612+
# Presets without GPU clustering can use any fabric in the region.
613+
availability = _get_best_availability(advice_by_fabric.values())
614+
return offer.with_availability(
615+
availability=availability,
616+
price=_get_price_adjusted_for_pricing_policy(offer),
617+
backend_data=backend_data,
618+
)
619+
620+
621+
def _get_best_availability(availabilities: Iterable[InstanceAvailability]) -> InstanceAvailability:
622+
"""Prefer AVAILABLE, UNKNOWN, NOT_AVAILABLE, then NO_QUOTA; no advice means UNKNOWN."""
623+
statuses = set(availabilities)
624+
# Report NO_QUOTA only when every allocation option explicitly reports a limit.
625+
for availability in (
626+
InstanceAvailability.AVAILABLE,
627+
InstanceAvailability.UNKNOWN,
628+
InstanceAvailability.NOT_AVAILABLE,
629+
InstanceAvailability.NO_QUOTA,
630+
):
631+
if availability in statuses:
632+
return availability
633+
return InstanceAvailability.UNKNOWN
634+
635+
481636
def _wait_for_instance(sdk: SDK, op: resources.NebiusOperation) -> None:
482637
start = time.monotonic()
483638
while True:

‎src/dstack/_internal/core/backends/nebius/models.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
from dstack._internal.core.backends.base.models import fill_data
88
from dstack._internal.core.models.common import CoreModel
9+
from dstack._internal.core.models.instances import InstanceAvailability
910

1011
DEFAULT_PROJECT_NAME_PREFIX = "default"
1112

@@ -139,6 +140,12 @@ class NebiusBackendConfig(CoreModel):
139140
)
140141
),
141142
] = None
143+
capacity_advisor: Annotated[
144+
bool,
145+
Field(
146+
description="Use Nebius Capacity Advisor to check quota and capacity when collecting offers"
147+
),
148+
] = False
142149
tags: Annotated[
143150
Optional[Dict[str, str]],
144151
Field(
@@ -181,6 +188,8 @@ class NebiusConfig(NebiusStoredConfig):
181188

182189
class NebiusOfferBackendData(CoreModel):
183190
fabrics: set[str] = set()
191+
fabric_availability: dict[str, InstanceAvailability] = {}
192+
"""Per-fabric offer status for cluster placement; missing entries mean UNKNOWN."""
184193
is_preemptible_flat_rate: bool = False
185194
"""
186195
True if the price of this spot instance does not change and pricing policies are not supported.

0 commit comments

Comments
 (0)