22import random
33import shlex
44import time
5+ from collections import defaultdict
6+ from collections .abc import Iterable
57from dataclasses import dataclass
68from decimal import ROUND_CEILING , Decimal
79from functools import cached_property
810from typing import List , Optional
911
12+ from cachetools import TTLCache , cachedmethod
13+ from cachetools .keys import hashkey
1014from grpc import StatusCode
1115from nebius .aio .service_error import RequestError
1216from nebius .api .nebius .billing .v1 import PricingPolicy , PricingPolicyStatus
17+ from nebius .api .nebius .capacity .v1 import ResourceAdvice , ResourceAdviceStatus
1318from nebius .api .nebius .common .v1 import ResourceMetadata
1419from nebius .api .nebius .compute .v1 import Disk , DiskStatus , Instance , InstanceStatus
1520from nebius .sdk import SDK
1621
1722from dstack ._internal .core .backends .base .backend import Compute
1823from dstack ._internal .core .backends .base .compute import (
1924 PROVIDER_REF_TAG_KEY ,
25+ ComputeTTLCache ,
2026 ComputeWithAllOffersCached ,
2127 ComputeWithCreateInstanceSupport ,
2228 ComputeWithInstanceVolumesSupport ,
4046from dstack ._internal .core .errors import (
4147 BackendError ,
4248 ComputeError ,
49+ NoCapacityError ,
4350 NotYetTerminated ,
4451 ProvisioningError ,
4552)
7683 min = Memory .parse ("40GB" ), # min for the ubuntu24.04-cuda13.0 image
7784 max = Memory .parse ("8192GB" ), # max for the NETWORK_SSD disk type
7885)
86+ RESOURCE_ADVICE_CACHE_TTL = 600
87+ """Seconds to cache tenant advice across offer refreshes; matches AWS's quota cache."""
7988WAIT_FOR_DISK_TIMEOUT = 20
8089WAIT_FOR_PRICING_POLICY_TIMEOUT = 10
8190WAIT_FOR_INSTANCE_TIMEOUT = 30
@@ -122,10 +131,17 @@ class NebiusCompute(
122131 ComputeWithResourceListingSupport ,
123132 Compute ,
124133):
125- def __init__ (self , config : NebiusConfig ):
134+ def __init__ (
135+ self , config : NebiusConfig , resource_advice_cache : Optional [ComputeTTLCache ] = None
136+ ):
126137 super ().__init__ ()
127138 self .config = config
128139 self ._subnet_id_cache : dict [str , str ] = {}
140+ if resource_advice_cache is None :
141+ resource_advice_cache = ComputeTTLCache (
142+ cache = TTLCache (maxsize = 1 , ttl = RESOURCE_ADVICE_CACHE_TTL )
143+ )
144+ self ._resource_advice_cache = resource_advice_cache
129145
130146 @cached_property
131147 def _sdk (self ) -> SDK :
@@ -139,12 +155,17 @@ def close(self) -> None:
139155 del self ._sdk
140156 resources .close_sdk (sdk )
141157
158+ @cached_property
159+ def _tenant_id (self ) -> str :
160+ return resources .get_tenant_id (self ._sdk )
161+
142162 @cached_property
143163 def _region_to_project_id (self ) -> dict [str , str ]:
144164 return resources .get_region_to_project_id_map (
145165 self ._sdk ,
146166 configured_regions = self .config .regions ,
147167 configured_project_ids = self .config .projects ,
168+ tenant_id = self ._tenant_id ,
148169 )
149170
150171 def _get_subnet_id (self , region : str ) -> str :
@@ -162,14 +183,63 @@ def get_all_offers_with_availability(
162183 locations = list (self ._region_to_project_id ),
163184 extra_filter = _supported_instances ,
164185 )
186+ # Capacity Advisor batches GPU advice; prices stay in the offline catalog.
187+ advice_by_offer : dict [tuple [str , str , bool ], dict [str , InstanceAvailability ]] = (
188+ defaultdict (dict )
189+ )
190+ if self .config .capacity_advisor and any (offer .instance .resources .gpus for offer in offers ):
191+ try :
192+ advice = self ._get_resource_advice ()
193+ except Exception :
194+ # Missing permissions or a Capacity Advisor outage must not hide catalog offers.
195+ logger .warning ("Failed to get Nebius capacity advice" , exc_info = True )
196+ else :
197+ for item in advice :
198+ instance = item .spec .compute_instance
199+ if instance is None :
200+ continue
201+ name = f"{ instance .platform } { instance .preset .name } "
202+ # AUTO reservation policy can use reserved capacity before the shared pool.
203+ # Advice is tenant-wide; project reservation limits remain unchecked, so
204+ # reserved capacity keeps offers eligible as UNKNOWN, not AVAILABLE.
205+ reserved = _get_capacity_availability (item .status .reserved )
206+ if reserved == InstanceAvailability .AVAILABLE :
207+ reserved = InstanceAvailability .UNKNOWN
208+ on_demand = _get_best_availability (
209+ (_get_capacity_availability (item .status .on_demand ), reserved )
210+ )
211+ for spot , availability in (
212+ (False , on_demand ),
213+ (True , _get_capacity_availability (item .status .preemptible )),
214+ ):
215+ key = (item .spec .region , name , spot )
216+ advice_by_offer [key ][item .spec .fabric ] = availability
165217 return [
166- offer .with_availability (
167- availability = InstanceAvailability .UNKNOWN ,
168- price = _get_price_adjusted_for_pricing_policy (offer ),
218+ _apply_resource_advice (
219+ offer ,
220+ advice_by_offer .get (
221+ (offer .region , offer .instance .name , offer .instance .resources .spot ), {}
222+ ),
169223 )
170224 for offer in offers
171225 ]
172226
227+ def _get_resource_advice_key (self ) -> tuple :
228+ return hashkey (
229+ self ._tenant_id ,
230+ self .config .creds .service_account_id ,
231+ self .config .creds .public_key_id ,
232+ )
233+
234+ @cachedmethod (
235+ cache = lambda self : self ._resource_advice_cache .cache ,
236+ key = _get_resource_advice_key ,
237+ lock = lambda self : self ._resource_advice_cache .lock ,
238+ )
239+ def _get_resource_advice (self ) -> list [ResourceAdvice ]:
240+ """Return advice for offer collection; fetch on cache miss and propagate API errors."""
241+ return resources .list_resource_advice (self ._sdk , self ._tenant_id )
242+
173243 def get_offer_disk_size_range (
174244 self , offer : InstanceOfferWithAvailability
175245 ) -> Optional [Range [Memory ]]:
@@ -394,14 +464,14 @@ def create_placement_group(
394464 master_instance_offer_backend_data = validate_extra_ignore (
395465 NebiusOfferBackendData , master_instance_offer .backend_data
396466 )
397- fabrics = list (master_instance_offer_backend_data . fabrics )
398- if self . config . fabrics is not None :
399- fabrics = [ f for f in fabrics if f in self . config . fabrics ]
467+ fabrics = self . _get_eligible_fabrics (master_instance_offer_backend_data )
468+ if master_instance_offer_backend_data . fabrics and not fabrics :
469+ raise NoCapacityError ( "No eligible InfiniBand fabrics for the instance offer" )
400470 placement_group_backend_data = NebiusPlacementGroupBackendData (cluster = None )
401471 # Only create a Nebius cluster if the instance supports it.
402472 # For other instances, return dummy PlacementGroupProvisioningData.
403473 if fabrics :
404- fabric = random .choice (fabrics )
474+ fabric = random .choice (list ( fabrics ) )
405475 op = resources .create_cluster (
406476 self ._sdk ,
407477 name = placement_group .name ,
@@ -442,9 +512,24 @@ def is_suitable_placement_group(
442512 )
443513 return (
444514 placement_group_backend_data .cluster is None
445- or placement_group_backend_data .cluster .fabric in instance_offer_backend_data .fabrics
515+ or placement_group_backend_data .cluster .fabric
516+ in self ._get_eligible_fabrics (instance_offer_backend_data )
446517 )
447518
519+ def _get_eligible_fabrics (self , data : NebiusOfferBackendData ) -> set [str ]:
520+ """Return compatible, allowed cluster fabrics, applying capacity advice when enabled."""
521+ return {
522+ fabric
523+ for fabric in data .fabrics
524+ if (self .config .fabrics is None or fabric in self .config .fabrics )
525+ and (
526+ not self .config .capacity_advisor
527+ or data .fabric_availability .get (
528+ fabric , InstanceAvailability .UNKNOWN
529+ ).is_available ()
530+ )
531+ }
532+
448533
449534class NebiusInstanceBackendData (CoreModel ):
450535 boot_disk_id : str
@@ -478,6 +563,76 @@ def load(cls, raw: Optional[str]) -> "NebiusPlacementGroupBackendData":
478563 return validate_json_extra_ignore (cls , raw )
479564
480565
566+ def _get_capacity_availability (
567+ status : ResourceAdviceStatus .Availability ,
568+ ) -> InstanceAvailability :
569+ """Convert fresh Nebius advice to offer status; stale or unknown data returns UNKNOWN."""
570+ if status .data_state != ResourceAdviceStatus .Availability .DataState .DATA_STATE_FRESH :
571+ return InstanceAvailability .UNKNOWN
572+ level = status .availability_level
573+ levels = ResourceAdviceStatus .Availability .AvailabilityLevel
574+ # Nebius defines LIMIT_REACHED as a customer quota or capacity-block limit.
575+ # `limit` is the maximum quota, not the remaining quota; zero `available` alone
576+ # does not mean quota exhaustion.
577+ if level == levels .AVAILABILITY_LEVEL_LIMIT_REACHED :
578+ return InstanceAvailability .NO_QUOTA
579+ if level not in (
580+ levels .AVAILABILITY_LEVEL_HIGH ,
581+ levels .AVAILABILITY_LEVEL_MEDIUM ,
582+ levels .AVAILABILITY_LEVEL_LOW ,
583+ ):
584+ return InstanceAvailability .UNKNOWN
585+ if status .available > 0 :
586+ return InstanceAvailability .AVAILABLE
587+ return InstanceAvailability .NOT_AVAILABLE
588+
589+
590+ def _apply_resource_advice (
591+ offer : InstanceOffer ,
592+ advice_by_fabric : dict [str , InstanceAvailability ],
593+ ) -> InstanceOfferWithAvailability :
594+ """Set offer status across compatible fabrics; keep per-fabric advice for cluster placement."""
595+ availability = InstanceAvailability .UNKNOWN
596+ backend_data = offer .backend_data
597+ if offer .instance .resources .gpus :
598+ data = validate_extra_ignore (NebiusOfferBackendData , offer .backend_data )
599+ if data .fabrics :
600+ fabrics = data .fabrics
601+ fabric_availability = {
602+ f : advice_by_fabric .get (f , InstanceAvailability .UNKNOWN ) for f in fabrics
603+ }
604+ availability = _get_best_availability (fabric_availability .values ())
605+ backend_data = {
606+ ** offer .backend_data ,
607+ "fabric_availability" : {
608+ fabric : status .value for fabric , status in fabric_availability .items ()
609+ },
610+ }
611+ else :
612+ # Presets without GPU clustering can use any fabric in the region.
613+ availability = _get_best_availability (advice_by_fabric .values ())
614+ return offer .with_availability (
615+ availability = availability ,
616+ price = _get_price_adjusted_for_pricing_policy (offer ),
617+ backend_data = backend_data ,
618+ )
619+
620+
621+ def _get_best_availability (availabilities : Iterable [InstanceAvailability ]) -> InstanceAvailability :
622+ """Prefer AVAILABLE, UNKNOWN, NOT_AVAILABLE, then NO_QUOTA; no advice means UNKNOWN."""
623+ statuses = set (availabilities )
624+ # Report NO_QUOTA only when every allocation option explicitly reports a limit.
625+ for availability in (
626+ InstanceAvailability .AVAILABLE ,
627+ InstanceAvailability .UNKNOWN ,
628+ InstanceAvailability .NOT_AVAILABLE ,
629+ InstanceAvailability .NO_QUOTA ,
630+ ):
631+ if availability in statuses :
632+ return availability
633+ return InstanceAvailability .UNKNOWN
634+
635+
481636def _wait_for_instance (sdk : SDK , op : resources .NebiusOperation ) -> None :
482637 start = time .monotonic ()
483638 while True :
0 commit comments