|
60 | 60 | OBJECT_NAME_MAX_LENGTH, |
61 | 61 | AnyKubernetesGPUResource, |
62 | 62 | KubernetesResource, |
| 63 | + Operator, |
63 | 64 | PodPhase, |
64 | 65 | ResourceLimits, |
65 | 66 | ResourceRequests, |
|
80 | 81 | get_registry_auth_secret_name, |
81 | 82 | is_hard_taint, |
82 | 83 | is_taint_tolerated, |
| 84 | + parse_node_selector_requirement, |
83 | 85 | parse_quantity, |
| 86 | + parse_toleration, |
84 | 87 | ) |
85 | 88 | from dstack._internal.core.backends.kubernetes.utils import ( |
86 | 89 | LEGACY_CURRENT_CONTEXT_REGION, |
|
139 | 142 | JOB_POD_SCHEDULING_TIMEOUT = 10 |
140 | 143 |
|
141 | 144 |
|
142 | | -class Operator(str, Enum): |
143 | | - EXISTS = "Exists" |
144 | | - IN = "In" |
145 | | - |
146 | | - |
147 | 145 | class RestartPolicy(str, Enum): |
148 | 146 | """ |
149 | 147 | This policy determines how Kubernetes reacts to containers exiting due to errors |
@@ -267,6 +265,8 @@ def run_job( |
267 | 265 | authorized_keys=project_authorized_keys, |
268 | 266 | cpu_request_limit=cluster.proxy_jump.cpu or JUMP_POD_DEFAULT_CPU, |
269 | 267 | memory_request_limit=cluster.proxy_jump.memory or JUMP_POD_DEFAULT_MEMORY, |
| 268 | + node_selector=cluster.proxy_jump.node_selector, |
| 269 | + tolerations=cluster.proxy_jump.tolerations, |
270 | 270 | ) |
271 | 271 |
|
272 | 272 | pod_name = generate_unique_instance_name_for_job( |
@@ -888,6 +888,8 @@ def _create_jump_pod_service_if_not_exists( |
888 | 888 | authorized_keys: list[str], |
889 | 889 | cpu_request_limit: str, |
890 | 890 | memory_request_limit: str, |
| 891 | + node_selector: Optional[list[str]], |
| 892 | + tolerations: Optional[list[str]], |
891 | 893 | ) -> None: |
892 | 894 | base_labels = build_base_labels( |
893 | 895 | component="ssh-proxy", |
@@ -959,35 +961,19 @@ def _create_jump_pod_service_if_not_exists( |
959 | 961 | # Otherwise, the existing legacy pod will be adopted by a newly created ReplicaSet. |
960 | 962 |
|
961 | 963 | if should_create_replica_set: |
962 | | - # False if we found at least one node without any "hard" taint, that is, if we don't need |
963 | | - # to specify the toleration. |
964 | | - toleration_required = True |
965 | | - # (key, effect) pairs. |
966 | | - tolerated_taints: set[tuple[str, str]] = set() |
967 | | - for node in api.list_node().items: |
968 | | - if (node_spec := node.spec) is None: |
969 | | - continue |
970 | | - # True if the node has at least one NoExecute or NoSchedule taint. |
971 | | - has_hard_taint = False |
972 | | - taints = node_spec.taints or [] |
973 | | - for taint in taints: |
974 | | - # A "soft" taint, ignore. |
975 | | - if not is_hard_taint(taint): |
976 | | - continue |
977 | | - has_hard_taint = True |
978 | | - if is_taint_tolerated(taint): |
979 | | - tolerated_taints.add((taint.key, taint.effect)) |
980 | | - if not has_hard_taint: |
981 | | - toleration_required = False |
982 | | - break |
983 | | - tolerations: list[client.V1Toleration] = [] |
984 | | - if toleration_required: |
985 | | - for key, effect in tolerated_taints: |
986 | | - tolerations.append( |
987 | | - client.V1Toleration(key=key, operator=Operator.EXISTS, effect=effect) |
988 | | - ) |
989 | | - if not tolerations: |
990 | | - logger.warning("No appropriate node found, the jump pod may never be scheduled") |
| 964 | + affinity: Optional[client.V1Affinity] = None |
| 965 | + # Not for an empty `node_selector`, as an empty node selector term matches no nodes |
| 966 | + if node_selector: |
| 967 | + affinity = _get_jump_pod_affinity(node_selector) |
| 968 | + pod_tolerations: list[client.V1Toleration] |
| 969 | + if tolerations is not None: |
| 970 | + pod_tolerations = [parse_toleration(toleration) for toleration in tolerations] |
| 971 | + elif node_selector is not None: |
| 972 | + # Autodetection considers all nodes, not only the selected ones, so the user |
| 973 | + # specifies tolerations for the selected nodes, if any are needed. |
| 974 | + pod_tolerations = [] |
| 975 | + else: |
| 976 | + pod_tolerations = _get_jump_pod_tolerations_from_node_taints(api) |
991 | 977 | commands = _get_jump_pod_commands(authorized_keys) |
992 | 978 | # Equal CPU and memory requests and limits for the Guaranteed QoS class |
993 | 979 | limits = { |
@@ -1030,7 +1016,8 @@ def _create_jump_pod_service_if_not_exists( |
1030 | 1016 | ) |
1031 | 1017 | ], |
1032 | 1018 | restart_policy=RestartPolicy.ALWAYS, |
1033 | | - tolerations=tolerations, |
| 1019 | + affinity=affinity, |
| 1020 | + tolerations=pod_tolerations, |
1034 | 1021 | ), |
1035 | 1022 | ), |
1036 | 1023 | ), |
@@ -1220,6 +1207,60 @@ def _check_and_configure_jump_pod_service( |
1220 | 1207 | ) |
1221 | 1208 |
|
1222 | 1209 |
|
| 1210 | +def _get_jump_pod_affinity(node_selector: list[str]) -> client.V1Affinity: |
| 1211 | + # A nodeSelector cannot express `Exists`. A single node affinity term matches like |
| 1212 | + # a nodeSelector: a node must meet all requirements. |
| 1213 | + return client.V1Affinity( |
| 1214 | + node_affinity=client.V1NodeAffinity( |
| 1215 | + required_during_scheduling_ignored_during_execution=client.V1NodeSelector( |
| 1216 | + node_selector_terms=[ |
| 1217 | + client.V1NodeSelectorTerm( |
| 1218 | + match_expressions=[ |
| 1219 | + parse_node_selector_requirement(requirement) |
| 1220 | + for requirement in node_selector |
| 1221 | + ], |
| 1222 | + ), |
| 1223 | + ], |
| 1224 | + ), |
| 1225 | + ), |
| 1226 | + ) |
| 1227 | + |
| 1228 | + |
| 1229 | +def _get_jump_pod_tolerations_from_node_taints( |
| 1230 | + api: client.CoreV1Api, |
| 1231 | +) -> list[client.V1Toleration]: |
| 1232 | + # False if we found at least one node without any "hard" taint, that is, if we don't need |
| 1233 | + # to specify the toleration. |
| 1234 | + toleration_required = True |
| 1235 | + # (key, effect) pairs. |
| 1236 | + tolerated_taints: set[tuple[str, str]] = set() |
| 1237 | + for node in api.list_node().items: |
| 1238 | + if (node_spec := node.spec) is None: |
| 1239 | + continue |
| 1240 | + # True if the node has at least one NoExecute or NoSchedule taint. |
| 1241 | + has_hard_taint = False |
| 1242 | + taints = node_spec.taints or [] |
| 1243 | + for taint in taints: |
| 1244 | + # A "soft" taint, ignore. |
| 1245 | + if not is_hard_taint(taint): |
| 1246 | + continue |
| 1247 | + has_hard_taint = True |
| 1248 | + if is_taint_tolerated(taint): |
| 1249 | + tolerated_taints.add((taint.key, taint.effect)) |
| 1250 | + if not has_hard_taint: |
| 1251 | + toleration_required = False |
| 1252 | + break |
| 1253 | + tolerations: list[client.V1Toleration] = [] |
| 1254 | + if toleration_required: |
| 1255 | + for key, effect in tolerated_taints: |
| 1256 | + tolerations.append( |
| 1257 | + client.V1Toleration(key=key, operator=Operator.EXISTS, effect=effect) |
| 1258 | + ) |
| 1259 | + if not tolerations: |
| 1260 | + logger.warning("No appropriate node found, the jump pod may never be scheduled") |
| 1261 | + return tolerations |
| 1262 | + |
| 1263 | + |
1223 | 1264 | def _get_jump_pod_commands(authorized_keys: list[str]) -> list[str]: |
1224 | 1265 | authorized_keys_content = "\n".join(authorized_keys) |
1225 | 1266 | commands = [ |
|
0 commit comments