diff --git a/assets/state-dcgm-exporter/0800_daemonset.yaml b/assets/state-dcgm-exporter/0800_daemonset.yaml index ba16c9a868..22dd7b7afa 100644 --- a/assets/state-dcgm-exporter/0800_daemonset.yaml +++ b/assets/state-dcgm-exporter/0800_daemonset.yaml @@ -3,6 +3,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm-exporter + nvidia.com/gpu-operator.dcgm-exporter: "true" name: nvidia-dcgm-exporter namespace: "FILLED BY THE OPERATOR" annotations: @@ -15,6 +16,7 @@ spec: metadata: labels: app: nvidia-dcgm-exporter + nvidia.com/gpu-operator.dcgm-exporter: "true" spec: nodeSelector: nvidia.com/gpu.deploy.dcgm-exporter: "true" diff --git a/assets/state-dcgm/0400_dcgm.yml b/assets/state-dcgm/0400_dcgm.yml index 6f5b348f6a..ab9597366e 100644 --- a/assets/state-dcgm/0400_dcgm.yml +++ b/assets/state-dcgm/0400_dcgm.yml @@ -3,6 +3,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm + nvidia.com/gpu-operator.dcgm: "true" name: nvidia-dcgm namespace: "FILLED BY THE OPERATOR" annotations: @@ -15,6 +16,7 @@ spec: metadata: labels: app: nvidia-dcgm + nvidia.com/gpu-operator.dcgm: "true" spec: nodeSelector: nvidia.com/gpu.deploy.dcgm: "true" diff --git a/controllers/object_controls.go b/controllers/object_controls.go index ffe8a281d0..2fc0c62fbd 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -4740,6 +4740,7 @@ func DaemonSet(n ClusterPolicyController) (gpuv1.State, error) { } maps.Copy(obj.Labels, n.singleton.Spec.Daemonsets.Labels) + enforceDCGMCommonLabel(obj) // Daemonsets will always have at least one annotation applied, so allocate if necessary if obj.Annotations == nil { @@ -4788,6 +4789,27 @@ func DaemonSet(n ClusterPolicyController) (gpuv1.State, error) { return isDaemonSetReady(obj.Name, n), nil } +func enforceDCGMCommonLabel(obj *appsv1.DaemonSet) { + var key string + switch obj.Name { + case "nvidia-dcgm", "nvidia-dcgm-dra": + key = "nvidia.com/gpu-operator.dcgm" + case "nvidia-dcgm-exporter", "nvidia-dcgm-exporter-dra": + key = "nvidia.com/gpu-operator.dcgm-exporter" + default: + return + } + + if obj.Labels == nil { + obj.Labels = make(map[string]string) + } + obj.Labels[key] = "true" + if obj.Spec.Template.Labels == nil { + obj.Spec.Template.Labels = make(map[string]string) + } + obj.Spec.Template.Labels[key] = "true" +} + // isDaemonsetSpecChanged returns true if the spec has changed between existing one // and new Daemonset spec compared by hash. func isDaemonsetSpecChanged(current *appsv1.DaemonSet, new *appsv1.DaemonSet) bool { diff --git a/controllers/object_controls_test.go b/controllers/object_controls_test.go index a6595134b6..a27cbd4c52 100644 --- a/controllers/object_controls_test.go +++ b/controllers/object_controls_test.go @@ -62,6 +62,7 @@ const ( sandboxDevicePluginAssetsPath = "assets/state-sandbox-device-plugin" kataDevicePluginAssetsPath = "assets/state-kata-device-plugin" devicePluginAssetsPath = "assets/state-device-plugin/" + dcgmAssetsPath = "assets/state-dcgm/" dcgmExporterAssetsPath = "assets/state-dcgm-exporter/" migManagerAssetsPath = "assets/state-mig-manager/" vGPUDeviceManagerAssetsPath = "assets/state-vgpu-device-manager/" @@ -773,6 +774,24 @@ func testDaemonsetCommon(t *testing.T, cp *gpuv1.ClusterPolicy, component string if err != nil { return nil, fmt.Errorf("unable to get mainCtrImage for dcgm-exporter: %v", err) } + case "DCGM": + spec = commonDaemonsetSpec{ + repository: cp.Spec.DCGM.Repository, + image: cp.Spec.DCGM.Image, + version: cp.Spec.DCGM.Version, + imagePullPolicy: cp.Spec.DCGM.ImagePullPolicy, + imagePullSecrets: getImagePullSecrets(cp.Spec.DCGM.ImagePullSecrets), + args: cp.Spec.DCGM.Args, + env: cp.Spec.DCGM.Env, + resources: cp.Spec.DCGM.Resources, + } + dsLabel = "nvidia-dcgm" + mainCtrName = "nvidia-dcgm-ctr" + manifestFile = filepath.Join(cfg.root, dcgmAssetsPath) + mainCtrImage, err = gpuv1.ImagePath(&cp.Spec.DCGM) + if err != nil { + return nil, fmt.Errorf("unable to get mainCtrImage for dcgm: %v", err) + } case "MIGManager": spec = commonDaemonsetSpec{ repository: cp.Spec.MIGManager.Repository, @@ -1681,6 +1700,63 @@ func getDCGMExporterTestOutput(testCase string) map[string]any { return output } +func TestDCGMCommonLabelEnforcement(t *testing.T) { + testCases := []struct { + description string + component string + label string + configure func(*gpuv1.ClusterPolicy) + }{ + { + description: "DCGM", + component: "DCGM", + label: "nvidia.com/gpu-operator.dcgm", + configure: func(cp *gpuv1.ClusterPolicy) { + enabled := true + cp.Spec.DCGM.Enabled = &enabled + cp.Spec.DCGM.Repository = "nvcr.io/nvidia/cloud-native" + cp.Spec.DCGM.Image = "dcgm" + cp.Spec.DCGM.Version = "3.3.0-1-ubuntu22.04" + }, + }, + { + description: "DCGMExporter", + component: "DCGMExporter", + label: "nvidia.com/gpu-operator.dcgm-exporter", + configure: func(cp *gpuv1.ClusterPolicy) { + cp.Spec.DCGMExporter.Repository = "nvcr.io/nvidia/k8s" + cp.Spec.DCGMExporter.Image = "dcgm-exporter" + cp.Spec.DCGMExporter.Version = "3.3.0-3.2.0-ubuntu22.04" + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + cp := clusterPolicy.DeepCopy() + cp.Spec.Validator.Repository = "nvcr.io/nvidia/cloud-native" + cp.Spec.Validator.Image = "gpu-operator-validator" + cp.Spec.Validator.Version = "v23.9.2" + tc.configure(cp) + cp.Spec.Daemonsets.Labels = map[string]string{ + tc.label: "false", + "custom": "label", + } + + ds, err := testDaemonsetCommon(t, cp, tc.component, 1) + require.NoError(t, err) + require.NotNil(t, ds) + require.Equal(t, "true", ds.Labels[tc.label]) + require.Equal(t, "true", ds.Spec.Template.Labels[tc.label]) + require.Equal(t, "label", ds.Labels["custom"]) + require.Equal(t, "label", ds.Spec.Template.Labels["custom"]) + + require.NoError(t, removeState(&clusterPolicyController, clusterPolicyController.idx-1)) + clusterPolicyController.idx-- + }) + } +} + // TestDCGMExporter tests that the GPU Operator correctly deploys the dcgm-exporter daemonset // under various scenarios/config options func TestDCGMExporter(t *testing.T) { diff --git a/internal/state/dcgm_exporter_test.go b/internal/state/dcgm_exporter_test.go index 6b228808d6..44be62625e 100644 --- a/internal/state/dcgm_exporter_test.go +++ b/internal/state/dcgm_exporter_test.go @@ -79,7 +79,7 @@ func TestDCGMExporterEnabledByDefault(t *testing.T) { assert.Equal(t, 1, kinds["RoleBinding"]) assert.Equal(t, 1, kinds["ResourceClaimTemplate"]) assert.Equal(t, 1, kinds["DaemonSet"]) - assert.Equal(t, 1, kinds["Service"]) + assert.Equal(t, 2, kinds["Service"]) // DRA attribution needs the ResourceSlice informer, so the read ClusterRole is // always bound. No ServiceMonitor by default. assert.Equal(t, 1, kinds["ClusterRole"]) @@ -89,6 +89,8 @@ func TestDCGMExporterEnabledByDefault(t *testing.T) { claimHasAdminAccess(t, findByKind(objs, "ResourceClaimTemplate")) ds := findDaemonSet(t, objs) + assert.Equal(t, "true", ds.Labels["nvidia.com/gpu-operator.dcgm-exporter"]) + assert.Equal(t, "true", ds.Spec.Template.Labels["nvidia.com/gpu-operator.dcgm-exporter"]) podSpec := ds.Spec.Template.Spec assert.Equal(t, "true", podSpec.NodeSelector["nvidia.com/gpu.deploy.dcgm-exporter-dra"]) require.NotNil(t, podSpec.AutomountServiceAccountToken) @@ -228,10 +230,31 @@ func TestDCGMExporterServiceType(t *testing.T) { objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) require.NoError(t, err) - svc := findByKind(objs, "Service") - require.NotNil(t, svc) - svcType, _, _ := unstructured.NestedString(svc.Object, "spec", "type") - assert.Equal(t, "NodePort", svcType) - itpValue, _, _ := unstructured.NestedString(svc.Object, "spec", "internalTrafficPolicy") - assert.Equal(t, "Local", itpValue) + for _, name := range []string{"nvidia-dcgm-exporter", "nvidia-dcgm-exporter-dra"} { + svc := findByKindAndName(objs, "Service", name) + require.NotNil(t, svc) + svcType, _, _ := unstructured.NestedString(svc.Object, "spec", "type") + assert.Equal(t, "NodePort", svcType) + itpValue, _, _ := unstructured.NestedString(svc.Object, "spec", "internalTrafficPolicy") + assert.Equal(t, "Local", itpValue) + selector, _, _ := unstructured.NestedStringMap(svc.Object, "spec", "selector") + assert.Equal(t, map[string]string{"app": "nvidia-dcgm-exporter-dra"}, selector) + if name == "nvidia-dcgm-exporter" { + assert.Equal(t, "true", svc.GetAnnotations()["prometheus.io/scrape"]) + } else { + assert.NotContains(t, svc.GetAnnotations(), "prometheus.io/scrape") + } + } +} + +func TestDCGMExporterCommonLabelCannotBeOverridden(t *testing.T) { + s := newTestDCGMExporterState(t, false) + cr := exporterCR(&nvidiav1.DCGMExporterSpec{}) + cr.Spec.Daemonsets.Labels = map[string]string{"nvidia.com/gpu-operator.dcgm-exporter": "false"} + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + require.NoError(t, err) + ds := findDaemonSet(t, objs) + assert.Equal(t, "true", ds.Labels["nvidia.com/gpu-operator.dcgm-exporter"]) + assert.Equal(t, "true", ds.Spec.Template.Labels["nvidia.com/gpu-operator.dcgm-exporter"]) } diff --git a/internal/state/dcgm_test.go b/internal/state/dcgm_test.go index 030c07f9b0..304b70ed7d 100644 --- a/internal/state/dcgm_test.go +++ b/internal/state/dcgm_test.go @@ -57,6 +57,15 @@ func findByKind(objs []*unstructured.Unstructured, kind string) *unstructured.Un return nil } +func findByKindAndName(objs []*unstructured.Unstructured, kind, name string) *unstructured.Unstructured { + for _, o := range objs { + if o.GetKind() == kind && o.GetName() == name { + return o + } + } + return nil +} + // claimHasAdminAccess verifies the ResourceClaimTemplate requests all GPUs with adminAccess. func claimHasAdminAccess(t *testing.T, rct *unstructured.Unstructured) { t.Helper() @@ -117,11 +126,13 @@ func TestDCGMEnabled(t *testing.T) { assert.Equal(t, 1, kinds["RoleBinding"]) assert.Equal(t, 1, kinds["ResourceClaimTemplate"]) assert.Equal(t, 1, kinds["DaemonSet"]) - assert.Equal(t, 1, kinds["Service"]) + assert.Equal(t, 2, kinds["Service"]) claimHasAdminAccess(t, findByKind(objs, "ResourceClaimTemplate")) ds := findDaemonSet(t, objs) + assert.Equal(t, "true", ds.Labels["nvidia.com/gpu-operator.dcgm"]) + assert.Equal(t, "true", ds.Spec.Template.Labels["nvidia.com/gpu-operator.dcgm"]) podSpec := ds.Spec.Template.Spec assert.Equal(t, "true", podSpec.NodeSelector["nvidia.com/gpu.deploy.dcgm-dra"]) require.Len(t, podSpec.Containers, 1) @@ -138,13 +149,32 @@ func TestDCGMEnabled(t *testing.T) { require.Len(t, ctr.Resources.Claims, 1) assert.Equal(t, "admin-gpus", ctr.Resources.Claims[0].Name) - // The exporter targets this Service on port 5555. - svc := findByKind(objs, "Service") - port, found, err := unstructured.NestedSlice(svc.Object, "spec", "ports") + for _, name := range []string{"nvidia-dcgm", "nvidia-dcgm-dra"} { + svc := findByKindAndName(objs, "Service", name) + require.NotNil(t, svc) + selector, found, err := unstructured.NestedStringMap(svc.Object, "spec", "selector") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, map[string]string{"app": "nvidia-dcgm-dra"}, selector) + port, found, err := unstructured.NestedSlice(svc.Object, "spec", "ports") + require.NoError(t, err) + require.True(t, found) + require.Len(t, port, 1) + assert.Equal(t, int64(5555), port[0].(map[string]any)["port"]) + } +} + +func TestDCGMCommonLabelCannotBeOverridden(t *testing.T) { + s := newTestDCGMState(t) + cr := sampleGPUCluster() + cr.Spec.DCGM = &nvidiav1.DCGMSpec{Enabled: new(true)} + cr.Spec.Daemonsets.Labels = map[string]string{"nvidia.com/gpu-operator.dcgm": "false"} + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) require.NoError(t, err) - require.True(t, found) - require.Len(t, port, 1) - assert.Equal(t, int64(5555), port[0].(map[string]any)["port"]) + ds := findDaemonSet(t, objs) + assert.Equal(t, "true", ds.Labels["nvidia.com/gpu-operator.dcgm"]) + assert.Equal(t, "true", ds.Spec.Template.Labels["nvidia.com/gpu-operator.dcgm"]) } func TestDCGMImageFromEnvFallback(t *testing.T) { diff --git a/internal/state/state_skel.go b/internal/state/state_skel.go index 21c37b86c9..9b81161e58 100644 --- a/internal/state/state_skel.go +++ b/internal/state/state_skel.go @@ -23,6 +23,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -442,9 +443,86 @@ func (s *stateSkel) mergeObjects(updated, current *unstructured.Unstructured) er if gvk.Group == "" && gvk.Kind == "ServiceAccount" { return s.mergeServiceAccount(updated, current) } + if gvk.Group == "" && gvk.Kind == "Service" { + return s.mergeService(updated, current) + } + return nil +} + +// mergeService preserves fields allocated by the API server. Clearing these fields in a full +// update is rejected because their values are immutable or must remain allocated. +func (s *stateSkel) mergeService(updated, current *unstructured.Unstructured) error { + updatedService := &corev1.Service{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(updated.Object, updatedService); err != nil { + return fmt.Errorf("failed to convert updated Service: %w", err) + } + currentService := &corev1.Service{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(current.Object, currentService); err != nil { + return fmt.Errorf("failed to convert current Service: %w", err) + } + + if updatedService.Spec.ClusterIP == "" && len(updatedService.Spec.ClusterIPs) == 0 { + updatedService.Spec.ClusterIP = currentService.Spec.ClusterIP + updatedService.Spec.ClusterIPs = currentService.Spec.ClusterIPs + } + if len(updatedService.Spec.IPFamilies) == 0 { + updatedService.Spec.IPFamilies = currentService.Spec.IPFamilies + } + if updatedService.Spec.IPFamilyPolicy == nil { + updatedService.Spec.IPFamilyPolicy = currentService.Spec.IPFamilyPolicy + } + + if serviceAllocatesNodePorts(updatedService) { + for i := range updatedService.Spec.Ports { + if updatedService.Spec.Ports[i].NodePort != 0 { + continue + } + if currentPort := findMatchingServicePort(updatedService.Spec.Ports[i], currentService.Spec.Ports); currentPort != nil { + updatedService.Spec.Ports[i].NodePort = currentPort.NodePort + } + } + } + if updatedService.Spec.Type == corev1.ServiceTypeLoadBalancer && + updatedService.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyLocal && + updatedService.Spec.HealthCheckNodePort == 0 { + updatedService.Spec.HealthCheckNodePort = currentService.Spec.HealthCheckNodePort + } + + merged, err := runtime.DefaultUnstructuredConverter.ToUnstructured(updatedService) + if err != nil { + return fmt.Errorf("failed to convert merged Service: %w", err) + } + updated.Object = merged + return nil +} + +func serviceAllocatesNodePorts(service *corev1.Service) bool { + if service.Spec.Type == corev1.ServiceTypeNodePort { + return true + } + return service.Spec.Type == corev1.ServiceTypeLoadBalancer && + (service.Spec.AllocateLoadBalancerNodePorts == nil || *service.Spec.AllocateLoadBalancerNodePorts) +} + +func findMatchingServicePort(updated corev1.ServicePort, current []corev1.ServicePort) *corev1.ServicePort { + for i := range current { + if updated.Name != "" && updated.Name == current[i].Name { + return ¤t[i] + } + if updated.Name == "" && current[i].Name == "" && servicePortProtocol(updated) == servicePortProtocol(current[i]) { + return ¤t[i] + } + } return nil } +func servicePortProtocol(port corev1.ServicePort) corev1.Protocol { + if port.Protocol == "" { + return corev1.ProtocolTCP + } + return port.Protocol +} + // For Service Account, keep secrets if exists func (s *stateSkel) mergeServiceAccount(updated, current *unstructured.Unstructured) error { curSecrets, ok, err := unstructured.NestedSlice(current.Object, "secrets") diff --git a/internal/state/state_skel_test.go b/internal/state/state_skel_test.go index 9b8702b03f..b12166d3c4 100644 --- a/internal/state/state_skel_test.go +++ b/internal/state/state_skel_test.go @@ -248,6 +248,101 @@ func TestMergeServiceAccount(t *testing.T) { assert.Equal(t, []any{map[string]any{"name": "pull-secret"}}, pullSecrets) } +func TestMergeServicePreservesAPIAllocatedFields(t *testing.T) { + skel := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + updatedService := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "service", + Namespace: "test-ns", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyLocal, + Ports: []corev1.ServicePort{{ + Name: "metrics", + Protocol: corev1.ProtocolTCP, + Port: 9400, + }}, + }, + } + currentService := updatedService.DeepCopy() + currentService.ResourceVersion = "42" + currentService.Spec.ClusterIP = "10.96.0.10" + currentService.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} + currentService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + currentService.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyRequireDualStack) + currentService.Spec.Ports[0].NodePort = 30400 + currentService.Spec.HealthCheckNodePort = 30900 + + updatedMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(updatedService) + require.NoError(t, err) + currentMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(currentService) + require.NoError(t, err) + updated := &unstructured.Unstructured{Object: updatedMap} + current := &unstructured.Unstructured{Object: currentMap} + + require.NoError(t, skel.mergeObjects(updated, current)) + + merged := &corev1.Service{} + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(updated.Object, merged)) + assert.Equal(t, currentService.ResourceVersion, merged.ResourceVersion) + assert.Equal(t, currentService.Spec.ClusterIP, merged.Spec.ClusterIP) + assert.Equal(t, currentService.Spec.ClusterIPs, merged.Spec.ClusterIPs) + assert.Equal(t, currentService.Spec.IPFamilies, merged.Spec.IPFamilies) + assert.Equal(t, currentService.Spec.IPFamilyPolicy, merged.Spec.IPFamilyPolicy) + assert.Equal(t, currentService.Spec.Ports[0].NodePort, merged.Spec.Ports[0].NodePort) + assert.Equal(t, currentService.Spec.HealthCheckNodePort, merged.Spec.HealthCheckNodePort) +} + +func TestMergeServiceKeepsManifestAllocatedFields(t *testing.T) { + skel := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + updatedService := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "service", + Namespace: "test-ns", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ClusterIP: "10.96.0.20", + ClusterIPs: []string{"10.96.0.20"}, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicySingleStack), + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyLocal, + HealthCheckNodePort: 30901, + Ports: []corev1.ServicePort{{ + Name: "metrics", + Protocol: corev1.ProtocolTCP, + Port: 9400, + NodePort: 30401, + }}, + }, + } + currentService := updatedService.DeepCopy() + currentService.Spec.ClusterIP = "10.96.0.10" + currentService.Spec.ClusterIPs = []string{"10.96.0.10", "fd00::10"} + currentService.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol} + currentService.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyRequireDualStack) + currentService.Spec.Ports[0].NodePort = 30400 + currentService.Spec.HealthCheckNodePort = 30900 + + updatedMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(updatedService) + require.NoError(t, err) + currentMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(currentService) + require.NoError(t, err) + updated := &unstructured.Unstructured{Object: updatedMap} + current := &unstructured.Unstructured{Object: currentMap} + + require.NoError(t, skel.mergeObjects(updated, current)) + + merged := &corev1.Service{} + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(updated.Object, merged)) + assert.Equal(t, updatedService.Spec, merged.Spec) +} + func TestCreateOrUpdateObjsCreatesNewObject(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() skel := newTestSkel(t, fakeClient) diff --git a/internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml b/internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml index 4cbcc6690e..ea98a53b7e 100644 --- a/internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml +++ b/internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml @@ -99,6 +99,23 @@ kind: Service metadata: annotations: prometheus.io/scrape: "true" + labels: + app: nvidia-dcgm-exporter + name: nvidia-dcgm-exporter + namespace: test-operator +spec: + ports: + - name: gpu-metrics + port: 9400 + protocol: TCP + targetPort: 9400 + selector: + app: nvidia-dcgm-exporter-dra + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: labels: app: nvidia-dcgm-exporter-dra name: nvidia-dcgm-exporter-dra @@ -118,6 +135,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" name: nvidia-dcgm-exporter-dra namespace: test-operator spec: @@ -128,6 +146,7 @@ spec: metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" spec: automountServiceAccountToken: true containers: diff --git a/internal/state/testdata/golden/gpucluster-dcgm-exporter-pod-metadata.yaml b/internal/state/testdata/golden/gpucluster-dcgm-exporter-pod-metadata.yaml index d3334775c1..7cd1d35b8c 100644 --- a/internal/state/testdata/golden/gpucluster-dcgm-exporter-pod-metadata.yaml +++ b/internal/state/testdata/golden/gpucluster-dcgm-exporter-pod-metadata.yaml @@ -99,6 +99,23 @@ kind: Service metadata: annotations: prometheus.io/scrape: "true" + labels: + app: nvidia-dcgm-exporter + name: nvidia-dcgm-exporter + namespace: test-operator +spec: + ports: + - name: gpu-metrics + port: 9400 + protocol: TCP + targetPort: 9400 + selector: + app: nvidia-dcgm-exporter-dra + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: labels: app: nvidia-dcgm-exporter-dra name: nvidia-dcgm-exporter-dra @@ -118,6 +135,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" name: nvidia-dcgm-exporter-dra namespace: test-operator spec: @@ -128,6 +146,7 @@ spec: metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" spec: automountServiceAccountToken: true containers: diff --git a/internal/state/testdata/golden/gpucluster-dcgm-exporter-remote-engine.yaml b/internal/state/testdata/golden/gpucluster-dcgm-exporter-remote-engine.yaml index cd8d28bfd4..c5830782fa 100644 --- a/internal/state/testdata/golden/gpucluster-dcgm-exporter-remote-engine.yaml +++ b/internal/state/testdata/golden/gpucluster-dcgm-exporter-remote-engine.yaml @@ -99,6 +99,23 @@ kind: Service metadata: annotations: prometheus.io/scrape: "true" + labels: + app: nvidia-dcgm-exporter + name: nvidia-dcgm-exporter + namespace: test-operator +spec: + ports: + - name: gpu-metrics + port: 9400 + protocol: TCP + targetPort: 9400 + selector: + app: nvidia-dcgm-exporter-dra + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: labels: app: nvidia-dcgm-exporter-dra name: nvidia-dcgm-exporter-dra @@ -118,6 +135,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" name: nvidia-dcgm-exporter-dra namespace: test-operator spec: @@ -128,6 +146,7 @@ spec: metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" spec: automountServiceAccountToken: true containers: diff --git a/internal/state/testdata/golden/gpucluster-dcgm.yaml b/internal/state/testdata/golden/gpucluster-dcgm.yaml index f60d64b712..443e3942cd 100644 --- a/internal/state/testdata/golden/gpucluster-dcgm.yaml +++ b/internal/state/testdata/golden/gpucluster-dcgm.yaml @@ -53,6 +53,7 @@ kind: DaemonSet metadata: labels: app: nvidia-dcgm-dra + nvidia.com/gpu-operator.dcgm: "true" name: nvidia-dcgm-dra namespace: test-operator spec: @@ -63,6 +64,7 @@ spec: metadata: labels: app: nvidia-dcgm-dra + nvidia.com/gpu-operator.dcgm: "true" spec: containers: - image: nvcr.io/nvidia/cloud-native/dcgm:test @@ -99,6 +101,24 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: nvidia-dcgm + name: nvidia-dcgm + namespace: test-operator +spec: + internalTrafficPolicy: Local + ports: + - name: dcgm + port: 5555 + protocol: TCP + targetPort: 5555 + selector: + app: nvidia-dcgm-dra + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: labels: app: nvidia-dcgm-dra diff --git a/manifests/state-dcgm-exporter/0490_service.yaml b/manifests/state-dcgm-exporter/0490_service.yaml new file mode 100644 index 0000000000..c1407ef227 --- /dev/null +++ b/manifests/state-dcgm-exporter/0490_service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: nvidia-dcgm-exporter + namespace: {{ .Namespace }} + labels: + app: nvidia-dcgm-exporter + annotations: + prometheus.io/scrape: "true" +spec: + type: {{ .ServiceType }} + {{- if .ServiceInternalTrafficPolicy }} + internalTrafficPolicy: {{ .ServiceInternalTrafficPolicy }} + {{- end }} + selector: + app: nvidia-dcgm-exporter-dra + ports: + - name: gpu-metrics + port: 9400 + targetPort: 9400 + protocol: TCP diff --git a/manifests/state-dcgm-exporter/0500_service.yaml b/manifests/state-dcgm-exporter/0500_service.yaml index c9efdc4c69..90c10a97cf 100644 --- a/manifests/state-dcgm-exporter/0500_service.yaml +++ b/manifests/state-dcgm-exporter/0500_service.yaml @@ -5,8 +5,6 @@ metadata: namespace: {{ .Namespace }} labels: app: nvidia-dcgm-exporter-dra - annotations: - prometheus.io/scrape: "true" spec: type: {{ .ServiceType }} {{- if .ServiceInternalTrafficPolicy }} diff --git a/manifests/state-dcgm-exporter/0700_daemonset.yaml b/manifests/state-dcgm-exporter/0700_daemonset.yaml index dcf9b4203b..823a120b1f 100644 --- a/manifests/state-dcgm-exporter/0700_daemonset.yaml +++ b/manifests/state-dcgm-exporter/0700_daemonset.yaml @@ -5,9 +5,12 @@ metadata: namespace: {{ .Namespace }} labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" {{- range $k, $v := .Daemonsets.Labels }} + {{- if ne $k "nvidia.com/gpu-operator.dcgm-exporter" }} {{ $k }}: {{ $v | quote }} {{- end }} + {{- end }} {{- if .Daemonsets.Annotations }} annotations: {{- range $k, $v := .Daemonsets.Annotations }} @@ -24,8 +27,9 @@ spec: metadata: labels: app: nvidia-dcgm-exporter-dra + nvidia.com/gpu-operator.dcgm-exporter: "true" {{- range $k, $v := .Daemonsets.Labels }} - {{- if and (ne $k "app") (ne $k "app.kubernetes.io/part-of") }} + {{- if and (ne $k "app") (ne $k "app.kubernetes.io/part-of") (ne $k "nvidia.com/gpu-operator.dcgm-exporter") }} {{ $k }}: {{ $v | quote }} {{- end }} {{- end }} diff --git a/manifests/state-dcgm/0500_daemonset.yaml b/manifests/state-dcgm/0500_daemonset.yaml index 1cc511a813..cdd565a8d1 100644 --- a/manifests/state-dcgm/0500_daemonset.yaml +++ b/manifests/state-dcgm/0500_daemonset.yaml @@ -5,9 +5,12 @@ metadata: namespace: {{ .Namespace }} labels: app: nvidia-dcgm-dra + nvidia.com/gpu-operator.dcgm: "true" {{- range $k, $v := .Daemonsets.Labels }} + {{- if ne $k "nvidia.com/gpu-operator.dcgm" }} {{ $k }}: {{ $v | quote }} {{- end }} + {{- end }} {{- if .Daemonsets.Annotations }} annotations: {{- range $k, $v := .Daemonsets.Annotations }} @@ -24,8 +27,9 @@ spec: metadata: labels: app: nvidia-dcgm-dra + nvidia.com/gpu-operator.dcgm: "true" {{- range $k, $v := .Daemonsets.Labels }} - {{- if and (ne $k "app") (ne $k "app.kubernetes.io/part-of") }} + {{- if and (ne $k "app") (ne $k "app.kubernetes.io/part-of") (ne $k "nvidia.com/gpu-operator.dcgm") }} {{ $k }}: {{ $v | quote }} {{- end }} {{- end }} diff --git a/manifests/state-dcgm/0550_service.yaml b/manifests/state-dcgm/0550_service.yaml new file mode 100644 index 0000000000..e4d14580fc --- /dev/null +++ b/manifests/state-dcgm/0550_service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: nvidia-dcgm + namespace: {{ .Namespace }} + labels: + app: nvidia-dcgm +spec: + type: ClusterIP + internalTrafficPolicy: Local + selector: + app: nvidia-dcgm-dra + ports: + - name: "dcgm" + port: 5555 + targetPort: 5555 + protocol: TCP