diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ed0b89c5a016..78984754885e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -5912,6 +5912,10 @@ public OMExecutionFlow getOmExecutionFlow() { return omExecutionFlow; } + public ProtocolMessageMetrics getOmClientProtocolMetrics() { + return omClientProtocolMetrics; + } + /** * OM Startup mode. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 94a328a77c17..9617687beb52 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -109,6 +109,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.ratis.grpc.GrpcTlsConfig; import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.util.UncheckedAutoCloseable; import org.rocksdb.RocksDBException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -526,7 +527,12 @@ public static GrpcTlsConfig createServerTlsConfig(SecurityConfig conf, public static OzoneManagerProtocolProtos.OMResponse submitRequest( OzoneManager om, OMRequest omRequest, ClientId clientId, long callId) throws ServiceException { - return om.getOmRatisServer().submitRequest(omRequest, clientId, callId); + // Internally-submitted requests (e.g. PurgeKeys) bypass the RPC endpoint dispatcher, so measure + // them here to populate the same OmClientProtocol per-type metrics that client requests get. + try (UncheckedAutoCloseable ignored = + om.getOmClientProtocolMetrics().measure(omRequest.getCmdType())) { + return om.getOmRatisServer().submitRequest(omRequest, clientId, callId); + } } public static OzoneManagerProtocolProtos.OMResponse createErrorResponse( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java index 5ee5067bfe55..0f7386e12ae2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashOzoneFileSystem.java @@ -20,6 +20,9 @@ import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE_MAX; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.File; @@ -30,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; @@ -39,7 +43,9 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.security.SecurityUtil; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -94,6 +100,82 @@ void testGetTrashRootsBeyondPageSize(BucketLayout bucketLayout, } } + /** + * The Trash emptier renames and deletes trash directories in FSO buckets through the + * TrashOzoneFileSystem, submitting internal {@code RenameKey} and {@code DeleteKey} requests. + * Both should be counted in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + void testRenameAndDeleteInFsoBucketIncrementMetrics(@TempDir File testDir) throws Exception { + OmTestManagers omTestManagers = newOmTestManagers(testDir); + try { + OzoneManager om = omTestManagers.getOzoneManager(); + OzoneManagerProtocol writeClient = omTestManagers.getWriteClient(); + final String volumeName = "vol-" + objectId.incrementAndGet(); + final String bucketName = "bucket-" + objectId.incrementAndGet(); + createVolumeAndBucket(omTestManagers, volumeName, bucketName, + BucketLayout.FILE_SYSTEM_OPTIMIZED, writeClient); + createDirectory(writeClient, volumeName, bucketName, TRASH_PREFIX + "/user1"); + + Path src = trashPath(volumeName, bucketName, "user1"); + Path dst = trashPath(volumeName, bucketName, "user2"); + try (FileSystem fs = SecurityUtil.doAsLoginUser( + (PrivilegedExceptionAction) () -> new TrashOzoneFileSystem(om))) { + // This OM is freshly created, so no RenameKey call has been recorded yet. + assertEquals(0, getRequestCount(om.getOmClientProtocolMetrics(), Type.RenameKey)); + assertEquals(0, getRequestTime(om.getOmClientProtocolMetrics(), Type.RenameKey)); + fs.rename(src, dst); + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), Type.RenameKey)).isGreaterThan(0L); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), Type.RenameKey)).isGreaterThan(0L); + + // Likewise no DeleteKey call has been recorded yet. + assertEquals(0, getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteKey)); + assertEquals(0, getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteKey)); + fs.delete(dst, true); + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteKey)).isGreaterThan(0L); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteKey)).isGreaterThan(0L); + } + } finally { + omTestManagers.stop(); + } + } + + /** + * The Trash emptier deletes trash contents in non-FSO buckets one key at a time through the + * TrashOzoneFileSystem, submitting an internal {@code DeleteKeys} request per key. These should + * be counted in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + void deleteInObjectStoreBucketIncrementsDeleteKeysMetric(@TempDir File testDir) throws Exception { + OmTestManagers omTestManagers = newOmTestManagers(testDir); + try { + OzoneManager om = omTestManagers.getOzoneManager(); + OzoneManagerProtocol writeClient = omTestManagers.getWriteClient(); + final String volumeName = "vol-" + objectId.incrementAndGet(); + final String bucketName = "bucket-" + objectId.incrementAndGet(); + createVolumeAndBucket(omTestManagers, volumeName, bucketName, + BucketLayout.OBJECT_STORE, writeClient); + createDirectory(writeClient, volumeName, bucketName, TRASH_PREFIX + "/user1"); + + Path trashRoot = new Path("/" + volumeName + "/" + bucketName + "/" + TRASH_PREFIX); + try (FileSystem fs = SecurityUtil.doAsLoginUser( + (PrivilegedExceptionAction) () -> new TrashOzoneFileSystem(om))) { + // This OM is freshly created, so no DeleteKeys call has been recorded yet. + assertEquals(0, getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteKeys)); + assertEquals(0, getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteKeys)); + fs.delete(trashRoot, true); + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteKeys)).isGreaterThan(0L); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteKeys)).isGreaterThan(0L); + } + } finally { + omTestManagers.stop(); + } + } + + private static Path trashPath(String volumeName, String bucketName, String userDir) { + return new Path("/" + volumeName + "/" + bucketName + "/" + TRASH_PREFIX + "/" + userDir); + } + private void createVolumeAndBucket(OmTestManagers omTestManagers, String volumeName, String bucketName, BucketLayout bucketLayout, OzoneManagerProtocol writeClient) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/ProtocolMessageMetricsTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/ProtocolMessageMetricsTestUtils.java new file mode 100644 index 000000000000..0c0e600c3263 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/ProtocolMessageMetricsTestUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.ratis.utils; + +import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl; + +/** + * Test helpers for reading counters back out of a live {@link ProtocolMessageMetrics}. + */ +public final class ProtocolMessageMetricsTestUtils { + + private ProtocolMessageMetricsTestUtils() { + } + + /** + * Reads the {@code counter} value (number of calls) recorded for the given request type from the + * live {@link ProtocolMessageMetrics} source. Returns {@code 0} if the type has no recorded calls. + */ + public static long getRequestCount(ProtocolMessageMetrics metrics, Enum type) { + return readMetric(metrics, type, "counter"); + } + + /** + * Reads the {@code time} value (summed call latency, in milliseconds) recorded for the given + * request type from the live {@link ProtocolMessageMetrics} source. Returns {@code 0} if the type + * has no recorded calls. + */ + public static long getRequestTime(ProtocolMessageMetrics metrics, Enum type) { + return readMetric(metrics, type, "time"); + } + + private static long readMetric(ProtocolMessageMetrics metrics, Enum type, String metricName) { + MetricsCollectorImpl collector = new MetricsCollectorImpl(); + metrics.getMetrics(collector, true); + for (MetricsRecord record : collector.getRecords()) { + boolean matchesType = false; + for (MetricsTag tag : record.tags()) { + if ("type".equals(tag.name()) && type.toString().equals(tag.value())) { + matchesType = true; + break; + } + } + if (!matchesType) { + continue; + } + for (AbstractMetric metric : record.metrics()) { + if (metricName.equals(metric.name())) { + return metric.value().longValue(); + } + } + } + return 0; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/TestOzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/TestOzoneManagerRatisUtils.java new file mode 100644 index 000000000000..e42b7903d352 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/utils/TestOzoneManagerRatisUtils.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.ratis.utils; + +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.ratis.protocol.ClientId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link OzoneManagerRatisUtils#submitRequest}, in particular that + * internally-submitted requests populate the OmClientProtocol per-type metrics. + */ +public class TestOzoneManagerRatisUtils { + + private OzoneManager ozoneManager; + private OzoneManagerRatisServer ratisServer; + private ProtocolMessageMetrics metrics; + + @BeforeEach + public void setup() { + ozoneManager = mock(OzoneManager.class); + ratisServer = mock(OzoneManagerRatisServer.class); + metrics = ProtocolMessageMetrics.create( + "OmClientProtocol", "Ozone Manager RPC endpoint", Type.class); + when(ozoneManager.getOmClientProtocolMetrics()).thenReturn(metrics); + when(ozoneManager.getOmRatisServer()).thenReturn(ratisServer); + } + + @Test + public void testSubmitRequestRecordsMetricForRequestType() throws Exception { + OMRequest request = newRequest(Type.PurgeKeys); + OMResponse expected = newResponse(Type.PurgeKeys); + mockSubmitRequestWithDelay(expected); + + OMResponse actual = OzoneManagerRatisUtils.submitRequest( + ozoneManager, request, ClientId.randomId(), 1L); + + assertSame(expected, actual); + assertEquals(1, getRequestCount(metrics, Type.PurgeKeys)); + assertThat(getRequestTime(metrics, Type.PurgeKeys)).isGreaterThan(0L); + // Only the submitted type should be counted. + assertEquals(0, getRequestCount(metrics, Type.RenameKey)); + assertEquals(0, getRequestTime(metrics, Type.RenameKey)); + } + + @Test + public void testSubmitRequestIncrementsMetricPerCall() throws Exception { + OMRequest request = newRequest(Type.PurgeKeys); + mockSubmitRequestWithDelay(newResponse(Type.PurgeKeys)); + + OzoneManagerRatisUtils.submitRequest(ozoneManager, request, ClientId.randomId(), 1L); + OzoneManagerRatisUtils.submitRequest(ozoneManager, request, ClientId.randomId(), 2L); + + assertEquals(2, getRequestCount(metrics, Type.PurgeKeys)); + assertThat(getRequestTime(metrics, Type.PurgeKeys)).isGreaterThan(0L); + } + + @Test + public void testSubmitRequestRecordsMetricOnFailure() throws Exception { + OMRequest request = newRequest(Type.PurgeKeys); + // Sleep briefly before failing so the measured latency is reliably greater than zero. + doAnswer(invocation -> { + Thread.sleep(2); + throw new ServiceException("submit failed"); + }).when(ratisServer).submitRequest(any(OMRequest.class), any(ClientId.class), anyLong()); + + assertThrows(ServiceException.class, () -> OzoneManagerRatisUtils.submitRequest( + ozoneManager, request, ClientId.randomId(), 1L)); + + // The measurement wraps the submission, so the metric is recorded even when it fails. + assertEquals(1, getRequestCount(metrics, Type.PurgeKeys)); + assertThat(getRequestTime(metrics, Type.PurgeKeys)).isGreaterThan(0L); + } + + private void mockSubmitRequestWithDelay(OMResponse expectedResponse) throws ServiceException { + // Sleep briefly inside the submission so the measured latency is reliably greater than zero. + doAnswer(invocation -> { + Thread.sleep(2); + return expectedResponse; + }).when(ratisServer).submitRequest(any(OMRequest.class), any(ClientId.class), anyLong()); + } + + private static OMRequest newRequest(Type type) { + return OMRequest.newBuilder() + .setCmdType(type) + .setClientId(ClientId.randomId().toString()) + .build(); + } + + private static OMResponse newResponse(Type type) { + return OMResponse.newBuilder() + .setCmdType(type) + .setStatus(Status.OK) + .setSuccess(true) + .build(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java index 167fbc354a3c..4030097a83fd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequestTests.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; +import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -118,6 +119,7 @@ public class OMKeyRequestTests { protected StorageContainerLocationProtocol scmContainerLocationProtocol; protected OMPerformanceMetrics perfMetrics; protected DeletingServiceMetrics delMetrics; + protected ProtocolMessageMetrics omClientProtocolMetrics; protected static final long CONTAINER_ID = 1000L; protected static final long LOCAL_ID = 100L; @@ -140,6 +142,9 @@ public void setup() throws Exception { omMetrics = OMMetrics.create(ozoneConfiguration); perfMetrics = OMPerformanceMetrics.register(); delMetrics = DeletingServiceMetrics.create(); + omClientProtocolMetrics = ProtocolMessageMetrics.create( + "OmClientProtocol", "Ozone Manager RPC endpoint", + OzoneManagerProtocolProtos.Type.class); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, folder.toAbsolutePath().toString()); ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, @@ -151,6 +156,7 @@ public void setup() throws Exception { when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getPerfMetrics()).thenReturn(perfMetrics); when(ozoneManager.getDeletionMetrics()).thenReturn(delMetrics); + when(ozoneManager.getOmClientProtocolMetrics()).thenReturn(omClientProtocolMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); when(ozoneManager.getConfig()).thenReturn(ozoneConfiguration.getObject(OmConfig.class)); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java index fdb723dc3196..e5e2a560791c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java @@ -20,7 +20,10 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -180,6 +183,72 @@ public void testDeleteDirectoryCrossingSizeLimit() throws Exception { assertThat(dirDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1); } + /** + * The DirectoryDeletingService submits an internal {@code PurgeDirectories} request, which should + * be counted in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + public void testDeleteDirectoryIncrementsPurgeDirectoriesMetric() throws Exception { + OzoneConfiguration conf = createConfAndInitValues(10); + OmTestManagers omTestManagers = new OmTestManagers(conf); + OzoneManagerProtocol writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + om.getMetadataManager(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + + String bucketKey = om.getMetadataManager().getBucketKey(volumeName, bucketName); + OmBucketInfo bucketInfo = om.getMetadataManager().getBucketTable().get(bucketKey); + + OmDirectoryInfo dir1 = OmDirectoryInfo.newBuilder() + .setName("dir1") + .setCreationTime(Time.now()) + .setModificationTime(Time.now()) + .setObjectID(1) + .setParentObjectID(bucketInfo.getObjectID()) + .setUpdateID(0) + .build(); + OMRequestTestUtils.addDirKeyToDirTable(true, dir1, volumeName, bucketName, + 1L, om.getMetadataManager()); + + for (int i = 0; i < 5; ++i) { + String keyName = "key" + i; + OmKeyInfo omKeyInfo = + OMRequestTestUtils.createOmKeyInfo(volumeName, bucketName, keyName, RatisReplicationConfig.getInstance(ONE)) + .setObjectID(dir1.getObjectID() + 1 + i) + .setParentObjectID(dir1.getObjectID()) + .setUpdateID(100L) + .build(); + OMRequestTestUtils.addFileToKeyTable(false, true, keyName, + omKeyInfo, 1234L, i + 1, om.getMetadataManager()); + } + + // This OM is freshly created, so no PurgeDirectories call has been recorded yet. + assertEquals(0, getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeDirectories)); + assertEquals(0, getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeDirectories)); + + // delete directory recursively + OmKeyArgs delArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName("dir1") + .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) + .setDataSize(0).setRecursive(true) + .build(); + writeClient.deleteKey(delArgs); + + // The DirectoryDeletingService should pick up the deleted directory and submit PurgeDirectories. + GenericTestUtils.waitFor( + () -> getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeDirectories) > 0, + 500, 60000); + // A real submission was measured, so the summed latency must be greater than zero. + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeDirectories)).isGreaterThan(0L); + } + @Test public void testMultithreadedDirectoryDeletion() throws Exception { int threadCount = 10; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index 411cd65bebc6..5c5e8a762dc1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -260,6 +262,32 @@ void checkIfDeleteServiceIsDeletingKeys() } } + /** + * The KeyDeletingService submits an internal {@code PurgeKeys} request, which should be counted + * in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + void testPurgeKeysMetricIsRecorded() + throws IOException, TimeoutException, InterruptedException { + final long beforeCount = getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeKeys); + final long beforeTime = getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeKeys); + final long initialDeletedCount = getDeletedKeyCount(); + + final int keyCount = 10; + createAndDeleteKeys(keyCount, 1); + + GenericTestUtils.waitFor( + () -> getDeletedKeyCount() >= initialDeletedCount + keyCount, + 100, 10000); + + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeKeys)).isGreaterThan(beforeCount); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.PurgeKeys)).isGreaterThan(beforeTime); + } + /** * Test that verifies zero-sized keys (keys with no blocks) are not sent to SCM. * The KeyDeletingService should filter out empty keys before calling SCM. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index ab725ba48000..1050a0dcfa99 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -44,7 +44,10 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; import static org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.ALL; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -335,6 +338,42 @@ void testAllKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I deleteLifecyclePolicy(volumeName, bucketName); } + /** + * The KeyLifecycleService submits an internal {@code DeleteKeys} request to remove expired keys, + * which should be counted in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + void testExpiredKeyDeletionIncrementsDeleteKeysMetric() throws IOException, + TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(BucketLayout.OBJECT_STORE); + final long beforeCount = getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.DeleteKeys); + final long beforeTime = getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.DeleteKeys); + // create keys + List keyList = + createKeys(volumeName, bucketName, BucketLayout.OBJECT_STORE, KEY_COUNT, 1, keyPrefix, null); + assertEquals(KEY_COUNT, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(BucketLayout.OBJECT_STORE) - initialKeyCount == KEY_COUNT, + WAIT_CHECK_INTERVAL, 1000); + // create Lifecycle configuration so the keys expire and get deleted + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + createLifecyclePolicy(volumeName, bucketName, BucketLayout.OBJECT_STORE, keyPrefix, null, date.toString(), true); + + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); + GenericTestUtils.waitFor(() -> getRequestCount(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.DeleteKeys) > beforeCount, WAIT_CHECK_INTERVAL, 10000); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.DeleteKeys)).isGreaterThan(beforeTime); + deleteLifecyclePolicy(volumeName, bucketName); + } + @ParameterizedTest @MethodSource("parameters1") void testScanStatePiggybackedOnDelete(BucketLayout bucketLayout, boolean createPrefix) throws IOException, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java index 780dc85da826..510f20c77d3d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java @@ -21,6 +21,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_PARTS_CLEANUP_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; import static org.apache.ozone.test.GenericTestUtils.waitFor; import static org.assertj.core.api.Assertions.assertThat; @@ -53,10 +55,12 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -146,6 +150,43 @@ void deletesExpiredUpload(int numDEFKeys, int numFSOKeys) throws Exception { .isGreaterThanOrEqualTo(oldMpuInfoCount + numDEFKeys + numFSOKeys); } + /** + * The MultipartUploadCleanupService submits an internal {@code AbortExpiredMultiPartUploads} + * request, which should be counted in the OmClientProtocol per-type metrics like a client request. + */ + @Test + void testCleanupIncrementsAbortExpiredMultiPartUploadsMetric() throws Exception { + MultipartUploadCleanupService multipartUploadCleanupService = + (MultipartUploadCleanupService) keyManager.getMultipartUploadCleanupService(); + + multipartUploadCleanupService.suspend(); + // wait for submitted tasks to complete + Thread.sleep(SERVICE_INTERVAL.toMillis()); + + final long beforeCount = getRequestCount(om.getOmClientProtocolMetrics(), + Type.AbortExpiredMultiPartUploads); + final long beforeTime = getRequestTime(om.getOmClientProtocolMetrics(), + Type.AbortExpiredMultiPartUploads); + + createIncompleteMPUKeys(5, BucketLayout.DEFAULT); + + // wait for MPU info to expire + Thread.sleep(EXPIRE_THRESHOLD.toMillis()); + assertThat(getExpiredMultipartUploads()).isNotEmpty(); + + multipartUploadCleanupService.resume(); + + // wait for requests to complete + waitFor(() -> getExpiredMultipartUploads().isEmpty(), + (int) SERVICE_INTERVAL.toMillis(), + 15 * (int) SERVICE_INTERVAL.toMillis()); + + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), + Type.AbortExpiredMultiPartUploads)).isGreaterThan(beforeCount); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), + Type.AbortExpiredMultiPartUploads)).isGreaterThan(beforeTime); + } + private List getExpiredMultipartUploads() { try { return keyManager.getExpiredMultipartUploads(EXPIRE_THRESHOLD, 10000); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOMRangerBGSyncService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOMRangerBGSyncService.java new file mode 100644 index 000000000000..836ea277f530 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOMRangerBGSyncService.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.service; + +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.ozone.om.OMMultiTenantManager; +import org.apache.hadoop.ozone.om.multitenant.AuthorizerLock; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.om.request.key.OMKeyRequestTests; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** + * Test that the {@link OMRangerBGSyncService} internal {@code SetRangerServiceVersion} request is + * counted in the OmClientProtocol per-type metrics just like a client request. + */ +@Timeout(120) +public class TestOMRangerBGSyncService extends OMKeyRequestTests { + + @Test + public void testSetOMDBRangerServiceVersionIncrementsMetric() throws Exception { + OzoneManagerProtocolProtos.OMResponse respMock = mock(OzoneManagerProtocolProtos.OMResponse.class); + OzoneManagerRatisServer ratisServerMock = mock(OzoneManagerRatisServer.class); + // Sleep briefly inside the submission so the measured latency is reliably greater than zero. + doAnswer(invocation -> { + Thread.sleep(2); + return respMock; + }).when(ratisServerMock).submitRequest(any(), any(), anyLong()); + when(ozoneManager.getOmRatisServer()).thenReturn(ratisServerMock); + when(ozoneManager.getThreadNamePrefix()).thenReturn(""); + + OMMultiTenantManager multiTenantManager = mock(OMMultiTenantManager.class); + when(multiTenantManager.getAuthorizerLock()).thenReturn(mock(AuthorizerLock.class)); + + // accessController may be null for unit tests; the service falls back to an in-memory controller. + OMRangerBGSyncService syncService = new OMRangerBGSyncService(ozoneManager, multiTenantManager, + null, 10, TimeUnit.SECONDS, 10_000); + + // The metrics source is created fresh for each test, so no SetRangerServiceVersion call is recorded yet. + assertEquals(0, getRequestCount(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.SetRangerServiceVersion)); + assertEquals(0, getRequestTime(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.SetRangerServiceVersion)); + + syncService.setOMDBRangerServiceVersion(1L); + + assertThat(getRequestCount(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.SetRangerServiceVersion)).isGreaterThan(0L); + assertThat(getRequestTime(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.SetRangerServiceVersion)).isGreaterThan(0L); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java index 4adfeb27d003..c24e74625fc6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LEASE_HARD_LIMIT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.anyLong; @@ -65,6 +67,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; @@ -412,6 +415,37 @@ public void testCleanupExpiredOpenMPUPartKeys( assertAtLeast(numOpenKeysCleaned + partCount, metrics.getNumOpenKeysCleaned()); } + /** + * Cleaning up expired open keys submits an internal {@code DeleteOpenKeys} request, which should + * be counted in the OmClientProtocol per-type metrics just like a client request. + */ + @Test + public void testCleanupIncrementsDeleteOpenKeysMetric() throws Exception { + OpenKeyCleanupService openKeyCleanupService = + (OpenKeyCleanupService) keyManager.getOpenKeyCleanupService(); + + openKeyCleanupService.suspend(); + // wait for submitted tasks to complete + Thread.sleep(SERVICE_INTERVAL); + + final long beforeCount = getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteOpenKeys); + final long beforeTime = getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteOpenKeys); + + createOpenKeys(5, false, BucketLayout.DEFAULT, false, false); + + // wait for open keys to expire + Thread.sleep(EXPIRE_THRESHOLD_MS); + assertExpiredOpenKeys(false, false, BucketLayout.DEFAULT); + + openKeyCleanupService.resume(); + waitForOpenKeyCleanup(false, BucketLayout.DEFAULT); + + assertThat(getRequestCount(om.getOmClientProtocolMetrics(), Type.DeleteOpenKeys)) + .isGreaterThan(beforeCount); + assertThat(getRequestTime(om.getOmClientProtocolMetrics(), Type.DeleteOpenKeys)) + .isGreaterThan(beforeTime); + } + private static void assertAtLeast(long expectedMinimum, long actual) { assertThat(actual).isGreaterThanOrEqualTo(expectedMinimum); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java index a956fb3cb216..f3d9d88aba79 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java @@ -19,6 +19,9 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestCount; +import static org.apache.hadoop.ozone.om.ratis.utils.ProtocolMessageMetricsTestUtils.getRequestTime; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -325,6 +328,42 @@ public void testQuotaRepairSnapshotDbDeletedTableQuota() throws Exception { assertEquals(1, repaired.getSnapshotUsedNamespace()); } + /** + * The QuotaRepairTask submits an internal {@code QuotaRepair} request, which should be counted in + * the OmClientProtocol per-type metrics just like a client request. + */ + @Test + public void testQuotaRepairIncrementsMetric() throws Exception { + OzoneManagerProtocolProtos.OMResponse respMock = mock(OzoneManagerProtocolProtos.OMResponse.class); + when(respMock.getSuccess()).thenReturn(true); + OzoneManagerRatisServer ratisServerMock = mock(OzoneManagerRatisServer.class); + // Sleep briefly inside the submission so the measured latency is reliably greater than zero. + doAnswer(invocation -> { + Thread.sleep(2); + return respMock; + }).when(ratisServerMock).submitRequest(any(), any(), anyLong()); + when(ozoneManager.getOmRatisServer()).thenReturn(ratisServerMock); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager, BucketLayout.OBJECT_STORE); + OMRequestTestUtils.addKeyToTableAndCache(volumeName, bucketName, + "/user/key0", -1, RatisReplicationConfig.getInstance(THREE), 1L, omMetadataManager); + + // The metrics source is created fresh for each test, so no QuotaRepair call is recorded yet. + assertEquals(0, getRequestCount(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.QuotaRepair)); + assertEquals(0, getRequestTime(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.QuotaRepair)); + + QuotaRepairTask quotaRepairTask = new QuotaRepairTask(ozoneManager); + assertTrue(awaitRepair(quotaRepairTask.repair())); + + assertThat(getRequestCount(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.QuotaRepair)).isGreaterThan(0L); + assertThat(getRequestTime(ozoneManager.getOmClientProtocolMetrics(), + OzoneManagerProtocolProtos.Type.QuotaRepair)).isGreaterThan(0L); + } + private void zeroOutBucketUsedBytes(String volumeName, String bucketName, long trxnLogIndex) throws IOException {