HDDS-17778. Drop httpclient and httpcore from the Ozone filesystem jar

hadoop-ozone/common/pom.xml

@@ -98,14 +98,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs-client</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpcore</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-client</artifactId>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OFSPath.java

@@ -40,7 +40,6 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.http.ParseException;
 
 /**
  * Utility class for Rooted Ozone Filesystem (OFS) path processing.
@@ -101,7 +100,7 @@ private void initOFSPath(URI uri, boolean endsWithSlash) {
     // Scheme is case-insensitive
     String scheme = uri.getScheme();
     if (scheme != null && !scheme.equalsIgnoreCase(OZONE_OFS_URI_SCHEME)) {
-      throw new ParseException("Can't parse schemes other than ofs.");
+      throw new IllegalArgumentException("Can't parse schemes other than ofs.");
     }
     // authority could be empty
     authority = uri.getAuthority() == null ? "" : uri.getAuthority();
@@ -124,7 +123,7 @@ private void initOFSPath(URI uri, boolean endsWithSlash) {
             bucketName = getTempMountBucketNameOfCurrentUser();
           }
         } catch (IOException ex) {
-          throw new ParseException(
+          throw new IllegalArgumentException(
               "Failed to get temp bucket name for current user.");
         }
       } else if (numToken >= 2) {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java

@@ -28,17 +28,18 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Collection;
+import java.util.StringJoiner;
 import org.apache.hadoop.hdds.NodeDetails;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.OmUtils;
 import org.apache.hadoop.ozone.ha.ConfUtils;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.NodeState;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo;
-import org.apache.http.client.utils.URIBuilder;
 
 /**
  * This class stores OM node details.
@@ -201,15 +202,13 @@ public URL getOMDBCheckpointEndpointUrl(boolean useV2, boolean isHttp,
         : OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
     URL url;
     try {
-      URIBuilder urlBuilder = new URIBuilder().
-          setScheme(isHttp ? "http" : "https").
-          setHost(isHttp ? getHttpAddress() : getHttpsAddress()).
-          setPath(path).
-          addParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA, "true").
-          addParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH,
-              flush ? "true" : "false");
-
-      url = urlBuilder.build().toURL();
+      String scheme = isHttp ? "http" : "https";
+      String address = isHttp ? getHttpAddress() : getHttpsAddress();
+      String query = new StringJoiner("&")
+          .add(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA + "=true")
+          .add(OZONE_DB_CHECKPOINT_REQUEST_FLUSH + "=" + flush)
+          .toString();
+      url = new URI(scheme, address, path, query, null).toURL();
     } catch (URISyntaxException | MalformedURLException e) {
       throw new IOException("Could not get OM DB Checkpoint Endpoint Url", e);
     }

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOMNodeDetails.java

@@ -32,6 +32,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.NodeState;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OMNodeInfo;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 
 /**
  * Test OMNodeDetails.
@@ -355,6 +357,35 @@ public void testGetOMDBCheckpointEndpointUrlHttps() throws IOException {
     assertEquals(HOST_ADDRESS + ":9875", url.getAuthority());
   }
 
+  /**
+   * The full checkpoint URL, as the leader's servlet sees it.
+   */
+  @ParameterizedTest
+  @CsvSource({
+      "localhost, true, true, false, http://localhost:9874/v2/dbCheckpoint"
+          + "?includeSnapshotData=true&flushBeforeCheckpoint=false",
+      "localhost, true, false, true, https://localhost:9875/v2/dbCheckpoint"
+          + "?includeSnapshotData=true&flushBeforeCheckpoint=true",
+      "localhost, false, true, true, http://localhost:9874/dbCheckpoint"
+          + "?includeSnapshotData=true&flushBeforeCheckpoint=true",
+      "[::1], true, false, false, https://[::1]:9875/v2/dbCheckpoint"
+          + "?includeSnapshotData=true&flushBeforeCheckpoint=false",
+  })
+  public void testGetOMDBCheckpointEndpointUrlExact(String host, boolean useV2, boolean isHttp, boolean flush,
+      String expected) throws IOException {
+    OMNodeDetails nodeDetails = new OMNodeDetails.Builder()
+        .setOMServiceId(OM_SERVICE_ID)
+        .setOMNodeId(OM_NODE_ID)
+        .setHostAddress(HOST_ADDRESS)
+        .setRpcPort(RPC_PORT)
+        .setRatisPort(RATIS_PORT)
+        .setHttpAddress(host + ":9874")
+        .setHttpsAddress(host + ":9875")
+        .build();
+
+    assertEquals(expected, nodeDetails.getOMDBCheckpointEndpointUrl(useV2, isHttp, flush).toString());
+  }
+
   /**
    * Test getOMDBCheckpointEndpointUrl with useV2 parameter.
    */

hadoop-ozone/ozonefs-common/pom.xml

@@ -59,10 +59,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs-client</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpclient</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-client</artifactId>

hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java

@@ -84,7 +84,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Progressable;
-import org.apache.http.client.utils.URIBuilder;
 import org.apache.ratis.util.function.CheckedFunction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -190,9 +189,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
     }
 
     try {
-      uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
-          .setHost(authority)
-          .build();
+      uri = new URI(OZONE_URI_SCHEME + "://" + authority);
       LOG.trace("Ozone URI for ozfs initialization is {}", uri);
 
       ConfigurationSource source = getConfSource();

hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java

@@ -93,7 +93,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Progressable;
-import org.apache.http.client.utils.URIBuilder;
 import org.apache.ratis.util.function.CheckedFunction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -188,9 +187,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
     String host = hostAndPort.getHost();
 
     try {
-      uri = new URIBuilder().setScheme(OZONE_OFS_URI_SCHEME)
-          .setHost(authority)
-          .build();
+      uri = new URI(OZONE_OFS_URI_SCHEME + "://" + authority);
       LOG.trace("Ozone URI for OFS initialization is " + uri);
 
       ConfigurationSource source = getConfSource();

hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestBasicOzoneFileSystems.java

@@ -185,6 +185,7 @@ public void testRootedAuthorityParsing(String uri, String expectedHost,
 
     // The rebuilt filesystem URI keeps the (bracketed) IPv6 authority intact.
     assertEquals(new URI(uri).getAuthority(), ofs.getUri().getAuthority());
+    assertEquals("ofs://" + new URI(uri).getAuthority(), ofs.getUri().toString());
   }
 
   @Test
@@ -242,6 +243,7 @@ public void testO3fsAuthorityParsing(String uri, String expectedHost, int expect
     assertEquals(expectedHost, hostCaptor.getValue());
     assertEquals(expectedPort, portCaptor.getValue().intValue());
     assertEquals(new URI(uri).getAuthority(), o3fs.getUri().getAuthority());
+    assertEquals("o3fs://" + new URI(uri).getAuthority(), o3fs.getUri().toString());
   }
 
   @Test
@@ -287,6 +289,26 @@ public void testO3fsMalformedAuthoritiesUseGeneralError(String uri) throws Excep
     assertTrue(exception.getMessage().contains("Ozone file system URL should be one of the following formats"));
   }
 
+  @Test
+  public void testRootedAuthorityWithEncodedIllegalCharacter() throws Exception {
+    BasicRootedOzoneFileSystem ofs = spy(new BasicRootedOzoneFileSystem());
+    doReturn(mock(BasicRootedOzoneClientAdapterImpl.class)).when(ofs).createAdapter(any(), anyString(), anyInt());
+    // getAuthority() decodes %20, and the rebuilt URI must reject the space
+    IOException e = assertThrows(IOException.class,
+        () -> ofs.initialize(new URI("ofs://om%20x/"), new OzoneConfiguration()));
+    assertTrue(e.getMessage().startsWith("Invalid Ozone endpoint"));
+  }
+
+  @Test
+  public void testO3fsAuthorityWithEncodedIllegalCharacter() throws Exception {
+    BasicOzoneFileSystem o3fs = spy(new BasicOzoneFileSystem());
+    doReturn(mock(BasicOzoneClientAdapterImpl.class))
+        .when(o3fs).createAdapter(any(), anyString(), anyString(), nullable(String.class), anyInt());
+    IOException e = assertThrows(IOException.class,
+        () -> o3fs.initialize(new URI("o3fs://bucket.volume.om%20x/"), new OzoneConfiguration()));
+    assertTrue(e.getMessage().startsWith("Invalid Ozone endpoint"));
+  }
+
   @Test
   public void testRootedAuthorityRejectsOutOfRangePort() {
     BasicRootedOzoneFileSystem ofs = spy(new BasicRootedOzoneFileSystem());

hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java

@@ -19,6 +19,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -34,6 +35,13 @@ public class TestOFSPath {
 
   private OzoneConfiguration conf = new OzoneConfiguration();
 
+  @Test
+  public void testRejectsSchemesOtherThanOfs() {
+    IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+        () -> new OFSPath("o3fs://bucket.volume.om/key", conf));
+    assertEquals("Can't parse schemes other than ofs.", e.getMessage());
+  }
+
   @Test
   public void testParsingPathWithSpace() {
     // Two most common cases: file key and dir key inside a bucket

hadoop-ozone/ozonefs-shaded/pom.xml

@@ -25,6 +25,7 @@
   <name>Apache Ozone FileSystem Shaded</name>
 
   <properties>
+    <ban-httpcomponents.skip>false</ban-httpcomponents.skip>
     <ban-rocksdb.skip>false</ban-rocksdb.skip>
     <maven.compiler.release>${ozone.java.version.client}</maven.compiler.release>
     <!-- no tests in this module so far -->

pom.xml

@@ -40,6 +40,7 @@
     <aspectj.java8.version>1.9.7</aspectj.java8.version>
     <assertj.version>3.27.7</assertj.version>
     <aws-java-sdk2.version>2.55.6</aws-java-sdk2.version>
+    <ban-httpcomponents.skip>true</ban-httpcomponents.skip>
     <ban-rocksdb.skip>true</ban-rocksdb.skip>
     <bonecp.version>0.8.0.RELEASE</bonecp.version>
     <bouncycastle.version>1.86</bouncycastle.version>
@@ -2144,6 +2145,25 @@
                 </rules>
               </configuration>
             </execution>
+            <execution>
+              <id>ban-httpcomponents</id>
+              <goals>
+                <goal>enforce</goal>
+              </goals>
+              <phase>validate</phase>
+              <configuration>
+                <!-- enabled in the shaded filesystem module; unshaded client modules get httpclient from Hadoop -->
+                <skip>${ban-httpcomponents.skip}</skip>
+                <rules>
+                  <bannedDependencies>
+                    <excludes>
+                      <exclude>org.apache.httpcomponents</exclude>
+                    </excludes>
+                    <message>The Ozone filesystem jar must not bundle Apache HttpComponents, use java.net.URI instead</message>
+                  </bannedDependencies>
+                </rules>
+              </configuration>
+            </execution>
             <execution>
               <id>ban-imports</id>
               <goals>