From 41ecc50487d9b8193ad34c9f21eb032c31fe4749 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:31:07 +0800 Subject: [PATCH 01/14] [kafka] Serve ApiVersions from registered capabilities Route requests through the dispatcher and advertise only implemented APIs. Return version-aware errors for unsupported APIs and invalid requests. Validated with mvn -o -pl fluss-kafka verify (23 unit tests and 1 IT). Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 313/313 AI-Contributed/UT: 137/137 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 232 ++---------------- .../api/versions/ApiVersionsHandler.java | 78 ++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 137 ++++++++--- 4 files changed, 207 insertions(+), 243 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2bd2f5d81bc..9a9e2e9ff6e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,7 +66,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return new KafkaRequestHandler(); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..2df16a8bd7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,24 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; - -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the implemented server capabilities. */ + public KafkaRequestHandler() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +44,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); - } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..c38d7bc6cb2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index bf8e23b4356..b0f25149e43 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,24 +17,32 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -53,7 +61,7 @@ public void testKafkaApiVersionsNotSupported() { new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); @@ -61,44 +69,111 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + newRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ctx); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequest newRequest( @@ -134,6 +209,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + return new KafkaRequestHandler(); } } From 4317d87922b826751d2a23283e8f3eb5e3044a0f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 17:28:33 +0800 Subject: [PATCH 02/14] [kafka] Cover ApiVersions wire negotiation and capability ranges Exercise future-version decoding, v0 error serialization, correlation IDs, unknown tagged fields, buffer release, and renegotiation on one connection. Verify registry-defined ranges, hidden APIs, ordering, and invalid client software fields across the supported ApiVersions schemas. Validated with Java 11 reactor clean verify: 37 unit tests and 2 integration tests passed, with Checkstyle, Spotless, RAT, and git diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 238/238 --- .../fluss/kafka/KafkaRequestHandlerTest.java | 149 ++++++++++++++---- .../api/versions/ApiVersionsHandlerTest.java | 89 +++++++++++ 2 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index b0f25149e43..40fdab04cc4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,15 +17,21 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; @@ -33,12 +39,15 @@ import org.apache.kafka.common.requests.CreateTopicsRequest; import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.ByteBuffer; import java.util.Collections; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; @@ -48,25 +57,49 @@ public class KafkaRequestHandlerTest { @Test - public void testKafkaApiVersionsNotSupported() { + public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); - KafkaRequest request = - newRequest( - ApiKeys.API_VERSIONS, - (short) (latestVersion + 1), // unsupported version - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.processRequest(request); + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + short unsupportedVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + try { + // An unknown request version must be rejected without parsing its body schema. + processWireRequest( + handler, + requests, + channel, + unsupportedVersion, + new ApiVersionsRequest.Builder().build((short) 4), + 41); + ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); + assertThat(unsupportedResponse.data().errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertBrokerCapabilities(unsupportedResponse); - ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); + ApiVersionsRequestData data = + new ApiVersionsRequestData() + .setClientSoftwareName("test-client") + .setClientSoftwareVersion("1.0"); + data.unknownTaggedFields().add(new RawTaggedField(42, new byte[] {1, 2, 3})); + processWireRequest( + handler, + requests, + channel, + (short) 4, + new ApiVersionsRequest(data, (short) 4), + 42); + ApiVersionsResponse response = parseWireResponse(channel, (short) 4, 42); + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + assertThat(channel.isActive()).isTrue(); + assertThat(requests.requestsCount()).isZero(); + Object extraResponse = channel.readOutbound(); + assertThat(extraResponse).isNull(); + } finally { + channel.finishAndReleaseAll(); + } } @ParameterizedTest @@ -112,30 +145,32 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly( - tuple( - ApiKeys.API_VERSIONS.id, - ApiKeys.API_VERSIONS.oldestVersion(), - ApiKeys.API_VERSIONS.latestVersion())); + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } - @Test - public void testInvalidApiVersionsRequest() { + @ParameterizedTest + @CsvSource({ + "3, invalid client name, 1.0", + "3, client, ''", + "4, invalid client name, 1.0", + "4, client, ''" + }) + public void testInvalidApiVersionsRequest( + short version, String softwareName, String softwareVersion) { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder( new ApiVersionsRequestData() - .setClientSoftwareName("invalid client name") - .setClientSoftwareVersion("1.0"), - latestVersion, - latestVersion) - .build(latestVersion); + .setClientSoftwareName(softwareName) + .setClientSoftwareVersion(softwareVersion), + version, + version) + .build(version); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, - latestVersion, - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), requestBody, new TestingChannelHandlerContext()); @@ -143,6 +178,7 @@ public void testInvalidApiVersionsRequest() { ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + assertThat(response.data().apiKeys()).isEmpty(); } @Test @@ -176,6 +212,53 @@ public void testUnregisteredApiIsNotRouted() { assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } + private static void processWireRequest( + KafkaRequestHandler handler, + RequestChannel requests, + EmbeddedChannel channel, + short version, + ApiVersionsRequest body, + int correlationId) { + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, version, "test-client", correlationId); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + channel.writeInbound(buffer); + KafkaRequest request = (KafkaRequest) requests.pollRequest(1000); + assertThat(request).isNotNull(); + try { + assertThat(request.apiVersion()).isEqualTo(version); + handler.processRequest(request); + assertThat(request.future()).isDone(); + channel.runPendingTasks(); + assertThat(buffer.refCnt()).isZero(); + } finally { + request.releaseBuffer(); + } + } + + private static ApiVersionsResponse parseWireResponse( + EmbeddedChannel channel, short version, int correlationId) { + ByteBuf buffer = channel.readOutbound(); + assertThat(buffer).isNotNull(); + try { + ByteBuffer bytes = buffer.nioBuffer(); + assertThat(ResponseHeader.parse(bytes, (short) 0).correlationId()) + .isEqualTo(correlationId); + ApiVersionsResponseData data = + new ApiVersionsResponseData(new ByteBufferAccessor(bytes), version); + assertThat(bytes.hasRemaining()).isFalse(); + return new ApiVersionsResponse(data); + } finally { + buffer.release(); + } + } + private static KafkaRequest newRequest( ApiKeys apiKey, short apiVersion, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java new file mode 100644 index 00000000000..31269af56c5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Tests registry-backed capability advertisement by {@link ApiVersionsHandler}. */ +class ApiVersionsHandlerTest { + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + ApiVersionsHandler handler = new ApiVersionsHandler(registry); + registry.register(handler); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); + registry.freeze(); + + ApiVersionsResponse response = + (ApiVersionsResponse) + handler.handle(null, new ApiVersionsRequest.Builder().build(version)) + .join(); + + assertThat(response.data().errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + } + + private static final class TestingApiHandler implements KafkaApiHandler { + private final KafkaApiSpec spec; + + private TestingApiHandler(KafkaApiSpec spec) { + this.spec = spec; + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, AbstractRequest request) { + throw new AssertionError( + "Capability advertisement must not invoke other API handlers."); + } + } +} From 229fc9434f705b4df9a894a5743c72166679cc45 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 09:43:06 +0800 Subject: [PATCH 03/14] [kafka] Adapt ApiVersions to immutable API registry Construct the registry once with all handlers and defer ApiVersions registry access until request handling. Advertised capabilities and dispatch continue to share the same immutable registry without a separate capability list. Update coverage to verify lazy access and handler identity across versions. Validated with Java 11 reactor clean verify: 44 unit and 2 integration cases passed in both Maven test executions, including inherited error-mapping and idle regressions. Kafka main and test sources also compile with --release 8. Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 28/28 AI-Contributed/UT: 33/33 --- .../fluss/kafka/KafkaRequestHandler.java | 13 ++++++-- .../api/versions/ApiVersionsHandler.java | 15 ++++++--- .../api/versions/ApiVersionsHandlerTest.java | 33 ++++++++++++++----- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2df16a8bd7f..b3b555a8f9a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -24,16 +24,19 @@ import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import java.util.Collections; + /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { + private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the implemented server capabilities. */ public KafkaRequestHandler() { - KafkaApiRegistry registry = new KafkaApiRegistry(); - registry.register(new ApiVersionsHandler(registry)); - registry.freeze(); + this.registry = + new KafkaApiRegistry( + Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -55,4 +58,8 @@ public void processRequest(KafkaRequest request) { } }); } + + private KafkaApiRegistry apiRegistry() { + return registry; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index c38d7bc6cb2..82e3559c5f4 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.requests.ApiVersionsResponse; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -45,11 +46,15 @@ public final class ApiVersionsHandler implements KafkaApiHandler registrySupplier; - /** Creates an ApiVersions handler backed by the server capability registry. */ - public ApiVersionsHandler(KafkaApiRegistry registry) { - this.registry = checkNotNull(registry); + /** + * Creates an ApiVersions handler backed by the same immutable registry used for dispatch. The + * supplier is read only when handling requests, so this handler can be included in the + * registry's construction snapshot. + */ + public ApiVersionsHandler(Supplier registrySupplier) { + this.registrySupplier = checkNotNull(registrySupplier); } @Override @@ -65,7 +70,7 @@ public CompletableFuture handle( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 31269af56c5..0492ee3572f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -32,7 +32,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.util.Arrays; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -43,16 +45,29 @@ class ApiVersionsHandlerTest { @ParameterizedTest @ValueSource(shorts = {0, 1, 2, 3, 4}) void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { - KafkaApiRegistry registry = new KafkaApiRegistry(); - ApiVersionsHandler handler = new ApiVersionsHandler(registry); - registry.register(handler); - registry.register( + AtomicReference registryReference = new AtomicReference<>(); + ApiVersionsHandler handler = + new ApiVersionsHandler( + () -> { + assertThat(registryReference.get()).isNotNull(); + return registryReference.get(); + }); + TestingApiHandler metadataHandler = new TestingApiHandler( - new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); - registry.register( - new TestingApiHandler( - new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); - registry.freeze(); + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true)); + KafkaApiRegistry registry = + new KafkaApiRegistry( + Arrays.asList( + handler, + metadataHandler, + new TestingApiHandler( + new KafkaApiSpec( + ApiKeys.FETCH, (short) 0, (short) 3, false)))); + registryReference.set(registry); + + assertThat(registry.lookup(ApiKeys.API_VERSIONS)).isSameAs(handler); + assertThat(registry.lookup(ApiKeys.METADATA)).isSameAs(metadataHandler); + assertThat(registry.lookup(ApiKeys.FETCH)).isNull(); ApiVersionsResponse response = (ApiVersionsResponse) From 10bf516b6a04c579079b6e69b3788ac633041525 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 18:06:21 +0800 Subject: [PATCH 04/14] [kafka] Adapt version fallback regression to PR02 handler Use the dispatcher-backed handler constructor in the fallback test added by PR01. Preserve its unsupported-version, listener, response, and buffer checks. Validated with Java 11 reactor clean verify: 39 unit and 2 integration cases passed in both Maven test executions. Kafka main and test sources compile with --release 8; Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 3/3 --- .../java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index f4a255b013d..dda2e52bde5 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -71,7 +70,7 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler(new TestingTabletGatewayService()).processRequest(request); + new KafkaRequestHandler().processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); From 062047def41ff5fe2d89b6c43b0b355e3772de1f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 21:00:25 +0800 Subject: [PATCH 05/14] [kafka] Clarify uninitialized API registry failures Check the deferred registry before reading advertised capabilities so an initialization-order failure reports an explicit diagnostic message. Validated with Java 11 reactor tests: 16 ApiVersions and request-handler cases passed, along with Spotless, Checkstyle, RAT, and diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 4/4 AI-Contributed/UT: 0/0 --- .../apache/fluss/kafka/api/versions/ApiVersionsHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index 82e3559c5f4..5cac0c3cca5 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -69,8 +69,10 @@ public CompletableFuture handle( return CompletableFuture.completedFuture( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } + KafkaApiRegistry registry = + checkNotNull(registrySupplier.get(), "Kafka API registry is not initialized."); ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() From 34964400fef3a407a9d05a4d35bfd9e71c5d6394 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:28:16 +0800 Subject: [PATCH 06/14] [kafka] Define DDL table mapping for Kafka compatibility Extract topic identity and the raw/string table mapping contract before Metadata. Validate table kinds, field projections and metadata columns independently of request handling and record decoding. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (47 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 769/769 AI-Contributed/UT: 480/480 --- fluss-kafka/pom.xml | 7 + .../fluss/kafka/format/KafkaDataFormat.java | 73 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 73 ++++ .../kafka/schema/KafkaFieldProjection.java | 123 +++++++ .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ++++++++ .../schema/KafkaTopicSchemaException.java | 30 ++ .../schema/KafkaTopicSchemaResolver.java | 313 +++++++++++++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 62 ++++ .../kafka/schema/KafkaTableMappingITCase.java | 88 +++++ .../schema/KafkaTopicSchemaResolverTest.java | 330 ++++++++++++++++++ 10 files changed, 1249 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index f48a94ab9e9..b77695848ca 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,13 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..ad0039b7e78 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Fluss table custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; + + /** Fluss table custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; + + /** Fluss fields populated from the Kafka record key. */ + public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; + + /** Strategy for deriving fields populated from the Kafka record value. */ + public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; + + /** Fluss column populated from the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; + + /** Fluss column populated from the Kafka record headers. */ + public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a table custom property value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted table property value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..9f8ed63f543 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + Topic.validate(topicName); + return TablePath.of(databaseName, topicName); + } + + /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return databaseName.equals(tablePath.getDatabaseName()) + && Topic.isValid(tablePath.getTableName()); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java new file mode 100644 index 00000000000..b1715758a9d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFieldNames().get(position)); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java new file mode 100644 index 00000000000..64263e69286 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ +@Internal +public final class KafkaTopicSchema { + + private final RowType rowType; + private final @Nullable KafkaDataFormat keyFormat; + private final KafkaFieldProjection keyProjection; + private final KafkaDataFormat valueFormat; + private final KafkaFieldProjection valueProjection; + private final int timestampPosition; + private final int headersPosition; + + /** Creates a resolved Kafka topic schema. */ + public KafkaTopicSchema( + RowType rowType, + @Nullable KafkaDataFormat keyFormat, + KafkaFieldProjection keyProjection, + KafkaDataFormat valueFormat, + KafkaFieldProjection valueProjection, + int timestampPosition, + int headersPosition) { + this.rowType = checkNotNull(rowType); + this.keyFormat = keyFormat; + this.keyProjection = checkNotNull(keyProjection); + this.valueFormat = checkNotNull(valueFormat); + this.valueProjection = checkNotNull(valueProjection); + this.timestampPosition = timestampPosition; + this.headersPosition = headersPosition; + } + + /** Returns the physical Fluss row type. */ + public RowType rowType() { + return rowType; + } + + /** Returns the key format, or null when the Kafka key is not mapped. */ + public @Nullable KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the key field projection. */ + public KafkaFieldProjection keyProjection() { + return keyProjection; + } + + /** Returns the value format. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + + /** Returns the value field projection. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the timestamp physical position, or -1 when it is not mapped. */ + public int timestampPosition() { + return timestampPosition; + } + + /** Returns the headers physical position, or -1 when they are not mapped. */ + public int headersPosition() { + return headersPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaTopicSchema)) { + return false; + } + KafkaTopicSchema that = (KafkaTopicSchema) obj; + return Objects.equals(rowType, that.rowType) + && Objects.equals(keyFormat, that.keyFormat) + && Objects.equals(keyProjection, that.keyProjection) + && Objects.equals(valueFormat, that.valueFormat) + && Objects.equals(valueProjection, that.valueProjection) + && Objects.equals(timestampPosition, that.timestampPosition) + && Objects.equals(headersPosition, that.headersPosition); + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + @Override + public String toString() { + return "KafkaTopicSchema{" + + "rowType=" + + rowType + + ", " + + "keyFormat=" + + keyFormat + + ", " + + "keyProjection=" + + keyProjection + + ", " + + "valueFormat=" + + valueFormat + + ", " + + "valueProjection=" + + valueProjection + + ", " + + "timestampPosition=" + + timestampPosition + + ", " + + "headersPosition=" + + headersPosition + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..840ce8a4f58 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +@Internal +public final class KafkaTopicSchemaResolver { + + private static final String INCLUDE_ALL = "ALL"; + private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; + + private static final Set SUPPORTED_PROPERTIES = + new HashSet<>( + Arrays.asList( + KafkaDataFormat.KEY_FORMAT_CONFIG, + KafkaDataFormat.KEY_FIELDS_CONFIG, + KafkaDataFormat.VALUE_FORMAT_CONFIG, + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, + KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + + /** Resolves one table's Kafka record mapping contract. */ + public KafkaTopicSchema resolve(TableDescriptor table) { + validateTableKind(table); + RowType rowType = table.getSchema().getRowType(); + Map properties = table.getCustomProperties(); + + for (String property : properties.keySet()) { + if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { + throw invalid("Unsupported Kafka table property '" + property + "'."); + } + } + + int timestampPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); + int headersPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + if (timestampPosition >= 0 && timestampPosition == headersPosition) { + throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); + } + validateMetadataColumns(rowType, timestampPosition, headersPosition); + + String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); + List keyPositions = + resolveKeyPositions( + rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); + for (Integer keyPosition : keyPositions) { + if (keyPosition == timestampPosition || keyPosition == headersPosition) { + throw invalid( + "Kafka key field '" + + rowType.getFieldNames().get(keyPosition) + + "' cannot be a Kafka metadata column."); + } + } + KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); + + String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); + if (valueFormatValue == null) { + throw invalid( + "Missing required table property '" + + KafkaDataFormat.VALUE_FORMAT_CONFIG + + "'."); + } + KafkaDataFormat valueFormat = parseFormat(valueFormatValue); + String fieldsInclude = + normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); + if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { + throw invalid( + "Mapping Kafka key fields requires " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + "=EXCEPT_KEY."); + } + + Set metadataPositions = new HashSet<>(); + if (timestampPosition >= 0) { + metadataPositions.add(timestampPosition); + } + if (headersPosition >= 0) { + metadataPositions.add(headersPosition); + } + List valuePositions = + resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); + KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); + if (valueProjection.isEmpty()) { + throw invalid("Kafka value projection must contain at least one Fluss column."); + } + + validateSingleFieldFormat(keyFormat, keyProjection, "key"); + validateSingleFieldFormat(valueFormat, valueProjection, "value"); + return new KafkaTopicSchema( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + private static void validateTableKind(TableDescriptor table) { + if (table.hasPrimaryKey()) { + throw invalid("Kafka topic table must be a log table."); + } + if (table.isPartitioned()) { + throw invalid("Partitioned Fluss tables are not supported."); + } + if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid("Kafka topic table must use the Arrow log format."); + } + } + + private static List resolveKeyPositions( + RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { + if (keyFormat == null) { + if (keyFieldsValue != null) { + throw invalid( + KafkaDataFormat.KEY_FIELDS_CONFIG + + " requires " + + KafkaDataFormat.KEY_FORMAT_CONFIG + + "."); + } + return Collections.emptyList(); + } + if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { + throw invalid( + "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); + } + String[] fieldNames = keyFieldsValue.split(",", -1); + List positions = new ArrayList<>(fieldNames.length); + Set uniquePositions = new HashSet<>(); + for (String fieldNameValue : fieldNames) { + String fieldName = fieldNameValue.trim(); + if (fieldName.isEmpty()) { + throw invalid("Kafka key field names must not be empty."); + } + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka key field '" + fieldName + "' does not exist."); + } + if (!uniquePositions.add(position)) { + throw invalid("Duplicate Kafka key field '" + fieldName + "'."); + } + positions.add(position); + } + return positions; + } + + private static String normalizeFieldsInclude(String value) { + String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); + if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { + throw invalid( + "Invalid " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + " '" + + value + + "'. Expected ALL or EXCEPT_KEY."); + } + return normalized; + } + + private static List resolveValuePositions( + RowType rowType, + String fieldsInclude, + List keyPositions, + Set metadataPositions) { + Set excludedKeyPositions = + INCLUDE_EXCEPT_KEY.equals(fieldsInclude) + ? new HashSet<>(keyPositions) + : Collections.emptySet(); + List positions = new ArrayList<>(); + for (int position = 0; position < rowType.getFieldCount(); position++) { + if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { + positions.add(position); + } + } + return positions; + } + + private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { + if (fieldNameValue == null) { + return -1; + } + if (fieldNameValue.trim().isEmpty()) { + throw invalid("Kafka metadata column name must not be empty."); + } + String fieldName = fieldNameValue.trim(); + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); + } + return position; + } + + private static void validateMetadataColumns( + RowType rowType, int timestampPosition, int headersPosition) { + if (timestampPosition >= 0) { + if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) + || rowType.getTypeAt(timestampPosition).isNullable() + || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) + .getPrecision() + != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + } + } + if (headersPosition >= 0) { + validateHeadersType(rowType.getTypeAt(headersPosition)); + } + } + + private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { + if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { + throw invalid( + "Kafka headers column must be nullable " + + "ARRAY>."); + } + ArrayType arrayType = (ArrayType) dataType; + if (!(arrayType.getElementType() instanceof RowType)) { + throw invalid("Kafka headers elements must be rows."); + } + RowType headerType = (RowType) arrayType.getElementType(); + if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) + || !(headerType.getTypeAt(0) instanceof StringType) + || !(headerType.getTypeAt(1) instanceof BytesType) + || !headerType.getTypeAt(1).isNullable()) { + throw invalid("Kafka headers elements must be ROW."); + } + } + + private static void validateSingleFieldFormat( + KafkaDataFormat format, KafkaFieldProjection projection, String component) { + if (format == null) { + return; + } + if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { + if (projection.size() != 1) { + throw invalid( + "Kafka " + + component + + " format " + + format.value() + + " requires exactly one Fluss field."); + } + boolean validType = + format == KafkaDataFormat.RAW + ? projection.dataTypeAt(0) instanceof BytesType + : projection.dataTypeAt(0) instanceof StringType; + if (!validType) { + throw invalid( + "Kafka " + + component + + " field '" + + projection.nameAt(0) + + "' must be " + + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") + + " for format " + + format.value() + + "."); + } + } + } + + private static KafkaDataFormat parseFormat(String value) { + try { + return KafkaDataFormat.parse(value); + } catch (IllegalArgumentException e) { + throw invalid(e.getMessage()); + } + } + + private static KafkaTopicSchemaException invalid(String message) { + return new KafkaTopicSchemaException(message); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..20c32ded941 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); + assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c92b8a7300 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ +public class KafkaTableMappingITCase { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + @Test + public void testResolveMappingFromCreatedTableMetadata() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("event_key", DataTypes.STRING()) + .column("event_body", DataTypes.BYTES()) + .build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build(); + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); + TablePath tablePath = mapper.toTablePath("events"); + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + MetadataRequest request = new MetadataRequest(); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + PbTableMetadata metadata = + FLUSS_CLUSTER_EXTENSION + .newCoordinatorClient() + .metadata(request) + .get() + .getTableMetadatasList() + .get(0); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + + assertThat(metadata.getTableId()).isEqualTo(tableId); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..4c698b5ad6b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the DDL contract independently of Kafka request handling and record decoding. */ +public class KafkaTopicSchemaResolverTest { + + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + public void testRawMappingSurvivesTableMetadataSerialization() { + Schema schema = + Schema.newBuilder() + .column("message", DataTypes.BYTES()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("attributes", headersType()) + .column("message_key", DataTypes.BYTES()) + .build(); + TableDescriptor table = + table(schema, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + KafkaTopicSchema mapping = + resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); + + assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(3); + assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping).isEqualTo(resolver.resolve(table)); + assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValueOnlyStringAndNullableColumns() { + for (boolean nullable : new boolean[] {true, false}) { + KafkaTopicSchema mapping = + resolver.resolve( + table( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING().copy(nullable)) + .build(), + " STRING ") + .build()); + assertThat(mapping.keyFormat()).isNull(); + assertThat(mapping.keyProjection().isEmpty()).isTrue(); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); + assertThat(mapping.timestampPosition()).isEqualTo(-1); + assertThat(mapping.headersPosition()).isEqualTo(-1); + } + } + + @Test + public void testMixedFormatsAndMetadataAreOptional() { + KafkaTopicSchema mapping = + resolver.resolve( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") + .build()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + } + + @Test + public void testRejectsUnsupportedTableKinds() { + Schema primaryKeySchema = + Schema.newBuilder() + .column("id", DataTypes.STRING().copy(false)) + .primaryKey("id") + .build(); + assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); + assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); + assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); + } + + @Test + public void testRequiresExplicitValueFormat() { + assertInvalid( + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(1) + .customProperty("fluss.value.format", "raw"), + KafkaDataFormat.VALUE_FORMAT_CONFIG); + } + + @ParameterizedTest + @ValueSource(strings = {"json", "avro", ""}) + public void testRejectsUnavailableFormats(String format) { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + } + + @Test + public void testRejectsUnsupportedKafkaOptions() { + assertInvalid( + valueTable().customProperty("kafka.value.rescue-column", "body"), + "Unsupported Kafka table property"); + assertInvalid( + valueTable().customProperty("kafka.key.field", "body"), + "Unsupported Kafka table property"); + assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) + .isNotNull(); + } + + @Test + public void testKeyFormatAndFieldMustBeSpecifiedTogether() { + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires kafka.key.format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), + "kafka.key.fields"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) + public void testRejectsInvalidKeyFields(String fields) { + assertThatThrownBy( + () -> + resolver.resolve( + keyValueTable() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty( + KafkaDataFormat.KEY_FIELDS_CONFIG, fields) + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + "EXCEPT_KEY") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class); + } + + @Test + public void testRejectsAmbiguousAndEmptyValueProjections() { + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), + "Expected ALL or EXCEPT_KEY"); + assertInvalid( + valueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + assertInvalid(keyValueTable(), "exactly one Fluss field"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + } + + @Test + public void testRejectsWrongPhysicalTypes() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), + "must be STRING"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "must be BYTES"); + } + + @Test + public void testRejectsWrongTimestampTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(6).copy(false) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), + "TIMESTAMP_LTZ(3) NOT NULL"); + } + } + + @Test + public void testRejectsWrongHeaderTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + headersType().copy(false), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("key", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("attributes", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), + "headers"); + } + } + + @Test + public void testRejectsConflictingOrMissingMetadataColumns() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), + "does not exist"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), + "must not be empty"); + TableDescriptor.Builder table = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), + "same Fluss column"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), + "metadata column"); + } + + private void assertInvalid(TableDescriptor.Builder table, String message) { + assertThatThrownBy(() -> resolver.resolve(table.build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(message); + } + + private static TableDescriptor.Builder keyValueTable() { + return table( + Schema.newBuilder() + .column("id", DataTypes.STRING()) + .column("body", DataTypes.BYTES()) + .build(), + "raw"); + } + + private static TableDescriptor.Builder valueTable() { + return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); + } + + private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); + } + + private static DataType headersType() { + return DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))); + } +} From 3252d38e1b7f28fb717234486bc363b5a64b39cc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 11:39:47 +0800 Subject: [PATCH 07/14] [kafka] Route qualified topics and streamline field projections Resolve database.table topic names directly to Fluss table paths and return qualified names for metadata. Validate name boundaries and cover same-named tables in different databases through native metadata round trips. Read projected field names directly from DataField to avoid rebuilding the complete field-name list for every projected column. Validated on Java 11: 67 Kafka unit tests and 2 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 52/52 AI-Contributed/UT: 167/167 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 50 ++++++--- .../kafka/schema/KafkaFieldProjection.java | 2 +- .../kafka/mapping/KafkaTopicMapperTest.java | 101 ++++++++++++++++-- .../kafka/schema/KafkaTableMappingITCase.java | 66 ++++++++---- 4 files changed, 172 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 9f8ed63f543..f8b5744f71d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -21,35 +21,57 @@ import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import static org.apache.fluss.utils.Preconditions.checkArgument; -import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +/** Maps fully qualified Kafka topic identities to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - private final String databaseName; - - /** Creates a topic mapper for one Fluss database. */ - public KafkaTopicMapper(String databaseName) { - this.databaseName = checkNotNull(databaseName); + /** Maps a Kafka topic in database.table form to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + if (!isValidTopic(topicName)) { + throw new InvalidTopicException( + "Kafka topic must be a valid database.table name: " + topicName); + } + int separator = topicName.indexOf('.'); + return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); } - /** Maps a Kafka topic name to its Fluss table path. */ - public TablePath toTablePath(String topicName) { - Topic.validate(topicName); - return TablePath.of(databaseName, topicName); + /** Returns the fully qualified Kafka name of a representable Fluss table. */ + public String toTopicName(TablePath tablePath) { + if (!isMappedTable(tablePath)) { + throw new InvalidTopicException( + "Fluss table cannot be represented as a Kafka database.table name: " + + tablePath); + } + return tablePath.toString(); } - /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return databaseName.equals(tablePath.getDatabaseName()) - && Topic.isValid(tablePath.getTableName()); + return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + } + + /** Returns whether a name uniquely represents a user table in a Fluss database. */ + public static boolean isValidTopic(String topicName) { + if (topicName == null || !Topic.isValid(topicName)) { + return false; + } + int separator = topicName.indexOf('.'); + if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + return false; + } + String database = topicName.substring(0, separator); + String table = topicName.substring(separator + 1); + return TablePath.of(database, table).isValid() + && TablePath.validatePrefix(database) == null + && TablePath.validatePrefix(table) == null; } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java index b1715758a9d..b6833475a20 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -50,7 +50,7 @@ public KafkaFieldProjection(RowType rowType, List positions) { "Invalid Kafka field projection position %s.", position); positionCopy.add(position); - projectedNames.add(rowType.getFieldNames().get(position)); + projectedNames.add(rowType.getFields().get(position).getName()); projectedTypes.add(rowType.getTypeAt(position)); } this.positions = Collections.unmodifiableList(positionCopy); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 20c32ded941..c5d4a582a88 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -22,6 +22,12 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidTopicException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,11 +35,88 @@ /** Tests for {@link KafkaTopicMapper}. */ public class KafkaTopicMapperTest { + private final KafkaTopicMapper mapper = new KafkaTopicMapper(); + + @Test + public void testQualifiedNamesAcrossDatabases() { + for (String database : Arrays.asList("sales", "archive", "sales_1-2026")) { + TablePath tablePath = TablePath.of(database, "orders_1-2026"); + String topicName = database + ".orders_1-2026"; + assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); + assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); + assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); + } + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + "orders", + ".orders", + "sales.", + "sales.orders.extra", + "sales..orders", + "sales.bad name", + " sales.orders", + "sales.orders ", + "sales/orders", + "__system.orders", + "sales.__internal", + "数据库.orders", + "sales.订单" + }) + public void testRejectsInvalidTopicNames(String topicName) { + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(topicName)) + .isInstanceOf(InvalidTopicException.class); + } + @Test - public void testTopicNameAndIdMapping() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + public void testNameLengthLimits() { + for (TablePath valid : + Arrays.asList( + TablePath.of(repeat('a', 200), "orders"), + TablePath.of("sales", repeat('a', 200)), + TablePath.of(repeat('a', 124), repeat('b', 124)))) { + assertThat(mapper.toTablePath(mapper.toTopicName(valid))).isEqualTo(valid); + } + for (TablePath invalid : + Arrays.asList( + TablePath.of(repeat('a', 201), "orders"), + TablePath.of("sales", repeat('a', 201)), + TablePath.of(repeat('a', 124), repeat('b', 125)))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testRejectsUnrepresentableTablePaths() { + for (TablePath invalid : + Arrays.asList( + null, + TablePath.of(null, "orders"), + TablePath.of("sales", null), + TablePath.of("", "orders"), + TablePath.of("sales", ""), + TablePath.of("sales.region", "orders"), + TablePath.of("sales", "orders.v1"), + TablePath.of("__system", "orders"), + TablePath.of("sales", "__internal"), + TablePath.of("sales", "bad name"))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } - assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + @Test + public void testTopicIdMapping() { Uuid topicId = mapper.toTopicId(123L); assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); assertThat(mapper.isMappedTopicId(topicId)).isTrue(); @@ -46,17 +129,15 @@ public void testTopicNameAndIdMapping() { } @Test - public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); - assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); - assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) - .isInstanceOf(InvalidTopicException.class); + public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) .isInstanceOf(IllegalArgumentException.class); assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) .isInstanceOf(IllegalArgumentException.class); } + + private static String repeat(char character, int count) { + return String.join("", Collections.nCopies(count, String.valueOf(character))); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 0c92b8a7300..2ded7bd17f6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -31,6 +31,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +47,7 @@ public class KafkaTableMappingITCase { FlussClusterExtension.builder().setNumOfTabletServers(1).build(); @Test - public void testResolveMappingFromCreatedTableMetadata() throws Exception { + public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { TableDescriptor descriptor = TableDescriptor.builder() .schema( @@ -57,32 +62,49 @@ public void testResolveMappingFromCreatedTableMetadata() throws Exception { .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); - TablePath tablePath = mapper.toTablePath("events"); - long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + KafkaTopicMapper mapper = new KafkaTopicMapper(); + Map tableIds = new LinkedHashMap<>(); MetadataRequest request = new MetadataRequest(); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - PbTableMetadata metadata = + for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { + TablePath tablePath = mapper.toTablePath(database + ".events"); + assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); + tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + } + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + List metadatas = FLUSS_CLUSTER_EXTENSION .newCoordinatorClient() .metadata(request) .get() - .getTableMetadatasList() - .get(0); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + .getTableMetadatasList(); + assertThat(metadatas).hasSize(2); + assertThat(metadatas) + .extracting(PbTableMetadata::getTableId) + .containsExactlyInAnyOrderElementsOf(tableIds.values()) + .doesNotHaveDuplicates(); + for (PbTableMetadata metadata : metadatas) { + TablePath tablePath = + TablePath.of( + metadata.getTablePath().getDatabaseName(), + metadata.getTablePath().getTableName()); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - assertThat(metadata.getTableId()).isEqualTo(tableId); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); + assertThat(mapper.toTopicName(tablePath)) + .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) + .isEqualTo(tableIds.get(tablePath)); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } } } From 1275b338946fb4aabd78af1533b026129502563e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:28:20 +0800 Subject: [PATCH 08/14] [kafka] Define Kafka timestamp metadata as TIMESTAMP(3) Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 16/16 AI-Contributed/UT: 33/33 --- .../fluss/kafka/format/KafkaDataFormat.java | 5 +++- .../schema/KafkaTopicSchemaResolver.java | 11 +++----- .../kafka/schema/KafkaTableMappingITCase.java | 5 ++++ .../schema/KafkaTopicSchemaResolverTest.java | 28 ++++++++++++++++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java index ad0039b7e78..df5d871907c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -39,7 +39,10 @@ public enum KafkaDataFormat { /** Strategy for deriving fields populated from the Kafka record value. */ public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - /** Fluss column populated from the Kafka record timestamp. */ + /** + * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar + * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. + */ public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; /** Fluss column populated from the Kafka record headers. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 840ce8a4f58..8959c420d15 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -25,9 +25,9 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.LocalZonedTimestampType; import org.apache.fluss.types.RowType; import org.apache.fluss.types.StringType; +import org.apache.fluss.types.TimestampType; import java.util.ArrayList; import java.util.Arrays; @@ -234,12 +234,9 @@ private static int resolveOptionalPosition(RowType rowType, String fieldNameValu private static void validateMetadataColumns( RowType rowType, int timestampPosition, int headersPosition) { if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) - || rowType.getTypeAt(timestampPosition).isNullable() - || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) - .getPrecision() - != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) + || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); } } if (headersPosition >= 0) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2ded7bd17f6..2f3e75bdd3d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -54,12 +54,14 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep Schema.newBuilder() .column("event_key", DataTypes.STRING()) .column("event_body", DataTypes.BYTES()) + .column("event_time", DataTypes.TIMESTAMP(3)) .build()) .distributedBy(2) .logFormat(LogFormat.ARROW) .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); KafkaTopicMapper mapper = new KafkaTopicMapper(); @@ -105,6 +107,9 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); assertThat(mapping.valueProjection().positions()).containsExactly(1); assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.timestampPosition()).isEqualTo(2); + assertThat(persisted.getSchema().getRowType().getTypeAt(2)) + .isEqualTo(DataTypes.TIMESTAMP(3)); } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 4c698b5ad6b..66c13a4711a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -41,7 +41,7 @@ public void testRawMappingSurvivesTableMetadataSerialization() { Schema schema = Schema.newBuilder() .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column("attributes", headersType()) .column("message_key", DataTypes.BYTES()) .build(); @@ -219,13 +219,33 @@ public void testRejectsWrongPhysicalTypes() { "must be BYTES"); } + @Test + public void testAcceptsTimestampWithEitherNullability() { + for (boolean nullable : new boolean[] {true, false}) { + TableDescriptor descriptor = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") + .build(); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + } + } + @Test public void testRejectsWrongTimestampTypes() { for (DataType type : new DataType[] { DataTypes.STRING(), DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(6).copy(false) + DataTypes.TIMESTAMP_LTZ(3).copy(false), + DataTypes.TIMESTAMP(2), + DataTypes.TIMESTAMP(6).copy(false) }) { assertInvalid( table( @@ -235,7 +255,7 @@ public void testRejectsWrongTimestampTypes() { .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP_LTZ(3) NOT NULL"); + "TIMESTAMP(3)"); } } @@ -279,7 +299,7 @@ public void testRejectsConflictingOrMissingMetadataColumns() { table( Schema.newBuilder() .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("ts", DataTypes.TIMESTAMP(3).copy(false)) .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); From dea30c267b3b5d5b14ebaf5134a3ee34fbb2de43 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 11 Oct 2026 23:16:34 +0800 Subject: [PATCH 09/14] [test] Wait for table readiness in snapshot lease validation Wait for the newly created log table to become ready before acquiring or releasing snapshot leases. This prevents the coordinator's asynchronous table-ID registration from masking the expected NonPrimaryKeyTableException. Validated on Java 11 with the same-worktree reactor: FlussAdminITCase#testKvSnapshotLeaseForLogTable passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 1/1 --- .../java/org/apache/fluss/client/admin/FlussAdminITCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index e67faf84b24..b39739c6b01 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -2831,6 +2831,7 @@ void testKvSnapshotLeaseForLogTable() throws Exception { TableDescriptor logTableDescriptor = TableDescriptor.builder().schema(logSchema).distributedBy(3).build(); long tableId = createTable(logTablePath, logTableDescriptor, true); + FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId); // Create a KvSnapshotLease KvSnapshotLease lease = admin.createKvSnapshotLease("test-lease-log", 60000); From 6e9dc555cd6e2db25e3def9a800d1966995e3179 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 15:55:33 +0800 Subject: [PATCH 10/14] [kafka] Align DDL table mapping with the FIP contract Register table.kafka options and validate the declared fixed-column mapping through a shared schema contract. Support binary, string and flat/nested JSON values with optional key, headers and rescue columns. Validate CREATE and ALTER before persistence, and fence schema publication with the table version and coordinator epoch so concurrent schema and property changes cannot bypass the contract. Resolve bare topic aliases through kafka.default.database and stop automatically creating that database. Keep real offset persistence deferred and reject Kafka enablement for tables that declare offset metadata until that capability is available. Validation: 146 targeted tests passed on Java 11; Checkstyle, Spotless, RAT and git diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 1131/1131 AI-Contributed/UT: 1451/1451 --- .../apache/fluss/config/ConfigOptions.java | 39 +- .../apache/fluss/config/FlussConfigUtils.java | 4 + .../org/apache/fluss/config/TableConfig.java | 5 + .../fluss/metadata}/KafkaFieldProjection.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 391 ++++++++++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 437 ++++++++++++++++++ .../flink/utils/FlinkConversionsTest.java | 45 ++ .../fluss/kafka/format/KafkaDataFormat.java | 76 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 49 +- .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ------ .../schema/KafkaTopicSchemaResolver.java | 292 +----------- .../apache/fluss/kafka/KafkaConfigsTest.java | 8 +- .../kafka/mapping/KafkaTopicMapperTest.java | 34 +- .../kafka/schema/KafkaTableMappingITCase.java | 410 +++++++++++++--- .../schema/KafkaTopicSchemaResolverTest.java | 335 ++------------ .../server/coordinator/CoordinatorServer.java | 8 - .../coordinator/CoordinatorService.java | 5 +- .../server/coordinator/MetadataManager.java | 36 +- .../utils/TableDescriptorValidation.java | 7 + .../fluss/server/zk/ZooKeeperClient.java | 30 ++ .../KafkaTableSchemaAlterTest.java | 182 ++++++++ website/docs/maintenance/configuration.md | 37 +- 22 files changed, 1672 insertions(+), 910 deletions(-) rename {fluss-kafka/src/main/java/org/apache/fluss/kafka/schema => fluss-common/src/main/java/org/apache/fluss/metadata}/KafkaFieldProjection.java (99%) create mode 100644 fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 3e86eb77579..667e1af5f68 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2269,6 +2269,38 @@ public class ConfigOptions { + "Downstream consumers must be upgraded to Fluss v1.0+ before enabling this option, " + "as older versions cannot parse the extended batch format."); + public static final ConfigOption TABLE_KAFKA_ENABLED = + key("table.kafka.enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether the table is exposed as a Kafka topic. Enabling validates the " + + "existing schema and conversion options without adding columns or defaults."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + key("table.kafka.convert.value.type") + .stringType() + .noDefaultValue() + .withDescription( + "Required when Kafka compatibility is enabled: binary, string, or parse_json. " + + "Binary and string target the declared _kafka_value column."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + key("table.kafka.convert.value.layout") + .stringType() + .noDefaultValue() + .withDescription( + "JSON layout: flat (the read-time default for parse_json) or nested. " + + "Nested targets _kafka_value ROW. Not allowed for binary or string."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = + key("table.kafka.convert.key.type") + .stringType() + .noDefaultValue() + .withDescription( + "Optional Kafka key conversion type. Only binary is supported and requires " + + "_kafka_key BYTES. Declaring that column implies binary without persisting a default."); + // ------------------------------------------------------------------------ // ConfigOptions for Kv // ------------------------------------------------------------------------ @@ -2905,12 +2937,13 @@ public class ConfigOptions { .withDescription( "The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma."); - public static final ConfigOption KAFKA_DATABASE = - key("kafka.database") + public static final ConfigOption KAFKA_DEFAULT_DATABASE = + key("kafka.default.database") .stringType() .defaultValue("kafka") .withDescription( - "The database for fluss kafka. The default database is `kafka`."); + "Default database for unqualified Kafka topic names. Fully qualified names may " + + "access other databases. Databases must already exist; this option does not create them."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 37d5cc9f88e..415103c46ea 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -47,6 +47,10 @@ public class FlussConfigUtils { CLIENT_OPTIONS = extractConfigOptions("client."); ALTERABLE_TABLE_OPTIONS = Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_MODE.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ed2c111c335..b1bc64dfc83 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -54,6 +54,11 @@ public TableConfig(Configuration config) { this.config = config; } + /** Whether the table explicitly enables Kafka compatibility. */ + public boolean isKafkaEnabled() { + return config.get(ConfigOptions.TABLE_KAFKA_ENABLED); + } + /** Gets the replication factor of the table. */ public int getReplicationFactor() { return config.get(ConfigOptions.TABLE_REPLICATION_FACTOR); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java similarity index 99% rename from fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java rename to fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java index b6833475a20..f8946107020 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.fluss.kafka.schema; +package org.apache.fluss.metadata; import org.apache.fluss.annotation.Internal; import org.apache.fluss.types.DataType; diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java new file mode 100644 index 00000000000..7c040f15871 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable Kafka record mapping, shared by native DDL validation and Kafka request handling. + * Resolving a contract never changes the schema or materializes configuration defaults. + */ +@Internal +public final class KafkaTableSchema { + + public static final String KEY_COLUMN = "_kafka_key"; + public static final String VALUE_COLUMN = "_kafka_value"; + public static final String HEADERS_COLUMN = "_kafka_headers"; + public static final String METADATA_COLUMN = "_kafka_metadata"; + public static final String RESCUE_COLUMN = "_kafka_rescue"; + + private static final Set RESERVED_NAMES = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + KEY_COLUMN, + VALUE_COLUMN, + HEADERS_COLUMN, + METADATA_COLUMN, + RESCUE_COLUMN))); + private static final Set OPTIONS = + new HashSet<>( + Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key())); + + /** Supported value conversion contracts; decoders are provided by the Kafka module. */ + public enum ValueType { + BINARY, + STRING, + PARSE_JSON + } + + /** JSON business fields either occupy the table row or the fixed value ROW column. */ + public enum ValueLayout { + FLAT, + NESTED + } + + private final RowType rowType; + private final ValueType valueType; + private final @Nullable ValueLayout valueLayout; + private final KafkaFieldProjection valueProjection; + private final int keyPosition; + private final int headersPosition; + private final int rescuePosition; + + private KafkaTableSchema( + RowType rowType, + ValueType valueType, + @Nullable ValueLayout valueLayout, + List valuePositions, + int keyPosition, + int headersPosition, + int rescuePosition) { + this.rowType = rowType; + this.valueType = valueType; + this.valueLayout = valueLayout; + this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); + this.keyPosition = keyPosition; + this.headersPosition = headersPosition; + this.rescuePosition = rescuePosition; + } + + /** Returns only the persisted opt-in flag; schema compatibility does not imply enablement. */ + public static boolean isEnabled(TableDescriptor table) { + return new TableConfig(Configuration.fromMap(table.getProperties())).isKafkaEnabled(); + } + + /** Validates the Kafka contract only when the final table state enables it. */ + public static void validate(TableDescriptor table) { + if (isEnabled(table)) { + resolve(table); + } + } + + /** Checks whether the canonical topic name satisfies Fluss and Kafka name restrictions. */ + public static boolean isValidTablePath(TablePath path) { + // Fluss identifiers are already restricted to Kafka's ASCII character set, without dots. + return path != null + && path.isValid() + && TablePath.validatePrefix(path.getDatabaseName()) == null + && TablePath.validatePrefix(path.getTableName()) == null + && path.toString().length() <= 249; + } + + /** Validates the canonical topic name before persisting an enabled table. */ + public static void validateTablePath(TablePath path, TableDescriptor table) { + if (isEnabled(table) && !isValidTablePath(path)) { + throw invalid("Table path cannot be represented as a Kafka topic: " + path); + } + } + + /** Returns whether a JSON member at the value mapping level has a reserved name. */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + + /** Resolves an explicitly enabled table or rejects its invalid contract. */ + public static KafkaTableSchema resolve(TableDescriptor table) { + if (!isEnabled(table)) { + throw invalid("Kafka compatibility is disabled; table.kafka.enabled must be true."); + } + validateOptions(table); + if (table.hasPrimaryKey() + || table.isPartitioned() + || new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid( + "Kafka requires a non-partitioned Arrow log table without a primary key."); + } + Configuration options = Configuration.fromMap(table.getProperties()); + String value = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); + if (value == null) { + throw invalid("Missing required table property 'table.kafka.convert.value.type'."); + } + ValueType valueType; + try { + valueType = ValueType.valueOf(value.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid( + "Unsupported table.kafka.convert.value.type '" + + value + + "'. Expected binary, string or parse_json."); + } + String layout = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT); + ValueLayout valueLayout = null; + if (valueType == ValueType.PARSE_JSON) { + try { + valueLayout = + layout == null + ? ValueLayout.FLAT + : ValueLayout.valueOf(layout.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid("Unsupported table.kafka.convert.value.layout '" + layout + "'."); + } + } else if (layout != null) { + throw invalid("table.kafka.convert.value.layout is only supported for parse_json."); + } + + RowType rowType = table.getSchema().getRowType(); + int keyPosition = rowType.getFieldIndex(KEY_COLUMN); + if (keyPosition >= 0 && !(rowType.getTypeAt(keyPosition) instanceof BytesType)) { + throw invalid("_kafka_key must be BYTES."); + } + String keyType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE); + if (keyType != null && (!"binary".equalsIgnoreCase(keyType.trim()) || keyPosition < 0)) { + throw invalid( + "table.kafka.convert.key.type only supports binary and requires _kafka_key BYTES."); + } + int headersPosition = rowType.getFieldIndex(HEADERS_COLUMN); + if (headersPosition >= 0) { + DataType headers = rowType.getTypeAt(headersPosition); + if (!(headers instanceof MapType) + || !(((MapType) headers).getKeyType() instanceof StringType) + || !(((MapType) headers).getValueType() instanceof BytesType)) { + throw invalid("_kafka_headers must be MAP."); + } + } + int rescuePosition = rowType.getFieldIndex(RESCUE_COLUMN); + if (rescuePosition >= 0 + && (!(rowType.getTypeAt(rescuePosition) instanceof StringType) + || !rowType.getTypeAt(rescuePosition).isNullable())) { + throw invalid("_kafka_rescue must be nullable STRING."); + } + int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); + if (metadataPosition >= 0) { + validateMetadataType(rowType.getTypeAt(metadataPosition)); + throw invalid( + "_kafka_metadata requires native append support for persisting actual per-row offsets; " + + "this capability is not available yet."); + } + + List valuePositions = new ArrayList<>(); + if (valueLayout == ValueLayout.FLAT) { + if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { + throw invalid( + "_kafka_value is reserved for binary, string or nested JSON; " + + "it cannot be a flat JSON business field."); + } + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (!isReservedName(rowType.getFieldNames().get(i))) { + valuePositions.add(i); + } + } + } else { + int valuePosition = rowType.getFieldIndex(VALUE_COLUMN); + if (valuePosition < 0) { + throw invalid("Missing required value column _kafka_value."); + } + DataType type = rowType.getTypeAt(valuePosition); + boolean valid = + valueType == ValueType.BINARY + ? type instanceof BytesType + : valueType == ValueType.STRING + ? type instanceof StringType + : type instanceof RowType; + if (!valid) { + throw invalid( + "_kafka_value must be " + + (valueType == ValueType.BINARY + ? "BYTES" + : valueType == ValueType.STRING ? "STRING" : "ROW") + + " for " + + valueType + + "."); + } + for (String name : rowType.getFieldNames()) { + if (!isReservedName(name)) { + throw invalid( + "Column '" + + name + + "' has no value mapping; this conversion targets _kafka_value only."); + } + } + if (type instanceof RowType) { + for (String name : ((RowType) type).getFieldNames()) { + if (isReservedName(name)) { + throw invalid( + "Reserved name '" + + name + + "' cannot be a nested JSON business field."); + } + } + } + valuePositions.add(valuePosition); + } + return new KafkaTableSchema( + rowType, + valueType, + valueLayout, + valuePositions, + keyPosition, + headersPosition, + rescuePosition); + } + + private static void validateOptions(TableDescriptor table) { + for (String name : table.getProperties().keySet()) { + if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { + throw invalid("Unsupported Kafka table property '" + name + "'."); + } + } + for (String name : table.getCustomProperties().keySet()) { + if (name.startsWith("table.kafka.") + || name.startsWith("kafka.") + || name.startsWith("fluss.")) { + throw invalid( + "Unsupported Kafka custom property '" + + name + + "'. Use registered table.kafka.* properties and fixed system columns."); + } + } + } + + private static void validateMetadataType(DataType type) { + String message = + "_kafka_metadata must be ROW."; + if (!(type instanceof RowType)) { + throw invalid(message); + } + RowType row = (RowType) type; + if (!row.getFieldNames() + .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) + || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + throw invalid(message); + } + } + + /** Returns the unchanged physical schema, including declared nullability. */ + public RowType rowType() { + return rowType; + } + + /** Returns the value conversion type. */ + public ValueType valueType() { + return valueType; + } + + /** Returns the JSON layout, or null for binary/string. */ + public @Nullable ValueLayout valueLayout() { + return valueLayout; + } + + /** Returns physical value targets; nested JSON projects its single ROW container. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the key column position, or -1 when keys are not stored. */ + public int keyPosition() { + return keyPosition; + } + + /** + * Returns the headers column position, or -1 when headers are not stored. Decoding maps names + * to binary values, with the last occurrence winning for duplicate header names. + */ + public int headersPosition() { + return headersPosition; + } + + /** + * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and + * reserved members here; binary/string decoders write null to this optional column. + */ + public int rescuePosition() { + return rescuePosition; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof KafkaTableSchema)) { + return false; + } + KafkaTableSchema that = (KafkaTableSchema) other; + return rowType.equals(that.rowType) + && valueType == that.valueType + && valueLayout == that.valueLayout + && valueProjection.equals(that.valueProjection) + && keyPosition == that.keyPosition + && headersPosition == that.headersPosition + && rescuePosition == that.rescuePosition; + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + valueType, + valueLayout, + valueProjection, + keyPosition, + headersPosition, + rescuePosition); + } + + private static InvalidTableException invalid(String message) { + return new InvalidTableException(message); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java new file mode 100644 index 00000000000..e2b9c5be187 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.FlussConfigUtils; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the shared DDL and Kafka conversion contract. */ +class KafkaTableSchemaTest { + + @Test + void testRegisteredAlterableOptionsAndReadTimeDefaults() { + for (String key : + Arrays.asList( + "table.kafka.enabled", + "table.kafka.convert.value.type", + "table.kafka.convert.value.layout", + "table.kafka.convert.key.type")) { + assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); + assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); + } + Configuration config = new Configuration(); + assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); + assertThat(config.toMap()).isEmpty(); + } + + @Test + void testExplicitOptInAndDisabledTables() { + TableDescriptor disabled = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + assertThat(KafkaTableSchema.isEnabled(disabled)).isFalse(); + assertThatCode(() -> KafkaTableSchema.validate(disabled)).doesNotThrowAnyException(); + assertThatThrownBy(() -> KafkaTableSchema.resolve(disabled)) + .hasMessageContaining("disabled"); + TableDescriptor arbitrary = + TableDescriptor.builder(disabled) + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .primaryKey("id") + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "future-format") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + assertThatCode(() -> KafkaTableSchema.validate(arbitrary)).doesNotThrowAnyException(); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testFixedValueColumnWithOptionalSystemFields(String format) { + DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); + for (int mask = 0; mask < 8; mask++) { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", valueType.copy(false)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 2) != 0) { + schema.column( + "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + TableDescriptor table = enabled(schema.build(), format).build(); + byte[] before = table.toJsonBytes(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueProjection().positions()).containsExactly(0); + assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); + assertThat(resolved.valueLayout()).isNull(); + assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); + assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); + assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(table.getProperties()) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(resolved); + } + } + + @Test + void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { + RowType nested = + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", DataTypes.STRING()), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("_kafka_rescue", DataTypes.STRING()) + .column("payload", nested) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); + assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + TableDescriptor noRescue = + enabled( + Schema.newBuilder() + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); + } + + @Test + void testNestedJsonTargetsOneRow() { + RowType row = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()), + DataTypes.FIELD( + "details", + DataTypes.ROW( + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_value", row) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); + assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); + assertThat(resolved.rescuePosition()).isZero(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_value", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue" + }) + void testReservedNamesAtValueMappingLevel(String name) { + assertThat(KafkaTableSchema.isReservedName(name)).isTrue(); + TableDescriptor nested = + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD(name, DataTypes.STRING()))) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + assertInvalid(nested, "Reserved name"); + } + + @ParameterizedTest + @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + void testRejectsUnsupportedValueTypes(String type) { + assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); + } + + @Test + void testRejectsMissingValueTypeAndTargetConflicts() { + assertInvalid( + TableDescriptor.builder() + .schema(valueSchema()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(), + "Missing required"); + assertInvalid( + enabled(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "binary") + .build(), + "Missing required value column"); + assertInvalid(enabled(valueSchema(), "string").build(), "must be STRING"); + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "must be ROW"); + assertInvalid(enabled(valueSchema(), "parse_json").build(), "flat JSON"); + assertInvalid( + enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unused", DataTypes.STRING()) + .build(), + "binary") + .build(), + "no value mapping"); + assertInvalid( + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()))) + .column("unused", DataTypes.STRING()) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "no value mapping"); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNonJsonRejectsExplicitLayout(String type) { + assertInvalid( + enabled(valueSchema(), type) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "flat") + .build(), + "only supported for parse_json"); + } + + @Test + void testInvalidLayoutAndKeyConversion() { + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "unknown") + .build(), + "value.layout"); + assertInvalid( + enabled(valueSchema(), "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .build(), + "requires _kafka_key"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build(); + assertInvalid( + enabled(schema, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build(), + "only supports binary"); + assertThat( + KafkaTableSchema.resolve( + enabled(schema, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + "binary") + .build()) + .keyPosition()) + .isEqualTo(1); + } + + @Test + void testSystemColumnTypes() { + for (DataType invalid : Arrays.asList(DataTypes.STRING(), DataTypes.BINARY(8))) { + assertInvalid(withSystem("_kafka_key", invalid), "_kafka_key must be BYTES"); + } + for (DataType invalid : + Arrays.asList( + DataTypes.BYTES(), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.MAP(DataTypes.INT(), DataTypes.BYTES()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))) { + assertInvalid(withSystem("_kafka_headers", invalid), "MAP"); + } + assertInvalid(withSystem("_kafka_rescue", DataTypes.INT()), "nullable STRING"); + assertInvalid( + withSystem("_kafka_rescue", DataTypes.STRING().copy(false)), "nullable STRING"); + } + + @Test + void testMetadataStructureAndUnavailableOffsetPersistence() { + RowType metadata = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); + assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))), + "TIMESTAMP_LTZ(3)"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("offset", DataTypes.BIGINT()), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), + "must be ROW"); + } + + @Test + void testUnknownAndPrototypeOptions() { + for (String key : + Arrays.asList("table.kafka.unknown", "table.kafka.convert.rescue-column")) { + assertInvalid( + enabled(valueSchema(), "binary").property(key, "anything").build(), + "Unsupported Kafka table property"); + } + for (String key : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "kafka.metadata.headers.column", + "fluss.value.format", + "table.kafka.convert.value.type")) { + assertInvalid( + enabled(valueSchema(), "binary").customProperty(key, "raw").build(), + "Unsupported Kafka custom property"); + } + assertThat( + KafkaTableSchema.resolve( + enabled(valueSchema(), "binary") + .customProperty("owner", "team") + .build())) + .isNotNull(); + } + + @Test + void testTableKindsAndCanonicalNameLimits() { + Schema keyed = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") + .build(); + assertInvalid(enabled(keyed, "binary").build(), "without a primary key"); + assertInvalid( + enabled(valueSchema(), "binary").partitionedBy("_kafka_value").build(), + "non-partitioned"); + assertInvalid( + enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); + String database = String.join("", Collections.nCopies(124, "d")); + String tableName = String.join("", Collections.nCopies(124, "t")); + TableDescriptor table = enabled(valueSchema(), "binary").build(); + assertThatCode( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName), table)) + .doesNotThrowAnyException(); + assertThatThrownBy( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName + "t"), table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("Kafka topic"); + } + + private static TableDescriptor withSystem(String name, DataType type) { + return enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column(name, type) + .build(), + "binary") + .build(); + } + + private static Schema valueSchema() { + return Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(); + } + + private static TableDescriptor.Builder enabled(Schema schema, String type) { + return TableDescriptor.builder() + .schema(schema) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, type); + } + + private static void assertInvalid(TableDescriptor table, String message) { + assertThatThrownBy(() -> KafkaTableSchema.resolve(table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining(message); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 67f843a7276..56e990d8cd5 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -75,6 +76,50 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { + @Test + void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { + ResolvedSchema schema = + new ResolvedSchema( + Arrays.asList( + Column.physical( + "_kafka_value", + org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_headers", + org.apache.flink.table.api.DataTypes.MAP( + org.apache.flink.table.api.DataTypes.STRING(), + org.apache.flink.table.api.DataTypes.BYTES())), + Column.physical( + "_kafka_rescue", + org.apache.flink.table.api.DataTypes.STRING())), + Collections.emptyList(), + null); + Map options = new HashMap<>(); + options.put("table.kafka.enabled", "true"); + options.put("table.kafka.convert.value.type", "binary"); + CatalogTable table = + CatalogTable.of( + Schema.newBuilder().fromResolvedSchema(schema).build(), + "Kafka DDL", + Collections.emptyList(), + options); + TableDescriptor descriptor = + FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); + assertThat(descriptor.getProperties()) + .containsAllEntriesOf(options) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(descriptor.getCustomProperties()) + .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); + KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.keyPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping.rescuePosition()).isEqualTo(3); + } + @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java deleted file mode 100644 index df5d871907c..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.format; - -import org.apache.fluss.annotation.Internal; - -import java.util.Locale; - -/** Supported interpretations of Kafka record key and value bytes. */ -@Internal -public enum KafkaDataFormat { - RAW("raw"), - STRING("string"); - - /** Fluss table custom property controlling the record key format. */ - public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; - - /** Fluss table custom property controlling the record value format. */ - public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; - - /** Fluss fields populated from the Kafka record key. */ - public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; - - /** Strategy for deriving fields populated from the Kafka record value. */ - public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - - /** - * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar - * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. - */ - public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; - - /** Fluss column populated from the Kafka record headers. */ - public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; - - private final String value; - - KafkaDataFormat(String value) { - this.value = value; - } - - /** Parses a table custom property value. */ - public static KafkaDataFormat parse(String value) { - if (value == null) { - throw new IllegalArgumentException("Kafka data format must not be null."); - } - String normalized = value.trim().toLowerCase(Locale.ROOT); - for (KafkaDataFormat format : values()) { - if (format.value.equals(normalized)) { - return format; - } - } - throw new IllegalArgumentException( - "Unsupported Kafka data format '" + value + "'. Expected raw or string."); - } - - /** Returns the persisted table property value. */ - public String value() { - return value; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index f8b5744f71d..704d322f2e0 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -18,6 +18,9 @@ package org.apache.fluss.kafka.mapping; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -26,21 +29,43 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Maps fully qualified Kafka topic identities to Fluss tables. */ +/** Maps canonical Kafka topic names and default-database aliases to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - /** Maps a Kafka topic in database.table form to its Fluss table path. */ + private final String defaultDatabase; + + /** Creates a mapper using the default database configuration. */ + public KafkaTopicMapper() { + this(new Configuration()); + } + + /** Creates a mapper using the server's default database; it never creates databases. */ + public KafkaTopicMapper(Configuration config) { + this.defaultDatabase = config.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + TablePath.validateDatabaseName(defaultDatabase); + } + + /** Maps a canonical topic or an unqualified alias to a table path, without fallback lookup. */ public TablePath toTablePath(String topicName) { if (!isValidTopic(topicName)) { throw new InvalidTopicException( - "Kafka topic must be a valid database.table name: " + topicName); + "Kafka topic must be a valid table or database.table name: " + topicName); } int separator = topicName.indexOf('.'); - return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); + TablePath path = + separator < 0 + ? TablePath.of(defaultDatabase, topicName) + : TablePath.of( + topicName.substring(0, separator), + topicName.substring(separator + 1)); + if (!isMappedTable(path)) { + throw new InvalidTopicException("Canonical Kafka topic name is invalid: " + path); + } + return path; } /** Returns the fully qualified Kafka name of a representable Fluss table. */ @@ -55,7 +80,7 @@ public String toTopicName(TablePath tablePath) { /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + return KafkaTableSchema.isValidTablePath(tablePath); } /** Returns whether a name uniquely represents a user table in a Fluss database. */ @@ -64,14 +89,16 @@ public static boolean isValidTopic(String topicName) { return false; } int separator = topicName.indexOf('.'); - if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + if (separator < 0) { + return TablePath.detectInvalidName(topicName) == null + && TablePath.validatePrefix(topicName) == null; + } + if (separator == 0 || separator != topicName.lastIndexOf('.')) { return false; } - String database = topicName.substring(0, separator); - String table = topicName.substring(separator + 1); - return TablePath.of(database, table).isValid() - && TablePath.validatePrefix(database) == null - && TablePath.validatePrefix(table) == null; + return KafkaTableSchema.isValidTablePath( + TablePath.of( + topicName.substring(0, separator), topicName.substring(separator + 1))); } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java deleted file mode 100644 index 64263e69286..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.schema; - -import org.apache.fluss.annotation.Internal; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.types.RowType; - -import javax.annotation.Nullable; - -import java.util.Objects; - -import static org.apache.fluss.utils.Preconditions.checkNotNull; - -/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ -@Internal -public final class KafkaTopicSchema { - - private final RowType rowType; - private final @Nullable KafkaDataFormat keyFormat; - private final KafkaFieldProjection keyProjection; - private final KafkaDataFormat valueFormat; - private final KafkaFieldProjection valueProjection; - private final int timestampPosition; - private final int headersPosition; - - /** Creates a resolved Kafka topic schema. */ - public KafkaTopicSchema( - RowType rowType, - @Nullable KafkaDataFormat keyFormat, - KafkaFieldProjection keyProjection, - KafkaDataFormat valueFormat, - KafkaFieldProjection valueProjection, - int timestampPosition, - int headersPosition) { - this.rowType = checkNotNull(rowType); - this.keyFormat = keyFormat; - this.keyProjection = checkNotNull(keyProjection); - this.valueFormat = checkNotNull(valueFormat); - this.valueProjection = checkNotNull(valueProjection); - this.timestampPosition = timestampPosition; - this.headersPosition = headersPosition; - } - - /** Returns the physical Fluss row type. */ - public RowType rowType() { - return rowType; - } - - /** Returns the key format, or null when the Kafka key is not mapped. */ - public @Nullable KafkaDataFormat keyFormat() { - return keyFormat; - } - - /** Returns the key field projection. */ - public KafkaFieldProjection keyProjection() { - return keyProjection; - } - - /** Returns the value format. */ - public KafkaDataFormat valueFormat() { - return valueFormat; - } - - /** Returns the value field projection. */ - public KafkaFieldProjection valueProjection() { - return valueProjection; - } - - /** Returns the timestamp physical position, or -1 when it is not mapped. */ - public int timestampPosition() { - return timestampPosition; - } - - /** Returns the headers physical position, or -1 when they are not mapped. */ - public int headersPosition() { - return headersPosition; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof KafkaTopicSchema)) { - return false; - } - KafkaTopicSchema that = (KafkaTopicSchema) obj; - return Objects.equals(rowType, that.rowType) - && Objects.equals(keyFormat, that.keyFormat) - && Objects.equals(keyProjection, that.keyProjection) - && Objects.equals(valueFormat, that.valueFormat) - && Objects.equals(valueProjection, that.valueProjection) - && Objects.equals(timestampPosition, that.timestampPosition) - && Objects.equals(headersPosition, that.headersPosition); - } - - @Override - public int hashCode() { - return Objects.hash( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - @Override - public String toString() { - return "KafkaTopicSchema{" - + "rowType=" - + rowType - + ", " - + "keyFormat=" - + keyFormat - + ", " - + "keyProjection=" - + keyProjection - + ", " - + "valueFormat=" - + valueFormat - + ", " - + "valueProjection=" - + valueProjection - + ", " - + "timestampPosition=" - + timestampPosition - + ", " - + "headersPosition=" - + headersPosition - + "}"; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 8959c420d15..c02879e922a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -18,293 +18,29 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.ArrayType; -import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.RowType; -import org.apache.fluss.types.StringType; -import org.apache.fluss.types.TimestampType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +/** Resolves the same persisted Kafka contract that native DDL validates. */ @Internal public final class KafkaTopicSchemaResolver { - private static final String INCLUDE_ALL = "ALL"; - private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; - - private static final Set SUPPORTED_PROPERTIES = - new HashSet<>( - Arrays.asList( - KafkaDataFormat.KEY_FORMAT_CONFIG, - KafkaDataFormat.KEY_FIELDS_CONFIG, - KafkaDataFormat.VALUE_FORMAT_CONFIG, - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, - KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - - /** Resolves one table's Kafka record mapping contract. */ - public KafkaTopicSchema resolve(TableDescriptor table) { - validateTableKind(table); - RowType rowType = table.getSchema().getRowType(); - Map properties = table.getCustomProperties(); - - for (String property : properties.keySet()) { - if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { - throw invalid("Unsupported Kafka table property '" + property + "'."); - } - } - - int timestampPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); - int headersPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - if (timestampPosition >= 0 && timestampPosition == headersPosition) { - throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); - } - validateMetadataColumns(rowType, timestampPosition, headersPosition); - - String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); - KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); - List keyPositions = - resolveKeyPositions( - rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); - for (Integer keyPosition : keyPositions) { - if (keyPosition == timestampPosition || keyPosition == headersPosition) { - throw invalid( - "Kafka key field '" - + rowType.getFieldNames().get(keyPosition) - + "' cannot be a Kafka metadata column."); - } - } - KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); - - String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); - if (valueFormatValue == null) { - throw invalid( - "Missing required table property '" - + KafkaDataFormat.VALUE_FORMAT_CONFIG - + "'."); - } - KafkaDataFormat valueFormat = parseFormat(valueFormatValue); - String fieldsInclude = - normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); - if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { - throw invalid( - "Mapping Kafka key fields requires " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + "=EXCEPT_KEY."); - } - - Set metadataPositions = new HashSet<>(); - if (timestampPosition >= 0) { - metadataPositions.add(timestampPosition); - } - if (headersPosition >= 0) { - metadataPositions.add(headersPosition); - } - List valuePositions = - resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); - KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); - if (valueProjection.isEmpty()) { - throw invalid("Kafka value projection must contain at least one Fluss column."); - } - - validateSingleFieldFormat(keyFormat, keyProjection, "key"); - validateSingleFieldFormat(valueFormat, valueProjection, "value"); - return new KafkaTopicSchema( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - private static void validateTableKind(TableDescriptor table) { - if (table.hasPrimaryKey()) { - throw invalid("Kafka topic table must be a log table."); - } - if (table.isPartitioned()) { - throw invalid("Partitioned Fluss tables are not supported."); - } - if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() - != LogFormat.ARROW) { - throw invalid("Kafka topic table must use the Arrow log format."); - } - } - - private static List resolveKeyPositions( - RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { - if (keyFormat == null) { - if (keyFieldsValue != null) { - throw invalid( - KafkaDataFormat.KEY_FIELDS_CONFIG - + " requires " - + KafkaDataFormat.KEY_FORMAT_CONFIG - + "."); - } - return Collections.emptyList(); - } - if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { - throw invalid( - "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); - } - String[] fieldNames = keyFieldsValue.split(",", -1); - List positions = new ArrayList<>(fieldNames.length); - Set uniquePositions = new HashSet<>(); - for (String fieldNameValue : fieldNames) { - String fieldName = fieldNameValue.trim(); - if (fieldName.isEmpty()) { - throw invalid("Kafka key field names must not be empty."); - } - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka key field '" + fieldName + "' does not exist."); - } - if (!uniquePositions.add(position)) { - throw invalid("Duplicate Kafka key field '" + fieldName + "'."); - } - positions.add(position); - } - return positions; - } - - private static String normalizeFieldsInclude(String value) { - String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); - if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { - throw invalid( - "Invalid " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + " '" - + value - + "'. Expected ALL or EXCEPT_KEY."); - } - return normalized; - } - - private static List resolveValuePositions( - RowType rowType, - String fieldsInclude, - List keyPositions, - Set metadataPositions) { - Set excludedKeyPositions = - INCLUDE_EXCEPT_KEY.equals(fieldsInclude) - ? new HashSet<>(keyPositions) - : Collections.emptySet(); - List positions = new ArrayList<>(); - for (int position = 0; position < rowType.getFieldCount(); position++) { - if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { - positions.add(position); - } - } - return positions; - } - - private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { - if (fieldNameValue == null) { - return -1; - } - if (fieldNameValue.trim().isEmpty()) { - throw invalid("Kafka metadata column name must not be empty."); - } - String fieldName = fieldNameValue.trim(); - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); - } - return position; - } - - private static void validateMetadataColumns( - RowType rowType, int timestampPosition, int headersPosition) { - if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) - || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); - } - } - if (headersPosition >= 0) { - validateHeadersType(rowType.getTypeAt(headersPosition)); - } - } - - private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { - if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { - throw invalid( - "Kafka headers column must be nullable " - + "ARRAY>."); - } - ArrayType arrayType = (ArrayType) dataType; - if (!(arrayType.getElementType() instanceof RowType)) { - throw invalid("Kafka headers elements must be rows."); - } - RowType headerType = (RowType) arrayType.getElementType(); - if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) - || !(headerType.getTypeAt(0) instanceof StringType) - || !(headerType.getTypeAt(1) instanceof BytesType) - || !headerType.getTypeAt(1).isNullable()) { - throw invalid("Kafka headers elements must be ROW."); - } - } - - private static void validateSingleFieldFormat( - KafkaDataFormat format, KafkaFieldProjection projection, String component) { - if (format == null) { - return; - } - if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { - if (projection.size() != 1) { - throw invalid( - "Kafka " - + component - + " format " - + format.value() - + " requires exactly one Fluss field."); - } - boolean validType = - format == KafkaDataFormat.RAW - ? projection.dataTypeAt(0) instanceof BytesType - : projection.dataTypeAt(0) instanceof StringType; - if (!validType) { - throw invalid( - "Kafka " - + component - + " field '" - + projection.nameAt(0) - + "' must be " - + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") - + " for format " - + format.value() - + "."); - } - } - } - - private static KafkaDataFormat parseFormat(String value) { + /** Returns the table's explicit opt-in flag without inferring it from schema compatibility. */ + public boolean isEnabled(TableDescriptor table) { try { - return KafkaDataFormat.parse(value); + return KafkaTableSchema.isEnabled(table); } catch (IllegalArgumentException e) { - throw invalid(e.getMessage()); + throw new KafkaTopicSchemaException(e.getMessage()); } } - private static KafkaTopicSchemaException invalid(String message) { - return new KafkaTopicSchemaException(message); + /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ + public KafkaTableSchema resolve(TableDescriptor table) { + try { + return KafkaTableSchema.resolve(table); + } catch (InvalidTableException | IllegalArgumentException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 7f502494f2e..55e8ba6fc57 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,13 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("fluss"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("fluss"); } @Test @@ -51,6 +52,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("kafka"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("kafka"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index c5d4a582a88..7ef7e281244 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka.mapping; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -49,11 +51,41 @@ public void testQualifiedNamesAcrossDatabases() { } } + @Test + void testDefaultDatabaseAliases() { + assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper custom = new KafkaTopicMapper(config); + assertThat(custom.toTablePath("orders")).isEqualTo(TablePath.of("sales", "orders")); + assertThat(custom.toTablePath("warehouse.orders")) + .isEqualTo(TablePath.of("warehouse", "orders")); + assertThat(custom.toTopicName(custom.toTablePath("orders"))).isEqualTo("sales.orders"); + config.setString("kafka.database", "legacy"); + assertThat(new KafkaTopicMapper(config).toTablePath("orders")) + .isEqualTo(TablePath.of("sales", "orders")); + } + + @Test + void testInvalidDefaultDatabaseAndCanonicalAliasLength() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "bad.name"); + assertThatThrownBy(() -> new KafkaTopicMapper(config)).hasMessageContaining("invalid"); + config.set( + ConfigOptions.KAFKA_DEFAULT_DATABASE, + String.join("", Collections.nCopies(128, "d"))); + KafkaTopicMapper longDatabase = new KafkaTopicMapper(config); + assertThatThrownBy( + () -> + longDatabase.toTablePath( + String.join("", Collections.nCopies(128, "t")))) + .isInstanceOf(InvalidTopicException.class); + } + @ParameterizedTest @NullAndEmptySource @ValueSource( strings = { - "orders", ".orders", "sales.", "sales.orders.extra", diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2f3e75bdd3d..45744dbc87a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,99 +17,371 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.InvalidConfigException; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.rpc.messages.MetadataRequest; -import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.AlterTableRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbAddColumn; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.json.DataTypeJsonSerde; +import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateDatabaseRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newGetTableInfoRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newTableExistsRequest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ -public class KafkaTableMappingITCase { +/** Verifies Kafka DDL contracts through real coordinator RPC and persisted metadata. */ +class KafkaTableMappingITCase { + + private static final Configuration CONFIG = new Configuration(); + + static { + CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.setString("kafka.database", "legacy_database"); + } @RegisterExtension - public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = - FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder().setNumOfTabletServers(1).setClusterConf(CONFIG).build(); + + private CoordinatorGateway gateway; + + @BeforeEach + void setup() { + gateway = CLUSTER.newCoordinatorClient(); + } @Test - public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { + void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { + for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) + .isFalse(); + } + assertThat( + gateway.databaseExists(newDatabaseExistsRequest(DEFAULT_DATABASE)) + .get() + .isExists()) + .isTrue(); + KafkaTopicMapper mapper = new KafkaTopicMapper(CONFIG); + TablePath sales = mapper.toTablePath("sales.orders"); + TablePath warehouse = mapper.toTablePath("warehouse.orders"); + long salesId = createTable(CLUSTER, sales, enabled().build()); + long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); + assertThat(salesId).isNotEqualTo(warehouseId); + assertThat(mapper.toTablePath("orders")) + .isEqualTo(TablePath.of("missing_default", "orders")); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + mapper.toTablePath("orders"), + enabled().build(), + false)) + .get()) + .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat( + gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + .get() + .isExists()) + .isFalse(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); + assertThat(info(aliasMapper.toTablePath("orders")).getTableId()) + .isEqualTo(info(sales).getTableId()); + assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); + } + + @Test + void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { + TablePath path = TablePath.of("ddl", "roundtrip"); + Schema schema = + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column( + "payload", + DataTypes.ROW(DataTypes.FIELD("nested", DataTypes.STRING()))) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build(); TableDescriptor descriptor = - TableDescriptor.builder() + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + createTable(CLUSTER, path, descriptor); + TableDescriptor stored = stored(path); + assertThat(stored.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(stored)).isEqualTo(kafkaProperties(descriptor)); + assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + .containsExactly(1); + } + + @Test + void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + TablePath path = TablePath.of("ddl", "enable_disable"); + TableDescriptor disabled = + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, disabled); + TableDescriptor before = stored(path); + assertThat(KafkaTableSchema.isEnabled(before)).isFalse(); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor enabled = stored(path); + assertThat(KafkaTableSchema.isEnabled(enabled)).isTrue(); + Map expected = new HashMap<>(before.getProperties()); + expected.put("table.kafka.enabled", "true"); + assertThat(enabled.getProperties()).isEqualTo(expected); + assertThat(enabled.getSchema()).isEqualTo(before.getSchema()); + Map invalid = new HashMap<>(); + invalid.put("table.kafka.convert.value.type", "string"); + invalid.put("table.kafka.convert.key.type", "binary"); + assertThatThrownBy(() -> set(path, invalid)) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + assertThatThrownBy(() -> reset(path, "table.kafka.convert.value.type")) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + reset(path, "table.kafka.enabled"); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + assertThat(stored(path).getProperties()).doesNotContainKey("table.kafka.enabled"); + // Disabled tables can stage incompatible Kafka conversion properties without Kafka + // validation. + set(path, Collections.singletonMap("table.kafka.convert.value.type", "string")); + TableDescriptor staged = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(staged); + } + + @Test + void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exception { + TablePath path = TablePath.of("ddl", "schema_change"); + createTable(CLUSTER, path, enabled().build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_headers", DataTypes.STRING())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(info(path).getTableJson()).isEqualTo(before.getTableJson()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + TableDescriptor withHeaders = stored(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(withHeaders); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "unmapped_business_field", DataTypes.INT()); + assertThat(stored(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped_business_field"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + + @Test + void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); + TablePath path = TablePath.of("ddl_invalid", "bad"); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "string") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + // Non-null added columns are rejected by generic schema evolution first, so exercise + // Kafka-specific rescue nullability through CREATE, where such columns are allowed. + TableDescriptor nonNullableRescue = + enabled() .schema( Schema.newBuilder() - .column("event_key", DataTypes.STRING()) - .column("event_body", DataTypes.BYTES()) - .column("event_time", DataTypes.TIMESTAMP(3)) + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_rescue", DataTypes.STRING().copy(false)) .build()) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper(); - Map tableIds = new LinkedHashMap<>(); - MetadataRequest request = new MetadataRequest(); - for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { - TablePath tablePath = mapper.toTablePath(database + ".events"); - assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); - tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - } - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - List metadatas = - FLUSS_CLUSTER_EXTENSION - .newCoordinatorClient() - .metadata(request) - .get() - .getTableMetadatasList(); - assertThat(metadatas).hasSize(2); - assertThat(metadatas) - .extracting(PbTableMetadata::getTableId) - .containsExactlyInAnyOrderElementsOf(tableIds.values()) - .doesNotHaveDuplicates(); - for (PbTableMetadata metadata : metadatas) { - TablePath tablePath = - TablePath.of( - metadata.getTablePath().getDatabaseName(), - metadata.getTablePath().getTableName()); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - - assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); - assertThat(mapper.toTopicName(tablePath)) - .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) - .isEqualTo(tableIds.get(tablePath)); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.timestampPosition()).isEqualTo(2); - assertThat(persisted.getSchema().getRowType().getTypeAt(2)) - .isEqualTo(DataTypes.TIMESTAMP(3)); - } + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, nonNullableRescue, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable STRING"); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + "table.kafka.convert.rescue-column", + "rescue") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidConfigException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + createTable(CLUSTER, path, enabled().build()); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); + } + + @Test + void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { + TablePath path = TablePath.of("ddl", "offset_capability"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + createTable( + CLUSTER, + path, + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("actual per-row offsets"); + assertThat(stored(path)).isEqualTo(before); + } + + @Test + void testCanonicalNameCheckedOnCreateAndEnable() throws Exception { + String database = String.join("", Collections.nCopies(128, "d")); + TablePath path = TablePath.of(database, String.join("", Collections.nCopies(128, "t"))); + gateway.createDatabase(newCreateDatabaseRequest(database, true)).get(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, enabled().build(), false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("Kafka topic"); + createTable( + CLUSTER, + path, + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(before); + } + + private GetTableInfoResponse info(TablePath path) throws Exception { + return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); + } + + private TableDescriptor stored(TablePath path) throws Exception { + return TableDescriptor.fromJsonBytes(info(path).getTableJson()); + } + + private void set(TablePath path, Map properties) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + properties, + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private void reset(TablePath path, String property) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(property), + Collections.emptyList(), + false)) + .get(); + } + + private void addColumn(TablePath path, String name, DataType type) throws Exception { + PbAddColumn column = + new PbAddColumn() + .setColumnName(name) + .setDataTypeJson( + JsonSerdeUtils.writeValueAsBytes(type, DataTypeJsonSerde.INSTANCE)) + .setColumnPositionType(0); + AlterTableRequest request = + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.emptyList(), + Collections.singletonList(column), + false); + gateway.alterTable(request).get(); + } + + private static Map kafkaProperties(TableDescriptor table) { + Map properties = new HashMap<>(); + table.getProperties() + .forEach( + (key, value) -> { + if (key.startsWith("table.kafka.")) { + properties.put(key, value); + } + }); + return properties; + } + + private static TableDescriptor.Builder enabled() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 66c13a4711a..b7e4174421c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,334 +17,61 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests the DDL contract independently of Kafka request handling and record decoding. */ -public class KafkaTopicSchemaResolverTest { - +/** Tests the Kafka adapter's use of the shared DDL contract. */ +class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); @Test - public void testRawMappingSurvivesTableMetadataSerialization() { - Schema schema = - Schema.newBuilder() - .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) - .column("attributes", headersType()) - .column("message_key", DataTypes.BYTES()) - .build(); - TableDescriptor table = - table(schema, "raw") - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") - .build(); - KafkaTopicSchema mapping = - resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); - - assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(3); - assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping).isEqualTo(resolver.resolve(table)); - assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) - .isInstanceOf(UnsupportedOperationException.class); - } - - @Test - public void testValueOnlyStringAndNullableColumns() { - for (boolean nullable : new boolean[] {true, false}) { - KafkaTopicSchema mapping = - resolver.resolve( - table( - Schema.newBuilder() - .column( - "body", - DataTypes.STRING().copy(nullable)) - .build(), - " STRING ") - .build()); - assertThat(mapping.keyFormat()).isNull(); - assertThat(mapping.keyProjection().isEmpty()).isTrue(); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); - assertThat(mapping.timestampPosition()).isEqualTo(-1); - assertThat(mapping.headersPosition()).isEqualTo(-1); - } + void testEnabledContractUsesSharedValidation() { + TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + assertThat(resolver.isEnabled(table)).isTrue(); + assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolver.resolve(table)); } @Test - public void testMixedFormatsAndMetadataAreOptional() { - KafkaTopicSchema mapping = - resolver.resolve( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") - .build()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().positions()).containsExactly(1); + void testCompatibleSchemaDoesNotEnableTable() { + assertThat(resolver.isEnabled(table().build())).isFalse(); + assertThatThrownBy(() -> resolver.resolve(table().build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); } @Test - public void testRejectsUnsupportedTableKinds() { - Schema primaryKeySchema = - Schema.newBuilder() - .column("id", DataTypes.STRING().copy(false)) - .primaryKey("id") + void testInvalidEnabledContractIsNotTreatedAsDisabled() { + TableDescriptor table = + table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "schema_latest") .build(); - assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); - assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); - assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); - } - - @Test - public void testRequiresExplicitValueFormat() { - assertInvalid( - TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) - .distributedBy(1) - .customProperty("fluss.value.format", "raw"), - KafkaDataFormat.VALUE_FORMAT_CONFIG); - } - - @ParameterizedTest - @ValueSource(strings = {"json", "avro", ""}) - public void testRejectsUnavailableFormats(String format) { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - } - - @Test - public void testRejectsUnsupportedKafkaOptions() { - assertInvalid( - valueTable().customProperty("kafka.value.rescue-column", "body"), - "Unsupported Kafka table property"); - assertInvalid( - valueTable().customProperty("kafka.key.field", "body"), - "Unsupported Kafka table property"); - assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) - .isNotNull(); - } - - @Test - public void testKeyFormatAndFieldMustBeSpecifiedTogether() { - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires kafka.key.format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), - "kafka.key.fields"); - } - - @ParameterizedTest - @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) - public void testRejectsInvalidKeyFields(String fields) { - assertThatThrownBy( - () -> - resolver.resolve( - keyValueTable() - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty( - KafkaDataFormat.KEY_FIELDS_CONFIG, fields) - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - "EXCEPT_KEY") - .build())) - .isInstanceOf(KafkaTopicSchemaException.class); - } - - @Test - public void testRejectsAmbiguousAndEmptyValueProjections() { - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), - "Expected ALL or EXCEPT_KEY"); - assertInvalid( - valueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - assertInvalid(keyValueTable(), "exactly one Fluss field"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - } - - @Test - public void testRejectsWrongPhysicalTypes() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), - "must be STRING"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "must be BYTES"); - } - - @Test - public void testAcceptsTimestampWithEitherNullability() { - for (boolean nullable : new boolean[] {true, false}) { - TableDescriptor descriptor = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") - .build(); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - } - } - - @Test - public void testRejectsWrongTimestampTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(3).copy(false), - DataTypes.TIMESTAMP(2), - DataTypes.TIMESTAMP(6).copy(false) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP(3)"); - } - } - - @Test - public void testRejectsWrongHeaderTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - headersType().copy(false), - DataTypes.ARRAY(DataTypes.STRING()), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("key", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("attributes", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), - "headers"); - } + assertThat(resolver.isEnabled(table)).isTrue(); + assertThatThrownBy(() -> resolver.resolve(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("schema_latest"); } @Test - public void testRejectsConflictingOrMissingMetadataColumns() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), - "does not exist"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), - "must not be empty"); - TableDescriptor.Builder table = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(false)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), - "same Fluss column"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), - "metadata column"); - } - - private void assertInvalid(TableDescriptor.Builder table, String message) { - assertThatThrownBy(() -> resolver.resolve(table.build())) + void testMalformedPersistedEnableFlagIsAnExplicitContractError() { + TableDescriptor table = table().property("table.kafka.enabled", "maybe").build(); + assertThatThrownBy(() -> resolver.isEnabled(table)) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining(message); + .hasMessageContaining("table.kafka.enabled"); } - private static TableDescriptor.Builder keyValueTable() { - return table( - Schema.newBuilder() - .column("id", DataTypes.STRING()) - .column("body", DataTypes.BYTES()) - .build(), - "raw"); - } - - private static TableDescriptor.Builder valueTable() { - return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); - } - - private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + private static TableDescriptor.Builder table() { return TableDescriptor.builder() - .schema(schema) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); - } - - private static DataType headersType() { - return DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))); + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 2cdd2c5bf97..aff8bb78761 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -553,14 +553,6 @@ private void createDefaultDatabase() { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } - // create Kafka default database if Kafka is enabled. - if (conf.get(ConfigOptions.KAFKA_ENABLED)) { - String kafkaDB = conf.get(ConfigOptions.KAFKA_DATABASE); - if (!databases.contains(kafkaDB)) { - metadataManager.createDatabase(kafkaDB, DatabaseDescriptor.EMPTY, true); - LOG.info("Created default database '{}' for Kafka protocol.", kafkaDB); - } - } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 6bed0ecd3f0..e8e09dab26d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent @@ -615,7 +615,8 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal()); + currentSession().getPrincipal(), + coordinatorZkVersionSupplier.get()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 14b97c927eb..b1a5fae3d04 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -39,6 +39,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.DatabaseInfo; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LakeTableUtil; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -58,6 +59,7 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -131,6 +133,12 @@ public void validateTableDescriptor(TableDescriptor tableDescriptor) { lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); } + /** Validates a descriptor and its Kafka topic identity before persistence. */ + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { + validateTableDescriptor(tableDescriptor); + KafkaTableSchema.validateTablePath(tablePath, tableDescriptor); + } + public void createDatabase( String databaseName, DatabaseDescriptor databaseDescriptor, boolean ignoreIfExists) throws DatabaseAlreadyExistException { @@ -458,8 +466,25 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - try { + alterTableSchema( + tablePath, + schemaChanges, + ignoreIfNotExists, + flussPrincipal, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + public void alterTableSchema( + TablePath tablePath, + List schemaChanges, + boolean ignoreIfNotExists, + FlussPrincipal flussPrincipal, + int coordinatorZkVersion) + throws TableNotExistException, TableNotPartitionedException { + try { + ZooKeeperClient.VersionedData versionedTable = + getTableRegistrationWithVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -480,7 +505,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); + zookeeperClient.registerSchemaAndBumpTableVersion( + tablePath, + newSchema, + table.getSchemaId() + 1, + versionedTable, + coordinatorZkVersion); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -852,7 +882,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index de40fbf01a8..fafbbf9a6c1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; @@ -122,6 +123,8 @@ public static void validateTableDescriptor( } } + KafkaTableSchema.validate(tableDescriptor); + // check distribution checkDistribution(tableDescriptor, maxBucketNum); checkPrimaryKey(tableDescriptor); @@ -146,6 +149,10 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { + TableDescriptor updated = + TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); + KafkaTableSchema.validate(updated); + KafkaTableSchema.validateTablePath(table.getTablePath(), updated); checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 01f897cff29..a7a706c214e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1315,6 +1315,36 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } + /** + * Publishes a schema and advances the table registration version in one fenced transaction. + * Property ALTER validates against a schema snapshot and compares the table version, so schema + * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + */ + public void registerSchemaAndBumpTableVersion( + TablePath tablePath, + Schema schema, + int schemaId, + VersionedData expectedTable, + int expectedCoordinatorEpochZkVersion) + throws Exception { + List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .setData() + .withVersion(expectedTable.zkVersion()) + .forPath( + TableZNode.path(tablePath), + TableZNode.encode(expectedTable.data()))); + ops.add( + zkClient.transactionOp() + .create() + .withMode(CreateMode.PERSISTENT) + .forPath( + SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); + zkClient.transaction() + .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java new file mode 100644 index 00000000000..e76dbda662d --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.entity.TablePropertyChanges; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.function.BiConsumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ +class KafkaTableSchemaAlterTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private ZooKeeperClient zk; + private MetadataManager manager; + private int epoch; + + @BeforeEach + void setup() throws Exception { + zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); + } + + @Test + void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { + TablePath path = create("schema_wins"); + assertThatThrownBy( + () -> + enable( + path, + (oldTable, updated) -> + manager.alterTableSchema( + path, + Collections.singletonList( + TableChange.addColumn( + "unmapped", + DataTypes.STRING(), + null, + TableChange.ColumnPosition + .last())), + false, + null, + epoch))) + .isInstanceOf(FlussRuntimeException.class) + .hasMessageContaining("Concurrent metadata change"); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped"); + } + + @Test + void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { + TablePath path = create("enable_wins"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema incompatible = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unmapped", DataTypes.STRING()) + .build(); + // This candidate is valid for the disabled snapshot, but not after the competing enable. + KafkaTableSchema.validate( + TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) + .schema(incompatible) + .build()); + enable(path, (oldTable, updated) -> {}); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, incompatible, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .containsExactly("_kafka_value"); + } + + @Test + void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + TablePath path = create("atomic_failure"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema schema = manager.getTable(path).getSchema(); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId, snapshot, epoch)) + .isInstanceOf(KeeperException.NodeExistsException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + zk.fenceBecomeCoordinatorLeader("next-coordinator"); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + } + + private TablePath create(String name) { + TablePath path = TablePath.of("ddl", name); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build() + .withReplicationFactor(1); + manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); + return path; + } + + private void enable( + TablePath path, + BiConsumer beforeUpdate) { + TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); + changes.setTableProperty("table.kafka.enabled", "true"); + manager.alterTableProperties( + path, + Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), + changes.build(), + false, + null, + beforeUpdate, + (oldTable, updated) -> {}, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } +} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index ce76beca9a1..491f7e83c6d 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,7 +254,42 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.database | String | kafka | The database for Fluss Kafka. The default database is `kafka`. | +| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. + +Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named +`sales.orders` maps to that table directly; `orders` maps to the same-named table in +`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +the fully qualified name. All TabletServers must use the same default database. Changing it +redirects unqualified names, so migrate clients to fully qualified names before changing the +configuration. The prototype `kafka.database` option is no longer used. + +### Kafka table properties + +| Option | Type | Default | Description | +|---|---|---|---| +| table.kafka.enabled | Boolean | false | Explicitly expose a non-partitioned Arrow log table as a Kafka topic. CREATE and ALTER validate the contract before persisting it. | +| table.kafka.convert.value.type | String | (None) | Required when enabled: `binary`, `string`, or `parse_json`. | +| table.kafka.convert.value.layout | String | (None) | JSON uses `flat` when omitted; `nested` targets `_kafka_value ROW<...>`. Setting this property for binary/string is invalid. | +| table.kafka.convert.key.type | String | (None) | Only `binary` is supported, and requires `_kafka_key BYTES`. Declaring that column implies binary without adding a property. | + +Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` +column respectively. Flat JSON targets business columns at the table level; nested JSON targets +the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. +Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and +collects unknown or reserved JSON members; it requires no mapping property. Binary/string +conversion writes null to a declared rescue column. + +Enabling Kafka compatibility does not add columns or persist default Kafka properties. When +disabled, ordinary Fluss tables are not subject to Kafka-specific schema validation. Prototype +`kafka.key.*`, `kafka.value.*`, `kafka.metadata.*`, and `fluss.*` conversion properties are not +accepted on enabled tables. + +The `_kafka_metadata` contract is +`ROW`. +Enabling tables with this column is currently rejected until native append can assign and +persist each row's actual offset. Other optional system columns can be used independently. +These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From 48c84f9a5324428ef22d77673066d9723dd8593a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:15:35 +0800 Subject: [PATCH 11/14] [kafka] Validate recursive JSON schemas and default to fluss Default bare topic names to the native fluss database while preserving explicit database overrides, canonical names and no automatic Kafka database creation. Validate JSON business types recursively before CREATE, ALTER and enablement. Reject non-STRING map keys at every nesting level and schemas deeper than the supported limit. Expose the validation entry point and depth constant for subsequent decoders. Count business fields equally in flat and nested layouts, without charging a level for the physical _kafka_value ROW container. Keep disabled tables outside Kafka validation and preserve user schemas and explicit properties. Cover complex scalar/container types, depth boundaries, failed DDL state preservation and successful enablement with real RPC tests. Validation: Java 11 clean reactor verify passed 161 targeted cases, including snapshot lease and concurrent ALTER regressions. Checkstyle, Spotless, RAT and git diff checks passed. The five prior PR03 commits were rebased onto PR02 97d20b18549ec34f49e4e608392cdf1ec3a1b68a and checked with range-diff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 92/92 AI-Contributed/UT: 292/292 --- .../apache/fluss/config/ConfigOptions.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 80 ++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 103 ++++++++++ .../apache/fluss/kafka/KafkaConfigsTest.java | 7 +- .../kafka/mapping/KafkaTopicMapperTest.java | 4 +- .../kafka/schema/KafkaTableMappingITCase.java | 178 ++++++++++++++++++ website/docs/maintenance/configuration.md | 10 +- 7 files changed, 377 insertions(+), 7 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 667e1af5f68..66f908d74d0 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2940,7 +2940,7 @@ public class ConfigOptions { public static final ConfigOption KAFKA_DEFAULT_DATABASE = key("kafka.default.database") .stringType() - .defaultValue("kafka") + .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " + "access other databases. Databases must already exist; this option does not create them."); diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 7c040f15871..990cec50416 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -53,6 +54,9 @@ public final class KafkaTableSchema { public static final String METADATA_COLUMN = "_kafka_metadata"; public static final String RESCUE_COLUMN = "_kafka_rescue"; + /** Maximum recursive depth below each top-level JSON business field. */ + public static final int MAX_JSON_SCHEMA_NESTING_DEPTH = 64; + private static final Set RESERVED_NAMES = Collections.unmodifiableSet( new HashSet<>( @@ -271,6 +275,20 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } valuePositions.add(valuePosition); } + if (valueType == ValueType.PARSE_JSON) { + for (int position : valuePositions) { + DataType type = rowType.getTypeAt(position); + if (valueLayout == ValueLayout.NESTED) { + // The physical value container represents the JSON object root, just like + // the table row in flat layout; it must not add a business nesting level. + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field); + } + } else { + validateJsonType(type); + } + } + } return new KafkaTableSchema( rowType, valueType, @@ -281,6 +299,68 @@ public static KafkaTableSchema resolve(TableDescriptor table) { rescuePosition); } + /** + * Validates a top-level JSON business field without parser or protocol dependencies. DDL and + * decoder construction share this contract. Each business field starts at depth zero; ROW + * fields, ARRAY elements and MAP values each advance the depth by one. Nullability and scalar + * value constraints are enforced when decoding individual records. The JSON object root + * (including the physical nested _kafka_value ROW) is not counted. + */ + public static void validateJsonType(DataType type) { + validateJsonType(type, 0); + } + + private static void validateJsonType(DataType type, int depth) { + if (depth > MAX_JSON_SCHEMA_NESTING_DEPTH) { + throw invalid( + "Kafka JSON schema exceeds the maximum nesting depth of " + + MAX_JSON_SCHEMA_NESTING_DEPTH + + "."); + } + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case BINARY: + case BYTES: + case DECIMAL: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return; + case ARRAY: + validateJsonType(((ArrayType) type).getElementType(), depth + 1); + return; + case MAP: + MapType map = (MapType) type; + if (!(map.getKeyType() instanceof StringType)) { + throw invalid( + "Kafka JSON format only supports STRING map keys, but found " + + type.asSummaryString() + + "."); + } + validateJsonType(map.getValueType(), depth + 1); + return; + case ROW: + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field, depth + 1); + } + return; + default: + throw invalid( + "Kafka JSON format does not support Fluss type " + + type.asSummaryString() + + "."); + } + } + private static void validateOptions(TableDescriptor table) { for (String name : table.getProperties().keySet()) { if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index e2b9c5be187..809f13ac553 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -186,6 +186,109 @@ void testNestedJsonTargetsOneRow() { assertThat(resolved.rescuePosition()).isZero(); } + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonRejectsNonStringMapKeysRecursively(String layout) { + for (DataType key : Arrays.asList(DataTypes.INT(), DataTypes.CHAR(10), DataTypes.BYTES())) { + DataType invalidMap = DataTypes.MAP(key, DataTypes.STRING()); + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TableDescriptor table = jsonTable(layout, type); + assertInvalid(table, "only supports STRING map keys"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(table) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build())) + .doesNotThrowAnyException(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { + for (DataType scalar : + Arrays.asList( + DataTypes.CHAR(10), + DataTypes.STRING(), + DataTypes.BOOLEAN(), + DataTypes.BINARY(4), + DataTypes.BYTES(), + DataTypes.DECIMAL(18, 2), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.DATE(), + DataTypes.TIME(3), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(9))) { + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING().copy(false), + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", scalar.copy(false))))); + TableDescriptor table = jsonTable(layout, type); + byte[] before = table.toJsonBytes(); + assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(KafkaTableSchema.resolve(table)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonBusinessNestingDepthBoundary(String layout) { + // Exercise every recursive edge, including map values (keys do not add a level). + DataType boundary = DataTypes.STRING(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + if (depth % 3 == 0) { + boundary = DataTypes.ARRAY(boundary); + } else if (depth % 3 == 1) { + boundary = DataTypes.MAP(DataTypes.STRING(), boundary); + } else { + boundary = DataTypes.ROW(DataTypes.FIELD("child", boundary)); + } + } + TableDescriptor valid = jsonTable(layout, boundary); + assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); + assertInvalid(tooDeep, "maximum nesting depth of 64"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build())) + .doesNotThrowAnyException(); + } + + private static TableDescriptor jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + return enabled(schema, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .build(); + } + @ParameterizedTest @ValueSource( strings = { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 55e8ba6fc57..17a8c6c0f9c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,14 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); + .isEqualTo("sales"); } @Test @@ -53,6 +53,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("kafka"); + .isEqualTo("fluss"); + assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 7ef7e281244..f70c7ffca22 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -53,7 +53,9 @@ public void testQualifiedNamesAcrossDatabases() { @Test void testDefaultDatabaseAliases() { - assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("fluss", "orders")); + assertThat(mapper.toTopicName(mapper.toTablePath("orders"))).isEqualTo("fluss.orders"); + assertThat(mapper.toTablePath("kafka.orders")).isEqualTo(TablePath.of("kafka", "orders")); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper custom = new KafkaTopicMapper(config); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 45744dbc87a..76b9c0f773d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -41,6 +41,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.Collections; @@ -121,6 +123,182 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } + @Test + void testDefaultAliasUsesExistingFlussDatabase() throws Exception { + KafkaTopicMapper mapper = new KafkaTopicMapper(); + TablePath path = mapper.toTablePath("default_alias"); + assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); + createTable(CLUSTER, path, enabled().build()); + assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) + .isEqualTo(info(path).getTableId()); + assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); + DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); + int index = 0; + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); + TableDescriptor descriptor = jsonTable(layout, type).build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + // The identical schema is legal for an ordinary, explicitly disabled Fluss table. + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy( + () -> + set( + path, + Collections.singletonMap( + "table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonComplexSchemaRoundtripAndEnable(String layout) throws Exception { + TablePath path = TablePath.of("ddl_json", layout + "_complex"); + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD("amount", DataTypes.DECIMAL(18, 2)), + DataTypes.FIELD("time", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("_kafka_key", DataTypes.BYTES())))); + TableDescriptor descriptor = + jsonTable(layout, type).property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, descriptor); + GetTableInfoResponse before = info(path); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor actual = stored(path); + assertThat(actual.getSchema()).isEqualTo(descriptor.getSchema()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + Map expected = new HashMap<>(kafkaProperties(descriptor)); + expected.put("table.kafka.enabled", "true"); + assertThat(kafkaProperties(actual)).isEqualTo(expected); + addColumn(path, "_kafka_rescue", DataTypes.STRING()); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(stored(path).getSchema().getRowType().getTypeAt(0)) + .isEqualTo(descriptor.getSchema().getRowType().getTypeAt(0)); + assertThat(KafkaTableSchema.resolve(stored(path)).rescuePosition()).isEqualTo(1); + assertThat(kafkaProperties(stored(path))).isEqualTo(expected); + } + + @Test + void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception { + TablePath path = TablePath.of("ddl_json", "alter_business_type"); + createTable(CLUSTER, path, jsonTable("flat", DataTypes.STRING()).build()); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "bad", invalid)) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + addColumn( + path, "good", DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT()))); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "ordinary", invalid); + GetTableInfoResponse disabled = info(path); + assertThat(stored(path).getSchema().getRowType().getFieldNames()).contains("ordinary"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, disabled); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonDepthValidationThroughDdl(String layout) throws Exception { + DataType boundary = DataTypes.STRING(); + for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { + boundary = DataTypes.ARRAY(boundary); + } + TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); + TableDescriptor descriptor = jsonTable(layout, boundary).build(); + createTable(CLUSTER, valid, descriptor); + assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); + TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); + createTable( + CLUSTER, + invalid, + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(invalid); + assertThatThrownBy( + () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertUnchanged(invalid, before); + } + + private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { + GetTableInfoResponse after = info(path); + assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); + assertThat(after.getSchemaId()).isEqualTo(before.getSchemaId()); + } + + private static TableDescriptor.Builder jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + TableDescriptor.Builder builder = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); + // Omitted flat layout must remain omitted through CREATE and ALTER. + if ("nested".equals(layout)) { + builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout); + } + return builder; + } + @Test void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { TablePath path = TablePath.of("ddl", "roundtrip"); diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 491f7e83c6d..f5fb15b670b 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,14 +254,14 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. @@ -278,6 +278,12 @@ configuration. The prototype `kafka.database` option is no longer used. Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` column respectively. Flat JSON targets business columns at the table level; nested JSON targets the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +JSON business schemas support nested ROW, ARRAY and MAP values. Every MAP key must be STRING, +including maps inside other containers. CREATE and ALTER reject unsupported types or a nesting +depth above 64, counted from each top-level JSON business field at depth zero; ROW fields, +ARRAY elements and MAP values each add one level. The JSON root and the physical nested +`_kafka_value ROW` container add no level, so equivalent flat/nested schemas have the same limit. +Scalar value and nullability checks apply when decoding records. Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and collects unknown or reserved JSON members; it requires no mapping property. Binary/string From e7673be5719768ad5a72d4711a6f184188beed71 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 10:54:36 +0800 Subject: [PATCH 12/14] [kafka] Enable metadata mapping and fix table change notifications Allow optional _kafka_metadata with timestamp, timestamp_type and partition, without a physical offset field. Cover optional fixed-column combinations, metadata validation and CREATE/ALTER behavior. Ignore unchanged table registrations when schema ALTER advances the ZK version. Preserve atomic concurrency fencing and notifications for real property and bucket changes, with deterministic regression coverage. Validation: Java 11 reactor verify passed 111 targeted test cases; Checkstyle, Spotless and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 57/57 AI-Contributed/UT: 265/265 --- .../fluss/metadata/KafkaTableSchema.java | 25 +++-- .../fluss/metadata/KafkaTableSchemaTest.java | 93 ++++++++++++++----- .../kafka/schema/KafkaTableMappingITCase.java | 79 +++++++++++++++- .../schema/KafkaTopicSchemaResolverTest.java | 20 +++- .../event/watcher/TableChangeWatcher.java | 21 +++-- .../event/watcher/TableChangeWatcherTest.java | 73 ++++++++++++++- website/docs/maintenance/configuration.md | 11 ++- 7 files changed, 277 insertions(+), 45 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 990cec50416..9ab16195fbd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -93,6 +93,7 @@ public enum ValueLayout { private final KafkaFieldProjection valueProjection; private final int keyPosition; private final int headersPosition; + private final int metadataPosition; private final int rescuePosition; private KafkaTableSchema( @@ -102,6 +103,7 @@ private KafkaTableSchema( List valuePositions, int keyPosition, int headersPosition, + int metadataPosition, int rescuePosition) { this.rowType = rowType; this.valueType = valueType; @@ -109,6 +111,7 @@ private KafkaTableSchema( this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); this.keyPosition = keyPosition; this.headersPosition = headersPosition; + this.metadataPosition = metadataPosition; this.rescuePosition = rescuePosition; } @@ -216,9 +219,6 @@ public static KafkaTableSchema resolve(TableDescriptor table) { int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); if (metadataPosition >= 0) { validateMetadataType(rowType.getTypeAt(metadataPosition)); - throw invalid( - "_kafka_metadata requires native append support for persisting actual per-row offsets; " - + "this capability is not available yet."); } List valuePositions = new ArrayList<>(); @@ -296,6 +296,7 @@ public static KafkaTableSchema resolve(TableDescriptor table) { valuePositions, keyPosition, headersPosition, + metadataPosition, rescuePosition); } @@ -382,17 +383,15 @@ private static void validateOptions(TableDescriptor table) { private static void validateMetadataType(DataType type) { String message = "_kafka_metadata must be ROW."; + + "timestamp_type INT, partition INT>."; if (!(type instanceof RowType)) { throw invalid(message); } RowType row = (RowType) type; - if (!row.getFieldNames() - .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + if (!row.getFieldNames().equals(Arrays.asList("timestamp", "timestamp_type", "partition")) || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT())) { throw invalid(message); } } @@ -430,6 +429,14 @@ public int headersPosition() { return headersPosition; } + /** + * Returns the metadata column position, or -1 when absent. The ROW stores the input record + * timestamp, batch timestamp type and requested partition; log offsets are not row fields. + */ + public int metadataPosition() { + return metadataPosition; + } + /** * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and * reserved members here; binary/string decoders write null to this optional column. @@ -450,6 +457,7 @@ public boolean equals(Object other) { && valueProjection.equals(that.valueProjection) && keyPosition == that.keyPosition && headersPosition == that.headersPosition + && metadataPosition == that.metadataPosition && rescuePosition == that.rescuePosition; } @@ -462,6 +470,7 @@ public int hashCode() { valueProjection, keyPosition, headersPosition, + metadataPosition, rescuePosition); } diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 809f13ac553..00aacbd51e5 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -89,7 +89,7 @@ void testExplicitOptInAndDisabledTables() { @ValueSource(strings = {"binary", "string"}) void testFixedValueColumnWithOptionalSystemFields(String format) { DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); - for (int mask = 0; mask < 8; mask++) { + for (int mask = 0; mask < 16; mask++) { Schema.Builder schema = Schema.newBuilder().column("_kafka_value", valueType.copy(false)); if ((mask & 1) != 0) { @@ -102,6 +102,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { if ((mask & 4) != 0) { schema.column("_kafka_rescue", DataTypes.STRING()); } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType()); + } TableDescriptor table = enabled(schema.build(), format).build(); byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); @@ -111,6 +114,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); + assertThat(resolved.metadataPosition()) + .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( @@ -134,6 +140,7 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { Schema.newBuilder() .column("id", DataTypes.BIGINT().copy(false)) .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("payload", nested) .column("_kafka_key", DataTypes.BYTES()) .column( @@ -147,9 +154,10 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); - assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); TableDescriptor noRescue = enabled( @@ -174,6 +182,7 @@ void testNestedJsonTargetsOneRow() { enabled( Schema.newBuilder() .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("_kafka_value", row) .build(), "parse_json") @@ -181,9 +190,10 @@ void testNestedJsonTargetsOneRow() { .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); - assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().positions()).containsExactly(2); assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); assertThat(resolved.rescuePosition()).isZero(); + assertThat(resolved.metadataPosition()).isEqualTo(1); } @ParameterizedTest @@ -425,33 +435,63 @@ void testSystemColumnTypes() { } @Test - void testMetadataStructureAndUnavailableOffsetPersistence() { + void testMetadataPreservesDeclaredNullability() { RowType metadata = DataTypes.ROW( DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), - DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT())); - assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); - assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + for (boolean nullable : new boolean[] {true, false}) { + DataType type = metadata.copy(nullable); + TableDescriptor table = withSystem("_kafka_metadata", type); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolved); + } + } + + @Test + void testMetadataRejectsInvalidStructureAndOffsetField() { + for (DataType invalid : + Arrays.asList( + DataTypes.STRING(), DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT()))), - "TIMESTAMP_LTZ(3)"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("offset", DataTypes.BIGINT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(6)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.STRING()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.BIGINT())), DataTypes.ROW( - DataTypes.FIELD("offset", DataTypes.BIGINT()), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), - "must be ROW"); + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("bucket", DataTypes.INT())))) { + assertInvalid( + withSystem("_kafka_metadata", invalid), + "ROW"); + } } @Test @@ -510,6 +550,13 @@ void testTableKindsAndCanonicalNameLimits() { .hasMessageContaining("Kafka topic"); } + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private static TableDescriptor withSystem(String name, DataType type) { return enabled( Schema.newBuilder() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 76b9c0f773d..97e75987f89 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -444,9 +444,62 @@ void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exceptio assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + Schema schema = + Schema.newBuilder() + .column("_kafka_metadata", metadataType()) + .column( + "_kafka_value", + "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + TablePath created = TablePath.of("ddl", "metadata_create_" + format); + createTable(CLUSTER, created, descriptor); + TableDescriptor persisted = stored(created); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + + TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + createTable( + CLUSTER, + altered, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + int schemaId = info(altered).getSchemaId(); + set(altered, Collections.singletonMap("table.kafka.enabled", "true")); + persisted = stored(altered); + assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + } + @Test - void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { - TablePath path = TablePath.of("ddl", "offset_capability"); + void testMetadataCanBeAddedToEnabledTable() throws Exception { + TablePath path = TablePath.of("ddl", "metadata_add"); + createTable(CLUSTER, path, enabled().build()); + TableDescriptor before = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + TableDescriptor after = stored(path); + KafkaTableSchema resolved = KafkaTableSchema.resolve(after); + assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) + .isEqualTo(metadataType()); + assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + } + + @Test + void testMetadataOffsetFieldRejectedOnCreateAndEnable() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl", true)).get(); + TablePath path = TablePath.of("ddl", "metadata_with_offset"); Schema schema = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) @@ -458,6 +511,18 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { DataTypes.FIELD("partition", DataTypes.INT()), DataTypes.FIELD("offset", DataTypes.BIGINT()))) .build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled().schema(schema).build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining( + "ROW"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); createTable( CLUSTER, path, @@ -468,7 +533,8 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { TableDescriptor before = stored(path); assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("actual per-row offsets"); + .hasStackTraceContaining( + "ROW"); assertThat(stored(path)).isEqualTo(before); } @@ -495,6 +561,13 @@ path, enabled().build(), false)) assertThat(stored(path)).isEqualTo(before); } + private static DataType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private GetTableInfoResponse info(TablePath path) throws Exception { return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index b7e4174421c..8327c6fb60e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -34,9 +34,27 @@ class KafkaTopicSchemaResolverTest { @Test void testEnabledContractUsesSharedValidation() { - TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + TableDescriptor table = + table().schema( + Schema.newBuilder() + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(); assertThat(resolver.isEnabled(table)).isTrue(); assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(table).metadataPosition()).isZero(); + assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) .isEqualTo(resolver.resolve(table)); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index 5e05877b4be..f38a7f0afe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,6 +17,7 @@ package org.apache.fluss.server.coordinator.event.watcher; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -87,7 +88,8 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - private final class TablePathChangeListener implements CuratorCacheListener { + @VisibleForTesting + final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -150,13 +152,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and properties change. + // Distinguish between table creation and registration updates. // If oldData exists and contains valid table registration data, - // it's a properties change; otherwise, it's a table creation. + // compare its contents; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, newData); + processTableRegistrationChange(tablePath, oldData, newData); } else { processCreateTable(tablePath, newData); } @@ -276,9 +278,16 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { + private void processTableRegistrationChange( + TablePath tablePath, ChildData oldData, ChildData newData) { + TableRegistration oldTable = TableZNode.decode(oldData.getData()); TableRegistration newTable = TableZNode.decode(newData.getData()); - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + // Schema ALTER bumps the table znode version to fence concurrent registration updates, + // even when its contents are unchanged. Compare decoded registrations so differences + // in JSON serialization do not trigger a registration change either. + if (!oldTable.equals(newTable)) { + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + } } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 28a4131ecd8..9a36280d891 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,11 +42,16 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -56,9 +61,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -302,7 +311,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() { + void testSchemaChanges() throws Exception { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -349,6 +358,8 @@ void testSchemaChanges() { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); + VersionedData beforeAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -359,6 +370,10 @@ void testSchemaChanges() { TableChange.ColumnPosition.last())), false, null); + VersionedData afterAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); + assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); + assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -378,6 +393,62 @@ void testSchemaChanges() { .containsExactlyInAnyOrderElementsOf(allEvents)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnchangedTableRegistration(boolean differentJsonFormatting) { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + byte[] oldData = TableZNode.encode(table); + byte[] newData = + differentJsonFormatting + ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) + .getBytes(StandardCharsets.UTF_8) + : oldData; + + // Deliver the callback synchronously so the absence of an event cannot pass before + // CuratorCache has observed the version bump. + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, oldData, 1), + tableNodeData(tablePath, newData, 2)); + + assertThat(eventManager.getEvents()).isEmpty(); + } + + @Test + void testChangedTableRegistrationContents() { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + List expectedEvents = new ArrayList<>(); + for (TableRegistration updatedTable : + Arrays.asList( + table.newProperties( + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), + table.customProperties), + table.newProperties( + table.properties, + Collections.singletonMap("custom.key", "custom.value")), + table.newBucketCount(4))) { + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, TableZNode.encode(table), 1), + tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); + expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); + assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); + } + } + + private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { + Stat stat = new Stat(); + stat.setVersion(version); + return new ChildData(TableZNode.path(tablePath), stat, data); + } + @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index f5fb15b670b..6344eee5207 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -295,7 +295,12 @@ disabled, ordinary Fluss tables are not subject to Kafka-specific schema validat accepted on enabled tables. The `_kafka_metadata` contract is -`ROW`. -Enabling tables with this column is currently rejected until native append can assign and -persist each row's actual offset. Other optional system columns can be used independently. +`ROW`. +This optional physical column stores the input Kafka record timestamp, the batch timestamp type +(`-1` for no timestamp, `0` for CreateTime, `1` for LogAppendTime), and the requested partition. +Without a timestamp, the timestamp field is null; declared NOT NULL constraints still apply +when converting records. These values are available before Arrow encoding. Log offsets remain +native log metadata and are not part of this ROW, so enabling the column requires no append-time +offset backfill. CREATE and ALTER reject extra fields, including the former offset field, and +incorrect field names, ordering or types. These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From b4e5a021235e5763b5ed2a07da640bdd736d0122 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 17:17:18 +0800 Subject: [PATCH 13/14] [kafka] Focus DDL mapping changes and streamline tests Create the configured Kafka default database when missing and preserve existing metadata. Fence schema and property ALTER with schema-parent version checks without rewriting table registrations, restoring the original table watcher. Remove redundant configuration, Flink conversion and mapping test coverage while retaining DDL validation, optional system columns and concurrent ALTER checks. Validation: targeted clean verify passed 173 tests with Checkstyle and Spotless. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 120/120 AI-Contributed/UT: 683/683 --- .../apache/fluss/config/ConfigOptions.java | 3 +- .../fluss/metadata/KafkaTableSchemaTest.java | 83 +----- .../flink/utils/FlinkConversionsTest.java | 45 ---- .../apache/fluss/kafka/KafkaConfigsTest.java | 59 ---- .../kafka/schema/KafkaTableMappingITCase.java | 255 ++++-------------- .../schema/KafkaTopicSchemaResolverTest.java | 28 -- .../server/coordinator/CoordinatorServer.java | 16 ++ .../server/coordinator/MetadataManager.java | 20 +- .../event/watcher/TableChangeWatcher.java | 21 +- .../fluss/server/zk/ZooKeeperClient.java | 54 +++- .../coordinator/KafkaDefaultDatabaseTest.java | 94 +++++++ .../KafkaTableSchemaAlterTest.java | 46 +++- .../event/watcher/TableChangeWatcherTest.java | 73 +---- website/docs/maintenance/configuration.md | 6 +- 14 files changed, 275 insertions(+), 528 deletions(-) delete mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 66f908d74d0..e05d981e995 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2943,7 +2943,8 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. Databases must already exist; this option does not create them."); + + "access other databases. When Kafka is enabled, the coordinator creates this " + + "database if it does not exist."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 00aacbd51e5..458242feadd 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -18,9 +18,6 @@ package org.apache.fluss.metadata; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.FlussConfigUtils; -import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; import org.apache.fluss.metadata.KafkaTableSchema.ValueType; @@ -33,7 +30,6 @@ import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; -import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -42,22 +38,6 @@ /** Tests the shared DDL and Kafka conversion contract. */ class KafkaTableSchemaTest { - @Test - void testRegisteredAlterableOptionsAndReadTimeDefaults() { - for (String key : - Arrays.asList( - "table.kafka.enabled", - "table.kafka.convert.value.type", - "table.kafka.convert.value.layout", - "table.kafka.convert.key.type")) { - assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); - assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); - } - Configuration config = new Configuration(); - assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); - assertThat(config.toMap()).isEmpty(); - } - @Test void testExplicitOptInAndDisabledTables() { TableDescriptor disabled = @@ -106,35 +86,22 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { schema.column("_kafka_metadata", metadataType()); } TableDescriptor table = enabled(schema.build(), format).build(); - byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueProjection().positions()).containsExactly(0); assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); - assertThat(resolved.valueLayout()).isNull(); assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); - assertThat(resolved.metadataPosition()) - .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); - assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(resolved); } } @Test void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { - RowType nested = - DataTypes.ROW( - DataTypes.FIELD("_kafka_key", DataTypes.STRING()), - DataTypes.FIELD( - "items", - DataTypes.ARRAY( - DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + RowType nested = DataTypes.ROW(DataTypes.FIELD("_kafka_key", DataTypes.STRING())); TableDescriptor table = enabled( Schema.newBuilder() @@ -159,14 +126,6 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { assertThat(resolved.rescuePosition()).isEqualTo(1); assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); - TableDescriptor noRescue = - enabled( - Schema.newBuilder() - .column("custom_rescue", DataTypes.STRING()) - .build(), - "parse_json") - .build(); - assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); } @Test @@ -209,15 +168,6 @@ void testJsonRejectsNonStringMapKeysRecursively(String layout) { DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { TableDescriptor table = jsonTable(layout, type); assertInvalid(table, "only supports STRING map keys"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(table) - .property( - ConfigOptions.TABLE_KAFKA_ENABLED, - false) - .build())) - .doesNotThrowAnyException(); } } } @@ -250,11 +200,7 @@ void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { DataTypes.ROW( DataTypes.FIELD("_kafka_key", scalar.copy(false))))); TableDescriptor table = jsonTable(layout, type); - byte[] before = table.toJsonBytes(); assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); - assertThat(table.toJsonBytes()).isEqualTo(before); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(KafkaTableSchema.resolve(table)); } } @@ -276,13 +222,6 @@ void testJsonBusinessNestingDepthBoundary(String layout) { assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); assertInvalid(tooDeep, "maximum nesting depth of 64"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build())) - .doesNotThrowAnyException(); } private static TableDescriptor jsonTable(String layout, DataType businessType) { @@ -325,7 +264,7 @@ void testReservedNamesAtValueMappingLevel(String name) { } @ParameterizedTest - @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + @ValueSource(strings = {"raw", "avro", ""}) void testRejectsUnsupportedValueTypes(String type) { assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); } @@ -446,8 +385,6 @@ void testMetadataPreservesDeclaredNullability() { TableDescriptor table = withSystem("_kafka_metadata", type); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolved); } } @@ -522,7 +459,7 @@ void testUnknownAndPrototypeOptions() { } @Test - void testTableKindsAndCanonicalNameLimits() { + void testUnsupportedTableKinds() { Schema keyed = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES().copy(false)) @@ -534,20 +471,6 @@ void testTableKindsAndCanonicalNameLimits() { "non-partitioned"); assertInvalid( enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); - String database = String.join("", Collections.nCopies(124, "d")); - String tableName = String.join("", Collections.nCopies(124, "t")); - TableDescriptor table = enabled(valueSchema(), "binary").build(); - assertThatCode( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName), table)) - .doesNotThrowAnyException(); - assertThatThrownBy( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName + "t"), table)) - .isInstanceOf(InvalidTableException.class) - .hasMessageContaining("Kafka topic"); } private static RowType metadataType() { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 56e990d8cd5..67f843a7276 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,7 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -76,50 +75,6 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { - @Test - void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { - ResolvedSchema schema = - new ResolvedSchema( - Arrays.asList( - Column.physical( - "_kafka_value", - org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_headers", - org.apache.flink.table.api.DataTypes.MAP( - org.apache.flink.table.api.DataTypes.STRING(), - org.apache.flink.table.api.DataTypes.BYTES())), - Column.physical( - "_kafka_rescue", - org.apache.flink.table.api.DataTypes.STRING())), - Collections.emptyList(), - null); - Map options = new HashMap<>(); - options.put("table.kafka.enabled", "true"); - options.put("table.kafka.convert.value.type", "binary"); - CatalogTable table = - CatalogTable.of( - Schema.newBuilder().fromResolvedSchema(schema).build(), - "Kafka DDL", - Collections.emptyList(), - options); - TableDescriptor descriptor = - FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); - assertThat(descriptor.getProperties()) - .containsAllEntriesOf(options) - .doesNotContainKeys( - "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(descriptor.getCustomProperties()) - .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); - KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.keyPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping.rescuePosition()).isEqualTo(3); - } - @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java deleted file mode 100644 index 17a8c6c0f9c..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for Kafka configuration. */ -public class KafkaConfigsTest { - @Test - public void testFromMap() throws Exception { - Map map = new HashMap<>(); - map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); - map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); - Configuration configuration = Configuration.fromMap(map); - - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("sales"); - } - - @Test - public void testFromDefault() throws Exception { - Configuration configuration = Configuration.fromMap(new HashMap<>()); - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); - assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); - } -} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 97e75987f89..a327df3d5a6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -19,8 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.DatabaseNotExistException; -import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.metadata.KafkaTableSchema; @@ -38,6 +36,7 @@ import org.apache.fluss.utils.json.DataTypeJsonSerde; import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -67,7 +66,7 @@ class KafkaTableMappingITCase { static { CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); - CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka_default"); CONFIG.setString("kafka.database", "legacy_database"); } @@ -77,14 +76,24 @@ class KafkaTableMappingITCase { private CoordinatorGateway gateway; + @BeforeAll + static void verifyConfiguredDatabaseCreatedOnStartup() throws Exception { + assertThat( + CLUSTER.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("kafka_default")) + .get() + .isExists()) + .isTrue(); + } + @BeforeEach void setup() { gateway = CLUSTER.newCoordinatorClient(); } @Test - void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { - for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + void testConfiguredKafkaDatabaseAndQualifiedAccess() throws Exception { + for (String database : Arrays.asList("kafka", "legacy_database")) { assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) .isFalse(); } @@ -99,22 +108,16 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro long salesId = createTable(CLUSTER, sales, enabled().build()); long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); assertThat(salesId).isNotEqualTo(warehouseId); - assertThat(mapper.toTablePath("orders")) - .isEqualTo(TablePath.of("missing_default", "orders")); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - mapper.toTablePath("orders"), - enabled().build(), - false)) - .get()) - .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("kafka_default", "orders")); + // The cluster fixture removes custom databases between tests; recreate this table's + // database through the usual DDL helper after checking startup creation in @BeforeAll. + createTable(CLUSTER, mapper.toTablePath("orders"), enabled().build()); assertThat( - gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + gateway.databaseExists(newDatabaseExistsRequest("kafka_default")) .get() .isExists()) - .isFalse(); + .isTrue(); + assertThat(KafkaTableSchema.isEnabled(stored(mapper.toTablePath("orders")))).isTrue(); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); @@ -123,60 +126,36 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } - @Test - void testDefaultAliasUsesExistingFlussDatabase() throws Exception { - KafkaTopicMapper mapper = new KafkaTopicMapper(); - TablePath path = mapper.toTablePath("default_alias"); - assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); - createTable(CLUSTER, path, enabled().build()); - assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) - .isEqualTo(info(path).getTableId()); - assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); - } - @ParameterizedTest @ValueSource(strings = {"flat", "nested"}) void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); - DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); - int index = 0; - for (DataType type : - Arrays.asList( - invalidMap, - DataTypes.ARRAY(invalidMap), - DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), - DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { - TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); - TableDescriptor descriptor = jsonTable(layout, type).build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest(path, descriptor, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - - // The identical schema is legal for an ordinary, explicitly disabled Fluss table. - createTable( - CLUSTER, - path, - TableDescriptor.builder(descriptor) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(path); - assertThatThrownBy( - () -> - set( - path, - Collections.singletonMap( - "table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertUnchanged(path, before); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); - } + TablePath path = TablePath.of("ddl_json", layout + "_invalid"); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + TableDescriptor descriptor = jsonTable(layout, invalid).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertUnchanged(path, before); } @ParameterizedTest @@ -238,41 +217,6 @@ void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception assertUnchanged(path, disabled); } - @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testJsonDepthValidationThroughDdl(String layout) throws Exception { - DataType boundary = DataTypes.STRING(); - for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { - boundary = DataTypes.ARRAY(boundary); - } - TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); - TableDescriptor descriptor = jsonTable(layout, boundary).build(); - createTable(CLUSTER, valid, descriptor); - assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); - TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); - TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); - assertThatThrownBy( - () -> - gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); - createTable( - CLUSTER, - invalid, - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(invalid); - assertThatThrownBy( - () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertUnchanged(invalid, before); - } - private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { GetTableInfoResponse after = info(path); assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); @@ -374,10 +318,12 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); - TableDescriptor withHeaders = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + assertThat(KafkaTableSchema.resolve(stored(path)).metadataPosition()).isEqualTo(2); + TableDescriptor beforeFailedRescue = stored(path); assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) .hasCauseInstanceOf(InvalidTableException.class); - assertThat(stored(path)).isEqualTo(withHeaders); + assertThat(stored(path)).isEqualTo(beforeFailedRescue); set(path, Collections.singletonMap("table.kafka.enabled", "false")); addColumn(path, "unmapped_business_field", DataTypes.INT()); assertThat(stored(path).getSchema().getRowType().getFieldNames()) @@ -388,85 +334,18 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio } @Test - void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { - gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); - TablePath path = TablePath.of("ddl_invalid", "bad"); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_TYPE, - "string") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - // Non-null added columns are rejected by generic schema evolution first, so exercise - // Kafka-specific rescue nullability through CREATE, where such columns are allowed. - TableDescriptor nonNullableRescue = - enabled() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("_kafka_rescue", DataTypes.STRING().copy(false)) - .build()) - .build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, nonNullableRescue, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("nullable STRING"); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - "table.kafka.convert.rescue-column", - "rescue") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidConfigException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - createTable(CLUSTER, path, enabled().build()); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); - } - - @ParameterizedTest - @ValueSource(strings = {"binary", "string"}) - void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + void testMetadataCanBeCreatedAndEnabledWithoutOffset() throws Exception { Schema schema = Schema.newBuilder() .column("_kafka_metadata", metadataType()) - .column( - "_kafka_value", - "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) - .build(); - TableDescriptor descriptor = - enabled() - .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .column("_kafka_value", DataTypes.BYTES()) .build(); - TablePath created = TablePath.of("ddl", "metadata_create_" + format); + TableDescriptor descriptor = enabled().schema(schema).build(); + TablePath created = TablePath.of("ddl", "metadata_create"); createTable(CLUSTER, created, descriptor); - TableDescriptor persisted = stored(created); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + assertThat(stored(created).getSchema()).isEqualTo(schema); - TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + TablePath altered = TablePath.of("ddl", "metadata_enable"); createTable( CLUSTER, altered, @@ -475,25 +354,9 @@ void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Excep .build()); int schemaId = info(altered).getSchemaId(); set(altered, Collections.singletonMap("table.kafka.enabled", "true")); - persisted = stored(altered); assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); - } - - @Test - void testMetadataCanBeAddedToEnabledTable() throws Exception { - TablePath path = TablePath.of("ddl", "metadata_add"); - createTable(CLUSTER, path, enabled().build()); - TableDescriptor before = stored(path); - addColumn(path, "_kafka_metadata", metadataType()); - TableDescriptor after = stored(path); - KafkaTableSchema resolved = KafkaTableSchema.resolve(after); - assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); - assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) - .isEqualTo(metadataType()); - assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + assertThat(stored(altered).getSchema()).isEqualTo(schema); + assertThat(KafkaTableSchema.resolve(stored(altered)).metadataPosition()).isZero(); } @Test diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 8327c6fb60e..119e8aff38f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -18,7 +18,6 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.DataTypes; @@ -32,33 +31,6 @@ class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); - @Test - void testEnabledContractUsesSharedValidation() { - TableDescriptor table = - table().schema( - Schema.newBuilder() - .column( - "_kafka_metadata", - DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD( - "timestamp_type", DataTypes.INT()), - DataTypes.FIELD( - "partition", DataTypes.INT()))) - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .build(); - assertThat(resolver.isEnabled(table)).isTrue(); - assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); - assertThat(resolver.resolve(table).metadataPosition()).isZero(); - assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); - assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolver.resolve(table)); - } - @Test void testCompatibleSchemaDoesNotEnableTable() { assertThat(resolver.isEnabled(table().build())).isFalse(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index aff8bb78761..ac161728107 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -548,11 +548,27 @@ private interface ThrowingRunnable { private void createDefaultDatabase() { MetadataManager metadataManager = new MetadataManager(zkClient, conf, lakeCatalogDynamicLoader); + createDefaultDatabases(metadataManager, conf); + } + + @VisibleForTesting + static void createDefaultDatabases(MetadataManager metadataManager, Configuration conf) { List databases = metadataManager.listDatabases(); if (databases.isEmpty()) { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } + if (conf.get(ConfigOptions.KAFKA_ENABLED)) { + String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + // An empty cluster has already created the native default database above. + if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { + return; + } + if (!databases.contains(kafkaDatabase)) { + metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); + LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); + } + } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index b1a5fae3d04..3354b869eb6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -474,7 +474,7 @@ public void alterTableSchema( ZkVersion.MATCH_ANY_VERSION.getVersion()); } - /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ public void alterTableSchema( TablePath tablePath, List schemaChanges, @@ -485,6 +485,7 @@ public void alterTableSchema( try { ZooKeeperClient.VersionedData versionedTable = getTableRegistrationWithVersion(tablePath); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -505,11 +506,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaAndBumpTableVersion( + zookeeperClient.registerSchemaWithVersionCheck( tablePath, newSchema, table.getSchemaId() + 1, - versionedTable, + versionedTable.zkVersion(), + schemaZkVersion, coordinatorZkVersion); } else { LOG.info( @@ -813,10 +815,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table - * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} - * may apply external lake side effects, a version conflict is surfaced instead of being retried - * automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and + * schema versions read here plus the coordinator epoch version. Since {@link + * #preAlterTableProperties} may apply external lake side effects, a version conflict is + * surfaced instead of being retried automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -832,6 +834,7 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -901,7 +904,8 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion); + coordinatorZkVersion, + schemaZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index f38a7f0afe4..5e05877b4be 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,7 +17,6 @@ package org.apache.fluss.server.coordinator.event.watcher; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -88,8 +87,7 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - @VisibleForTesting - final class TablePathChangeListener implements CuratorCacheListener { + private final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -152,13 +150,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and registration updates. + // Distinguish between table creation and properties change. // If oldData exists and contains valid table registration data, - // compare its contents; otherwise, it's a table creation. + // it's a properties change; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, oldData, newData); + processTableRegistrationChange(tablePath, newData); } else { processCreateTable(tablePath, newData); } @@ -278,16 +276,9 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange( - TablePath tablePath, ChildData oldData, ChildData newData) { - TableRegistration oldTable = TableZNode.decode(oldData.getData()); + private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { TableRegistration newTable = TableZNode.decode(newData.getData()); - // Schema ALTER bumps the table znode version to fence concurrent registration updates, - // even when its contents are unchanged. Compare decoded registrations so differences - // in JSON serialization do not trigger a registration change either. - if (!oldTable.equals(newTable)) { - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); - } + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index a7a706c214e..99d2c6c01ae 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,7 +1159,32 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { + updateTableWithPartitionBucketCountBackfill( + tablePath, + tableRegistration, + expectedTableZkVersion, + partitionBackfills, + expectedCoordinatorEpochZkVersion, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + + /** Updates table properties with an additional fence on the schema used for validation. */ + public void updateTableWithPartitionBucketCountBackfill( + TablePath tablePath, + TableRegistration tableRegistration, + int expectedTableZkVersion, + Map> partitionBackfills, + int expectedCoordinatorEpochZkVersion, + int expectedSchemaZkVersion) + throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); + if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath))); + } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1316,25 +1341,29 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro } /** - * Publishes a schema and advances the table registration version in one fenced transaction. - * Property ALTER validates against a schema snapshot and compares the table version, so schema - * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + * Publishes a schema and advances the schema version in one fenced transaction. The table + * registration is checked without being rewritten; property ALTER checks the schema version to + * prevent enabling a Kafka contract against a stale schema. */ - public void registerSchemaAndBumpTableVersion( + public void registerSchemaWithVersionCheck( TablePath tablePath, Schema schema, int schemaId, - VersionedData expectedTable, + int expectedTableZkVersion, + int expectedSchemaZkVersion, int expectedCoordinatorEpochZkVersion) throws Exception { List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedTableZkVersion) + .forPath(TableZNode.path(tablePath))); ops.add( zkClient.transactionOp() .setData() - .withVersion(expectedTable.zkVersion()) - .forPath( - TableZNode.path(tablePath), - TableZNode.encode(expectedTable.data()))); + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath), new byte[0])); ops.add( zkClient.transactionOp() .create() @@ -1345,6 +1374,13 @@ public void registerSchemaAndBumpTableVersion( .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); } + /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ + public int getSchemaZkVersion(TablePath tablePath) throws Exception { + Stat stat = new Stat(); + zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); + return stat.getVersion(); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java new file mode 100644 index 00000000000..d846c95f3e6 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode; +import org.apache.fluss.testutils.common.AllCallbackWrapper; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests startup database creation for native and Kafka default database configurations. */ +class KafkaDefaultDatabaseTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private MetadataManager manager; + + @BeforeEach + void setup() throws Exception { + ZooKeeperClient zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + if (zk.getStat(DatabasesZNode.path()).isPresent()) { + zk.getCuratorClient() + .delete() + .deletingChildrenIfNeeded() + .forPath(DatabasesZNode.path()); + } + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + } + + @ParameterizedTest + @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) + void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + DatabaseDescriptor descriptor = + DatabaseDescriptor.builder().comment("existing default").build(); + if (exists) { + manager.createDatabase(database, descriptor, false); + } + Configuration config = kafkaConfiguration(database); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactlyInAnyOrder("existing", database); + assertThat(manager.getDatabase(database).getDatabaseDescriptor().toJsonBytes()) + .isEqualTo((exists ? descriptor : DatabaseDescriptor.EMPTY).toJsonBytes()); + } + + @Test + void testDisabledKafkaDoesNotCreateConfiguredDatabase() { + Configuration config = kafkaConfiguration("kafka_default"); + config.set(ConfigOptions.KAFKA_ENABLED, false); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactly(DEFAULT_DATABASE); + } + + private static Configuration kafkaConfiguration(String database) { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + return config; + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java index e76dbda662d..dc08ed10385 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -21,7 +21,6 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -72,6 +71,9 @@ void setup() throws Exception { @Test void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { TablePath path = create("schema_wins"); + ZooKeeperClient.VersionedData before = + zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); assertThatThrownBy( () -> enable( @@ -91,6 +93,10 @@ void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { epoch))) .isInstanceOf(FlussRuntimeException.class) .hasMessageContaining("Concurrent metadata change"); + ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); + assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); + assertThat(after.data()).isEqualTo(before.data()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) .contains("unmapped"); @@ -101,22 +107,23 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { TablePath path = create("enable_wins"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema incompatible = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) .column("unmapped", DataTypes.STRING()) .build(); - // This candidate is valid for the disabled snapshot, but not after the competing enable. - KafkaTableSchema.validate( - TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) - .schema(incompatible) - .build()); enable(path, (oldTable, updated) -> {}); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, incompatible, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + incompatible, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); @@ -125,26 +132,39 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { } @Test - void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { TablePath path = create("atomic_failure"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema schema = manager.getTable(path).getSchema(); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.NodeExistsException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); zk.fenceBecomeCoordinatorLeader("next-coordinator"); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 9a36280d891..28a4131ecd8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,16 +42,11 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -61,13 +56,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -311,7 +302,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() throws Exception { + void testSchemaChanges() { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -358,8 +349,6 @@ void testSchemaChanges() throws Exception { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); - VersionedData beforeAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -370,10 +359,6 @@ void testSchemaChanges() throws Exception { TableChange.ColumnPosition.last())), false, null); - VersionedData afterAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); - assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); - assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -393,62 +378,6 @@ void testSchemaChanges() throws Exception { .containsExactlyInAnyOrderElementsOf(allEvents)); } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testUnchangedTableRegistration(boolean differentJsonFormatting) { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - byte[] oldData = TableZNode.encode(table); - byte[] newData = - differentJsonFormatting - ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) - .getBytes(StandardCharsets.UTF_8) - : oldData; - - // Deliver the callback synchronously so the absence of an event cannot pass before - // CuratorCache has observed the version bump. - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, oldData, 1), - tableNodeData(tablePath, newData, 2)); - - assertThat(eventManager.getEvents()).isEmpty(); - } - - @Test - void testChangedTableRegistrationContents() { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - List expectedEvents = new ArrayList<>(); - for (TableRegistration updatedTable : - Arrays.asList( - table.newProperties( - Collections.singletonMap( - ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), - table.customProperties), - table.newProperties( - table.properties, - Collections.singletonMap("custom.key", "custom.value")), - table.newBucketCount(4))) { - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, TableZNode.encode(table), 1), - tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); - expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); - assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); - } - } - - private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { - Stat stat = new Stat(); - stat.setVersion(version); - return new ChildData(TableZNode.path(tablePath), stat, data); - } - @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 6344eee5207..5f4a5ef345a 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,14 +254,16 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default +database when Kafka is enabled and the database is missing. Existing databases are preserved; +other databases and all tables must be created explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. From 7bcda8b4a17282b9e956137ea31ae281cbf5795b Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 20:11:19 +0800 Subject: [PATCH 14/14] [kafka] Simplify DDL validation and default database setup Reuse native initialization for the fluss database and create a custom Kafka default database only when it is missing. Remove PR03-only schema version and Coordinator epoch checks, preserving PR02 metadata writes and existing protections. Drop the corresponding concurrency tests while retaining Kafka DDL mapping validation. Validation: 121 focused tests passed; Maven verify checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 132/132 AI-Contributed/UT: 224/224 --- .../apache/fluss/config/ConfigOptions.java | 5 +- .../kafka/schema/KafkaTableMappingITCase.java | 2 +- .../server/coordinator/CoordinatorServer.java | 7 +- .../coordinator/CoordinatorService.java | 3 +- .../server/coordinator/MetadataManager.java | 41 +--- .../fluss/server/zk/ZooKeeperClient.java | 66 ------ .../coordinator/KafkaDefaultDatabaseTest.java | 20 +- .../KafkaTableSchemaAlterTest.java | 202 ------------------ website/docs/maintenance/configuration.md | 10 +- 9 files changed, 37 insertions(+), 319 deletions(-) delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index e05d981e995..6d6adb6a5ee 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2943,8 +2943,9 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. When Kafka is enabled, the coordinator creates this " - + "database if it does not exist."); + + "access other databases. When Kafka is enabled, a configured database other " + + "than 'fluss' is created if missing. The native 'fluss' database is created " + + "only when the cluster has no databases."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index a327df3d5a6..0c7ff17eda3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -271,7 +271,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { } @Test - void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + void testEnableDisableAndRejectInvalidProperties() throws Exception { TablePath path = TablePath.of("ddl", "enable_disable"); TableDescriptor disabled = enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index ac161728107..3b93e2c8c50 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -560,11 +560,8 @@ static void createDefaultDatabases(MetadataManager metadataManager, Configuratio } if (conf.get(ConfigOptions.KAFKA_ENABLED)) { String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); - // An empty cluster has already created the native default database above. - if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { - return; - } - if (!databases.contains(kafkaDatabase)) { + // The native default database follows the cluster initialization logic above. + if (!DEFAULT_DATABASE.equals(kafkaDatabase) && !databases.contains(kafkaDatabase)) { metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index e8e09dab26d..df8713338a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -615,8 +615,7 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal(), - coordinatorZkVersionSupplier.get()); + currentSession().getPrincipal()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 3354b869eb6..41259eed034 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -59,7 +59,6 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -466,26 +465,8 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - alterTableSchema( - tablePath, - schemaChanges, - ignoreIfNotExists, - flussPrincipal, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ - public void alterTableSchema( - TablePath tablePath, - List schemaChanges, - boolean ignoreIfNotExists, - FlussPrincipal flussPrincipal, - int coordinatorZkVersion) - throws TableNotExistException, TableNotPartitionedException { try { - ZooKeeperClient.VersionedData versionedTable = - getTableRegistrationWithVersion(tablePath); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); + TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -506,13 +487,7 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaWithVersionCheck( - tablePath, - newSchema, - table.getSchemaId() + 1, - versionedTable.zkVersion(), - schemaZkVersion, - coordinatorZkVersion); + zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -815,10 +790,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and - * schema versions read here plus the coordinator epoch version. Since {@link - * #preAlterTableProperties} may apply external lake side effects, a version conflict is - * surfaced instead of being retried automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table + * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} + * may apply external lake side effects, a version conflict is surfaced instead of being retried + * automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -834,7 +809,6 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -904,8 +878,7 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion, - schemaZkVersion); + coordinatorZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 99d2c6c01ae..01f897cff29 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,32 +1159,7 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { - updateTableWithPartitionBucketCountBackfill( - tablePath, - tableRegistration, - expectedTableZkVersion, - partitionBackfills, - expectedCoordinatorEpochZkVersion, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Updates table properties with an additional fence on the schema used for validation. */ - public void updateTableWithPartitionBucketCountBackfill( - TablePath tablePath, - TableRegistration tableRegistration, - int expectedTableZkVersion, - Map> partitionBackfills, - int expectedCoordinatorEpochZkVersion, - int expectedSchemaZkVersion) - throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); - if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath))); - } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1340,47 +1315,6 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } - /** - * Publishes a schema and advances the schema version in one fenced transaction. The table - * registration is checked without being rewritten; property ALTER checks the schema version to - * prevent enabling a Kafka contract against a stale schema. - */ - public void registerSchemaWithVersionCheck( - TablePath tablePath, - Schema schema, - int schemaId, - int expectedTableZkVersion, - int expectedSchemaZkVersion, - int expectedCoordinatorEpochZkVersion) - throws Exception { - List ops = new ArrayList<>(); - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedTableZkVersion) - .forPath(TableZNode.path(tablePath))); - ops.add( - zkClient.transactionOp() - .setData() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath), new byte[0])); - ops.add( - zkClient.transactionOp() - .create() - .withMode(CreateMode.PERSISTENT) - .forPath( - SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); - zkClient.transaction() - .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); - } - - /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ - public int getSchemaZkVersion(TablePath tablePath) throws Exception { - Stat stat = new Stat(); - zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); - return stat.getVersion(); - } - /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java index d846c95f3e6..5e4cc8bc3a5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.assertj.core.api.Assertions.assertThat; @@ -61,8 +61,22 @@ void setup() throws Exception { } @ParameterizedTest - @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) - void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + @ValueSource(booleans = {false, true}) + void testNativeDefaultFollowsClusterInitialization(boolean hasDatabase) { + if (hasDatabase) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + } + Configuration config = kafkaConfiguration(DEFAULT_DATABASE); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()) + .containsExactly(hasDatabase ? "existing" : DEFAULT_DATABASE); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreateMissingCustomDefaultAndPreserveExisting(boolean exists) { + String database = "kafka_default"; manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); DatabaseDescriptor descriptor = DatabaseDescriptor.builder().comment("existing default").build(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java deleted file mode 100644 index dc08ed10385..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.coordinator; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.FlussRuntimeException; -import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.Schema; -import org.apache.fluss.metadata.TableChange; -import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.server.entity.TablePropertyChanges; -import org.apache.fluss.server.zk.NOPErrorHandler; -import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperExtension; -import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; -import org.apache.fluss.testutils.common.AllCallbackWrapper; -import org.apache.fluss.types.DataTypes; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Collections; -import java.util.function.BiConsumer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ -class KafkaTableSchemaAlterTest { - - @RegisterExtension - public static final AllCallbackWrapper ZK = - new AllCallbackWrapper<>(new ZooKeeperExtension()); - - private ZooKeeperClient zk; - private MetadataManager manager; - private int epoch; - - @BeforeEach - void setup() throws Exception { - zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); - epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); - manager = - new MetadataManager( - zk, - new Configuration(), - new LakeCatalogDynamicLoader(new Configuration(), null, true)); - manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); - } - - @Test - void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { - TablePath path = create("schema_wins"); - ZooKeeperClient.VersionedData before = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - assertThatThrownBy( - () -> - enable( - path, - (oldTable, updated) -> - manager.alterTableSchema( - path, - Collections.singletonList( - TableChange.addColumn( - "unmapped", - DataTypes.STRING(), - null, - TableChange.ColumnPosition - .last())), - false, - null, - epoch))) - .isInstanceOf(FlussRuntimeException.class) - .hasMessageContaining("Concurrent metadata change"); - ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); - assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); - assertThat(after.data()).isEqualTo(before.data()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .contains("unmapped"); - } - - @Test - void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { - TablePath path = create("enable_wins"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema incompatible = - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("unmapped", DataTypes.STRING()) - .build(); - enable(path, (oldTable, updated) -> {}); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - incompatible, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .containsExactly("_kafka_value"); - } - - @Test - void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { - TablePath path = create("atomic_failure"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema schema = manager.getTable(path).getSchema(); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.NodeExistsException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - zk.fenceBecomeCoordinatorLeader("next-coordinator"); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - } - - private TablePath create(String name) { - TablePath path = TablePath.of("ddl", name); - TableDescriptor table = - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(1) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") - .build() - .withReplicationFactor(1); - manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); - return path; - } - - private void enable( - TablePath path, - BiConsumer beforeUpdate) { - TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); - changes.setTableProperty("table.kafka.enabled", "true"); - manager.alterTableProperties( - path, - Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), - changes.build(), - false, - null, - beforeUpdate, - (oldTable, updated) -> {}, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } -} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 5f4a5ef345a..b183783fd27 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -254,16 +254,18 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, a configured database other than `fluss` is created if missing. The native `fluss` database is created only when the cluster has no databases. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default -database when Kafka is enabled and the database is missing. Existing databases are preserved; -other databases and all tables must be created explicitly. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The native `fluss` database is created only +when the cluster has no databases; Kafka reuses this database when its default is `fluss`. +If Kafka is enabled and a different default database is configured, the coordinator creates it +if missing. Existing databases are preserved; other databases and all tables must be created +explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used.