From a4fba444c102fa8dce97c91e639f437960ef82e7 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 16:09:12 +0200 Subject: [PATCH 1/4] refactor: read the first value of a constant array once ConstantEncodingEncoder#isConstant and #readFirstRaw each read element 0 (and, for floats, converted it to raw bits) again; constantBits does both in one pass, with the first value in a local outside the loop and the empty array an explicit early return. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Sko6B2519bE1TkpPx7WKxf --- .../encode/ConstantEncodingEncoder.java | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index a4ea6d84..0aabb4ad 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -8,6 +8,7 @@ import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; +import java.util.OptionalLong; /// Write-only encoder for `vortex.constant`. public final class ConstantEncodingEncoder implements EncodingEncoder { @@ -64,10 +65,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "encode only supports Primitive or Bool dtype, got " + dtype); } PType ptype = p.ptype(); - if (!isConstant(data, ptype)) { - throw new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array"); - } - long firstRaw = readFirstRaw(data, ptype); + long firstRaw = constantBits(data, ptype) + .orElseThrow(() -> new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array")); ProtoScalarValue scalar = buildScalar(ptype, firstRaw); byte[] scalarBytes = scalar.encode(); // A constant array's min and max are both the one repeated value, by construction -- no @@ -91,7 +90,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC } return CascadeStep.terminal(encode(bool, data, encodeCtx)); } - if (!isConstant(data, ((DType.Primitive) dtype).ptype())) { + if (constantBits(data, ((DType.Primitive) dtype).ptype()).isEmpty()) { return CascadeStep.notApplicable(); } return CascadeStep.terminal(encode(dtype, data, encodeCtx)); @@ -138,73 +137,78 @@ private static boolean isConstantBool(boolean[] data) { return true; } - private static long readFirstRaw(Object data, PType ptype) { - return switch (ptype) { - case I8, U8 -> ((byte[]) data).length > 0 ? ((byte[]) data)[0] : 0L; - case I16, U16 -> ((short[]) data).length > 0 ? ((short[]) data)[0] : 0L; - case I32, U32 -> ((int[]) data).length > 0 ? ((int[]) data)[0] : 0L; - case I64, U64 -> ((long[]) data).length > 0 ? ((long[]) data)[0] : 0L; - case F32 -> ((float[]) data).length > 0 ? Float.floatToRawIntBits(((float[]) data)[0]) : 0L; - case F64 -> ((double[]) data).length > 0 ? Double.doubleToRawLongBits(((double[]) data)[0]) : 0L; - default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); - }; - } - - // One typed loop per width, the ptype switch hoisted out (CLAUDE.md hot-loop rule). Floats - // compare raw bits, so distinct NaN payloads or -0.0 vs 0.0 are not constant. - private static boolean isConstant(Object data, PType ptype) { + // The raw bits of the one repeated value, or empty if the array is not constant. An empty array + // is constant (bits 0), as `expectedRatio` already treats valueCount 0. One typed loop per width, + // the ptype switch hoisted out (CLAUDE.md hot-loop rule) and the first element read once + // outside the loop. Floats compare raw bits, so distinct NaN payloads or -0.0 vs 0.0 are not + // constant. + private static OptionalLong constantBits(Object data, PType ptype) { + if (Array.getLength(data) == 0) { + return OptionalLong.of(0L); + } switch (ptype) { case I8, U8 -> { byte[] a = (byte[]) data; + byte first = a[0]; for (int i = 1; i < a.length; i++) { - if (a[i] != a[0]) { - return false; + if (a[i] != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } case I16, U16 -> { short[] a = (short[]) data; + short first = a[0]; for (int i = 1; i < a.length; i++) { - if (a[i] != a[0]) { - return false; + if (a[i] != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } case I32, U32 -> { int[] a = (int[]) data; + int first = a[0]; for (int i = 1; i < a.length; i++) { - if (a[i] != a[0]) { - return false; + if (a[i] != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } case I64, U64 -> { long[] a = (long[]) data; + long first = a[0]; for (int i = 1; i < a.length; i++) { - if (a[i] != a[0]) { - return false; + if (a[i] != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } case F32 -> { float[] a = (float[]) data; + int first = Float.floatToRawIntBits(a[0]); for (int i = 1; i < a.length; i++) { - if (Float.floatToRawIntBits(a[i]) != Float.floatToRawIntBits(a[0])) { - return false; + if (Float.floatToRawIntBits(a[i]) != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } case F64 -> { double[] a = (double[]) data; + long first = Double.doubleToRawLongBits(a[0]); for (int i = 1; i < a.length; i++) { - if (Double.doubleToRawLongBits(a[i]) != Double.doubleToRawLongBits(a[0])) { - return false; + if (Double.doubleToRawLongBits(a[i]) != first) { + return OptionalLong.empty(); } } + return OptionalLong.of(first); } default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); } - return true; } private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { From 21ab5f5e7c96e066db6401659e48de9d07726647 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 16:16:40 +0200 Subject: [PATCH 2/4] fix: constant encoding handles F16 columns (#515) ConstantEncodingEncoder and ConstantEncodingDecoder had no F16 case, so a constant F16 column failed to write. The scalar is Rust's f16_value (the half bits as a u64); the decoder also accepts the legacy uint64_value form. The ptype switches lose their default branch, so a future PType fails to compile instead of at write time. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Sko6B2519bE1TkpPx7WKxf --- CHANGELOG.md | 1 + .../array/LazyConstantFloat16Array.java | 50 ++++++++++++ .../decode/ConstantEncodingDecoder.java | 6 ++ .../reader/array/LazyConstantArrayTest.java | 38 +++++++++ .../encode/ConstantEncodingEncoder.java | 42 ++++++---- .../vortex/writer/ConstantF16ColumnTest.java | 58 ++++++++++++++ .../encode/ConstantEncodingEncoderTest.java | 77 +++++++++++++++++++ 7 files changed, 255 insertions(+), 17 deletions(-) create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantFloat16Array.java create mode 100644 writer/src/test/java/io/github/dfa1/vortex/writer/ConstantF16ColumnTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a9de99e..32cb6725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `WriteOptions` is a final class instead of a record. ([#474](https://github.com/dfa1/vortex-java/issues/474)) ### Fixed +- A constant `F16` column can be written and read; it failed with `unsupported ptype: F16` ([#515](https://github.com/dfa1/vortex-java/issues/515)). - Pco columns follow Rust: 2^18-value chunks of 8192-value pages, Rust's IntMult filter, delta sample and histogram; integer columns are byte-identical to pco 1.0.1. ([#513](https://github.com/dfa1/vortex-java/pull/513)) - Integer columns that are an exact arithmetic sequence (timestamps at a fixed step) are written as `vortex.sequence` instead of bit-packed, as Rust does (a 4.2M-row klines file 155 → 128 MB). ([#507](https://github.com/dfa1/vortex-java/issues/507)) - ALP-RD trains its dictionary on the sample Rust's `alp` uses (64 spread runs of 64 values) and ranks ties as Rust does, so the cascade picks it where Rust does (klines `quote_volume` column −0.5 MB). ([#507](https://github.com/dfa1/vortex-java/issues/507)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantFloat16Array.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantFloat16Array.java new file mode 100644 index 00000000..5580c832 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantFloat16Array.java @@ -0,0 +1,50 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.util.Objects; + +/// Metadata-only [Float16Array] for `vortex.constant` columns. +/// +/// Holds a single F16 value broadcast across `length` logical rows. No buffer is allocated — +/// `getFloat(i)` returns the stored value, widened from half precision, for any valid index. +/// The value is kept as its 16 raw bits so a NaN payload or `-0.0` survives a re-encode. +/// +/// @param dtype logical primitive type (F16) +/// @param length total logical row count +/// @param bits broadcast value as raw half-precision bits +public record LazyConstantFloat16Array(DType dtype, long length, short bits) implements Float16Array { + + @Override + public float getFloat(long i) { + Objects.checkIndex(i, length); + return Float.float16ToFloat(bits); + } + + /// Zero-copy truncation: every row holds the same value, so only the row count shrinks. + /// + /// @param rows number of leading rows to keep + /// @return a length-`rows` constant over the same value + @Override + public Array limited(long rows) { + return rows >= length ? this : new LazyConstantFloat16Array(dtype, rows, bits); + } + + /// Materializes the constant into a fresh little-endian half-precision segment, on demand + /// for consumers that need a contiguous buffer. + /// + /// @param arena allocator for the output segment + /// @return a little-endian `f16` segment of `length()` elements + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n * 2L, 2); + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, bits); + } + return dst; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java index 2cf4fdf4..81a8a0e5 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java @@ -10,6 +10,7 @@ import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantDecimalArray; import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; +import io.github.dfa1.vortex.reader.array.LazyConstantFloat16Array; import io.github.dfa1.vortex.reader.array.LazyConstantFloatArray; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; import io.github.dfa1.vortex.reader.array.LazyConstantLongArray; @@ -105,6 +106,7 @@ private static Array constantPrimitive(DType outDtype, PType ptype, ProtoScalarV case I32, U32 -> new LazyConstantIntArray(outDtype, n, (int) rawBits); case F64 -> new LazyConstantDoubleArray(outDtype, n, Double.longBitsToDouble(rawBits)); case F32 -> new LazyConstantFloatArray(outDtype, n, Float.intBitsToFloat((int) rawBits)); + case F16 -> new LazyConstantFloat16Array(outDtype, n, (short) rawBits); case I16, U16 -> new LazyConstantShortArray(outDtype, n, (short) rawBits); case I8, U8 -> new LazyConstantByteArray(outDtype, n, (byte) rawBits); default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype " + ptype); @@ -154,6 +156,10 @@ private static long scalarToRawBits(ProtoScalarValue scalar) { if (scalar.uint64_value() != null) { return scalar.uint64_value(); } + if (scalar.f16_value() != null) { + // Rust writes an f16 as its 16 bits in a u64; the legacy uint64_value form is read above + return scalar.f16_value(); + } if (scalar.f32_value() != null) { return Float.floatToRawIntBits(scalar.f32_value()); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java index ae68ad0f..f2d510dd 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java @@ -104,6 +104,44 @@ void getInt_outOfBounds_throws() { } } + @Nested + class Float16Constant { + private static final short ONE_AND_A_HALF = (short) 0x3E00; + + @Test + void getFloat_widensTheBroadcastValue() { + LazyConstantFloat16Array sut = new LazyConstantFloat16Array(DType.F16, 3, ONE_AND_A_HALF); + assertThat(sut.getFloat(0)).isEqualTo(1.5f); + assertThat(sut.getFloat(2)).isEqualTo(1.5f); + } + + @Test + void getFloat_outOfBounds_throws() { + LazyConstantFloat16Array sut = new LazyConstantFloat16Array(DType.F16, 3, ONE_AND_A_HALF); + assertThatThrownBy(() -> sut.getFloat(-1)).isInstanceOf(IndexOutOfBoundsException.class); + assertThatThrownBy(() -> sut.getFloat(3)).isInstanceOf(IndexOutOfBoundsException.class); + } + + @Test + void limited_shrinksTheLengthOnly() { + LazyConstantFloat16Array sut = new LazyConstantFloat16Array(DType.F16, 10, ONE_AND_A_HALF); + assertThat(sut.limited(4)).isEqualTo(new LazyConstantFloat16Array(DType.F16, 4, ONE_AND_A_HALF)); + assertThat(sut.limited(10)).isSameAs(sut); + } + + @Test + void materialize_writesEveryRowLittleEndian() { + try (java.lang.foreign.Arena arena = java.lang.foreign.Arena.ofConfined()) { + java.lang.foreign.MemorySegment seg = + new LazyConstantFloat16Array(DType.F16, 3, ONE_AND_A_HALF).materialize(arena); + assertThat(seg.byteSize()).isEqualTo(6L); + for (int i = 0; i < 3; i++) { + assertThat(seg.get(io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT, i * 2L)).isEqualTo(ONE_AND_A_HALF); + } + } + } + } + @Nested class ShortConstant { private static final short RAW = (short) 0xFF00; // high bit set diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 0aabb4ad..3676ca10 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -71,7 +71,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { byte[] scalarBytes = scalar.encode(); // A constant array's min and max are both the one repeated value, by construction -- no // scan needed. Empty arrays report no stats, matching every other encoder's convention. - byte[] stats = Array.getLength(data) > 0 ? scalarBytes : null; + // F16 min/max travel as f32 scalars, as PrimitiveEncodingEncoder writes them. + byte[] stats = Array.getLength(data) > 0 ? statsScalar(ptype, firstRaw, scalarBytes) : null; return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), stats, stats); } @@ -146,78 +147,85 @@ private static OptionalLong constantBits(Object data, PType ptype) { if (Array.getLength(data) == 0) { return OptionalLong.of(0L); } - switch (ptype) { + // No default: every PType is handled, so a new one fails to compile here instead of at write time + return switch (ptype) { case I8, U8 -> { byte[] a = (byte[]) data; byte first = a[0]; for (int i = 1; i < a.length; i++) { if (a[i] != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } - case I16, U16 -> { + case I16, U16, F16 -> { short[] a = (short[]) data; short first = a[0]; for (int i = 1; i < a.length; i++) { if (a[i] != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } case I32, U32 -> { int[] a = (int[]) data; int first = a[0]; for (int i = 1; i < a.length; i++) { if (a[i] != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } case I64, U64 -> { long[] a = (long[]) data; long first = a[0]; for (int i = 1; i < a.length; i++) { if (a[i] != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } case F32 -> { float[] a = (float[]) data; int first = Float.floatToRawIntBits(a[0]); for (int i = 1; i < a.length; i++) { if (Float.floatToRawIntBits(a[i]) != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } case F64 -> { double[] a = (double[]) data; long first = Double.doubleToRawLongBits(a[0]); for (int i = 1; i < a.length; i++) { if (Double.doubleToRawLongBits(a[i]) != first) { - return OptionalLong.empty(); + yield OptionalLong.empty(); } } - return OptionalLong.of(first); + yield OptionalLong.of(first); } - default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); + }; + } + + private static byte[] statsScalar(PType ptype, long rawBits, byte[] scalarBytes) { + if (ptype != PType.F16) { + return scalarBytes; } + return ProtoScalarValue.ofF32Value(Float.float16ToFloat((short) rawBits)).encode(); } private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { return switch (ptype) { case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(rawBits); case I8, I16, I32, I64 -> ProtoScalarValue.ofInt64Value(rawBits); + case F16 -> ProtoScalarValue.ofF16Value(rawBits & 0xFFFFL); case F32 -> ProtoScalarValue.ofF32Value(Float.intBitsToFloat((int) rawBits)); case F64 -> ProtoScalarValue.ofF64Value(Double.longBitsToDouble(rawBits)); - default -> throw new VortexException(EncodingId.VORTEX_CONSTANT, "unsupported ptype: " + ptype); }; } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ConstantF16ColumnTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ConstantF16ColumnTest.java new file mode 100644 index 00000000..6a5bc56c --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ConstantF16ColumnTest.java @@ -0,0 +1,58 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.Float16Array; +import io.github.dfa1.vortex.reader.array.LazyConstantFloat16Array; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// #515: a constant F16 column hit `vortex.constant: unsupported ptype: F16` on write, because +/// the constant encoder and decoder had no F16 case. +class ConstantF16ColumnTest { + + private static final ColumnName COLUMN = ColumnName.of("h"); + private static final DType.Struct SCHEMA = new DType.Struct(List.of(COLUMN), + List.of(new DType.Primitive(PType.F16, false)), false); + + @TempDir + Path tmp; + + @Test + void constantF16Column_roundTripsAsALazyConstant() throws IOException { + // Given 1000 rows of 1.5, which the default cascade picks vortex.constant for + short[] data = new short[1000]; + Arrays.fill(data, Float.floatToFloat16(1.5f)); + Path file = tmp.resolve("f16.vtx"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = VortexWriter.create(ch, SCHEMA, WriteOptions.defaults())) { + sut.writeChunk(Map.of(COLUMN, data)); + } + + // When + List result = new ArrayList<>(); + try (VortexReader reader = VortexReader.open(file)) { + reader.scan(ScanOptions.all()).forEachRemaining(c -> result.add(c.column(COLUMN))); + } + + // Then the column stayed one metadata-only constant, not 1000 stored halves + assertThat(result).hasSize(1); + assertThat(result.getFirst()).isInstanceOf(LazyConstantFloat16Array.class); + assertThat(result.getFirst().length()).isEqualTo(1000L); + assertThat(result.getFirst().getFloat(999)).isEqualTo(1.5f); + } +} diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 0bb37030..9f292940 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -8,9 +8,11 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LazyConstantBoolArray; +import io.github.dfa1.vortex.reader.array.LazyConstantFloat16Array; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; import io.github.dfa1.vortex.reader.array.LazyConstantLongArray; import io.github.dfa1.vortex.reader.array.LongArray; @@ -143,6 +145,68 @@ void encodeDecode_i64_isLossless(long[] data) { assertThat(result).isInstanceOf(LazyConstantLongArray.class); assertThat(((LazyConstantLongArray) result).value()).isEqualTo(data[0]); } + + @ParameterizedTest + @ValueSource(shorts = {0x3E00, (short) 0x8000, 0x7C01, 0x0001, (short) 0xFFFF}) + void encodeDecode_f16_keepsTheRawBits(short bits) { + // Given — 1.5, -0.0, a NaN payload, the smallest subnormal and the top NaN: every one + // must come back bit for bit, so the value is compared as raw bits, not as a number + short[] data = {bits, bits, bits}; + + // When + EncodeResult resultEncoded = ENCODER.encode(DTypes.F16, data, EncodeTestHelper.testCtx()); + DecodeContext ctx = DecodeTestHelper.toDecodeContext(resultEncoded, data.length, DTypes.F16, REGISTRY); + Array result = DECODER.decode(ctx); + + // Then + assertThat(result).isInstanceOf(LazyConstantFloat16Array.class); + assertThat(((LazyConstantFloat16Array) result).bits()).isEqualTo(bits); + // Float.compare, not isEqualTo(float): the NaN cases are never == themselves + assertThat(Float.compare(((Float16Array) result).getFloat(2), Float.float16ToFloat(bits))).isZero(); + } + + @Test + void encode_f16_writesRustsF16ScalarField() throws java.io.IOException { + // Given — Rust reads an F16 constant from the f16_value field (the half bits as a u64) + short[] data = {Float.floatToFloat16(1.5f), Float.floatToFloat16(1.5f)}; + + // When + EncodeResult result = ENCODER.encode(DTypes.F16, data, EncodeTestHelper.testCtx()); + + // Then + MemorySegment seg = result.encodedBuffers().getFirst().data(); + ProtoScalarValue scalar = ProtoScalarValue.decode(seg, 0, seg.byteSize()); + assertThat(scalar.f16_value()).isEqualTo(0x3E00L); + } + + @Test + void decode_f16_acceptsTheLegacyUint64Scalar() { + // Given — older Rust writers stored an f16 as uint64_value (its bits) + EncodeResult encoded = ENCODER.encode(DTypes.F16, new short[]{0}, EncodeTestHelper.testCtx()); + byte[] legacy = ProtoScalarValue.ofUint64Value(0x3E00L).encode(); + EncodeResult swapped = EncodeResult.simple(EncodingId.VORTEX_CONSTANT, + EncodedBuffer.bytes(MemorySegment.ofArray(legacy)), null, null); + + // When + Array result = DECODER.decode(DecodeTestHelper.toDecodeContext(swapped, 4, DTypes.F16, REGISTRY)); + + // Then + assertThat(((Float16Array) result).getFloat(3)).isEqualTo(1.5f); + assertThat(encoded).isNotNull(); + } + + @Test + void encodeCascade_f16WithNegativeZeroAndZero_notApplicable() { + // Given — +0.0 and -0.0 compare equal as numbers but not as bits; a constant would + // make every row the first one's sign + short[] data = {0, (short) 0x8000}; + + // When + CascadeStep step = ENCODER.encodeCascade(DTypes.F16, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(step.applicable()).isFalse(); + } } /// ConstantEncoding stores 1 element in the buffer but reports length=N. @@ -352,6 +416,19 @@ void encode_f64_reportsValueAsMinAndMax() throws java.io.IOException { assertThat(scalar(result.statsMax()).f64_value()).isEqualTo(constant); } + @Test + void encode_f16_reportsValueAsF32MinAndMax() throws java.io.IOException { + // Given — F16 min/max travel as f32 scalars, as PrimitiveEncodingEncoder writes them + short[] data = {Float.floatToFloat16(2.5f), Float.floatToFloat16(2.5f)}; + + // When + EncodeResult result = ENCODER.encode(DTypes.F16, data, EncodeTestHelper.testCtx()); + + // Then + assertThat(scalar(result.statsMin()).f32_value()).isEqualTo(2.5f); + assertThat(scalar(result.statsMax()).f32_value()).isEqualTo(2.5f); + } + @Test void encode_empty_statsAreNull() { // Given From 198d46077633a7b76368f756856ec4fb6c5c07ef Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 16:24:48 +0200 Subject: [PATCH 3/4] fix: dict, rle and sparse encodings handle F16 columns (#515) DictEncodingDecoder, DictLayoutDecoder and RleEncodingDecoder had no F16 case although the cascade writes both encodings for F16, so such files could not be read back; SparseEncodingEncoder and SparseEncodingDecoder had none at all. New DictFloat16Array, LazyRleFloat16Array and LazySparseFloat16Array keep the 16 raw bits (a NaN payload or -0.0 survives materialize) and ignore the pool's widened floats. Float16EncodingsRoundTripTest writes eight data shapes under the default cascade and each of the four encodings forced, and compares raw bits. The Rust interop test for F16 stays disabled (vortex-jni exports no F16). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Sko6B2519bE1TkpPx7WKxf --- CHANGELOG.md | 2 +- .../vortex/reader/array/DictFloat16Array.java | 96 ++++++++++++++++ .../reader/array/LazyRleFloat16Array.java | 101 ++++++++++++++++ .../reader/array/LazySparseFloat16Array.java | 64 +++++++++++ .../reader/decode/DictEncodingDecoder.java | 5 +- .../reader/decode/RleEncodingDecoder.java | 7 +- .../reader/decode/SparseEncodingDecoder.java | 9 +- .../reader/layout/DictLayoutDecoder.java | 6 +- .../reader/array/DictRecordSmokeTest.java | 68 +++++++++++ .../vortex/reader/array/LazyRleArrayTest.java | 84 ++++++++++++++ .../reader/array/LazySparseArrayTest.java | 73 ++++++++++++ .../decode/DictEncodingDecoderTest.java | 21 ++-- .../encode/ConstantEncodingEncoder.java | 2 +- .../writer/encode/SparseEncodingEncoder.java | 8 +- .../writer/Float16EncodingsRoundTripTest.java | 108 ++++++++++++++++++ 15 files changed, 630 insertions(+), 24 deletions(-) create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloat16Array.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyRleFloat16Array.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/array/LazySparseFloat16Array.java create mode 100644 writer/src/test/java/io/github/dfa1/vortex/writer/Float16EncodingsRoundTripTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 32cb6725..cc178097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `WriteOptions` is a final class instead of a record. ([#474](https://github.com/dfa1/vortex-java/issues/474)) ### Fixed -- A constant `F16` column can be written and read; it failed with `unsupported ptype: F16` ([#515](https://github.com/dfa1/vortex-java/issues/515)). +- `F16` columns with constant, low-cardinality, run-heavy or mostly-one-value data can be written and read back; they failed with `unsupported ptype: F16` or wrote files the reader rejected. ([#515](https://github.com/dfa1/vortex-java/issues/515)) - Pco columns follow Rust: 2^18-value chunks of 8192-value pages, Rust's IntMult filter, delta sample and histogram; integer columns are byte-identical to pco 1.0.1. ([#513](https://github.com/dfa1/vortex-java/pull/513)) - Integer columns that are an exact arithmetic sequence (timestamps at a fixed step) are written as `vortex.sequence` instead of bit-packed, as Rust does (a 4.2M-row klines file 155 → 128 MB). ([#507](https://github.com/dfa1/vortex-java/issues/507)) - ALP-RD trains its dictionary on the sample Rust's `alp` uses (64 spread runs of 64 values) and ranks ties as Rust does, so the cascade picks it where Rust does (klines `quote_volume` column −0.5 MB). ([#507](https://github.com/dfa1/vortex-java/issues/507)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloat16Array.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloat16Array.java new file mode 100644 index 00000000..ec102407 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloat16Array.java @@ -0,0 +1,96 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; + +/// Dict-encoded [Float16Array] view. ADR 0012 shape. +/// +/// Stores `values` (the dictionary pool) and `codes` (one index per row into `values`). Scalar +/// access resolves on demand: `getFloat(i) = values.getFloat(codes.getCode(i))`, so a +/// categorical column costs the pool plus the codes. +/// +/// The `codes` array is typed as [Array] because the codes ptype varies with dictionary size — +/// U8/U16/U32/U64 backed by [ByteArray]/[ShortArray]/[IntArray]/[LongArray]. +/// +/// @param dtype logical element type (matches `values.dtype()`) +/// @param length total logical row count (matches `codes.length()`) +/// @param values dictionary pool — element at code `c` is `values.getFloat(c)` +/// @param codes per-row index into `values`; must be one of +/// [ByteArray], [ShortArray], [IntArray], [LongArray] +public record DictFloat16Array(DType dtype, long length, Float16Array values, Array codes) implements Float16Array { + + /// Builds a [DictFloat16Array], validating that `codes` is one of the four narrow-int code + /// array types and that its length matches `length`. + /// + /// @param dtype logical element type + /// @param length total logical row count + /// @param values dictionary pool + /// @param codes per-row code array (must be [ByteArray], [ShortArray], [IntArray], or [LongArray]) + /// @return a new [DictFloat16Array] + /// @throws VortexException if `codes` is not a supported code-array type or its length does + /// not equal `length` + public static DictFloat16Array of(DType dtype, long length, Float16Array values, Array codes) { + DictArrays.validateCodes(codes, length); + return new DictFloat16Array(dtype, length, values, codes); + } + + @Override + public float getFloat(long i) { + return values.getFloat(DictArrays.readCode(codes, i)); + } + + /// Zero-copy truncation: the pool is shared, only the codes are cut. + /// + /// @param rows number of leading rows to keep + /// @return a length-`rows` dict over the same pool + @Override + public Array limited(long rows) { + return rows >= length ? this : new DictFloat16Array(dtype, rows, values, Array.limited(codes, rows)); + } + + /// Materializes by gathering one pool entry per code, copying the half's 16 bits rather than + /// widening and narrowing them, so NaN payloads survive. The codes switch is hoisted outside + /// the loop so each branch is a uniform gather over a single code width. + /// + /// @param arena allocator for the output segment + /// @return a little-endian `f16` segment of gathered values + /// @throws VortexException if `codes` is not a supported code-array type + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n * 2L, 2); + MemorySegment pool = values.materialize(arena); + switch (CanonicalArrays.of(codes, arena)) { + case ByteArray ba -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, + pool.getAtIndex(VortexFormat.LE_SHORT, Byte.toUnsignedLong(ba.getByte(i)))); + } + } + case ShortArray sa -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, + pool.getAtIndex(VortexFormat.LE_SHORT, Short.toUnsignedLong(sa.getShort(i)))); + } + } + case IntArray ia -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, + pool.getAtIndex(VortexFormat.LE_SHORT, Integer.toUnsignedLong(ia.getInt(i)))); + } + } + case LongArray la -> { + for (long i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, pool.getAtIndex(VortexFormat.LE_SHORT, la.getLong(i))); + } + } + default -> throw new VortexException("DictFloat16Array: invalid codes type: " + + codes.getClass().getSimpleName()); + } + return dst.asReadOnly(); + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyRleFloat16Array.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyRleFloat16Array.java new file mode 100644 index 00000000..b3435d73 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyRleFloat16Array.java @@ -0,0 +1,101 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.ValueLayout; + +/// Lazy FastLanes-RLE-encoded [Float16Array]. See [LazyRleLongArray] for semantics. The pool and +/// the materialized output keep the half's 16 bits, so NaN payloads survive. +/// +/// @param dtype logical element type +/// @param length total logical row count +/// @param values concatenated distinct values per chunk, as a little-endian +/// `f16` segment of exactly `valuesLen` elements +/// @param indices per-row local index table, as a `u8`/`u16` segment +/// @param wideIndices `true` when `indices` holds `u16` elements, `false` for `u8` +/// @param valuesIdxOffsets per-chunk values-pool start offsets +/// @param firstOffset absolute origin of the values pool +/// @param valuesLen total values pool length +/// @param numChunks number of FastLanes chunks covered +/// @param offset starting absolute position +@SuppressWarnings("java:S6218") // internal data carrier; the long[] offsets component is never compared. +public record LazyRleFloat16Array( + DType dtype, long length, MemorySegment values, MemorySegment indices, boolean wideIndices, + long[] valuesIdxOffsets, long firstOffset, long valuesLen, + int numChunks, int offset) + implements Float16Array { + + @Override + public float getFloat(long i) { + int absRow = (int) (i + offset); + int chunkIdx = absRow >>> RleArrays.FL_LOG2; + int rowInChunk = absRow & RleArrays.FL_MASK; + long valueIdxOffset = valuesIdxOffsets[chunkIdx] - firstOffset; + int numChunkValues = RleArrays.chunkValueCount(chunkIdx, numChunks, valuesIdxOffsets, firstOffset, valuesLen); + if (numChunkValues <= 1) { + return numChunkValues == 1 ? Float.float16ToFloat(values.getAtIndex(VortexFormat.LE_SHORT, valueIdxOffset)) : 0.0f; + } + int localIdx = RleArrays.localIndex(indices, wideIndices, + (long) chunkIdx * RleArrays.FL_CHUNK_SIZE + rowInChunk); + return Float.float16ToFloat(values.getAtIndex(VortexFormat.LE_SHORT, + valueIdxOffset + Math.min(localIdx, numChunkValues - 1))); + } + + /// Zero-copy truncation: the chunk tables and the values pool are unchanged, only the row + /// count shrinks. + /// + /// @param rows number of leading rows to keep + /// @return a length-`rows` view over the same tables + @Override + public Array limited(long rows) { + return rows >= length ? this : new LazyRleFloat16Array(dtype, rows, values, indices, wideIndices, + valuesIdxOffsets, firstOffset, valuesLen, numChunks, offset); + } + + /// Bulk-decodes chunk by chunk into a fresh little-endian `f16` segment, + /// with the constant-run fast path (`numChunkValues <= 1`) emitting each + /// value once. See [LazyRleLongArray] for the chunk-walk rationale. + /// + /// @param arena allocator for the output segment + /// @return a little-endian `f16` segment of `length()` decoded elements + @Override + public MemorySegment materialize(SegmentAllocator arena) { + long n = length; + MemorySegment dst = arena.allocate(n * 2L, 2); + RleArrays.walkChunks(length, offset, numChunks, + (chunkIdx, rowInChunk, end) -> processChunk(chunkIdx, rowInChunk, end, dst)); + return dst; + } + + /// Writes one chunk's rows. The `u8`/`u16` index width is hoisted into its own loop + /// (rather than tested per row) so each body stays uniform — see the CLAUDE.md hot-loop rule. + private void processChunk(int chunkIdx, int rowInChunk, int end, MemorySegment dst) { + long chunkBase = (long) chunkIdx * RleArrays.FL_CHUNK_SIZE; + long valueIdxOffset = valuesIdxOffsets[chunkIdx] - firstOffset; + int numChunkValues = RleArrays.chunkValueCount(chunkIdx, numChunks, valuesIdxOffsets, firstOffset, valuesLen); + int maxIdx = numChunkValues - 1; + // dstIdx tracks the logical output row; chunk 0 may start mid-chunk when offset != 0. + long dstIdx = chunkBase + rowInChunk - offset; + if (numChunkValues <= 1) { + short v = numChunkValues == 1 ? values.getAtIndex(VortexFormat.LE_SHORT, valueIdxOffset) : (short) 0; + for (int r = rowInChunk; r < end; r++) { + dst.setAtIndex(VortexFormat.LE_SHORT, dstIdx++, v); + } + } else if (wideIndices) { + for (int r = rowInChunk; r < end; r++) { + int localIdx = Short.toUnsignedInt(indices.getAtIndex(VortexFormat.LE_SHORT, chunkBase + r)); + dst.setAtIndex(VortexFormat.LE_SHORT, dstIdx++, + values.getAtIndex(VortexFormat.LE_SHORT, valueIdxOffset + Math.min(localIdx, maxIdx))); + } + } else { + for (int r = rowInChunk; r < end; r++) { + int localIdx = Byte.toUnsignedInt(indices.get(ValueLayout.JAVA_BYTE, chunkBase + r)); + dst.setAtIndex(VortexFormat.LE_SHORT, dstIdx++, + values.getAtIndex(VortexFormat.LE_SHORT, valueIdxOffset + Math.min(localIdx, maxIdx))); + } + } + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazySparseFloat16Array.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazySparseFloat16Array.java new file mode 100644 index 00000000..c88265c8 --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazySparseFloat16Array.java @@ -0,0 +1,64 @@ +package io.github.dfa1.vortex.reader.array; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.DType; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; + +/// Lazy Sparse-encoded [Float16Array]. See [LazySparseLongArray] for semantics. The fill is kept +/// as its 16 raw bits, so a NaN payload or `-0.0` survives materialize. +/// +/// @param dtype logical element type +/// @param length total logical row count +/// @param fillBits raw half-precision bits at every unpatched position +/// @param patchValues values for patched positions +/// @param patchIndices sorted absolute positions of patches +/// @param offset starting absolute position +public record LazySparseFloat16Array( + DType dtype, long length, short fillBits, + Float16Array patchValues, Array patchIndices, long offset) + implements Float16Array { + + @Override + public float getFloat(long i) { + if (patchValues == null) { + return Float.float16ToFloat(fillBits); + } + int p = SparseArrays.findPatch(patchIndices, patchValues.length(), i + offset); + return p >= 0 ? patchValues.getFloat(p) : Float.float16ToFloat(fillBits); + } + + /// Zero-copy truncation: patches are found by absolute position, so only the row count + /// shrinks. + /// + /// @param rows number of leading rows to keep + /// @return a length-`rows` view over the same patches + @Override + public Array limited(long rows) { + return rows >= length ? this : new LazySparseFloat16Array(dtype, rows, fillBits, patchValues, patchIndices, offset); + } + + /// Materializes into a fresh little-endian half-precision segment: one walk over the rows in + /// order, the fill for an unpatched row and the patch's own bits for a patched one. Patch + /// values are copied through their pool's segment, not widened and narrowed. + /// + /// @param arena allocator for the output segment + /// @return a little-endian `f16` segment of `length()` elements + @Override + public MemorySegment materialize(SegmentAllocator arena) { + MemorySegment dst = arena.allocate(length * 2L, 2); + long[] row = {0}; + if (patchValues == null) { + SparseArrays.walkPatches(patchIndices, 0, offset, offset + length, + () -> dst.setAtIndex(VortexFormat.LE_SHORT, row[0]++, fillBits), + p -> { }); + return dst; + } + MemorySegment pool = patchValues.materialize(arena); + SparseArrays.walkPatches(patchIndices, patchValues.length(), offset, offset + length, + () -> dst.setAtIndex(VortexFormat.LE_SHORT, row[0]++, fillBits), + p -> dst.setAtIndex(VortexFormat.LE_SHORT, row[0]++, pool.getAtIndex(VortexFormat.LE_SHORT, p))); + return dst; + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 758687e7..63774c9d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -12,11 +12,13 @@ import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictDoubleArray; +import io.github.dfa1.vortex.reader.array.DictFloat16Array; import io.github.dfa1.vortex.reader.array.DictFloatArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DictLongArray; import io.github.dfa1.vortex.reader.array.DictShortArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; @@ -169,8 +171,7 @@ private static Array buildLazyDict(DType dtype, PType valPType, long n, Array va case I8, U8 -> DictByteArray.of(dtype, n, (ByteArray) values, codes); case F64 -> DictDoubleArray.of(dtype, n, (DoubleArray) values, codes); case F32 -> DictFloatArray.of(dtype, n, (FloatArray) values, codes); - // F16 has no Array subtype yet, matching DictLayoutDecoder. - default -> throw new VortexException(EncodingId.VORTEX_DICT, "unsupported ptype " + valPType); + case F16 -> DictFloat16Array.of(dtype, n, (Float16Array) values, codes); }; } catch (ClassCastException e) { // The values child is untrusted and may decode to any Array family regardless of diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index d91c7f6b..8f5d6f88 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -11,6 +11,7 @@ import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; +import io.github.dfa1.vortex.reader.array.LazyConstantFloat16Array; import io.github.dfa1.vortex.reader.array.LazyConstantFloatArray; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; import io.github.dfa1.vortex.reader.array.LazyConstantLongArray; @@ -18,6 +19,7 @@ import io.github.dfa1.vortex.reader.array.LazyRleBoolArray; import io.github.dfa1.vortex.reader.array.LazyRleByteArray; import io.github.dfa1.vortex.reader.array.LazyRleDoubleArray; +import io.github.dfa1.vortex.reader.array.LazyRleFloat16Array; import io.github.dfa1.vortex.reader.array.LazyRleFloatArray; import io.github.dfa1.vortex.reader.array.LazyRleIntArray; import io.github.dfa1.vortex.reader.array.LazyRleLongArray; @@ -134,7 +136,8 @@ public Array decode(DecodeContext ctx) { indices, wideIndices, valuesIdxOffsets, firstOffset, valuesLen, numChunks, offset); case F32 -> new LazyRleFloatArray(ctx.dtype(), rowCount, values, indices, wideIndices, valuesIdxOffsets, firstOffset, valuesLen, numChunks, offset); - default -> throw new VortexException(EncodingId.FASTLANES_RLE, "unsupported ptype " + ptype); + case F16 -> new LazyRleFloat16Array(ctx.dtype(), rowCount, values, + indices, wideIndices, valuesIdxOffsets, firstOffset, valuesLen, numChunks, offset); }; if (indicesValidity == null) { @@ -154,7 +157,7 @@ private static Array emptyArray(DecodeContext ctx) { case I8, U8 -> new LazyConstantByteArray(dt, 0L, (byte) 0); case F64 -> new LazyConstantDoubleArray(dt, 0L, 0.0); case F32 -> new LazyConstantFloatArray(dt, 0L, 0.0f); - default -> throw new VortexException(EncodingId.FASTLANES_RLE, "unsupported ptype " + ptype); + case F16 -> new LazyConstantFloat16Array(dt, 0L, (short) 0); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java index 519c1878..49215e68 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java @@ -11,11 +11,13 @@ import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LazySparseBoolArray; import io.github.dfa1.vortex.reader.array.LazySparseByteArray; import io.github.dfa1.vortex.reader.array.LazySparseDoubleArray; +import io.github.dfa1.vortex.reader.array.LazySparseFloat16Array; import io.github.dfa1.vortex.reader.array.LazySparseFloatArray; import io.github.dfa1.vortex.reader.array.LazySparseIntArray; import io.github.dfa1.vortex.reader.array.LazySparseLongArray; @@ -158,6 +160,8 @@ public Array decode(DecodeContext ctx) { checkedCast(valData, DoubleArray.class, ROLE_VALUES), idxData, offset); case F32 -> new LazySparseFloatArray(ctx.dtype(), n, Float.intBitsToFloat((int) fillBits), checkedCast(valData, FloatArray.class, ROLE_VALUES), idxData, offset); + case F16 -> new LazySparseFloat16Array(ctx.dtype(), n, (short) fillBits, + checkedCast(valData, Float16Array.class, ROLE_VALUES), idxData, offset); case I16 -> new LazySparseShortArray(ctx.dtype(), n, (short) fillBits, (short) fillBits, checkedCast(valData, ShortArray.class, ROLE_VALUES), idxData, offset); case U16 -> new LazySparseShortArray(ctx.dtype(), n, (short) fillBits, (int) (fillBits & 0xFFFFL), @@ -166,7 +170,6 @@ public Array decode(DecodeContext ctx) { checkedCast(valData, ByteArray.class, ROLE_VALUES), idxData, offset); case U8 -> new LazySparseByteArray(ctx.dtype(), n, (byte) fillBits, (int) (fillBits & 0xFFL), checkedCast(valData, ByteArray.class, ROLE_VALUES), idxData, offset); - default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype " + valuePtype); }; return withSparseValidity(ctx, result, fillValid, patchValidity, idxData, numPatches, n, offset); } @@ -330,6 +333,10 @@ private static long scalarToLong(ProtoScalarValue scalar) { if (scalar.uint64_value() != null) { return scalar.uint64_value(); } + if (scalar.f16_value() != null) { + // Rust writes an f16 as its 16 bits in a u64; the legacy uint64_value form is read above + return scalar.f16_value(); + } if (scalar.f32_value() != null) { return Float.floatToRawIntBits(scalar.f32_value()); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index af671801..0af5e790 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -15,11 +15,13 @@ import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictDoubleArray; +import io.github.dfa1.vortex.reader.array.DictFloat16Array; import io.github.dfa1.vortex.reader.array.DictFloatArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DictLongArray; import io.github.dfa1.vortex.reader.array.DictShortArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; import io.github.dfa1.vortex.reader.array.LongArray; @@ -276,9 +278,7 @@ private static Array buildLazyDictPrimitive(DType.Primitive dtype, long n, Array case I8, U8 -> DictByteArray.of(dtype, n, (ByteArray) valuesData, codesData); case F64 -> DictDoubleArray.of(dtype, n, (DoubleArray) valuesData, codesData); case F32 -> DictFloatArray.of(dtype, n, (FloatArray) valuesData, codesData); - // F16 has no Array subtype yet - default -> throw new VortexException(EncodingId.VORTEX_DICT, - "layout: unsupported ptype for lazy dict: " + ptype); + case F16 -> DictFloat16Array.of(dtype, n, (Float16Array) valuesData, codesData); }; if (poolValidity == null && codesValidity == null) { return dict; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java index c1ab3650..f1ae1f40 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java @@ -21,6 +21,7 @@ class DictRecordSmokeTest { private static final DType I32 = DType.I32; private static final DType F64 = DType.F64; private static final DType F32 = DType.F32; + private static final DType F16 = DType.F16; private static final DType U8 = DType.U8; private static final DType U16 = DType.U16; private static final DType U32 = DType.U32; @@ -386,6 +387,65 @@ void bulkOps_invalidCodesType_throw() { } } + @Nested + class DictFloat16 { + + // 1.5, 2.5 and a NaN with a payload: the payload only survives if the gather copies bits + private static final short[] POOL = {(short) 0x3E00, (short) 0x4100, (short) 0x7C01}; + + @Test + void allCodeTypes_getAndMaterialize() { + try (Arena arena = Arena.ofConfined()) { + Float16Array values = float16Array(arena, POOL); + short[] expectedBits = {POOL[2], POOL[0], POOL[1]}; + List codeVariants = List.of( + byteArray(arena, (byte) 2, (byte) 0, (byte) 1), + shortArray(arena, U16, (short) 2, (short) 0, (short) 1), + intArray(arena, U32, 2, 0, 1), + longArray(arena, U64, 2L, 0L, 1L)); + + for (Array codes : codeVariants) { + DictFloat16Array sut = DictFloat16Array.of(F16, 3, values, codes); + String label = codes.getClass().getSimpleName(); + + assertThat(sut.getFloat(1)).as(label).isEqualTo(1.5f); + assertThat(sut.getFloat(2)).as(label).isEqualTo(2.5f); + MemorySegment m = sut.materialize(arena); + for (int i = 0; i < 3; i++) { + assertThat(m.getAtIndex(VortexFormat.LE_SHORT, i)).as(label).isEqualTo(expectedBits[i]); + } + } + } + } + + @Test + void limited_keepsThePoolAndCutsTheCodes() { + try (Arena arena = Arena.ofConfined()) { + Float16Array values = float16Array(arena, POOL); + DictFloat16Array sut = DictFloat16Array.of(F16, 3, values, byteArray(arena, (byte) 1, (byte) 0, (byte) 1)); + + Array result = sut.limited(2); + + assertThat(result.length()).isEqualTo(2L); + assertThat(((Float16Array) result).getFloat(0)).isEqualTo(2.5f); + assertThat(((Float16Array) result).getFloat(1)).isEqualTo(1.5f); + assertThat(sut.limited(3)).isSameAs(sut); + } + } + + @Test + void invalidCodes_throw() { + try (Arena arena = Arena.ofConfined()) { + Float16Array values = float16Array(arena, POOL); + + assertThatThrownBy(() -> DictFloat16Array.of(F16, 2, values, byteArray(arena, (byte) 0))) + .isInstanceOf(VortexException.class).hasMessageContaining("length"); + assertThatThrownBy(() -> new DictFloat16Array(F16, 1, values, doubleArray(arena, 0.0)).materialize(arena)) + .isInstanceOf(VortexException.class); + } + } + } + @Nested class DictFloat { @@ -759,6 +819,14 @@ private static FloatArray floatArray(Arena arena, float... values) { return new MaterializedFloatArray(F32, values.length, seg.asReadOnly()); } + private static Float16Array float16Array(Arena arena, short... bits) { + MemorySegment seg = arena.allocate(bits.length * 2L, 2); + for (int i = 0; i < bits.length; i++) { + seg.setAtIndex(VortexFormat.LE_SHORT, i, bits[i]); + } + return new MaterializedFloat16Array(F16, bits.length, seg.asReadOnly()); + } + private static ShortArray shortArray(Arena arena, DType dtype, short... values) { MemorySegment seg = arena.allocate(values.length * 2L); for (int i = 0; i < values.length; i++) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java index b19080ef..480f321b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyRleArrayTest.java @@ -31,6 +31,7 @@ class LazyRleArrayTest { private static final DType I16 = DType.I16; private static final DType F64 = DType.F64; private static final DType F32 = DType.F32; + private static final DType F16 = DType.F16; @Nested class LongDispatch { @@ -524,6 +525,89 @@ void indexedClampAndEmptyChunk() { } } + /// F16 RLE columns follow the F32 layout with a 2-byte value child (#515); the pool stays in + /// half-precision bits, so a NaN payload survives materialize. + @Nested + class Float16Dispatch { + + private static final short ONE_AND_A_HALF = (short) 0x3E00; + private static final short TWO_AND_A_HALF = (short) 0x4100; + private static final short NAN_PAYLOAD = (short) 0x7C01; + + @Test + void singleChunkWithIndices_perRowLookup() { + // Given one chunk with 3 distinct halves in a 0,1,2 cycle; a wrong-width read + // (4 bytes per value) would shift every later value + var sut = new LazyRleFloat16Array(F16, 6, + TestSegments.leShorts(ONE_AND_A_HALF, TWO_AND_A_HALF, NAN_PAYLOAD), + u8Indices(1024, 0, 1, 2, 0, 1, 2), false, new long[]{0L}, 0L, 3L, 1, 0); + + // When / Then + assertThat(sut.getFloat(0)).isEqualTo(1.5f); + assertThat(sut.getFloat(4)).isEqualTo(2.5f); + assertThat(sut.getFloat(5)).isNaN(); + } + + @Test + void constantRunAndEmptyChunk() { + // Given a constant chunk (1 distinct value) and an empty chunk (0 distinct values) + var constant = new LazyRleFloat16Array(F16, 3, TestSegments.leShorts(TWO_AND_A_HALF), + u8Indices(1024), false, new long[]{0L}, 0L, 1L, 1, 0); + var empty = new LazyRleFloat16Array(F16, 2, TestSegments.leShorts(), + u8Indices(1024), false, new long[]{0L}, 0L, 0L, 1, 0); + + // When / Then + assertThat(constant.getFloat(2)).isEqualTo(2.5f); + assertThat(empty.getFloat(0)).isZero(); + } + + @Test + void indexOverrun_clampsToTheLastValue() { + // Given index[2] past the pool + var sut = new LazyRleFloat16Array(F16, 3, + TestSegments.leShorts(ONE_AND_A_HALF, TWO_AND_A_HALF), + u8Indices(1024, 0, 1, 9), false, new long[]{0L}, 0L, 2L, 1, 0); + + // When / Then + assertThat(sut.getFloat(2)).isEqualTo(2.5f); + } + + @Test + void materialize_keepsTheHalfBitsOfEveryRow() { + // Given an indexed chunk holding a NaN payload and a constant chunk after it + var sut = new LazyRleFloat16Array(F16, 1026, + TestSegments.leShorts(ONE_AND_A_HALF, NAN_PAYLOAD, TWO_AND_A_HALF), + u8Indices(2 * 1024, 0, 1, 0), false, new long[]{0L, 2L}, 0L, 3L, 2, 0); + + // When + try (var arena = Arena.ofConfined()) { + var result = sut.materialize(arena); + + // Then widening and narrowing again would have changed the payload + assertThat(result.byteSize()).isEqualTo(2052L); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 0)).isEqualTo(ONE_AND_A_HALF); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 1)).isEqualTo(NAN_PAYLOAD); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 1025)).isEqualTo(TWO_AND_A_HALF); + } + } + + @Test + void limited_cutsTheRowCountOnly() { + // Given + var sut = new LazyRleFloat16Array(F16, 6, + TestSegments.leShorts(ONE_AND_A_HALF, TWO_AND_A_HALF), + u8Indices(1024, 0, 1, 0, 1, 0, 1), false, new long[]{0L}, 0L, 2L, 1, 0); + + // When + Array result = sut.limited(2); + + // Then + assertThat(result.length()).isEqualTo(2L); + assertThat(((Float16Array) result).getFloat(1)).isEqualTo(2.5f); + assertThat(sut.limited(6)).isSameAs(sut); + } + } + /// F32 RLE columns follow the same wire layout as F64 with a 4-byte value child /// (issue #209); [LazyRleFloatArray] materializes rather than exposing a forEach. @Nested diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazySparseArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazySparseArrayTest.java index 48c923b4..e15729b6 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazySparseArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazySparseArrayTest.java @@ -1,6 +1,8 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.testing.TestSegments; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -10,6 +12,7 @@ import java.util.ArrayList; import static io.github.dfa1.vortex.core.testing.DTypes.BOOL; +import static io.github.dfa1.vortex.core.testing.DTypes.F16; import static io.github.dfa1.vortex.core.testing.DTypes.F32; import static io.github.dfa1.vortex.core.testing.DTypes.F64; import static io.github.dfa1.vortex.core.testing.DTypes.I16; @@ -314,6 +317,76 @@ void forEachBooleanEmitsFillAndPatches() { } } + @Nested + class Float16 { + + private static final short ONE_AND_A_HALF = (short) 0x3E00; + private static final short TWO_AND_A_HALF = (short) 0x4100; + private static final short NINE = (short) 0x4880; + private static final short NAN_PAYLOAD = (short) 0x7C01; + + @Test + void patchAndFillDispatch() { + // Given + Float16Array values = float16s(ONE_AND_A_HALF, TWO_AND_A_HALF); + var sut = new LazySparseFloat16Array(F16, 3, NINE, values, ints(0, 2), 0L); + + // When / Then + assertThat(sut.getFloat(0)).isEqualTo(1.5f); + assertThat(sut.getFloat(1)).isEqualTo(9.0f); + assertThat(sut.getFloat(2)).isEqualTo(2.5f); + } + + @Test + void materialize_keepsFillAndPatchBitsAndHonorsOffset() { + // Given length 3 covering abs [4..7): fill is a NaN payload, patches at abs 4 and 6 + Float16Array values = float16s(ONE_AND_A_HALF, TWO_AND_A_HALF, NINE); + var sut = new LazySparseFloat16Array(F16, 3, NAN_PAYLOAD, values, ints(1, 4, 6), 4L); + + // When + try (Arena arena = Arena.ofConfined()) { + MemorySegment result = sut.materialize(arena); + + // Then bits, not widened floats: the payload would not survive a round trip + assertThat(result.byteSize()).isEqualTo(6L); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 0)).isEqualTo(TWO_AND_A_HALF); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 1)).isEqualTo(NAN_PAYLOAD); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 2)).isEqualTo(NINE); + } + } + + @Test + void nullPatchesIsAllFill() { + // Given patchValues == null, the no-patch fast path + var sut = new LazySparseFloat16Array(F16, 3, NINE, null, null, 0L); + + // When / Then + assertThat(sut.getFloat(2)).isEqualTo(9.0f); + try (Arena arena = Arena.ofConfined()) { + assertThat(sut.materialize(arena).getAtIndex(VortexFormat.LE_SHORT, 1)).isEqualTo(NINE); + } + } + + @Test + void limited_cutsTheRowCountOnly() { + // Given + Float16Array values = float16s(ONE_AND_A_HALF); + var sut = new LazySparseFloat16Array(F16, 5, NINE, values, ints(1), 0L); + + // When + Array result = sut.limited(2); + + // Then + assertThat(result.length()).isEqualTo(2L); + assertThat(((Float16Array) result).getFloat(1)).isEqualTo(1.5f); + assertThat(sut.limited(5)).isSameAs(sut); + } + + private static Float16Array float16s(short... bits) { + return new MaterializedFloat16Array(F16, bits.length, TestSegments.leShorts(bits)); + } + } + @Nested class Float { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index f768fbca..54169442 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -10,6 +10,7 @@ import io.github.dfa1.vortex.core.proto.ProtoVarBinMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.DictFloat16Array; import io.github.dfa1.vortex.reader.array.DictByteArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DictLongArray; @@ -174,16 +175,18 @@ void nonPrimitiveNonVarBinDtype_throwsVortexException() { } @Test - void unsupportedValuePType_throws() { - // Given — F16 expands fine (2 bytes) but typedArray has no F16 mapping - MemorySegment codes = u8Codes(0, 1); - MemorySegment values = TestSegments.leShorts((short) 1, (short) 2); - DType dtype = DType.F16; + void f16Values_decodeToDictFloat16Array() { + // Given — F16 had no dict array, so the cascade wrote a dict the reader then rejected (#515) + MemorySegment codes = u8Codes(1, 0); + MemorySegment values = TestSegments.leShorts(Float.floatToFloat16(1.5f), Float.floatToFloat16(2.5f)); - // When / Then - assertThatThrownBy(() -> decodeProtoSegments(dtype, PType.U8, codes, values, 2, 2)) - .isInstanceOf(VortexException.class) - .hasMessageContaining("unsupported ptype"); + // When + Array result = decodeProtoSegments(DType.F16, PType.U8, codes, values, 2, 2); + + // Then + assertThat(result).isInstanceOf(DictFloat16Array.class); + assertThat(((DictFloat16Array) result).getFloat(0)).isEqualTo(2.5f); + assertThat(((DictFloat16Array) result).getFloat(1)).isEqualTo(1.5f); } @Test diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 3676ca10..3d67a839 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -223,7 +223,7 @@ private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { return switch (ptype) { case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(rawBits); case I8, I16, I32, I64 -> ProtoScalarValue.ofInt64Value(rawBits); - case F16 -> ProtoScalarValue.ofF16Value(rawBits & 0xFFFFL); + case F16 -> ProtoScalarValue.ofF16Value(Short.toUnsignedLong((short) rawBits)); case F32 -> ProtoScalarValue.ofF32Value(Float.intBitsToFloat((int) rawBits)); case F64 -> ProtoScalarValue.ofF64Value(Double.longBitsToDouble(rawBits)); }; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java index fd01b150..8c60cbee 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoder.java @@ -252,12 +252,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { private static int arrayLength(Object data, PType ptype) { return switch (ptype) { case I8, U8 -> ((byte[]) data).length; - case I16, U16 -> ((short[]) data).length; + case I16, U16, F16 -> ((short[]) data).length; case I32, U32 -> ((int[]) data).length; case I64, U64 -> ((long[]) data).length; case F32 -> ((float[]) data).length; case F64 -> ((double[]) data).length; - default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype: " + ptype); }; } @@ -266,13 +265,12 @@ private static long readBits(Object data, PType ptype, int i) { case I8 -> ((byte[]) data)[i]; case U8 -> Byte.toUnsignedLong(((byte[]) data)[i]); case I16 -> ((short[]) data)[i]; - case U16 -> Short.toUnsignedLong(((short[]) data)[i]); + case U16, F16 -> Short.toUnsignedLong(((short[]) data)[i]); case I32 -> ((int[]) data)[i]; case U32 -> Integer.toUnsignedLong(((int[]) data)[i]); case I64, U64 -> ((long[]) data)[i]; case F32 -> Float.floatToRawIntBits(((float[]) data)[i]); case F64 -> Double.doubleToRawLongBits(((double[]) data)[i]); - default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype: " + ptype); }; } @@ -286,9 +284,9 @@ private static ProtoScalarValue scalar(PType ptype, long bits) { return switch (ptype) { case I8, I16, I32, I64 -> ProtoScalarValue.ofInt64Value(bits); case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(bits); + case F16 -> ProtoScalarValue.ofF16Value(Short.toUnsignedLong((short) bits)); case F32 -> ProtoScalarValue.ofF32Value(Float.intBitsToFloat((int) bits)); case F64 -> ProtoScalarValue.ofF64Value(Double.longBitsToDouble(bits)); - default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported ptype: " + ptype); }; } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/Float16EncodingsRoundTripTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/Float16EncodingsRoundTripTest.java new file mode 100644 index 00000000..633f461b --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/Float16EncodingsRoundTripTest.java @@ -0,0 +1,108 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.Float16Array; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.IntUnaryOperator; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// #515: F16 had no case in the constant, dict, rle and sparse encodings, so some F16 columns +/// could not be written and others were written but could not be read back. Every shape below is +/// written with the default cascade and with each of those encodings forced, then compared as raw +/// half-precision bits, so a NaN payload rounded through `float` or a `-0.0` turned into `0.0` +/// fails too. +class Float16EncodingsRoundTripTest { + + private static final ColumnName COLUMN = ColumnName.of("h"); + private static final DType.Struct SCHEMA = new DType.Struct(List.of(COLUMN), + List.of(new DType.Primitive(PType.F16, false)), false); + private static final int ROWS = 3000; + + @TempDir + Path tmp; + + static Stream optionsAndShapes() { + Map options = new java.util.LinkedHashMap<>(); + options.put("default cascade", WriteOptions.defaults()); + for (EncodingId id : List.of(EncodingId.VORTEX_CONSTANT, EncodingId.VORTEX_DICT, + EncodingId.FASTLANES_RLE, EncodingId.VORTEX_SPARSE)) { + options.put(id.toString(), WriteOptions.defaults().withoutEditions() + .withColumnEncoding(COLUMN, ColumnEncoding.candidates(id))); + } + Random random = new Random(515); + Map shapes = new java.util.LinkedHashMap<>(); + shapes.put("constant 1.5", i -> bits(1.5f)); + shapes.put("constant NaN payload", i -> 0x7C01); + shapes.put("mostly one value", i -> i % 50 == 0 ? bits(random.nextInt(100)) : bits(1.5f)); + shapes.put("runs", i -> bits((i / 50) % 7)); + shapes.put("low cardinality", i -> bits(random.nextInt(5))); + shapes.put("signed zeros", i -> i % 2 == 0 ? 0x0000 : 0x8000); + shapes.put("ramp", i -> bits(i % 2048)); + shapes.put("random", i -> bits(random.nextFloat() * 100)); + List cases = new ArrayList<>(); + options.forEach((optionsName, writeOptions) -> shapes.forEach((shapeName, shape) -> { + short[] data = new short[ROWS]; + for (int i = 0; i < ROWS; i++) { + data[i] = (short) shape.applyAsInt(i); + } + cases.add(Arguments.of(optionsName + " / " + shapeName, writeOptions, data)); + })); + return cases.stream(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("optionsAndShapes") + void writeThenRead_keepsEveryHalfBit(String label, WriteOptions options, short[] data) throws IOException { + // Given + Path file = tmp.resolve("f16.vtx"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = VortexWriter.create(ch, SCHEMA, options)) { + sut.writeChunk(Map.of(COLUMN, data)); + } + + // When + List result = new ArrayList<>(); + try (VortexReader reader = VortexReader.open(file); + Arena arena = Arena.ofConfined()) { + reader.scan(ScanOptions.all()).forEachRemaining(chunk -> { + Float16Array column = chunk.column(COLUMN); + MemorySegment bits = column.materialize(arena); + for (long i = 0; i < column.length(); i++) { + result.add(bits.getAtIndex(VortexFormat.LE_SHORT, i)); + } + }); + } + + // Then + assertThat(result).as(label).hasSize(ROWS); + for (int i = 0; i < ROWS; i++) { + assertThat(result.get(i)).as("%s row %d", label, i).isEqualTo(data[i]); + } + } + + private static int bits(float value) { + return Short.toUnsignedInt(Float.floatToFloat16(value)); + } +} From ce8dea70ae21b20d0ee3c24b8d28b66c3ec1ff23 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 16:41:17 +0200 Subject: [PATCH 4/4] fix: write F16 min/max statistics as F16 scalars (#515) PrimitiveEncodingEncoder wrote an F16 column's array-stat min/max as f32 scalars, which Rust rejects (expected F32 dtype for F32Value, got f16), so vortex-jni could not read any F16 file Java wrote. The scalars are now f16_value, as Rust types them, and the zone-map codec and the reader's ArrayStats read that form. Float16InteropIntegrationTest checks both directions against vortex-jni (Java writes/JNI reads with the default cascade and each of constant, dict, rle and sparse forced; JNI writes/Java reads), comparing raw half bits; the old disabled javaWriter_jniReader_f16_nanAndInf test passes and is enabled. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Sko6B2519bE1TkpPx7WKxf --- CHANGELOG.md | 1 + .../Float16InteropIntegrationTest.java | 244 ++++++++++++++++++ .../JavaWritesRustReadsIntegrationTest.java | 3 +- .../github/dfa1/vortex/reader/ArrayStats.java | 3 + .../dfa1/vortex/writer/ZoneMapStatCodec.java | 9 +- .../encode/ConstantEncodingEncoder.java | 10 +- .../encode/PrimitiveEncodingEncoder.java | 8 +- .../encode/ConstantEncodingEncoderTest.java | 8 +- 8 files changed, 267 insertions(+), 19 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/Float16InteropIntegrationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index cc178097..ab3c63af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `WriteOptions` is a final class instead of a record. ([#474](https://github.com/dfa1/vortex-java/issues/474)) ### Fixed +- `F16` files written by vortex-java can be read by vortex-jni: min/max statistics were written as `f32` scalars, which Rust rejects with `expected F32 dtype for F32Value, got f16`. ([#515](https://github.com/dfa1/vortex-java/issues/515)) - `F16` columns with constant, low-cardinality, run-heavy or mostly-one-value data can be written and read back; they failed with `unsupported ptype: F16` or wrote files the reader rejected. ([#515](https://github.com/dfa1/vortex-java/issues/515)) - Pco columns follow Rust: 2^18-value chunks of 8192-value pages, Rust's IntMult filter, delta sample and histogram; integer columns are byte-identical to pco 1.0.1. ([#513](https://github.com/dfa1/vortex-java/pull/513)) - Integer columns that are an exact arithmetic sequence (timestamps at a fixed step) are written as `vortex.sequence` instead of bit-packed, as Rust does (a 4.2M-row klines file 155 → 128 MB). ([#507](https://github.com/dfa1/vortex-java/issues/507)) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/Float16InteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/Float16InteropIntegrationTest.java new file mode 100644 index 00000000..c4a781f2 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/Float16InteropIntegrationTest.java @@ -0,0 +1,244 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Expression; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.inspect.VortexInspector; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.Float16Array; +import io.github.dfa1.vortex.writer.ColumnEncoding; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.Float2Vector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.IntUnaryOperator; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// F16 against vortex-jni (#515), in both directions. Java had no F16 case in the constant, dict, +/// rle and sparse encodings, and wrote F16 min/max statistics as `f32` scalars, which Rust rejects +/// ("expected F32 dtype for F32Value, got f16"), so no F16 file Java wrote could be read by Rust. +/// Every shape is compared as raw half-precision bits, so a NaN payload or a `-0.0` that went +/// through `float` fails. +class Float16InteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final Schema ARROW_SCHEMA = new Schema(List.of( + Field.notNullable("v", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)))); + private static final ColumnName COLUMN = ColumnName.of("v"); + private static final DType.Struct SCHEMA = new DType.Struct(List.of(COLUMN), List.of(DType.F16), false); + private static final int ROWS = 3000; + + static { + NativeLoader.loadJni(); + } + + private static Map shapes() { + Random random = new Random(515); + Map generators = new LinkedHashMap<>(); + generators.put("constant 1.5", i -> bits(1.5f)); + generators.put("constant NaN payload", i -> 0x7C01); + generators.put("mostly one value", i -> i % 50 == 0 ? bits(random.nextInt(100)) : bits(1.5f)); + generators.put("runs", i -> bits((i / 50) % 7)); + generators.put("low cardinality", i -> bits(random.nextInt(5))); + generators.put("signed zeros", i -> i % 2 == 0 ? 0x0000 : 0x8000); + generators.put("ramp", i -> bits(i % 2048)); + generators.put("random", i -> bits(random.nextFloat() * 100)); + Map result = new LinkedHashMap<>(); + generators.forEach((name, generator) -> { + short[] data = new short[ROWS]; + for (int i = 0; i < ROWS; i++) { + data[i] = (short) generator.applyAsInt(i); + } + result.put(name, data); + }); + return result; + } + + static Stream shapeArguments() { + return shapes().entrySet().stream().map(e -> Arguments.of(e.getKey(), e.getValue())); + } + + /// Each encoding forced onto the shape it exists for, plus the default cascade on every shape: + /// the forced runs prove the file really holds that encoding, not a canonical fallback. + static Stream javaWritesCases() { + Map shapes = shapes(); + List cases = new ArrayList<>(); + shapes.forEach((shape, data) -> cases.add(Arguments.of("default cascade / " + shape, WriteOptions.defaults(), null, data))); + cases.add(forced(EncodingId.VORTEX_CONSTANT, "constant 1.5", shapes)); + cases.add(forced(EncodingId.VORTEX_CONSTANT, "constant NaN payload", shapes)); + cases.add(forced(EncodingId.VORTEX_DICT, "low cardinality", shapes)); + cases.add(forced(EncodingId.VORTEX_DICT, "runs", shapes)); + cases.add(forced(EncodingId.FASTLANES_RLE, "runs", shapes)); + cases.add(forced(EncodingId.FASTLANES_RLE, "signed zeros", shapes)); + cases.add(forced(EncodingId.VORTEX_SPARSE, "mostly one value", shapes)); + return cases.stream(); + } + + private static Arguments forced(EncodingId id, String shape, Map shapes) { + WriteOptions options = WriteOptions.defaults().withoutEditions() + .withColumnEncoding(COLUMN, ColumnEncoding.candidates(id)); + return Arguments.of(id + " / " + shape, options, id.toString(), shapes.get(shape)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("javaWritesCases") + void javaWriter_jniReader_keepsEveryHalfBit(String label, WriteOptions options, String expectedEncoding, + short[] data, @TempDir Path tmp) throws IOException { + // Given + Path file = tmp.resolve("java_f16.vtx"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = io.github.dfa1.vortex.writer.VortexWriter.create(ch, SCHEMA, options)) { + sut.writeChunk(Map.of(COLUMN, data)); + } + + // When + short[] result = readWithJni(file); + + // Then + assertThat(result).as(label).containsExactly(data); + if (expectedEncoding != null) { + try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll())) { + assertThat(VortexInspector.inspect(reader)).as("%s: the file must hold the forced encoding", label) + .contains(expectedEncoding); + } + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("shapeArguments") + void jniWriter_javaReader_keepsEveryHalfBit(String shape, short[] data, @TempDir Path tmp) throws IOException { + // Given Rust's compressor picks whatever encoding it likes for the shape + Path file = tmp.resolve("jni_f16.vtx"); + writeWithJni(file, data); + + // When + short[] result = readWithJava(file); + + // Then + assertThat(result).as(shape).containsExactly(data); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 1023, 1024, 1025}) + void jniWriter_javaReader_constantColumnOfAnyLength(int rows, @TempDir Path tmp) throws IOException { + // Given a constant column, which Rust writes as vortex.constant, at lengths around the + // 1024-row FastLanes chunk + short[] data = new short[rows]; + java.util.Arrays.fill(data, Float.floatToFloat16(1.5f)); + Path file = tmp.resolve("jni_f16_constant.vtx"); + writeWithJni(file, data); + + // When + short[] result = readWithJava(file); + + // Then + assertThat(result).containsExactly(data); + } + + private static void writeWithJni(Path file, short[] data) throws IOException { + String uri = file.toAbsolutePath().toUri().toString(); + try (VortexWriter writer = VortexWriter.builder(SESSION, uri, ARROW_SCHEMA, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(ARROW_SCHEMA, ALLOCATOR)) { + Float2Vector vec = (Float2Vector) root.getVector("v"); + vec.allocateNew(data.length); + for (int i = 0; i < data.length; i++) { + vec.setSafe(i, data[i]); + } + root.setRowCount(data.length); + try (ArrowArray arr = ArrowArray.allocateNew(ALLOCATOR); + ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, arr, schema); + writer.writeBatch(arr.memoryAddress(), schema.memoryAddress()); + } + } + } + + private static short[] readWithJava(Path file) throws IOException { + List bits = new ArrayList<>(); + try (VortexReader reader = VortexReader.open(file, ReadRegistry.loadAll()); + Arena arena = Arena.ofConfined()) { + reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all()).forEachRemaining(chunk -> { + Float16Array column = chunk.column(COLUMN); + MemorySegment segment = column.materialize(arena); + for (long i = 0; i < column.length(); i++) { + bits.add(segment.getAtIndex(VortexFormat.LE_SHORT, i)); + } + }); + } + return toArray(bits); + } + + private static short[] readWithJni(Path file) throws IOException { + String uri = file.toAbsolutePath().toUri().toString(); + ScanOptions options = ScanOptions.builder() + .projection(Expression.select(new String[]{"v"}, Expression.root())) + .build(); + List bits = new ArrayList<>(); + DataSource source = DataSource.open(SESSION, uri); + Scan scan = source.scan(options); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + Float2Vector vec = (Float2Vector) root.getVector("v"); + for (int i = 0; i < root.getRowCount(); i++) { + bits.add(vec.get(i)); + } + } + } + } + return toArray(bits); + } + + private static short[] toArray(List bits) { + short[] result = new short[bits.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = bits.get(i); + } + return result; + } + + private static int bits(float value) { + return Short.toUnsignedInt(Float.floatToFloat16(value)); + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index e8776d0b..a6952638 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -1393,8 +1393,7 @@ void javaWriter_jniReader_f32_nanAndInf(@TempDir Path tmp) throws IOException { } /// F16: PrimitiveEncoding stores raw short bits — NaN and Inf are just bit patterns. - /// Disabled: vortex-jni does not export F16 via Arrow C Data Interface (not fixed as of 0.74.0). - @Disabled + /// Failed until #515: Java wrote F16 min/max statistics as f32 scalars, which Rust rejects. @Test void javaWriter_jniReader_f16_nanAndInf(@TempDir Path tmp) throws IOException { // Given diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java index 7ae682f6..0dbd2137 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java @@ -65,6 +65,9 @@ private static Object decodeScalar(MemorySegment seg) { if (sv.uint64_value() != null) { return sv.uint64_value(); } + if (sv.f16_value() != null) { + return Float.float16ToFloat((short) sv.f16_value().longValue()); + } if (sv.f32_value() != null) { return sv.f32_value(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java index 2be0a552..98676721 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapStatCodec.java @@ -266,7 +266,7 @@ private static Object statColumn(PType ptype, List statBytes, boolean[] yield a; } case F16 -> { - // F16 min/max are serialized as f32 scalars; re-pack to float16 storage. + // F16 min/max are f16 scalars; widen and re-pack to float16 storage. short[] a = new short[n]; for (int i = 0; i < n; i++) { valid[i] = statBytes.get(i) != null; @@ -284,12 +284,15 @@ private static long scalarLong(byte[] bytes) throws IOException { } private static double scalarDouble(byte[] bytes) throws IOException { - // Float columns serialize min/max as f64 (F64) or f32 (F32). Branch rather than use a - // ternary so the F32 path widens Float -> double explicitly instead of mixing boxed types. + // Float columns serialize min/max as f64 (F64), f32 (F32) or f16 (F16, the half bits). + // Branch rather than use a ternary so each path widens explicitly instead of mixing boxed types. ProtoScalarValue sv = decodeScalar(bytes); if (sv.f64_value() != null) { return sv.f64_value(); } + if (sv.f16_value() != null) { + return Float.float16ToFloat((short) sv.f16_value().longValue()); + } return sv.f32_value(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 3d67a839..2057a8fe 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -71,8 +71,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { byte[] scalarBytes = scalar.encode(); // A constant array's min and max are both the one repeated value, by construction -- no // scan needed. Empty arrays report no stats, matching every other encoder's convention. - // F16 min/max travel as f32 scalars, as PrimitiveEncodingEncoder writes them. - byte[] stats = Array.getLength(data) > 0 ? statsScalar(ptype, firstRaw, scalarBytes) : null; + byte[] stats = Array.getLength(data) > 0 ? scalarBytes : null; return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalarBytes)), stats, stats); } @@ -212,13 +211,6 @@ private static OptionalLong constantBits(Object data, PType ptype) { }; } - private static byte[] statsScalar(PType ptype, long rawBits, byte[] scalarBytes) { - if (ptype != PType.F16) { - return scalarBytes; - } - return ProtoScalarValue.ofF32Value(Float.float16ToFloat((short) rawBits)).encode(); - } - private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { return switch (ptype) { case U8, U16, U32, U64 -> ProtoScalarValue.ofUint64Value(rawBits); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index 70db78f7..a061d897 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -235,7 +235,7 @@ public static byte[][] minMaxStats(PType ptype, Object data) { max = fv; } } - yield min > max ? null : new byte[][]{scalarF32(min), scalarF32(max)}; + yield min > max ? null : new byte[][]{scalarF16(min), scalarF16(max)}; } }; } @@ -411,6 +411,12 @@ private static byte[] scalarU64(long v) { return ProtoScalarValue.ofUint64Value(v).encode(); } + // Rust types an F16 column's min/max as F16 scalars (`f16_value`, the half bits as a u64); an + // f32 scalar on an F16 column is rejected on read ("expected F32 dtype for F32Value, got f16"). + private static byte[] scalarF16(float v) { + return ProtoScalarValue.ofF16Value(Short.toUnsignedLong(Float.floatToFloat16(v))).encode(); + } + private static byte[] scalarF32(float v) { return ProtoScalarValue.ofF32Value(v).encode(); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 9f292940..04787d81 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -417,16 +417,16 @@ void encode_f64_reportsValueAsMinAndMax() throws java.io.IOException { } @Test - void encode_f16_reportsValueAsF32MinAndMax() throws java.io.IOException { - // Given — F16 min/max travel as f32 scalars, as PrimitiveEncodingEncoder writes them + void encode_f16_reportsValueAsF16MinAndMax() throws java.io.IOException { + // Given — Rust types an F16 column's min/max as F16 scalars; an f32 one is rejected on read short[] data = {Float.floatToFloat16(2.5f), Float.floatToFloat16(2.5f)}; // When EncodeResult result = ENCODER.encode(DTypes.F16, data, EncodeTestHelper.testCtx()); // Then - assertThat(scalar(result.statsMin()).f32_value()).isEqualTo(2.5f); - assertThat(scalar(result.statsMax()).f32_value()).isEqualTo(2.5f); + assertThat(scalar(result.statsMin()).f16_value()).isEqualTo(0x4100L); + assertThat(scalar(result.statsMax()).f16_value()).isEqualTo(0x4100L); } @Test