From 5ec6377d1d4f5f1ce237d6de3762aa3f37572d7f Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 17:07:34 +0200 Subject: [PATCH 1/5] refactor: rename the SIMD kernel classes (#516) ScalarOperations -> AutoVectorizedSimdOperations: the default is C2 auto-vectorized loops, and "Scalar" collides with Vortex's scalar values. VectorSupport -> SimdOperationsSupport, operations() -> preferred() (JDK ArraysSupport / VectorSpecies.ofPreferred vocabulary). The selector drops nothing else: it only ever exposed operations(). Pure rename, no behavior change. Package stays core.simd. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 4 ++-- .../vortex/core/compute/PrimitiveArrays.java | 6 +++--- ...java => AutoVectorizedSimdOperations.java} | 7 ++++--- .../dfa1/vortex/core/simd/SimdOperations.java | 6 +++--- .../core/simd/SimdOperationsSupport.java | 21 +++++++++++++++++++ .../dfa1/vortex/core/simd/VectorSupport.java | 21 ------------------- .../vortex/core/simd/SimdOperationsTest.java | 2 +- .../performance/MaxUnsignedBenchmark.java | 4 ++-- .../performance/SimdKernelBenchmark.java | 4 ++-- .../reader/decode/DeltaEncodingDecoder.java | 4 ++-- .../reader/decode/DictEncodingDecoder.java | 4 ++-- .../encode/BitpackedEncodingEncoder.java | 4 ++-- .../writer/encode/DeltaEncodingEncoder.java | 4 ++-- 13 files changed, 46 insertions(+), 45 deletions(-) rename core/src/main/java/io/github/dfa1/vortex/core/simd/{ScalarOperations.java => AutoVectorizedSimdOperations.java} (96%) create mode 100644 core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java delete mode 100644 core/src/main/java/io/github/dfa1/vortex/core/simd/VectorSupport.java diff --git a/CLAUDE.md b/CLAUDE.md index 9c5b2250e..6b9c90ac4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,8 +39,8 @@ core — everything lives under `io.github.dfa1.vortex.core.*`: core.io IoBounds, PTypeIO, VortexFormat core.error VortexException core.compute FastLanes, PrimitiveArrays, Utf8Order - core.simd SimdOperations (kernels shared by reader+writer), VectorSupport (picks the impl); - scalar only today, Vector API impl planned (ADR 0005, #483, #484) + core.simd SimdOperations (kernels shared by reader+writer), SimdOperationsSupport#preferred() (picks the impl); + best-effort (C2 auto-vectorized) only today, Vector API impl planned (ADR 0005, #483, #484) core.fbs / core.proto — generated wire codecs + their runtimes reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, Zone, ScanOptions, RowFilter; file internals (Footer, Trailer, PostscriptParser, …) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java index 03352c3ec..7d754b385 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -159,7 +159,7 @@ public static void toLongsInto(MemorySegment seg, long fromElement, int count, P if (ptype.isFloating()) { throw new VortexException(encoding, "unsupported ptype: " + ptype); } - VectorSupport.operations().widenInto(seg, fromElement, count, ptype, out); + SimdOperationsSupport.preferred().widenInto(seg, fromElement, count, ptype, out); } /// Writes a `long[]` to a freshly allocated little-endian off-heap segment whose element width @@ -179,7 +179,7 @@ public static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocato } int n = longs.length; MemorySegment seg = arena.allocate((long) n * ptype.byteSize()); - VectorSupport.operations().narrowInto(longs, ptype, seg); + SimdOperationsSupport.preferred().narrowInto(longs, ptype, seg); return seg; } diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/ScalarOperations.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java similarity index 96% rename from core/src/main/java/io/github/dfa1/vortex/core/simd/ScalarOperations.java rename to core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java index 555b61eef..d1bc265bf 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/ScalarOperations.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java @@ -7,9 +7,10 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -/// The scalar [SimdOperations]: one specialized loop per width with the `ptype` switch hoisted -/// out (CLAUDE.md hot-loop rule), so each body is uniform and left to C2 superword. -final class ScalarOperations implements SimdOperations { +/// The auto-vectorized [SimdOperations]: plain Java loops that rely on C2 auto-vectorization, with one +/// specialized loop per width and the `ptype` switch hoisted out (CLAUDE.md hot-loop rule), so each +/// body is uniform. +final class AutoVectorizedSimdOperations implements SimdOperations { @Override public void widenInto(MemorySegment src, long fromElement, int count, PType ptype, long[] out) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java index 98084cba4..df996dca6 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java @@ -5,11 +5,11 @@ import java.lang.foreign.MemorySegment; /// The vectorizable kernels shared by the reader and the writer: the tight loops where a SIMD -/// implementation can replace the scalar one without changing a single output byte. +/// implementation can replace the auto-vectorized one without changing a single output byte. /// -/// [ScalarOperations] holds the plain loops, written to the hot-loop rule so C2 can auto-vectorize +/// [AutoVectorizedSimdOperations] holds the plain loops, written to the hot-loop rule so C2 can auto-vectorize /// them. It is always complete, so correctness never depends on which implementation -/// [VectorSupport#operations()] returns. Callers validate arguments and attribute errors; kernels +/// [SimdOperationsSupport#preferred()] returns. Callers validate arguments and attribute errors; kernels /// assume well-formed input and throw only [IllegalArgumentException] for a `ptype` they do not cover. public interface SimdOperations { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java new file mode 100644 index 000000000..c2e13ed79 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java @@ -0,0 +1,21 @@ +package io.github.dfa1.vortex.core.simd; + +/// Selects the [SimdOperations] implementation, once per JVM. +/// +/// Only the auto-vectorized kernels exist today. The Vector API implementation +/// (ADR 0005, #483, #484) will be chosen here by a guarded static initializer that falls back to them when +/// `jdk.incubator.vector` is absent, so no call site changes when it lands. +public final class SimdOperationsSupport { + + private static final SimdOperations INSTANCE = new AutoVectorizedSimdOperations(); + + private SimdOperationsSupport() { + } + + /// Returns the kernel implementation in effect for this JVM. + /// + /// @return the shared, stateless [SimdOperations] + public static SimdOperations preferred() { + return INSTANCE; + } +} diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorSupport.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorSupport.java deleted file mode 100644 index beff6a089..000000000 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorSupport.java +++ /dev/null @@ -1,21 +0,0 @@ -package io.github.dfa1.vortex.core.simd; - -/// Selects the [SimdOperations] implementation, once per JVM. -/// -/// Only the scalar kernels exist today. The Vector API implementation (ADR 0005, #483, #484) will be -/// chosen here by a guarded static initializer that falls back to scalar when -/// `jdk.incubator.vector` is absent, so no call site changes when it lands. -public final class VectorSupport { - - private static final SimdOperations INSTANCE = new ScalarOperations(); - - private VectorSupport() { - } - - /// Returns the kernel implementation in effect for this JVM. - /// - /// @return the shared, stateless [SimdOperations] - public static SimdOperations operations() { - return INSTANCE; - } -} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java index 24f77704a..e1f7e8363 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java @@ -24,7 +24,7 @@ class SimdOperationsTest { private static final long[] VALUES = {0L, -1L, 1L, 0x80L, 0xFFL, 0x8000L, 0xFFFFL, 0x80000000L, 0xFFFFFFFFL, Long.MIN_VALUE, Long.MAX_VALUE, 0x123456789ABCDEFL}; - private final SimdOperations sut = VectorSupport.operations(); + private final SimdOperations sut = SimdOperationsSupport.preferred(); @ParameterizedTest @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) diff --git a/performance/src/main/java/io/github/dfa1/vortex/performance/MaxUnsignedBenchmark.java b/performance/src/main/java/io/github/dfa1/vortex/performance/MaxUnsignedBenchmark.java index c299490b4..c1ddcc899 100644 --- a/performance/src/main/java/io/github/dfa1/vortex/performance/MaxUnsignedBenchmark.java +++ b/performance/src/main/java/io/github/dfa1/vortex/performance/MaxUnsignedBenchmark.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.simd.SimdOperations; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -39,7 +39,7 @@ public class MaxUnsignedBenchmark { @Param({"U8", "U16", "U32"}) public PType ptype; - private final SimdOperations ops = VectorSupport.operations(); + private final SimdOperations ops = SimdOperationsSupport.preferred(); private Arena arena; private MemorySegment codes; diff --git a/performance/src/main/java/io/github/dfa1/vortex/performance/SimdKernelBenchmark.java b/performance/src/main/java/io/github/dfa1/vortex/performance/SimdKernelBenchmark.java index 0d66e5d9c..4d8f03dd4 100644 --- a/performance/src/main/java/io/github/dfa1/vortex/performance/SimdKernelBenchmark.java +++ b/performance/src/main/java/io/github/dfa1/vortex/performance/SimdKernelBenchmark.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.compute.FastLanes; import io.github.dfa1.vortex.core.simd.SimdOperations; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -34,7 +34,7 @@ public class SimdKernelBenchmark { @Param({"8", "16", "32", "64"}) public int typeBits; - private final SimdOperations ops = VectorSupport.operations(); + private final SimdOperations ops = SimdOperationsSupport.preferred(); private long[] input; private long[] bases; private long[] out; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index 6d57efe4e..36d6cdd2e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.FastLanes; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; @@ -107,7 +107,7 @@ public Array decode(DecodeContext ctx) { for (long chunk = firstChunk; chunk <= lastChunk; chunk++) { readElements(basesSeg, ptype, basesCap, chunk * lanes, lanes, chunkBases); readElements(deltasSeg, ptype, deltasCap, chunk * FastLanes.CHUNK, FastLanes.CHUNK, chunkDeltas); - VectorSupport.operations().undeltaChunk(chunkDeltas, chunkBases, lanes, typeBits, mask, chunkUndelta); + SimdOperationsSupport.preferred().undeltaChunk(chunkDeltas, chunkBases, lanes, typeBits, mask, chunkUndelta); scatterChunk(out, ptype, chunkUndelta, chunk * FastLanes.CHUNK - offset, rowCount); } // A nullable column carries its validity as an optional third child, not under a diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 63774c9dc..d6f4b85ff 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -224,7 +224,7 @@ private static void validateCodesInRange(Array codes, PType codePType, long pool } Optional segment = codes.segmentIfPresent(); long max = segment.isPresent() - ? VectorSupport.operations().maxUnsigned(segment.get(), + ? SimdOperationsSupport.preferred().maxUnsigned(segment.get(), Math.min(n, segment.get().byteSize() / codePType.byteSize()), codePType) : maxCode(codes); if (Long.compareUnsigned(max, poolLength) >= 0) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java index f5ec3c865..cb0277715 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java @@ -9,7 +9,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.simd.SimdOperations; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; @@ -202,7 +202,7 @@ private static MemorySegment packFastLanes(long[] values, int n, int bitWidth, i long[] words = new long[wordsPerBlock]; // The kernel packs full blocks only; a partial tail block is zero-padded into scratch. long[] tail = null; - SimdOperations ops = VectorSupport.operations(); + SimdOperations ops = SimdOperationsSupport.preferred(); for (int block = 0; block < blockCount; block++) { int blockStart = block * 1024; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java index ecc53ff9e..a46b37947 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.FastLanes; -import io.github.dfa1.vortex.core.simd.VectorSupport; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; @@ -139,7 +139,7 @@ private static Deltas deltas(long[] longs, PType ptype) { int basesOff = chunk * lanes; System.arraycopy(transposed, 0, basesAll, basesOff, lanes); System.arraycopy(basesAll, basesOff, chunkBases, 0, lanes); - VectorSupport.operations().deltaChunk(transposed, chunkBases, lanes, typeBits, mask, chunkDelta); + SimdOperationsSupport.preferred().deltaChunk(transposed, chunkBases, lanes, typeBits, mask, chunkDelta); System.arraycopy(chunkDelta, 0, deltasAll, chunk * FastLanes.CHUNK, FastLanes.CHUNK); } From 64047da6961e6b8b492cc57c513d7bf6c1a73cd4 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 22:32:00 +0200 Subject: [PATCH 2/5] refactor: one SimdOperations kernel per primitive-array loop, for all ptypes (#516) The writer and the reader each carried their own copies of the same loops: min/max in ArrayStats and again in PrimitiveEncodingEncoder, three widening loops, four all-values-equal checks, the run counter, the zone-map sums, the dictionary code bound. They now live in SimdOperations (minMax, sum, sumFloating, runs, allEqual, widenInto and widenArrayInto, narrowInto and narrowArrayInto, maxUnsigned) and the callers use them: ArrayStats, PrimitiveEncodingEncoder, PrimitiveArrays, ConstantEncodingEncoder, MaskedEncodingEncoder, AlpRdEncodingEncoder, FrameOfReferenceEncodingEncoder, ZigZagEncodingEncoder, ZoneMapStats and DictLayoutDecoder. Every kernel accepts all eleven ptypes: floats widen as their raw bits and narrow back into float carriers, the 8-byte types narrow as a copy, maxUnsigned covers U64, and minMax covers each type in its natural order. Comparing with the Rust source turned up two statistics differences, fixed here: float min/max now order -0.0 before 0.0 and skip NaN (Rust's total_compare with skip_nans; the old first-zero rule was a divergence), and U64 min/max and ArrayStats' dense span are unsigned. Rust's zone Sum also skips NaN and ours does not; that is #523, documented in docs/compatibility.md and left alone. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01JbquncyA7mTMJbDJo1s5HD --- CHANGELOG.md | 3 + CLAUDE.md | 5 +- .../vortex/core/compute/PrimitiveArrays.java | 90 +-- .../simd/AutoVectorizedSimdOperations.java | 559 +++++++++++++++++- .../dfa1/vortex/core/simd/SimdOperations.java | 99 +++- .../vortex/core/simd/SimdOperationsTest.java | 474 ++++++++++++++- docs/compatibility.md | 12 + .../reader/layout/DictLayoutDecoder.java | 29 +- .../writer/encode/AlpRdEncodingEncoder.java | 12 +- .../dfa1/vortex/writer/encode/ArrayStats.java | 189 +----- .../encode/ConstantEncodingEncoder.java | 97 +-- .../FrameOfReferenceEncodingEncoder.java | 9 +- .../writer/encode/MaskedEncodingEncoder.java | 16 +- .../encode/PrimitiveEncodingEncoder.java | 347 +---------- .../writer/encode/ZigZagEncodingEncoder.java | 7 +- .../vortex/writer/encode/ZoneMapStats.java | 15 +- 16 files changed, 1218 insertions(+), 745 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 791163d9b..099a1a3fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Float min/max statistics order `-0.0` before `0.0` and skip `NaN`, and `U64` min/max compare unsigned, as Rust does. ([#516](https://github.com/dfa1/vortex-java/issues/516)) + ### Fixed - Nulls written by Rust come back as null, not as zeros or empty strings, in fixed-size list, FSST, decimal, bytebool, varbinview and delta columns, and a fixed-size list can hold nullable elements or a list, struct or fixed-size list as its element. ([0c910d2f](https://github.com/dfa1/vortex-java/commit/0c910d2f), [4404effe](https://github.com/dfa1/vortex-java/commit/4404effe), [6018bd57](https://github.com/dfa1/vortex-java/commit/6018bd57), [581bf894](https://github.com/dfa1/vortex-java/commit/581bf894), [692c991f](https://github.com/dfa1/vortex-java/commit/692c991f), [7079aed0](https://github.com/dfa1/vortex-java/commit/7079aed0), [c74e7432](https://github.com/dfa1/vortex-java/commit/c74e7432), [07171ffa](https://github.com/dfa1/vortex-java/commit/07171ffa)) diff --git a/CLAUDE.md b/CLAUDE.md index 6b9c90ac4..734f33796 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,8 +39,9 @@ core — everything lives under `io.github.dfa1.vortex.core.*`: core.io IoBounds, PTypeIO, VortexFormat core.error VortexException core.compute FastLanes, PrimitiveArrays, Utf8Order - core.simd SimdOperations (kernels shared by reader+writer), SimdOperationsSupport#preferred() (picks the impl); - best-effort (C2 auto-vectorized) only today, Vector API impl planned (ADR 0005, #483, #484) + core.simd SimdOperations (kernels shared by reader+writer, all eleven ptypes), + AutoVectorizedSimdOperations (C2-shaped loops, the Rust-parity default), + SimdOperationsSupport#preferred() (picks the impl); Vector API impl planned (ADR 0005, #483, #484) core.fbs / core.proto — generated wire codecs + their runtimes reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, Zone, ScanOptions, RowFilter; file internals (Footer, Trailer, PostscriptParser, …) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java index 7d754b385..d74ee291e 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.simd.SimdOperations; import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; @@ -34,58 +35,15 @@ private PrimitiveArrays() { /// @return a `long[]` holding every element of `data` widened to 64 bits /// @throws VortexException if `ptype` is not an integer ptype public static long[] toLongs(Object data, PType ptype, EncodingId encoding) { - return switch (ptype) { - case I8 -> { - byte[] arr = (byte[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = arr[i]; - } - yield r; - } - case U8 -> { - byte[] arr = (byte[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = Byte.toUnsignedLong(arr[i]); - } - yield r; - } - case I16 -> { - short[] arr = (short[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = arr[i]; - } - yield r; - } - case U16 -> { - short[] arr = (short[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = Short.toUnsignedLong(arr[i]); - } - yield r; - } - case I32 -> { - int[] arr = (int[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = arr[i]; - } - yield r; - } - case U32 -> { - int[] arr = (int[]) data; - long[] r = new long[arr.length]; - for (int i = 0; i < arr.length; i++) { - r[i] = Integer.toUnsignedLong(arr[i]); - } - yield r; - } - case I64, U64 -> (long[]) data; - default -> throw new VortexException(encoding, "unsupported ptype: " + ptype); - }; + if (ptype.isFloating()) { + throw new VortexException(encoding, "unsupported ptype: " + ptype); + } + if (ptype == PType.I64 || ptype == PType.U64) { + return (long[]) data; + } + long[] result = new long[Array.getLength(data)]; + SimdOperationsSupport.preferred().widenArrayInto(data, 0, result.length, ptype, result); + return result; } /// Throws unless `ptype` is one of the four unsigned integer types. Several wire fields (run @@ -220,26 +178,21 @@ public static MemorySegment toSegment(Object data, PType ptype, SegmentAllocator /// @throws VortexException for floating-point or other non-integer types public static Object fromLongsArray(long[] longs, PType ptype, EncodingId encoding) { int n = longs.length; + SimdOperations ops = SimdOperationsSupport.preferred(); return switch (ptype) { case I8, U8 -> { byte[] r = new byte[n]; - for (int i = 0; i < n; i++) { - r[i] = (byte) longs[i]; - } + ops.narrowArrayInto(longs, ptype, r); yield r; } case I16, U16 -> { short[] r = new short[n]; - for (int i = 0; i < n; i++) { - r[i] = (short) longs[i]; - } + ops.narrowArrayInto(longs, ptype, r); yield r; } case I32, U32 -> { int[] r = new int[n]; - for (int i = 0; i < n; i++) { - r[i] = (int) longs[i]; - } + ops.narrowArrayInto(longs, ptype, r); yield r; } case I64, U64 -> longs; @@ -259,20 +212,21 @@ public static Object fromLongsArray(long[] longs, PType ptype, EncodingId encodi /// @throws VortexException for a type with no primitive carrier public static Object fromBitsArray(long[] bits, PType ptype, EncodingId encoding) { int n = bits.length; + SimdOperations ops = SimdOperationsSupport.preferred(); return switch (ptype) { - case F16 -> fromLongsArray(bits, PType.U16, encoding); + case F16 -> { + short[] r = new short[n]; + ops.narrowArrayInto(bits, ptype, r); + yield r; + } case F32 -> { float[] r = new float[n]; - for (int i = 0; i < n; i++) { - r[i] = Float.intBitsToFloat((int) bits[i]); - } + ops.narrowArrayInto(bits, ptype, r); yield r; } case F64 -> { double[] r = new double[n]; - for (int i = 0; i < n; i++) { - r[i] = Double.longBitsToDouble(bits[i]); - } + ops.narrowArrayInto(bits, ptype, r); yield r; } default -> fromLongsArray(bits, ptype, encoding); diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java index d1bc265bf..8b03c8d8d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/AutoVectorizedSimdOperations.java @@ -6,12 +6,17 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.lang.reflect.Array; +import java.util.OptionalLong; /// The auto-vectorized [SimdOperations]: plain Java loops that rely on C2 auto-vectorization, with one /// specialized loop per width and the `ptype` switch hoisted out (CLAUDE.md hot-loop rule), so each /// body is uniform. final class AutoVectorizedSimdOperations implements SimdOperations { + /// Elements compared per branch-free block of [#allEqual(Object, PType)]. + private static final int EQUAL_BLOCK = 256; + @Override public void widenInto(MemorySegment src, long fromElement, int count, PType ptype, long[] out) { switch (ptype) { @@ -30,7 +35,7 @@ public void widenInto(MemorySegment src, long fromElement, int count, PType ptyp out[i] = src.getAtIndex(VortexFormat.LE_SHORT, fromElement + i); } } - case U16 -> { + case U16, F16 -> { for (int i = 0; i < count; i++) { out[i] = Short.toUnsignedLong(src.getAtIndex(VortexFormat.LE_SHORT, fromElement + i)); } @@ -40,17 +45,71 @@ public void widenInto(MemorySegment src, long fromElement, int count, PType ptyp out[i] = src.getAtIndex(VortexFormat.LE_INT, fromElement + i); } } - case U32 -> { + case U32, F32 -> { for (int i = 0; i < count; i++) { out[i] = Integer.toUnsignedLong(src.getAtIndex(VortexFormat.LE_INT, fromElement + i)); } } - case I64, U64 -> { + case I64, U64, F64 -> { for (int i = 0; i < count; i++) { out[i] = src.getAtIndex(VortexFormat.LE_LONG, fromElement + i); } } - default -> throw new IllegalArgumentException("not an integer ptype: " + ptype); + } + } + + @Override + public void widenArrayInto(Object values, int from, int count, PType ptype, long[] out) { + switch (ptype) { + case I8 -> { + byte[] a = (byte[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i]; + } + } + case U8 -> { + byte[] a = (byte[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i] & 0xFFL; + } + } + case I16 -> { + short[] a = (short[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i]; + } + } + case U16, F16 -> { + short[] a = (short[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i] & 0xFFFFL; + } + } + case I32 -> { + int[] a = (int[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i]; + } + } + case U32 -> { + int[] a = (int[]) values; + for (int i = 0; i < count; i++) { + out[i] = a[from + i] & 0xFFFF_FFFFL; + } + } + case I64, U64 -> System.arraycopy((long[]) values, from, out, 0, count); + case F32 -> { + float[] a = (float[]) values; + for (int i = 0; i < count; i++) { + out[i] = Float.floatToRawIntBits(a[from + i]) & 0xFFFF_FFFFL; + } + } + case F64 -> { + double[] a = (double[]) values; + for (int i = 0; i < count; i++) { + out[i] = Double.doubleToRawLongBits(a[from + i]); + } + } } } @@ -73,7 +132,55 @@ public void narrowInto(long[] values, PType ptype, MemorySegment dst) { dst.setAtIndex(VortexFormat.LE_INT, i, (int) values[i]); } } - default -> throw new IllegalArgumentException("not a 1-4 byte ptype: " + ptype); + case I64, U64, F64 -> { + for (int i = 0; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_LONG, i, values[i]); + } + } + } + } + + @Override + public void narrowArrayInto(long[] values, PType ptype, Object out) { + int n = values.length; + switch (ptype) { + case I8, U8 -> { + byte[] a = (byte[]) out; + for (int i = 0; i < n; i++) { + a[i] = (byte) values[i]; + } + } + case I16, U16 -> { + short[] a = (short[]) out; + for (int i = 0; i < n; i++) { + a[i] = (short) values[i]; + } + } + case I32, U32 -> { + int[] a = (int[]) out; + for (int i = 0; i < n; i++) { + a[i] = (int) values[i]; + } + } + case I64, U64 -> System.arraycopy(values, 0, (long[]) out, 0, n); + case F16 -> { + short[] a = (short[]) out; + for (int i = 0; i < n; i++) { + a[i] = (short) values[i]; + } + } + case F32 -> { + float[] a = (float[]) out; + for (int i = 0; i < n; i++) { + a[i] = Float.intBitsToFloat((int) values[i]); + } + } + case F64 -> { + double[] a = (double[]) out; + for (int i = 0; i < n; i++) { + a[i] = Double.longBitsToDouble(values[i]); + } + } } } @@ -87,7 +194,8 @@ public long maxUnsigned(MemorySegment src, long count, PType ptype) { case U8 -> maxU8(src, count); case U16 -> maxU16(src, count); case U32 -> maxU32(src, count); - default -> throw new IllegalArgumentException("not a U8/U16/U32 ptype: " + ptype); + case U64 -> maxU64(src, count); + default -> throw new IllegalArgumentException("not an unsigned integer ptype: " + ptype); }; } @@ -127,6 +235,16 @@ private static long maxU16(MemorySegment src, long count) { return Math.max(Math.max(m0, m1), Math.max(m2, m3)); } + // Flipping the sign bit orders the signed max as unsigned; NEON has no 64-bit max, so this is a + // scalar compare-and-select loop whatever the flip. + private static long maxU64(MemorySegment src, long count) { + long max = Long.MIN_VALUE; + for (long i = 0; i < count; i++) { + max = Math.max(max, src.getAtIndex(VortexFormat.LE_LONG, i) ^ Long.MIN_VALUE); + } + return max ^ Long.MIN_VALUE; + } + private static long maxU32(MemorySegment src, long count) { int max = Integer.MIN_VALUE; for (long i = 0; i < count; i++) { @@ -135,6 +253,435 @@ private static long maxU32(MemorySegment src, long count) { return Integer.toUnsignedLong(max ^ Integer.MIN_VALUE); } + // Reduced in the int domain: NEON has no 64-bit integer min/max, so C2 vectorizes these loops for + // `int`, `short` and `byte` but not over a widened `long` buffer (#476). Unsigned widths are masked + // in the loop, which keeps them in their carrier's lanes. + @Override + public long[] minMax(Object values, PType ptype) { + if (Array.getLength(values) == 0) { + if (ptype.isFloating()) { + return new long[0]; + } + throw new IllegalArgumentException("empty array has no min/max"); + } + return switch (ptype) { + case I8 -> { + byte[] a = (byte[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (byte v : a) { + min = Math.min(min, v); + max = Math.max(max, v); + } + yield new long[]{min, max}; + } + case U8 -> { + byte[] a = (byte[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (byte v : a) { + min = Math.min(min, v & 0xFF); + max = Math.max(max, v & 0xFF); + } + yield new long[]{min, max}; + } + case I16 -> { + short[] a = (short[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (short v : a) { + min = Math.min(min, v); + max = Math.max(max, v); + } + yield new long[]{min, max}; + } + case U16 -> { + short[] a = (short[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (short v : a) { + min = Math.min(min, v & 0xFFFF); + max = Math.max(max, v & 0xFFFF); + } + yield new long[]{min, max}; + } + case I32 -> { + int[] a = (int[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (int v : a) { + min = Math.min(min, v); + max = Math.max(max, v); + } + yield new long[]{min, max}; + } + case U32 -> { + // Flipping the sign bit maps unsigned order onto signed order, so the loop stays + // a plain int min/max. + int[] a = (int[]) values; + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (int v : a) { + min = Math.min(min, v ^ Integer.MIN_VALUE); + max = Math.max(max, v ^ Integer.MIN_VALUE); + } + yield new long[]{Integer.toUnsignedLong(min ^ Integer.MIN_VALUE), + Integer.toUnsignedLong(max ^ Integer.MIN_VALUE)}; + } + case I64 -> { + long[] a = (long[]) values; + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + for (long v : a) { + min = Math.min(min, v); + max = Math.max(max, v); + } + yield new long[]{min, max}; + } + case U64 -> { + // Flipping the sign bit maps unsigned order onto signed order + long[] a = (long[]) values; + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + for (long v : a) { + min = Math.min(min, v ^ Long.MIN_VALUE); + max = Math.max(max, v ^ Long.MIN_VALUE); + } + yield new long[]{min ^ Long.MIN_VALUE, max ^ Long.MIN_VALUE}; + } + // Floating-point: NaN is skipped (Rust's `skip_nans`), and Math.min/Math.max give Rust's total + // order for zeros, -0.0 before 0.0. Starting from the infinities, an array with no ordered + // element ends with min > max. + case F16 -> { + short[] a = (short[]) values; + float min = 0f; + float max = 0f; + short minBits = 0; + short maxBits = 0; + boolean seen = false; + for (short v : a) { + float f = Float.float16ToFloat(v); + if (f != f) { + continue; + } + if (!seen) { + min = f; + max = f; + minBits = v; + maxBits = v; + seen = true; + continue; + } + if (Float.compare(f, min) < 0) { + min = f; + minBits = v; + } + if (Float.compare(f, max) > 0) { + max = f; + maxBits = v; + } + } + yield seen ? new long[]{Short.toUnsignedLong(minBits), Short.toUnsignedLong(maxBits)} : new long[0]; + } + case F32 -> { + float[] a = (float[]) values; + float min = Float.POSITIVE_INFINITY; + float max = Float.NEGATIVE_INFINITY; + for (float v : a) { + if (v == v) { + min = Math.min(min, v); + max = Math.max(max, v); + } + } + yield min > max ? new long[0] : new long[]{ + Float.floatToRawIntBits(min) & 0xFFFF_FFFFL, Float.floatToRawIntBits(max) & 0xFFFF_FFFFL}; + } + case F64 -> { + double[] a = (double[]) values; + double min = Double.POSITIVE_INFINITY; + double max = Double.NEGATIVE_INFINITY; + for (double v : a) { + if (v == v) { + min = Math.min(min, v); + max = Math.max(max, v); + } + } + yield min > max ? new long[0] : new long[]{Double.doubleToRawLongBits(min), Double.doubleToRawLongBits(max)}; + } + }; + } + + // Block-wise: each block ORs the XOR against the first element, which is branch-free and so + // vectorizes, and the early exit happens between blocks. A non-constant array, the common case, + // is rejected after one block; a per-element exit would stop C2 from vectorizing the body. + @Override + public boolean allEqual(Object values, PType ptype) { + if (Array.getLength(values) == 0) { + return true; + } + return switch (ptype) { + case I8, U8 -> allEqualBytes((byte[]) values); + case I16, U16, F16 -> allEqualShorts((short[]) values); + case I32, U32 -> allEqualInts((int[]) values); + case I64, U64 -> allEqualLongs((long[]) values); + case F32 -> allEqualFloatBits((float[]) values); + case F64 -> allEqualDoubleBits((double[]) values); + }; + } + + @Override + public boolean allEqual(boolean[] values) { + if (values.length == 0) { + return true; + } + boolean first = values[0]; + for (int base = 0; base < values.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, values.length); + int diff = 0; + for (int i = base; i < end; i++) { + diff |= values[i] != first ? 1 : 0; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualBytes(byte[] a) { + byte first = a[0]; + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + int diff = 0; + for (int i = base; i < end; i++) { + diff |= a[i] ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualShorts(short[] a) { + short first = a[0]; + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + int diff = 0; + for (int i = base; i < end; i++) { + diff |= a[i] ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualInts(int[] a) { + int first = a[0]; + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + int diff = 0; + for (int i = base; i < end; i++) { + diff |= a[i] ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualLongs(long[] a) { + long first = a[0]; + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + long diff = 0; + for (int i = base; i < end; i++) { + diff |= a[i] ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualFloatBits(float[] a) { + int first = Float.floatToRawIntBits(a[0]); + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + int diff = 0; + for (int i = base; i < end; i++) { + diff |= Float.floatToRawIntBits(a[i]) ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + private static boolean allEqualDoubleBits(double[] a) { + long first = Double.doubleToRawLongBits(a[0]); + for (int base = 0; base < a.length; base += EQUAL_BLOCK) { + int end = Math.min(base + EQUAL_BLOCK, a.length); + long diff = 0; + for (int i = base; i < end; i++) { + diff |= Double.doubleToRawLongBits(a[i]) ^ first; + } + if (diff != 0) { + return false; + } + } + return true; + } + + @Override + public OptionalLong sum(Object values, PType ptype) { + return switch (ptype) { + case I8 -> { + long s = 0; + for (byte v : (byte[]) values) { + s += v; + } + yield OptionalLong.of(s); + } + case I16 -> { + long s = 0; + for (short v : (short[]) values) { + s += v; + } + yield OptionalLong.of(s); + } + case I32 -> { + long s = 0; + for (int v : (int[]) values) { + s += v; + } + yield OptionalLong.of(s); + } + case I64 -> { + long s = 0; + for (long v : (long[]) values) { + try { + s = Math.addExact(s, v); + } catch (ArithmeticException _) { + yield OptionalLong.empty(); + } + } + yield OptionalLong.of(s); + } + case U8 -> { + long s = 0; + for (byte v : (byte[]) values) { + s += Byte.toUnsignedLong(v); + } + yield OptionalLong.of(s); + } + case U16 -> { + long s = 0; + for (short v : (short[]) values) { + s += Short.toUnsignedLong(v); + } + yield OptionalLong.of(s); + } + case U32 -> { + long s = 0; + for (int v : (int[]) values) { + s += Integer.toUnsignedLong(v); + } + yield OptionalLong.of(s); + } + case U64 -> { + long s = 0; + for (long v : (long[]) values) { + long next = s + v; + if (Long.compareUnsigned(next, s) < 0) { + yield OptionalLong.empty(); + } + s = next; + } + yield OptionalLong.of(s); + } + default -> throw new IllegalArgumentException("not an integer ptype: " + ptype); + }; + } + + @Override + public double sumFloating(Object values, PType ptype) { + double s = 0; + switch (ptype) { + case F16 -> { + for (short v : (short[]) values) { + s += Float.float16ToFloat(v); + } + } + case F32 -> { + for (float v : (float[]) values) { + s += v; + } + } + case F64 -> { + for (double v : (double[]) values) { + s += v; + } + } + default -> throw new IllegalArgumentException("not a floating-point ptype: " + ptype); + } + return s; + } + + // Branch-free body per carrier, so C2 vectorizes it (CLAUDE.md hot-loop rule). + @Override + public long runs(Object values, PType ptype) { + int n = Array.getLength(values); + if (n == 0) { + return 0; + } + long changes = 0; + switch (ptype) { + case I8, U8 -> { + byte[] a = (byte[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + case I16, U16, F16 -> { + short[] a = (short[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + case I32, U32 -> { + int[] a = (int[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + case I64, U64 -> { + long[] a = (long[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + case F32 -> { + float[] a = (float[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + case F64 -> { + double[] a = (double[]) values; + for (int i = 1; i < n; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + } + } + return changes + 1; + } + // Only the result is masked: the low typeBits bits of a sum or difference depend on the operands' // low bits alone, so masking each operand too would be redundant work. // Rows outer, lanes inner: lanes are independent and contiguous (iterateIndex(row, lane) is diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java index df996dca6..1e5ad3f92 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperations.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.model.PType; import java.lang.foreign.MemorySegment; +import java.util.OptionalLong; /// The vectorizable kernels shared by the reader and the writer: the tight loops where a SIMD /// implementation can replace the auto-vectorized one without changing a single output byte. @@ -13,37 +14,113 @@ /// assume well-formed input and throw only [IllegalArgumentException] for a `ptype` they do not cover. public interface SimdOperations { - /// Widens `count` contiguous little-endian integer elements starting at element index - /// `fromElement` into `out[0, count)`, zero-extending unsigned ptypes and sign-extending signed ones. + /// Widens `count` contiguous little-endian elements starting at element index `fromElement` + /// into `out[0, count)`: signed integers are sign-extended, unsigned integers zero-extended, and + /// floating-point values contribute their raw bits, zero-extended. /// /// @param src the source segment /// @param fromElement starting element index (not byte offset) within `src` /// @param count number of elements to widen - /// @param ptype the elements' physical type, an integer type + /// @param ptype the elements' physical type /// @param out destination array, at least `count` long - /// @throws IllegalArgumentException if `ptype` is not an integer type void widenInto(MemorySegment src, long fromElement, int count, PType ptype, long[] out); + /// The heap-array counterpart of [#widenInto]: widens `count` elements of a primitive array + /// starting at index `from` into `out[0, count)`, with the same extension rules. + /// + /// @param values an array matching `ptype`'s carrier (`short[]` for `F16`) + /// @param from starting element index within `values` + /// @param count number of elements to widen + /// @param ptype the elements' physical type + /// @param out destination array, at least `count` long + void widenArrayInto(Object values, int from, int count, PType ptype, long[] out); + /// Narrows `values` to the width of `ptype`, keeping the low bytes, and writes them - /// little-endian into `dst` starting at byte 0. Handles the 1-, 2- and 4-byte types; the - /// 8-byte types are a plain bulk copy that needs no kernel. + /// little-endian into `dst` starting at byte 0. Floating-point types keep their raw bits, and the + /// 8-byte types are written as they are. /// /// @param values the wide values - /// @param ptype the target physical type, 1 to 4 bytes wide (floats keep their raw bits) + /// @param ptype the target physical type /// @param dst destination segment, at least `values.length * ptype.byteSize()` bytes - /// @throws IllegalArgumentException if `ptype` is 8 bytes wide void narrowInto(long[] values, PType ptype, MemorySegment dst); + /// The heap-array counterpart of [#narrowInto]: narrows `values` to the width of `ptype`, keeping + /// the low bytes, into the primitive array `out`. Floating-point carriers are rebuilt from the raw + /// bits, the inverse of [#widenArrayInto]. + /// + /// @param values the wide values + /// @param ptype the target physical type + /// @param out an array matching `ptype`'s carrier (`short[]` for `F16`), at least `values.length` long + void narrowArrayInto(long[] values, PType ptype, Object out); + /// Returns the largest of `count` contiguous little-endian unsigned elements of `src`, zero when /// `count` is zero: the dictionary code bound check (Rust's `vmaxq_u8` small-table guard). /// /// @param src the source segment, at least `count * ptype.byteSize()` bytes /// @param count number of elements to scan - /// @param ptype the elements' physical type: `U8`, `U16` or `U32` - /// @return the maximum, zero-extended - /// @throws IllegalArgumentException if `ptype` is not `U8`, `U16` or `U32` + /// @param ptype the elements' physical type: `U8`, `U16`, `U32` or `U64` + /// @return the maximum, zero-extended (as its bit pattern for `U64`) + /// @throws IllegalArgumentException if `ptype` is not an unsigned integer type long maxUnsigned(MemorySegment src, long count, PType ptype); + /// Returns the smallest and largest element in the ptype's natural order, widened as + /// [#widenArrayInto] does (a float contributes its raw bits). Integers compare as their type + /// does, `U64` unsigned. Floating-point values skip `NaN` (Rust's `skip_nans`) and follow the total + /// order for zeros, `-0.0` before `0.0`, as Rust does. + /// + /// @param values an array matching `ptype`'s carrier (`short[]` for `F16`) + /// @param ptype the elements' physical type + /// @return `{min, max}`; for a floating-point array with no non-`NaN` element (including an + /// empty one) an empty array + /// @throws IllegalArgumentException if `values` is empty and `ptype` is an integer type + long[] minMax(Object values, PType ptype); + + /// Returns whether every element of `values` equals the first, comparing floating-point values by + /// their raw bits: distinct NaN payloads, and `-0.0` against `0.0`, are not equal. An empty array + /// is trivially equal. + /// + /// @param values an array matching `ptype`'s carrier (`short[]` for `F16`) + /// @param ptype the elements' physical type + /// @return `true` if all elements are equal + boolean allEqual(Object values, PType ptype); + + /// Returns whether every element of `values` equals the first; an empty array is trivially equal. + /// + /// @param values the flags + /// @return `true` if all elements are equal + boolean allEqual(boolean[] values); + + /// Sums an integer array in Rust's widened shape: signed types as a signed `long`, unsigned types + /// as an unsigned one (returned as its bit pattern). Overflow is checked per addition, in order, + /// so a partial sum that overflows reports overflow even if later terms would bring the total + /// back in range (Rust's `checked_add`). Only `I64` and `U64` can overflow: the narrower widths + /// cannot, for any array that fits in memory. + /// + /// @param values an array matching `ptype`'s carrier; empty sums to zero + /// @param ptype the elements' physical type, an integer type + /// @return the sum, or empty on overflow + /// @throws IllegalArgumentException if `ptype` is not an integer type + OptionalLong sum(Object values, PType ptype); + + /// Sums a floating-point array into a `double`, strictly left to right. The order is part of the + /// contract: reassociating a float sum changes its rounding, and with it the stats bits we write. + /// + /// @param values a `short[]` (`F16`), `float[]` (`F32`) or `double[]` (`F64`) + /// @param ptype the elements' physical type, a floating-point type + /// @return the sum, zero for an empty array + /// @throws IllegalArgumentException if `ptype` is not a floating-point type + double sumFloating(Object values, PType ptype); + + /// Counts the runs of equal neighbors: one plus every position whose element differs from its + /// predecessor, and zero for an empty array. Floating-point elements are compared by value, as + /// `!=`: a `NaN` always differs from its neighbor, and `0.0` equals `-0.0`. That is unlike + /// [#allEqual(Object, PType)], which compares raw bits. + /// + /// @param values an array matching `ptype`'s carrier (`short[]` for `F16`, compared as raw bits) + /// @param ptype the elements' physical type + /// @return the number of runs + long runs(Object values, PType ptype); + /// Reverses the FastLanes delta transform of one 1024-element chunk. Each of the `lanes` /// independent lanes is a prefix sum over `typeBits` rows, wrapping at the element width. /// Element `(row, lane)` sits at index `FastLanes.iterateIndex(row, lane)` in both `deltas` and `out`. diff --git a/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java index e1f7e8363..f80f6ea22 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.compute.FastLanes; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.PType; import org.junit.jupiter.api.Test; @@ -13,6 +14,8 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.lang.reflect.Array; +import java.util.OptionalLong; import java.util.Random; import static org.assertj.core.api.Assertions.assertThat; @@ -50,6 +53,361 @@ void widenInto_roundTripsNarrowedValues(PType ptype) { } } + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void allEqual_integerArrays(PType ptype) { + // Given lengths around the 256-element block edge, where a block-boundary slip would hide a mismatch + for (int length : new int[]{1, 2, 255, 256, 257, 600}) { + Object constant = constantArray(ptype, length); + + // When / Then a constant array is equal, and so is the empty one + assertThat(sut.allEqual(constant, ptype)).isTrue(); + + // And changing any single element, first or last of a block included, breaks it + for (int position : new int[]{0, length / 2, length - 1, Math.min(256, length - 1)}) { + Object changed = constantArray(ptype, length); + bump(changed, position); + assertThat(sut.allEqual(changed, ptype)).as("%s[%d] differs at %d", ptype, length, position) + .isEqualTo(length == 1); + } + } + assertThat(sut.allEqual(constantArray(ptype, 0), ptype)).isTrue(); + } + + @Test + void allEqual_floatsCompareRawBits() { + // Given arrays equal under ==/Double.equals-ish rules but not bit for bit + float[] zeros = {0.0f, -0.0f}; + double[] nans = {Double.longBitsToDouble(0x7ff8000000000000L), Double.longBitsToDouble(0x7ff8000000000001L)}; + float[] sameNan = {Float.NaN, Float.NaN}; + + // When / Then + assertThat(sut.allEqual(zeros, PType.F32)).isFalse(); + assertThat(sut.allEqual(nans, PType.F64)).isFalse(); + assertThat(sut.allEqual(sameNan, PType.F32)).isTrue(); + assertThat(sut.allEqual(new short[]{1, 1, 1}, PType.F16)).isTrue(); + assertThat(sut.allEqual(new short[]{1, 1, 2}, PType.F16)).isFalse(); + } + + @Test + void allEqual_booleans() { + // Given + boolean[] longTail = new boolean[600]; + longTail[599] = true; + + // When / Then + assertThat(sut.allEqual(new boolean[0])).isTrue(); + assertThat(sut.allEqual(new boolean[]{true, true})).isTrue(); + assertThat(sut.allEqual(new boolean[]{false, true})).isFalse(); + assertThat(sut.allEqual(longTail)).isFalse(); + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32"}) + void sum_narrowIntegers_matchWidenedSum(PType ptype) { + // Given full-range values; the narrow widths cannot overflow a long + Object values = randomArray(ptype, 1000, new Random(ptype.ordinal())); + long expected = 0; + for (long v : PrimitiveArrays.toLongs(values, ptype, EncodingId.VORTEX_PRIMITIVE)) { + expected += v; + } + + // When + OptionalLong result = sut.sum(values, ptype); + + // Then + assertThat(result).hasValue(expected); + } + + @Test + void sum_i64_overflowOfAPartialSumIsOverflowEvenIfTheTotalFits() { + // Given a prefix sum past Long.MAX_VALUE that the last term brings back: Rust's checked_add + // drops the sum, while a final-total check would not notice + long[] values = {Long.MAX_VALUE, 1L, -1L}; + + // When + OptionalLong result = sut.sum(values, PType.I64); + + // Then + assertThat(result).isEmpty(); + } + + @Test + void sum_i64_fitsAtTheBoundary() { + // Given + long[] values = {Long.MAX_VALUE, -1L, 1L}; + + // When / Then + assertThat(sut.sum(values, PType.I64)).hasValue(Long.MAX_VALUE); + assertThat(sut.sum(new long[]{Long.MIN_VALUE, -1L}, PType.I64)).isEmpty(); + } + + @Test + void sum_u64_overflowPastTwoToThe64IsEmpty_andLargeUnsignedSumsFit() { + // Given + long[] fits = {Long.MIN_VALUE, Long.MAX_VALUE}; // 2^63 + (2^63 - 1) = 2^64 - 1 + long[] overflows = {-1L, 1L}; // 2^64 - 1 + 1 + + // When / Then + assertThat(sut.sum(fits, PType.U64)).hasValue(-1L); + assertThat(sut.sum(overflows, PType.U64)).isEmpty(); + } + + @Test + void sum_emptyArray_isZero() { + // Given / When / Then + assertThat(sut.sum(new int[0], PType.I32)).hasValue(0L); + assertThat(sut.sumFloating(new double[0], PType.F64)).isZero(); + } + + @Test + void sum_floatingPType_throws() { + // Given / When / Then + assertThatThrownBy(() -> sut.sum(new float[]{1f}, PType.F32)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> sut.sumFloating(new int[]{1}, PType.I32)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void sumFloating_addsStrictlyLeftToRight() { + // Given terms where reassociation changes the result: (1e16 + 1) + 1 = 1e16, but 1e16 + (1 + 1) is larger + double[] values = {1e16, 1.0, 1.0}; + + // When + double result = sut.sumFloating(values, PType.F64); + + // Then the sequential result, not the reassociated one + assertThat(result).isEqualTo(1e16); + assertThat(sut.sumFloating(new float[]{0.5f, 0.25f}, PType.F32)).isEqualTo(0.75); + assertThat(sut.sumFloating(new short[]{Float.floatToFloat16(1.5f), Float.floatToFloat16(2.0f)}, PType.F16)) + .isEqualTo(3.5); + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void runs_countsChangesBetweenNeighbors(PType ptype) { + // Given a constant array and then one with a change at known positions + Object constant = constantArray(ptype, 500); + Object twoChanges = constantArray(ptype, 500); + bump(twoChanges, 100); + bump(twoChanges, 101); + + // When / Then a constant array is one run; bumping 100 and 101 to the same new value + // adds a change into it and one out of it + assertThat(sut.runs(constant, ptype)).isEqualTo(1); + assertThat(sut.runs(twoChanges, ptype)).isEqualTo(3); + assertThat(sut.runs(constantArray(ptype, 1), ptype)).isEqualTo(1); + assertThat(sut.runs(constantArray(ptype, 0), ptype)).isZero(); + } + + @Test + void runs_floatsCompareByValue() { + // Given NaN, which never equals itself, and a signed zero pair, which compares equal + float[] nans = {Float.NaN, Float.NaN, Float.NaN}; + double[] zeros = {0.0, -0.0, 0.0}; + + // When / Then (unlike allEqual, which compares raw bits) + assertThat(sut.runs(nans, PType.F32)).isEqualTo(3); + assertThat(sut.runs(zeros, PType.F64)).isEqualTo(1); + assertThat(sut.runs(new short[]{1, 1, 2, 2, 3}, PType.F16)).isEqualTo(3); + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32"}) + void narrowArrayInto_keepsTheLowBytes_andWidenRestoresThem(PType ptype) { + // Given full-range wide values, so the dropped high bytes are not all zero + Object carrier = randomArray(ptype, VALUES.length, new Random(1)); + long[] result = VALUES.clone(); + + // When narrowed into an array and widened back + sut.narrowArrayInto(VALUES, ptype, carrier); + sut.widenArrayInto(carrier, 0, VALUES.length, ptype, result); + + // Then each value equals the original truncated to the width and re-extended + for (int i = 0; i < VALUES.length; i++) { + long truncated = ptype.isSigned() + ? VALUES[i] << (64 - 8 * ptype.byteSize()) >> (64 - 8 * ptype.byteSize()) + : VALUES[i] & (-1L >>> (64 - 8 * ptype.byteSize())); + assertThat(result[i]).as("%s[%d]", ptype, i).isEqualTo(truncated); + } + } + + @Test + void narrowArrayInto_eightByteTypes_copy_andFloatCarriersAreRebuiltFromRawBits() { + // Given the full-range wide values + long[] copied = new long[VALUES.length]; + short[] halves = new short[VALUES.length]; + float[] floats = new float[VALUES.length]; + double[] doubles = new double[VALUES.length]; + + // When + sut.narrowArrayInto(VALUES, PType.U64, copied); + sut.narrowArrayInto(VALUES, PType.F16, halves); + sut.narrowArrayInto(VALUES, PType.F32, floats); + sut.narrowArrayInto(VALUES, PType.F64, doubles); + + // Then the 8-byte types are unchanged and each float carrier holds the low bits of the value + assertThat(copied).containsExactly(VALUES); + for (int i = 0; i < VALUES.length; i++) { + assertThat(halves[i]).isEqualTo((short) VALUES[i]); + assertThat(Float.floatToRawIntBits(floats[i])).isEqualTo((int) VALUES[i]); + assertThat(Double.doubleToRawLongBits(doubles[i])).isEqualTo(VALUES[i]); + } + } + + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void widenArrayInto_matchesSegmentWiden(PType ptype) { + // Given the same full-range values as a heap array and as the equivalent segment + Object values = randomArray(ptype, 100, new Random(ptype.ordinal())); + try (Arena arena = Arena.ofConfined()) { + MemorySegment segment = PrimitiveArrays.toSegment(values, ptype, arena); + long[] expected = new long[60]; + sut.widenInto(segment, 17, 60, ptype, expected); + + // When widened from the same element, into an oversized out + long[] result = new long[61]; + sut.widenArrayInto(values, 17, 60, ptype, result); + + // Then the heap path agrees with the segment path, and leaves the tail untouched + assertThat(result).startsWith(expected); + assertThat(result[60]).isZero(); + } + } + + @Test + void widenArrayInto_floats_contributeTheirRawBitsZeroExtended() { + // Given a negative float, whose sign bit must not be sign-extended + float[] floats = {-1.5f, 2.5f}; + double[] doubles = {-1.5, 2.5}; + short[] halves = {(short) 0xBE00, 0x4100}; + long[] out = new long[2]; + + // When / Then + sut.widenArrayInto(floats, 0, 2, PType.F32, out); + assertThat(out).containsExactly(Float.floatToRawIntBits(-1.5f) & 0xFFFF_FFFFL, Float.floatToRawIntBits(2.5f) & 0xFFFF_FFFFL); + sut.widenArrayInto(doubles, 0, 2, PType.F64, out); + assertThat(out).containsExactly(Double.doubleToRawLongBits(-1.5), Double.doubleToRawLongBits(2.5)); + sut.widenArrayInto(halves, 0, 2, PType.F16, out); + assertThat(out).containsExactly(0xBE00L, 0x4100L); + } + + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void minMax_matchesWidenedReference(PType ptype) { + // Given seeded-random arrays of full-range values, so a wrong width or sign/zero extension + // changes the answer; lengths straddle the vector lane counts + Random random = new Random(ptype.ordinal()); + for (int length : new int[]{1, 2, 7, 33, 1000}) { + Object values = randomArray(ptype, length, random); + long[] widened = PrimitiveArrays.toLongs(values, ptype, EncodingId.VORTEX_PRIMITIVE); + long expectedMin = widened[0]; + long expectedMax = widened[0]; + for (long v : widened) { + // U64 is ordered unsigned; the narrower widths are zero-extended, so signed order is theirs + boolean lower = ptype == PType.U64 ? Long.compareUnsigned(v, expectedMin) < 0 : v < expectedMin; + boolean higher = ptype == PType.U64 ? Long.compareUnsigned(v, expectedMax) > 0 : v > expectedMax; + expectedMin = lower ? v : expectedMin; + expectedMax = higher ? v : expectedMax; + } + + // When + long[] result = sut.minMax(values, ptype); + + // Then the signed min and max of the widened values + assertThat(result).containsExactly(expectedMin, expectedMax); + } + } + + @Test + void minMax_u32AboveSignedRange_isZeroExtended() { + // Given U32 values straddling 2^31, where sign-bit handling would swap min and max + int[] values = {0x8000_0000, 5, 0xFFFF_FFFF}; + + // When + long[] result = sut.minMax(values, PType.U32); + + // Then + assertThat(result).containsExactly(5L, 0xFFFF_FFFFL); + } + + @Test + void minMax_u64AboveSignedRange_isOrderedUnsigned() { + // Given a U64 value with the top bit set, which sits at a negative signed position + long[] values = {1L, -1L, 5L}; + + // When + long[] result = sut.minMax(values, PType.U64); + + // Then the natural unsigned order: 2^64 - 1 is the maximum + assertThat(result).containsExactly(1L, -1L); + } + + + @Test + void minMax_emptyArray_throws() { + // Given + int[] values = {}; + + // When / Then + assertThatThrownBy(() -> sut.minMax(values, PType.I32)).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void minMax_floats_skipNaN_andOrderNegativeZeroFirst() { + // Given NaN around the extremes, and zeros of both signs + float[] floats = {Float.NaN, 3f, -0.0f, 0.0f, -2f, Float.NaN, 7f}; + double[] zeros = {0.0, -0.0, 5.0}; + double[] negativeZeros = {-0.0, 0.0, -5.0}; + short[] halves = {(short) 0x7E00, 0x4200, (short) 0xC000}; + + // When / Then NaN never wins, and -0.0 sorts before 0.0 (Rust's total order) + assertThat(sut.minMax(floats, PType.F32)).containsExactly( + Float.floatToRawIntBits(-2f) & 0xFFFF_FFFFL, Float.floatToRawIntBits(7f) & 0xFFFF_FFFFL); + assertThat(sut.minMax(zeros, PType.F64)).containsExactly( + Double.doubleToRawLongBits(-0.0), Double.doubleToRawLongBits(5.0)); + assertThat(sut.minMax(negativeZeros, PType.F64)).containsExactly( + Double.doubleToRawLongBits(-5.0), Double.doubleToRawLongBits(0.0)); + assertThat(sut.minMax(halves, PType.F16)).containsExactly(0xC000L, 0x4200L); + } + + @Test + void minMax_floatZerosOnly_pickNegativeZeroAsMinAndPositiveZeroAsMax() { + // Given only zeros of both signs + double[] zeros = {0.0, -0.0, 0.0}; + + // When + long[] result = sut.minMax(zeros, PType.F64); + + // Then + assertThat(result).containsExactly(Double.doubleToRawLongBits(-0.0), Double.doubleToRawLongBits(0.0)); + } + + @Test + void minMax_halfAllPositiveInfinity_returnsInfinityNotZero() { + // Given F16 +Infinity (0x7C00), which a bit-tracking loop seeded with zeros would lose + short[] halves = {0x7C00, 0x7C00}; + + // When + long[] result = sut.minMax(halves, PType.F16); + + // Then + assertThat(result).containsExactly(0x7C00L, 0x7C00L); + } + + @Test + void minMax_floatsWithoutAnOrderedElement_haveNoMinMax() { + // Given only NaN, and no elements at all + float[] nans = {Float.NaN, Float.NaN}; + + // When / Then + assertThat(sut.minMax(nans, PType.F32)).isEmpty(); + assertThat(sut.minMax(new double[0], PType.F64)).isEmpty(); + assertThat(sut.minMax(new short[0], PType.F16)).isEmpty(); + } + + @ParameterizedTest @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "F16", "I32", "U32", "F32"}) void narrowInto_matchesPTypeIoSet(PType ptype) { @@ -111,7 +469,7 @@ void maxUnsigned_findsMaxAtEveryPosition(PType ptype, int count) { } @ParameterizedTest - @EnumSource(value = PType.class, names = {"I8", "I16", "I32", "U64", "F32"}) + @EnumSource(value = PType.class, names = {"I8", "I16", "I32", "I64", "F32"}) void maxUnsigned_nonCodePType_throws(PType ptype) { // When / Then assertThatThrownBy(() -> sut.maxUnsigned(MemorySegment.NULL, 0, ptype)) @@ -120,20 +478,67 @@ void maxUnsigned_nonCodePType_throws(PType ptype) { @ParameterizedTest @EnumSource(value = PType.class, names = {"F16", "F32", "F64"}) - void widenInto_floatingPType_throws(PType ptype) { - // When / Then - assertThatThrownBy(() -> sut.widenInto(MemorySegment.NULL, 0, 0, ptype, new long[0])) - .isInstanceOf(IllegalArgumentException.class); + void widenInto_floats_readLikeTheUnsignedTypeOfTheSameWidth(PType ptype) { + // Given a segment of full-range bits + PType unsigned = switch (ptype) { + case F16 -> PType.U16; + case F32 -> PType.U32; + default -> PType.U64; + }; + try (Arena arena = Arena.ofConfined()) { + MemorySegment src = arena.allocate(VALUES.length * ptype.byteSize()); + for (int i = 0; i < VALUES.length; i++) { + PTypeIO.set(src, i * ptype.byteSize(), unsigned, VALUES[i]); + } + long[] expected = new long[VALUES.length]; + long[] result = new long[VALUES.length]; + + // When + sut.widenInto(src, 0, VALUES.length, unsigned, expected); + sut.widenInto(src, 0, VALUES.length, ptype, result); + + // Then + assertThat(result).containsExactly(expected); + } } + @ParameterizedTest @EnumSource(value = PType.class, names = {"I64", "U64", "F64"}) - void narrowInto_eightByteType_throws(PType ptype) { - // When / Then - assertThatThrownBy(() -> sut.narrowInto(new long[0], ptype, MemorySegment.NULL)) - .isInstanceOf(IllegalArgumentException.class); + void narrowInto_eightByteTypes_writeTheValuesLittleEndian(PType ptype) { + // Given + try (Arena arena = Arena.ofConfined()) { + MemorySegment dst = arena.allocate(VALUES.length * 8L); + + // When + sut.narrowInto(VALUES, ptype, dst); + + // Then + for (int i = 0; i < VALUES.length; i++) { + assertThat(dst.getAtIndex(VortexFormat.LE_LONG, i)).isEqualTo(VALUES[i]); + } + } } + @Test + void maxUnsigned_u64_ordersTheHighBitAboveEverything() { + // Given a code with the top bit set among small ones + try (Arena arena = Arena.ofConfined()) { + MemorySegment src = arena.allocate(4 * 8L); + long[] codes = {5L, -2L, 7L, 1L}; + for (int i = 0; i < codes.length; i++) { + src.setAtIndex(VortexFormat.LE_LONG, i, codes[i]); + } + + // When + long result = sut.maxUnsigned(src, 4, PType.U64); + + // Then + assertThat(result).isEqualTo(-2L); + } + } + + @ParameterizedTest @ValueSource(ints = {8, 16, 32, 64}) void undeltaChunk_matchesLaneByLaneReference(int typeBits) { @@ -226,4 +631,55 @@ void packBlock_matchesBitByBitReference(int typeBits, int bitWidth) { // Then assertThat(result).isEqualTo(expected); } + + private static Object randomArray(PType ptype, int length, Random random) { + return switch (ptype) { + case I8, U8 -> { + byte[] a = new byte[length]; + random.nextBytes(a); + yield a; + } + case I16, U16 -> { + short[] a = new short[length]; + for (int i = 0; i < length; i++) { + a[i] = (short) random.nextInt(); + } + yield a; + } + case I32, U32 -> { + int[] a = new int[length]; + for (int i = 0; i < length; i++) { + a[i] = random.nextInt(); + } + yield a; + } + case I64, U64 -> { + long[] a = new long[length]; + for (int i = 0; i < length; i++) { + a[i] = random.nextLong(); + } + yield a; + } + default -> throw new IllegalArgumentException(ptype.toString()); + }; + } + + private static Object constantArray(PType ptype, int length) { + Object a = randomArray(ptype, 1, new Random(7)); + Object result = Array.newInstance(a.getClass().getComponentType(), length); + for (int i = 0; i < length; i++) { + Array.set(result, i, Array.get(a, 0)); + } + return result; + } + + private static void bump(Object array, int index) { + switch (array) { + case byte[] a -> a[index]++; + case short[] a -> a[index]++; + case int[] a -> a[index]++; + case long[] a -> a[index]++; + default -> throw new IllegalArgumentException(array.getClass().toString()); + } + } } diff --git a/docs/compatibility.md b/docs/compatibility.md index 7d1592ea5..a79071980 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -44,6 +44,18 @@ only the built-in decoders in `reader`; no encoder class is loaded. | Duplicate struct field names | Rust writer rejects ("StructLayout must have unique field names"); Rust reader tolerates foreign files (first-match access) | ⚠️ Deliberate divergence on read: Java rejects such files with `VortexException("duplicate field name in file schema")` instead of tolerating them — the name-keyed `Chunk` API cannot represent both columns, and silent column loss is worse than a loud failure on a file the reference writer refuses to produce. Java's writer mirrors the Rust writer's rejection. | | Blank / control-character field names | Wire-legal; the Rust writer produces `""` and whitespace-only names. NUL (`U+0000`) additionally aborts the Rust toolchain: Arrow FFI schema export hits a panic-cannot-unwind in `arrow-rs` (`ffi_stream::get_schema`) and SIGABRTs the process (measured against vortex-jni 0.75.0) | ⚠️ Deliberate strictness BOTH ways: vortex-java's writer refuses blank and control-character field names (`IllegalArgumentException`), and its reader rejects files carrying them (`VortexException` naming the producing pipeline as the likely bug) — the JSON-`""`-key principle: wire-legal is a floor, not a policy. Printable names of any shape (`$`-runs, spaces inside, emoji) are legal and round-trip intact both directions (measured; pinned by `ColumnNameEdgeCasesIntegrationTest`). | +## Primitive statistics (min, max, sum) + +The bulk loops behind the min, max and sum statistics live in `SimdOperations`. Compared with the Rust source: + +- Float min/max statistics order `-0.0` before `0.0` and skip `NaN`, as Rust's `total_compare` with + `skip_nans` does. Before, the first zero met in array order was kept. +- `U64` min/max and the dense-counter span in `ArrayStats` are computed in unsigned order, as Rust's + `typed_int_stats` does. +- **Open divergence:** Rust binds the zoned `Min`, `Max` **and `Sum`** aggregates with `skip_nans()`, so a + float zone's sum ignores `NaN`. Java's float sum adds `NaN` and so becomes `NaN` + ([#523](https://github.com/dfa1/vortex-java/issues/523)). + ## Real-world conformance: the Raincloud corpus Cross-implementation conformance against [Raincloud](https://github.com/spiraldb/raincloud) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index 0af5e790a..30eabdfde 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -2,7 +2,6 @@ import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; -import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; @@ -31,6 +30,7 @@ import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.array.VarBinConstantArray; import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -183,22 +183,13 @@ private static long checkedPoolExtent(MemorySegment codesSeg, PType codesPType, long max = -1; long min = 0; switch (codesPType) { - case U8 -> { - for (long i = 0; i < n; i++) { - long code = Byte.toUnsignedLong(codesSeg.get(ValueLayout.JAVA_BYTE, i)); - max = Math.max(max, code); - } - } - case U16 -> { - for (long i = 0; i < n; i++) { - long code = Short.toUnsignedLong(codesSeg.getAtIndex(LE_SHORT, i)); - max = Math.max(max, code); - } - } - case U32 -> { - for (long i = 0; i < n; i++) { - long code = Integer.toUnsignedLong(codesSeg.getAtIndex(LE_INT, i)); - max = Math.max(max, code); + // An unsigned code past Long.MAX_VALUE reads back negative, which the min test rejects: it + // is out of range for any pool a file can actually hold anyway. + case U8, U16, U32, U64 -> { + if (n > 0) { + long largest = SimdOperationsSupport.preferred().maxUnsigned(codesSeg, n, codesPType); + max = Math.max(largest, 0L); + min = Math.min(largest, 0L); } } case I32 -> { @@ -208,9 +199,7 @@ private static long checkedPoolExtent(MemorySegment codesSeg, PType codesPType, min = Math.min(min, code); } } - // A u64 code past Long.MAX_VALUE reads back negative, which the min test rejects — - // it is out of range for any pool a file can actually hold anyway. - case I64, U64 -> { + case I64 -> { for (long i = 0; i < n; i++) { long code = codesSeg.getAtIndex(LE_LONG, i); max = Math.max(max, code); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java index bae50b7b0..d52bbd6d8 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpRdEncodingEncoder.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.util.ArrayList; @@ -334,7 +335,7 @@ private static EncodeResult buildEncodeResult( EncodeNode valNode; int valOffset = allBuffers.size(); - if (allEqual(excValsArr)) { + if (SimdOperationsSupport.preferred().allEqual(excValsArr, PType.I16)) { EncodeResult constant = new ConstantEncodingEncoder().encode(DType.U16, excValsArr, ctx); allBuffers.addAll(constant.encodedBuffers()); valNode = EncodeNode.remapBufferIndices(constant.rootNode(), valOffset); @@ -365,15 +366,6 @@ private static EncodeResult buildEncodeResult( return new EncodeResult(root, List.copyOf(allBuffers), statsMin, statsMax); } - private static boolean allEqual(short[] values) { - for (short value : values) { - if (value != values[0]) { - return false; - } - } - return true; - } - private static byte[] scalarF64(double v) { return ProtoScalarValue.ofF64Value(v).encode(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java index 431ee8de1..9f0e264c1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayStats.java @@ -1,8 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.reflect.Array; @@ -76,7 +75,7 @@ public static ArrayStats compute(PType ptype, Object data, StatsOptions options) if (n == 0) { return EMPTY; } - long averageRunLength = n / runs(ptype, data, n); + long averageRunLength = n / SimdOperationsSupport.preferred().runs(data, ptype); boolean accumulate = options.countDistinct() || options.trackMostFrequent(); // Stop once the distinct count passes half the rows: past that point every consumer's @@ -96,23 +95,12 @@ public static ArrayStats compute(PType ptype, Object data, StatsOptions options) // width without copying the whole array to a long[] (8 bytes per row of garbage on // every int[] column, #476). long[] buffer = new long[BUFFER]; - long[] minMax = minMax(ptype, data); + long[] minMax = SimdOperationsSupport.preferred().minMax(data, ptype); long min = minMax[0]; long max = minMax[1]; ArrayStats counted = accumulate ? integerStats(ptype, data, n, buffer, cap, options.countDistinct(), averageRunLength, min, max) : new ArrayStats(n, -1, 0, 0, false, averageRunLength, 0, 0); - if (ptype == PType.U64) { - // Narrower unsigned widths are zero-extended, so signed order is already theirs. - long[] values = (long[]) data; - long unsignedMin = -1L; - long unsignedMax = 0L; - for (int i = 0; i < n; i++) { - unsignedMin = Long.compareUnsigned(values[i], unsignedMin) < 0 ? values[i] : unsignedMin; - unsignedMax = Long.compareUnsigned(values[i], unsignedMax) > 0 ? values[i] : unsignedMax; - } - return counted.withMinMax(unsignedMin, unsignedMax); - } return counted.withMinMax(min, max); } if (!accumulate) { @@ -241,137 +229,13 @@ private static ArrayStats fromDense(int n, int[] counts, long min, boolean count averageRunLength, 0, 0); } - /// Signed min and max of the widened values, computed on the array's own carrier: NEON has - /// no 64-bit integer min/max, so C2 vectorizes these loops for `int`, `short` and `byte` - /// but not over a widened `long` buffer, where this pass measured ~8% of a write (#476). - /// Unsigned widths are masked in the loop, which keeps them in their carrier's lanes. - /// - /// @return `{min, max}` - private static long[] minMax(PType ptype, Object data) { - return switch (ptype) { - case I8 -> { - byte[] a = (byte[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (byte v : a) { - min = Math.min(min, v); - max = Math.max(max, v); - } - yield new long[]{min, max}; - } - case U8 -> { - byte[] a = (byte[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (byte v : a) { - min = Math.min(min, v & 0xFF); - max = Math.max(max, v & 0xFF); - } - yield new long[]{min, max}; - } - case I16 -> { - short[] a = (short[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (short v : a) { - min = Math.min(min, v); - max = Math.max(max, v); - } - yield new long[]{min, max}; - } - case U16 -> { - short[] a = (short[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (short v : a) { - min = Math.min(min, v & 0xFFFF); - max = Math.max(max, v & 0xFFFF); - } - yield new long[]{min, max}; - } - case I32 -> { - int[] a = (int[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (int v : a) { - min = Math.min(min, v); - max = Math.max(max, v); - } - yield new long[]{min, max}; - } - case U32 -> { - // Flipping the sign bit maps unsigned order onto signed order, so the loop stays - // a plain int min/max. - int[] a = (int[]) data; - int min = Integer.MAX_VALUE; - int max = Integer.MIN_VALUE; - for (int v : a) { - min = Math.min(min, v ^ Integer.MIN_VALUE); - max = Math.max(max, v ^ Integer.MIN_VALUE); - } - yield new long[]{Integer.toUnsignedLong(min ^ Integer.MIN_VALUE), - Integer.toUnsignedLong(max ^ Integer.MIN_VALUE)}; - } - case I64, U64 -> { - long[] a = (long[]) data; - long min = Long.MAX_VALUE; - long max = Long.MIN_VALUE; - for (long v : a) { - min = Math.min(min, v); - max = Math.max(max, v); - } - yield new long[]{min, max}; - } - default -> throw new VortexException(EncodingId.VORTEX_PRIMITIVE, "not an integer ptype: " + ptype); - }; - } - /// Copies up to `chunk.length` values of `data` from `from` into `chunk`, sign-extending the signed /// widths and zero-extending the unsigned ones; one switch per chunk, not per value. /// /// @return how many values were copied private static int widen(PType ptype, Object data, int from, int n, long[] chunk) { int len = Math.min(chunk.length, n - from); - switch (ptype) { - case I8 -> { - byte[] a = (byte[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i]; - } - } - case U8 -> { - byte[] a = (byte[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i] & 0xFFL; - } - } - case I16 -> { - short[] a = (short[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i]; - } - } - case U16 -> { - short[] a = (short[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i] & 0xFFFFL; - } - } - case I32 -> { - int[] a = (int[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i]; - } - } - case U32 -> { - int[] a = (int[]) data; - for (int i = 0; i < len; i++) { - chunk[i] = a[from + i] & 0xFFFF_FFFFL; - } - } - case I64, U64 -> System.arraycopy((long[]) data, from, chunk, 0, len); - default -> throw new VortexException(EncodingId.VORTEX_PRIMITIVE, "not an integer ptype: " + ptype); - } + SimdOperationsSupport.preferred().widenArrayInto(data, from, len, ptype, chunk); return len; } @@ -384,51 +248,6 @@ private static ArrayStats fromCounts(int n, LongIntMap counts, boolean capped, b return new ArrayStats(n, distinct, topFreqBits, topFreq, capped, averageRunLength, 0, 0); } - /// Runs of equal values: 1 plus every change between neighbors. One loop per carrier with a - /// branch-free body, so C2 vectorizes it (CLAUDE.md hot-loop rule). - private static long runs(PType ptype, Object data, int n) { - long changes = 0; - switch (ptype) { - case I8, U8 -> { - byte[] a = (byte[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - case I16, U16, F16 -> { - short[] a = (short[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - case I32, U32 -> { - int[] a = (int[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - case I64, U64 -> { - long[] a = (long[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - case F32 -> { - float[] a = (float[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - case F64 -> { - double[] a = (double[]) data; - for (int i = 1; i < n; i++) { - changes += a[i] != a[i - 1] ? 1 : 0; - } - } - } - return changes + 1; - } - private static boolean scanF16(short[] a, int n, LongIntMap counts, long cap) { for (int i = 0; i < n; i++) { counts.increment(Short.toUnsignedLong(a[i])); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index 2057a8fe0..1e56368a4 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.lang.reflect.Array; @@ -85,7 +86,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC return CascadeStep.terminal(encode(dtype, data, encodeCtx)); } if (dtype instanceof DType.Bool bool) { - if (!isConstantBool((boolean[]) data)) { + if (!SimdOperationsSupport.preferred().allEqual((boolean[]) data)) { return CascadeStep.notApplicable(); } return CascadeStep.terminal(encode(bool, data, encodeCtx)); @@ -116,7 +117,7 @@ private static boolean isConstantStrings(String[] data) { } private static EncodeResult encodeBool(boolean[] data) { - if (!isConstantBool(data)) { + if (!SimdOperationsSupport.preferred().allEqual(data)) { throw new VortexException(EncodingId.VORTEX_CONSTANT, "not a constant array"); } boolean value = data.length == 0 || data[0]; @@ -124,91 +125,25 @@ private static EncodeResult encodeBool(boolean[] data) { return EncodeResult.simple(EncodingId.VORTEX_CONSTANT, EncodedBuffer.bytes(MemorySegment.ofArray(scalar.encode()))); } - private static boolean isConstantBool(boolean[] data) { - if (data.length == 0) { - return true; - } - boolean first = data[0]; - for (boolean b : data) { - if (b != first) { - return false; - } - } - return true; - } - // The raw bits of the one repeated value, or empty if the array is not constant. An empty array - // is constant (bits 0), as `expectedRatio` already treats valueCount 0. One typed loop per width, - // the ptype switch hoisted out (CLAUDE.md hot-loop rule) and the first element read once - // outside the loop. Floats compare raw bits, so distinct NaN payloads or -0.0 vs 0.0 are not - // constant. + // is constant (bits 0), as `expectedRatio` already treats valueCount 0. Floats compare raw bits, + // so distinct NaN payloads or -0.0 vs 0.0 are not constant. private static OptionalLong constantBits(Object data, PType ptype) { if (Array.getLength(data) == 0) { return OptionalLong.of(0L); } + if (!SimdOperationsSupport.preferred().allEqual(data, ptype)) { + return OptionalLong.empty(); + } // No default: every PType is handled, so a new one fails to compile here instead of at write time - return switch (ptype) { - case I8, U8 -> { - byte[] a = (byte[]) data; - byte first = a[0]; - for (int i = 1; i < a.length; i++) { - if (a[i] != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - case I16, U16, F16 -> { - short[] a = (short[]) data; - short first = a[0]; - for (int i = 1; i < a.length; i++) { - if (a[i] != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - case I32, U32 -> { - int[] a = (int[]) data; - int first = a[0]; - for (int i = 1; i < a.length; i++) { - if (a[i] != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - case I64, U64 -> { - long[] a = (long[]) data; - long first = a[0]; - for (int i = 1; i < a.length; i++) { - if (a[i] != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - case F32 -> { - float[] a = (float[]) data; - int first = Float.floatToRawIntBits(a[0]); - for (int i = 1; i < a.length; i++) { - if (Float.floatToRawIntBits(a[i]) != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - case F64 -> { - double[] a = (double[]) data; - long first = Double.doubleToRawLongBits(a[0]); - for (int i = 1; i < a.length; i++) { - if (Double.doubleToRawLongBits(a[i]) != first) { - yield OptionalLong.empty(); - } - } - yield OptionalLong.of(first); - } - }; + return OptionalLong.of(switch (ptype) { + case I8, U8 -> ((byte[]) data)[0]; + case I16, U16, F16 -> ((short[]) data)[0]; + case I32, U32 -> ((int[]) data)[0]; + case I64, U64 -> ((long[]) data)[0]; + case F32 -> Float.floatToRawIntBits(((float[]) data)[0]); + case F64 -> Double.doubleToRawLongBits(((double[]) data)[0]); + }); } private static ProtoScalarValue buildScalar(PType ptype, long rawBits) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java index 55dbfce0d..a5a007b0a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoder.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.util.Set; @@ -95,13 +96,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext encodeC } private static long computeRef(long[] longs, int n) { - long ref = n > 0 ? longs[0] : 0L; - for (long v : longs) { - if (v < ref) { - ref = v; - } - } - return ref; + return n > 0 ? SimdOperationsSupport.preferred().minMax(longs, PType.I64)[0] : 0L; } private static MemorySegment buildForMeta(long ref, PType ptype) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java index c395c5dbe..a09f735a6 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.util.ArrayList; @@ -136,7 +137,7 @@ private static EncodeResult encodeValues(DType nonNullable, Object values, Encod /// @param ctx the encode context /// @return the encoded validity child static EncodeResult encodeValidity(boolean[] validity, EncodeContext ctx) { - if (isConstantValidity(validity)) { + if (SimdOperationsSupport.preferred().allEqual(validity)) { return new ConstantEncodingEncoder().encode(DType.BOOL, validity, ctx); } EncodeResult best = new BoolEncodingEncoder().encode(DType.BOOL, validity, ctx); @@ -166,19 +167,6 @@ private static long totalBytes(EncodeResult result) { return total; } - private static boolean isConstantValidity(boolean[] validity) { - if (validity.length == 0) { - return true; - } - boolean first = validity[0]; - for (boolean b : validity) { - if (b != first) { - return false; - } - } - return true; - } - /// Returns `values` unchanged, except a `String[]`/`byte[][]` with null elements is copied /// with each null replaced by an empty placeholder so cascade encoders (Dict, FSST, Zstd, /// VarBinView) can read every element's bytes safely. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index a061d897e..7fe068553 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -5,8 +5,12 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperations; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; +import java.lang.reflect.Array; +import java.util.OptionalLong; /// Write-only encoder for `vortex.primitive` — raw little-endian primitive arrays. public final class PrimitiveEncodingEncoder implements EncodingEncoder { @@ -47,196 +51,23 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { /// @param data the raw primitive array (e.g. `long[]`, `int[]`, `String`-free) /// @return a two-element `{min, max}` array of encoded scalars, or `null` if `data` is empty public static byte[][] minMaxStats(PType ptype, Object data) { + if (Array.getLength(data) == 0) { + return null; + } + // Floating-point arrays skip NaN, so an all-NaN one has no min/max: the kernel returns nothing + long[] minMax = SimdOperationsSupport.preferred().minMax(data, ptype); + if (minMax.length == 0) { + return null; + } return switch (ptype) { - case I8 -> { - byte[] arr = (byte[]) data; - if (arr.length == 0) { - yield null; - } - long min = arr[0]; - long max = arr[0]; - for (byte v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield new byte[][]{scalarI64(min), scalarI64(max)}; - } - case I16 -> { - short[] arr = (short[]) data; - if (arr.length == 0) { - yield null; - } - long min = arr[0]; - long max = arr[0]; - for (short v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield new byte[][]{scalarI64(min), scalarI64(max)}; - } - case I32 -> { - int[] arr = (int[]) data; - if (arr.length == 0) { - yield null; - } - long min = arr[0]; - long max = arr[0]; - for (int v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield new byte[][]{scalarI64(min), scalarI64(max)}; - } - case I64 -> { - long[] arr = (long[]) data; - if (arr.length == 0) { - yield null; - } - long min = arr[0]; - long max = arr[0]; - for (long v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield new byte[][]{scalarI64(min), scalarI64(max)}; - } - case U8 -> { - byte[] arr = (byte[]) data; - if (arr.length == 0) { - yield null; - } - long min = Byte.toUnsignedInt(arr[0]); - long max = Byte.toUnsignedInt(arr[0]); - for (byte v : arr) { - long uv = Byte.toUnsignedInt(v); - if (uv < min) { - min = uv; - } - if (uv > max) { - max = uv; - } - } - yield new byte[][]{scalarU64(min), scalarU64(max)}; - } - case U16 -> { - short[] arr = (short[]) data; - if (arr.length == 0) { - yield null; - } - long min = Short.toUnsignedInt(arr[0]); - long max = Short.toUnsignedInt(arr[0]); - for (short v : arr) { - long uv = Short.toUnsignedInt(v); - if (uv < min) { - min = uv; - } - if (uv > max) { - max = uv; - } - } - yield new byte[][]{scalarU64(min), scalarU64(max)}; - } - case U32 -> { - int[] arr = (int[]) data; - if (arr.length == 0) { - yield null; - } - long min = Integer.toUnsignedLong(arr[0]); - long max = Integer.toUnsignedLong(arr[0]); - for (int v : arr) { - long uv = Integer.toUnsignedLong(v); - if (uv < min) { - min = uv; - } - if (uv > max) { - max = uv; - } - } - yield new byte[][]{scalarU64(min), scalarU64(max)}; - } - case U64 -> { - long[] arr = (long[]) data; - if (arr.length == 0) { - yield null; - } - long min = arr[0]; - long max = arr[0]; - for (long v : arr) { - if (Long.compareUnsigned(v, min) < 0) { - min = v; - } - if (Long.compareUnsigned(v, max) > 0) { - max = v; - } - } - yield new byte[][]{scalarU64(min), scalarU64(max)}; - } - case F32 -> { - float[] arr = (float[]) data; - // NaN-skipping, as Rust's min/max (skip_nans): every comparison with NaN is false, so - // starting from the infinities skips NaN without a per-element branch; an all-NaN - // (or empty) array ends with min > max and has no min/max. - float min = Float.POSITIVE_INFINITY; - float max = Float.NEGATIVE_INFINITY; - for (float v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield min > max ? null : new byte[][]{scalarF32(min), scalarF32(max)}; - } - case F64 -> { - double[] arr = (double[]) data; - // NaN-skipping, as Rust's min/max (skip_nans): every comparison with NaN is false, so - // starting from the infinities skips NaN without a per-element branch; an all-NaN - // (or empty) array ends with min > max and has no min/max. - double min = Double.POSITIVE_INFINITY; - double max = Double.NEGATIVE_INFINITY; - for (double v : arr) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - yield min > max ? null : new byte[][]{scalarF64(min), scalarF64(max)}; - } - case F16 -> { - short[] arr = (short[]) data; - // NaN-skipping like F32/F64 above. - float min = Float.POSITIVE_INFINITY; - float max = Float.NEGATIVE_INFINITY; - for (short v : arr) { - float fv = Float.float16ToFloat(v); - if (fv < min) { - min = fv; - } - if (fv > max) { - max = fv; - } - } - yield min > max ? null : new byte[][]{scalarF16(min), scalarF16(max)}; - } + case I8, I16, I32, I64 -> new byte[][]{scalarI64(minMax[0]), scalarI64(minMax[1])}; + case U8, U16, U32, U64 -> new byte[][]{scalarU64(minMax[0]), scalarU64(minMax[1])}; + case F16 -> new byte[][]{scalarF16(Float.float16ToFloat((short) minMax[0])), + scalarF16(Float.float16ToFloat((short) minMax[1]))}; + case F32 -> new byte[][]{scalarF32(Float.intBitsToFloat((int) minMax[0])), + scalarF32(Float.intBitsToFloat((int) minMax[1]))}; + case F64 -> new byte[][]{scalarF64(Double.longBitsToDouble(minMax[0])), + scalarF64(Double.longBitsToDouble(minMax[1]))}; }; } @@ -270,136 +101,20 @@ public static byte[] maxOf(byte[][] stats) { /// @param data the raw primitive array /// @return the encoded sum scalar, or `null` on overflow or empty input public static byte[] sumStat(PType ptype, Object data) { + if (Array.getLength(data) == 0) { + return null; + } + SimdOperations ops = SimdOperationsSupport.preferred(); return switch (ptype) { - case I8 -> { - byte[] a = (byte[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (byte v : a) { - s += v; - } - yield scalarI64(s); - } - case I16 -> { - short[] a = (short[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (short v : a) { - s += v; - } - yield scalarI64(s); - } - case I32 -> { - int[] a = (int[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (int v : a) { - s += v; - } - yield scalarI64(s); - } - case I64 -> { - long[] a = (long[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (long v : a) { - try { - s = Math.addExact(s, v); - } catch (ArithmeticException _) { - yield null; - } - } - yield scalarI64(s); - } - case U8 -> { - byte[] a = (byte[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (byte v : a) { - s += Byte.toUnsignedLong(v); - } - yield scalarU64(s); - } - case U16 -> { - short[] a = (short[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (short v : a) { - s += Short.toUnsignedLong(v); - } - yield scalarU64(s); - } - case U32 -> { - int[] a = (int[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (int v : a) { - s += Integer.toUnsignedLong(v); - } - yield scalarU64(s); - } - case U64 -> { - long[] a = (long[]) data; - if (a.length == 0) { - yield null; - } - long s = 0; - for (long v : a) { - long next = s + v; - if (Long.compareUnsigned(next, s) < 0) { - yield null; - } - s = next; - } - yield scalarU64(s); - } - case F32 -> { - float[] a = (float[]) data; - if (a.length == 0) { - yield null; - } - double s = 0; - for (float v : a) { - s += v; - } - yield scalarF64(s); - } - case F64 -> { - double[] a = (double[]) data; - if (a.length == 0) { - yield null; - } - double s = 0; - for (double v : a) { - s += v; - } - yield scalarF64(s); + case I8, I16, I32, I64 -> { + OptionalLong sum = ops.sum(data, ptype); + yield sum.isEmpty() ? null : scalarI64(sum.getAsLong()); } - case F16 -> { - short[] a = (short[]) data; - if (a.length == 0) { - yield null; - } - double s = 0; - for (short v : a) { - s += Float.float16ToFloat(v); - } - yield scalarF64(s); + case U8, U16, U32, U64 -> { + OptionalLong sum = ops.sum(data, ptype); + yield sum.isEmpty() ? null : scalarU64(sum.getAsLong()); } + case F16, F32, F64 -> scalarF64(ops.sumFloating(data, ptype)); }; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java index 17af3316d..7bdd31431 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -157,11 +158,7 @@ public CascadeStep encodeCascade(DType dtype, Object data, EncodeContext ctx) { int n = values.length; // Rust skips on stats before sampling; ours carry no minimum, so find it with a read-only // scan before allocating anything — the common all-non-negative column stops here - long min = 0L; - for (long v : values) { - min = Math.min(min, v); - } - if (min >= 0L) { + if (n == 0 || SimdOperationsSupport.preferred().minMax(values, PType.I64)[0] >= 0L) { return CascadeStep.notApplicable(); } long[] encoded = new long[n]; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java index 4f04e122c..cd636a62d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZoneMapStats.java @@ -2,7 +2,9 @@ import io.github.dfa1.vortex.core.compute.PrimitiveArrays; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -113,17 +115,8 @@ static byte[][] ofDecimal(long[] values) { if (values.length == 0) { return null; } - long min = values[0]; - long max = values[0]; - for (long v : values) { - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } - } - return new byte[][]{decimalScalar(min), decimalScalar(max)}; + long[] minMax = SimdOperationsSupport.preferred().minMax(values, PType.I64); + return new byte[][]{decimalScalar(minMax[0]), decimalScalar(minMax[1])}; } /// Compares two equal-width little-endian two's-complement values in place: the most From 8ac3fb3bcec1a5cbdb38f84b9f452ea4fbbb7ee9 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 22:33:02 +0200 Subject: [PATCH 3/5] feat: an opt-in Vector API implementation of the SIMD kernels (#516) VectorApiSimdOperations implements every SimdOperations kernel and width as an explicit Vector API loop over the CPU's preferred species with a scalar tail, using the API's own reductions (reduceLanes); only F16, which has no half-float lanes, stays a plain loop. SimdOperationsSupport selects it when the JVM is launched with --add-modules jdk.incubator.vector and the CPU has 128-bit vectors, and otherwise keeps the auto-vectorized implementation, also if the Vector API cannot be linked. The flag is the opt-in (ADR 0005 rules out a per-scan switch). The shape follows Hardwood's VectorOperations (Apache-2.0), credited in the class. It departs from Rust in two reductions, as a recorded decision (docs/compatibility.md, CLAUDE.md, the class javadoc): float sums accumulate lane-wise, so their last bits can differ from Rust's sequential f64 sum, and I64/U64 sum overflow is detected per lane instead of per prefix. The default implementation stays Rust-parity, and 131 differential tests pin every other kernel of the two implementations to each other. Measured with JMH on NEON, AVX2 and AVX-512, bytes and shorts are 2x to 98x faster than the C2 loop (min/max, allEqual, maxUnsigned, runs, sum), delta/undelta/pack 1.3x to 3x, float sums 2x to 8x; widening and narrowing, and 32-bit-lane min/max, still lose to C2. Narrow sums and widening are built without cross-width Vector API conversions, which are not intrinsified (about 40x slower) or are slow on AVX2. Build wiring: core compiles against the incubating module and every test JVM runs with it; a second surefire pass runs the SIMD tests without it to cover the fallback. JDK 26 reports the incubating module as a compiler warning and its javadoc cannot see it, so core silences that lint category and a JDK 26+ profile adds the module for javadoc. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01JbquncyA7mTMJbDJo1s5HD --- CHANGELOG.md | 3 + CLAUDE.md | 7 +- core/pom.xml | 65 + .../core/simd/SimdOperationsSupport.java | 32 +- .../core/simd/VectorApiSimdOperations.java | 1197 +++++++++++++++++ .../core/simd/SimdOperationsSupportTest.java | 44 + .../simd/VectorApiSimdOperationsTest.java | 649 +++++++++ docs/compatibility.md | 27 +- integration/pom.xml | 1 + pom.xml | 2 +- 10 files changed, 2016 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperations.java create mode 100644 core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupportTest.java create mode 100644 core/src/test/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperationsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 099a1a3fc..bdf520495 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Opt-in Vector API kernels for min/max, sums, run counting, widening and bit-packing, enabled by `--add-modules jdk.incubator.vector` (over 10× faster on byte and short columns); float sums and 64-bit overflow checks differ slightly from Rust's order, see `docs/compatibility.md`. ([#516](https://github.com/dfa1/vortex-java/issues/516)) + ### Changed - Float min/max statistics order `-0.0` before `0.0` and skip `NaN`, and `U64` min/max compare unsigned, as Rust does. ([#516](https://github.com/dfa1/vortex-java/issues/516)) diff --git a/CLAUDE.md b/CLAUDE.md index 734f33796..281d5288f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,8 @@ core — everything lives under `io.github.dfa1.vortex.core.*`: core.compute FastLanes, PrimitiveArrays, Utf8Order core.simd SimdOperations (kernels shared by reader+writer, all eleven ptypes), AutoVectorizedSimdOperations (C2-shaped loops, the Rust-parity default), - SimdOperationsSupport#preferred() (picks the impl); Vector API impl planned (ADR 0005, #483, #484) + VectorApiSimdOperations (opt-in via --add-modules jdk.incubator.vector; a few documented + differences from Rust, see docs/compatibility.md), SimdOperationsSupport#preferred() (picks one) core.fbs / core.proto — generated wire codecs + their runtimes reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, Zone, ScanOptions, RowFilter; file internals (Footer, Trailer, PostscriptParser, …) @@ -273,6 +274,10 @@ When stuck on encode/decode behavior, consult **in this order**: starting point and `withXxx` methods return copies. Never a record and never a public constructor — they hold callbacks or executors with no meaningful `equals`, and a constructor freezes the field list as API. +- **The Vector API implementation may differ from Rust in a few reductions** (decision recorded in + `docs/compatibility.md` and `VectorApiSimdOperations`): lane-wise float sums and per-lane `I64`/`U64` + overflow detection, instead of forcing scalar loops to keep Rust's exact order. It is opt-in + (`--add-modules jdk.incubator.vector`); the default `AutoVectorizedSimdOperations` stays Rust-parity. - **Small public APIs.** Don't expose internals — when in doubt, leave it out or make it private. - **POM deps** grouped with comments: `` then ``, each with project-internal (`io.github.dfa1.vortex:*`) deps first, then external. Omit empty sections. diff --git a/core/pom.xml b/core/pom.xml index 704340e91..2d649cb51 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -35,6 +35,21 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + + + --add-modules + jdk.incubator.vector + -Xlint:-incubating + + + @@ -45,6 +60,31 @@ true + + + org.apache.maven.plugins + maven-surefire-plugin + + + without-vector-module + test + + test + + + @{argLine} --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED + + **/SimdOperationsSupportTest.java + **/SimdOperationsTest.java + + ${project.build.directory}/surefire-reports-without-vector-module + + + + @@ -72,6 +112,31 @@ Then commit the updated files. --> + + + javadoc-sees-incubating-module + + [26,) + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + false + + --add-modules + jdk.incubator.vector + + + + + + regenerate-sources diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java index c2e13ed79..9f97da244 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupport.java @@ -2,12 +2,15 @@ /// Selects the [SimdOperations] implementation, once per JVM. /// -/// Only the auto-vectorized kernels exist today. The Vector API implementation -/// (ADR 0005, #483, #484) will be chosen here by a guarded static initializer that falls back to them when -/// `jdk.incubator.vector` is absent, so no call site changes when it lands. +/// The Vector API implementation is chosen only when `jdk.incubator.vector` is on the module graph +/// (`--add-modules jdk.incubator.vector`) and the CPU has vectors of at least 128 bits; otherwise the +/// auto-vectorized implementation stays in effect. That launch flag is the opt-in, and omitting it +/// is the opt-out, as in Hardwood: ADR 0005 rules out a per-scan flag. public final class SimdOperationsSupport { - private static final SimdOperations INSTANCE = new AutoVectorizedSimdOperations(); + private static final System.Logger LOG = System.getLogger(SimdOperationsSupport.class.getName()); + + private static final SimdOperations INSTANCE = select(); private SimdOperationsSupport() { } @@ -18,4 +21,25 @@ private SimdOperationsSupport() { public static SimdOperations preferred() { return INSTANCE; } + + private static SimdOperations select() { + SimdOperations autoVectorized = new AutoVectorizedSimdOperations(); + if (ModuleLayer.boot().findModule("jdk.incubator.vector").isEmpty()) { + LOG.log(System.Logger.Level.DEBUG, "SIMD: auto-vectorized (jdk.incubator.vector not on the module graph)"); + return autoVectorized; + } + try { + if (!VectorApiSimdOperations.isUsable()) { + LOG.log(System.Logger.Level.DEBUG, "SIMD: auto-vectorized (preferred vectors narrower than 128 bits)"); + return autoVectorized; + } + SimdOperations vectorApi = new VectorApiSimdOperations(); + LOG.log(System.Logger.Level.DEBUG, "SIMD: Vector API, {0}-bit vectors", VectorApiSimdOperations.vectorBitSize()); + return vectorApi; + } catch (LinkageError e) { + // The module is present but the Vector API could not be linked or initialized + LOG.log(System.Logger.Level.DEBUG, "SIMD: auto-vectorized (Vector API unavailable: {0})", e.toString()); + return autoVectorized; + } + } } diff --git a/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperations.java b/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperations.java new file mode 100644 index 000000000..4b1ab7256 --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperations.java @@ -0,0 +1,1197 @@ +package io.github.dfa1.vortex.core.simd; + +import io.github.dfa1.vortex.core.compute.FastLanes; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.PType; +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.DoubleVector; +import jdk.incubator.vector.FloatVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.ShortVector; +import jdk.incubator.vector.VectorMask; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.lang.reflect.Array; +import java.nio.ByteOrder; +import java.util.OptionalLong; + +/// The [SimdOperations] written with the incubating Vector API: explicit vector loops over the +/// preferred species of the CPU, each with a scalar tail for the elements that do not fill a vector. +/// The Vector API does the work, including the reductions; no kernel is forced back to a scalar loop +/// to protect an ordering, with the two exceptions below. +/// +/// Only [SimdOperationsSupport] instantiates it, and only when `jdk.incubator.vector` is on the +/// module graph (`--add-modules jdk.incubator.vector`); without it this class fails to link and the +/// [AutoVectorizedSimdOperations] stays in effect. A differential test pins every kernel to that +/// implementation, which follows the Rust reference. +/// +/// ## Where it differs from Rust (a deliberate decision) +/// +/// - **Floating-point sums** accumulate lane-wise and reduce once, so they add in a different order +/// from Rust's sequential `f64` sum. The result can differ in its last bits, and the JDK documents +/// `reduceLanes(ADD)` on floats as using "an arbitrary order of operations, which may even vary over +/// time", so it may also differ between runs and between CPUs with different vector widths. Integer +/// results are exact. +/// - **`I64`/`U64` sums** detect overflow on each lane's running sum, not on the sequential prefix Rust +/// checks (a checked add per element). Both can report an overflow the other does not, on arrays whose +/// partial sums approach the limits: for example `[MAX, 1, -1]` overflows Rust's prefix but not the +/// total. +/// - **`F16`** sums and min/max are scalar loops: the Vector API has no half-float lanes. +/// +/// Outputs derived from these (the zone-map sum statistic) therefore depend on whether the module is +/// enabled; the auto-vectorized implementation, used without the flag, is the Rust-parity one. See +/// `docs/compatibility.md`. +/// +/// The shape follows Hardwood's `VectorOperations` (Apache-2.0, hardwood-hq/hardwood): the +/// preferred species, a main loop over whole vectors, and a scalar tail. +final class VectorApiSimdOperations implements SimdOperations { + + private static final VectorSpecies BYTES = ByteVector.SPECIES_PREFERRED; + private static final VectorSpecies SHORTS = ShortVector.SPECIES_PREFERRED; + private static final VectorSpecies INTS = IntVector.SPECIES_PREFERRED; + private static final VectorSpecies LONGS = LongVector.SPECIES_PREFERRED; + private static final VectorSpecies FLOATS = FloatVector.SPECIES_PREFERRED; + private static final VectorSpecies DOUBLES = DoubleVector.SPECIES_PREFERRED; + + /// Vector iterations a narrow-element sum accumulates in `int` lanes before flushing to a `long`. + /// A lane gains at most `parts` elements per iteration, each below 2^16, so 2^13 iterations stay + /// far under 2^31. + private static final int SUM_FLUSH_ITERATIONS = 1 << 13; + + /// Whether the Vector API is worth using on this CPU: a preferred vector of at least 128 bits. + /// Anything narrower cannot beat the scalar loops. + /// + /// @return `true` if the preferred species holds at least four ints and sixteen bytes + static boolean isUsable() { + return INTS.length() >= 4 && BYTES.length() >= 16; + } + + /// The width of the vectors in use, for diagnostics. + /// + /// @return the preferred vector size in bits + static int vectorBitSize() { + return INTS.vectorBitSize(); + } + + // ---- widen ---- + + // Narrow lanes widen in stages (byte to int to long, short to int to long): the Vector API does not + // intrinsify a direct conversion across a lane ratio of four or more into 64-bit lanes, and that + // direct form measured about 40x slower than the staged one (JMH, 128-bit NEON). The ratios of the + // stages (4 and 2, 2 and 2) are fixed by the types, so they hold at any vector width. + @Override + public void widenInto(MemorySegment src, long fromElement, int count, PType ptype, long[] out) { + switch (ptype) { + case I8, U8 -> { + int i = 0; + for (; i + BYTES.length() <= count; i += BYTES.length()) { + widenBytes(ByteVector.fromMemorySegment(BYTES, src, fromElement + i, ByteOrder.LITTLE_ENDIAN), + ptype == PType.U8, out, i); + } + for (; i < count; i++) { + long v = src.get(ValueLayout.JAVA_BYTE, fromElement + i); + out[i] = ptype == PType.U8 ? v & 0xFFL : v; + } + } + case I16, U16, F16 -> { + int i = 0; + for (; i + SHORTS.length() <= count; i += SHORTS.length()) { + widenShorts(ShortVector.fromMemorySegment(SHORTS, src, (fromElement + i) * 2, ByteOrder.LITTLE_ENDIAN), + ptype != PType.I16, out, i); + } + for (; i < count; i++) { + long v = src.getAtIndex(VortexFormat.LE_SHORT, fromElement + i); + out[i] = ptype == PType.I16 ? v : v & 0xFFFFL; + } + } + case I32, U32, F32 -> { + int i = 0; + for (; i + INTS.length() <= count; i += INTS.length()) { + widenInts(IntVector.fromMemorySegment(INTS, src, (fromElement + i) * 4, ByteOrder.LITTLE_ENDIAN), + ptype != PType.I32, out, i); + } + for (; i < count; i++) { + long v = src.getAtIndex(VortexFormat.LE_INT, fromElement + i); + out[i] = ptype == PType.I32 ? v : v & 0xFFFF_FFFFL; + } + } + // A copy is a copy: the JDK's bulk copy is what a vector loop would be at best + case I64, U64, F64 -> MemorySegment.copy(src, VortexFormat.LE_LONG, fromElement * 8, out, 0, count); + } + } + + @Override + public void widenArrayInto(Object values, int from, int count, PType ptype, long[] out) { + switch (ptype) { + case I8, U8 -> { + byte[] a = (byte[]) values; + int i = 0; + for (; i + BYTES.length() <= count; i += BYTES.length()) { + widenBytes(ByteVector.fromArray(BYTES, a, from + i), ptype == PType.U8, out, i); + } + for (; i < count; i++) { + out[i] = ptype == PType.U8 ? a[from + i] & 0xFFL : a[from + i]; + } + } + case I16, U16, F16 -> { + short[] a = (short[]) values; + int i = 0; + for (; i + SHORTS.length() <= count; i += SHORTS.length()) { + widenShorts(ShortVector.fromArray(SHORTS, a, from + i), ptype != PType.I16, out, i); + } + for (; i < count; i++) { + out[i] = ptype == PType.I16 ? a[from + i] : a[from + i] & 0xFFFFL; + } + } + case I32, U32 -> { + int[] a = (int[]) values; + int i = 0; + for (; i + INTS.length() <= count; i += INTS.length()) { + widenInts(IntVector.fromArray(INTS, a, from + i), ptype == PType.U32, out, i); + } + for (; i < count; i++) { + out[i] = ptype == PType.U32 ? a[from + i] & 0xFFFF_FFFFL : a[from + i]; + } + } + case I64, U64 -> System.arraycopy((long[]) values, from, out, 0, count); + case F32 -> { + float[] a = (float[]) values; + int i = 0; + for (; i + FLOATS.length() <= count; i += FLOATS.length()) { + widenInts(FloatVector.fromArray(FLOATS, a, from + i).reinterpretAsInts(), true, out, i); + } + for (; i < count; i++) { + out[i] = Float.floatToRawIntBits(a[from + i]) & 0xFFFF_FFFFL; + } + } + case F64 -> { + double[] a = (double[]) values; + int i = 0; + for (; i + DOUBLES.length() <= count; i += DOUBLES.length()) { + DoubleVector.fromArray(DOUBLES, a, from + i).reinterpretAsLongs().intoArray(out, i); + } + for (; i < count; i++) { + out[i] = Double.doubleToRawLongBits(a[from + i]); + } + } + } + } + + /// Widens a byte vector to `BYTES.length()` longs at `out[at]`: byte to int (four parts), int to long. + private static void widenBytes(ByteVector v, boolean unsigned, long[] out, int at) { + VectorOperators.Conversion toInt = unsigned ? VectorOperators.ZERO_EXTEND_B2I : VectorOperators.B2I; + for (int part = 0; part < BYTES.length() / INTS.length(); part++) { + widenInts((IntVector) v.convertShape(toInt, INTS, part), false, out, at + part * INTS.length()); + } + } + + private static void widenShorts(ShortVector v, boolean unsigned, long[] out, int at) { + VectorOperators.Conversion toInt = unsigned ? VectorOperators.ZERO_EXTEND_S2I : VectorOperators.S2I; + for (int part = 0; part < SHORTS.length() / INTS.length(); part++) { + widenInts((IntVector) v.convertShape(toInt, INTS, part), false, out, at + part * INTS.length()); + } + } + + /// Widens an int vector to `INTS.length()` longs at `out[at]`, sign- or zero-extending the lanes. + private static void widenInts(IntVector v, boolean unsigned, long[] out, int at) { + VectorOperators.Conversion toLong = unsigned ? VectorOperators.ZERO_EXTEND_I2L : VectorOperators.I2L; + for (int part = 0; part < INTS.length() / LONGS.length(); part++) { + ((LongVector) v.convertShape(toLong, LONGS, part)).intoArray(out, at + part * LONGS.length()); + } + } + + // ---- narrow ---- + + @Override + public void narrowInto(long[] values, PType ptype, MemorySegment dst) { + int n = values.length; + switch (ptype) { + case I8, U8 -> { + int i = 0; + for (; i + BYTES.length() <= n; i += BYTES.length()) { + narrowToBytes(values, i).intoMemorySegment(dst, i, ByteOrder.LITTLE_ENDIAN); + } + for (; i < n; i++) { + dst.set(ValueLayout.JAVA_BYTE, i, (byte) values[i]); + } + } + case I16, U16, F16 -> { + int i = 0; + for (; i + SHORTS.length() <= n; i += SHORTS.length()) { + narrowToShorts(values, i).intoMemorySegment(dst, i * 2L, ByteOrder.LITTLE_ENDIAN); + } + for (; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_SHORT, i, (short) values[i]); + } + } + case I32, U32, F32 -> { + int i = 0; + for (; i + INTS.length() <= n; i += INTS.length()) { + narrowToInts(values, i).intoMemorySegment(dst, i * 4L, ByteOrder.LITTLE_ENDIAN); + } + for (; i < n; i++) { + dst.setAtIndex(VortexFormat.LE_INT, i, (int) values[i]); + } + } + case I64, U64, F64 -> MemorySegment.copy(values, 0, dst, VortexFormat.LE_LONG, 0, n); + } + } + + @Override + public void narrowArrayInto(long[] values, PType ptype, Object out) { + int n = values.length; + switch (ptype) { + case I8, U8 -> { + byte[] a = (byte[]) out; + int i = 0; + for (; i + BYTES.length() <= n; i += BYTES.length()) { + narrowToBytes(values, i).intoArray(a, i); + } + for (; i < n; i++) { + a[i] = (byte) values[i]; + } + } + case I16, U16, F16 -> { + short[] a = (short[]) out; + int i = 0; + for (; i + SHORTS.length() <= n; i += SHORTS.length()) { + narrowToShorts(values, i).intoArray(a, i); + } + for (; i < n; i++) { + a[i] = (short) values[i]; + } + } + case I32, U32 -> { + int[] a = (int[]) out; + int i = 0; + for (; i + INTS.length() <= n; i += INTS.length()) { + narrowToInts(values, i).intoArray(a, i); + } + for (; i < n; i++) { + a[i] = (int) values[i]; + } + } + case I64, U64 -> System.arraycopy(values, 0, (long[]) out, 0, n); + case F32 -> { + float[] a = (float[]) out; + int i = 0; + for (; i + INTS.length() <= n; i += INTS.length()) { + narrowToInts(values, i).reinterpretAsFloats().intoArray(a, i); + } + for (; i < n; i++) { + a[i] = Float.intBitsToFloat((int) values[i]); + } + } + case F64 -> { + double[] a = (double[]) out; + int i = 0; + for (; i + LONGS.length() <= n; i += LONGS.length()) { + LongVector.fromArray(LONGS, values, i).reinterpretAsDoubles().intoArray(a, i); + } + for (; i < n; i++) { + a[i] = Double.longBitsToDouble(values[i]); + } + } + } + } + + // Narrowing mirrors widening: two long vectors contract into one int vector (parts 0 and -1), and + // 4 (bytes) or 2 (shorts) int vectors contract into one byte or short vector, each into its own slice + // of the result (part -k) with the other lanes zero, combined with an OR. A direct long to byte + // contraction measured about 25x slower than this staged form. + private static IntVector narrowToInts(long[] values, int from) { + return ((IntVector) LongVector.fromArray(LONGS, values, from).convertShape(VectorOperators.L2I, INTS, 0)) + .or((IntVector) LongVector.fromArray(LONGS, values, from + LONGS.length()) + .convertShape(VectorOperators.L2I, INTS, -1)); + } + + private static ByteVector narrowToBytes(long[] values, int from) { + ByteVector result = ByteVector.zero(BYTES); + for (int part = 0; part < BYTES.length() / INTS.length(); part++) { + result = result.or((ByteVector) narrowToInts(values, from + part * INTS.length()) + .convertShape(VectorOperators.I2B, BYTES, -part)); + } + return result; + } + + private static ShortVector narrowToShorts(long[] values, int from) { + ShortVector result = ShortVector.zero(SHORTS); + for (int part = 0; part < SHORTS.length() / INTS.length(); part++) { + result = result.or((ShortVector) narrowToInts(values, from + part * INTS.length()) + .convertShape(VectorOperators.I2S, SHORTS, -part)); + } + return result; + } + + // ---- maxUnsigned ---- + + // The sign bit is flipped so the signed lane max orders unsigned values, and flipped back at the end. + @Override + public long maxUnsigned(MemorySegment src, long count, PType ptype) { + return switch (ptype) { + case U8 -> maxUnsignedBytes(src, count); + case U16 -> maxUnsignedShorts(src, count); + case U32 -> maxUnsignedInts(src, count); + case U64 -> maxUnsignedLongs(src, count); + default -> throw new IllegalArgumentException("not an unsigned integer ptype: " + ptype); + }; + } + + private static long maxUnsignedBytes(MemorySegment src, long count) { + ByteVector flip = ByteVector.broadcast(BYTES, Byte.MIN_VALUE); + ByteVector max = ByteVector.broadcast(BYTES, Byte.MIN_VALUE); + long i = 0; + for (; i + BYTES.length() <= count; i += BYTES.length()) { + max = max.max(ByteVector.fromMemorySegment(BYTES, src, i, ByteOrder.LITTLE_ENDIAN) + .lanewise(VectorOperators.XOR, flip)); + } + int result = max.reduceLanes(VectorOperators.MAX); + for (; i < count; i++) { + result = Math.max(result, (byte) (src.get(ValueLayout.JAVA_BYTE, i) ^ Byte.MIN_VALUE)); + } + return Byte.toUnsignedLong((byte) (result ^ Byte.MIN_VALUE)); + } + + private static long maxUnsignedShorts(MemorySegment src, long count) { + ShortVector flip = ShortVector.broadcast(SHORTS, Short.MIN_VALUE); + ShortVector max = ShortVector.broadcast(SHORTS, Short.MIN_VALUE); + long i = 0; + for (; i + SHORTS.length() <= count; i += SHORTS.length()) { + max = max.max(ShortVector.fromMemorySegment(SHORTS, src, i * 2, ByteOrder.LITTLE_ENDIAN) + .lanewise(VectorOperators.XOR, flip)); + } + int result = max.reduceLanes(VectorOperators.MAX); + for (; i < count; i++) { + result = Math.max(result, (short) (src.getAtIndex(VortexFormat.LE_SHORT, i) ^ Short.MIN_VALUE)); + } + return Short.toUnsignedLong((short) (result ^ Short.MIN_VALUE)); + } + + private static long maxUnsignedInts(MemorySegment src, long count) { + IntVector flip = IntVector.broadcast(INTS, Integer.MIN_VALUE); + IntVector max = IntVector.broadcast(INTS, Integer.MIN_VALUE); + long i = 0; + for (; i + INTS.length() <= count; i += INTS.length()) { + max = max.max(IntVector.fromMemorySegment(INTS, src, i * 4, ByteOrder.LITTLE_ENDIAN) + .lanewise(VectorOperators.XOR, flip)); + } + int result = max.reduceLanes(VectorOperators.MAX); + for (; i < count; i++) { + result = Math.max(result, src.getAtIndex(VortexFormat.LE_INT, i) ^ Integer.MIN_VALUE); + } + return Integer.toUnsignedLong(result ^ Integer.MIN_VALUE); + } + + private static long maxUnsignedLongs(MemorySegment src, long count) { + LongVector flip = LongVector.broadcast(LONGS, Long.MIN_VALUE); + LongVector max = LongVector.broadcast(LONGS, Long.MIN_VALUE); + long i = 0; + for (; i + LONGS.length() <= count; i += LONGS.length()) { + max = max.max(LongVector.fromMemorySegment(LONGS, src, i * 8, ByteOrder.LITTLE_ENDIAN) + .lanewise(VectorOperators.XOR, flip)); + } + long result = max.reduceLanes(VectorOperators.MAX); + for (; i < count; i++) { + result = Math.max(result, src.getAtIndex(VortexFormat.LE_LONG, i) ^ Long.MIN_VALUE); + } + return result ^ Long.MIN_VALUE; + } + + // ---- minMax ---- + + // Lane-wise min and max accumulators, reduced once at the end. Unsigned widths XOR the sign bit + // first, which maps unsigned order onto signed order so the loop stays a plain signed min/max. + @Override + public long[] minMax(Object values, PType ptype) { + if (Array.getLength(values) == 0) { + if (ptype.isFloating()) { + return new long[0]; + } + throw new IllegalArgumentException("empty array has no min/max"); + } + return switch (ptype) { + case I8 -> minMax((byte[]) values, (byte) 0); + case U8 -> unsigned(minMax((byte[]) values, Byte.MIN_VALUE), 0xFFL, Byte.MIN_VALUE); + case I16 -> minMax((short[]) values, (short) 0); + case U16 -> unsigned(minMax((short[]) values, Short.MIN_VALUE), 0xFFFFL, Short.MIN_VALUE); + case I32 -> minMax((int[]) values, 0); + case U32 -> unsigned(minMax((int[]) values, Integer.MIN_VALUE), 0xFFFF_FFFFL, Integer.MIN_VALUE); + case I64 -> minMax((long[]) values, 0L); + case U64 -> unsigned(minMax((long[]) values, Long.MIN_VALUE), -1L, Long.MIN_VALUE); + case F16 -> minMaxHalf((short[]) values); + case F32 -> minMax((float[]) values); + case F64 -> minMax((double[]) values); + }; + } + + /// Undoes the sign-bit flip of a minimum and maximum found in flipped space. + private static long[] unsigned(long[] flipped, long mask, long signBit) { + return new long[]{(flipped[0] ^ signBit) & mask, (flipped[1] ^ signBit) & mask}; + } + + private static long[] minMax(byte[] a, byte flip) { + ByteVector flipVector = ByteVector.broadcast(BYTES, flip); + ByteVector min = ByteVector.broadcast(BYTES, Byte.MAX_VALUE); + ByteVector max = ByteVector.broadcast(BYTES, Byte.MIN_VALUE); + int i = 0; + for (; i + BYTES.length() <= a.length; i += BYTES.length()) { + ByteVector v = ByteVector.fromArray(BYTES, a, i).lanewise(VectorOperators.XOR, flipVector); + min = min.min(v); + max = max.max(v); + } + int lo = min.reduceLanes(VectorOperators.MIN); + int hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + int v = (byte) (a[i] ^ flip); + lo = Math.min(lo, v); + hi = Math.max(hi, v); + } + return new long[]{lo, hi}; + } + + private static long[] minMax(short[] a, short flip) { + ShortVector flipVector = ShortVector.broadcast(SHORTS, flip); + ShortVector min = ShortVector.broadcast(SHORTS, Short.MAX_VALUE); + ShortVector max = ShortVector.broadcast(SHORTS, Short.MIN_VALUE); + int i = 0; + for (; i + SHORTS.length() <= a.length; i += SHORTS.length()) { + ShortVector v = ShortVector.fromArray(SHORTS, a, i).lanewise(VectorOperators.XOR, flipVector); + min = min.min(v); + max = max.max(v); + } + int lo = min.reduceLanes(VectorOperators.MIN); + int hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + int v = (short) (a[i] ^ flip); + lo = Math.min(lo, v); + hi = Math.max(hi, v); + } + return new long[]{lo, hi}; + } + + private static long[] minMax(int[] a, int flip) { + IntVector flipVector = IntVector.broadcast(INTS, flip); + IntVector min = IntVector.broadcast(INTS, Integer.MAX_VALUE); + IntVector max = IntVector.broadcast(INTS, Integer.MIN_VALUE); + int i = 0; + for (; i + INTS.length() <= a.length; i += INTS.length()) { + IntVector v = IntVector.fromArray(INTS, a, i).lanewise(VectorOperators.XOR, flipVector); + min = min.min(v); + max = max.max(v); + } + int lo = min.reduceLanes(VectorOperators.MIN); + int hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + int v = a[i] ^ flip; + lo = Math.min(lo, v); + hi = Math.max(hi, v); + } + return new long[]{lo, hi}; + } + + private static long[] minMax(long[] a, long flip) { + LongVector flipVector = LongVector.broadcast(LONGS, flip); + LongVector min = LongVector.broadcast(LONGS, Long.MAX_VALUE); + LongVector max = LongVector.broadcast(LONGS, Long.MIN_VALUE); + int i = 0; + for (; i + LONGS.length() <= a.length; i += LONGS.length()) { + LongVector v = LongVector.fromArray(LONGS, a, i).lanewise(VectorOperators.XOR, flipVector); + min = min.min(v); + max = max.max(v); + } + long lo = min.reduceLanes(VectorOperators.MIN); + long hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + long v = a[i] ^ flip; + lo = Math.min(lo, v); + hi = Math.max(hi, v); + } + return new long[]{lo, hi}; + } + + // Masked lane-wise min/max: a NaN lane is left out of the reduction (Rust's `skip_nans`), and the + // lane-wise and cross-lane min/max follow Math.min/Math.max, which order -0.0 before 0.0 exactly as + // Rust's total order does. + private static long[] minMax(float[] a) { + FloatVector min = FloatVector.broadcast(FLOATS, Float.POSITIVE_INFINITY); + FloatVector max = FloatVector.broadcast(FLOATS, Float.NEGATIVE_INFINITY); + int i = 0; + for (; i + FLOATS.length() <= a.length; i += FLOATS.length()) { + FloatVector v = FloatVector.fromArray(FLOATS, a, i); + VectorMask ordered = v.test(VectorOperators.IS_NAN).not(); + min = min.lanewise(VectorOperators.MIN, v, ordered); + max = max.lanewise(VectorOperators.MAX, v, ordered); + } + float lo = min.reduceLanes(VectorOperators.MIN); + float hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + if (a[i] == a[i]) { + lo = Math.min(lo, a[i]); + hi = Math.max(hi, a[i]); + } + } + return lo > hi ? new long[0] + : new long[]{Float.floatToRawIntBits(lo) & 0xFFFF_FFFFL, Float.floatToRawIntBits(hi) & 0xFFFF_FFFFL}; + } + + private static long[] minMax(double[] a) { + DoubleVector min = DoubleVector.broadcast(DOUBLES, Double.POSITIVE_INFINITY); + DoubleVector max = DoubleVector.broadcast(DOUBLES, Double.NEGATIVE_INFINITY); + int i = 0; + for (; i + DOUBLES.length() <= a.length; i += DOUBLES.length()) { + DoubleVector v = DoubleVector.fromArray(DOUBLES, a, i); + VectorMask ordered = v.test(VectorOperators.IS_NAN).not(); + min = min.lanewise(VectorOperators.MIN, v, ordered); + max = max.lanewise(VectorOperators.MAX, v, ordered); + } + double lo = min.reduceLanes(VectorOperators.MIN); + double hi = max.reduceLanes(VectorOperators.MAX); + for (; i < a.length; i++) { + if (a[i] == a[i]) { + lo = Math.min(lo, a[i]); + hi = Math.max(hi, a[i]); + } + } + return lo > hi ? new long[0] : new long[]{Double.doubleToRawLongBits(lo), Double.doubleToRawLongBits(hi)}; + } + + // The Vector API has no half-float lanes, so this is a plain loop, the same as the reference. + private static long[] minMaxHalf(short[] a) { + float lo = 0f; + float hi = 0f; + short loBits = 0; + short hiBits = 0; + boolean seen = false; + for (short v : a) { + float f = Float.float16ToFloat(v); + if (f != f) { + continue; + } + if (!seen) { + lo = f; + hi = f; + loBits = v; + hiBits = v; + seen = true; + continue; + } + if (Float.compare(f, lo) < 0) { + lo = f; + loBits = v; + } + if (Float.compare(f, hi) > 0) { + hi = f; + hiBits = v; + } + } + return seen ? new long[]{Short.toUnsignedLong(loBits), Short.toUnsignedLong(hiBits)} : new long[0]; + } + + // ---- allEqual ---- + + /// Vectors compared per early-exit check of [#allEqual(Object, PType)]. Each block ORs the XOR against + /// the first element and tests the accumulator once, so the loop body has no branch; a non-constant + /// array is still rejected after one block. + private static final int EQUAL_BLOCK_VECTORS = 16; + + // The differences from the first element are ORed over a block and tested once per block; floats by + // their raw bits. + @Override + public boolean allEqual(Object values, PType ptype) { + if (Array.getLength(values) == 0) { + return true; + } + return switch (ptype) { + case I8, U8 -> allEqual((byte[]) values); + case I16, U16, F16 -> allEqual((short[]) values); + case I32, U32 -> allEqual((int[]) values); + case I64, U64 -> allEqual((long[]) values); + case F32 -> allEqual((float[]) values); + case F64 -> allEqual((double[]) values); + }; + } + + @Override + public boolean allEqual(boolean[] values) { + if (values.length == 0) { + return true; + } + ByteVector first = ByteVector.broadcast(BYTES, (byte) (values[0] ? 1 : 0)); + int step = BYTES.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= values.length) { + ByteVector diff = ByteVector.zero(BYTES); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(ByteVector.fromBooleanArray(BYTES, values, i).lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, (byte) 0).anyTrue()) { + return false; + } + } + for (; i < values.length; i++) { + if (values[i] != values[0]) { + return false; + } + } + return true; + } + + private static boolean allEqual(byte[] a) { + ByteVector first = ByteVector.broadcast(BYTES, a[0]); + int step = BYTES.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + ByteVector diff = ByteVector.zero(BYTES); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(ByteVector.fromArray(BYTES, a, i).lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, (byte) 0).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (a[i] != a[0]) { + return false; + } + } + return true; + } + + private static boolean allEqual(short[] a) { + ShortVector first = ShortVector.broadcast(SHORTS, a[0]); + int step = SHORTS.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + ShortVector diff = ShortVector.zero(SHORTS); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(ShortVector.fromArray(SHORTS, a, i).lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, (short) 0).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (a[i] != a[0]) { + return false; + } + } + return true; + } + + private static boolean allEqual(int[] a) { + IntVector first = IntVector.broadcast(INTS, a[0]); + int step = INTS.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + IntVector diff = IntVector.zero(INTS); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(IntVector.fromArray(INTS, a, i).lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, 0).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (a[i] != a[0]) { + return false; + } + } + return true; + } + + private static boolean allEqual(long[] a) { + LongVector first = LongVector.broadcast(LONGS, a[0]); + int step = LONGS.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + LongVector diff = LongVector.zero(LONGS); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(LongVector.fromArray(LONGS, a, i).lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, 0L).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (a[i] != a[0]) { + return false; + } + } + return true; + } + + private static boolean allEqual(float[] a) { + int firstBits = Float.floatToRawIntBits(a[0]); + IntVector first = IntVector.broadcast(INTS, firstBits); + int step = FLOATS.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + IntVector diff = IntVector.zero(INTS); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(FloatVector.fromArray(FLOATS, a, i).reinterpretAsInts().lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, 0).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (Float.floatToRawIntBits(a[i]) != firstBits) { + return false; + } + } + return true; + } + + private static boolean allEqual(double[] a) { + long firstBits = Double.doubleToRawLongBits(a[0]); + LongVector first = LongVector.broadcast(LONGS, firstBits); + int step = DOUBLES.length(); + int i = 0; + while (i + step * EQUAL_BLOCK_VECTORS <= a.length) { + LongVector diff = LongVector.zero(LONGS); + for (int end = i + step * EQUAL_BLOCK_VECTORS; i < end; i += step) { + diff = diff.or(DoubleVector.fromArray(DOUBLES, a, i).reinterpretAsLongs().lanewise(VectorOperators.XOR, first)); + } + if (diff.compare(VectorOperators.NE, 0L).anyTrue()) { + return false; + } + } + for (; i < a.length; i++) { + if (Double.doubleToRawLongBits(a[i]) != firstBits) { + return false; + } + } + return true; + } + + // ---- runs ---- + + // Each vector is compared against the same elements shifted by one, so the changes are counted + // without a dependency between iterations. NE on floating-point lanes is `!=`: NaN always differs. + @Override + public long runs(Object values, PType ptype) { + if (Array.getLength(values) == 0) { + return 0; + } + return 1 + switch (ptype) { + case I8, U8 -> changes((byte[]) values); + case I16, U16, F16 -> changes((short[]) values); + case I32, U32 -> changes((int[]) values); + case I64, U64 -> changes((long[]) values); + case F32 -> changes((float[]) values); + case F64 -> changes((double[]) values); + }; + } + + private static long changes(byte[] a) { + long changes = 0; + int i = 1; + for (; i + BYTES.length() <= a.length; i += BYTES.length()) { + changes += ByteVector.fromArray(BYTES, a, i) + .compare(VectorOperators.NE, ByteVector.fromArray(BYTES, a, i - 1)).trueCount(); + } + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + private static long changes(short[] a) { + long changes = 0; + int i = 1; + for (; i + SHORTS.length() <= a.length; i += SHORTS.length()) { + changes += ShortVector.fromArray(SHORTS, a, i) + .compare(VectorOperators.NE, ShortVector.fromArray(SHORTS, a, i - 1)).trueCount(); + } + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + // The comparison masks are accumulated as -1 lanes and reduced once, instead of counting each mask + // with trueCount(), a cross-lane operation per vector. A lane gains at most one per vector, so the + // 32-bit counters hold any array that fits in memory. + private static long changes(int[] a) { + IntVector count = IntVector.zero(INTS); + int i = 1; + for (; i + INTS.length() <= a.length; i += INTS.length()) { + count = count.sub(IntVector.fromArray(INTS, a, i) + .compare(VectorOperators.NE, IntVector.fromArray(INTS, a, i - 1)).toVector()); + } + long changes = laneSum(count); + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + private static long changes(long[] a) { + LongVector count = LongVector.zero(LONGS); + int i = 1; + for (; i + LONGS.length() <= a.length; i += LONGS.length()) { + count = count.sub(LongVector.fromArray(LONGS, a, i) + .compare(VectorOperators.NE, LongVector.fromArray(LONGS, a, i - 1)).toVector()); + } + long changes = count.reduceLanes(VectorOperators.ADD); + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + private static long changes(float[] a) { + IntVector count = IntVector.zero(INTS); + int i = 1; + for (; i + FLOATS.length() <= a.length; i += FLOATS.length()) { + count = count.sub(FloatVector.fromArray(FLOATS, a, i) + .compare(VectorOperators.NE, FloatVector.fromArray(FLOATS, a, i - 1)).cast(INTS).toVector()); + } + long changes = laneSum(count); + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + private static long changes(double[] a) { + LongVector count = LongVector.zero(LONGS); + int i = 1; + for (; i + DOUBLES.length() <= a.length; i += DOUBLES.length()) { + count = count.sub(DoubleVector.fromArray(DOUBLES, a, i) + .compare(VectorOperators.NE, DoubleVector.fromArray(DOUBLES, a, i - 1)).cast(LONGS).toVector()); + } + long changes = count.reduceLanes(VectorOperators.ADD); + for (; i < a.length; i++) { + changes += a[i] != a[i - 1] ? 1 : 0; + } + return changes; + } + + // ---- sum ---- + + @Override + public OptionalLong sum(Object values, PType ptype) { + return switch (ptype) { + case I8 -> OptionalLong.of(sumBytes((byte[]) values, false)); + case U8 -> OptionalLong.of(sumBytes((byte[]) values, true)); + case I16 -> OptionalLong.of(sumShorts((short[]) values, false)); + case U16 -> OptionalLong.of(sumShorts((short[]) values, true)); + case I32 -> OptionalLong.of(sum((int[]) values, VectorOperators.I2L)); + case U32 -> OptionalLong.of(sum((int[]) values, VectorOperators.ZERO_EXTEND_I2L)); + case I64 -> sumSigned((long[]) values); + case U64 -> sumUnsigned((long[]) values); + case F16, F32, F64 -> throw new IllegalArgumentException("not an integer ptype: " + ptype); + }; + } + + // Narrow integers are summed without converting between lane widths, which AVX2 does far less well than + // NEON (a convertShape-per-part version ran 3-5x slower than C2 on x86). Each vector is reinterpreted as + // lanes twice as wide, so a lane holds two elements; the two are split off with shifts and masks and + // added into that wider accumulator, which is flushed to a long before it can overflow. The split is + // endianness-independent: both halves are always added. + + /// Vectors a 16-bit accumulator takes before a flush: a lane gains at most 510 (unsigned) or 256 in + /// magnitude (signed) per vector, so 127 stays under 2^16 and 2^15. + private static final int BYTE_SUM_FLUSH_ITERATIONS = 127; + + private static long sumBytes(byte[] a, boolean unsigned) { + long total = 0; + ShortVector acc = ShortVector.zero(SHORTS); + int iterations = 0; + int i = 0; + for (; i + BYTES.length() <= a.length; i += BYTES.length()) { + ShortVector pairs = ByteVector.fromArray(BYTES, a, i).reinterpretAsShorts(); + ShortVector low; + ShortVector high; + if (unsigned) { + low = pairs.and((short) 0xFF); + high = pairs.lanewise(VectorOperators.ASHR, 8).and((short) 0xFF); + } else { + low = pairs.lanewise(VectorOperators.LSHL, 8).lanewise(VectorOperators.ASHR, 8); + high = pairs.lanewise(VectorOperators.ASHR, 8); + } + acc = acc.add(low).add(high); + if (++iterations == BYTE_SUM_FLUSH_ITERATIONS) { + total += laneSum(acc, unsigned); + acc = ShortVector.zero(SHORTS); + iterations = 0; + } + } + total += laneSum(acc, unsigned); + for (; i < a.length; i++) { + total += unsigned ? Byte.toUnsignedLong(a[i]) : a[i]; + } + return total; + } + + private static long sumShorts(short[] a, boolean unsigned) { + long total = 0; + IntVector acc = IntVector.zero(INTS); + int iterations = 0; + int i = 0; + for (; i + SHORTS.length() <= a.length; i += SHORTS.length()) { + IntVector pairs = ShortVector.fromArray(SHORTS, a, i).reinterpretAsInts(); + IntVector low; + IntVector high; + if (unsigned) { + low = pairs.and(0xFFFF); + high = pairs.lanewise(VectorOperators.LSHR, 16); + } else { + low = pairs.lanewise(VectorOperators.LSHL, 16).lanewise(VectorOperators.ASHR, 16); + high = pairs.lanewise(VectorOperators.ASHR, 16); + } + acc = acc.add(low).add(high); + if (++iterations == SUM_FLUSH_ITERATIONS) { + total += laneSum(acc); + acc = IntVector.zero(INTS); + iterations = 0; + } + } + total += laneSum(acc); + for (; i < a.length; i++) { + total += unsigned ? Short.toUnsignedLong(a[i]) : a[i]; + } + return total; + } + + /// Adds the lanes of a 16-bit accumulator as `long`s, widening each half through `int` lanes. + private static long laneSum(ShortVector acc, boolean unsigned) { + VectorOperators.Conversion toInt = unsigned ? VectorOperators.ZERO_EXTEND_S2I : VectorOperators.S2I; + long total = 0; + for (int part = 0; part < SHORTS.length() / INTS.length(); part++) { + total += ((IntVector) acc.convertShape(toInt, INTS, part)).reduceLanes(VectorOperators.ADD); + } + return total; + } + + private static long sum(int[] a, VectorOperators.Conversion widen) { + int parts = INTS.length() / LONGS.length(); + LongVector acc = LongVector.zero(LONGS); + int i = 0; + for (; i + INTS.length() <= a.length; i += INTS.length()) { + IntVector v = IntVector.fromArray(INTS, a, i); + for (int part = 0; part < parts; part++) { + acc = acc.add((LongVector) v.convertShape(widen, LONGS, part)); + } + } + long total = acc.reduceLanes(VectorOperators.ADD); + boolean unsigned = widen == VectorOperators.ZERO_EXTEND_I2L; + for (; i < a.length; i++) { + total += unsigned ? Integer.toUnsignedLong(a[i]) : a[i]; + } + return total; + } + + /// Adds the lanes of an `int` accumulator as `long`s: reducing in `int` would wrap even when every + /// lane is in range (`reduceLanesToLong` casts after the `int` reduction, so it would wrap too). + private static long laneSum(IntVector acc) { + long total = 0; + int parts = INTS.length() / LONGS.length(); + for (int part = 0; part < parts; part++) { + total += ((LongVector) acc.convertShape(VectorOperators.I2L, LONGS, part)).reduceLanes(VectorOperators.ADD); + } + return total; + } + + // Overflow is detected on each lane's running sum (the sign bit of (acc ^ sum) & (v ^ sum) is set + // when the addition overflowed), not on the sequential prefix Rust checks, so the two can disagree + // in both directions on arrays whose partial sums approach the limits. The lanes and the scalar + // tail are then combined with exact checked adds. + private static OptionalLong sumSigned(long[] a) { + LongVector acc = LongVector.zero(LONGS); + LongVector overflow = LongVector.zero(LONGS); + int i = 0; + for (; i + LONGS.length() <= a.length; i += LONGS.length()) { + LongVector v = LongVector.fromArray(LONGS, a, i); + LongVector next = acc.add(v); + overflow = overflow.or(acc.lanewise(VectorOperators.XOR, next).and(v.lanewise(VectorOperators.XOR, next))); + acc = next; + } + if (overflow.compare(VectorOperators.LT, 0L).anyTrue()) { + return OptionalLong.empty(); + } + long total = 0; + try { + for (long lane : acc.toArray()) { + total = Math.addExact(total, lane); + } + for (; i < a.length; i++) { + total = Math.addExact(total, a[i]); + } + } catch (ArithmeticException _) { + return OptionalLong.empty(); + } + return OptionalLong.of(total); + } + + private static OptionalLong sumUnsigned(long[] a) { + LongVector acc = LongVector.zero(LONGS); + LongVector flip = LongVector.broadcast(LONGS, Long.MIN_VALUE); + VectorMask carry = LongVector.zero(LONGS).compare(VectorOperators.LT, 0L); + int i = 0; + for (; i + LONGS.length() <= a.length; i += LONGS.length()) { + LongVector next = acc.add(LongVector.fromArray(LONGS, a, i)); + // An unsigned add wrapped when the sum is below an addend; XOR with the sign bit orders unsigned as signed + carry = carry.or(next.lanewise(VectorOperators.XOR, flip).compare(VectorOperators.LT, acc.lanewise(VectorOperators.XOR, flip))); + acc = next; + } + if (carry.anyTrue()) { + return OptionalLong.empty(); + } + long total = 0; + for (long lane : acc.toArray()) { + long next = total + lane; + if (Long.compareUnsigned(next, total) < 0) { + return OptionalLong.empty(); + } + total = next; + } + for (; i < a.length; i++) { + long next = total + a[i]; + if (Long.compareUnsigned(next, total) < 0) { + return OptionalLong.empty(); + } + total = next; + } + return OptionalLong.of(total); + } + + // Lane-wise accumulation, reduced once at the end. The order of additions therefore differs from + // Rust's sequential f64 sum (and the JDK says reduceLanes(ADD) on floats may use "an arbitrary order + // of operations, which may even vary over time"), so the last bits of the result can differ from + // Rust's and from the auto-vectorized implementation's. F16 has no vector lanes: a plain loop. + @Override + public double sumFloating(Object values, PType ptype) { + return switch (ptype) { + case F32 -> sum((float[]) values); + case F64 -> sum((double[]) values); + case F16 -> { + double s = 0; + for (short v : (short[]) values) { + s += Float.float16ToFloat(v); + } + yield s; + } + default -> throw new IllegalArgumentException("not a floating-point ptype: " + ptype); + }; + } + + private static double sum(float[] a) { + int parts = FLOATS.length() / DOUBLES.length(); + DoubleVector acc = DoubleVector.zero(DOUBLES); + int i = 0; + for (; i + FLOATS.length() <= a.length; i += FLOATS.length()) { + FloatVector v = FloatVector.fromArray(FLOATS, a, i); + for (int part = 0; part < parts; part++) { + acc = acc.add((DoubleVector) v.convertShape(VectorOperators.F2D, DOUBLES, part)); + } + } + double total = acc.reduceLanes(VectorOperators.ADD); + for (; i < a.length; i++) { + total += a[i]; + } + return total; + } + + private static double sum(double[] a) { + DoubleVector acc = DoubleVector.zero(DOUBLES); + int i = 0; + for (; i + DOUBLES.length() <= a.length; i += DOUBLES.length()) { + acc = acc.add(DoubleVector.fromArray(DOUBLES, a, i)); + } + double total = acc.reduceLanes(VectorOperators.ADD); + for (; i < a.length; i++) { + total += a[i]; + } + return total; + } + + // ---- FastLanes ---- + + // Rows outer, lanes inner: each row is a vector add of the previous row. The lane count is a power + // of two of at least 16, so it is always a whole number of vectors. + @Override + public void undeltaChunk(long[] deltas, long[] bases, int lanes, int typeBits, long mask, long[] out) { + LongVector maskVector = LongVector.broadcast(LONGS, mask); + int rowBase = FastLanes.iterateIndex(0, 0); + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector.fromArray(LONGS, deltas, rowBase + lane).add(LongVector.fromArray(LONGS, bases, lane)) + .and(maskVector).intoArray(out, rowBase + lane); + } + for (int row = 1; row < typeBits; row++) { + int base = FastLanes.iterateIndex(row, 0); + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector.fromArray(LONGS, deltas, base + lane).add(LongVector.fromArray(LONGS, out, rowBase + lane)) + .and(maskVector).intoArray(out, base + lane); + } + rowBase = base; + } + } + + @Override + public void deltaChunk(long[] values, long[] bases, int lanes, int typeBits, long mask, long[] out) { + LongVector maskVector = LongVector.broadcast(LONGS, mask); + int rowBase = FastLanes.iterateIndex(0, 0); + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector.fromArray(LONGS, values, rowBase + lane).sub(LongVector.fromArray(LONGS, bases, lane)) + .and(maskVector).intoArray(out, rowBase + lane); + } + for (int row = 1; row < typeBits; row++) { + int base = FastLanes.iterateIndex(row, 0); + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector.fromArray(LONGS, values, base + lane).sub(LongVector.fromArray(LONGS, values, rowBase + lane)) + .and(maskVector).intoArray(out, base + lane); + } + rowBase = base; + } + } + + // Same structure as the auto-vectorized kernel: whether a row completes a word depends on the row + // alone, so the flush branch is hoisted out of the lane loop, and each lane loop is vector ops over + // contiguous lanes with the per-lane accumulators in a scratch array. + @Override + public void packBlock(long[] values, int offset, int bitWidth, int typeBits, long[] words) { + int lanes = FastLanes.CHUNK / typeBits; + long typeMask = FastLanes.lowMask(typeBits); + long widthMask = bitWidth >= 64 ? -1L : (1L << bitWidth) - 1L; + LongVector widthMaskVector = LongVector.broadcast(LONGS, widthMask); + LongVector typeMaskVector = LongVector.broadcast(LONGS, typeMask); + long[] acc = new long[lanes]; + int shift = 0; + int word = 0; + for (int row = 0; row < typeBits; row++) { + int base = offset + FastLanes.iterateIndex(row, 0); + int filled = shift + bitWidth; + if (filled < typeBits) { + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector.fromArray(LONGS, acc, lane) + .or(LongVector.fromArray(LONGS, values, base + lane).and(widthMaskVector) + .lanewise(VectorOperators.LSHL, shift)) + .intoArray(acc, lane); + } + shift = filled; + } else { + int wordBase = word * lanes; + int carry = filled - typeBits; + int carryShift = bitWidth - carry; + for (int lane = 0; lane < lanes; lane += LONGS.length()) { + LongVector value = LongVector.fromArray(LONGS, values, base + lane).and(widthMaskVector); + LongVector.fromArray(LONGS, acc, lane).or(value.lanewise(VectorOperators.LSHL, shift)) + .and(typeMaskVector).intoArray(words, wordBase + lane); + if (carry > 0) { + value.lanewise(VectorOperators.LSHR, carryShift).intoArray(acc, lane); + } else { + LongVector.zero(LONGS).intoArray(acc, lane); + } + } + word++; + shift = carry; + } + } + } +} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupportTest.java b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupportTest.java new file mode 100644 index 000000000..3a2329b89 --- /dev/null +++ b/core/src/test/java/io/github/dfa1/vortex/core/simd/SimdOperationsSupportTest.java @@ -0,0 +1,44 @@ +package io.github.dfa1.vortex.core.simd; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +class SimdOperationsSupportTest { + + @Test + void preferred_withTheVectorModulePresent_isTheVectorApiImplementation() { + // Given the test JVM launched with --add-modules jdk.incubator.vector (the surefire argLine) + assumeTrue(ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent(), "run without the module"); + assumeTrue(VectorApiSimdOperations.isUsable(), "CPU vectors narrower than 128 bits"); + + // When + SimdOperations result = SimdOperationsSupport.preferred(); + + // Then the whole suite exercises the Vector API path, not only its own differential test + assertThat(result).isInstanceOf(VectorApiSimdOperations.class); + } + + @Test + void preferred_withoutTheVectorModule_isTheAutoVectorizedImplementation() { + // Given a JVM launched without --add-modules jdk.incubator.vector (the without-vector-module pass) + assumeTrue(ModuleLayer.boot().findModule("jdk.incubator.vector").isEmpty(), "run with the module"); + + // When + SimdOperations result = SimdOperationsSupport.preferred(); + + // Then the fallback works: the Vector API class is never loaded, so nothing fails to link + assertThat(result).isInstanceOf(AutoVectorizedSimdOperations.class); + } + + @Test + void preferred_isTheSameInstanceEveryCall() { + // Given / When + SimdOperations first = SimdOperationsSupport.preferred(); + SimdOperations result = SimdOperationsSupport.preferred(); + + // Then + assertThat(result).isSameAs(first); + } +} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperationsTest.java b/core/src/test/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperationsTest.java new file mode 100644 index 000000000..6415d4a8d --- /dev/null +++ b/core/src/test/java/io/github/dfa1/vortex/core/simd/VectorApiSimdOperationsTest.java @@ -0,0 +1,649 @@ +package io.github.dfa1.vortex.core.simd; + +import io.github.dfa1.vortex.core.compute.FastLanes; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.model.PType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.OptionalLong; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/// Differential test: the Vector API implementation must agree with the auto-vectorized one, which +/// the unit tests of [SimdOperationsTest] pin down, for every input. Lengths straddle the lane +/// counts of every vector width so the main loop, the tail and the empty case are all hit. +class VectorApiSimdOperationsTest { + + private static final int[] LENGTHS = {0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 100, 255, 256, 257, 1000}; + + private final SimdOperations reference = new AutoVectorizedSimdOperations(); + private SimdOperations sut; + + @BeforeEach + void setUp() { + assumeTrue(ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent(), "run without the module"); + assumeTrue(VectorApiSimdOperations.isUsable(), "CPU vectors narrower than 128 bits"); + sut = new VectorApiSimdOperations(); + } + + @ParameterizedTest + @EnumSource(PType.class) + void runs_matchesReference(PType ptype) { + // Given arrays with long runs, short runs and no runs at every length, and NaN / signed zero floats + for (int length : LENGTHS) { + for (int runLength : new int[]{1, 2, 5, 64}) { + Object values = arrayWithRuns(ptype, length, runLength, new Random(length * 31L + runLength)); + + // When + long result = sut.runs(values, ptype); + + // Then + assertThat(result).as("%s length=%d runLength=%d", ptype, length, runLength) + .isEqualTo(reference.runs(values, ptype)); + } + } + } + + @ParameterizedTest + @EnumSource(PType.class) + void allEqual_matchesReference(PType ptype) { + // Given a constant array at every length, and the same array with one element changed at the + // first, a middle, the last and the first-tail position + for (int length : LENGTHS) { + Object constant = arrayWithRuns(ptype, length, Integer.MAX_VALUE, new Random(length)); + assertThat(sut.allEqual(constant, ptype)).as("%s constant length=%d", ptype, length) + .isEqualTo(reference.allEqual(constant, ptype)); + for (int position : new int[]{0, length / 2, length - 1, length - length % 16 - 1}) { + if (position < 0 || position >= length) { + continue; + } + Object changed = arrayWithRuns(ptype, length, Integer.MAX_VALUE, new Random(length)); + flip(changed, position); + + // When / Then + assertThat(sut.allEqual(changed, ptype)).as("%s length=%d differs at %d", ptype, length, position) + .isEqualTo(reference.allEqual(changed, ptype)); + } + } + } + + @ParameterizedTest + @EnumSource(PType.class) + void minMax_matchesReference(PType ptype) { + // Given random full-range data at every length: values near the sign boundary, where the + // unsigned flip would swap min and max if wrong + for (int length : LENGTHS) { + if (length == 0 && !ptype.isFloating()) { + continue; + } + Object values = arrayWithRuns(ptype, length, 1, new Random(length * 17L)); + + // When + long[] result = sut.minMax(values, ptype); + + // Then + assertThat(result).as("%s length=%d", ptype, length).containsExactly(reference.minMax(values, ptype)); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void minMax_extremes_matchReference(PType ptype) { + // Given arrays holding only the extremes of the carrier, in a vector-length run and a tail + for (int length : new int[]{1, 5, 17, 64, 70}) { + Object values = arrayWithRuns(ptype, length, 1, new Random(1)); + extremes(values); + + // When / Then + assertThat(sut.minMax(values, ptype)).as("%s length=%d", ptype, length) + .containsExactly(reference.minMax(values, ptype)); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"U8", "U16", "U32", "U64"}) + void maxUnsigned_matchesReference(PType ptype) { + // Given random codes, with the maximum planted at every position class: first, last, a lane boundary + try (Arena arena = Arena.ofConfined()) { + for (int length : LENGTHS) { + MemorySegment segment = arena.allocate(Math.max(1L, (long) length * ptype.byteSize())); + Random random = new Random(length); + for (long b = 0; b < segment.byteSize(); b++) { + segment.set(ValueLayout.JAVA_BYTE, b, (byte) random.nextInt(256)); + } + + // When / Then + assertThat(sut.maxUnsigned(segment, length, ptype)).as("%s length=%d", ptype, length) + .isEqualTo(reference.maxUnsigned(segment, length, ptype)); + } + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"U8", "U16", "U32", "U64"}) + void maxUnsigned_highBitCodes_areUnsigned(PType ptype) { + // Given a code with the top bit set, which a signed max would rank below a small code + try (Arena arena = Arena.ofConfined()) { + int length = 70; + MemorySegment segment = arena.allocate((long) length * ptype.byteSize()); + PTypeIO.set(segment, 69L * ptype.byteSize(), ptype, -1L); + PTypeIO.set(segment, 3L * ptype.byteSize(), ptype, 5L); + + // When + long result = sut.maxUnsigned(segment, length, ptype); + + // Then + assertThat(result).isEqualTo(reference.maxUnsigned(segment, length, ptype)); + assertThat(Long.compareUnsigned(result, 5L)).isPositive(); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32", "I64", "U64"}) + void sum_matchesReference(PType ptype) { + // Given random data at every length; the 64-bit types hold values below 2^40 so no partial sum + // overflows, which is the case where lane-wise and sequential overflow checks must agree + for (int length : LENGTHS) { + Object values = arrayWithRuns(ptype, length, 1, new Random(length * 13L)); + if (values instanceof long[] longs) { + for (int i = 0; i < longs.length; i++) { + longs[i] >>= 24; + longs[i] = ptype == PType.U64 ? longs[i] & 0xFF_FFFF_FFFFL : longs[i]; + } + } + + // When / Then + assertThat(sut.sum(values, ptype)).as("%s length=%d", ptype, length).isEqualTo(reference.sum(values, ptype)); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16", "I32", "U32"}) + void sum_pastTheFlushWindow_doesNotOverflowALane(PType ptype) { + // Given more elements than one int-lane flush window holds, every one at the carrier's extreme + // (255 / 65535 / -32768): a missing flush would wrap an int lane here + int length = 600_000; + Object values = arrayWithRuns(ptype, length, Integer.MAX_VALUE, new Random(1)); + extremes(values); + fillConstant(values); + + // When + OptionalLong result = sut.sum(values, ptype); + + // Then + assertThat(result).isEqualTo(reference.sum(values, ptype)); + } + + @ParameterizedTest + @EnumSource(PType.class) + void widenArrayInto_matchesReference(PType ptype) { + // Given full-range data widened from several start offsets, so the vector body starts unaligned + for (int length : LENGTHS) { + Object values = arrayWithRuns(ptype, length + 5, 1, new Random(length * 7L)); + for (int from : new int[]{0, 1, 5}) { + long[] expected = new long[length + 1]; + long[] result = new long[length + 1]; + + // When + reference.widenArrayInto(values, from, length, ptype, expected); + sut.widenArrayInto(values, from, length, ptype, result); + + // Then, including the untouched slot past count + assertThat(result).as("%s length=%d from=%d", ptype, length, from).containsExactly(expected); + } + } + } + + @ParameterizedTest + @EnumSource(PType.class) + void widenInto_matchesReference(PType ptype) { + // Given full-range data in a segment, widened from several start elements + try (Arena arena = Arena.ofConfined()) { + for (int length : LENGTHS) { + Object values = arrayWithRuns(ptype, length + 5, 1, new Random(length * 11L)); + MemorySegment segment = PrimitiveArrays.toSegment(values, ptype, arena); + for (long from : new long[]{0, 1, 5}) { + long[] expected = new long[length + 1]; + long[] result = new long[length + 1]; + + // When + reference.widenInto(segment, from, length, ptype, expected); + sut.widenInto(segment, from, length, ptype, result); + + // Then + assertThat(result).as("%s length=%d from=%d", ptype, length, from).containsExactly(expected); + } + } + } + } + + @ParameterizedTest + @EnumSource(PType.class) + void narrowArrayInto_matchesReference(PType ptype) { + // Given full-range wide values at every length + for (int length : LENGTHS) { + long[] values = new Random(length * 3L).longs(length).toArray(); + Object expected = narrowCarrier(ptype, length); + Object result = narrowCarrier(ptype, length); + + // When + reference.narrowArrayInto(values, ptype, expected); + sut.narrowArrayInto(values, ptype, result); + + // Then + assertThat(result).as("%s length=%d", ptype, length).isEqualTo(expected); + } + } + + @ParameterizedTest + @EnumSource(PType.class) + void narrowInto_matchesReference(PType ptype) { + // Given full-range wide values written into segments of exactly the narrowed size + try (Arena arena = Arena.ofConfined()) { + for (int length : LENGTHS) { + long[] values = new Random(length * 5L).longs(length).toArray(); + MemorySegment expected = arena.allocate(Math.max(1L, (long) length * ptype.byteSize())); + MemorySegment result = arena.allocate(Math.max(1L, (long) length * ptype.byteSize())); + + // When + reference.narrowInto(values, ptype, expected); + sut.narrowInto(values, ptype, result); + + // Then + assertThat(result.mismatch(expected)).as("%s length=%d", ptype, length).isEqualTo(-1L); + } + } + } + + @ParameterizedTest + @ValueSource(ints = {8, 16, 32, 64}) + void undeltaAndDeltaChunk_matchReference(int typeBits) { + // Given a random chunk and bases + int lanes = FastLanes.CHUNK / typeBits; + long mask = FastLanes.lowMask(typeBits); + Random random = new Random(typeBits); + long[] chunk = new long[FastLanes.CHUNK]; + for (int i = 0; i < chunk.length; i++) { + chunk[i] = random.nextLong() & mask; + } + long[] bases = new long[lanes]; + for (int i = 0; i < lanes; i++) { + bases[i] = random.nextLong() & mask; + } + long[] expectedUndelta = new long[FastLanes.CHUNK]; + long[] resultUndelta = new long[FastLanes.CHUNK]; + long[] expectedDelta = new long[FastLanes.CHUNK]; + long[] resultDelta = new long[FastLanes.CHUNK]; + + // When + reference.undeltaChunk(chunk, bases, lanes, typeBits, mask, expectedUndelta); + sut.undeltaChunk(chunk, bases, lanes, typeBits, mask, resultUndelta); + reference.deltaChunk(chunk, bases, lanes, typeBits, mask, expectedDelta); + sut.deltaChunk(chunk, bases, lanes, typeBits, mask, resultDelta); + + // Then + assertThat(resultUndelta).as("undelta %d", typeBits).containsExactly(expectedUndelta); + assertThat(resultDelta).as("delta %d", typeBits).containsExactly(expectedDelta); + } + + @ParameterizedTest + @ValueSource(ints = {8, 16, 32, 64}) + void packBlock_matchesReference_atEveryBitWidth(int typeBits) { + // Given a random block, including bits above the width (which packBlock must mask off) + Random random = new Random(typeBits * 101L); + long[] values = random.longs(FastLanes.CHUNK + 7).toArray(); + for (int bitWidth = 1; bitWidth <= typeBits; bitWidth++) { + long[] expected = new long[bitWidth * (FastLanes.CHUNK / typeBits)]; + long[] result = new long[expected.length]; + + // When + reference.packBlock(values, 7, bitWidth, typeBits, expected); + sut.packBlock(values, 7, bitWidth, typeBits, result); + + // Then + assertThat(result).as("typeBits=%d bitWidth=%d", typeBits, bitWidth).containsExactly(expected); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"F16", "F32", "F64"}) + void sumFloating_agreesWithTheReferenceToRoundingError(PType ptype) { + // Given values small enough that the sum is well inside the double range. The Vector API sums + // lane-wise, so the last bits legitimately differ from the sequential reference. + for (int length : LENGTHS) { + Object values = arrayWithRuns(ptype, length, 1, new Random(length * 19L)); + if (values instanceof float[] f) { + for (int i = 0; i < f.length; i++) { + f[i] = Float.isFinite(f[i]) ? (float) (f[i] % 1e6) : 1f; + } + } else if (values instanceof double[] d) { + for (int i = 0; i < d.length; i++) { + d[i] = Double.isFinite(d[i]) ? d[i] % 1e12 : 1d; + } + } else if (values instanceof short[] h) { + for (int i = 0; i < h.length; i++) { + h[i] = Float.isFinite(Float.float16ToFloat(h[i])) ? h[i] : Float.floatToFloat16(1f); + } + } + double expected = reference.sumFloating(values, ptype); + + // When + double result = sut.sumFloating(values, ptype); + + // Then within a relative 1e-9 of the sum of the magnitudes (the error scale of reordering) + double scale = Math.max(1.0, magnitude(values, ptype)); + assertThat(result).as("%s length=%d", ptype, length).isCloseTo(expected, + org.assertj.core.data.Offset.offset(scale * 1e-9)); + } + } + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"I8", "U8", "I16", "U16"}) + void sum_narrowIntegersAtTheirMostExtremeValues_neverOverflowAnAccumulator(PType ptype) { + // Given long arrays of the value that gains the most per vector in a 16-bit (bytes) or 32-bit + // (shorts) accumulator: the minimum for signed types, the maximum for unsigned ones + int length = 300_000; + Object values = switch (ptype) { + case I8 -> filled(new byte[length], Byte.MIN_VALUE); + case U8 -> filled(new byte[length], (byte) 0xFF); + case I16 -> filled(new short[length], Short.MIN_VALUE); + default -> filled(new short[length], (short) 0xFFFF); + }; + + // When + OptionalLong result = sut.sum(values, ptype); + + // Then + assertThat(result).isEqualTo(reference.sum(values, ptype)); + } + + @Test + void sumFloating_exactlyRepresentableValues_areEqualWhateverTheOrder() { + // Given small integers, whose sum is exact in any order + double[] doubles = new Random(3).doubles(1000, 0, 100).map(Math::floor).toArray(); + float[] floats = new float[1000]; + for (int i = 0; i < floats.length; i++) { + floats[i] = (float) doubles[i]; + } + + // When / Then + assertThat(sut.sumFloating(doubles, PType.F64)).isEqualTo(reference.sumFloating(doubles, PType.F64)); + assertThat(sut.sumFloating(floats, PType.F32)).isEqualTo(reference.sumFloating(floats, PType.F32)); + } + + @Test + void sum_signed64_clearOverflowIsReportedLikeTheReference() { + // Given enough Long.MAX_VALUEs that every lane and every order overflows + long[] values = new long[100]; + java.util.Arrays.fill(values, Long.MAX_VALUE); + + // When / Then + assertThat(sut.sum(values, PType.I64)).isEmpty(); + assertThat(reference.sum(values, PType.I64)).isEmpty(); + } + + @Test + void sum_unsigned64_clearOverflowIsReportedLikeTheReference() { + // Given enough -1s (2^64 - 1 each) that every lane and every order carries + long[] values = new long[100]; + java.util.Arrays.fill(values, -1L); + + // When / Then + assertThat(sut.sum(values, PType.U64)).isEmpty(); + assertThat(reference.sum(values, PType.U64)).isEmpty(); + } + + @Test + void sum_signed64_oppositeHugeValuesInOneLane_overflowPerLane_unlikeTheSequentialReference() { + // Given alternating huge values of opposite sign. Every vector length is even, so every positive + // value lands in the same lane: that lane's running sum overflows although the total is zero + long[] values = new long[64]; + for (int i = 0; i < values.length; i++) { + values[i] = i % 2 == 0 ? Long.MAX_VALUE / 2 : -(Long.MAX_VALUE / 2); + } + + // When / Then the documented difference from Rust: lane-wise overflow detection (docs/compatibility.md) + assertThat(reference.sum(values, PType.I64)).hasValue(0L); + assertThat(sut.sum(values, PType.I64)).isEmpty(); + } + + @Test + void minMax_floatsWithNaNAndZerosOfBothSigns_matchReference() { + // Given every arrangement of NaN, +0.0, -0.0 and ordinary values, long enough for the vector + // body, so the lane merge sees a zero of each sign in different lanes + float[] pool = {Float.NaN, 0.0f, -0.0f, 1f, -1f, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}; + Random random = new Random(42); + for (int trial = 0; trial < 200; trial++) { + int length = 1 + random.nextInt(70); + float[] floats = new float[length]; + double[] doubles = new double[length]; + for (int i = 0; i < length; i++) { + floats[i] = pool[random.nextInt(pool.length)]; + doubles[i] = floats[i]; + } + + // When / Then + assertThat(sut.minMax(floats, PType.F32)).as("F32 %s", java.util.Arrays.toString(floats)) + .containsExactly(reference.minMax(floats, PType.F32)); + assertThat(sut.minMax(doubles, PType.F64)).as("F64 %s", java.util.Arrays.toString(doubles)) + .containsExactly(reference.minMax(doubles, PType.F64)); + } + } + + @Test + void minMax_emptyArray_throws() { + // Given / When / Then + org.assertj.core.api.Assertions.assertThatThrownBy(() -> sut.minMax(new int[0], PType.I32)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void allEqual_floatsCompareRawBits() { + // Given arrays equal under == but not bit for bit, long enough for the vector loop + float[] zeros = new float[40]; + zeros[39] = -0.0f; + double[] nans = new double[40]; + java.util.Arrays.fill(nans, Double.NaN); + nans[20] = Double.longBitsToDouble(0x7ff8000000000001L); + + // When / Then + assertThat(sut.allEqual(zeros, PType.F32)).isFalse().isEqualTo(reference.allEqual(zeros, PType.F32)); + assertThat(sut.allEqual(nans, PType.F64)).isFalse().isEqualTo(reference.allEqual(nans, PType.F64)); + } + + @Test + void allEqual_booleans_matchReference() { + // Given + for (int length : LENGTHS) { + boolean[] constant = new boolean[length]; + java.util.Arrays.fill(constant, true); + assertThat(sut.allEqual(constant)).as("constant length=%d", length).isEqualTo(reference.allEqual(constant)); + for (int position : new int[]{0, length / 2, length - 1}) { + if (position < 0 || position >= length) { + continue; + } + boolean[] changed = constant.clone(); + changed[position] = false; + + // When / Then + assertThat(sut.allEqual(changed)).as("length=%d differs at %d", length, position) + .isEqualTo(reference.allEqual(changed)); + } + } + } + + @Test + void runs_floatsWithNaNAndSignedZero_matchReference() { + // Given NaN, which always differs from itself, and 0.0 / -0.0, which compare equal + float[] floats = {Float.NaN, Float.NaN, 0.0f, -0.0f, 0.0f, 1f, 1f, Float.NaN, 2f, 2f, 0f, -0f, 7f, 7f, 7f, 7f, 8f}; + double[] doubles = {Double.NaN, Double.NaN, 0.0, -0.0, 0.0, 1.0, 1.0, Double.NaN, 2.0, 2.0, 0.0, -0.0, 7.0, 7.0}; + + // When / Then + assertThat(sut.runs(floats, PType.F32)).isEqualTo(reference.runs(floats, PType.F32)); + assertThat(sut.runs(doubles, PType.F64)).isEqualTo(reference.runs(doubles, PType.F64)); + } + + /// An array of `length` elements in runs of `runLength` equal values, full-range per carrier. + private static Object arrayWithRuns(PType ptype, int length, int runLength, Random random) { + long[] base = new long[length]; + long current = random.nextLong(); + for (int i = 0; i < length; i++) { + if (i % runLength == 0) { + current = random.nextLong(); + } + base[i] = current; + } + return switch (ptype) { + case I8, U8 -> { + byte[] a = new byte[length]; + for (int i = 0; i < length; i++) { + a[i] = (byte) base[i]; + } + yield a; + } + case I16, U16, F16 -> { + short[] a = new short[length]; + for (int i = 0; i < length; i++) { + a[i] = (short) base[i]; + } + yield a; + } + case I32, U32 -> { + int[] a = new int[length]; + for (int i = 0; i < length; i++) { + a[i] = (int) base[i]; + } + yield a; + } + case I64, U64 -> base; + case F32 -> { + float[] a = new float[length]; + for (int i = 0; i < length; i++) { + a[i] = Float.intBitsToFloat((int) base[i]); + } + yield a; + } + case F64 -> { + double[] a = new double[length]; + for (int i = 0; i < length; i++) { + a[i] = Double.longBitsToDouble(base[i]); + } + yield a; + } + }; + } + + /// Changes one element so that it differs from its neighbors in the raw bits. + private static void flip(Object array, int index) { + switch (array) { + case byte[] a -> a[index] ^= 0x55; + case short[] a -> a[index] ^= 0x5555; + case int[] a -> a[index] ^= 0x55555555; + case long[] a -> a[index] ^= 0x5555555555555555L; + case float[] a -> a[index] = Float.intBitsToFloat(Float.floatToRawIntBits(a[index]) ^ 1); + case double[] a -> a[index] = Double.longBitsToDouble(Double.doubleToRawLongBits(a[index]) ^ 1L); + default -> throw new IllegalArgumentException(array.getClass().toString()); + } + } + + /// Overwrites every other element with the carrier's minimum and maximum bit patterns. + private static void extremes(Object array) { + switch (array) { + case byte[] a -> fill(a); + case short[] a -> fill(a); + case int[] a -> fill(a); + case long[] a -> fill(a); + default -> throw new IllegalArgumentException(array.getClass().toString()); + } + } + + private static void fill(byte[] a) { + for (int i = 0; i < a.length; i++) { + a[i] = i % 3 == 0 ? Byte.MIN_VALUE : i % 3 == 1 ? Byte.MAX_VALUE : (byte) -1; + } + } + + private static void fill(short[] a) { + for (int i = 0; i < a.length; i++) { + a[i] = i % 3 == 0 ? Short.MIN_VALUE : i % 3 == 1 ? Short.MAX_VALUE : (short) -1; + } + } + + private static void fill(int[] a) { + for (int i = 0; i < a.length; i++) { + a[i] = i % 3 == 0 ? Integer.MIN_VALUE : i % 3 == 1 ? Integer.MAX_VALUE : -1; + } + } + + private static void fill(long[] a) { + for (int i = 0; i < a.length; i++) { + a[i] = i % 3 == 0 ? Long.MIN_VALUE : i % 3 == 1 ? Long.MAX_VALUE : -1L; + } + } + + /// Sets every element to the carrier's all-ones pattern: 255 / 65535 as unsigned, -1 as signed. + private static void fillConstant(Object array) { + switch (array) { + case byte[] a -> java.util.Arrays.fill(a, (byte) -1); + case short[] a -> java.util.Arrays.fill(a, (short) -1); + case int[] a -> java.util.Arrays.fill(a, -1); + default -> throw new IllegalArgumentException(array.getClass().toString()); + } + } + + private static Object narrowCarrier(PType ptype, int length) { + return switch (ptype) { + case I8, U8 -> new byte[length]; + case I16, U16 -> new short[length]; + case I32, U32 -> new int[length]; + case I64, U64 -> new long[length]; + case F16 -> new short[length]; + case F32 -> new float[length]; + case F64 -> new double[length]; + }; + } + + /// The sum of the absolute values: the scale of the rounding error a reordered float sum can have. + private static double magnitude(Object values, PType ptype) { + double total = 0; + switch (values) { + case float[] f -> { + for (float v : f) { + total += Math.abs(v); + } + } + case double[] d -> { + for (double v : d) { + total += Math.abs(v); + } + } + case short[] h -> { + for (short v : h) { + total += Math.abs(Float.float16ToFloat(v)); + } + } + default -> throw new IllegalArgumentException(ptype.toString()); + } + return total; + } + + private static byte[] filled(byte[] array, byte value) { + java.util.Arrays.fill(array, value); + return array; + } + + private static short[] filled(short[] array, short value) { + java.util.Arrays.fill(array, value); + return array; + } +} diff --git a/docs/compatibility.md b/docs/compatibility.md index a79071980..14ab6660b 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -44,17 +44,34 @@ only the built-in decoders in `reader`; no encoder class is loaded. | Duplicate struct field names | Rust writer rejects ("StructLayout must have unique field names"); Rust reader tolerates foreign files (first-match access) | ⚠️ Deliberate divergence on read: Java rejects such files with `VortexException("duplicate field name in file schema")` instead of tolerating them — the name-keyed `Chunk` API cannot represent both columns, and silent column loss is worse than a loud failure on a file the reference writer refuses to produce. Java's writer mirrors the Rust writer's rejection. | | Blank / control-character field names | Wire-legal; the Rust writer produces `""` and whitespace-only names. NUL (`U+0000`) additionally aborts the Rust toolchain: Arrow FFI schema export hits a panic-cannot-unwind in `arrow-rs` (`ffi_stream::get_schema`) and SIGABRTs the process (measured against vortex-jni 0.75.0) | ⚠️ Deliberate strictness BOTH ways: vortex-java's writer refuses blank and control-character field names (`IllegalArgumentException`), and its reader rejects files carrying them (`VortexException` naming the producing pipeline as the likely bug) — the JSON-`""`-key principle: wire-legal is a floor, not a policy. Printable names of any shape (`$`-runs, spaces inside, emoji) are legal and round-trip intact both directions (measured; pinned by `ColumnNameEdgeCasesIntegrationTest`). | -## Primitive statistics (min, max, sum) +## SIMD kernels and the Vector API -The bulk loops behind the min, max and sum statistics live in `SimdOperations`. Compared with the Rust source: +Bulk loops (min/max, sums, run counting, widening, bit-packing, ...) live behind `SimdOperations`. +Two implementations exist: the default, plain Java loops shaped for C2's auto-vectorizer, and one written +with the incubating Vector API, selected only when the JVM is launched with +`--add-modules jdk.incubator.vector` (omitting the flag is the opt-out; the JVM prints +`WARNING: Using incubator modules` once). The default follows Rust. **The Vector API implementation is a +deliberate, documented departure in the cases below**, because forcing a scalar loop to keep Rust's exact +order would defeat the point of using the Vector API. -- Float min/max statistics order `-0.0` before `0.0` and skip `NaN`, as Rust's `total_compare` with +| Kernel | Rust | Vector API implementation | Effect | +|--------|------|---------------------------|--------| +| Floating-point sum (zone-map `Sum`) | Sequential `f64` accumulation | Lane-wise accumulation, one reduction at the end. The JDK documents `reduceLanes(ADD)` on floats as using "an arbitrary order of operations, which may even vary over time" | The sum can differ in its last bits from Rust's and from the default implementation, and between CPUs of different vector width. Integer sums are exact. | +| `I64` / `U64` sum | A checked add per element: any overflowing prefix drops the sum | Overflow detected on each lane's running sum, then lanes and tail combined with checked adds | Can report an overflow Rust does not, or the reverse, when partial sums approach the limits (`[MAX, 1, -1]` overflows Rust's prefix but not the total). Narrower integer sums are exact in both. | +| `F16` sum and min/max | Scalar | Scalar | The Vector API has no half-float lanes. No difference. | + +Statistics derived from these (the zone-map sum) therefore depend on whether the module is enabled. The +other kernels give identical results in both implementations, pinned by a differential test. + +Separate from the Vector API, found while comparing against the Rust source: + +- Float min/max statistics now order `-0.0` before `0.0` and skip `NaN`, as Rust's `total_compare` with `skip_nans` does. Before, the first zero met in array order was kept. - `U64` min/max and the dense-counter span in `ArrayStats` are computed in unsigned order, as Rust's `typed_int_stats` does. - **Open divergence:** Rust binds the zoned `Min`, `Max` **and `Sum`** aggregates with `skip_nans()`, so a - float zone's sum ignores `NaN`. Java's float sum adds `NaN` and so becomes `NaN` - ([#523](https://github.com/dfa1/vortex-java/issues/523)). + float zone's sum ignores `NaN`. Java's float sum adds `NaN` and so becomes `NaN`. This predates the + Vector API work and is not changed by it ([#523](https://github.com/dfa1/vortex-java/issues/523)). ## Real-world conformance: the Raincloud corpus diff --git a/integration/pom.xml b/integration/pom.xml index d98248713..0e351c043 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -247,6 +247,7 @@ --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow + --add-modules jdk.incubator.vector diff --git a/pom.xml b/pom.xml index 96721ef95..0f866c0ae 100644 --- a/pom.xml +++ b/pom.xml @@ -387,7 +387,7 @@ {@code net.bytebuddy:byte-buddy-agent:jar} is resolved to an absolute path by {@code maven-dependency-plugin:properties} (declared in build/plugins); loading it as a javaagent suppresses the Mockito self-attach warning. --> - @{argLine} -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED + @{argLine} -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector From c1f91371352eec9247c7b148ab3e789b6cbb0b8f Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 10 Oct 2026 22:33:19 +0200 Subject: [PATCH 4/5] ci: benchmark the SIMD kernels on x86, ARM and Apple silicon (#516) SimdVectorApiBenchmark runs each kernel with both implementations in one JVM (it lives in the core.simd package to reach the package-private classes), and scripts/simd-benchmark.sh runs the valid (kernel, ptype) groups and prints a table of how many times faster the Vector API is, with JMH's confidence interval and a verdict: a row is a win or a loss only when the whole range clears 1, otherwise it says "within noise". The SIMD benchmark workflow runs it on demand and when the SIMD code changes, on x86-64 (as allocated and capped at AVX2 with -XX:MaxVectorSize=32), 64-bit ARM Linux and Apple silicon, one fork by default, and writes the table to the job summary. Runners with the same label are not the same CPU (an ubuntu-latest run came back AVX2 once and AVX-512 the next), which is why the x86 width is pinned. It gates nothing: single-fork numbers on shared runners move by tens of percent. It is the second-architecture evidence ADR 0005 asks for. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01JbquncyA7mTMJbDJo1s5HD --- .github/workflows/simd-benchmark.yml | 88 ++++++ CLAUDE.md | 1 + .../core/simd/SimdVectorApiBenchmark.java | 267 ++++++++++++++++++ scripts/simd-benchmark.sh | 75 +++++ 4 files changed, 431 insertions(+) create mode 100644 .github/workflows/simd-benchmark.yml create mode 100644 performance/src/main/java/io/github/dfa1/vortex/core/simd/SimdVectorApiBenchmark.java create mode 100755 scripts/simd-benchmark.sh diff --git a/.github/workflows/simd-benchmark.yml b/.github/workflows/simd-benchmark.yml new file mode 100644 index 000000000..a6d230a2f --- /dev/null +++ b/.github/workflows/simd-benchmark.yml @@ -0,0 +1,88 @@ +name: SIMD benchmark + +# Informational, never a gate: numbers from shared runners are noisy, so the table says which rows are within +# JMH's confidence interval of a tie instead of pretending to precision. One fork keeps the job short; run it +# by hand with more forks for numbers worth recording. It exists to put the +# Vector API kernels next to the auto-vectorized ones on more than one architecture (ADR 0005 asks for +# two), as a table in the job summary. Runs on demand and when the SIMD code or this workflow changes. +on: + workflow_dispatch: + inputs: + forks: + description: 'JMH forks per benchmark (3 or more for numbers worth recording)' + required: false + default: '1' + pull_request: + branches: [ main ] + paths: + - 'core/src/main/java/io/github/dfa1/vortex/core/simd/**' + - 'performance/src/main/java/io/github/dfa1/vortex/core/simd/**' + - 'scripts/simd-benchmark.sh' + - '.github/workflows/simd-benchmark.yml' + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + # x86-64 Linux, 64-bit ARM Linux and Apple silicon. The ubuntu-latest label does not pin the CPU: runners + # have come back with 256-bit (AVX2) and 512-bit (AVX-512) vectors, so the x86 job runs twice, once as + # allocated and once capped at 32 bytes (AVX2), which makes runs comparable and shows how the kernels + # scale with the vector width. + include: + - { name: x86-64-default, os: ubuntu-latest } + - { name: x86-64-avx2, os: ubuntu-latest, max-vector-bytes: '32' } + - { name: arm64-linux, os: ubuntu-24.04-arm } + - { name: apple-silicon, os: macos-latest } + + steps: + - uses: actions/checkout@v7 + + - name: Set up Azul Zulu JDK 25 + uses: actions/setup-java@v6.0.1 + with: + distribution: zulu + java-version: '25' + + - name: Cache Maven repository + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-${{ runner.arch }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-maven- + + - name: Build the benchmarks jar + run: ./mvnw --batch-mode -q package -DskipTests -DskipShade=false -pl performance -am + + - name: Report the vector unit + run: | + { + echo "### ${{ matrix.name }} (${{ matrix.os }}, ${{ runner.arch }})" + echo + echo -n "Preferred int vector: " + echo 'System.out.println(jdk.incubator.vector.IntVector.SPECIES_PREFERRED)' \ + | jshell --add-modules jdk.incubator.vector ${{ matrix.max-vector-bytes && format('-R-XX:MaxVectorSize={0}', matrix.max-vector-bytes) || '' }} -s - 2>/dev/null | tail -1 + } | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Run the benchmarks + env: + FORKS: ${{ inputs.forks || '1' }} + MAX_VECTOR_BYTES: ${{ matrix.max-vector-bytes }} + run: | + scripts/simd-benchmark.sh performance/target/simd-benchmark > table.md + { echo; cat table.md; } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: simd-benchmark-${{ matrix.name }} + path: | + performance/target/simd-benchmark/*.json + table.md diff --git a/CLAUDE.md b/CLAUDE.md index 281d5288f..a65d2c32b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,7 @@ and merge commits are disabled): squash a PR's commits locally into one, force-p ./mvnw verify -pl integration -am # integration (failsafe, NOT surefire) ./mvnw verify -pl integration -am -Dit.test="RustWritesJavaReadsIntegrationTest#method" ./bench JavaVsJniReadBenchmark.javaReadVolume # benchmark — always ClassName.methodName filter +scripts/simd-benchmark.sh # auto-vectorized vs Vector API kernels, table of speedups (also a CI job) scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), then: ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test="RaincloudConformanceIntegrationTest" ./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # Jazzer regression mode (ADR 0020) diff --git a/performance/src/main/java/io/github/dfa1/vortex/core/simd/SimdVectorApiBenchmark.java b/performance/src/main/java/io/github/dfa1/vortex/core/simd/SimdVectorApiBenchmark.java new file mode 100644 index 000000000..4d4778cf4 --- /dev/null +++ b/performance/src/main/java/io/github/dfa1/vortex/core/simd/SimdVectorApiBenchmark.java @@ -0,0 +1,267 @@ +package io.github.dfa1.vortex.core.simd; + +import io.github.dfa1.vortex.core.model.PType; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/// Per-kernel A/B of the auto-vectorized [SimdOperations] against the Vector API one, in one JVM so +/// the two share a CPU state. Lives in the `core.simd` package to reach the package-private +/// implementations; the `impl` parameter picks which one a trial runs. One op processes `size` +/// elements. Run one kernel at a time, `-f 3` for numbers worth recording: +/// `./bench SimdVectorApiBenchmark.runs`. +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(value = 1, jvmArgsAppend = {"--add-modules", "jdk.incubator.vector", "--enable-native-access=ALL-UNNAMED"}) +public class SimdVectorApiBenchmark { + + @Param({"auto", "vector"}) + public String impl; + + private static final int CHUNKS = 256; + + @Param({"4096", "262144"}) + public int size; + + @Param({"I8", "I16", "I32", "I64", "F32", "F64"}) + public PType ptype; + + private SimdOperations ops; + private Object runsFew; + private Object runsMany; + private Object constant; + private Object sumData; + private boolean[] constantFlags; + private MemorySegment codes; + private long[] widened; + private long[] wide; + private Object narrowed; + private MemorySegment narrowSegment; + private long[] chunk; + private long[] bases; + private long[] chunkOut; + private long[] words; + private long chunkMask; + private int lanes; + private int typeBits; + + @Setup + public void setup() { + SimdOperations auto = new AutoVectorizedSimdOperations(); + ops = switch (impl) { + case "auto" -> auto; + case "vector" -> new VectorApiSimdOperations(); + default -> throw new IllegalArgumentException(impl); + }; + runsMany = array(ptype, size, 1, new Random(1)); + runsFew = array(ptype, size, 64, new Random(2)); + constant = array(ptype, size, Integer.MAX_VALUE, new Random(3)); + // 64-bit sums need values small enough that no partial sum overflows, or they return at once + sumData = ptype == PType.I64 ? new Random(7).longs(size, 0, 1 << 20).toArray() : runsMany; + constantFlags = new boolean[size]; + widened = new long[size]; + wide = new Random(5).longs(size).toArray(); + narrowed = switch (ptype) { + case I8 -> new byte[size]; + case I16 -> new short[size]; + case I32 -> new int[size]; + case I64 -> new long[size]; + case F32 -> new float[size]; + case F64 -> new double[size]; + default -> null; + }; + narrowSegment = Arena.ofAuto().allocate((long) size * 8); + typeBits = ptype.byteSize() * 8; + lanes = io.github.dfa1.vortex.core.compute.FastLanes.CHUNK / typeBits; + chunkMask = io.github.dfa1.vortex.core.compute.FastLanes.lowMask(typeBits); + Random fastLanes = new Random(6); + chunk = new long[io.github.dfa1.vortex.core.compute.FastLanes.CHUNK * CHUNKS]; + for (int i = 0; i < chunk.length; i++) { + chunk[i] = fastLanes.nextLong() & chunkMask; + } + bases = new long[lanes]; + chunkOut = new long[io.github.dfa1.vortex.core.compute.FastLanes.CHUNK]; + words = new long[typeBits * lanes]; + codes = Arena.ofAuto().allocate((long) size * 8); + Random fill = new Random(4); + for (long b = 0; b < codes.byteSize(); b++) { + codes.set(ValueLayout.JAVA_BYTE, b, (byte) fill.nextInt(256)); + } + } + + /// Counts runs over data with no runs at all: every neighbor pair differs. + @Benchmark + public long runs_noRuns() { + return ops.runs(runsMany, ptype); + } + + /// Counts runs over data in runs of 64 equal values. + @Benchmark + public long runs_longRuns() { + return ops.runs(runsFew, ptype); + } + + /// The dictionary code bound check: the largest unsigned code (the I8/I16/I32 params run it as U8/U16/U32). + @Benchmark + public long maxUnsigned() { + PType unsigned = switch (ptype) { + case I8 -> PType.U8; + case I16 -> PType.U16; + case I32 -> PType.U32; + case I64 -> PType.U64; + default -> throw new IllegalArgumentException(ptype.toString()); + }; + return ops.maxUnsigned(codes, size, unsigned); + } + + /// Sums random full-range integers into a long (I8, I16, I32; I64 keeps its per-add overflow check). + @Benchmark + public java.util.OptionalLong sum() { + return ops.sum(sumData, ptype); + } + + /// Widens a heap array of random full-range integers to longs (I8, I16, I32). + @Benchmark + public long[] widenArray() { + ops.widenArrayInto(runsMany, 0, size, ptype, widened); + return widened; + } + + /// Widens a segment of random full-range integers to longs, as the decoders do (I8, I16, I32). + @Benchmark + public long[] widenSegment() { + ops.widenInto(codes, 0, size, ptype, widened); + return widened; + } + + /// Narrows random longs into a heap array of the target width (I8, I16, I32). + @Benchmark + public Object narrowArray() { + ops.narrowArrayInto(wide, ptype, narrowed); + return narrowed; + } + + /// Narrows random longs into a segment, as the writer does (I8, I16, I32). + @Benchmark + public MemorySegment narrowSegment() { + ops.narrowInto(wide, ptype, narrowSegment); + return narrowSegment; + } + + /// Undeltas 256 FastLanes chunks (I8, I16, I32, I64). + @Benchmark + public long[] undelta() { + for (int c = 0; c < CHUNKS; c++) { + ops.undeltaChunk(chunk, bases, lanes, typeBits, chunkMask, chunkOut); + } + return chunkOut; + } + + /// Deltas 256 FastLanes chunks (I8, I16, I32, I64). + @Benchmark + public long[] delta() { + for (int c = 0; c < CHUNKS; c++) { + ops.deltaChunk(chunk, bases, lanes, typeBits, chunkMask, chunkOut); + } + return chunkOut; + } + + /// Bit-packs 256 FastLanes blocks at half the type width (I8, I16, I32, I64). + @Benchmark + public long[] pack() { + for (int c = 0; c < CHUNKS; c++) { + ops.packBlock(chunk, c * 1024, typeBits / 2, typeBits, words); + } + return words; + } + + /// Sums floats strictly left to right (the F64 param). + @Benchmark + public double sumFloating() { + return ops.sumFloating(runsMany, ptype); + } + + /// Finds the smallest and largest element of random full-range data (integer ptypes only). + @Benchmark + public long[] minMax() { + return ops.minMax(runsMany, ptype); + } + + /// Checks an all-equal array, the worst case for an early exit: every element is compared. + @Benchmark + public boolean allEqual_constant() { + return ops.allEqual(constant, ptype); + } + + /// Checks an all-equal flag array. + @Benchmark + public boolean allEqual_booleans() { + return ops.allEqual(constantFlags); + } + + private static Object array(PType ptype, int n, int runLength, Random random) { + long[] base = new long[n]; + long current = 0; + for (int i = 0; i < n; i++) { + if (i % runLength == 0) { + current = random.nextLong(); + } + base[i] = current; + } + return switch (ptype) { + case I8, U8 -> { + byte[] a = new byte[n]; + for (int i = 0; i < n; i++) { + a[i] = (byte) base[i]; + } + yield a; + } + case I16, U16, F16 -> { + short[] a = new short[n]; + for (int i = 0; i < n; i++) { + a[i] = (short) base[i]; + } + yield a; + } + case I32, U32 -> { + int[] a = new int[n]; + for (int i = 0; i < n; i++) { + a[i] = (int) base[i]; + } + yield a; + } + case I64, U64 -> base; + case F32 -> { + float[] a = new float[n]; + for (int i = 0; i < n; i++) { + a[i] = Float.intBitsToFloat((int) base[i]); + } + yield a; + } + case F64 -> { + double[] a = new double[n]; + for (int i = 0; i < n; i++) { + a[i] = Double.longBitsToDouble(base[i]); + } + yield a; + } + }; + } +} diff --git a/scripts/simd-benchmark.sh b/scripts/simd-benchmark.sh new file mode 100755 index 000000000..d501634ce --- /dev/null +++ b/scripts/simd-benchmark.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# A/B of the auto-vectorized SIMD kernels against the Vector API ones (SimdVectorApiBenchmark), +# one group per set of valid (kernel, ptype) pairs, then a Markdown table of how many times faster +# the Vector API is than the auto-vectorized loop (above 1 = the Vector API wins). +# +# Each score carries JMH's 99.9% confidence interval. The speedup range divides the intervals the +# conservative way (auto low / Vector API high, auto high / Vector API low), and a row is a win or a +# loss only when that whole range is clear of 1: otherwise it is within noise and says so. +# +# scripts/simd-benchmark.sh [output-dir] +# +# Environment (the defaults give intervals tight enough to call most rows; a quick smoke run can use +# FORKS=1 WARMUP=1 ITERATIONS=1 SIZE=4096): +# JAR benchmarks jar (default performance/target/benchmarks.jar; build it with +# ./mvnw package -DskipTests -DskipShade=false -pl performance -am) +# SIZE elements per op (default 262144) +# FORKS, WARMUP, ITERATIONS JMH forks, warmup and measurement iterations (1 s each) +# MAX_VECTOR_BYTES cap the preferred vector size (HotSpot -XX:MaxVectorSize, x86 only), e.g. 32 for AVX2 +# on a machine that offers AVX-512: runners with the same label do not all have the same CPU, +# so pin the width when two runs have to be comparable +set -euo pipefail + +OUT=${1:-performance/target/simd-benchmark} +JAR=${JAR:-performance/target/benchmarks.jar} +SIZE=${SIZE:-262144} +FORKS=${FORKS:-3} +WARMUP=${WARMUP:-3} +ITERATIONS=${ITERATIONS:-5} + +mkdir -p "$OUT" + +# The command line replaces the benchmark class's own fork arguments, so the module is named here too +FORK_ARGS="--add-modules jdk.incubator.vector --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow" +if [ -n "${MAX_VECTOR_BYTES:-}" ]; then + FORK_ARGS="$FORK_ARGS -XX:MaxVectorSize=$MAX_VECTOR_BYTES" +fi + +run() { # name, benchmark regex, ptypes + java --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED \ + --sun-misc-unsafe-memory-access=allow -jar "$JAR" "$2" \ + -p "size=$SIZE" -p "ptype=$3" -f "$FORKS" -wi "$WARMUP" -w 1 -i "$ITERATIONS" -r 1 \ + -jvmArgsAppend "$FORK_ARGS" \ + -rf json -rff "$OUT/$1.json" > "$OUT/$1.log" 2>&1 +} + +B='SimdVectorApiBenchmark\.' +run general "${B}(runs_noRuns|allEqual_constant|minMax|widenArray|widenSegment|narrowSegment)\$" I8,I16,I32,I64,F64 +run narrow "${B}narrowArray\$" I8,I16,I32,I64,F64 +run maxu "${B}maxUnsigned\$" I8,I16,I32,I64 +run sum "${B}sum\$" I8,I16,I32,I64 +run sumf "${B}sumFloating\$" F32,F64 +run bools "${B}allEqual_booleans\$" I8 +run fastlanes "${B}(undelta|delta|pack)\$" I8,I16,I32,I64 + +echo "| kernel | type | auto-vectorized (us) | Vector API (us) | Vector API is | |" +echo "|---|---|---:|---:|---:|---|" +jq -s -r ' + def f: . * 100 | round / 100; + [.[][] | {b: (.benchmark | split(".") | last), p: .params.ptype, i: .params.impl, + s: .primaryMetric.score, e: .primaryMetric.scoreError, + lo: .primaryMetric.scoreConfidence[0], hi: .primaryMetric.scoreConfidence[1]}] + | group_by([.b, .p])[] + | select(length == 2) + | (map(select(.i == "auto"))[0]) as $a + | (map(select(.i == "vector"))[0]) as $v + | ($a.s / $v.s) as $x + | (if ($a.lo | type) == "number" and ($v.hi | type) == "number" then ($a.lo / $v.hi) else null end) as $worst + | (if ($a.hi | type) == "number" and ($v.lo | type) == "number" then ($a.hi / $v.lo) else null end) as $best + | (if $worst == null or $best == null then "n/a (one iteration)" + elif $worst > 1 then "wins" elif $best < 1 then "loses" else "within noise" end) as $verdict + | (if $worst == null then "" else " (\($worst | f)–\($best | f))" end) as $range + | (if ($a.e | type) == "number" then " ± \($a.e | f)" else "" end) as $ae + | (if ($v.e | type) == "number" then " ± \($v.e | f)" else "" end) as $ve + | "| \(.[0].b) | \(.[0].p) | \($a.s | f)\($ae) | \($v.s | f)\($ve) | \($x | f)x\($range) | \($verdict) |" +' "$OUT"/*.json From d1be05546ca4a9191a2af30bc8b937b78ba7c738 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 11 Oct 2026 17:08:15 +0200 Subject: [PATCH 5/5] build: run the tests without jdk.incubator.vector for now (#516) The Vector API implementation stays opt-in at run time; the test JVMs no longer add the module, so its kernel tests are skipped until the argLine gets the add-modules option back. Co-Authored-By: Claude Sonnet 5 --- core/pom.xml | 9 +++++---- integration/pom.xml | 1 - pom.xml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 2d649cb51..27712cdc2 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -60,10 +60,11 @@ true - + org.apache.maven.plugins maven-surefire-plugin diff --git a/integration/pom.xml b/integration/pom.xml index 0e351c043..d98248713 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -247,7 +247,6 @@ --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow - --add-modules jdk.incubator.vector diff --git a/pom.xml b/pom.xml index 0f866c0ae..96721ef95 100644 --- a/pom.xml +++ b/pom.xml @@ -387,7 +387,7 @@ {@code net.bytebuddy:byte-buddy-agent:jar} is resolved to an absolute path by {@code maven-dependency-plugin:properties} (declared in build/plugins); loading it as a javaagent suppresses the Mockito self-attach warning. --> - @{argLine} -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-modules jdk.incubator.vector + @{argLine} -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED