diff --git a/.clang-format b/.clang-format index 587f746da78..0dfb09d7dfe 100644 --- a/.clang-format +++ b/.clang-format @@ -7,7 +7,7 @@ SortIncludes: false TabWidth: 4 IndentWidth: 4 ColumnLimit: 110 -AllowShortFunctionsOnASingleLine: false +AllowShortFunctionsOnASingleLine: true --- AccessModifierOffset: -4 AlignAfterOpenBracket: Align @@ -25,7 +25,7 @@ DerivePointerAlignment: false IncludeBlocks: Regroup InsertBraces: true Language: Cpp -NamespaceIndentation: Inner +NamespaceIndentation: None PointerAlignment: Right SpaceBeforeCpp11BracedList: true SpaceBeforeCtorInitializerColon: true diff --git a/vortex-cxx/.clang-tidy b/.clang-tidy similarity index 95% rename from vortex-cxx/.clang-tidy rename to .clang-tidy index 80c94310cbc..8b08b258113 100644 --- a/vortex-cxx/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors Checks: | -*, @@ -43,7 +43,7 @@ Checks: | -bugprone-unused-local-non-trivial-variable, -bugprone-empty-catch, WarningsAsErrors: '*' -HeaderFilterRegex: '(cpp|examples)/.*\.(cpp|hpp)$' +HeaderFilterRegex: '(examples)/.*\.(cpp|hpp)$' FormatStyle: none CheckOptions: - key: readability-identifier-naming.ClassCase @@ -90,4 +90,3 @@ CheckOptions: value: lower_case - key: cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams value: true - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23af8e2c5ff..56b62e333a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -523,8 +523,8 @@ jobs: command: check ${{ matrix.checks }} cxx-test: - name: "C++ build" - timeout-minutes: 30 + name: "C++ API tests" + timeout-minutes: 5 runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) @@ -538,20 +538,42 @@ jobs: - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - - name: Build and run C++ unit tests + - name: Install Rust nightly toolchain + run: | + rustup toolchain install $NIGHTLY_TOOLCHAIN + rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview + - name: Build FFI library (asan) + run: | + RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ + -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ + -Zsanitizer=address,leak" \ + cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ + --target x86_64-unknown-linux-gnu -Zbuild-std \ + -p vortex-ffi + - name: Build C++ tests (asan) run: | mkdir -p vortex-cxx/build - cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON + cmake -S vortex-cxx -B vortex-cxx/build -DBUILD_TESTS=ON -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" cmake --build vortex-cxx/build --parallel $(nproc) - ctest --test-dir vortex-cxx/build -j $(nproc) -V - - name: Build and run the example in release mode + - name: Run C++ tests + run: | + vortex-cxx/build/tests/vortex_cxx_test + - name: Build and run examples run: | - cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release + cmake -S vortex-cxx -B vortex-cxx/examples/build -DSANITIZER=asan -DBUILD_EXAMPLES=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" cmake --build vortex-cxx/examples/build --parallel $(nproc) - vortex-cxx/examples/build/hello-vortex vortex-cxx/examples/goldenfiles/example.vortex - - uses: ./.github/actions/check-rebuild - with: - command: "cargo build --profile ci --locked -p vortex-cxx --lib" + cd vortex-cxx/examples/build/examples + # filesystem loop detected + rm -fr ../_deps/nanoarrow-src/python/subprojects/arrow-nanoarrow + + ./writer people0.vortex + ./writer people1.vortex + ./writer me.vortex + + ./scan '*.vortex' + ./dtype me.vortex + + ./reader sqllogic-test: name: "SQL logic tests" diff --git a/Cargo.lock b/Cargo.lock index 7a3b82f3dce..278ce65401d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1436,17 +1436,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "codespan-reporting" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" -dependencies = [ - "serde", - "termcolor", - "unicode-width 0.2.2", -] - [[package]] name = "codspeed" version = "5.0.1" @@ -1998,68 +1987,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "cxx" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6f8908ed0826ab0aec7c8358a4de3a9b26c5ed391a5dc5135765d2fd1aa8fb" -dependencies = [ - "cc", - "cxx-build", - "cxxbridge-cmd", - "cxxbridge-flags", - "cxxbridge-macro", - "foldhash 0.2.0", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8ae25c0ce72ac21a77b5deec4f49b452238ef97072f697dd6fd752b1355ecd" -dependencies = [ - "cc", - "codespan-reporting", - "indexmap 2.14.0", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.118", -] - -[[package]] -name = "cxxbridge-cmd" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d53791812143bd27f74ab6055f22e875f04c59bbef0ca03d714ebec6f6f484" -dependencies = [ - "clap", - "codespan-reporting", - "indexmap 2.14.0", - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd557619f7dc2252bf7373f6bec5600ce60a2b477e5b3b84eb4e074bb297795e" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe465fc5b9d0231ea141c4fae714a08f4f907855e1a7ca10cc90ab6c8b12ece" -dependencies = [ - "indexmap 2.14.0", - "proc-macro2", - "quote", - "syn 2.0.118", -] - [[package]] name = "darling" version = "0.23.0" @@ -5903,15 +5830,6 @@ dependencies = [ "bitflags 2.13.0", ] -[[package]] -name = "link-cplusplus" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -8259,12 +8177,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scratch" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" - [[package]] name = "scroll" version = "0.11.0" @@ -8990,12 +8902,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - [[package]] name = "tap" version = "1.0.1" @@ -9041,15 +8947,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.4.4" @@ -10114,21 +10011,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "vortex-cxx" -version = "0.1.0" -dependencies = [ - "anyhow", - "arrow-array 58.3.0", - "arrow-schema 58.3.0", - "async-fs", - "cxx", - "futures", - "paste", - "take_mut", - "vortex", -] - [[package]] name = "vortex-datafusion" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e781ec222d1..543aa93b71b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ members = [ "vortex-cuda/gpu-scan-cli", "vortex-cuda/macros", "vortex-cuda/nvcomp", - "vortex-cxx", "vortex-ffi", "fuzz", "vortex-jni", diff --git a/docs/api/cpp/dtypes.rst b/docs/api/cpp/dtypes.rst deleted file mode 100644 index ae026af604d..00000000000 --- a/docs/api/cpp/dtypes.rst +++ /dev/null @@ -1,17 +0,0 @@ -Data Types -========== - -Logical data types for Vortex arrays. ``DType`` represents a logical type, and ``PType`` -enumerates the physical primitive types. - -PType ------ - -.. doxygenenum:: vortex::PType - -DType ------ - -.. doxygennamespace:: vortex::dtype - :members: - :undoc-members: diff --git a/docs/api/cpp/expr.rst b/docs/api/cpp/expr.rst deleted file mode 100644 index 4e904fff558..00000000000 --- a/docs/api/cpp/expr.rst +++ /dev/null @@ -1,9 +0,0 @@ -Expressions -=========== - -Expression trees used for filter and projection pushdowns in the scan API. Build expressions -from columns, literals, and comparison operators. - -.. doxygennamespace:: vortex::expr - :members: - :undoc-members: diff --git a/docs/api/cpp/file.rst b/docs/api/cpp/file.rst deleted file mode 100644 index 26736a88979..00000000000 --- a/docs/api/cpp/file.rst +++ /dev/null @@ -1,17 +0,0 @@ -File I/O -======== - -Read and write Vortex files from C++. ``VortexFile`` opens an existing file for scanning, while -``VortexWriteOptions`` writes an Arrow array stream to a new file. - -VortexFile ----------- - -.. doxygenclass:: vortex::VortexFile - :members: - -VortexWriteOptions ------------------- - -.. doxygenclass:: vortex::VortexWriteOptions - :members: diff --git a/docs/api/cpp/index.rst b/docs/api/cpp/index.rst index befb980f6a9..1e768b421aa 100644 --- a/docs/api/cpp/index.rst +++ b/docs/api/cpp/index.rst @@ -1,82 +1,260 @@ C++ API ======= -The Vortex C++ API provides an idiomatic C++ wrapper around the Vortex C FFI, built using -`cxx `_. It currently supports reading and writing Vortex files and integrates -with the Arrow C Data Interface via `nanoarrow `_. - -In the future we will expand the C++ API to cover Vortex's plugin and extension points. Please -reach out if you are interested in extending Vortex from C++ so we can prioritize these features. +Vortex C++ API allows you to read and write `.vortex` files directly or via +Arrow compatibility layer `nanoarrow `_. +The only dependency apart from nanoarrow are Vortex C bindings. .. note:: - Both the C++ API and this documentation are a work in progress. The API surface may change - significantly. Please reach out if you are interested in using Vortex from C++ so we can - prioritize stabilization. - + C++ API is a work in progress. Please reach out to us if you are interested + in using Vortex from C++ or you want a feature not covered yet e.g. + extension DataType support. Installation ------------ -The C++ bindings are built using CMake. Requirements: +We don't provide prebuilt library files (yet) so you will need to build Vortex +from source, and for that you will need: -* CMake 3.22 or higher -* C++20 compatible compiler -* Rust toolchain (for building the underlying Rust library) +- C++20, +- Rust toolchain, +- and CMake 3.10. .. code-block:: bash + git clone --depth 1 https://github.com/vortex-data/vortex + cd vortex + cargo build --release -p vortex-ffi cd vortex-cxx - mkdir build && cd build - cmake .. - make -j$(nproc) + mkdir build + cmake -Bbuild -DCMAKE_BUILD_TYPE=Release + cmake --build build -j + +This will produce a shared and a static library in `vortex/target/release/` +which you can use directly or via + +.. code-block:: bash + + # static library + target_link_libraries(target PRIVATE vortex_cxx nanoarrow) + # shared library + target_link_libraries(target PRIVATE vortex_cxx_shared nanoarrow_shared) +Have a look at `examples `_ +directory as well. -Compatibility +Reading files ------------- -The C++ bindings are supported on the following architectures: +Full source code for this example is `reader.cpp +`_. +For brevity we omit `main()`, system includes and some `using namespace` +directives. + +Assuming you have Vortex files `people0`, `people1`, and `me` in a local folder, +each containing U8 column "age" and U16 column "height", this is how you +print all ages for specific heights: + +.. code-block:: cpp + + using namespace vortex::expr::ops; + + const Session session; + const DataSource ds = DataSource::open(session, {"people*.vortex", "me.vortex"}); + Scan scan = ds.scan({.filter = col("height") >= lit(50)}); + + for (Partition &partition : scan.partitions()) { + for (Array &array : partition.batches()) { + const Array age = array.field("age"); + const PrimitiveView age_view = age.values(session); + const std::span age_values = age_view.values(); + for (uint8_t value : age_values) { + std::cout << int(value) << " "; + } + } + } + std::cout << "\n"; + +DataSource and Scan +^^^^^^^^^^^^^^^^^^^ + +First, you need to create a Vortex session which holds stuff like object store +credentials and does extension bookkeeping (we'll get back to it later). Then +you need to create a DataSource, which is a view over multiple files which may +also be remote. You can specify globs for every item as it's shown. + +Once you have a DataSource, you can create Scans out of it. A Scan is a single +traversal of a DataSource which projects columns and filters on them. Our Scan +is consumed by following calls so it needs to be non-`const`. ScanOptions which +are passed to Scan are a simple C++ aggregate so you can initialize any fields +you want or avoid them altogether (`auto scan = ds.scan()`). Options have +reasonable defaults: empty projection returns all fields, empty filter doesn't +filter rows, and so on. + +Expressions +^^^^^^^^^^^ + +We want to return both column so we omit `.projection`. If we'd want to return +only the "age" column we'd write `.projection = col("age")`. We pass an +Expression to a filter which returns false for some "height" values, and the +scan filters them out. + +Two additional things to look for in `.filter`: first, we allow overloading +Expression operators which produce Expressions themselves a-la Eigen. This is +opt-in via `using namespace vortex::expr::ops`. If you prefer, you can use +`eq(col("height"), lit(180))` instead. Second thing is that we +explicitly pass the type to the `lit` expression, which creates a literal +constant. We don't do any type coercion in Vortex so if you were to write +`lit(180)`, C++ would likely deduce the type to `uint8_t` and fail in runtime. + +Once we're done with the scan, we need to consume the data it provides. If +you want, you can get the data in Arrow format, but now we want our own +Partitions and Arrays. + +Partitions and Arrays +^^^^^^^^^^^^^^^^^^^^^ + +A Partition is an independent unit of work. Assuming your Vortex file +processing will likely be multithreaded, each thread can pull Partitions from +a Scan and handle them in parallel. For simplicity we don't cover it here, but +look for the next sections! Each Partition produces Arrays which are batches of +rows and columns. As we've requested "age", each Array will have one named +column "age". -* x86_64 Linux -* ARM64 Linux -* Apple Silicon macOS +.. note:: + Arrays, DataTypes, and Expressions are reference-counted so copying them is + cheap. + +Once we have an Array, we can get access to its values. First, we need to +extract "age" field. + +.. note:: + Default projection returns the root Array which is a Struct so to get + a field from it we need to use `field()`. If we were to use `col()` in + projection, the array we got would be "age" column already, so we wouldn't + need to use `field()`. + + +Then we want to get access to the raw bytes. However, +Array likely references compressed data, so now we need to learn about +canonicalization. -They support any Linux distribution with a GLIBC version >= 2.31. This includes +Canonicalization +^^^^^^^^^^^^^^^^ -* Amazon Linux 2022 or newer -* Ubuntu 20.04 or newer +Vortex files hold layers of compressed data. Each layer is a specific encoding +(zstd, FSST, delta) on top of another layer. This is good for performance +because we defer decompression and we can also pass compressed data directly to +other systems. Say, if a reader knows how to deal with bitpacked integers but +not RLE and we have `RLE(Bitpacked(U64))`, we can decompress just RLE and pass +bitpacked array directly to the reader. +In this example we want to remove all encodings and uncompress all data fully. +This form is called a canonical Array, and the process is canonicalization. -Usage Example +When we request `.values(session)`, we canonicalize the Array and get a +PrimitiveView because we already know the type of the column. As PrimitiveView +holds uncompressed data, we can get raw `uint64_t` numbers by calling +`.values()` on the view. + +Writing files ------------- -Here's a basic example of reading a Vortex file into an Arrow array stream: +Now let's write the first files to be read by our previous example. +Source code for this example is `writer.cpp +`_. .. code-block:: cpp - #include "vortex/file.hpp" - #include "vortex/scan.hpp" + using dtype::Nullable; - // Open a Vortex file and scan with a row limit - auto stream = vortex::VortexFile::Open("data.vortex") - .CreateScanBuilder() - .WithLimit(1000) - .IntoStream(); + const Session session; + const DataType dtype = dtype::struct_({ + {"age", dtype::uint8()}, + {"height", dtype::uint16(Nullable)}, + }); - // Consume the Arrow C Data stream - ArrowArray array; - while (stream.get_next(&stream, &array) == 0 && array.release != nullptr) { - // Process each batch... + constexpr size_t SAMPLE_ROWS = 100; + std::vector age_buffer(SAMPLE_ROWS); + std::vector height_buffer(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + age_buffer[i] = static_cast(i); + height_buffer[i] = static_cast((i + 1) % 200); } + Array age = Array::primitive(age_buffer); + Array array = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, AllValid)}, + }); -API Reference -------------- + Expression age_gt_10 = expr::col("age") > expr::lit(10); + Array validity_array = array.apply(age_gt_10); + + const Validity validity {ValidityType::Array, validity_array}; + Array array2 = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, validity)}, + }); + + Writer writer = Writer::open(session, argv[1], dtype); + writer.push(array); + writer.push(array2); + writer.finish(); + +DataType +^^^^^^^^ + +First, you want to create a schema. DataType is a logical representation of +Vortex's type system. "Logical" as opposed to "physical" means DataType doesn't +know what encodings the data use. As another example, Arrow and Parquet's types +are physical. There's a convenience function `struct_` which creates a +`DataTypeVariant::Struct` with fields passed as a `initializer_list`. + +Each DataType has a notion of nullability: whether some items in the row can be +invalid (Vortex uses "null" and "invalid" terms interchangeably). DataTypes are +non-nullable by default, so "age" column is not nullable, but "height" is. + +Then we prepare individual columns and use `make_struct` to assemble them into +Array, and here we see Validity. + +Validity +^^^^^^^^ + +DataType's Nullable flag tells us we may theoretically have invalid items in +the Array, but to know this for sure we need Validity. "age" is non-Nullable +so its Validity is NonNullable which means, again, we can't have invalid +elements. + +"height" in first Array is AllValid which means in this Array we don't have +invalid items. For the second Array we reuse age by copying it (which is a +reference clone so it's cheap). But for "height" we say Validity is determined +by another Array. This array consists of Bools and we obtained it by applying a +comparison Expression to the "age" column of the first array. So, "height" field +in `array2` will have null/invalid values if "age"'s item at same index was +less than 11. + +Applying Expressions +^^^^^^^^^^^^^^^^^^^^ + +One important thing to know is that applying an Expression is a constant-time +operation, and the returned array contains the original values along with some +meta-information about the expression. Real application happens when you execute +the array. One example of executing the array is canonicalization, which in this +example happens later when we write both arrays to the file. + +Writing to a file +^^^^^^^^^^^^^^^^^ + +Once we're done with the arrays, we need to initialize the Writer and push +arrays into it. `finish()` call writes the file footer. If you don't call it and +let Writer go out of scope, the file will be visibly corrupted. + +Now you can build the example and read back the generated files: -.. toctree:: - :maxdepth: 2 +.. code-block:: - dtypes - scalars - expr - file - scan + ./build/examples/writer people0.vortex + ./build/examples/writer people1.vortex + ./build/examples/writer me.vortex + ./build/examples/reader diff --git a/docs/api/cpp/scalars.rst b/docs/api/cpp/scalars.rst deleted file mode 100644 index ab52cc22af0..00000000000 --- a/docs/api/cpp/scalars.rst +++ /dev/null @@ -1,9 +0,0 @@ -Scalars -======= - -A scalar is a single typed value. Factory functions create scalars of each primitive type, and -``cast`` converts between types. - -.. doxygennamespace:: vortex::scalar - :members: - :undoc-members: diff --git a/docs/api/cpp/scan.rst b/docs/api/cpp/scan.rst deleted file mode 100644 index dbbd9e3bd36..00000000000 --- a/docs/api/cpp/scan.rst +++ /dev/null @@ -1,18 +0,0 @@ -Scanning -======== - -The scan API provides a builder pattern for reading data from a Vortex file with optional -filter, projection, row range, and limit pushdowns. The resulting stream exposes the -Arrow C Data Interface (``ArrowArrayStream``). - -ScanBuilder ------------ - -.. doxygenclass:: vortex::ScanBuilder - :members: - -StreamDriver ------------- - -.. doxygenclass:: vortex::StreamDriver - :members: diff --git a/vortex-cuda/kernels/src/for.cu b/vortex-cuda/kernels/src/for.cu index 702a0bfdf7b..eb4f123b46c 100644 --- a/vortex-cuda/kernels/src/for.cu +++ b/vortex-cuda/kernels/src/for.cu @@ -9,9 +9,7 @@ template struct ForOp { T reference; - __device__ inline T operator()(T value) const { - return value + reference; - } + __device__ inline T operator()(T value) const { return value + reference; } }; // Macro to generate in-place FoR kernel for each type. diff --git a/vortex-cxx/CMakeLists.txt b/vortex-cxx/CMakeLists.txt index ab5474d0617..9a477d34685 100644 --- a/vortex-cxx/CMakeLists.txt +++ b/vortex-cxx/CMakeLists.txt @@ -1,16 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.22) +cmake_minimum_required(VERSION 3.10) +project(VortexCXX + VERSION 0.0.1 + LANGUAGES CXX) -# FetchContent from URL timestamp handling -cmake_policy(SET CMP0135 NEW) - -project(vortex) +include(FetchContent) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) find_program(SCCACHE_PROGRAM sccache) if (SCCACHE_PROGRAM) @@ -21,100 +21,125 @@ else () message(STATUS "Sccache not found") endif () -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) -endif() +# GCC's -Wmissing-field-initializers is a false positive on ScanOptions +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wpedantic -Wno-missing-field-initializers") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-dollar-in-identifier-extension") +option(BUILD_TESTS "Build tests" OFF) +option(BUILD_EXAMPLES "Build examples" OFF) -option(VORTEX_ENABLE_TESTING "Enable building test binary for vortex-cxx" OFF) -option(VORTEX_ENABLE_ASAN "Enable address sanitizer" OFF) - -include(FetchContent) -FetchContent_Declare( - Corrosion - GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git - GIT_TAG v0.5 -) -FetchContent_MakeAvailable(Corrosion) +set(SANITIZER "" CACHE STRING "Build with sanitizers") +set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") +set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library") -set(RUST_SOURCE_FILE lib.rs) +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() -# Import Rust crate using Corrosion -corrosion_import_crate( - MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml - FEATURES ${CORROSION_FEATURES} - CRATES vortex-cxx -) +if (NOT SANITIZER STREQUAL "") + message(NOTICE "Sanitizer: ${SANITIZER}") + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(FATAL_ERROR "Only debug build is supported for sanitizer builds") + endif() -# Enable CXX bridge for the Rust crate -corrosion_add_cxxbridge(vortex_cxx_bridge CRATE vortex_cxx FILES ${RUST_SOURCE_FILE}) + if (SANITIZER STREQUAL "asan") + set(SANITIZER_FLAGS "-fsanitize=address,undefined,leak") + elseif (SANITIZER STREQUAL "tsan") + set(SANITIZER_FLAGS "-fsanitize=thread") + else() + message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}") + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZER_FLAGS}") +endif() -FetchContent_Declare( - nanoarrow - GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow.git - GIT_TAG a579fbf5d192e85b6249935e117de7d02a6dc4e9 # v0.8.0 -) -FetchContent_MakeAvailable(nanoarrow) +if (RUST_BUILD_PROFILE STREQUAL "") + if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + set(RUST_BUILD_PROFILE "release_debug") + else() + string(TOLOWER "${CMAKE_BUILD_TYPE}" RUST_BUILD_PROFILE) + endif() +endif() -file(GLOB_RECURSE CPP_SOURCE_FILE CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/*.cpp" -) +set(VORTEX_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi") +set(VORTEX_FFI_LIB_DIR "${VORTEX_FFI_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}") +set(VORTEX_FFI_HEADERS "${VORTEX_FFI_DIR}/cinclude") + +if(APPLE) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dylib") +elseif(WIN32) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.lib") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dll") +else() + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.so") +endif() -# Public headers -set(CPP_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/include -) +if(NOT EXISTS "${VORTEX_FFI_STATIC}") + message(FATAL_ERROR + "vortex-ffi static library not found at ${VORTEX_FFI_STATIC}. " + "Run: cargo build --profile -p vortex-ffi") +endif() -# Private headers -set(CPP_PRIVATE_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src -) +add_library(vortex_ffi STATIC IMPORTED) +set_target_properties(vortex_ffi PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_STATIC}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}") + +if(EXISTS "${VORTEX_FFI_SHARED}") + add_library(vortex_ffi_shared SHARED IMPORTED) + set_target_properties(vortex_ffi_shared PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_SHARED}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}" + INTERFACE_LINK_OPTIONS "LINKER:-rpath,${VORTEX_FFI_LIB_DIR}") +endif() -# Create the main library combining C++ and Rust code -add_library(vortex STATIC ${CPP_SOURCE_FILE}) -target_include_directories(vortex PUBLIC ${CPP_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR}/corrosion_generated/cxxbridge/vortex_cxx_bridge/include) -target_include_directories(vortex PRIVATE - ${CPP_PRIVATE_INCLUDE_DIRS} -) -target_link_libraries(vortex - PUBLIC nanoarrow_static - PRIVATE vortex_cxx_bridge +file(GLOB VORTEX_CXX_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") +add_library(vortex_cxx STATIC ${VORTEX_CXX_SOURCES}) +target_include_directories(vortex_cxx PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude ) +target_link_libraries(vortex_cxx PUBLIC vortex_ffi) +add_library(vortex::cxx ALIAS vortex_cxx) + +if(TARGET vortex_ffi_shared) + add_library(vortex_cxx_shared SHARED ${VORTEX_CXX_SOURCES}) + set_target_properties(vortex_cxx_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(vortex_cxx_shared PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude + ) + target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared) + add_library(vortex::cxx_shared ALIAS vortex_cxx_shared) +endif() -if (VORTEX_ENABLE_ASAN) - target_compile_options(vortex PRIVATE -fsanitize=leak,address,undefined -fno-omit-frame-pointer -fno-common -O1) - target_link_options(vortex PRIVATE -fsanitize=leak,address,undefined) +if (BUILD_TESTS OR BUILD_EXAMPLES) + FetchContent_Declare( + Nanoarrow + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0 + ) + FetchContent_MakeAvailable(Nanoarrow) endif() -# Tests -if (VORTEX_ENABLE_TESTING) +if (BUILD_TESTS) FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.17.0 - GIT_SHALLOW TRUE + Catch + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.1 ) - FetchContent_MakeAvailable(googletest) + FetchContent_MakeAvailable(Catch) + include(Catch) + target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) enable_testing() - add_executable(vortex_cxx_test cpp/tests/basic_test.cpp cpp/tests/test_data_generator.cpp) - target_include_directories(vortex_cxx_test PUBLIC ${CPP_INCLUDE_DIRS}) - target_include_directories(vortex_cxx_test PRIVATE cpp/tests) - target_link_libraries(vortex_cxx_test PRIVATE gtest_main vortex nanoarrow_static) - target_include_directories(vortex_cxx_test PRIVATE - ${CMAKE_CURRENT_BINARY_DIR}/corrosion_generated/cxxbridge/vortex_cxx_bridge/include - ) - # Platform-specific configuration - if(APPLE) - set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") - endif() - target_link_libraries(vortex_cxx_test PRIVATE vortex_cxx_bridge ${APPLE_LINK_FLAGS}) - if (VORTEX_ENABLE_ASAN) - target_compile_options(vortex_cxx_test PRIVATE -fsanitize=leak,address,undefined -fno-omit-frame-pointer -fno-common -O1) - target_link_options(vortex_cxx_test PRIVATE -fsanitize=leak,address,undefined) - endif() - include(GoogleTest) - gtest_discover_tests(vortex_cxx_test) + add_subdirectory(tests) +endif() + +if (BUILD_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/vortex-cxx/Cargo.toml b/vortex-cxx/Cargo.toml deleted file mode 100644 index d75f7b74cee..00000000000 --- a/vortex-cxx/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "vortex-cxx" -authors = { workspace = true } -categories = { workspace = true } -description = "C++ bindings for Vortex generated from Rust using cxx" -edition = { workspace = true } -homepage = { workspace = true } -include = { workspace = true } -keywords = { workspace = true } -license = { workspace = true } -publish = false -readme = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[lints] -workspace = true - -[lib] -crate-type = ["staticlib"] - -[dependencies] -anyhow = { workspace = true } -arrow-array = { workspace = true, features = ["ffi"] } -arrow-schema = { workspace = true } -async-fs = { workspace = true } -cxx = "1.0" -futures = { workspace = true } -paste = { workspace = true } -take_mut = { workspace = true } -vortex = { workspace = true } diff --git a/vortex-cxx/README.md b/vortex-cxx/README.md index aa761f98b1a..a0c4b5a847f 100644 --- a/vortex-cxx/README.md +++ b/vortex-cxx/README.md @@ -1,35 +1,37 @@ -# Vortex C++ Bindings +# Vortex C++ bindings -This directory contains C++ bindings for Vortex using the [cxx](https://cxx.rs/) crate. The bindings provide a C++ interface to Vortex file operations, including roundtripping with Arrow Array stream with advanced pushdown support. +For a usage guide, see docs/api/cpp/index.rst. -## Building +## Requirements -### Requirements +- CMake 3.10+ +- C++20 compiler +- Rust toolchain for building `vortex-ffi`. -- CMake 3.22 or higher -- C++20 compatible compiler -- Rust toolchain (for building the Rust components) -- (optional) Ninja (`ninja-build`) +## Build -### Build Steps - -```bash -mkdir build -cmake -Bbuild -GNinja +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DCMAKE_BUILD_TYPE=Release cmake --build build -j ``` -### Running Tests +This will generate `libvortex_cxx` shared and static libraries. +You can use `vortex_cxx` and `vortex_cxx_shared` CMake targets. + +## Test -```bash -# Enable tests in CMake -cmake -Bbuild -DVORTEX_ENABLE_TESTING=ON -GNinja +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DBUILD_TESTS=ON cmake --build build -j -./vortex_cxx_test +ctest --test-dir build -j "$(nproc)" ``` -## C++ Coding Convention - -We use `.clang-tidy` and `.clang-format` to setup the coding convention. Both are borrowed from DuckDB. +## Run examples -`cppcoreguidelines-avoid-non-const-global-variables` is removed from `.clang-tidy` because GTest violates it. +```sh +cmake -Bbuild -DBUILD_EXAMPLES=ON +cmake --build build -j +./build/examples/hello-vortex +``` diff --git a/vortex-cxx/cpp/include/vortex.hpp b/vortex-cxx/cpp/include/vortex.hpp deleted file mode 100644 index 1f2dd1625c1..00000000000 --- a/vortex-cxx/cpp/include/vortex.hpp +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include "vortex/exception.hpp" -#include "vortex/expr.hpp" -#include "vortex/scalar.hpp" \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/dtype.hpp b/vortex-cxx/cpp/include/vortex/dtype.hpp deleted file mode 100644 index 9bf4a16d9bc..00000000000 --- a/vortex-cxx/cpp/include/vortex/dtype.hpp +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { - -enum class PType : uint8_t { - U8 = 0, - U16, - U32, - U64, - I8, - I16, - I32, - I64, - F16, - F32, - F64, -}; - -namespace dtype { - class DType { - public: - DType() = delete; - explicit DType(rust::Box impl) : impl_(std::move(impl)) { - } - DType(DType &&other) noexcept = default; - DType &operator=(DType &&other) = default; - ~DType() = default; - - DType(const DType &) = delete; - DType &operator=(const DType &) = delete; - - std::string ToString() const; - - const rust::Box &GetImpl() { - return impl_; - } - - private: - rust::Box impl_; - }; - - // Factory functions - DType null(); - DType bool_(bool nullable = false); - DType primitive(PType ptype, bool nullable = false); - DType int8(bool nullable = false); - DType int16(bool nullable = false); - DType int32(bool nullable = false); - DType int64(bool nullable = false); - DType uint8(bool nullable = false); - DType uint16(bool nullable = false); - DType uint32(bool nullable = false); - DType uint64(bool nullable = false); - DType float16(bool nullable = false); - DType float32(bool nullable = false); - DType float64(bool nullable = false); - DType decimal(uint8_t precision = 10, int8_t scale = 0, bool nullable = false); - DType utf8(bool nullable = false); - DType binary(bool nullable = false); - /// TODO: Other DTypes are only supported by creating from Arrow for now. - DType from_arrow(struct ArrowSchema &schema, bool non_nullable = false); -} // namespace dtype - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/exception.hpp b/vortex-cxx/cpp/include/vortex/exception.hpp deleted file mode 100644 index a7fe44fb91c..00000000000 --- a/vortex-cxx/cpp/include/vortex/exception.hpp +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include - -namespace vortex { - -/// TODO(xinyu): better error handling -class VortexException : public std::runtime_error { -public: - explicit VortexException(const std::string &message) : std::runtime_error(message) { - } -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/expr.hpp b/vortex-cxx/cpp/include/vortex/expr.hpp deleted file mode 100644 index 3060a8e3536..00000000000 --- a/vortex-cxx/cpp/include/vortex/expr.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/scalar.hpp" -#include "vortex_cxx_bridge/lib.h" -#include - -namespace vortex::expr { -class Expr { -public: - Expr() = delete; - explicit Expr(rust::Box impl) : impl_(std::move(impl)) { - } - Expr(Expr &&other) noexcept = default; - Expr &operator=(Expr &&other) noexcept = default; - ~Expr() = default; - - Expr(const Expr &) = delete; - Expr &operator=(const Expr &) = delete; - - rust::Box IntoImpl() && { - return std::move(impl_); - } - - const ffi::Expr &Impl() const & { - return *impl_; - } - -private: - rust::Box impl_; -}; - -Expr literal(scalar::Scalar scalar); -Expr root(); -Expr column(std::string_view name); -Expr get_item(std::string_view field, Expr expr); -Expr not_(Expr expr); -Expr is_null(Expr expr); -Expr eq(Expr lhs, Expr rhs); -Expr not_eq_(Expr lhs, Expr rhs); -Expr gt(Expr lhs, Expr rhs); -Expr gt_eq(Expr lhs, Expr rhs); -Expr lt(Expr lhs, Expr rhs); -Expr lt_eq(Expr lhs, Expr rhs); -Expr and_(Expr lhs, Expr rhs); -Expr or_(Expr lhs, Expr rhs); -Expr checked_add(Expr lhs, Expr rhs); -Expr select(const std::vector &fields, Expr child); -} // namespace vortex::expr \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/file.hpp b/vortex-cxx/cpp/include/vortex/file.hpp deleted file mode 100644 index c7b93e326d8..00000000000 --- a/vortex-cxx/cpp/include/vortex/file.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { -class ScanBuilder; - -class VortexFile { -public: - static VortexFile Open(const std::string &path); - static VortexFile Open(const uint8_t *data, size_t length); - - VortexFile(VortexFile &&other) noexcept = default; - VortexFile &operator=(VortexFile &&other) noexcept = default; - ~VortexFile() = default; - - VortexFile(const VortexFile &) = delete; - VortexFile &operator=(const VortexFile &) = delete; - - /// Get the number of rows in the file. - uint64_t RowCount() const; - - /// Create a scan builder for the file. - /// The scan builder can be used to scan the file. - ScanBuilder CreateScanBuilder() const; - -private: - explicit VortexFile(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/scalar.hpp b/vortex-cxx/cpp/include/vortex/scalar.hpp deleted file mode 100644 index 77706aef98a..00000000000 --- a/vortex-cxx/cpp/include/vortex/scalar.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "dtype.hpp" -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::scalar { -class Scalar { -public: - Scalar() = delete; - explicit Scalar(rust::Box impl) : impl_(std::move(impl)) { - } - Scalar(Scalar &&other) noexcept = default; - Scalar &operator=(Scalar &&other) noexcept = default; - ~Scalar() = default; - - Scalar(const Scalar &) = delete; - Scalar &operator=(const Scalar &) = delete; - - rust::Box IntoImpl() && { - return std::move(impl_); - } - -private: - rust::Box impl_; -}; - -// Factory functions for creating scalar values -Scalar bool_(bool value); -Scalar int8(int8_t value); -Scalar int16(int16_t value); -Scalar int32(int32_t value); -Scalar int64(int64_t value); -Scalar uint8(uint8_t value); -Scalar uint16(uint16_t value); -Scalar uint32(uint32_t value); -Scalar uint64(uint64_t value); -Scalar float32(float value); -Scalar float64(double value); -Scalar string(std::string_view value); -Scalar binary(const uint8_t *data, size_t length); -/// TODO: Other Scalars are only supported by casting for now. -Scalar cast(Scalar scalar, dtype::DType dtype); -} // namespace vortex::scalar diff --git a/vortex-cxx/cpp/include/vortex/scan.hpp b/vortex-cxx/cpp/include/vortex/scan.hpp deleted file mode 100644 index 7debb9ef6aa..00000000000 --- a/vortex-cxx/cpp/include/vortex/scan.hpp +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/expr.hpp" -#include -#include "vortex_cxx_bridge/lib.h" - -#include -#include - -namespace vortex { -/// The StreamDriver internally holds a `RecordBatchIteratorAdapter` from the Rust side, which is thread-safe -/// and cloneable. The `RecordBatchIteratorAdapter` internally holds a `WorkStealingArrayIterator`. -class StreamDriver { -public: - StreamDriver(StreamDriver &&other) noexcept = default; - StreamDriver &operator=(StreamDriver &&other) noexcept = default; - ~StreamDriver() = default; - - StreamDriver(const StreamDriver &) = delete; - StreamDriver &operator=(const StreamDriver &) = delete; - - /// Create a stream of record batches. - /// - /// This function is thread-safe and can be called from multiple threads to create one stream per - /// thread to make progress on the same StreamDriver that is built from a ScanBuilder concurrently. - /// - /// Within each thread, the record batches will be emitted in the original order they are within - /// the scan. Between threads, the order is not guaranteed. - /// - /// Example: If the scan contains batches [b0, b1, b2, b3, b4, b5] and two threads call this - /// function respectively to make progress on their own stream, Thread 1 might receive [b0, - /// b2, b4] and Thread 2 might receive [b1, b3, b5]. Each thread maintains order within its - /// subset, but overall ordering between threads is not guaranteed (e.g., Thread 2 could emit b1 - /// before Thread 1 emits b0). - ArrowArrayStream CreateArrayStream() const; - -private: - friend class ScanBuilder; - - explicit StreamDriver(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; - -class ScanBuilder { -public: - ScanBuilder(ScanBuilder &&other) noexcept = default; - ScanBuilder &operator=(ScanBuilder &&other) noexcept { - if (this != &other) { - impl_ = std::move(other.impl_); - } - return *this; - } - ~ScanBuilder() = default; - - ScanBuilder(const ScanBuilder &) = delete; - ScanBuilder &operator=(const ScanBuilder &) = delete; - - /// Only include rows that match the filter expressions. - ScanBuilder &WithFilter(expr::Expr &&expr) &; - ScanBuilder &WithFilter(const expr::Expr &expr) &; - ScanBuilder &&WithFilter(expr::Expr &&expr) &&; - ScanBuilder &&WithFilter(const expr::Expr &expr) &&; - - /// Only include columns that match the projection expressions. - ScanBuilder &WithProjection(expr::Expr &&expr) &; - ScanBuilder &WithProjection(const expr::Expr &expr) &; - ScanBuilder &&WithProjection(expr::Expr &&expr) &&; - ScanBuilder &&WithProjection(const expr::Expr &expr) &&; - - /// Only include rows in the range [row_range_start, row_range_end). - ScanBuilder &WithRowRange(uint64_t row_range_start, uint64_t row_range_end) &; - ScanBuilder &&WithRowRange(uint64_t row_range_start, uint64_t row_range_end) &&; - - /// Only include rows with the given indices. - ScanBuilder &WithIncludeByIndex(const uint64_t *indices, std::size_t size) &; - ScanBuilder &&WithIncludeByIndex(const uint64_t *indices, std::size_t size) &&; - - /// Set the limit on the number of rows to scan out. - ScanBuilder &WithLimit(uint64_t limit) &; - ScanBuilder &&WithLimit(uint64_t limit) &&; - - /// Set the output schema on the scan builder. - /// TODO: currently if pass in this option, the schema needs to be the schema after adding projection. - ScanBuilder &WithOutputSchema(ArrowSchema &output_schema) &; - ScanBuilder &&WithOutputSchema(ArrowSchema &output_schema) &&; - - /// Take ownership and consume the scan builder to a stream of record batches. - ArrowArrayStream IntoStream() &&; - - /// Take ownership and consume the scan builder to a stream driver. - /// Under the hood, this function calls `ScanBuilder::into_record_batch_reader` and holds a - /// `WorkStealingArrayIterator` in StreamDriver. - StreamDriver IntoStreamDriver() &&; - -private: - friend class VortexFile; - - explicit ScanBuilder(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/write_options.hpp b/vortex-cxx/cpp/include/vortex/write_options.hpp deleted file mode 100644 index 08b903bf2fb..00000000000 --- a/vortex-cxx/cpp/include/vortex/write_options.hpp +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { - -class VortexWriteOptions { -public: - VortexWriteOptions() : impl_(ffi::write_options_new()) { - } - VortexWriteOptions(VortexWriteOptions &&other) noexcept = default; - VortexWriteOptions &operator=(VortexWriteOptions &&other) noexcept = default; - ~VortexWriteOptions() = default; - - VortexWriteOptions(const VortexWriteOptions &) = delete; - VortexWriteOptions &operator=(const VortexWriteOptions &) = delete; - - /// Write an ArrowArrayStream to a Vortex file - void WriteArrayStream(ArrowArrayStream &stream, const std::string &path); - -private: - rust::Box impl_; -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/dtype.cpp b/vortex-cxx/cpp/src/dtype.cpp deleted file mode 100644 index 5b85f36835e..00000000000 --- a/vortex-cxx/cpp/src/dtype.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/dtype.hpp" -#include "vortex/exception.hpp" - -#include "rust/cxx.h" - -namespace vortex::dtype { -DType null() { - return DType(ffi::dtype_null()); -} - -DType bool_(bool nullable) { - return DType(ffi::dtype_bool(nullable)); -} - -DType primitive(PType ptype, bool nullable) { - return DType(ffi::dtype_primitive(static_cast(ptype), nullable)); -} - -DType int8(bool nullable) { - return primitive(PType::I8, nullable); -} - -DType int16(bool nullable) { - return primitive(PType::I16, nullable); -} - -DType int32(bool nullable) { - return primitive(PType::I32, nullable); -} - -DType int64(bool nullable) { - return primitive(PType::I64, nullable); -} - -DType uint8(bool nullable) { - return primitive(PType::U8, nullable); -} - -DType uint16(bool nullable) { - return primitive(PType::U16, nullable); -} - -DType uint32(bool nullable) { - return primitive(PType::U32, nullable); -} - -DType uint64(bool nullable) { - return primitive(PType::U64, nullable); -} - -DType float16(bool nullable) { - return primitive(PType::F16, nullable); -} - -DType float32(bool nullable) { - return primitive(PType::F32, nullable); -} - -DType float64(bool nullable) { - return primitive(PType::F64, nullable); -} - -DType decimal(uint8_t precision, int8_t scale, bool nullable) { - return DType(ffi::dtype_decimal(precision, scale, nullable)); -} - -DType utf8(bool nullable) { - return DType(ffi::dtype_utf8(nullable)); -} - -DType binary(bool nullable) { - return DType(ffi::dtype_binary(nullable)); -} - -DType from_arrow(struct ArrowSchema &schema, bool non_nullable) { - try { - return DType(ffi::from_arrow(reinterpret_cast(&schema), non_nullable)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} -// Methods -std::string DType::ToString() const { - auto rust_str = impl_->to_string(); - return std::string(rust_str.data(), rust_str.length()); -} -} // namespace vortex::dtype diff --git a/vortex-cxx/cpp/src/expr.cpp b/vortex-cxx/cpp/src/expr.cpp deleted file mode 100644 index 0e2d67395c1..00000000000 --- a/vortex-cxx/cpp/src/expr.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/expr.hpp" -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::expr { - -Expr literal(scalar::Scalar scalar) { - return Expr(ffi::literal(std::move(scalar).IntoImpl())); -} - -Expr root() { - return Expr(ffi::root()); -} - -Expr column(std::string_view name) { - return Expr(ffi::column(rust::String(name.data(), name.length()))); -} - -Expr get_item(std::string_view field, Expr child) { - return Expr(ffi::get_item(rust::String(field.data(), field.length()), std::move(child).IntoImpl())); -} - -Expr not_(Expr child) { - return Expr(ffi::not_(std::move(child).IntoImpl())); -} - -Expr is_null(Expr child) { - return Expr(ffi::is_null(std::move(child).IntoImpl())); -} - -// Macro to define binary operator functions -#define DEFINE_BINARY_OP(name) \ - Expr name(Expr lhs, Expr rhs) { \ - return Expr(ffi::name(std::move(lhs).IntoImpl(), std::move(rhs).IntoImpl())); \ - } - -DEFINE_BINARY_OP(eq) -DEFINE_BINARY_OP(not_eq_) -DEFINE_BINARY_OP(gt) -DEFINE_BINARY_OP(gt_eq) -DEFINE_BINARY_OP(lt) -DEFINE_BINARY_OP(lt_eq) -DEFINE_BINARY_OP(and_) -DEFINE_BINARY_OP(or_) -DEFINE_BINARY_OP(checked_add) - -#undef DEFINE_BINARY_OP - -Expr select(const std::vector &fields, Expr child) { - ::rust::Vec<::rust::String> rs_fields; - rs_fields.reserve(fields.size()); - for (std::string_view f : fields) { - rs_fields.emplace_back(f.data(), f.length()); - } - return Expr(ffi::select(rs_fields, std::move(child).IntoImpl())); -} - -} // namespace vortex::expr \ No newline at end of file diff --git a/vortex-cxx/cpp/src/file.cpp b/vortex-cxx/cpp/src/file.cpp deleted file mode 100644 index 078c305639d..00000000000 --- a/vortex-cxx/cpp/src/file.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/exception.hpp" -#include "rust/cxx.h" - -namespace vortex { - -VortexFile VortexFile::Open(const uint8_t *data, size_t length) { - try { - rust::Slice slice(data, length); - return VortexFile(ffi::open_file_from_buffer(slice)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -VortexFile VortexFile::Open(const std::string &path) { - try { - return VortexFile(ffi::open_file(path)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -uint64_t VortexFile::RowCount() const { - return impl_->row_count(); -} - -ScanBuilder VortexFile::CreateScanBuilder() const { - return ScanBuilder(impl_->scan_builder()); -} - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/scalar.cpp b/vortex-cxx/cpp/src/scalar.cpp deleted file mode 100644 index 87502ee2f00..00000000000 --- a/vortex-cxx/cpp/src/scalar.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/scalar.hpp" - -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::scalar { - -Scalar bool_(bool value) { - return Scalar(ffi::bool_scalar_new(value)); -} - -Scalar int8(int8_t value) { - return Scalar(ffi::i8_scalar_new(value)); -} - -Scalar int16(int16_t value) { - return Scalar(ffi::i16_scalar_new(value)); -} - -Scalar int32(int32_t value) { - return Scalar(ffi::i32_scalar_new(value)); -} - -Scalar int64(int64_t value) { - return Scalar(ffi::i64_scalar_new(value)); -} - -Scalar uint8(uint8_t value) { - return Scalar(ffi::u8_scalar_new(value)); -} - -Scalar uint16(uint16_t value) { - return Scalar(ffi::u16_scalar_new(value)); -} - -Scalar uint32(uint32_t value) { - return Scalar(ffi::u32_scalar_new(value)); -} - -Scalar uint64(uint64_t value) { - return Scalar(ffi::u64_scalar_new(value)); -} - -Scalar float32(float value) { - return Scalar(ffi::f32_scalar_new(value)); -} - -Scalar float64(double value) { - return Scalar(ffi::f64_scalar_new(value)); -} - -Scalar string(std::string_view value) { - return Scalar(ffi::string_scalar_new(rust::Str(value.data(), value.length()))); -} - -Scalar binary(const uint8_t *data, size_t length) { - return Scalar(ffi::binary_scalar_new(rust::Slice(data, length))); -} - -Scalar cast(Scalar scalar, dtype::DType dtype) { - return Scalar(std::move(scalar).IntoImpl()->cast_scalar(*std::move(dtype).GetImpl())); -} - -} // namespace vortex::scalar \ No newline at end of file diff --git a/vortex-cxx/cpp/src/scan.cpp b/vortex-cxx/cpp/src/scan.cpp deleted file mode 100644 index 9b40ec00e0d..00000000000 --- a/vortex-cxx/cpp/src/scan.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/scan.hpp" -#include "vortex/exception.hpp" -#include "rust/cxx.h" -#include "vortex/expr.hpp" - -namespace vortex { -ScanBuilder &ScanBuilder::WithFilter(expr::Expr &&expr) & { - impl_->with_filter(std::move(expr).IntoImpl()); - return *this; -} -ScanBuilder &ScanBuilder::WithFilter(const expr::Expr &expr) & { - impl_->with_filter_ref(expr.Impl()); - return *this; -} -ScanBuilder &&ScanBuilder::WithFilter(expr::Expr &&expr) && { - impl_->with_filter(std::move(expr).IntoImpl()); - return std::move(*this); -} -ScanBuilder &&ScanBuilder::WithFilter(const expr::Expr &expr) && { - impl_->with_filter_ref(expr.Impl()); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithProjection(expr::Expr &&expr) & { - impl_->with_projection(std::move(expr).IntoImpl()); - return *this; -} -ScanBuilder &ScanBuilder::WithProjection(const expr::Expr &expr) & { - impl_->with_projection_ref(expr.Impl()); - return *this; -} -ScanBuilder &&ScanBuilder::WithProjection(expr::Expr &&expr) && { - impl_->with_projection(std::move(expr).IntoImpl()); - return std::move(*this); -} -ScanBuilder &&ScanBuilder::WithProjection(const expr::Expr &expr) && { - impl_->with_projection_ref(expr.Impl()); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithRowRange(uint64_t row_range_start, uint64_t row_range_end) & { - impl_->with_row_range(row_range_start, row_range_end); - return *this; -} -ScanBuilder &&ScanBuilder::WithRowRange(uint64_t row_range_start, uint64_t row_range_end) && { - impl_->with_row_range(row_range_start, row_range_end); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithLimit(uint64_t limit) & { - impl_->with_limit(limit); - return *this; -} - -ScanBuilder &&ScanBuilder::WithLimit(uint64_t limit) && { - impl_->with_limit(limit); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithIncludeByIndex(const uint64_t *indices, std::size_t size) & { - impl_->with_include_by_index(rust::Slice(indices, size)); - return *this; -} - -ScanBuilder &&ScanBuilder::WithIncludeByIndex(const uint64_t *indices, std::size_t size) && { - impl_->with_include_by_index(rust::Slice(indices, size)); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithOutputSchema(ArrowSchema &output_schema) & { - try { - impl_->with_output_schema(reinterpret_cast(&output_schema)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } - return *this; -} - -ScanBuilder &&ScanBuilder::WithOutputSchema(ArrowSchema &output_schema) && { - try { - impl_->with_output_schema(reinterpret_cast(&output_schema)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } - return std::move(*this); -} - -ArrowArrayStream ScanBuilder::IntoStream() && { - try { - ArrowArrayStream stream; - ffi::scan_builder_into_stream(std::move(impl_), reinterpret_cast(&stream)); - return stream; - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -StreamDriver ScanBuilder::IntoStreamDriver() && { - try { - rust::Box reader = - ffi::scan_builder_into_threadsafe_cloneable_reader(std::move(impl_)); - return StreamDriver(std::move(reader)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -ArrowArrayStream StreamDriver::CreateArrayStream() const { - ArrowArrayStream stream; - impl_->clone_a_stream(reinterpret_cast(&stream)); - return stream; -} -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/write_options.cpp b/vortex-cxx/cpp/src/write_options.cpp deleted file mode 100644 index 712a6933208..00000000000 --- a/vortex-cxx/cpp/src/write_options.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/write_options.hpp" -#include "vortex/exception.hpp" - -#include "rust/cxx.h" - -namespace vortex { -void VortexWriteOptions::WriteArrayStream(ArrowArrayStream &stream, const std::string &path) { - try { - ffi::write_array_stream(std::move(impl_), reinterpret_cast(&stream), path); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/tests/basic_test.cpp b/vortex-cxx/cpp/tests/basic_test.cpp deleted file mode 100644 index eb44715285a..00000000000 --- a/vortex-cxx/cpp/tests/basic_test.cpp +++ /dev/null @@ -1,510 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include "vortex/scalar.hpp" -#include "test_data_generator.hpp" -#include "vortex_cxx_bridge/lib.h" - -#include -#include - -class VortexTest : public ::testing::Test { -protected: - // Helper function to create unique temporary files for each test - static std::string GetUniqueTempFile(const std::string &suffix = "vortex") { - std::filesystem::path temp_dir = std::filesystem::temp_directory_path(); - std::filesystem::path vortex_test_dir = temp_dir / "vortex_test"; - - if (!std::filesystem::exists(vortex_test_dir)) { - std::filesystem::create_directories(vortex_test_dir); - } - - // Use a unique random filename to prevent races between parallel test runs - std::string unique_name = "test_" + std::to_string(std::random_device {}()) + "_" + suffix; - return (vortex_test_dir / unique_name).string(); - } - - // Write test data to a unique temporary file and return the path - static std::string WriteTestData(const std::string &suffix = "test_data.vortex") { - std::string path = GetUniqueTempFile(suffix); - auto stream = vortex::testing::CreateTestDataStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream), - path.c_str()); - return path; - } - - // Helper function to create and initialize array view - nanoarrow::UniqueArrayView CreateArrayView(const nanoarrow::UniqueArray &array, - const nanoarrow::UniqueSchema &schema) { - nanoarrow::UniqueArrayView array_view; - ArrowError error; - ArrowErrorCode init_result = ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error); - if (init_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - ArrowErrorCode set_result = ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr); - if (set_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - return array_view; - } - - std::pair - StreamToUniqueStreamSchema(ArrowArrayStream &stream) { - nanoarrow::UniqueArrayStream array_stream; - ArrowArrayStreamMove(&stream, array_stream.get()); - ArrowError error; - nanoarrow::UniqueSchema schema; - ArrowErrorCode set_result = ArrowArrayStreamGetSchema(array_stream.get(), schema.get(), &error); - if (set_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - return {std::move(array_stream), std::move(schema)}; - } - - nanoarrow::UniqueArray ReadFirstArrayFromUniqueStream(nanoarrow::UniqueArrayStream &array_stream) { - nanoarrow::UniqueArray array; - int get_next_result = array_stream->get_next(array_stream.get(), array.get()); - EXPECT_EQ(get_next_result, 0); - return array; - } - - std::pair - ReadFirstArrayFromStream(ArrowArrayStream reference_stream) { - auto [ref_array_stream, ref_schema] = StreamToUniqueStreamSchema(reference_stream); - auto ref_array = ReadFirstArrayFromUniqueStream(ref_array_stream); - return {std::move(ref_array), std::move(ref_schema)}; - } - - /// Both array are struct of int64 - void ValidateArray(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema) { - // Basic properties validation - ASSERT_EQ(actual_schema->n_children, ref_schema->n_children); - - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, ref_array->length); - - // Compare all fields - for (int64_t field_idx = 0; field_idx < actual_schema->n_children; ++field_idx) { - auto actual_field = actual_view->children[field_idx]; - auto expected_field = ref_view->children[field_idx]; - - ASSERT_EQ(actual_field->array->length, expected_field->array->length); - - for (int64_t i = 0; i < actual_field->array->length; ++i) { - int64_t actual_value = ArrowArrayViewGetIntUnsafe(actual_field, i); - int64_t expected_value = ArrowArrayViewGetIntUnsafe(expected_field, i); - - ASSERT_EQ(actual_value, expected_value); - } - } - } - - /// Both array are struct of int64 - void ValidateArrayWithSelection(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema, - const std::vector &row_indices) { - // Basic properties validation - ASSERT_EQ(actual_schema->n_children, ref_schema->n_children); - if (row_indices.empty()) { - ASSERT_EQ(actual_array->length, 0); - return; - } - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, row_indices.size()); - - // Selective row comparison using indices - ASSERT_EQ(actual_array->length, static_cast(row_indices.size())); - - for (int64_t i = 0; i < static_cast(row_indices.size()); ++i) { - int64_t ref_idx = row_indices[i]; - - for (int64_t field_idx = 0; field_idx < actual_schema->n_children; ++field_idx) { - int64_t actual_val = ArrowArrayViewGetIntUnsafe(actual_view->children[field_idx], i); - int64_t expected_val = ArrowArrayViewGetIntUnsafe(ref_view->children[field_idx], ref_idx); - - ASSERT_EQ(actual_val, expected_val); - } - } - } - - // Helper to execute scan builder and get array+schema - std::pair ScanFirstArrayFromTestData( - const std::function &configureScanBuilder) { - auto test_data_path = WriteTestData(); - auto file = vortex::VortexFile::Open(test_data_path); - auto scan_builder = file.CreateScanBuilder(); - auto stream = configureScanBuilder(scan_builder); - - auto [array_stream, schema] = StreamToUniqueStreamSchema(stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - - return {std::move(array), std::move(schema)}; - } - - /// Validate array with projection - only checks specified field indices - void ValidateArrayWithProjection(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema, - const std::vector &field_idxs) { - ASSERT_EQ(actual_schema->n_children, field_idxs.size()); - - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, ref_array->length); - - // Compare only the specified fields - for (int64_t i = 0; i < actual_array->length; ++i) { - for (size_t proj_idx = 0; proj_idx < field_idxs.size(); ++proj_idx) { - int64_t ref_field_idx = field_idxs[proj_idx]; - int64_t actual_val = ArrowArrayViewGetIntUnsafe(actual_view->children[proj_idx], i); - int64_t expected_val = ArrowArrayViewGetIntUnsafe(ref_view->children[ref_field_idx], i); - ASSERT_EQ(actual_val, expected_val); - } - } - } - - // Top-level test helper that all tests can use - void - RunScanBuilderTest(const std::function &configureScanBuilder, - ArrowArrayStream expected_stream, - const std::vector &expected_row_indices = {}, - bool selection = false) { - - auto [array, schema] = ScanFirstArrayFromTestData(configureScanBuilder); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(expected_stream); - selection == false - ? ValidateArray(array, schema, ref_array, ref_schema) - : ValidateArrayWithSelection(array, schema, ref_array, ref_schema, expected_row_indices); - } - - // New helper for projection tests - void RunScanBuilderProjectionTest( - const std::function &configureScanBuilder, - ArrowArrayStream expected_stream, - const std::vector &field_idxs) { - - auto [array, schema] = ScanFirstArrayFromTestData(configureScanBuilder); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(expected_stream); - - ValidateArrayWithProjection(array, schema, ref_array, ref_schema, field_idxs); - } -}; - -TEST_F(VortexTest, ScanToStream) { - RunScanBuilderTest([](vortex::ScanBuilder &builder) { return std::move(builder).IntoStream(); }, - vortex::testing::CreateTestDataStream()); -} - -TEST_F(VortexTest, ScanBuilderWithLimitWithRowRange) { - // Test field "a" and "b" - should contain values from rows 1-2 from original data (indices 1 and - // 2) - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - return std::move(scan_builder.WithLimit(2).WithRowRange(1, 4)).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {1, 2}, - true); -} - -TEST_F(VortexTest, ScanBuilderWithIncludeByIndex) { - std::vector include_by_index = {1, 3}; - - RunScanBuilderTest( - [&include_by_index](vortex::ScanBuilder &scan_builder) { - return std::move( - scan_builder.WithIncludeByIndex(include_by_index.data(), include_by_index.size())) - .IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {1, 3}, - true); -} - -TEST_F(VortexTest, ScanBuilderWithRowRangeWithIncludeByIndex) { - std::vector include_by_index = {1, 3, 4}; - - RunScanBuilderTest( - [&include_by_index](vortex::ScanBuilder &scan_builder) { - return std::move(scan_builder.WithRowRange(2, 5).WithIncludeByIndex(include_by_index.data(), - include_by_index.size())) - .IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {3, 4}, - true); -} - -TEST_F(VortexTest, WriteArrayStream) { - auto test_data_path = WriteTestData(); - auto file = vortex::VortexFile::Open(test_data_path); - auto stream = file.CreateScanBuilder().IntoStream(); - - // Write the stream to a new Vortex file - std::string test_output_path = GetUniqueTempFile("write_output.vortex"); - vortex::VortexWriteOptions write_options; - ASSERT_NO_THROW(write_options.WriteArrayStream(stream, test_output_path)); - - // Verify the written file - auto written_file = vortex::VortexFile::Open(test_output_path); - ASSERT_EQ(written_file.RowCount(), 5); - - // Verify data integrity by reading from the written file - auto out_stream = written_file.CreateScanBuilder().IntoStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(out_stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestDataStream()); - ValidateArray(array, schema, ref_array, ref_schema); -} - -TEST_F(VortexTest, ConcurrentMultiStreamRead) { - std::string test_data_path_1m = GetUniqueTempFile("concurrent_1m.vortex"); - auto stream_1m = vortex::testing::CreateTestData1MStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream_1m), - test_data_path_1m.c_str()); - - auto file = vortex::VortexFile::Open(test_data_path_1m); - auto stream_driver = file.CreateScanBuilder().IntoStreamDriver(); - - // Structure to hold batch data with first ID and nanoarrow array - struct BatchData { - int64_t first_id; - nanoarrow::UniqueArray array; - - BatchData(int64_t first_id, nanoarrow::UniqueArray array) - : first_id(first_id), array(std::move(array)) { - } - }; - - std::vector thread1_batches; - std::vector thread2_batches; - - // Helper function to read from a stream and collect batches - auto read_stream = [&](std::vector &batches) { - // Each thread creates its own stream - auto stream = stream_driver.CreateArrayStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(stream); - - std::vector local_batches; - - while (true) { - nanoarrow::UniqueArray array; - int get_next_result = array_stream->get_next(array_stream.get(), array.get()); - - if (get_next_result != 0) { - std::cerr << "Error: " << array_stream->get_last_error(array_stream.get()) << '\n'; - std::abort(); - } - - if (array->length == 0) { - break; // Empty array indicates end - } - - auto array_view = CreateArrayView(array, schema); - - int64_t first_id = ArrowArrayViewGetIntUnsafe(array_view->children[0], 0); - - local_batches.emplace_back(first_id, std::move(array)); - } - batches = std::move(local_batches); - }; - - // Launch two threads - std::thread thread1(read_stream, std::ref(thread1_batches)); - std::thread thread2(read_stream, std::ref(thread2_batches)); - - // Wait for both threads to complete - thread1.join(); - thread2.join(); - - // Combine all batches from both threads - std::vector all_batches; - all_batches.insert(all_batches.end(), - std::make_move_iterator(thread1_batches.begin()), - std::make_move_iterator(thread1_batches.end())); - all_batches.insert(all_batches.end(), - std::make_move_iterator(thread2_batches.begin()), - std::make_move_iterator(thread2_batches.end())); - - // Sort batches by first ID to ensure proper validation order - std::sort(all_batches.begin(), all_batches.end(), [](const BatchData &a, const BatchData &b) { - return a.first_id < b.first_id; - }); - - // Create reference data for validation - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestData1MStream()); - auto ref_array_view = CreateArrayView(ref_array, ref_schema); - - // Validate all data against reference - constexpr size_t EXPECTED_ROWS = static_cast(1024) * 1024; - size_t total_rows_read = 0; - int64_t reference_offset = 0; - - auto stream_for_schema = stream_driver.CreateArrayStream(); - auto [_, schema] = StreamToUniqueStreamSchema(stream_for_schema); - for (const auto &batch : all_batches) { - auto array_view = CreateArrayView(batch.array, schema); - - for (int64_t i = 0; i < batch.array->length; ++i) { - - int64_t actual_id = ArrowArrayViewGetIntUnsafe(array_view->children[0], i); - int32_t actual_value = - static_cast(ArrowArrayViewGetIntUnsafe(array_view->children[1], i)); - - int64_t expected_id = ArrowArrayViewGetIntUnsafe(ref_array_view->children[0], reference_offset); - int32_t expected_value = static_cast( - ArrowArrayViewGetIntUnsafe(ref_array_view->children[1], reference_offset)); - - ASSERT_EQ(actual_id, expected_id); - ASSERT_EQ(actual_value, expected_value); - reference_offset++; - } - total_rows_read += batch.array->length; - } - - // Verify we read all expected data - ASSERT_EQ(total_rows_read, EXPECTED_ROWS) - << "Expected to read " << EXPECTED_ROWS << " rows, but read " << total_rows_read << " rows"; - ASSERT_EQ(reference_offset, EXPECTED_ROWS) << "Reference validation didn't cover all rows"; - - ASSERT_GT(all_batches.size(), 1) << "Expected multiple batches, but got " << all_batches.size(); -} - -namespace ve = vortex::expr; -namespace vs = vortex::scalar; - -TEST_F(VortexTest, ScanBuilderWithFilter) { - // Test filtering with eq(column("a"), val) - should return only rows where column "a" equals 30 - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - auto filter = ve::eq(ve::column("a"), ve::literal(vs::int32(30))); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithFilterLvalueref) { - // Test filtering with eq(column("a"), val) - should return only rows where column "a" equals 30 - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - const auto filter = ve::eq(ve::column("a"), ve::literal(vs::int32(30))); - return std::move(scan_builder.WithFilter(filter)).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithFilterNoMatches) { - // Test filtering with eq(column("a"), val) where no rows match - should return empty result - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - auto filter = ve::eq(ve::column("a"), - ve::literal(vs::int32(999)) // Value that doesn't exist in test data - ); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {}, - true); // No matching rows -} - -TEST_F(VortexTest, ScanBuilderWithFilterUsingDTypeFromArrowAndScalarCast) { - // Test filtering using DType::from_arrow and Scalar::cast functionality - // This test creates a filter expression by casting a scalar to match the column type - - // Test DType::from_arrow with int32 schema - nanoarrow::UniqueSchema int32_schema; - ArrowSchemaInit(int32_schema.get()); - ArrowErrorCode result = ArrowSchemaSetType(int32_schema.get(), NANOARROW_TYPE_INT32); - EXPECT_EQ(result, NANOARROW_OK); - result = ArrowSchemaSetName(int32_schema.get(), "test_field"); - EXPECT_EQ(result, NANOARROW_OK); - - auto dtype = vortex::dtype::from_arrow(*int32_schema.get()); - - // Use the casted scalar in filter expression - create a new scalar for lambda - RunScanBuilderTest( - [&](vortex::ScanBuilder &scan_builder) { - auto test_scalar = vs::cast(vs::int64(30), std::move(dtype)); - auto filter = ve::eq(ve::column("a"), ve::literal(std::move(test_scalar))); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithProjectionSingleColumn) { - // Test projection selecting only column "a" (field index 0) - RunScanBuilderProjectionTest( - [](vortex::ScanBuilder &scan_builder) { - auto projection = ve::select({"a"}, ve::root()); - return std::move(scan_builder.WithProjection(std::move(projection))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {0}); -} - -TEST_F(VortexTest, OpenFromBuffer) { - std::string test_file_path = GetUniqueTempFile("buffer.vortex"); - auto stream = vortex::testing::CreateTestDataStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream), - test_file_path.c_str()); - - std::ifstream file(test_file_path, std::ios::binary | std::ios::ate); - ASSERT_TRUE(file.is_open()) << "Failed to open file: " << test_file_path; - - std::streamsize file_size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(file_size); - ASSERT_TRUE(file.read(reinterpret_cast(buffer.data()), file_size)) - << "Failed to read file into buffer"; - file.close(); - - auto vortex_file = vortex::VortexFile::Open(buffer.data(), buffer.size()); - ASSERT_EQ(vortex_file.RowCount(), 5); - - auto scan_stream = vortex_file.CreateScanBuilder().IntoStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(scan_stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestDataStream()); - ValidateArray(array, schema, ref_array, ref_schema); -} diff --git a/vortex-cxx/cpp/tests/test_data_generator.cpp b/vortex-cxx/cpp/tests/test_data_generator.cpp deleted file mode 100644 index 67dcb290509..00000000000 --- a/vortex-cxx/cpp/tests/test_data_generator.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "test_data_generator.hpp" -#include -#include -#include -#include -#include - -namespace vortex { -namespace testing { - - ArrowArrayStream CreateTestDataStream() { - // Create a simple two-column struct with int32 data - // Schema: struct{a: int32, b: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray field_a, field_b; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_a.get(), NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_b.get(), NANOARROW_TYPE_INT32)); - - // Reserve for 5 elements - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_a.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_b.get())); - - // Add data: [10, 20, 30, 40, 50] - std::vector data = {10, 20, 30, 40, 50}; - for (int32_t value : data) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_a.get(), value)); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_b.get(), value)); - } - - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(field_a.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(field_b.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = 5; - ArrowArrayMove(field_a.get(), struct_array->children[0]); - ArrowArrayMove(field_b.get(), struct_array->children[1]); - - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); - - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); - - return stream; - } - - ArrowArrayStream CreateTestData1MStream() { - constexpr size_t NUM_ROWS = 1024UL * 1024; - - // Create schema: struct{id: int64, value: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "id")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "value")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray id_field, value_field; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(id_field.get(), NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(value_field.get(), NANOARROW_TYPE_INT32)); - - // Reserve space - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(id_field.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(value_field.get())); - - // Add data - for (size_t i = 0; i < NUM_ROWS; ++i) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(id_field.get(), static_cast(i))); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(value_field.get(), static_cast(i * 2))); - } - - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(id_field.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(value_field.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = NUM_ROWS; - ArrowArrayMove(id_field.get(), struct_array->children[0]); - ArrowArrayMove(value_field.get(), struct_array->children[1]); - - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); - - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); - - return stream; - } - -} // namespace testing -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/tests/test_data_generator.hpp b/vortex-cxx/cpp/tests/test_data_generator.hpp deleted file mode 100644 index ec5dba55d50..00000000000 --- a/vortex-cxx/cpp/tests/test_data_generator.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include - -namespace vortex { -namespace testing { - - /// Create test data with structure {a: [10, 20, 30, 40, 50], b: [10, 20, 30, 40, 50]} - /// This stream only has one Array - ArrowArrayStream CreateTestDataStream(); - - /// Create 1M rows of test data with structure {id: [0..1M], value: [0, 2, 4, ..., 2M]} - /// This stream only has one Array - ArrowArrayStream CreateTestData1MStream(); - -} // namespace testing -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/examples/.clang-tidy b/vortex-cxx/examples/.clang-tidy new file mode 100644 index 00000000000..e302d65b598 --- /dev/null +++ b/vortex-cxx/examples/.clang-tidy @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors +InheritParentConfig: true +Checks: | + -google-build-using-namespace, + -bugprone-unchecked-optional-access, diff --git a/vortex-cxx/examples/CMakeLists.txt b/vortex-cxx/examples/CMakeLists.txt index 348d80e1547..a8f90119fd7 100644 --- a/vortex-cxx/examples/CMakeLists.txt +++ b/vortex-cxx/examples/CMakeLists.txt @@ -1,34 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.22) - -project(vortex-examples) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -find_program(SCCACHE_PROGRAM sccache) -if (SCCACHE_PROGRAM) - set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - message(STATUS "Sccache found: ${SCCACHE_PROGRAM}") -else () - message(STATUS "Sccache not found") -endif () - -include(FetchContent) -FetchContent_Declare( - vortex - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../ - SOURCE_SUBDIR vortex-cxx -) -FetchContent_MakeAvailable(vortex) - -add_executable(hello-vortex hello-vortex.cpp) - -if(APPLE) - set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") -endif() - -target_link_libraries(hello-vortex PRIVATE vortex ${APPLE_LINK_FLAGS}) +foreach(name reader writer dtype scan scan_to_arrow) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE vortex_cxx_shared nanoarrow_shared) + if(APPLE) + target_link_libraries(${name} PRIVATE + "-framework CoreFoundation" "-framework Security") + endif() +endforeach() diff --git a/vortex-cxx/examples/README.md b/vortex-cxx/examples/README.md deleted file mode 100644 index 40e7708fa33..00000000000 --- a/vortex-cxx/examples/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# C++ examples - -This example shows how to interface with the C++ API using CMake. - -```bash -mkdir -p build -cd build -cmake .. -make -j$(nproc) -./hello-vortex -``` \ No newline at end of file diff --git a/vortex-cxx/examples/dtype.cpp b/vortex-cxx/examples/dtype.cpp new file mode 100644 index 00000000000..bdaaab1b3df --- /dev/null +++ b/vortex-cxx/examples/dtype.cpp @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +#include + +using vortex::DataSource; +using vortex::DataType; +using vortex::DataTypeVariant; +using vortex::Field; +using vortex::Session; +using vortex::VortexException; + +static void print_dtype(const DataType &dtype); + +static std::string_view ptype_name(vortex::PType p) { + using enum vortex::PType; + switch (p) { + case U8: + return "uint8_t"; + case U16: + return "uint16_t"; + case U32: + return "uint32_t"; + case U64: + return "uint64_t"; + case I8: + return "int8_t"; + case I16: + return "int16_t"; + case I32: + return "int32_t"; + case I64: + return "int64_t"; + case F16: + return "float16"; + case F32: + return "float"; + case F64: + return "double"; + } + return "?"; +} + +static void print_struct(const DataType &dtype) { + std::cout << "struct(\n"; + for (const Field &field : dtype.fields()) { + std::cout << " " << field.name << " = "; + print_dtype(field.dtype); + } + std::cout << ")"; +} + +static void print_dtype(const DataType &dtype) { + using enum DataTypeVariant; + switch (dtype.variant()) { + case Null: + std::cout << "null"; + break; + case Bool: + std::cout << "bool"; + break; + case Utf8: + std::cout << "utf8"; + break; + case Binary: + std::cout << "binary"; + break; + case Extension: + std::cout << "extension"; + break; + case Primitive: + std::cout << "primitive(" << ptype_name(dtype.primitive_type()) << ")"; + break; + case Struct: + print_struct(dtype); + break; + case List: + std::cout << "list("; + print_dtype(dtype.list_element()); + std::cout << ")"; + break; + case FixedSizeList: + std::cout << "fixed_list("; + print_dtype(dtype.fixed_size_list_element()); + std::cout << ")"; + break; + case Decimal: + std::cout << "decimal(precision=" << static_cast(dtype.decimal_precision()) + << ", scale=" << static_cast(dtype.decimal_scale()) << ")"; + break; + } + std::cout << (dtype.nullable() ? '?' : ' ') << '\n'; +} + +int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Usage: dtype \n"; + return 1; + } + + Session session; + DataSource ds = DataSource::open(session, {argv[1]}); + DataType dt = ds.dtype(); + std::cout << "dtype: "; + print_dtype(dt); + return 0; +} diff --git a/vortex-cxx/examples/hello-vortex.cpp b/vortex-cxx/examples/hello-vortex.cpp deleted file mode 100644 index c6a7fec3b9a..00000000000 --- a/vortex-cxx/examples/hello-vortex.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "nanoarrow/common/inline_types.h" -#include "nanoarrow/hpp/unique.hpp" -#include "nanoarrow/nanoarrow.hpp" -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include -#include -#include -#include -#include - -/// Create test data with structure {a: [10, 20, 30, 40, 50], b: [100, 200, 300, 400, 500]} -ArrowArrayStream CreateTestDataStream() { - // Create a simple two-column struct with int32 data - // Schema: struct{a: int32, b: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray field_a, field_b; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_a.get(), NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_b.get(), NANOARROW_TYPE_INT32)); - - // Reserve for 5 elements - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_a.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_b.get())); - - // Add data: a=[10, 20, 30, 40, 50], b=[100, 200, 300, 400, 500] - std::vector data_a = {10, 20, 30, 40, 50}; - std::vector data_b = {100, 200, 300, 400, 500}; - for (size_t i = 0; i < data_a.size(); ++i) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_a.get(), data_a[i])); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_b.get(), data_b[i])); - } - - NANOARROW_THROW_NOT_OK(ArrowArrayFinishBuilding(field_a.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK(ArrowArrayFinishBuilding(field_b.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = 5; - ArrowArrayMove(field_a.get(), struct_array->children[0]); - ArrowArrayMove(field_b.get(), struct_array->children[1]); - - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); - - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); - - return stream; -} - -int main() { - // Create a temporary file path - std::filesystem::path temp_dir = std::filesystem::temp_directory_path(); - std::string vortex_file = (temp_dir / "hello_vortex_example.vortex").string(); - - std::cout << "=== Vortex C++ Example ===" << '\n'; - std::cout << "Writing to: " << vortex_file << '\n'; - - // Write test data to a Vortex file - { - auto stream = CreateTestDataStream(); - vortex::VortexWriteOptions write_options; - write_options.WriteArrayStream(stream, vortex_file); - std::cout << "Wrote test data to file" << '\n'; - } - - auto check_stream = [](ArrowArrayStream &stream) { - nanoarrow::UniqueArray array; - int get_next_result = stream.get_next(&stream, array.get()); - assert(get_next_result == 0); - std::cout << "Number of rows: " << array->length << '\n'; - std::cout << "Number of columns in schema: " << array->n_children << '\n'; - }; - - // 1. Classic C++ builder pattern - std::cout << "\n1. Classic C++ builder pattern:" << '\n'; - { - auto builder = vortex::VortexFile::Open(vortex_file).CreateScanBuilder(); - builder.WithLimit(3); - auto stream = std::move(builder).IntoStream(); - check_stream(stream); - } - // 2. One-line Rusty function chain - std::cout << "\n2. One-line Rusty function chain:" << '\n'; - { - auto stream = vortex::VortexFile::Open(vortex_file).CreateScanBuilder().WithLimit(3).IntoStream(); - check_stream(stream); - } - // 3. Conditionally set the builder - std::cout << "\n3. Conditionally set the builder:" << '\n'; - { - auto limit = 1; - auto builder = vortex::VortexFile::Open(vortex_file).CreateScanBuilder(); - if (limit > 0) { - // prefer C++ way - builder.WithLimit(1); - // Rusty way is Ok, but you have to move the builder to an rvalue. - // builder = std::move(builder).WithLimit(3); - } - auto stream = std::move(builder).IntoStream(); - check_stream(stream); - } - - // Clean up - std::filesystem::remove(vortex_file); - std::cout << "\nCleaned up temporary file" << '\n'; - - return 0; -} \ No newline at end of file diff --git a/vortex-cxx/examples/reader.cpp b/vortex-cxx/examples/reader.cpp new file mode 100644 index 00000000000..48656c046f7 --- /dev/null +++ b/vortex-cxx/examples/reader.cpp @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +#include +#include + +using namespace std::string_view_literals; +using namespace vortex; +using namespace expr; +using namespace ops; // overloaded >= for Expressions +namespace fs = std::filesystem; + +int main() { + const Session session; + const DataSource ds = DataSource::open(session, {"people*.vortex", "me.vortex"}); + Scan scan = ds.scan({.filter = col("height") >= lit(50)}); + + for (Partition &partition : scan.partitions()) { + for (Array &array : partition.batches()) { + const Array age = array.field("age"); + const PrimitiveView age_view = age.values(session); + const std::span age_values = age_view.values(); + for (uint8_t value : age_values) { + std::cout << int(value) << " "; + } + } + } + std::cout << "\n"; + + return 0; +} diff --git a/vortex-cxx/examples/scan.cpp b/vortex-cxx/examples/scan.cpp new file mode 100644 index 00000000000..5a8d6fcf14c --- /dev/null +++ b/vortex-cxx/examples/scan.cpp @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +#include +#include +#include +#include +#include + +using vortex::DataSource; +using vortex::Estimate; +using vortex::Scan; +using vortex::Session; + +static void print_estimate(const char *what, const Estimate &est) { + if (!est.known()) { + std::cout << what << ": unknown\n"; + } else if (est.exact()) { + std::cout << what << ": " << est.value() << '\n'; + } else { + std::cout << what << ": at most " << est.value() << '\n'; + } +} + +struct ScanStats { + size_t partitions = 0; + size_t arrays = 0; + size_t rows = 0; +}; + +static ScanStats worker(Scan &scan) { + ScanStats stats; + while (auto partition = scan.next_partition()) { + ++stats.partitions; + while (auto array = partition->next()) { + ++stats.arrays; + stats.rows += array->size(); + } + } + return stats; +} + +int main(int argc, char **argv) { + size_t num_threads = 0; + int opt = 0; + while ((opt = getopt(argc, argv, "j:")) != -1) { + switch (opt) { + case 'j': + num_threads = static_cast(std::atoi(optarg)); + break; + default: + std::cerr << "Multi-threaded file scan\nUsage: scan [-j " + "threads] \n"; + return 1; + } + } + if (optind + 1 != argc) { + std::cerr << "Multi-threaded file scan\nUsage: scan [-j threads] \n"; + return 1; + } + + const Session session; + std::cout << "Opening files: " << argv[optind] << '\n'; + const DataSource ds = DataSource::open(session, {argv[optind]}); + + print_estimate("Data source row count", ds.row_count()); + + Scan scan = ds.scan(); + print_estimate("Partition count", scan.partition_count()); + + if (num_threads == 0) { + num_threads = scan.partition_count().value_or(1); + } + + std::cout << "Starting scan, using " << num_threads << " threads\n"; + std::vector threads; + threads.reserve(num_threads); + std::vector results(num_threads); + + for (size_t i = 0; i < num_threads; ++i) { + threads.emplace_back([i, &scan, &results] { results[i] = worker(scan); }); + } + for (auto &t : threads) { + t.join(); + } + + ScanStats total; + for (const auto &r : results) { + total.partitions += r.partitions; + total.arrays += r.arrays; + total.rows += r.rows; + } + std::cout << "Finished scan, processed " << total.partitions << " partitions, " << total.arrays + << " arrays, " << total.rows << " rows\n"; + return 0; +} diff --git a/vortex-cxx/examples/scan_to_arrow.cpp b/vortex-cxx/examples/scan_to_arrow.cpp new file mode 100644 index 00000000000..707cd4a4103 --- /dev/null +++ b/vortex-cxx/examples/scan_to_arrow.cpp @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include +#include +#include +#include + +using vortex::ArrowStream; +using vortex::DataSource; +using vortex::DataType; +using vortex::Scan; +using vortex::Session; +using vortex::VortexException; + +int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Scan vortex files to Arrow\nUsage: scan_to_arrow \n"; + return 1; + } + const char *paths = argv[1]; + + Session session; + DataSource ds = DataSource::open(session, {paths}); + Scan scan = ds.scan(); + + DataType out_dtype = ds.dtype(); + ArrowSchema schema = out_dtype.to_arrow(); + char schema_buf[10 * 1024]; + const int64_t schema_len = ArrowSchemaToString(&schema, schema_buf, sizeof schema_buf, 1); + std::cout << "arrow schema: " << std::string_view {schema_buf, static_cast(schema_len)} << '\n'; + if (schema.release != nullptr) { + schema.release(&schema); + } + + ArrowError arrow_error {}; + ArrowErrorInit(&arrow_error); + + size_t partition_idx = 0; + while (auto partition = scan.next_partition()) { + ArrowStream stream = std::move(*partition).into_arrow_stream(); + + size_t rows = 0; + size_t arrays = 0; + ArrowArray array = {}; + while (ArrowArrayStreamGetNext(stream.raw(), &array, &arrow_error) == NANOARROW_OK && + array.release != nullptr) { + rows += array.length; + ++arrays; + array.release(&array); + std::memset(&array, 0, sizeof(array)); + } + std::cout << "Read partition " << partition_idx << " to Arrow, " << arrays << " arrays, " << rows + << " rows\n"; + ++partition_idx; + } + return 0; +} diff --git a/vortex-cxx/examples/writer.cpp b/vortex-cxx/examples/writer.cpp new file mode 100644 index 00000000000..b4c88e458e1 --- /dev/null +++ b/vortex-cxx/examples/writer.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +#include +#include + +using namespace vortex; +using namespace expr::ops; +using dtype::Nullable; + +int main(int argc, char **argv) { + const Session session; + const DataType dtype = dtype::struct_({ + {"age", dtype::uint8()}, + {"height", dtype::uint16(Nullable)}, + }); + + constexpr size_t SAMPLE_ROWS = 100; + std::vector age_buffer(SAMPLE_ROWS); + std::vector height_buffer(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + age_buffer[i] = static_cast(i); + height_buffer[i] = static_cast((i + 1) % 200); + } + + Array age = Array::primitive(age_buffer); + Array array = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, AllValid)}, + }); + + Expression age_gt_10 = expr::col("age") > expr::lit(10); + Array validity_array = array.apply(age_gt_10); + + const Validity validity {ValidityType::Array, validity_array}; + Array array2 = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, validity)}, + }); + + Writer writer = Writer::open(session, argv[1], dtype); + writer.push(array); + writer.push(array2); + writer.finish(); + + return 0; +} diff --git a/vortex-cxx/include/vortex/array.hpp b/vortex-cxx/include/vortex/array.hpp new file mode 100644 index 00000000000..bd9a761278c --- /dev/null +++ b/vortex-cxx/include/vortex/array.hpp @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/expression.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace vortex { + +// Types that a PrimitiveView can hold +template +concept primitive_view = primitive_type || std::is_same_v; + +template +class PrimitiveView; + +class Array; +class StringView; +class BytesView; + +enum class ValidityType { + // Items can't be null + NonNullable = VX_VALIDITY_NON_NULLABLE, + // All items are valid + AllValid = VX_VALIDITY_ALL_VALID, + // All items are invalid + AllInvalid = VX_VALIDITY_ALL_INVALID, + // Item validity is set in a boolean array: true = valid, false = invalid + Array = VX_VALIDITY_ARRAY, +}; + +inline constexpr ValidityType NonNullable = ValidityType::NonNullable; +inline constexpr ValidityType AllValid = ValidityType::AllValid; +inline constexpr ValidityType AllInvalid = ValidityType::AllInvalid; + +/** + * Array per-element validity of type ValidityType. + * If ValidityType is ValidityType::Array, holds a boolean array + * with validity items. + * + * You can use shortcut constants NonNullable/AllValid/AllInvalid and + * function ValidityArray(validity_bools); + */ +class Validity { +public: + // NOLINTNEXTLINE(google-explicit-constructor) for NonNullable constructor + Validity(ValidityType type) : type_(type), array_(nullptr) {} + Validity(ValidityType type, const Array &array); + + Validity(const Validity &other); + Validity(Validity &&other) noexcept; + Validity &operator=(const Validity &other); + Validity &operator=(Validity &&other) noexcept; + ~Validity(); + + ValidityType type() const noexcept { return type_; } + + // Boolean validity array. Throws if type() != ValidityType::Array. + Array array() const; + +private: + friend struct detail::Access; + friend Validity ValidityArray(const Array &bools); + Validity(ValidityType type, const vx_array *owned) : type_(type), array_(owned) {} + + ValidityType type_; + const vx_array *array_; +}; + +// Validity determined by a boolean array, true = valid, false = invalid. +Validity ValidityArray(const Array &bools); + +namespace detail { +// Validity bitmap for typed views. Owns the arrays that back the bits +class ValidityBits { +public: + ValidityBits(ValidityBits &&other) noexcept; + ValidityBits &operator=(ValidityBits &&other) noexcept; + ValidityBits(const ValidityBits &) = delete; + ValidityBits &operator=(const ValidityBits &) = delete; + ~ValidityBits(); + + bool is_null(size_t index) const noexcept; + +private: + friend class vortex::Array; + // Materialize validity of "canonical" + ValidityBits(const Session &session, const vx_array *canonical); + + const vx_array *owner_ = nullptr; + const uint8_t *bits_ = nullptr; + size_t bit_offset_ = 0; + bool all_invalid_ = false; +}; +} // namespace detail + +// A reference-counted handle to columnar data in some encoding +class Array { +public: + Array(const Array &other); + Array(Array &&) noexcept = default; + Array &operator=(const Array &other); + Array &operator=(Array &&) noexcept = default; + + // An all-null array with DataType Null. + static Array null(size_t len); + + /** + * A Primitive array copied from a typed buffer. + * + * Example: + * + * std::array buffer = {0, 1, 2}; + * auto array = Array::primitive(buffer); + */ + template + static Array primitive(std::span data, const Validity &validity = NonNullable) { + return primitive_raw(detail::to_ptype(), data.data(), data.size(), validity); + } + + /** + * Import an Arrow array. Consumes both "array" and "schema", do not use + * or release them afterwards. For a record batch pass nullable = false. + */ + static Array from_arrow(ArrowArray *array, ArrowSchema *schema, bool nullable); + + size_t size() const; + bool nullable() const; + bool has_dtype(DataTypeVariant variant) const; + bool is_primitive(PType ptype) const; + DataType dtype() const; + Validity validity() const; + size_t null_count() const; + + // Struct field accessors. Throw if Array's DataType is not Struct + + Array field(size_t index) const; + Array field(std::string_view name) const; + + /* + * Zero-copy view of rows [begin, end). + * + * Example: + * + * std::array buffer = {0, 1, 2}; + * auto array = Array::primitive(buffer); + * auto sliced = array.slice(1, 2); + */ + Array slice(size_t begin, size_t end) const; + + /** + * Apply an expression to an array. + * + * This function operates in constant time and doesn't execute the result + * array. To execute the array, canonicalise it. + * + * Example: + * + * using namespace vortex::expr::ops; + * + * std::array buffer = {0, 1, 2}; + * Array array = Array::primitive(buffer); + * Expression expr = expr::root() > expr::lit(0); + * Array result = array.apply(expr); + */ + Array apply(const Expression &expr) const; + + /** + * Bulk view over values. Canonicalizes the array. + * Throws if T does not match the array's ptype. + * + * + * Example: + * + * const Session session; + * std::array buffer = {0, 1, 2}; + * Array array = Array::primitive(buffer); + * auto view = array.values(session); + */ + template + PrimitiveView values(const Session &session) const; + + // Bulk view over Bool values + PrimitiveView bools(const Session &session) const; + + // Bulk view over Utf8 values. + StringView strings(const Session &session) const; + + // Bulk view over Binary values. + BytesView bytes(const Session &session) const; + +private: + friend struct detail::Access; + friend class StringView; + friend class BytesView; + template + friend class PrimitiveView; + + explicit Array(const vx_array *owned) : handle_(owned) {} + const vx_array *release() && noexcept { return handle_.release(); } + + static Array primitive_raw(vx_ptype ptype, const void *data, size_t len, const Validity &validity); + Array canonicalize(const Session &session) const; + + struct Deleter { + void operator()(const vx_array *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +/** + * Create a Struct array from named columns of equal length. + * + * Example: + * + * std::array age_buffer = {0, 1, 2}; + * std::array height_buffer = {0, 1, 2}; + * Array ages = Array::primitive(age_buffer); + * Array heights = Array::primitive(height_buffer); + * Array result = make_struct( + * {{"age", ages}, {"height", heights}}, + * NonNullable); + */ +Array make_struct(std::initializer_list> fields, + const Validity &validity = NonNullable); + +/** + * Create a Struct Array from a dynamic number of fields. + * Prefer make_struct() if you know all fields beforehand. + * + * Example: + * + * std::array age_buffer = {0, 1, 2}; + * std::array height_buffer = {0, 1, 2}; + * Array ages = Array::primitive(age_buffer); + * Array heights = Array::primitive(height_buffer); + * + * StructArrayBuilder b(NonNullable, 2); + * Array result = builder + * .add("age", ages) + * .add("height", heights) + * .build(); + */ +class StructArrayBuilder { +public: + explicit StructArrayBuilder(const Validity &validity, size_t capacity = 0); + StructArrayBuilder(const StructArrayBuilder &) = delete; + StructArrayBuilder &operator=(const StructArrayBuilder &) = delete; + StructArrayBuilder(StructArrayBuilder &&) noexcept = default; + StructArrayBuilder &operator=(StructArrayBuilder &&) noexcept = default; + + StructArrayBuilder &add(std::string_view name, const Array &field) &; + StructArrayBuilder &&add(std::string_view name, const Array &field) &&; + + // Consume the builder and return and Array + Array build() &&; + +private: + struct Deleter { + void operator()(vx_struct_column_builder *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +/** + * Typed read-only view over a Primitive array. + * + * Owns a canonicalized copy of Array. values() and anything derived from + * it are valid as long as the view lives. + */ +template +class PrimitiveView { +public: + /* + * Get raw values from this view. Values at null/invalid positions are + * unspecified. + */ + std::span values() const noexcept { return {data_, size_}; } + bool is_null(size_t index) const noexcept { return validity_.is_null(index); } + size_t size() const noexcept { return size_; } + +private: + friend class Array; + PrimitiveView(Array canonical, detail::ValidityBits validity, const T *data, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), data_(data), size_(size) {} + + Array canonical_; + detail::ValidityBits validity_; + const T *data_; + size_t size_; +}; + +/** + * Read-only view over a Bool array. As Bool values are bit-packed, there's no + * span. Read individual values with value(i). + */ +template <> +class PrimitiveView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + bool value(size_t index) const; + bool is_null(size_t index) const noexcept { return validity_.is_null(index); } + size_t size() const noexcept { return size_; } + +private: + friend class Array; + PrimitiveView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) {} + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +/** + * Read-only view over a Utf8 array. + * + * operator[] is O(1) and borrows from the view's canonical copy. Returned + * string_views are valid as long as the view lives. + */ +class StringView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + std::string_view operator[](size_t index) const; + bool is_null(size_t index) const noexcept { return validity_.is_null(index); } + size_t size() const noexcept { return size_; } + +private: + friend class Array; + StringView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) {} + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +/** + * Read-only view over a Binary array. + * + * Byte spans borrow from the view's canonical copy and are valid as long as + * the view lives. + */ +class BytesView { +public: + /* + * Get raw value from this view. Values at null/invalid positions are + * unspecified. + */ + BinaryView operator[](size_t index) const; + bool is_null(size_t index) const noexcept { return validity_.is_null(index); } + size_t size() const noexcept { return size_; } + +private: + friend class Array; + BytesView(Array canonical, detail::ValidityBits validity, size_t size) + : canonical_(std::move(canonical)), validity_(std::move(validity)), size_(size) {} + + Array canonical_; + detail::ValidityBits validity_; + size_t size_; +}; + +template +PrimitiveView Array::values(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = detail::Access::c_ptr(canonical); + if (!vx_array_is_primitive(raw, detail::to_ptype())) { + throw VortexException("values: T does not match the array's ptype", ErrorCode::MismatchedTypes); + } + vx_error *error = nullptr; + const void *data = vx_array_data_ptr_primitive(raw, &error); + detail::throw_on_error(error); + detail::ValidityBits validity(session, raw); + const size_t n = vx_array_len(raw); + return PrimitiveView(std::move(canonical), std::move(validity), static_cast(data), n); +} +} // namespace vortex diff --git a/vortex-cxx/include/vortex/common.hpp b/vortex-cxx/include/vortex/common.hpp new file mode 100644 index 00000000000..79c79491520 --- /dev/null +++ b/vortex-cxx/include/vortex/common.hpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include +#include +#include +#include +#include + +namespace vortex { +#if __STDCPP_FLOAT64_T__ != 1 +struct float16_t { + uint16_t bits; + friend bool operator==(float16_t, float16_t) = default; +}; +static_assert(sizeof(float16_t) == 2 && std::is_trivially_copyable_v); +#else +#include +using float16_t = std::float16_t; +#endif + +float to_float(float16_t f16t); +} // namespace vortex + +namespace vortex::detail { +struct Access { + template + static T adopt(Args &&...args) { + return T(std::forward(args)...); + } + template + static auto release(T &&t) noexcept { + return std::forward(t).release(); + } + template + static auto c_ptr(const T &t) noexcept { + return t.handle_.get(); + } +}; + +template +constexpr vx_ptype to_ptype() { + if constexpr (std::is_same_v) { + return PTYPE_U8; + } else if constexpr (std::is_same_v) { + return PTYPE_U16; + } else if constexpr (std::is_same_v) { + return PTYPE_U32; + } else if constexpr (std::is_same_v) { + return PTYPE_U64; + } else if constexpr (std::is_same_v) { + return PTYPE_I8; + } else if constexpr (std::is_same_v) { + return PTYPE_I16; + } else if constexpr (std::is_same_v) { + return PTYPE_I32; + } else if constexpr (std::is_same_v) { + return PTYPE_I64; + } else if constexpr (std::is_same_v) { + return PTYPE_F16; + } else if constexpr (std::is_same_v) { + return PTYPE_F32; + } else { + static_assert(std::is_same_v); + return PTYPE_F64; + } +} + +template +inline constexpr bool is_numeric_element = + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v; +} // namespace vortex::detail + +namespace vortex { + +// View over a single Binary byte range. +using BinaryView = std::span; + +// Types that can be stored in a Primitive array +template +concept primitive_type = detail::is_numeric_element; + +// Types constructible as scalars or literals. +template +concept element_type = primitive_type || std::is_same_v || std::is_same_v || + std::is_same_v; +} // namespace vortex diff --git a/vortex-cxx/include/vortex/data_source.hpp b/vortex-cxx/include/vortex/data_source.hpp new file mode 100644 index 00000000000..f48e305f737 --- /dev/null +++ b/vortex-cxx/include/vortex/data_source.hpp @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/estimate.hpp" +#include "vortex/scan.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace vortex { + +/** + * A reference to one or more (possibly remote, possibly glob) paths. + * + * Creating a DataSource opens the first matched path to read the schema. All + * other IO is deferred until a scan is requested. Multiple scans may be + * requested from one data source. + */ +class DataSource { +public: + static DataSource open(const Session &session, std::initializer_list paths); + static DataSource open(const Session &session, std::span paths); + static DataSource open(const Session &session, std::span paths); + + /** + * Create a DataSource from an in-memory Vortex file. Borrows the buffer: + * caller must keep it alive and unmodified while DataSource is alive. + */ + static DataSource from_buffer(const Session &session, std::span data); + + DataSource(const DataSource &other); + DataSource(DataSource &&) noexcept = default; + DataSource &operator=(const DataSource &other); + DataSource &operator=(DataSource &&) noexcept = default; + + Estimate row_count() const; + DataType dtype() const; + Scan scan(ScanOptions options = {}) const; + +private: + friend struct detail::Access; + DataSource(const vx_data_source *owned, Session session) : handle_(owned), session_(std::move(session)) {} + + struct Deleter { + void operator()(const vx_data_source *ptr) const noexcept; + }; + std::unique_ptr handle_; + Session session_; +}; +} // namespace vortex diff --git a/vortex-cxx/include/vortex/dtype.hpp b/vortex-cxx/include/vortex/dtype.hpp new file mode 100644 index 00000000000..e3e272e3312 --- /dev/null +++ b/vortex-cxx/include/vortex/dtype.hpp @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace vortex { + +struct Field; + +enum class DataTypeVariant { + Null = DTYPE_NULL, + Bool = DTYPE_BOOL, + // Primitives e.g., u8, i16, f32 + Primitive = DTYPE_PRIMITIVE, + // Variable-length UTF-8 string + Utf8 = DTYPE_UTF8, + // Variable-length binary + Binary = DTYPE_BINARY, + // Nested struct + Struct = DTYPE_STRUCT, + // Nested list + List = DTYPE_LIST, + // User-defined extension + Extension = DTYPE_EXTENSION, + // Decimal with fixed precision and scale + Decimal = DTYPE_DECIMAL, + // Nested fixed-size list + FixedSizeList = DTYPE_FIXED_SIZE_LIST, +}; + +// Primitive type +enum class PType { + U8 = PTYPE_U8, + U16 = PTYPE_U16, + U32 = PTYPE_U32, + U64 = PTYPE_U64, + I8 = PTYPE_I8, + I16 = PTYPE_I16, + I32 = PTYPE_I32, + I64 = PTYPE_I64, + F16 = PTYPE_F16, + F32 = PTYPE_F32, + F64 = PTYPE_F64, +}; + +/** + * A Vortex data type. Data types are logical: they say nothing about physical + * representation. + */ +class DataType { +public: + DataType(const DataType &other); + DataType(DataType &&) noexcept = default; + DataType &operator=(const DataType &other); + DataType &operator=(DataType &&) noexcept = default; + + /** + * Consume an ArrowSchema and convert it into a DataType. The schema must + * not be used after this call. + */ + static DataType from_arrow(ArrowSchema *schema); + + /** + * Convert dtype to an Arrow C schema. Caller is responsible for invoking + * schema's release() callback. + */ + ArrowSchema to_arrow() const; + + DataTypeVariant variant() const; + bool nullable() const; + + PType primitive_type() const; + uint8_t decimal_precision() const; + int8_t decimal_scale() const; + + /** + * For a Struct dtype, return its fields in order. + * Throws if DataType is not Struct. + */ + std::vector fields() const; + + // List accessors. Valid only on List and FixedSizeList dtypes + + DataType list_element() const; + DataType fixed_size_list_element() const; + uint32_t fixed_size_list_size() const; + +private: + friend struct detail::Access; + explicit DataType(const vx_dtype *owned); + const vx_dtype *release() && noexcept { return handle_.release(); } + + struct Deleter { + void operator()(const vx_dtype *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +// Field of a Struct dtype +struct Field { + std::string name; + DataType dtype; +}; + +namespace dtype { + +inline constexpr bool Nullable = true; + +DataType null(); +DataType boolean(bool nullable = false); +DataType primitive(PType ptype, bool nullable = false); +DataType int8(bool nullable = false); +DataType int16(bool nullable = false); +DataType int32(bool nullable = false); +DataType int64(bool nullable = false); +DataType uint8(bool nullable = false); +DataType uint16(bool nullable = false); +DataType uint32(bool nullable = false); +DataType uint64(bool nullable = false); +DataType float16(bool nullable = false); +DataType float32(bool nullable = false); +DataType float64(bool nullable = false); +DataType utf8(bool nullable = false); +DataType binary(bool nullable = false); +DataType decimal(uint8_t precision, int8_t scale, bool nullable = false); +DataType list(DataType element, bool nullable = false); +DataType fixed_size_list(DataType element, uint32_t size, bool nullable = false); + +/** + * Create a DataTypeVariant::Struct from a field list. + * + * Example: + * + * using dtype::Nullable; + * DataType dtype = dtype::struct_({ + * {"age", dtype::uint8()}, + * {"height", dtype::uint16(Nullable)}} + * ); + */ +DataType struct_(std::initializer_list> fields, bool nullable = false); +} // namespace dtype + +/* + * Create a DataTypeVariant::Struct dynamically. + * Prefer dtype::struct_ if fields are known beforehand. + * + * Example: + * + * using dtype::Nullable; + * DataType dtype = StructFieldsBuilder() + * .add("age", dtype::uint8()) + * .add("height", dtype::uint16(Nullable)}) + * .build(Nullable); + * ); + */ +class StructFieldsBuilder { +public: + StructFieldsBuilder(); + StructFieldsBuilder(const StructFieldsBuilder &) = delete; + StructFieldsBuilder &operator=(const StructFieldsBuilder &) = delete; + StructFieldsBuilder(StructFieldsBuilder &&) noexcept = default; + StructFieldsBuilder &operator=(StructFieldsBuilder &&) noexcept = default; + + StructFieldsBuilder &add(std::string_view name, const DataType &dtype) &; + StructFieldsBuilder &&add(std::string_view name, const DataType &dtype) &&; + + // Consume the builder and return a DataType + DataType build(bool nullable = false) &&; + +private: + struct Deleter { + void operator()(vx_struct_fields_builder *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; +} // namespace vortex diff --git a/vortex-cxx/include/vortex/error.hpp b/vortex-cxx/include/vortex/error.hpp new file mode 100644 index 00000000000..2c85d700026 --- /dev/null +++ b/vortex-cxx/include/vortex/error.hpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include + +#include +#include +#include + +namespace vortex { +enum class ErrorCode { + Other = VX_ERROR_CODE_OTHER, + OutOfBounds = VX_ERROR_CODE_OUT_OF_BOUNDS, + Compute = VX_ERROR_CODE_COMPUTE, + InvalidArgument = VX_ERROR_CODE_INVALID_ARGUMENT, + Serialization = VX_ERROR_CODE_SERIALIZATION, + NotImplemented = VX_ERROR_CODE_NOT_IMPLEMENTED, + MismatchedTypes = VX_ERROR_CODE_MISMATCHED_TYPES, + AssertionFailed = VX_ERROR_CODE_ASSERTION_FAILED, + Io = VX_ERROR_CODE_IO, + Panic = VX_ERROR_CODE_PANIC, +}; + +class VortexException : public std::runtime_error { +public: + VortexException(const std::string &message, ErrorCode code) : std::runtime_error(message), code_(code) {} + + ErrorCode code() const noexcept { return code_; } + +private: + ErrorCode code_; +}; + +namespace detail { +// Throw VortexException and free "error" if it is non-nullptr. +inline void throw_on_error(vx_error *error) { + if (error == nullptr) { + return; + } + const vx_view str = vx_error_message(error); + const std::string message {str.ptr, str.len}; + const auto code = static_cast(vx_error_get_code(error)); + vx_error_free(error); + throw VortexException(message, code); +} + +inline vx_view to_view(std::string_view view) noexcept { return {view.data(), view.size()}; } +} // namespace detail +} // namespace vortex diff --git a/vortex-cxx/include/vortex/estimate.hpp b/vortex-cxx/include/vortex/estimate.hpp new file mode 100644 index 00000000000..ea84e612c7e --- /dev/null +++ b/vortex-cxx/include/vortex/estimate.hpp @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/error.hpp" +#include + +#include + +namespace vortex { + +enum class EstimateType { + Unknown = VX_ESTIMATE_UNKNOWN, + Exact = VX_ESTIMATE_EXACT, + Inexact = VX_ESTIMATE_INEXACT, +}; + +// Estimated count (rows in a partition, partitions in a scan) +class Estimate { +public: + explicit Estimate(vx_estimate raw) : raw_(raw) {} + + bool known() const noexcept { return raw_.type != VX_ESTIMATE_UNKNOWN; } + bool exact() const noexcept { return raw_.type == VX_ESTIMATE_EXACT; } + + /** + * Estimated count. Throws if not known(). For inexact estimates this is + * is an upper bound. + */ + inline uint64_t value() const { + if (!known()) { + throw VortexException("estimate is unknown", ErrorCode::InvalidArgument); + } + return raw_.estimate; + } + + inline uint64_t value_or(uint64_t fallback) const noexcept { return known() ? raw_.estimate : fallback; } + +private: + vx_estimate raw_; +}; + +} // namespace vortex diff --git a/vortex-cxx/include/vortex/expression.hpp b/vortex-cxx/include/vortex/expression.hpp new file mode 100644 index 00000000000..b9e691523c2 --- /dev/null +++ b/vortex-cxx/include/vortex/expression.hpp @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/scalar.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +// A node in an expression tree used for scan filters and projections. +class Expression { +public: + Expression(const Expression &); + Expression &operator=(const Expression &); + Expression(Expression &&) noexcept = default; + Expression &operator=(Expression &&) noexcept = default; + + /** + * Extract field from a Struct. Output DataType is field's DataType. + * + * Errors at scan/apply time if field does not exist or if root() is + * not a Struct. + * + * Example: + * + * Expression field = root()["age"]["nested"]; + */ + Expression operator[](std::string_view field) const; + + Expression is_null() const; + + /* + * Extract fields from a Struct. Output DataType is a Struct. + * + * Errors at scan/apply time if any of fields does not exist or if root() + * is not a Struct. + */ + Expression select(std::span names) const; + Expression select(std::span names) const; + Expression select(std::initializer_list names) const; + +private: + friend struct detail::Access; + explicit Expression(const vx_expression *owned) : handle_(owned) {} + const vx_expression *release() && noexcept { return handle_.release(); } + + struct Deleter { + void operator()(const vx_expression *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +// Add, Sub, Mul, and Div error in runtime on overflow and underflow +enum class BinaryOperator { + // x == y + Eq = VX_OPERATOR_EQ, + // x != y + NotEq = VX_OPERATOR_NOT_EQ, + // x > y + Gt = VX_OPERATOR_GT, + // x >= y + Gte = VX_OPERATOR_GTE, + // x < y + Lt = VX_OPERATOR_LT, + // x <= y + Lte = VX_OPERATOR_LTE, + // boolean x AND y, Kleene semantics + KleeneAnd = VX_OPERATOR_KLEENE_AND, + // boolean x OR y, Kleene semantics + KleeneOr = VX_OPERATOR_KLEENE_OR, + // x + y + Add = VX_OPERATOR_ADD, + // x - y + Sub = VX_OPERATOR_SUB, + // x * y + Mul = VX_OPERATOR_MUL, + // x / y + Div = VX_OPERATOR_DIV, +}; + +namespace expr { + +// scanned/applied array. +Expression root(); + +// root()'s named column. +Expression col(std::string_view name); + +// Literal expression +Expression lit(const Scalar &value); + +/* + * Literal expression. + * + * Literal's DataType must match column it's compared against, otherwise scan + * fails at runtime. No type coercion is performed. + */ +template +Expression lit(T value) { + return lit(scalar::of(value)); +} + +Expression eq(const Expression &l, const Expression &r); +Expression neq(const Expression &l, const Expression &r); +Expression lt(const Expression &l, const Expression &r); +Expression lte(const Expression &l, const Expression &r); +Expression gt(const Expression &l, const Expression &r); +Expression gte(const Expression &l, const Expression &r); +Expression add(const Expression &l, const Expression &r); +Expression sub(const Expression &l, const Expression &r); +Expression mul(const Expression &l, const Expression &r); +Expression div(const Expression &l, const Expression &r); +Expression binary_op(BinaryOperator op, const Expression &l, const Expression &r); + +// Kleene AND of children. Errors on an empty list. +Expression and_all(std::span children); +// Kleene OR of children. Errors on an empty list. +Expression or_all(std::span children); + +Expression logical_not(const Expression &child); +Expression is_null(const Expression &child); +Expression list_contains(const Expression &list, const Expression &value); + +/** + * Opt-in operator overloads like in Eigen. + * Note && and || don't short-circuit. + */ +namespace ops { + +inline Expression operator==(const Expression &l, const Expression &r) { return eq(l, r); } +inline Expression operator!=(const Expression &l, const Expression &r) { return neq(l, r); } +inline Expression operator<(const Expression &l, const Expression &r) { return lt(l, r); } +inline Expression operator<=(const Expression &l, const Expression &r) { return lte(l, r); } +inline Expression operator>(const Expression &l, const Expression &r) { return gt(l, r); } +inline Expression operator>=(const Expression &l, const Expression &r) { return gte(l, r); } +inline Expression operator&&(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::KleeneAnd, l, r); +} +inline Expression operator||(const Expression &l, const Expression &r) { + return binary_op(BinaryOperator::KleeneOr, l, r); +} +inline Expression operator!(const Expression &e) { return logical_not(e); } +inline Expression operator+(const Expression &l, const Expression &r) { return add(l, r); } +inline Expression operator-(const Expression &l, const Expression &r) { return sub(l, r); } +inline Expression operator*(const Expression &l, const Expression &r) { return mul(l, r); } +inline Expression operator/(const Expression &l, const Expression &r) { return div(l, r); } + +} // namespace ops +} // namespace expr +} // namespace vortex diff --git a/vortex-cxx/include/vortex/scalar.hpp b/vortex-cxx/include/vortex/scalar.hpp new file mode 100644 index 00000000000..92643faf2f3 --- /dev/null +++ b/vortex-cxx/include/vortex/scalar.hpp @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +// A single value with an associated DataType +class Scalar { +public: + Scalar(const Scalar &other); + Scalar(Scalar &&) noexcept = default; + Scalar &operator=(const Scalar &other); + Scalar &operator=(Scalar &&) noexcept = default; + + bool is_null() const; + DataType dtype() const; + +private: + friend struct detail::Access; + explicit Scalar(vx_scalar *owned) : handle_(owned) {} + vx_scalar *release() && noexcept { return handle_.release(); } + + struct Deleter { + void operator()(vx_scalar *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +namespace detail { +vx_scalar *make_bool(bool value, bool nullable); +vx_scalar *make_primitive(vx_ptype ptype, const void *value, bool nullable); +vx_scalar *make_utf8(std::string_view value, bool nullable); +vx_scalar *make_binary(BinaryView value, bool nullable); +Scalar adopt(vx_scalar *raw); +} // namespace detail + +namespace scalar { +/** + * A scalar of DataType selected by T: bool, primitive, string_view (utf8), + * or a BinaryView (binary). + * + * Bytes are copied for utf8 and binary scalars. + */ +template +Scalar of(T value, bool nullable = false) { + if constexpr (std::is_same_v) { + return detail::adopt(detail::make_bool(value, nullable)); + } else if constexpr (std::is_same_v) { + return detail::adopt(detail::make_utf8(value, nullable)); + } else if constexpr (std::is_same_v) { + return detail::adopt(detail::make_binary(value, nullable)); + } else { + return detail::adopt(detail::make_primitive(detail::to_ptype(), &value, nullable)); + } +} + +// A typed null of (a nullable copy of) a given DataType. +Scalar null(const DataType &dtype); + +Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable = false); +Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable = false); +} // namespace scalar +} // namespace vortex diff --git a/vortex-cxx/include/vortex/scan.hpp b/vortex-cxx/include/vortex/scan.hpp new file mode 100644 index 00000000000..898b553d2fe --- /dev/null +++ b/vortex-cxx/include/vortex/scan.hpp @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/estimate.hpp" +#include "vortex/expression.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace vortex { + +namespace detail { +// range-for support for Scan and Partition +template +class PullRange { +public: + class iterator { + public: + using value_type = Item; + using difference_type = std::ptrdiff_t; + + iterator() = default; + iterator(Source *src, std::optional first) : src_(src), cur_(std::move(first)) {} + + Item &operator*() const { return *cur_; } + iterator &operator++() { + cur_ = src_->pull(); + return *this; + } + void operator++(int) { ++*this; } + bool operator==(std::default_sentinel_t) const { return !cur_.has_value(); } + + private: + Source *src_ = nullptr; + mutable std::optional cur_; + }; + + explicit PullRange(Source &src) : src_(&src) {} + iterator begin() { return iterator(src_, src_->pull()); } + std::default_sentinel_t end() { return std::default_sentinel; } + +private: + Source *src_; +}; +} // namespace detail + +// Wrapper around ArrowArrayStream which releases stream in destructor +class ArrowStream { +public: + ArrowStream(const ArrowStream &) = delete; + ArrowStream &operator=(const ArrowStream &) = delete; + ArrowStream(ArrowStream &&other) noexcept; + ArrowStream &operator=(ArrowStream &&other) noexcept; + ~ArrowStream(); + + ArrowArrayStream *raw() noexcept { return &stream_; } + +private: + friend struct detail::Access; + ArrowStream(Session session, ArrowArrayStream stream) noexcept + : session_(std::move(session)), stream_(stream) {} + + Session session_; + ArrowArrayStream stream_ {}; +}; + +/** + * An independent unit of scan work. Single-threaded: drive each partition + * from one worker thread. + */ +class Partition { +public: + Partition(const Partition &) = delete; + Partition &operator=(const Partition &) = delete; + Partition(Partition &&) noexcept = default; + Partition &operator=(Partition &&) noexcept = default; + + // Estimated row count. Throws if called after next(). + Estimate row_count() const; + + // Next batch or nullopt when partition is exhausted. + std::optional next(); + + // range-for over batches + auto batches() { return detail::PullRange(*this); } + + // Consume the partition into an Arrow stream + // + // This function blocks until partition is drained. + ArrowStream into_arrow_stream() &&; + +private: + friend struct detail::Access; + friend class detail::PullRange; + Partition(vx_partition *owned, Session session) : handle_(owned), session_(std::move(session)) {} + + std::optional pull() { return next(); } + + struct Deleter { + void operator()(vx_partition *ptr) const noexcept; + }; + std::unique_ptr handle_; + Session session_; +}; + +struct RowRange { + uint64_t begin = 0; + uint64_t end = 0; +}; + +struct Selection { + enum class Kind { + Include = VX_SELECTION_INCLUDE_RANGE, + Exclude = VX_SELECTION_EXCLUDE_RANGE, + }; + Kind kind = Kind::Include; + std::vector indices; +}; + +/** + * Scan configuration. Fields are append-only and must be set via designated + * initializers. + * + * Example: + * + * DataSource ds = DataSource::open(session, "file.vortex", dtype); + * Scan scan = ds.scan({.limit = 100}); + */ +struct ScanOptions { + std::optional projection; + std::optional filter; + std::optional row_range; + // Row-index filter applied after row_range. + std::optional selection; + // Maximum number of rows to return. 0 = no limit. + uint64_t limit = 0; + // If true, return rows in storage order. + bool ordered = false; +}; + +/** + * A single traversal of a DataSource. A scan can be consumed only once. + * + * next_partition() is internally synchronized, give each partition to its own + * worker thread. + */ +class Scan { +public: + Scan(const Scan &) = delete; + Scan &operator=(const Scan &) = delete; + Scan(Scan &&) noexcept = default; + Scan &operator=(Scan &&) noexcept = default; + + Estimate partition_count() const noexcept { return estimate_; } + + // Throws if called after next_partition(). + DataType dtype() const; + + // Next partition or nullopt when the scan is exhausted. Thread-safe. + std::optional next_partition(); + + // range-for over partitions + auto partitions() { return detail::PullRange(*this); } + +private: + friend struct detail::Access; + friend class detail::PullRange; + Scan(vx_scan *owned, Estimate estimate, Session session) + : handle_(owned), mutex_(std::make_unique()), estimate_(estimate), + session_(std::move(session)) {} + + std::optional pull() { return next_partition(); } + + struct Deleter { + void operator()(vx_scan *ptr) const noexcept; + }; + std::unique_ptr handle_; + std::unique_ptr mutex_; + Estimate estimate_; + Session session_; +}; +} // namespace vortex diff --git a/vortex-cxx/include/vortex/session.hpp b/vortex-cxx/include/vortex/session.hpp new file mode 100644 index 00000000000..29522e7fd74 --- /dev/null +++ b/vortex-cxx/include/vortex/session.hpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/common.hpp" + +#include + +#include + +namespace vortex { + +/** + * A handle to a Vortex session, registry of encodings and compute kernels. + * Copying shares the underlying session. + */ +class Session { +public: + Session(); + + Session(const Session &other); + Session(Session &&) noexcept = default; + Session &operator=(const Session &other); + Session &operator=(Session &&) noexcept = default; + +private: + friend struct detail::Access; + + struct Deleter { + void operator()(vx_session *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; + +} // namespace vortex diff --git a/vortex-cxx/include/vortex/writer.hpp b/vortex-cxx/include/vortex/writer.hpp new file mode 100644 index 00000000000..c1b9c223592 --- /dev/null +++ b/vortex-cxx/include/vortex/writer.hpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/array.hpp" +#include "vortex/dtype.hpp" +#include "vortex/session.hpp" + +#include + +#include +#include + +namespace vortex { + +/** + * Writes arrays into a Vortex file. + * + * finish() writes the footer and finalizes the file. + * Not calling finish() leaves file corrupted. + */ +class Writer { +public: + static Writer open(const Session &session, std::string_view path, const DataType &dtype); + + Writer(const Writer &) = delete; + Writer &operator=(const Writer &) = delete; + Writer(Writer &&) noexcept = default; + Writer &operator=(Writer &&) noexcept = default; + + // Append a chunk. Chunk's DataType must match writer's DataType. + void push(const Array &chunk); + + /* + * Write footer and finalize the file. + * Throws on failure. Writer is closed afterwards and further uses throws. + */ + void finish(); + +private: + explicit Writer(vx_array_sink *sink) : handle_(sink) {} + + struct Deleter { + void operator()(vx_array_sink *ptr) const noexcept; + }; + std::unique_ptr handle_; +}; +} // namespace vortex diff --git a/vortex-cxx/src/array.cpp b/vortex-cxx/src/array.cpp new file mode 100644 index 00000000000..c11b58c2ca8 --- /dev/null +++ b/vortex-cxx/src/array.cpp @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; + +Validity::Validity(const Validity &other) + : type_(other.type_), array_(other.array_ != nullptr ? vx_array_clone(other.array_) : nullptr) {} + +Validity::Validity(ValidityType type, const Array &array) + : type_(type), array_(vx_array_clone(Access::c_ptr(array))) {} + +Validity::Validity(Validity &&other) noexcept : type_(other.type_), array_(other.array_) { + other.array_ = nullptr; + other.type_ = ValidityType::NonNullable; +} + +Validity &Validity::operator=(const Validity &other) { + if (this != &other) { + *this = Validity(other); + } + return *this; +} + +Validity &Validity::operator=(Validity &&other) noexcept { + if (this != &other) { + if (array_ != nullptr) { + vx_array_free(array_); + } + type_ = other.type_; + array_ = other.array_; + other.array_ = nullptr; + other.type_ = ValidityType::NonNullable; + } + return *this; +} + +Validity::~Validity() { + if (array_ != nullptr) { + vx_array_free(array_); + } +} + +Array Validity::array() const { + if (type_ != ValidityType::Array || array_ == nullptr) { + throw VortexException("validity has no backing array", ErrorCode::InvalidArgument); + } + return Access::adopt(vx_array_clone(array_)); +} + +Validity ValidityArray(const Array &bools) { + return Validity(ValidityType::Array, vx_array_clone(Access::c_ptr(bools))); +} + +bool detail::ValidityBits::is_null(size_t index) const noexcept { + if (all_invalid_) { + return true; + } + if (bits_ == nullptr) { + return false; + } + const size_t bit = bit_offset_ + index; + return (bits_[bit / 8] >> (bit % 8) & 1) == 0; +} + +namespace detail { +ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) { + vx_validity raw {}; + vx_error *error = nullptr; + vx_array_get_validity(canonical, &raw, &error); + throw_on_error(error); + + switch (static_cast(raw.type)) { + case ValidityType::NonNullable: + case ValidityType::AllValid: + return; + case ValidityType::AllInvalid: + all_invalid_ = true; + return; + case ValidityType::Array: + break; + } + + owner_ = vx_array_canonicalize(Access::c_ptr(session), raw.array, &error); + vx_array_free(raw.array); + throw_on_error(error); + + bits_ = static_cast(vx_array_data_ptr_bool(owner_, &bit_offset_, &error)); + throw_on_error(error); +} + +ValidityBits::ValidityBits(ValidityBits &&other) noexcept + : owner_(other.owner_), bits_(other.bits_), bit_offset_(other.bit_offset_), + all_invalid_(other.all_invalid_) { + other.owner_ = nullptr; + other.bits_ = nullptr; +} + +ValidityBits &ValidityBits::operator=(ValidityBits &&other) noexcept { + if (this != &other) { + if (owner_ != nullptr) { + vx_array_free(owner_); + } + owner_ = other.owner_; + bits_ = other.bits_; + bit_offset_ = other.bit_offset_; + all_invalid_ = other.all_invalid_; + other.owner_ = nullptr; + other.bits_ = nullptr; + } + return *this; +} + +ValidityBits::~ValidityBits() { + if (owner_ != nullptr) { + vx_array_free(owner_); + } +} +} // namespace detail + +static const vx_struct_fields *struct_fields_or_throw(const vx_dtype *dtype) { + const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); + if (fields == nullptr) { + throw VortexException("dtype is not a struct", ErrorCode::MismatchedTypes); + } + return fields; +} + +void Array::Deleter::operator()(const vx_array *ptr) const noexcept { vx_array_free(ptr); } + +Array::Array(const Array &other) : handle_(vx_array_clone(other.handle_.get())) {} + +Array &Array::operator=(const Array &other) { + if (this != &other) { + handle_.reset(vx_array_clone(other.handle_.get())); + } + return *this; +} + +Array Array::null(size_t len) { return Access::adopt(vx_array_new_null(len)); } + +Array Array::primitive_raw(vx_ptype ptype, const void *data, size_t len, const Validity &validity) { + std::optional keep_alive; + vx_validity raw {}; + raw.type = static_cast(validity.type()); + if (validity.type() == ValidityType::Array) { + keep_alive = validity.array(); + raw.array = Access::c_ptr(*keep_alive); + } + + vx_error *error = nullptr; + const vx_array *out = vx_array_new_primitive(ptype, data, len, &raw, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::from_arrow(ArrowArray *array, ArrowSchema *schema, bool nullable) { + vx_error *error = nullptr; + const vx_array *out = vx_array_from_arrow(array, schema, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +size_t Array::size() const { return vx_array_len(handle_.get()); } + +bool Array::nullable() const { return vx_array_is_nullable(handle_.get()); } + +bool Array::has_dtype(DataTypeVariant v) const { + return vx_array_has_dtype(handle_.get(), static_cast(v)); +} + +bool Array::is_primitive(PType p) const { + return vx_array_is_primitive(handle_.get(), static_cast(p)); +} + +DataType Array::dtype() const { return Access::adopt(vx_array_dtype(handle_.get())); } + +Validity Array::validity() const { + vx_validity raw {}; + vx_error *error = nullptr; + vx_array_get_validity(handle_.get(), &raw, &error); + throw_on_error(error); + return Access::adopt(static_cast(raw.type), raw.array); +} + +size_t Array::null_count() const { + vx_error *error = nullptr; + const size_t count = vx_array_invalid_count(handle_.get(), &error); + throw_on_error(error); + return count; +} + +Array Array::field(size_t index) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_get_field(handle_.get(), index, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::field(std::string_view name) const { + const DataType dt = dtype(); + const std::unique_ptr fields( + struct_fields_or_throw(detail::Access::c_ptr(dt)), + &vx_struct_fields_free); + const uint64_t fields_size = vx_struct_fields_nfields(fields.get()); + for (uint64_t i = 0; i < fields_size; ++i) { + const vx_view field = vx_struct_fields_field_name(fields.get(), i); + if (std::string_view {field.ptr, field.len} == name) { + return this->field(i); + } + } + throw VortexException("no field named \"" + std::string(name) + "\"", ErrorCode::InvalidArgument); +} + +Array Array::slice(size_t begin, size_t end) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_slice(handle_.get(), begin, end, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::apply(const Expression &expr) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_apply(handle_.get(), Access::c_ptr(expr), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array Array::canonicalize(const Session &session) const { + vx_error *error = nullptr; + const vx_array *out = vx_array_canonicalize(Access::c_ptr(session), handle_.get(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Array make_struct(std::initializer_list> fields, + const Validity &validity) { + StructArrayBuilder builder(validity, fields.size()); + for (const auto &[name, field] : fields) { + builder.add(name, field); + } + return std::move(builder).build(); +} + +void StructArrayBuilder::Deleter::operator()(vx_struct_column_builder *ptr) const noexcept { + vx_struct_column_builder_free(ptr); +} + +StructArrayBuilder::StructArrayBuilder(const Validity &validity, size_t capacity) { + std::optional keep_alive; + vx_validity raw {}; + raw.type = static_cast(validity.type()); + if (validity.type() == ValidityType::Array) { + keep_alive = validity.array(); + raw.array = Access::c_ptr(*keep_alive); + } + handle_.reset(vx_struct_column_builder_new(&raw, capacity)); +} + +StructArrayBuilder &StructArrayBuilder::add(std::string_view name, const Array &field) & { + vx_error *error = nullptr; + vx_struct_column_builder_add_field(handle_.get(), detail::to_view(name), Access::c_ptr(field), &error); + throw_on_error(error); + return *this; +} + +StructArrayBuilder &&StructArrayBuilder::add(std::string_view name, const Array &field) && { + add(name, field); + return std::move(*this); +} + +Array StructArrayBuilder::build() && { + vx_error *error = nullptr; + const vx_array *out = vx_struct_column_builder_finalize(handle_.release(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +PrimitiveView Array::bools(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_BOOL)) { + throw VortexException("bools(): array is not a Bool array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return PrimitiveView(std::move(canonical), std::move(validity), len); +} + +StringView Array::strings(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_UTF8)) { + throw VortexException("strings(): array is not a Utf8 array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return StringView(std::move(canonical), std::move(validity), len); +} + +BytesView Array::bytes(const Session &session) const { + Array canonical = canonicalize(session); + const vx_array *raw = Access::c_ptr(canonical); + if (!vx_array_has_dtype(raw, DTYPE_BINARY)) { + throw VortexException("bytes(): array is not a Binary array", ErrorCode::MismatchedTypes); + } + detail::ValidityBits validity(session, raw); + const size_t len = vx_array_len(raw); + return BytesView(std::move(canonical), std::move(validity), len); +} + +bool PrimitiveView::value(size_t i) const { return vx_array_get_bool(Access::c_ptr(canonical_), i); } + +std::string_view StringView::operator[](size_t i) const { + vx_error *error = nullptr; + const vx_view out = vx_array_utf8_at(Access::c_ptr(canonical_), i, &error); + throw_on_error(error); + return {out.ptr, out.len}; +} + +BinaryView BytesView::operator[](size_t i) const { + vx_error *error = nullptr; + const vx_view out = vx_array_binary_at(Access::c_ptr(canonical_), i, &error); + throw_on_error(error); + return {reinterpret_cast(out.ptr), out.len}; +} +} // namespace vortex diff --git a/vortex-cxx/src/data_source.cpp b/vortex-cxx/src/data_source.cpp new file mode 100644 index 00000000000..e77fe60da6c --- /dev/null +++ b/vortex-cxx/src/data_source.cpp @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/data_source.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void DataSource::Deleter::operator()(const vx_data_source *ptr) const noexcept { vx_data_source_free(ptr); } + +DataSource::DataSource(const DataSource &other) + : handle_(vx_data_source_clone(other.handle_.get())), session_(other.session_) {} + +DataSource &DataSource::operator=(const DataSource &other) { + if (this != &other) { + handle_.reset(vx_data_source_clone(other.handle_.get())); + session_ = other.session_; + } + return *this; +} + +template +static DataSource open_paths(const Session &session, std::span paths) { + std::vector raw; + raw.reserve(paths.size()); + for (const auto &path : paths) { + raw.push_back(to_view(path)); + } + + vx_data_source_options options {}; + options.paths = raw.data(); + options.paths_len = raw.size(); + + vx_error *error = nullptr; + const vx_data_source *ds = vx_data_source_new(Access::c_ptr(session), &options, &error); + throw_on_error(error); + return Access::adopt(ds, session); +} + +DataSource DataSource::open(const Session &session, std::initializer_list paths) { + std::span span {paths.begin(), paths.end()}; + return open_paths(session, span); +} + +DataSource DataSource::open(const Session &session, std::span paths) { + return open_paths(session, paths); +} + +DataSource DataSource::open(const Session &session, std::span paths) { + return open_paths(session, paths); +} + +DataSource DataSource::from_buffer(const Session &session, std::span data) { + vx_error *error = nullptr; + const vx_data_source *ds = + vx_data_source_new_buffer(Access::c_ptr(session), data.data(), data.size(), &error); + throw_on_error(error); + return Access::adopt(ds, session); +} + +Estimate DataSource::row_count() const { + vx_estimate raw {}; + vx_data_source_get_row_count(handle_.get(), &raw); + return Estimate(raw); +} + +DataType DataSource::dtype() const { return Access::adopt(vx_data_source_dtype(handle_.get())); } + +Scan DataSource::scan(ScanOptions options) const { + vx_scan_options raw {}; + raw.projection = options.projection.has_value() ? Access::c_ptr(*options.projection) : nullptr; + raw.filter = options.filter.has_value() ? Access::c_ptr(*options.filter) : nullptr; + if (options.row_range.has_value()) { + raw.row_range_begin = options.row_range->begin; + raw.row_range_end = options.row_range->end; + } + if (options.selection.has_value()) { + raw.selection.idx = options.selection->indices.data(); + raw.selection.idx_len = options.selection->indices.size(); + raw.selection.include = static_cast(options.selection->kind); + } else { + raw.selection.include = VX_SELECTION_INCLUDE_ALL; + } + raw.limit = options.limit; + raw.ordered = options.ordered; + + vx_estimate estimate {}; + vx_error *error = nullptr; + vx_scan *scan = vx_data_source_scan(handle_.get(), &raw, &estimate, &error); + throw_on_error(error); + return Access::adopt(scan, Estimate(estimate), session_); +} + +} // namespace vortex diff --git a/vortex-cxx/src/dtype.cpp b/vortex-cxx/src/dtype.cpp new file mode 100644 index 00000000000..95c0102187d --- /dev/null +++ b/vortex-cxx/src/dtype.cpp @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void DataType::Deleter::operator()(const vx_dtype *ptr) const noexcept { vx_dtype_free(ptr); } + +DataType::DataType(const vx_dtype *owned) : handle_(owned) {} + +DataType::DataType(const DataType &other) : handle_(vx_dtype_clone(other.handle_.get())) {} + +DataType &DataType::operator=(const DataType &other) { + if (this != &other) { + handle_.reset(vx_dtype_clone(other.handle_.get())); + } + return *this; +} + +DataType DataType::from_arrow(ArrowSchema *schema) { + vx_error *error = nullptr; + const vx_dtype *dtype = vx_dtype_from_arrow_schema(schema, &error); + throw_on_error(error); + return DataType(dtype); +} + +ArrowSchema DataType::to_arrow() const { + ArrowSchema schema {}; + vx_error *error = nullptr; + vx_dtype_to_arrow_schema(handle_.get(), &schema, &error); + throw_on_error(error); + return schema; +} + +DataTypeVariant DataType::variant() const { + return static_cast(vx_dtype_get_variant(handle_.get())); +} + +bool DataType::nullable() const { return vx_dtype_is_nullable(handle_.get()); } + +PType DataType::primitive_type() const { return static_cast(vx_dtype_primitive_ptype(handle_.get())); } + +uint8_t DataType::decimal_precision() const { return vx_dtype_decimal_precision(handle_.get()); } + +int8_t DataType::decimal_scale() const { return vx_dtype_decimal_scale(handle_.get()); } + +namespace { +const vx_struct_fields *struct_fields_or_throw(const vx_dtype *dtype) { + const vx_struct_fields *fields = vx_dtype_struct_dtype(dtype); + if (fields == nullptr) { + throw VortexException("dtype is not a struct", ErrorCode::MismatchedTypes); + } + return fields; +} +} // namespace + +std::vector DataType::fields() const { + const std::unique_ptr fields( + struct_fields_or_throw(handle_.get()), + &vx_struct_fields_free); + const uint64_t fields_size = vx_struct_fields_nfields(fields.get()); + std::vector out; + out.reserve(fields_size); + for (uint64_t idx = 0; idx < fields_size; ++idx) { + const vx_view name = vx_struct_fields_field_name(fields.get(), idx); + const vx_dtype *dtype = vx_struct_fields_field_dtype(fields.get(), idx); + out.push_back(Field {std::string {name.ptr, name.len}, DataType(dtype)}); + } + return out; +} + +DataType DataType::list_element() const { + const vx_dtype *element = vx_dtype_list_element(handle_.get()); + if (element == nullptr) { + throw VortexException("dtype is not a list", ErrorCode::MismatchedTypes); + } + return DataType(element); +} + +DataType DataType::fixed_size_list_element() const { + const vx_dtype *element = vx_dtype_fixed_size_list_element(handle_.get()); + if (element == nullptr) { + throw VortexException("dtype is not a fixed-size list", ErrorCode::MismatchedTypes); + } + return DataType(element); +} + +uint32_t DataType::fixed_size_list_size() const { return vx_dtype_fixed_size_list_size(handle_.get()); } + +namespace dtype { + +DataType null() { return Access::adopt(vx_dtype_new_null()); } +DataType boolean(bool nullable) { return Access::adopt(vx_dtype_new_bool(nullable)); } +DataType primitive(PType ptype, bool nullable) { + return Access::adopt(vx_dtype_new_primitive(static_cast(ptype), nullable)); +} +DataType int8(bool nullable) { return primitive(PType::I8, nullable); } +DataType int16(bool nullable) { return primitive(PType::I16, nullable); } +DataType int32(bool nullable) { return primitive(PType::I32, nullable); } +DataType int64(bool nullable) { return primitive(PType::I64, nullable); } +DataType uint8(bool nullable) { return primitive(PType::U8, nullable); } +DataType uint16(bool nullable) { return primitive(PType::U16, nullable); } +DataType uint32(bool nullable) { return primitive(PType::U32, nullable); } +DataType uint64(bool nullable) { return primitive(PType::U64, nullable); } +DataType float16(bool nullable) { return primitive(PType::F16, nullable); } +DataType float32(bool nullable) { return primitive(PType::F32, nullable); } +DataType float64(bool nullable) { return primitive(PType::F64, nullable); } +DataType utf8(bool nullable) { return Access::adopt(vx_dtype_new_utf8(nullable)); } +DataType binary(bool nullable) { return Access::adopt(vx_dtype_new_binary(nullable)); } +DataType decimal(uint8_t precision, int8_t scale, bool nullable) { + return Access::adopt(vx_dtype_new_decimal(precision, scale, nullable)); +} +DataType list(DataType element, bool nullable) { + return Access::adopt(vx_dtype_new_list(Access::release(std::move(element)), nullable)); +} +DataType fixed_size_list(DataType element, uint32_t size, bool nullable) { + return Access::adopt( + vx_dtype_new_fixed_size_list(Access::release(std::move(element)), size, nullable)); +} + +DataType struct_(std::initializer_list> fields, bool nullable) { + StructFieldsBuilder builder; + for (const auto &[name, field_dtype] : fields) { + builder.add(name, field_dtype); + } + return std::move(builder).build(nullable); +} + +} // namespace dtype + +void StructFieldsBuilder::Deleter::operator()(vx_struct_fields_builder *ptr) const noexcept { + vx_struct_fields_builder_free(ptr); +} + +StructFieldsBuilder::StructFieldsBuilder() : handle_(vx_struct_fields_builder_new()) {} + +StructFieldsBuilder &StructFieldsBuilder::add(std::string_view name, const DataType &dtype) & { + vx_error *error = nullptr; + vx_struct_fields_builder_add_field(handle_.get(), + to_view(name), + vx_dtype_clone(Access::c_ptr(dtype)), + &error); + throw_on_error(error); + return *this; +} + +StructFieldsBuilder &&StructFieldsBuilder::add(std::string_view name, const DataType &dtype) && { + add(name, dtype); + return std::move(*this); +} + +DataType StructFieldsBuilder::build(bool nullable) && { + vx_struct_fields *fields = vx_struct_fields_builder_finalize(handle_.release()); + return Access::adopt(vx_dtype_new_struct(fields, nullable)); +} + +} // namespace vortex diff --git a/vortex-cxx/src/dtype.rs b/vortex-cxx/src/dtype.rs deleted file mode 100644 index 0c125bcf822..00000000000 --- a/vortex-cxx/src/dtype.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; - -use anyhow::Result; -use arrow_array::ffi::FFI_ArrowSchema; -use arrow_schema::Field; -use vortex::dtype::DType as RustDType; -use vortex::dtype::DecimalDType; -use vortex::dtype::Nullability; -use vortex::dtype::PType as RustPType; -use vortex::dtype::arrow::FromArrowType; - -use crate::ffi; -pub(crate) struct DType { - pub(crate) inner: RustDType, -} - -pub(crate) fn dtype_null() -> Box { - Box::new(DType { - inner: RustDType::Null, - }) -} - -pub(crate) fn dtype_bool(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Bool(nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_primitive(ptype: ffi::PType, nullable: bool) -> Box { - let vortex_ptype = match ptype { - ffi::PType::U8 => RustPType::U8, - ffi::PType::U16 => RustPType::U16, - ffi::PType::U32 => RustPType::U32, - ffi::PType::U64 => RustPType::U64, - ffi::PType::I8 => RustPType::I8, - ffi::PType::I16 => RustPType::I16, - ffi::PType::I32 => RustPType::I32, - ffi::PType::I64 => RustPType::I64, - ffi::PType::F16 => RustPType::F16, - ffi::PType::F32 => RustPType::F32, - ffi::PType::F64 => RustPType::F64, - _ => unreachable!(), - }; - Box::new(DType { - inner: RustDType::Primitive(vortex_ptype, nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_decimal(precision: u8, scale: i8, nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Decimal( - DecimalDType::new(precision, scale), - nullability_from_bool(nullable), - ), - }) -} - -pub(crate) fn dtype_utf8(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Utf8(nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_binary(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Binary(nullability_from_bool(nullable)), - }) -} - -impl Display for DType { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{0}", self.inner) - } -} - -pub(crate) fn nullability_from_bool(nullable: bool) -> Nullability { - if nullable { - Nullability::Nullable - } else { - Nullability::NonNullable - } -} - -pub(crate) unsafe fn from_arrow(ffi_schema: *mut u8, non_nullable: bool) -> Result> { - let arrow_schema = unsafe { FFI_ArrowSchema::from_raw(ffi_schema as *mut FFI_ArrowSchema) }; - let arrow_dtype = arrow_schema::DataType::try_from(&arrow_schema)?; - Ok(Box::new(DType { - inner: RustDType::from_arrow(&Field::new("_", arrow_dtype, !non_nullable)), - })) -} diff --git a/vortex-cxx/src/expr.rs b/vortex-cxx/src/expr.rs deleted file mode 100644 index b980db7111b..00000000000 --- a/vortex-cxx/src/expr.rs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex::dtype::FieldName; -use vortex::expr::Expression; - -use crate::scalar::Scalar; - -pub(crate) struct Expr { - pub(crate) inner: Expression, -} - -pub(crate) fn literal(scalar: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::lit(scalar.inner), - }) -} - -pub(crate) fn root() -> Box { - Box::new(Expr { - inner: vortex::expr::root(), - }) -} - -pub(crate) fn column(name: String) -> Box { - Box::new(Expr { - inner: vortex::expr::get_item(name, vortex::expr::root()), - }) -} - -pub(crate) fn get_item(field: String, child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::get_item(field, child.inner), - }) -} - -pub(crate) fn not_(child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::not(child.inner), - }) -} - -pub(crate) fn is_null(child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::is_null(child.inner), - }) -} - -macro_rules! binary_op { - ($fn_name:ident $(, $suffix:tt)?) => { - paste::paste! { - pub(crate) fn [<$fn_name $($suffix)?>]( - lhs: Box, - rhs: Box, - ) -> Box { - Box::new(Expr { - inner: vortex::expr::$fn_name(lhs.inner, rhs.inner), - }) - } - } - }; -} - -binary_op!(eq); -binary_op!(not_eq, _); -binary_op!(gt); -binary_op!(gt_eq); -binary_op!(lt); -binary_op!(lt_eq); -binary_op!(and, _); -binary_op!(or, _); -binary_op!(checked_add); - -pub(crate) fn select(fields: Vec, child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::select( - fields.into_iter().map(FieldName::from).collect::>(), - child.inner, - ), - }) -} diff --git a/vortex-cxx/src/expression.cpp b/vortex-cxx/src/expression.cpp new file mode 100644 index 00000000000..4cbc769595f --- /dev/null +++ b/vortex-cxx/src/expression.cpp @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/error.hpp" +#include "vortex/expression.hpp" + +#include + +#include +#include +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Expression::Deleter::operator()(const vx_expression *ptr) const noexcept { vx_expression_free(ptr); } + +Expression::Expression(const Expression &other) : handle_(vx_expression_clone(other.handle_.get())) {} + +Expression &Expression::operator=(const Expression &other) { + if (this != &other) { + handle_.reset(vx_expression_clone(other.handle_.get())); + } + return *this; +} + +Expression Expression::operator[](std::string_view field) const { + vx_expression *out = vx_expression_get_item(to_view(field), handle_.get()); + if (out == nullptr) { + throw VortexException("get_item: field name is not valid UTF-8", ErrorCode::InvalidArgument); + } + return Access::adopt(out); +} + +Expression Expression::is_null() const { return expr::is_null(*this); } + +template +static Expression select_impl(std::span names, const vx_expression *expr) { + std::vector raw; + raw.reserve(names.size()); + for (const auto &name : names) { + raw.push_back(to_view(name)); + } + return Access::adopt(vx_expression_select(raw.data(), raw.size(), expr)); +} + +Expression Expression::select(std::span names) const { + return select_impl(names, handle_.get()); +} + +Expression Expression::select(std::span names) const { + return select_impl(names, handle_.get()); +} + +Expression Expression::select(std::initializer_list names) const { + std::span span {names.begin(), names.end()}; + return select_impl(span, handle_.get()); +} + +namespace expr { + +Expression root() { return Access::adopt(vx_expression_root()); } + +Expression col(std::string_view name) { return root()[name]; } + +Expression lit(const Scalar &value) { + vx_error *error = nullptr; + vx_expression *out = vx_expression_literal(Access::c_ptr(value), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Expression binary_op(BinaryOperator op, const Expression &l, const Expression &r) { + return Access::adopt( + vx_expression_binary(static_cast(op), Access::c_ptr(l), Access::c_ptr(r))); +} + +Expression eq(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Eq, l, r); } +Expression neq(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::NotEq, l, r); } +Expression lt(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Lt, l, r); } +Expression lte(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Lte, l, r); } +Expression gt(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Gt, l, r); } +Expression gte(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Gte, l, r); } +Expression add(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Add, l, r); } +Expression sub(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Sub, l, r); } +Expression mul(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Mul, l, r); } +Expression div(const Expression &l, const Expression &r) { return binary_op(BinaryOperator::Div, l, r); } + +static Expression combine(vx_expression *(*combiner)(const vx_expression *const *, size_t), + std::span children) { + std::vector raw; + raw.reserve(children.size()); + for (const auto &child : children) { + raw.push_back(Access::c_ptr(child)); + } + vx_expression *out = combiner(raw.data(), raw.size()); + if (out == nullptr) { + throw VortexException("empty expression list", ErrorCode::InvalidArgument); + } + return Access::adopt(out); +} + +Expression and_all(std::span children) { return combine(vx_expression_and, children); } + +Expression or_all(std::span children) { return combine(vx_expression_or, children); } + +Expression logical_not(const Expression &child) { + return Access::adopt(vx_expression_not(Access::c_ptr(child))); +} + +Expression is_null(const Expression &child) { + return Access::adopt(vx_expression_is_null(Access::c_ptr(child))); +} + +Expression list_contains(const Expression &list, const Expression &value) { + return Access::adopt(vx_expression_list_contains(Access::c_ptr(list), Access::c_ptr(value))); +} + +} // namespace expr + +} // namespace vortex diff --git a/vortex-cxx/src/f16.cpp b/vortex-cxx/src/f16.cpp new file mode 100644 index 00000000000..c62a79b27e7 --- /dev/null +++ b/vortex-cxx/src/f16.cpp @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" + +#include +#include + +namespace vortex { + +#if __STDCPP_FLOAT64_T__ != 1 +float to_float(float16_t f) { + const uint32_t sign = (f.bits >> 15) & 1; + const uint32_t exponent = (f.bits >> 10) & 0x1F; + const uint32_t mantissa = f.bits & 0x3FF; + + uint32_t out; + if (exponent == 0x1F) { + out = (sign << 31) | 0x7F800000 | (mantissa << 13); + } else if (exponent == 0) { + if (mantissa == 0) { + out = sign << 31; + } else { + // Renormalize a subnormal half into a normal float. + uint32_t m = mantissa; + int e = -1; + do { + m <<= 1; + ++e; + } while ((m & 0x400) == 0); + out = (sign << 31) | ((127 - 15 - e) << 23) | ((m & 0x3FF) << 13); + } + } else { + out = (sign << 31) | ((exponent - 15 + 127) << 23) | (mantissa << 13); + } + float result; + std::memcpy(&result, &out, sizeof(result)); + return result; +} +#endif +} // namespace vortex diff --git a/vortex-cxx/src/lib.rs b/vortex-cxx/src/lib.rs deleted file mode 100644 index b6d002513f6..00000000000 --- a/vortex-cxx/src/lib.rs +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#![allow(clippy::boxed_local)] - -mod dtype; -mod expr; -mod read; -mod scalar; -mod write; - -use std::sync::LazyLock; - -use dtype::*; -use expr::*; -use read::*; -use scalar::*; -use vortex::VortexSessionDefault; -use vortex::io::runtime::BlockingRuntime; -use vortex::io::runtime::current::CurrentThreadRuntime; -use vortex::io::session::RuntimeSessionExt; -use vortex::session::VortexSession; -use write::*; - -/// By default, the C++ API uses a current-thread runtime, providing control of the threading -/// model to the C++ side. -// TODO(ngates): in the future, we could expose an API for C++ to spawn threads that can drive -// this runtime. -pub(crate) static RUNTIME: LazyLock = - LazyLock::new(CurrentThreadRuntime::new); -pub(crate) static SESSION: LazyLock = - LazyLock::new(|| VortexSession::default().with_handle(RUNTIME.handle())); - -#[cxx::bridge(namespace = "vortex::ffi")] -#[allow(let_underscore_drop)] -#[allow(clippy::absolute_paths)] -mod ffi { - extern "Rust" { - type DType; - // Factory functions for creating DType - fn dtype_null() -> Box; - fn dtype_bool(nullable: bool) -> Box; - fn dtype_primitive(ptype: PType, nullable: bool) -> Box; - fn dtype_decimal(precision: u8, scale: i8, nullable: bool) -> Box; - fn dtype_utf8(nullable: bool) -> Box; - fn dtype_binary(nullable: bool) -> Box; - unsafe fn from_arrow(ffi_schema: *mut u8, non_nullable: bool) -> Result>; - // Methods for DType - fn to_string(self: &DType) -> String; - - type Scalar; - fn bool_scalar_new(value: bool) -> Box; - fn i8_scalar_new(value: i8) -> Box; - fn i16_scalar_new(value: i16) -> Box; - fn i32_scalar_new(value: i32) -> Box; - fn i64_scalar_new(value: i64) -> Box; - fn u8_scalar_new(value: u8) -> Box; - fn u16_scalar_new(value: u16) -> Box; - fn u32_scalar_new(value: u32) -> Box; - fn u64_scalar_new(value: u64) -> Box; - fn f32_scalar_new(value: f32) -> Box; - fn f64_scalar_new(value: f64) -> Box; - fn string_scalar_new(value: &str) -> Box; - fn binary_scalar_new(value: &[u8]) -> Box; - fn cast_scalar(self: &Scalar, dtype: &DType) -> Result>; - - type Expr; - fn literal(scalar: Box) -> Box; - fn root() -> Box; - fn column(name: String) -> Box; - fn get_item(field: String, child: Box) -> Box; - fn not_(child: Box) -> Box; - fn is_null(child: Box) -> Box; - // binary op - fn eq(lhs: Box, rhs: Box) -> Box; - fn not_eq_(lhs: Box, rhs: Box) -> Box; - fn gt(lhs: Box, rhs: Box) -> Box; - fn gt_eq(lhs: Box, rhs: Box) -> Box; - fn lt(lhs: Box, rhs: Box) -> Box; - fn lt_eq(lhs: Box, rhs: Box) -> Box; - fn and_(lhs: Box, rhs: Box) -> Box; - fn or_(lhs: Box, rhs: Box) -> Box; - fn checked_add(lhs: Box, rhs: Box) -> Box; - fn select(fields: Vec, child: Box) -> Box; - - type VortexFile; - fn row_count(self: &VortexFile) -> u64; - fn scan_builder(self: &VortexFile) -> Result>; - fn open_file(path: &str) -> Result>; - fn open_file_from_buffer(data: &[u8]) -> Result>; - - type VortexScanBuilder; - fn with_filter(self: &mut VortexScanBuilder, filter: Box); - fn with_filter_ref(self: &mut VortexScanBuilder, filter: &Expr); - fn with_projection(self: &mut VortexScanBuilder, projection: Box); - fn with_projection_ref(self: &mut VortexScanBuilder, projection: &Expr); - fn with_row_range(self: &mut VortexScanBuilder, row_range_start: u64, row_range_end: u64); - fn with_include_by_index(self: &mut VortexScanBuilder, include_by_index: &[u64]); - fn with_limit(self: &mut VortexScanBuilder, limit: usize); - unsafe fn with_output_schema( - self: &mut VortexScanBuilder, - output_schema: *mut u8, - ) -> Result<()>; - unsafe fn scan_builder_into_stream( - builder: Box, - out_stream: *mut u8, - ) -> Result<()>; - fn scan_builder_into_threadsafe_cloneable_reader( - builder: Box, - ) -> Result>; - - type ThreadsafeCloneableReader; - unsafe fn clone_a_stream(self: &ThreadsafeCloneableReader, out_stream: *mut u8); - - type VortexWriteOptions; - fn write_options_new() -> Box; - unsafe fn write_array_stream( - options: Box, - input_stream: *mut u8, - path: &str, - ) -> Result<()>; - } - - #[repr(u8)] - #[derive(Debug, Clone, Copy)] - enum PType { - U8, - U16, - U32, - U64, - I8, - I16, - I32, - I64, - F16, - F32, - F64, - } -} diff --git a/vortex-cxx/src/read.rs b/vortex-cxx/src/read.rs deleted file mode 100644 index 110e42d70f0..00000000000 --- a/vortex-cxx/src/read.rs +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::sync::Arc; - -use anyhow::Result; -use arrow_array::RecordBatch; -use arrow_array::RecordBatchReader; -use arrow_array::cast::AsArray; -use arrow_array::ffi::FFI_ArrowSchema; -use arrow_array::ffi_stream::FFI_ArrowArrayStream; -use arrow_schema::ArrowError; -use arrow_schema::Field; -use arrow_schema::Schema; -use arrow_schema::SchemaRef; -use futures::stream::TryStreamExt; -use vortex::array::ArrayRef; -use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; -use vortex::buffer::Buffer; -use vortex::file::OpenOptionsSessionExt; -use vortex::io::runtime::BlockingRuntime; -use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; -use vortex::layout::scan::scan_builder::ScanBuilder; -use vortex::scan::selection::Selection; - -use crate::RUNTIME; -use crate::SESSION; -use crate::expr::Expr; - -pub(crate) struct VortexFile { - inner: vortex::file::VortexFile, -} - -impl VortexFile { - pub(crate) fn row_count(&self) -> u64 { - self.inner.row_count() - } - - pub(crate) fn scan_builder(&self) -> Result> { - Ok(Box::new(VortexScanBuilder { - inner: self.inner.scan()?, - output_schema: None, - })) - } -} - -/// File operations - using blocking operations for simplicity -/// TODO(xinyu): object store (see vortex-ffi) -pub(crate) fn open_file(path: &str) -> Result> { - let file = RUNTIME.block_on(SESSION.open_options().open_path(std::path::Path::new(path)))?; - Ok(Box::new(VortexFile { inner: file })) -} - -pub(crate) fn open_file_from_buffer(data: &[u8]) -> Result> { - let buffer = Buffer::from(data.to_vec()); - let file = SESSION.open_options().open_buffer(buffer)?; - Ok(Box::new(VortexFile { inner: file })) -} - -pub(crate) struct VortexScanBuilder { - inner: ScanBuilder, - output_schema: Option, -} - -impl VortexScanBuilder { - pub(crate) fn with_filter(&mut self, filter: Box) { - take_mut::take(&mut self.inner, |inner| inner.with_filter(filter.inner)); - } - - pub(crate) fn with_filter_ref(&mut self, filter: &Expr) { - take_mut::take(&mut self.inner, |inner| { - inner.with_filter(filter.inner.clone()) - }); - } - - pub(crate) fn with_projection(&mut self, filter: Box) { - take_mut::take(&mut self.inner, |inner| inner.with_projection(filter.inner)); - } - - pub(crate) fn with_projection_ref(&mut self, filter: &Expr) { - take_mut::take(&mut self.inner, |inner| { - inner.with_projection(filter.inner.clone()) - }); - } - - pub(crate) fn with_row_range(&mut self, row_range_start: u64, row_range_end: u64) { - take_mut::take(&mut self.inner, |inner| { - inner.with_row_range(row_range_start..row_range_end) - }); - } - - pub(crate) fn with_include_by_index(&mut self, include_by_index: &[u64]) { - let selection = Selection::IncludeByIndex(Buffer::copy_from(include_by_index)); - take_mut::take(&mut self.inner, |inner| inner.with_selection(selection)); - } - - pub(crate) fn with_limit(&mut self, limit: usize) { - take_mut::take(&mut self.inner, |inner| inner.with_limit(limit as u64)); - } - - pub(crate) unsafe fn with_output_schema(&mut self, output_schema: *mut u8) -> Result<()> { - let ffi_schema = - unsafe { FFI_ArrowSchema::from_raw(output_schema as *mut FFI_ArrowSchema) }; - self.output_schema = Some(Arc::new(Schema::try_from(&ffi_schema)?)); - Ok(()) - } -} - -/// # Safety -/// -/// out_stream should be properly aligned according to the Arrow C stream interface and valid for write. -pub(crate) unsafe fn scan_builder_into_stream( - builder: Box, - out_stream: *mut u8, -) -> Result<()> { - let schema = match builder.output_schema { - Some(schema) => schema, - None => { - let dtype = builder.inner.dtype()?; - let arrow_schema = dtype.to_arrow_schema()?; - Arc::new(arrow_schema) - } - }; - let reader = builder.inner.into_record_batch_reader(schema, &*RUNTIME)?; - let stream = FFI_ArrowArrayStream::new(Box::new(reader)); - let out_stream = out_stream as *mut FFI_ArrowArrayStream; - // # Safety - // Arrow C stream interface - unsafe { std::ptr::write(out_stream, stream) }; - Ok(()) -} - -trait ThreadsafeCloneableReaderTrait: RecordBatchReader + Send + 'static { - fn clone_boxed(&self) -> Box; -} - -impl ThreadsafeCloneableReaderTrait for T -where - T: RecordBatchReader + Send + Clone + 'static, -{ - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } -} - -pub(crate) struct ThreadsafeCloneableReader { - inner: Box, -} - -#[allow(clippy::disallowed_methods)] -pub(crate) fn scan_builder_into_threadsafe_cloneable_reader( - builder: Box, -) -> Result, Box> { - let schema = match builder.output_schema { - Some(schema) => schema, - None => { - let dtype = builder.inner.dtype()?; - let arrow_schema = dtype.to_arrow_schema()?; - Arc::new(arrow_schema) - } - }; - let target = Field::new_struct("", schema.fields.clone(), false); - - let stream = builder - .inner - .map(move |b| { - SESSION - .arrow() - .execute_arrow(b, Some(&target), &mut SESSION.create_execution_ctx()) - .map(|struct_array| RecordBatch::from(struct_array.as_struct())) - }) - .into_stream()? - .map_err(|e| ArrowError::ExternalError(Box::new(e))); - - let iter = RUNTIME.block_on_stream_thread_safe(|_h| stream); - let rbr = RecordBatchIteratorAdapter::new(iter, schema); - - Ok(Box::new(ThreadsafeCloneableReader { - inner: Box::new(rbr), - })) -} - -impl ThreadsafeCloneableReader { - pub(crate) fn clone_a_stream(&self, out_stream: *mut u8) { - let cloned_reader = self.inner.clone_boxed(); - let stream = FFI_ArrowArrayStream::new(cloned_reader); - let out_stream = out_stream as *mut FFI_ArrowArrayStream; - // # Safety - // Arrow C stream interface - unsafe { std::ptr::write(out_stream, stream) }; - } -} diff --git a/vortex-cxx/src/scalar.cpp b/vortex-cxx/src/scalar.cpp new file mode 100644 index 00000000000..64eecbf03fc --- /dev/null +++ b/vortex-cxx/src/scalar.cpp @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/common.hpp" +#include "vortex/error.hpp" +#include "vortex/scalar.hpp" + +#include + +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Scalar::Deleter::operator()(vx_scalar *ptr) const noexcept { vx_scalar_free(ptr); } + +Scalar::Scalar(const Scalar &other) : handle_(vx_scalar_clone(other.handle_.get())) {} + +Scalar &Scalar::operator=(const Scalar &other) { + if (this != &other) { + handle_.reset(vx_scalar_clone(other.handle_.get())); + } + return *this; +} + +bool Scalar::is_null() const { return vx_scalar_is_null(handle_.get()); } + +DataType Scalar::dtype() const { return Access::adopt(vx_scalar_dtype(handle_.get())); } + +namespace detail { + +Scalar adopt(vx_scalar *raw) { return Access::adopt(raw); } + +vx_scalar *make_bool(bool value, bool nullable) { return vx_scalar_new_bool(value, nullable); } + +vx_scalar *make_primitive(vx_ptype ptype, const void *value, bool nullable) { + switch (ptype) { + case PTYPE_U8: + return vx_scalar_new_u8(*static_cast(value), nullable); + case PTYPE_U16: + return vx_scalar_new_u16(*static_cast(value), nullable); + case PTYPE_U32: + return vx_scalar_new_u32(*static_cast(value), nullable); + case PTYPE_U64: + return vx_scalar_new_u64(*static_cast(value), nullable); + case PTYPE_I8: + return vx_scalar_new_i8(*static_cast(value), nullable); + case PTYPE_I16: + return vx_scalar_new_i16(*static_cast(value), nullable); + case PTYPE_I32: + return vx_scalar_new_i32(*static_cast(value), nullable); + case PTYPE_I64: + return vx_scalar_new_i64(*static_cast(value), nullable); +#if __STDCPP_FLOAT64_T__ != 1 + case PTYPE_F16: + return vx_scalar_new_f16_bits(static_cast(value)->bits, nullable); +#else + case PTYPE_F16: + return vx_scalar_new_f16_bits(static_cast(value), nullable); +#endif + case PTYPE_F32: + return vx_scalar_new_f32(*static_cast(value), nullable); + case PTYPE_F64: + return vx_scalar_new_f64(*static_cast(value), nullable); + } + throw VortexException("unsupported ptype", ErrorCode::InvalidArgument); +} + +vx_scalar *make_utf8(std::string_view value, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_utf8(to_view(value), nullable, &error); + throw_on_error(error); + return out; +} + +vx_scalar *make_binary(BinaryView value, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = + vx_scalar_new_binary(reinterpret_cast(value.data()), value.size(), nullable, &error); + throw_on_error(error); + return out; +} + +} // namespace detail + +namespace scalar { +Scalar null(const DataType &dtype) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_null(Access::c_ptr(dtype), &error); + throw_on_error(error); + return Access::adopt(out); +} + +Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_decimal_i32(value, precision, scale, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool nullable) { + vx_error *error = nullptr; + vx_scalar *out = vx_scalar_new_decimal_i64(value, precision, scale, nullable, &error); + throw_on_error(error); + return Access::adopt(out); +} + +} // namespace scalar + +} // namespace vortex diff --git a/vortex-cxx/src/scalar.rs b/vortex-cxx/src/scalar.rs deleted file mode 100644 index 24cd38be0f1..00000000000 --- a/vortex-cxx/src/scalar.rs +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use anyhow::Result; - -use crate::dtype::DType; - -pub(crate) struct Scalar { - pub(crate) inner: vortex::scalar::Scalar, -} - -macro_rules! primitive_scalar_new { - ($name:ident, $type:ty) => { - pub(crate) fn $name(value: $type) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) - } - }; -} - -primitive_scalar_new!(bool_scalar_new, bool); // bool is not primitive but reuse the macro here -primitive_scalar_new!(i8_scalar_new, i8); -primitive_scalar_new!(i16_scalar_new, i16); -primitive_scalar_new!(i32_scalar_new, i32); -primitive_scalar_new!(i64_scalar_new, i64); -primitive_scalar_new!(u8_scalar_new, u8); -primitive_scalar_new!(u16_scalar_new, u16); -primitive_scalar_new!(u32_scalar_new, u32); -primitive_scalar_new!(u64_scalar_new, u64); -primitive_scalar_new!(f32_scalar_new, f32); -primitive_scalar_new!(f64_scalar_new, f64); - -pub(crate) fn string_scalar_new(value: &str) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) -} - -pub(crate) fn binary_scalar_new(value: &[u8]) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) -} - -impl Scalar { - pub(crate) fn cast_scalar(&self, dtype: &DType) -> Result> { - Ok(Box::new(Scalar { - inner: self.inner.cast(&dtype.inner)?, - })) - } -} diff --git a/vortex-cxx/src/scan.cpp b/vortex-cxx/src/scan.cpp new file mode 100644 index 00000000000..381f3db8cd0 --- /dev/null +++ b/vortex-cxx/src/scan.cpp @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/scan.hpp" + +#include + +#include +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; + +ArrowStream::ArrowStream(ArrowStream &&other) noexcept + : session_(std::move(other.session_)), stream_(other.stream_) { + other.stream_ = ArrowArrayStream {}; +} + +ArrowStream &ArrowStream::operator=(ArrowStream &&other) noexcept { + if (this != &other) { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } + session_ = std::move(other.session_); + stream_ = other.stream_; + other.stream_ = ArrowArrayStream {}; + } + return *this; +} + +ArrowStream::~ArrowStream() { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } +} + +void Partition::Deleter::operator()(vx_partition *ptr) const noexcept { vx_partition_free(ptr); } + +Estimate Partition::row_count() const { + vx_estimate raw {}; + vx_error *error = nullptr; + vx_partition_row_count(handle_.get(), &raw, &error); + throw_on_error(error); + return Estimate(raw); +} + +std::optional Partition::next() { + vx_error *error = nullptr; + const vx_array *array = vx_partition_next(handle_.get(), &error); + throw_on_error(error); + if (array == nullptr) { + return std::nullopt; + } + return Access::adopt(array); +} + +ArrowStream Partition::into_arrow_stream() && { + ArrowArrayStream stream {}; + vx_error *error = nullptr; + // Consumes handle even on error + vx_partition_scan_arrow(Access::c_ptr(session_), handle_.release(), &stream, &error); + throw_on_error(error); + return Access::adopt(std::move(session_), stream); +} + +void Scan::Deleter::operator()(vx_scan *ptr) const noexcept { vx_scan_free(ptr); } + +DataType Scan::dtype() const { + vx_error *error = nullptr; + const vx_dtype *out = vx_scan_dtype(handle_.get(), &error); + throw_on_error(error); + return Access::adopt(out); +} + +std::optional Scan::next_partition() { + vx_partition *partition = nullptr; + { + const std::lock_guard guard(*mutex_); + vx_error *error = nullptr; + partition = vx_scan_next_partition(handle_.get(), &error); + throw_on_error(error); + } + if (partition == nullptr) { + return std::nullopt; + } + return Access::adopt(partition, session_); +} +} // namespace vortex diff --git a/vortex-cxx/src/session.cpp b/vortex-cxx/src/session.cpp new file mode 100644 index 00000000000..ffa06de149a --- /dev/null +++ b/vortex-cxx/src/session.cpp @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/session.hpp" + +#include + +namespace vortex { + +void Session::Deleter::operator()(vx_session *ptr) const noexcept { vx_session_free(ptr); } + +Session::Session() : handle_(vx_session_new()) {} + +Session::Session(const Session &other) : handle_(vx_session_clone(other.handle_.get())) {} + +Session &Session::operator=(const Session &other) { + if (this != &other) { + handle_.reset(vx_session_clone(other.handle_.get())); + } + return *this; +} + +} // namespace vortex diff --git a/vortex-cxx/src/write.rs b/vortex-cxx/src/write.rs deleted file mode 100644 index ff77c5a3dc5..00000000000 --- a/vortex-cxx/src/write.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use anyhow::Result; -use arrow_array::RecordBatchReader; -use arrow_array::ffi_stream::ArrowArrayStreamReader; -use arrow_array::ffi_stream::FFI_ArrowArrayStream; -use vortex::array::ArrayRef; -use vortex::array::arrow::FromArrowArray; -use vortex::array::iter::ArrayIteratorAdapter; -use vortex::array::iter::ArrayIteratorExt; -use vortex::array::stream::ArrayStream; -use vortex::dtype::DType; -use vortex::dtype::arrow::FromArrowType; -use vortex::error::VortexError; -use vortex::file::VortexWriteOptions as WriteOptions; -use vortex::file::WriteOptionsSessionExt; -use vortex::io::VortexWrite; -use vortex::io::runtime::BlockingRuntime; - -use crate::RUNTIME; -use crate::SESSION; - -pub(crate) struct VortexWriteOptions { - inner: WriteOptions, -} - -pub(crate) fn write_options_new() -> Box { - Box::new(VortexWriteOptions { - inner: SESSION.write_options(), - }) -} - -/// Convert an ArrowArrayStreamReader to a Vortex ArrayStream -fn arrow_stream_to_vortex_stream(reader: ArrowArrayStreamReader) -> Result { - let array_iter = ArrayIteratorAdapter::new( - DType::from_arrow(reader.schema()), - reader.map(|result| { - result - .map_err(VortexError::from) - .and_then(|record_batch| ArrayRef::from_arrow(record_batch, false)) - }), - ); - - Ok(array_iter.into_array_stream()) -} - -/// # Safety -/// -/// input_stream should be valid FFI_ArrowArrayStream. -/// See [`FFI_ArrowArrayStream::from_raw`] -pub(crate) unsafe fn write_array_stream( - options: Box, - input_stream: *mut u8, - path: &str, -) -> Result<()> { - let path = path.to_string(); - - let stream_reader = - unsafe { ArrowArrayStreamReader::from_raw(input_stream as *mut FFI_ArrowArrayStream) }?; - - let vortex_stream = arrow_stream_to_vortex_stream(stream_reader)?; - - RUNTIME.block_on(async { - let mut file = async_fs::File::create(path).await?; - options.inner.write(&mut file, vortex_stream).await?; - file.shutdown().await?; - Ok(()) - }) -} diff --git a/vortex-cxx/src/writer.cpp b/vortex-cxx/src/writer.cpp new file mode 100644 index 00000000000..5212a6fbd10 --- /dev/null +++ b/vortex-cxx/src/writer.cpp @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/array.hpp" +#include "vortex/common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/writer.hpp" +#include "vortex/session.hpp" + +#include + +#include + +namespace vortex { + +using detail::Access; +using detail::throw_on_error; +using detail::to_view; + +void Writer::Deleter::operator()(vx_array_sink *ptr) const noexcept { vx_array_sink_abort(ptr); } + +Writer Writer::open(const Session &session, std::string_view path, const DataType &dtype) { + vx_error *error = nullptr; + vx_array_sink *sink = + vx_array_sink_open_file(Access::c_ptr(session), to_view(path), Access::c_ptr(dtype), &error); + throw_on_error(error); + return Writer(sink); +} + +void Writer::push(const Array &chunk) { + if (handle_ == nullptr) { + throw VortexException("push() after finish()", ErrorCode::InvalidArgument); + } + vx_error *error = nullptr; + vx_array_sink_push(handle_.get(), Access::c_ptr(chunk), &error); + throw_on_error(error); +} + +void Writer::finish() { + if (handle_ == nullptr) { + throw VortexException("finish() called twice", ErrorCode::InvalidArgument); + } + vx_error *error = nullptr; + vx_array_sink_close(handle_.release(), &error); + throw_on_error(error); +} +} // namespace vortex diff --git a/vortex-cxx/tests/.clang-tidy b/vortex-cxx/tests/.clang-tidy new file mode 100644 index 00000000000..ffb9e2daf91 --- /dev/null +++ b/vortex-cxx/tests/.clang-tidy @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +InheritParentConfig: true +Checks: | + -google-build-using-namespace, + -performance-unnecessary-copy-initialization, + -bugprone-unchecked-optional-access, diff --git a/vortex-cxx/tests/CMakeLists.txt b/vortex-cxx/tests/CMakeLists.txt new file mode 100644 index 00000000000..c9465609677 --- /dev/null +++ b/vortex-cxx/tests/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors +FetchContent_Declare( + magic_enum + GIT_REPOSITORY https://github.com/Neargye/magic_enum.git + GIT_TAG v0.9.7 +) +FetchContent_MakeAvailable(magic_enum) + +file(GLOB TEST_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +add_executable(vortex_cxx_test ${TEST_FILES}) +target_include_directories(vortex_cxx_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(vortex_cxx_test PRIVATE + vortex_cxx_shared + Catch2::Catch2WithMain + magic_enum::magic_enum + nanoarrow_shared) + +if(APPLE) + target_link_libraries(vortex_cxx_test PRIVATE + "-framework CoreFoundation" "-framework Security") +endif() + +catch_discover_tests(vortex_cxx_test) diff --git a/vortex-cxx/tests/common.hpp b/vortex-cxx/tests/common.hpp new file mode 100644 index 00000000000..be1fc24922e --- /dev/null +++ b/vortex-cxx/tests/common.hpp @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace vortex_test { + +namespace fs = std::filesystem; +using namespace vortex; + +class TempPath { +public: + TempPath() = default; + explicit TempPath(fs::path p) : path_(std::move(p)) {} + + TempPath(const TempPath &) = delete; + TempPath &operator=(const TempPath &) = delete; + + TempPath(TempPath &&other) noexcept : path_(std::move(other.path_)) { other.path_.clear(); } + TempPath &operator=(TempPath &&other) noexcept { + if (this != &other) { + reset(); + path_ = std::move(other.path_); + other.path_.clear(); + } + return *this; + } + + ~TempPath() { reset(); } + + const fs::path &path() const noexcept { return path_; } + std::string string() const { return path_.string(); } + + static TempPath unique() { + auto dir = fs::temp_directory_path() / "vortex_cxx_test"; + fs::create_directories(dir); + std::string name = std::to_string(std::random_device {}()) + ".vortex"; + return TempPath {dir / name}; + } + +private: + void reset() noexcept { + if (!path_.empty()) { + std::error_code ec; + fs::remove(path_, ec); + } + } + + fs::path path_; +}; + +inline DataType sample_dtype() { + return dtype::struct_({ + {"age", dtype::uint8()}, + {"height", dtype::uint16(dtype::Nullable)}, + }); +} + +constexpr size_t SAMPLE_ROWS = 100; + +inline std::vector sample_ages() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast(i); + } + return buf; +} + +inline std::vector sample_heights() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast((i + 1) % 200); + } + return buf; +} + +inline Array sample_array() { + auto ages = sample_ages(); + auto heights = sample_heights(); + return make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights, AllValid)}, + }); +} + +inline TempPath write_sample(const Session &session) { + TempPath path = TempPath::unique(); + Writer writer = Writer::open(session, path.string(), sample_dtype()); + writer.push(sample_array()); + writer.finish(); + return path; +} +} // namespace vortex_test diff --git a/vortex-cxx/tests/test_array.cpp b/vortex-cxx/tests/test_array.cpp new file mode 100644 index 00000000000..d672d8af54e --- /dev/null +++ b/vortex-cxx/tests/test_array.cpp @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +using namespace vortex; +using namespace vortex::expr::ops; + +namespace { +using enum vortex::PType; + +TEST_CASE("Null array", "[array]") { + Array a = Array::null(1999); + REQUIRE(a.size() == 1999); + REQUIRE(a.nullable()); + REQUIRE(a.has_dtype(DataTypeVariant::Null)); +} + +TEST_CASE("Primitive array and values view", "[array]") { + Session session; + + int32_t c_array[3] = {10, 20, 30}; + auto b = Array::primitive(c_array); + + const int32_t const_c_array[3] = {10, 20, 30}; + auto c = Array::primitive(const_c_array); + + const std::array cpp_array = {10, 20, 30}; + auto d = Array::primitive(cpp_array); + + std::vector data = {10, 20, 30}; + Array a = Array::primitive(data); + + REQUIRE(a.size() == 3); + REQUIRE(a.is_primitive(I32)); + REQUIRE_FALSE(a.nullable()); + REQUIRE(a.null_count() == 0); + + auto view = a.values(session); + REQUIRE(view.size() == 3); + REQUIRE(std::equal(view.values().begin(), view.values().end(), data.begin())); + REQUIRE_FALSE(view.is_null(1)); +} + +TEST_CASE("values with wrong type", "[array]") { + Session session; + std::vector data = {1}; + Array a = Array::primitive(data); + REQUIRE_THROWS_AS(a.values(session), VortexException); +} + +TEST_CASE("Validity from a boolean mask", "[array]") { + Session session; + std::vector data = {10, 20, 30}; + std::vector mask_bytes = {1, 0, 1}; + + Array mask_u8 = Array::primitive(std::span(mask_bytes)); + Array mask = mask_u8.apply(expr::root() == expr::lit(1)); + + Array a = Array::primitive(std::span(data), ValidityArray(mask)); + REQUIRE(a.nullable()); + REQUIRE(a.null_count() == 1); + + auto view = a.values(session); + REQUIRE_FALSE(view.is_null(0)); + REQUIRE(view.is_null(1)); + REQUIRE_FALSE(view.is_null(2)); + REQUIRE(view.values()[0] == 10); + REQUIRE(view.values()[2] == 30); + + Validity validity = a.validity(); + REQUIRE(validity.type() == ValidityType::Array); + REQUIRE(validity.array().size() == 3); +} + +TEST_CASE("AllInvalid", "[array]") { + Session session; + std::vector data = {1, 2}; + Array a = Array::primitive(std::span(data), AllInvalid); + REQUIRE(a.null_count() == 2); + auto view = a.values(session); + REQUIRE(view.is_null(0)); + REQUIRE(view.is_null(1)); +} + +TEST_CASE("make_struct and fields", "[array]") { + Session session; + std::vector ages = {10, 20, 30}; + std::vector heights = {150, 160, 170}; + + Array s = make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights, AllValid)}, + }); + + REQUIRE(s.size() == 3); + REQUIRE(s.has_dtype(DataTypeVariant::Struct)); + REQUIRE(s.dtype().fields().size() == 2); + + Array by_index = s.field(0); + REQUIRE(by_index.is_primitive(U8)); + + Array by_name = s.field("height"); + REQUIRE(by_name.is_primitive(U16)); + auto view = by_name.values(session); + REQUIRE(view.values()[2] == 170); + + REQUIRE_THROWS_AS(s.field(2), VortexException); + REQUIRE_THROWS_AS(s.field("nope"), VortexException); +} + +TEST_CASE("StructArrayBuilder", "[array]") { + std::vector ages = {1, 2}; + StructArrayBuilder builder(NonNullable, 1); + builder.add("age", Array::primitive(ages)); + Array s = std::move(builder).build(); + REQUIRE(s.size() == 2); + REQUIRE(s.has_dtype(DataTypeVariant::Struct)); +} + +TEST_CASE("Mismatched field length", "[array]") { + std::vector a = {1, 2}; + std::vector b = {1, 2, 3}; + REQUIRE_THROWS_AS(make_struct({ + {"a", Array::primitive(a)}, + {"b", Array::primitive(b)}, + }), + VortexException); +} + +TEST_CASE("Slice", "[array]") { + Session session; + std::vector data = {0, 1, 2, 3, 4, 5}; + Array a = Array::primitive(data); + Array sliced = a.slice(2, 5); + REQUIRE(sliced.size() == 3); + auto view = sliced.values(session); + REQUIRE(view.values()[0] == 2); + REQUIRE(view.values()[2] == 4); + + REQUIRE_THROWS_AS(a.slice(2, 100), VortexException); +} + +TEST_CASE("Error with a code", "[array]") { + std::vector data = {0}; + Array a = Array::primitive(data); + try { + (void)a.slice(2, 100); + FAIL("expected exception"); + } catch (const VortexException &e) { + REQUIRE_FALSE(std::string(e.what()).empty()); + (void)e.code(); + } +} + +TEST_CASE("float16_t to_float", "[array]") { + REQUIRE(to_float(float16_t {0x3C00}) == 1.0F); + REQUIRE(to_float(float16_t {0xC000}) == -2.0F); + REQUIRE(to_float(float16_t {0}) == 0.0F); +} +} // namespace diff --git a/vortex-cxx/tests/test_arrow.cpp b/vortex-cxx/tests/test_arrow.cpp new file mode 100644 index 00000000000..d127fe45c8c --- /dev/null +++ b/vortex-cxx/tests/test_arrow.cpp @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include + +#include "common.hpp" + +using namespace vortex; +using vortex_test::sample_dtype; +using vortex_test::TempPath; +using vortex_test::write_sample; + +namespace { +using enum vortex::PType; + +TEST_CASE("dtype to ArrowSchema", "[arrow]") { + DataType d = sample_dtype(); + ArrowSchema schema = d.to_arrow(); + + nanoarrow::UniqueSchema unique_schema; + ArrowSchemaMove(&schema, unique_schema.get()); + REQUIRE(unique_schema->format != nullptr); + REQUIRE(unique_schema->n_children == 2); +} + +TEST_CASE("dtype from ArrowSchema", "[arrow]") { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 1) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT64) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "n") == NANOARROW_OK); + + ArrowSchema raw = {}; + ArrowSchemaMove(schema.get(), &raw); + DataType d = DataType::from_arrow(&raw); + REQUIRE(d.variant() == DataTypeVariant::Struct); + const std::vector fields = d.fields(); + REQUIRE(fields.size() == 1); + REQUIRE(fields[0].name == "n"); + REQUIRE(fields[0].dtype.primitive_type() == I64); +} + +TEST_CASE("Import Arrow array as Vortex array", "[arrow]") { + Session session; + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 1) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT32) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "a") == NANOARROW_OK); + + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (int i : {10, 20, 30}) { + REQUIRE(ArrowArrayAppendInt(arr->children[0], i) == NANOARROW_OK); + REQUIRE(ArrowArrayFinishElement(arr.get()) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + + Array vx = Array::from_arrow(&raw_arr, &raw_schema, false); + REQUIRE(vx.size() == 3); + REQUIRE(vx.has_dtype(DataTypeVariant::Struct)); + + Array a = vx.field(0); + REQUIRE(a.is_primitive(I32)); + auto view = a.values(session); + REQUIRE(view.values()[0] == 10); + REQUIRE(view.values()[2] == 30); +} + +TEST_CASE("Scan partition to ArrowArrayStream", "[arrow]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + ArrowStream vx_stream = std::move(partition.value()).into_arrow_stream(); + nanoarrow::UniqueArrayStream owned; + ArrowArrayStreamMove(vx_stream.raw(), owned.get()); + + nanoarrow::UniqueSchema schema; + ArrowError err {}; + REQUIRE(ArrowArrayStreamGetSchema(owned.get(), schema.get(), &err) == NANOARROW_OK); + REQUIRE(schema->n_children == 2); + + size_t rows = 0; + while (true) { + nanoarrow::UniqueArray chunk; + int rc = owned->get_next(owned.get(), chunk.get()); + REQUIRE(rc == NANOARROW_OK); + if (chunk->release == nullptr) { + break; + } + rows += chunk->length; + } + REQUIRE(rows == vortex_test::SAMPLE_ROWS); +} +} // namespace diff --git a/vortex-cxx/tests/test_dtype.cpp b/vortex-cxx/tests/test_dtype.cpp new file mode 100644 index 00000000000..6a5d5f97081 --- /dev/null +++ b/vortex-cxx/tests/test_dtype.cpp @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +using namespace vortex; + +namespace { +using enum vortex::PType; + +TEST_CASE("Null dtype", "[dtype]") { + auto d = dtype::null(); + REQUIRE(d.variant() == DataTypeVariant::Null); + REQUIRE(d.nullable()); +} + +TEST_CASE("Decimal dtype", "[dtype]") { + auto d = dtype::decimal(5, 2, false); + REQUIRE(d.variant() == DataTypeVariant::Decimal); + REQUIRE(d.decimal_precision() == 5); + REQUIRE(d.decimal_scale() == 2); + REQUIRE_FALSE(d.nullable()); + + REQUIRE_THROWS_AS(d.fields(), VortexException); + REQUIRE_THROWS_AS(d.list_element(), VortexException); +} + +TEST_CASE("copy dtype", "[dtype]") { + auto d = dtype::int32(true); + DataType d2 = d; + REQUIRE(d2.variant() == DataTypeVariant::Primitive); + REQUIRE(d2.primitive_type() == I32); + REQUIRE(d2.nullable()); + REQUIRE(d.variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("list dtype", "[dtype]") { + auto d = dtype::list(dtype::float64(), dtype::Nullable); + REQUIRE(d.variant() == DataTypeVariant::List); + REQUIRE(d.nullable()); + REQUIRE(d.list_element().primitive_type() == F64); + + auto fsl = dtype::fixed_size_list(dtype::int16(), 4); + REQUIRE(fsl.variant() == DataTypeVariant::FixedSizeList); + REQUIRE(fsl.fixed_size_list_size() == 4); + REQUIRE(fsl.fixed_size_list_element().primitive_type() == I16); +} + +TEST_CASE("struct dtype from initializer", "[dtype]") { + DataType d = dtype::struct_({ + {"col1", dtype::uint8()}, + {"col2", dtype::binary(dtype::Nullable)}, + }); + + REQUIRE(d.variant() == DataTypeVariant::Struct); + REQUIRE_FALSE(d.nullable()); + const std::vector fields = d.fields(); + REQUIRE(fields.size() == 2); + REQUIRE(fields[0].name == "col1"); + REQUIRE(fields[1].name == "col2"); + REQUIRE(fields[0].dtype.primitive_type() == U8); + REQUIRE(fields[1].dtype.variant() == DataTypeVariant::Binary); + REQUIRE(fields[1].dtype.nullable()); +} + +TEST_CASE("Struct fields builder", "[dtype]") { + StructFieldsBuilder b; + b.add("col1", dtype::uint8()); + b.add("col2", dtype::utf8()); + DataType d = std::move(b).build(dtype::Nullable); + + REQUIRE(d.variant() == DataTypeVariant::Struct); + REQUIRE(d.nullable()); + const std::vector built = d.fields(); + REQUIRE(built.size() == 2); + REQUIRE(built[1].name == "col2"); +} + +TEST_CASE("Non-UTF8 field name throws", "[dtype]") { + StructFieldsBuilder b; + REQUIRE_THROWS_AS(b.add("\xFF\xFE", dtype::uint8()), VortexException); +} +} // namespace diff --git a/vortex-cxx/tests/test_enum_sizes.cpp b/vortex-cxx/tests/test_enum_sizes.cpp new file mode 100644 index 00000000000..a100fa3b583 --- /dev/null +++ b/vortex-cxx/tests/test_enum_sizes.cpp @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +using magic_enum::enum_count; + +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); diff --git a/vortex-cxx/tests/test_expression.cpp b/vortex-cxx/tests/test_expression.cpp new file mode 100644 index 00000000000..1a55364df02 --- /dev/null +++ b/vortex-cxx/tests/test_expression.cpp @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +#include +#include +#include + +using namespace vortex; + +namespace { +using enum vortex::PType; + +TEST_CASE("Expression copy", "[expr]") { + Expression clone = expr::root(); + { + Expression orig = expr::col("age"); + clone = orig; + } + Expression another = clone; + (void)another; + SUCCEED(); +} + +TEST_CASE("Apply root()", "[expr]") { + Session session; + std::vector data = {10, 20, 30, 40, 50}; + Array array = Array::primitive(data); + + Array applied = array.apply(expr::root()); + + REQUIRE(applied.size() == data.size()); + REQUIRE(applied.is_primitive(I32)); + auto values = applied.values(session); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(values.values()[i] == data[i]); + } +} + +TEST_CASE("Apply projection", "[expr]") { + Session session; + std::vector ages = {10, 20, 30}; + std::vector heights = {150, 160, 170}; + + Array struct_arr = make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights)}, + }); + + Array projected = struct_arr.apply(expr::col("age")); + REQUIRE(projected.size() == ages.size()); + REQUIRE(projected.is_primitive(U8)); + auto values = projected.values(session); + for (size_t i = 0; i < ages.size(); ++i) { + REQUIRE(values.values()[i] == ages[i]); + } +} + +TEST_CASE("Apply arithmetic", "[expr]") { + Session session; + std::vector data = {1, 2, 3, 4, 5}; + Array array = Array::primitive(data); + + Expression e = expr::add(expr::root(), expr::lit(10)); + Array applied = array.apply(e); + + REQUIRE(applied.is_primitive(I32)); + auto values = applied.values(session); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(values.values()[i] == data[i] + 10); + } +} + +TEST_CASE("Operator overloading", "[expr]") { + using namespace vortex::expr::ops; + Session session; + std::vector data = {1, 2, 2, 7}; + Array array = Array::primitive(data); + + Array applied = array.apply(expr::root() == expr::lit(2)); + auto bits = applied.bools(session); + REQUIRE(bits.size() == data.size()); + REQUIRE_FALSE(bits.value(0)); + REQUIRE(bits.value(1)); + REQUIRE(bits.value(2)); + REQUIRE_FALSE(bits.value(3)); +} + +TEST_CASE("Apply error", "[expr]") { + std::vector data = {1, 2, 3}; + Array array = Array::primitive(data); + + Expression bad = expr::add(expr::root(), expr::lit(1)); + REQUIRE_THROWS_AS(array.apply(bad), VortexException); +} + +TEST_CASE("Empty conjunction", "[expr]") { + REQUIRE_THROWS_AS(expr::and_all({}), VortexException); + REQUIRE_THROWS_AS(expr::or_all({}), VortexException); +} +} // namespace diff --git a/vortex-cxx/tests/test_scalar.cpp b/vortex-cxx/tests/test_scalar.cpp new file mode 100644 index 00000000000..9decd8e7f42 --- /dev/null +++ b/vortex-cxx/tests/test_scalar.cpp @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex/error.hpp" +#include +#include +#include + +using namespace vortex; +using namespace std::string_view_literals; + +namespace { +using enum vortex::PType; + +TEST_CASE("Boolean scalar", "[scalar]") { + Scalar s = scalar::of(true); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Bool); +} + +TEST_CASE("Integer scalars", "[scalar]") { + REQUIRE(scalar::of(42).dtype().primitive_type() == U8); + REQUIRE(scalar::of(42).dtype().primitive_type() == U16); + REQUIRE(scalar::of(42).dtype().primitive_type() == U32); + REQUIRE(scalar::of(42).dtype().primitive_type() == U64); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I8); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I16); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I32); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I64); +} + +TEST_CASE("Float scalars", "[scalar]") { + REQUIRE(scalar::of(1.5F).dtype().primitive_type() == F32); + REQUIRE(scalar::of(1.5).dtype().primitive_type() == F64); + REQUIRE(scalar::of(float16_t {0x3C00}).dtype().primitive_type() == F16); +} + +TEST_CASE("Nullable scalars", "[scalar]") { + Scalar s = scalar::of(0, true); + REQUIRE(s.dtype().nullable()); + REQUIRE_FALSE(s.is_null()); +} + +TEST_CASE("Null scalar", "[scalar]") { + Scalar s = scalar::null(dtype::int32(true)); + REQUIRE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("UTF-8 scalar", "[scalar]") { + Scalar s = scalar::of("hello"sv); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); + + REQUIRE_FALSE(scalar::of(""sv).is_null()); + s = scalar::of("Широкая строка"sv); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); + + REQUIRE_THROWS_AS(scalar::of("\xFF\xFE"sv), VortexException); +} + +TEST_CASE("Binary scalar", "[scalar]") { + const std::byte bytes[] = {std::byte {1}, std::byte {2}, std::byte {0}, std::byte {4}}; + Scalar s = scalar::of(std::span {bytes}); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Binary); +} + +TEST_CASE("Decimal scalars", "[scalar]") { + Scalar d32 = scalar::decimal_i32(1234, 5, 2); + REQUIRE(d32.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d32.dtype().decimal_precision() == 5); + REQUIRE(d32.dtype().decimal_scale() == 2); + + Scalar d64 = scalar::decimal_i64(99999, 12, 3); + REQUIRE(d64.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d64.dtype().decimal_precision() == 12); + REQUIRE(d64.dtype().decimal_scale() == 3); +} + +TEST_CASE("Copy scalar", "[scalar]") { + Scalar a = scalar::of(42); + Scalar b = a; + REQUIRE(a.dtype().primitive_type() == I64); + REQUIRE(b.dtype().primitive_type() == I64); + + Scalar c = scalar::of(1); + c = b; + REQUIRE(c.dtype().primitive_type() == I64); +} +} // namespace diff --git a/vortex-cxx/tests/test_scan.cpp b/vortex-cxx/tests/test_scan.cpp new file mode 100644 index 00000000000..b2a1bc111d0 --- /dev/null +++ b/vortex-cxx/tests/test_scan.cpp @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include +#include +#include +#include + +#include "common.hpp" + +using namespace vortex; +using Catch::Matchers::ContainsSubstring; +using vortex_test::SAMPLE_ROWS; +using vortex_test::TempPath; +using vortex_test::write_sample; + +namespace { +using enum vortex::PType; + +TEST_CASE("Non-existent data source", "[datasource]") { + Session session; + + try { + DataSource::open(session, {"nonexistent"}); + FAIL("expected exception"); + } catch (const VortexException &e) { + REQUIRE_THAT(std::string(e.what()), ContainsSubstring("No files matched")); + } +} + +TEST_CASE("Read dtype and row count", "[datasource]") { + Session session; + TempPath path = write_sample(session); + + std::array paths = {path.string()}; + + DataSource ds = DataSource::open(session, paths); + + Estimate row_count = ds.row_count(); + REQUIRE(row_count.exact()); + REQUIRE(row_count.value() == SAMPLE_ROWS); + + DataType dt = ds.dtype(); + REQUIRE(dt.variant() == DataTypeVariant::Struct); + const std::vector fields = dt.fields(); + REQUIRE(fields.size() == 2); + REQUIRE(fields[0].name == "age"); + REQUIRE(fields[1].name == "height"); +} + +void verify_age_field(const Session &session, + const Array &age, + uint8_t first = 0, + size_t rows = SAMPLE_ROWS) { + REQUIRE(age.is_primitive(U8)); + REQUIRE(age.size() == rows); + auto view = age.values(session); + for (size_t i = 0; i < rows; ++i) { + REQUIRE(view.values()[i] == static_cast(first + i)); + } +} + +void verify_sample_array(const Session &session, const Array &array) { + REQUIRE(array.size() == SAMPLE_ROWS); + REQUIRE(array.has_dtype(DataTypeVariant::Struct)); + verify_age_field(session, array.field(0)); + + Array height = array.field(1); + REQUIRE(height.is_primitive(U16)); + auto view = height.values(session); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(view.values()[i] == (i + 1) % 200); + } + + REQUIRE_THROWS_AS(array.field(2), VortexException); +} + +TEST_CASE("Basic scan", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + REQUIRE(scan.partition_count().exact()); + REQUIRE(scan.partition_count().value() == 1); + REQUIRE(scan.dtype().variant() == DataTypeVariant::Struct); + + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + Estimate rows = partition->row_count(); + REQUIRE(rows.exact()); + REQUIRE(rows.value() == SAMPLE_ROWS); + + REQUIRE_FALSE(scan.next_partition().has_value()); + + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE_FALSE(partition->next().has_value()); + + verify_sample_array(session, *array); +} + +TEST_CASE("Range-for over partitions and batches", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + size_t partitions = 0; + size_t rows = 0; + for (Partition &partition : scan.partitions()) { + ++partitions; + for (Array &batch : partition.batches()) { + rows += batch.size(); + } + } + REQUIRE(partitions == 1); + REQUIRE(rows == SAMPLE_ROWS); +} + +TEST_CASE("Scan from buffer", "[scan]") { + Session session; + TempPath path = write_sample(session); + + std::ifstream file(path.path(), std::ios::binary | std::ios::ate); + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(static_cast(size)); + REQUIRE(file.read(reinterpret_cast(buffer.data()), size)); + + DataSource ds = DataSource::from_buffer(session, buffer); + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + verify_sample_array(session, *array); +} + +TEST_CASE("Multiple paths", "[scan]") { + Session session; + TempPath first = write_sample(session); + TempPath second = write_sample(session); + + const std::vector paths = {first.string(), second.string()}; + DataSource ds = DataSource::open(session, paths); + REQUIRE(ds.row_count().value_or(0) >= SAMPLE_ROWS); +} + +TEST_CASE("Multithreaded scan", "[scan]") { + Session session; + + constexpr size_t NUM_FILES = 8; + std::vector paths; + std::vector path_strings; + for (size_t i = 0; i < NUM_FILES; ++i) { + paths.push_back(write_sample(session)); + path_strings.push_back(paths.back().string()); + } + + DataSource ds = DataSource::open(session, path_strings); + Scan scan = ds.scan(); + + std::vector threads; + threads.reserve(NUM_FILES); + std::vector rows(NUM_FILES, 0); + + for (size_t i = 0; i < NUM_FILES; ++i) { + threads.emplace_back([&, i] { + while (auto partition = scan.next_partition()) { + while (auto batch = partition->next()) { + rows[i] += batch->size(); + } + } + }); + } + for (auto &t : threads) { + t.join(); + } + + size_t total = 0; + for (size_t n : rows) { + total += n; + } + REQUIRE(total == NUM_FILES * SAMPLE_ROWS); +} + +TEST_CASE("Project field", "[projection]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({.projection = expr::col("age")}); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + verify_age_field(session, *array); +} + +TEST_CASE("Filter age", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + constexpr uint8_t threshold = 50; + Scan scan = ds.scan({ + .filter = expr::gte(expr::col("age"), expr::lit(threshold)), + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + + REQUIRE(array->size() == SAMPLE_ROWS - threshold); + verify_age_field(session, array->field(0), threshold, SAMPLE_ROWS - threshold); +} + +TEST_CASE("Filter with operators", "[filter]") { + using namespace vortex::expr::ops; + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .filter = expr::col("age") >= expr::lit(90) && expr::col("age") < expr::lit(95), + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->size() == 5); +} + +TEST_CASE("Type-mismatched filter", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .filter = expr::eq(expr::col("age"), expr::lit(67)), + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + REQUIRE_THROWS_AS(partition->next(), VortexException); +} + +TEST_CASE("Row range and limit", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .row_range = RowRange {10, 25}, + .limit = 5, + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + + REQUIRE(array->size() == 5); + verify_age_field(session, array->field(0), 10, 5); +} + +TEST_CASE("Selection", "[scan]") { + using enum Selection::Kind; + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .selection = {{Include, {3, 5, 8}}}, + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->size() == 3); + + auto ages = array->field(0).values(session); + REQUIRE(ages.values()[0] == 3); + REQUIRE(ages.values()[1] == 5); + REQUIRE(ages.values()[2] == 8); +} + +TEST_CASE("Unfinished writer", "[writer]") { + Session session; + TempPath path = TempPath::unique(); + + { + Writer writer = Writer::open(session, path.string(), vortex_test::sample_dtype()); + writer.push(vortex_test::sample_array()); + } + + REQUIRE_THROWS_AS(DataSource::open(session, {path.string()}), VortexException); +} + +TEST_CASE("Write finish() called twice", "[writer]") { + Session session; + TempPath path = TempPath::unique(); + + Writer writer = Writer::open(session, path.string(), vortex_test::sample_dtype()); + writer.push(vortex_test::sample_array()); + writer.finish(); + REQUIRE_THROWS_AS(writer.finish(), VortexException); + REQUIRE_THROWS_AS(writer.push(vortex_test::sample_array()), VortexException); +} +} // namespace diff --git a/vortex-cxx/tests/test_string_binary.cpp b/vortex-cxx/tests/test_string_binary.cpp new file mode 100644 index 00000000000..16ddb7eaa3c --- /dev/null +++ b/vortex-cxx/tests/test_string_binary.cpp @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include +#include +#include +#include + +#include "common.hpp" + +using namespace vortex; +using namespace std::string_view_literals; +using vortex_test::TempPath; + +namespace { + +Array strings_from_arrow(std::span values, bool with_null) { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRING) == NANOARROW_OK); + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (const auto &value : values) { + const ArrowStringView view {value.data(), static_cast(value.size())}; + REQUIRE(ArrowArrayAppendString(arr.get(), view) == NANOARROW_OK); + } + if (with_null) { + REQUIRE(ArrowArrayAppendNull(arr.get(), 1) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + return Array::from_arrow(&raw_arr, &raw_schema, true); +} + +Array bytes_from_arrow(std::span values) { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_BINARY) == NANOARROW_OK); + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (const auto &value : values) { + const ArrowBufferView view {{reinterpret_cast(value.data())}, + static_cast(value.size())}; + REQUIRE(ArrowArrayAppendBytes(arr.get(), view) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + return Array::from_arrow(&raw_arr, &raw_schema, true); +} + +TEST_CASE("String view over utf8 array", "[strings]") { + Session session; + + const std::string long1(40, 'x'); + const std::vector values = {"short"sv, "Широкая строка"sv, long1, ""sv}; + Array array = strings_from_arrow(values, true); + + StringView view = array.strings(session); + REQUIRE(view.size() == values.size() + 1); + for (size_t i = 0; i < values.size(); ++i) { + REQUIRE_FALSE(view.is_null(i)); + REQUIRE(view[i] == values[i]); + } + REQUIRE(view.is_null(values.size())); + REQUIRE_THROWS_AS(view[view.size()], VortexException); +} + +TEST_CASE("Bytes view over binary array", "[strings]") { + Session session; + + // Includes a NUL byte + const std::vector values = {std::string_view {"abc\0def", 7}, "ffff"sv}; + Array array = bytes_from_arrow(values); + + BytesView view = array.bytes(session); + REQUIRE(view.size() == 2); + for (size_t i = 0; i < values.size(); ++i) { + const auto bytes = view[i]; + REQUIRE(std::string_view(reinterpret_cast(bytes.data()), bytes.size()) == values[i]); + } +} + +TEST_CASE("Strings roundtrip", "[strings]") { + Session session; + TempPath path = TempPath::unique(); + + const std::string long1(64, 'y'); + const std::vector values = {"inlined"sv, long1}; + Array strings = strings_from_arrow(values, false); + + Writer writer = + Writer::open(session, path.string(), dtype::struct_({{"s", dtype::utf8(dtype::Nullable)}})); + writer.push(make_struct({{"s", strings}})); + writer.finish(); + + DataSource ds = DataSource::open(session, {path.string()}); + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto batch = partition->next(); + REQUIRE(batch.has_value()); + + StringView view = batch->field(0).strings(session); + REQUIRE(view.size() == 2); + REQUIRE(view[0] == "inlined"sv); + REQUIRE(view[1] == long1); +} + +TEST_CASE("strings() on a non-utf8 array", "[strings]") { + Session session; + std::vector data = {1}; + Array a = Array::primitive(data); + REQUIRE_THROWS_AS(a.strings(session), VortexException); + REQUIRE_THROWS_AS(a.bytes(session), VortexException); +} +} // namespace diff --git a/vortex-duckdb/cpp/cast_pushdown.cpp b/vortex-duckdb/cpp/cast_pushdown.cpp index 3acd5b04161..f439812d279 100644 --- a/vortex-duckdb/cpp/cast_pushdown.cpp +++ b/vortex-duckdb/cpp/cast_pushdown.cpp @@ -170,9 +170,7 @@ ExpressionPtr CastReplace::VisitReplace(BoundCastExpression &expr, ExpressionPtr } CastCollect::CastCollect(Analyses &analyses, const Projections &projections) - : analyses(analyses), projections(projections) { -} + : analyses(analyses), projections(projections) {} CastReplace::CastReplace(Analyses &analyses, const Projections &projections) - : analyses(analyses), projections(projections) { -} + : analyses(analyses), projections(projections) {} diff --git a/vortex-duckdb/cpp/copy_function.cpp b/vortex-duckdb/cpp/copy_function.cpp index 4740b92bbc3..33dc52f63f6 100644 --- a/vortex-duckdb/cpp/copy_function.cpp +++ b/vortex-duckdb/cpp/copy_function.cpp @@ -14,14 +14,12 @@ using namespace duckdb; struct CopyBindData final : TableFunctionData { - CopyBindData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) { - } + CopyBindData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) {} unique_ptr ffi_data; }; struct CopyGlobalData final : GlobalFunctionData { - CopyGlobalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) { - } + CopyGlobalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) {} unique_ptr ffi_data; }; diff --git a/vortex-duckdb/cpp/include/vector.hpp b/vortex-duckdb/cpp/include/vector.hpp index f59d51ef2e5..49ee21865a7 100644 --- a/vortex-duckdb/cpp/include/vector.hpp +++ b/vortex-duckdb/cpp/include/vector.hpp @@ -12,6 +12,5 @@ class ExternalVectorBuffer final : public duckdb::VectorBuffer { duckdb::unique_ptr data; public: - explicit inline ExternalVectorBuffer(duckdb::unique_ptr data) : data(std::move(data)) { - } + explicit inline ExternalVectorBuffer(duckdb::unique_ptr data) : data(std::move(data)) {} }; diff --git a/vortex-duckdb/cpp/scalar_fn_pushdown.cpp b/vortex-duckdb/cpp/scalar_fn_pushdown.cpp index b3986ef49cc..0166790aeed 100644 --- a/vortex-duckdb/cpp/scalar_fn_pushdown.cpp +++ b/vortex-duckdb/cpp/scalar_fn_pushdown.cpp @@ -116,12 +116,10 @@ ExpressionPtr ScalarFnReplace::VisitReplace(BoundFunctionExpression &expr, Expre } ScalarFnCollect::ScalarFnCollect(Analyses &analyses, const Projections &projections) - : analyses(analyses), projections(projections) { -} + : analyses(analyses), projections(projections) {} ScalarFnReplace::ScalarFnReplace(Analyses &analyses, const Projections &projections) - : analyses(analyses), projections(projections) { -} + : analyses(analyses), projections(projections) {} namespace { @@ -129,8 +127,7 @@ namespace { // data, so nothing here depends on its internals. class StDWithinRestore final : public LogicalOperatorVisitor { public: - explicit StDWithinRestore(ClientContext &context) : context(context) { - } + explicit StDWithinRestore(ClientContext &context) : context(context) {} // Restore join conditions, filters must keep the radius visible so // DuckDB's filter pushdown can offer them to Vortex scans. diff --git a/vortex-duckdb/cpp/table_filter.cpp b/vortex-duckdb/cpp/table_filter.cpp index 27630cc7c5b..72e5d8421e7 100644 --- a/vortex-duckdb/cpp/table_filter.cpp +++ b/vortex-duckdb/cpp/table_filter.cpp @@ -80,8 +80,7 @@ namespace { struct DynamicFilterDataWrapper { shared_ptr data; - explicit DynamicFilterDataWrapper(shared_ptr d) : data(std::move(d)) { - } + explicit DynamicFilterDataWrapper(shared_ptr d) : data(std::move(d)) {} }; } // namespace diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index 42c4285c383..fbb43b15d67 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -26,8 +26,7 @@ constexpr column_t COLUMN_IDENTIFIER_FILE_ROW_NUMBER = MultiFileReader::COLUMN_I struct CTableBindData final : FunctionData { CTableBindData(unique_ptr ffi_data_p, const vector &types) - : ffi_data(std::move(ffi_data_p)), types(types) { - } + : ffi_data(std::move(ffi_data_p)), types(types) {} unique_ptr Copy() const override; bool Equals(const FunctionData &other_base) const override; @@ -49,19 +48,15 @@ bool CTableBindData::Equals(const FunctionData &other_base) const { } struct CTableGlobalData final : GlobalTableFunctionState { - explicit CTableGlobalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) { - } + explicit CTableGlobalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) {} - idx_t MaxThreads() const override { - return GlobalTableFunctionState::MAX_THREADS; - } + idx_t MaxThreads() const override { return GlobalTableFunctionState::MAX_THREADS; } unique_ptr ffi_data; }; struct CTableLocalData final : LocalTableFunctionState { - explicit CTableLocalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) { - } + explicit CTableLocalData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) {} unique_ptr ffi_data; }; @@ -72,9 +67,7 @@ table_scan_progress(ClientContext &, const FunctionData *, const GlobalTableFunc return duckdb_table_function_scan_progress(c_global_state); } -static Value &UnwrapValue(duckdb_value value) { - return *(reinterpret_cast(value)); -} +static Value &UnwrapValue(duckdb_value value) { return *(reinterpret_cast(value)); } unique_ptr numeric_stats(duckdb_column_statistics &stats, LogicalType type) { BaseStatistics out = StringStats::CreateUnknown(type); diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp index b6666ff6dcb..a224cfc8d33 100644 --- a/vortex-duckdb/cpp/vector.cpp +++ b/vortex-duckdb/cpp/vector.cpp @@ -42,17 +42,11 @@ duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, i // VortexVector doesn't add any members. class VortexVector final : public Vector { public: - inline void SetDataBuffer(buffer_ptr new_buffer) { - buffer = std::move(new_buffer); - }; + inline void SetDataBuffer(buffer_ptr new_buffer) { buffer = std::move(new_buffer); }; - inline void SetDataPtr(data_ptr_t ptr) { - data = ptr; - }; + inline void SetDataPtr(data_ptr_t ptr) { data = ptr; }; - inline ValidityMask &GetValidity() { - return validity; - }; + inline ValidityMask &GetValidity() { return validity; }; }; static_assert(sizeof(VortexVector) == sizeof(Vector)); diff --git a/vortex-duckdb/cpp/vortex_duckdb.cpp b/vortex-duckdb/cpp/vortex_duckdb.cpp index 8b023c960dd..a6d5e9634e4 100644 --- a/vortex-duckdb/cpp/vortex_duckdb.cpp +++ b/vortex-duckdb/cpp/vortex_duckdb.cpp @@ -60,8 +60,7 @@ extern "C" char *duckdb_vx_value_to_string(duckdb_value value) { } } -CData::CData(void *data_ptr, duckdb_delete_callback_t callback) : data(data_ptr), delete_callback(callback) { -} +CData::CData(void *data_ptr, duckdb_delete_callback_t callback) : data(data_ptr), delete_callback(callback) {} CData::~CData() { if (data && delete_callback) { @@ -71,9 +70,7 @@ CData::~CData() { delete_callback = nullptr; } -void *CData::DataPtr() const { - return data; -} +void *CData::DataPtr() const { return data; } extern "C" duckdb_vx_data duckdb_vx_data_create(void *data, duckdb_delete_callback_t delete_callback) { return reinterpret_cast(new CData(data, delete_callback)); @@ -199,8 +196,7 @@ extern "C" duckdb_state duckdb_vx_register_scan_replacement(duckdb_database duck using Buffer = buffer_ptr; struct ReusableDict { Buffer buffer; - ReusableDict(Buffer buffer) : buffer(std::move(buffer)) { - } + ReusableDict(Buffer buffer) : buffer(std::move(buffer)) {} }; extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type ffi_type, idx_t size) { @@ -281,8 +277,7 @@ static void VortexPreOptimizeFunction(OptimizerExtensionInput &input, unique_ptr struct VortexOptimizerExtension final : OptimizerExtension { inline VortexOptimizerExtension() - : OptimizerExtension(VortexOptimizeFunction, VortexPreOptimizeFunction, {}) { - } + : OptimizerExtension(VortexOptimizeFunction, VortexPreOptimizeFunction, {}) {} }; extern "C" duckdb_state duckdb_vx_optimizer_extension_register(duckdb_database ffi_db) { diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index cbd9f7f1406..d9213adbfa7 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -1161,6 +1161,8 @@ void vx_expression_free(const vx_expression *ptr); */ vx_expression *vx_expression_root(void); +vx_expression *vx_expression_clone(const vx_expression *ptr); + /** * Create a literal expression from a scalar. * @@ -1253,7 +1255,7 @@ vx_expression_binary(vx_binary_operator operator_, const vx_expression *lhs, con * * Returns the logical negation of the input boolean expression. */ -const vx_expression *vx_expression_not(const vx_expression *child); +vx_expression *vx_expression_not(const vx_expression *child); /** * Create an expression that checks for null values. diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 87e15c5f843..3a0003b1b16 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -63,6 +63,16 @@ pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression { vx_expression::new(root()) } +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_expression_clone( + ptr: *const vx_expression, +) -> *mut vx_expression { + if ptr.is_null() { + return ptr::null_mut(); + } + vx_expression::new(vx_expression::as_ref(ptr).clone()) +} + /// Create a literal expression from a scalar. /// /// Literal expressions are useful for constants in expression trees, especially scan @@ -260,9 +270,9 @@ pub unsafe extern "C" fn vx_expression_binary( /// /// Returns the logical negation of the input boolean expression. #[unsafe(no_mangle)] -pub unsafe extern "C" fn vx_expression_not(child: *const vx_expression) -> *const vx_expression { +pub unsafe extern "C" fn vx_expression_not(child: *const vx_expression) -> *mut vx_expression { if child.is_null() { - return child; + return ptr::null_mut(); } vx_expression::new(not(vx_expression::as_ref(child).clone())) } diff --git a/vortex-ffi/src/struct_fields.rs b/vortex-ffi/src/struct_fields.rs index 6950b62a302..83b2f3dec74 100644 --- a/vortex-ffi/src/struct_fields.rs +++ b/vortex-ffi/src/struct_fields.rs @@ -103,10 +103,10 @@ pub unsafe extern "C-unwind" fn vx_struct_fields_builder_add_field( ) { try_or_default(error_out, || { let builder = vx_struct_fields_builder::as_mut(builder); - builder.names.push(Arc::from(unsafe { name.as_str() }?)); builder .fields .push(vx_dtype::into_arc(dtype).deref().clone()); + builder.names.push(Arc::from(unsafe { name.as_str() }?)); Ok(()) }) } diff --git a/vortex-ffi/test/array.cpp b/vortex-ffi/test/array.cpp index f8db2661b8d..737640e6644 100644 --- a/vortex-ffi/test/array.cpp +++ b/vortex-ffi/test/array.cpp @@ -10,9 +10,7 @@ TEST_CASE("Null array creation", "[array]") { REQUIRE(vx_array_is_nullable(array)); REQUIRE(vx_array_has_dtype(array, DTYPE_NULL)); const vx_dtype *dtype = vx_array_dtype(array); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; REQUIRE(vx_dtype_get_variant(dtype) == DTYPE_NULL); REQUIRE(vx_array_len(array) == 1999); vx_array_free(array); @@ -32,9 +30,7 @@ TEST_CASE("Primitive array creation", "[array]") { REQUIRE(vx_array_has_dtype(array, DTYPE_PRIMITIVE)); const vx_dtype *dtype = vx_array_dtype(array); REQUIRE(vx_dtype_get_variant(dtype) == DTYPE_PRIMITIVE); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; REQUIRE(vx_array_is_primitive(array, PTYPE_U8)); REQUIRE(vx_array_len(array) == buffer.size()); @@ -65,9 +61,7 @@ TEST_CASE("Struct array creation", "[array]") { vx_struct_column_builder_add_field(builder, vx_view_from_cstr("age"), field_array, &error); vx_array_free(field_array); - SECTION("Struct array builder free") { - vx_struct_column_builder_free(builder); - } + SECTION("Struct array builder free") { vx_struct_column_builder_free(builder); } SECTION("Struct array builder finalize") { const vx_array *struct_array = vx_struct_column_builder_finalize(builder, &error); diff --git a/vortex-ffi/test/common.h b/vortex-ffi/test/common.h index 81974b65bb9..3a2facae316 100644 --- a/vortex-ffi/test/common.h +++ b/vortex-ffi/test/common.h @@ -10,9 +10,7 @@ inline std::string to_string(vx_error *err) { return {msg.ptr, msg.len}; } -inline std::string_view to_string_view(vx_view str) { - return {str.ptr, str.len}; -} +inline std::string_view to_string_view(vx_view str) { return {str.ptr, str.len}; } inline void require_no_error(vx_error *error, bool assert = true) { if (!error) { @@ -29,11 +27,8 @@ inline void require_no_error(vx_error *error, bool assert = true) { template struct Defer { - Defer(F &&f) : f(std::move(f)) { - } - ~Defer() { - f(); - } + Defer(F &&f) : f(std::move(f)) {} + ~Defer() { f(); } F f; }; #define CONCAT(x, y) x##y diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp index e1f629dfde0..8ae4f83acaf 100644 --- a/vortex-ffi/test/scan.cpp +++ b/vortex-ffi/test/scan.cpp @@ -30,14 +30,12 @@ using nanoarrow::UniqueSchema; struct TempPath : fs::path { TempPath() = default; - explicit TempPath(fs::path p) : fs::path(std::move(p)) { - } + explicit TempPath(fs::path p) : fs::path(std::move(p)) {} TempPath(const TempPath &) = delete; TempPath &operator=(const TempPath &) = delete; - TempPath(TempPath &&other) noexcept : fs::path(std::move(other)) { - } + TempPath(TempPath &&other) noexcept : fs::path(std::move(other)) {} TempPath &operator=(TempPath &&other) noexcept { if (this != &other) { fs::remove(*this); @@ -97,9 +95,7 @@ std::vector sample_height() { std::vector age_buffer = sample_age(); const vx_array *age_array = vx_array_new_primitive(PTYPE_U8, age_buffer.data(), age_buffer.size(), &validity, &error); - defer { - vx_array_free(age_array); - }; + defer { vx_array_free(age_array); }; require_no_error(error); vx_struct_column_builder_add_field(builder, vx_view_from_cstr("age"), age_array, &error); @@ -109,9 +105,7 @@ std::vector sample_height() { validity.type = VX_VALIDITY_ALL_VALID; const vx_array *height_array = vx_array_new_primitive(PTYPE_U16, height_buffer.data(), height_buffer.size(), &validity, &error); - defer { - vx_array_free(height_array); - }; + defer { vx_array_free(height_array); }; require_no_error(error); vx_struct_column_builder_add_field(builder, vx_view_from_cstr("height"), height_array, &error); @@ -162,9 +156,7 @@ UniqueArrayStream sample_array_stream() { fs::path("test-" + std::to_string(std::random_device {}()) + ".vortex"); const vx_dtype *dtype = sample_dtype(); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; vx_error *error = nullptr; vx_array_sink *sink = vx_array_sink_open_file(session, vx_view_from_cstr(path.c_str()), dtype, &error); @@ -172,9 +164,7 @@ UniqueArrayStream sample_array_stream() { require_no_error(error); const vx_array *array = sample_array(); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; vx_array_sink_push(sink, array, &error); require_no_error(error); @@ -187,9 +177,7 @@ UniqueArrayStream sample_array_stream() { TEST_CASE("Creating datasources", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; vx_error *error = nullptr; const vx_data_source *ds = vx_data_source_new(session, nullptr, &error); @@ -235,17 +223,13 @@ TEST_CASE("Creating datasources", "[datasource]") { TEST_CASE("Write file", "[sink]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); } TEST_CASE("Write file and read dtypes", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -257,9 +241,7 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { const vx_data_source *ds = vx_data_source_new(session, &opts, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_estimate row_count; vx_data_source_get_row_count(ds, &row_count); @@ -268,23 +250,17 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { CHECK(row_count.estimate == SAMPLE_ROWS); const vx_dtype *data_source_dtype = vx_data_source_dtype(ds); - defer { - vx_dtype_free(data_source_dtype); - }; + defer { vx_dtype_free(data_source_dtype); }; REQUIRE(vx_dtype_get_variant(data_source_dtype) == DTYPE_STRUCT); const vx_struct_fields *fields = vx_dtype_struct_dtype(data_source_dtype); - defer { - vx_struct_fields_free(fields); - }; + defer { vx_struct_fields_free(fields); }; const size_t len = vx_struct_fields_nfields(fields); REQUIRE(len == 2); const vx_dtype *age_dtype = vx_struct_fields_field_dtype(fields, 0); const vx_view age_name = vx_struct_fields_field_name(fields, 0); - defer { - vx_dtype_free(age_dtype); - }; + defer { vx_dtype_free(age_dtype); }; REQUIRE(vx_dtype_get_variant(age_dtype) == DTYPE_PRIMITIVE); REQUIRE(vx_dtype_primitive_ptype(age_dtype) == PTYPE_U8); @@ -293,9 +269,7 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { const vx_dtype *height_dtype = vx_struct_fields_field_dtype(fields, 1); const vx_view height_name = vx_struct_fields_field_name(fields, 1); - defer { - vx_dtype_free(height_dtype); - }; + defer { vx_dtype_free(height_dtype); }; REQUIRE(vx_dtype_get_variant(height_dtype) == DTYPE_PRIMITIVE); REQUIRE(vx_dtype_primitive_ptype(height_dtype) == PTYPE_U16); REQUIRE(vx_dtype_is_nullable(height_dtype)); @@ -305,9 +279,7 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { void verify_age_field(const vx_array *age_field) { REQUIRE(vx_array_has_dtype(age_field, DTYPE_PRIMITIVE)); const vx_dtype *dtype = vx_array_dtype(age_field); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; REQUIRE(vx_dtype_primitive_ptype(dtype) == PTYPE_U8); REQUIRE(vx_array_len(age_field) == SAMPLE_ROWS); for (size_t i = 0; i < SAMPLE_ROWS; ++i) { @@ -318,9 +290,7 @@ void verify_age_field(const vx_array *age_field) { void verify_height_field(const vx_array *height_field) { REQUIRE(vx_array_has_dtype(height_field, DTYPE_PRIMITIVE)); const vx_dtype *dtype = vx_array_dtype(height_field); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; REQUIRE(vx_dtype_primitive_ptype(dtype) == PTYPE_U16); REQUIRE(vx_array_len(height_field) == SAMPLE_ROWS); for (size_t i = 0; i < SAMPLE_ROWS; ++i) { @@ -378,9 +348,7 @@ void verify_sample_array(const vx_array *array) { TEST_CASE("Requesting scans", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_data_source_options ds_options = {}; @@ -392,9 +360,7 @@ TEST_CASE("Requesting scans", "[datasource]") { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; { vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); @@ -418,18 +384,14 @@ void basic_scan(const vx_data_source *ds) { vx_estimate estimate = {}; vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); require_no_error(error); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; REQUIRE(scan != nullptr); REQUIRE(estimate.type == VX_ESTIMATE_EXACT); REQUIRE(estimate.estimate == 1); vx_partition *partition = vx_scan_next_partition(scan, &error); require_no_error(error); - defer { - vx_partition_free(partition); - }; + defer { vx_partition_free(partition); }; estimate = {}; vx_partition_row_count(partition, &estimate, &error); @@ -443,9 +405,7 @@ void basic_scan(const vx_data_source *ds) { const vx_array *array = vx_partition_next(partition, &error); require_no_error(error); REQUIRE(array != nullptr); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; REQUIRE(vx_partition_next(partition, &error) == nullptr); require_no_error(error); @@ -455,9 +415,7 @@ void basic_scan(const vx_data_source *ds) { TEST_CASE("Basic scan", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -468,18 +426,14 @@ TEST_CASE("Basic scan", "[datasource]") { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; basic_scan(ds); } TEST_CASE("Basic scan from memory", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); std::ifstream file(path, std::ios::binary | std::ios::ate); @@ -492,18 +446,14 @@ TEST_CASE("Basic scan from memory", "[datasource]") { const vx_data_source *ds = vx_data_source_new_buffer(session, buffer.data(), size, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; basic_scan(ds); } TEST_CASE("Multithreaded scan", "[datasource]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; constexpr size_t NUM_FILES = 10; std::vector paths(NUM_FILES); @@ -521,16 +471,12 @@ TEST_CASE("Multithreaded scan", "[datasource]") { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_estimate estimate = {}; vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); require_no_error(error); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; REQUIRE(scan != nullptr); REQUIRE(estimate.type == VX_ESTIMATE_INEXACT); REQUIRE(estimate.estimate == NUM_FILES); @@ -553,9 +499,7 @@ TEST_CASE("Multithreaded scan", "[datasource]") { } } - defer { - vx_partition_free(partition); - }; + defer { vx_partition_free(partition); }; estimate = {}; vx_partition_row_count(partition, &estimate, &error_tl); @@ -587,18 +531,14 @@ TEST_CASE("Multithreaded scan", "[datasource]") { for (const vx_array *array : arrays) { REQUIRE(array != nullptr); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; verify_sample_array(array); } } const vx_array *scan_with_options(vx_scan_options &options) { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -610,23 +550,17 @@ const vx_array *scan_with_options(vx_scan_options &options) { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_scan *scan = vx_data_source_scan(ds, &options, nullptr, &error); require_no_error(error); REQUIRE(scan != nullptr); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; vx_partition *partition = vx_scan_next_partition(scan, &error); require_no_error(error); REQUIRE(partition != nullptr); - defer { - vx_partition_free(partition); - }; + defer { vx_partition_free(partition); }; const vx_array *array = vx_partition_next(partition, &error); require_no_error(error); @@ -638,103 +572,75 @@ const vx_array *scan_with_options(vx_scan_options &options) { TEST_CASE("Project all fields", "[projection]") { vx_scan_options opts = {}; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; verify_sample_array(array); } TEST_CASE("Project root", "[projection]") { vx_expression *root = vx_expression_root(); - defer { - vx_expression_free(root); - }; + defer { vx_expression_free(root); }; vx_scan_options opts = {}; opts.projection = root; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; verify_sample_array(array); } TEST_CASE("Project single field", "[projection]") { vx_expression *root = vx_expression_root(); - defer { - vx_expression_free(root); - }; + defer { vx_expression_free(root); }; vx_scan_options opts = {}; vx_expression *age_field = vx_expression_get_item(vx_view_from_cstr("age"), root); REQUIRE(age_field != nullptr); - defer { - vx_expression_free(age_field); - }; + defer { vx_expression_free(age_field); }; { opts.projection = age_field; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; verify_age_field(array); } vx_expression *height_field = vx_expression_get_item(vx_view_from_cstr("height"), root); REQUIRE(height_field != nullptr); - defer { - vx_expression_free(height_field); - }; + defer { vx_expression_free(height_field); }; { opts.projection = height_field; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; verify_height_field(array); } } TEST_CASE("Filter with literal expression", "[filter]") { vx_expression *root = vx_expression_root(); - defer { - vx_expression_free(root); - }; + defer { vx_expression_free(root); }; vx_expression *age_field = vx_expression_get_item(vx_view_from_cstr("age"), root); REQUIRE(age_field != nullptr); - defer { - vx_expression_free(age_field); - }; + defer { vx_expression_free(age_field); }; uint8_t threshold = 50; vx_scalar *threshold_scalar = vx_scalar_new_u8(threshold, false); REQUIRE(threshold_scalar != nullptr); - defer { - vx_scalar_free(threshold_scalar); - }; + defer { vx_scalar_free(threshold_scalar); }; vx_error *literal_error = nullptr; vx_expression *threshold_expr = vx_expression_literal(threshold_scalar, &literal_error); require_no_error(literal_error); REQUIRE(threshold_expr != nullptr); - defer { - vx_expression_free(threshold_expr); - }; + defer { vx_expression_free(threshold_expr); }; vx_expression *filter = vx_expression_binary(VX_OPERATOR_GTE, age_field, threshold_expr); REQUIRE(filter != nullptr); - defer { - vx_expression_free(filter); - }; + defer { vx_expression_free(filter); }; vx_scan_options opts = {}; opts.filter = filter; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; REQUIRE(vx_array_len(array) == SAMPLE_ROWS - threshold); @@ -742,9 +648,7 @@ TEST_CASE("Filter with literal expression", "[filter]") { const vx_array *filtered_age = vx_array_get_field(array, 0, &error); require_no_error(error); REQUIRE(filtered_age != nullptr); - defer { - vx_array_free(filtered_age); - }; + defer { vx_array_free(filtered_age); }; for (size_t i = 0; i < vx_array_len(filtered_age); ++i) { REQUIRE(vx_array_get_u8(filtered_age, i) == static_cast(threshold + i)); @@ -758,37 +662,27 @@ TEST_CASE("Project UTF-8 literal expression", "[projection]") { vx_scalar_new_utf8(vx_view {value.data(), value.size()}, false, &scalar_error); require_no_error(scalar_error); REQUIRE(literal_scalar != nullptr); - defer { - vx_scalar_free(literal_scalar); - }; + defer { vx_scalar_free(literal_scalar); }; vx_error *literal_error = nullptr; vx_expression *literal_expr = vx_expression_literal(literal_scalar, &literal_error); require_no_error(literal_error); REQUIRE(literal_expr != nullptr); - defer { - vx_expression_free(literal_expr); - }; + defer { vx_expression_free(literal_expr); }; vx_scan_options opts = {}; opts.projection = literal_expr; const vx_array *array = scan_with_options(opts); - defer { - vx_array_free(array); - }; + defer { vx_array_free(array); }; REQUIRE(vx_array_len(array) == SAMPLE_ROWS); vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; vx_error *canon_error = nullptr; const vx_array *canonical = vx_array_canonicalize(session, array, &canon_error); require_no_error(canon_error); - defer { - vx_array_free(canonical); - }; + defer { vx_array_free(canonical); }; for (size_t i : {size_t {0}, SAMPLE_ROWS - 1}) { vx_error *at_error = nullptr; const vx_view actual = vx_array_utf8_at(canonical, i, &at_error); @@ -809,9 +703,7 @@ void compare_schemas(const UniqueSchema &left, const UniqueSchema &right) { } } -void compare_schema_with_sample(const UniqueSchema &left) { - compare_schemas(left, sample_schema()); -} +void compare_schema_with_sample(const UniqueSchema &left) { compare_schemas(left, sample_schema()); } void compare_stream_with_sample(UniqueArrayStream &left) { UniqueArrayStream right = sample_array_stream(); @@ -865,9 +757,7 @@ void compare_stream_with_sample(UniqueArrayStream &left) { TEST_CASE("Scan Arrow schema", "[scan]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -880,24 +770,18 @@ TEST_CASE("Scan Arrow schema", "[scan]") { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); require_no_error(error); REQUIRE(scan != nullptr); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; ArrowSchema schema; const vx_dtype *dtype = vx_scan_dtype(scan, &error); require_no_error(error); REQUIRE(dtype != nullptr); - defer { - vx_dtype_free(dtype); - }; + defer { vx_dtype_free(dtype); }; int res = vx_dtype_to_arrow_schema(dtype, &schema, &error); REQUIRE(res == 0); @@ -910,9 +794,7 @@ TEST_CASE("Scan Arrow schema", "[scan]") { TEST_CASE("Scan to Arrow", "[scan]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -924,16 +806,12 @@ TEST_CASE("Scan to Arrow", "[scan]") { const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); require_no_error(error); REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_scan *scan = vx_data_source_scan(ds, nullptr, nullptr, &error); require_no_error(error); REQUIRE(scan != nullptr); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; vx_partition *partition = vx_scan_next_partition(scan, &error); require_no_error(error); @@ -952,9 +830,7 @@ TEST_CASE("Scan to Arrow", "[scan]") { TEST_CASE("Broken scan with DType mismatch in filter", "[filter]") { vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; + defer { vx_session_free(session); }; TempPath path = write_sample(session); vx_error *error = nullptr; @@ -964,54 +840,38 @@ TEST_CASE("Broken scan with DType mismatch in filter", "[filter]") { ds_opts.paths_len = 1; const vx_data_source *ds = vx_data_source_new(session, &ds_opts, &error); require_no_error(error); - defer { - vx_data_source_free(ds); - }; + defer { vx_data_source_free(ds); }; vx_expression *root = vx_expression_root(); - defer { - vx_expression_free(root); - }; + defer { vx_expression_free(root); }; vx_expression *age_col = vx_expression_get_item(vx_view_from_cstr("age"), root); REQUIRE(age_col != nullptr); - defer { - vx_expression_free(age_col); - }; + defer { vx_expression_free(age_col); }; vx_scalar *lit = vx_scalar_new_i32(67, false); - defer { - vx_scalar_free(lit); - }; + defer { vx_scalar_free(lit); }; vx_expression *lit_expr = vx_expression_literal(lit, &error); require_no_error(error); - defer { - vx_expression_free(lit_expr); - }; + defer { vx_expression_free(lit_expr); }; // DType mismatch between age_col (u8) and lit (i32) vx_expression *filter = vx_expression_binary(VX_OPERATOR_EQ, age_col, lit_expr); REQUIRE(filter != nullptr); - defer { - vx_expression_free(filter); - }; + defer { vx_expression_free(filter); }; vx_scan_options scan_opts = {}; scan_opts.filter = filter; vx_scan *scan = vx_data_source_scan(ds, &scan_opts, nullptr, &error); require_no_error(error); - defer { - vx_scan_free(scan); - }; + defer { vx_scan_free(scan); }; vx_partition *partition = vx_scan_next_partition(scan, &error); require_no_error(error); REQUIRE(partition != nullptr); - defer { - vx_partition_free(partition); - }; + defer { vx_partition_free(partition); }; // This call must set vx_error and return nullptr, not panic const vx_array *array = vx_partition_next(partition, &error); diff --git a/vortex-ffi/test/struct.cpp b/vortex-ffi/test/struct.cpp index 2a45fccf56f..0445262e410 100644 --- a/vortex-ffi/test/struct.cpp +++ b/vortex-ffi/test/struct.cpp @@ -20,16 +20,12 @@ TEST_CASE("Struct builder", "[struct]") { vx_struct_fields_builder_add_field(builder, vx_view_from_cstr("col2"), col2_dtype, &error); require_no_error(error); - SECTION("Struct builder free") { - vx_struct_fields_builder_free(builder); - } + SECTION("Struct builder free") { vx_struct_fields_builder_free(builder); } SECTION("Struct builder finalize") { vx_struct_fields *fields = vx_struct_fields_builder_finalize(builder); - SECTION("struct fields free") { - vx_struct_fields_free(fields); - } + SECTION("struct fields free") { vx_struct_fields_free(fields); } SECTION("struct fields finalize") { const vx_dtype *dtype = vx_dtype_new_struct(fields, false);