diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 34ec6993aa4..6b1c52fe49b 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include "arrow/status.h" #include "arrow/util/endian.h" #include "arrow/util/future.h" +#include "arrow/util/int_util_overflow.h" #include "arrow/util/logging_internal.h" #include "arrow/util/ubsan.h" @@ -328,12 +330,19 @@ struct ReadMessageState { : listener(std::make_shared(&result)), decoder(std::make_shared(listener)) {} + explicit ReadMessageState(const IpcReadOptions& options) + : listener(std::make_shared(&result)), + decoder(std::make_shared(listener, options)) {} + // ReadMessageState points into itself, so it shouldn't be moved ReadMessageState(ReadMessageState&&) = delete; ReadMessageState& operator=(ReadMessageState&&) = delete; }; Status CheckBodyLength(int64_t offset, int64_t body_length, io::RandomAccessFile* file) { + if (body_length < 0) { + return Status::Invalid("IPC body length ", body_length, " must be non-negative"); + } ARROW_ASSIGN_OR_RAISE(auto file_size, file->GetSize()); if (body_length > file_size - offset) { return Status::IOError( @@ -418,11 +427,18 @@ static Result> ReadMessageContinued( } // namespace Result> ReadMessage(std::shared_ptr metadata, - std::shared_ptr body) { + std::shared_ptr body, + const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); std::unique_ptr result; auto listener = std::make_shared(&result); // If the user does not pass in a body buffer then we assume they are skipping it - MessageDecoder decoder(listener, default_memory_pool(), body == nullptr); + MessageDecoder decoder(listener, options, body == nullptr); + + RETURN_NOT_OK(options.CheckMetadataLength(metadata->size())); + if (body != nullptr) { + RETURN_NOT_OK(options.CheckBodyLength(body->size())); + } if (metadata->size() < decoder.next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -465,15 +481,25 @@ Result> ReadMessage(std::shared_ptr metadata, } } +Result> ReadMessage(std::shared_ptr metadata, + std::shared_ptr body) { + return ReadMessage(std::move(metadata), std::move(body), IpcReadOptions::Defaults()); +} + // Common helper for the two ReadMessage overloads that take a file + offset. // When body_length is provided, metadata and body are read in a single IO. // When body_length is absent, metadata is read first, then the body is read // separately. static Result> ReadMessageInternal( int64_t offset, int32_t metadata_length, std::optional body_length, - io::RandomAccessFile* file, MemoryPool* pool, + io::RandomAccessFile* file, const IpcReadOptions& options, const FieldsLoaderFunction& fields_loader) { - ReadMessageState state; + RETURN_NOT_OK(options.Validate()); + RETURN_NOT_OK(options.CheckMetadataLength(metadata_length)); + if (body_length.has_value()) { + RETURN_NOT_OK(options.CheckBodyLength(*body_length)); + } + ReadMessageState state(options); if (metadata_length < state.decoder->next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -489,15 +515,17 @@ static Result> ReadMessageInternal( file->ReadAt(offset, read_size, /*allow_short_read=*/false)); - return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, pool, - fields_loader, &state); + return ReadMessageContinued(offset, metadata_length, body_length, metadata, file, + options.memory_pool, fields_loader, &state); } Result> ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = file->io_context().pool(); return ReadMessageInternal(offset, metadata_length, /*body_length=*/std::nullopt, file, - file->io_context().pool(), fields_loader); + options, fields_loader); } Result> ReadMessage(const int64_t offset, @@ -505,17 +533,33 @@ Result> ReadMessage(const int64_t offset, const int64_t body_length, io::RandomAccessFile* file, MemoryPool* pool, const FieldsLoaderFunction& fields_loader) { - return ReadMessageInternal(offset, metadata_length, body_length, file, pool, + auto options = IpcReadOptions::Defaults(); + options.memory_pool = pool; + return ReadMessageInternal(offset, metadata_length, body_length, file, options, + fields_loader); +} + +Result> ReadMessage(const int64_t offset, + const int32_t metadata_length, + const int64_t body_length, + io::RandomAccessFile* file, + const IpcReadOptions& options, + const FieldsLoaderFunction& fields_loader) { + return ReadMessageInternal(offset, metadata_length, body_length, file, options, fields_loader); } Future> ReadMessageAsync(int64_t offset, int32_t metadata_length, int64_t body_length, io::RandomAccessFile* file, + const IpcReadOptions& options, const io::IOContext& context) { + RETURN_NOT_OK(options.Validate()); + RETURN_NOT_OK(options.CheckMetadataLength(metadata_length)); + RETURN_NOT_OK(options.CheckBodyLength(body_length)); // Make a std::shared_ptr so as to have a stable ReadMessageState pointer, // since state->listener will point back to state->result. - auto state = std::make_shared(); + auto state = std::make_shared(options); if (metadata_length < state->decoder->next_required_size()) { return Status::Invalid("metadata_length should be at least ", @@ -528,11 +572,20 @@ Future> ReadMessageAsync(int64_t offset, int32_t metada // Pass a nullptr file to ensure that no further IO occurs // (we have fetched all the required bytes). return ReadMessageContinued(offset, metadata_length, body_length, metadata, - /*file=*/nullptr, context.pool(), + /*file=*/nullptr, options.memory_pool, /*fields_loader=*/{}, state.get()); }); } +Future> ReadMessageAsync(int64_t offset, int32_t metadata_length, + int64_t body_length, + io::RandomAccessFile* file, + const io::IOContext& context) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = context.pool(); + return ReadMessageAsync(offset, metadata_length, body_length, file, options, context); +} + Status AlignStream(io::InputStream* stream, int32_t alignment) { ARROW_ASSIGN_OR_RAISE(int64_t position, stream->Tell()); return stream->Advance(PaddedLength(position, alignment) - position); @@ -612,10 +665,11 @@ Status DecodeMessage(MessageDecoder* decoder, io::InputStream* file) { } } -Result> ReadMessage(io::InputStream* file, MemoryPool* pool) { +Result> ReadMessage(io::InputStream* file, + const IpcReadOptions& options) { std::unique_ptr message; auto listener = std::make_shared(&message); - MessageDecoder decoder(listener, pool); + MessageDecoder decoder(listener, options); ARROW_RETURN_NOT_OK(DecodeMessage(&decoder, file)); if (!message) { return nullptr; @@ -624,6 +678,12 @@ Result> ReadMessage(io::InputStream* file, MemoryPool* } } +Result> ReadMessage(io::InputStream* file, MemoryPool* pool) { + auto options = IpcReadOptions::Defaults(); + options.memory_pool = pool; + return ReadMessage(file, options); +} + Status WriteMessage(const Buffer& message, const IpcWriteOptions& options, io::OutputStream* file, int32_t* message_length) { const int32_t prefix_size = options.write_legacy_ipc_format ? 4 : 8; @@ -674,8 +734,21 @@ class MessageDecoder::MessageDecoderImpl { explicit MessageDecoderImpl(std::shared_ptr listener, State initial_state, int64_t initial_next_required_size, MemoryPool* pool, bool skip_body) + : MessageDecoderImpl( + std::move(listener), initial_state, initial_next_required_size, + [pool] { + auto opts = IpcReadOptions::Defaults(); + opts.memory_pool = pool; + return opts; + }(), + skip_body) {} + + explicit MessageDecoderImpl(std::shared_ptr listener, + State initial_state, int64_t initial_next_required_size, + IpcReadOptions options, bool skip_body) : listener_(std::move(listener)), - pool_(pool), + options_(std::move(options)), + pool_(options_.memory_pool), memory_manager_(CPUDevice::memory_manager(pool_)), state_(initial_state), next_required_size_(initial_next_required_size), @@ -685,16 +758,35 @@ class MessageDecoder::MessageDecoderImpl { skip_body_(skip_body) {} Status ConsumeData(const uint8_t* data, int64_t size) { + RETURN_NOT_OK(options_.Validate()); + if (size < 0) { + return Status::Invalid("Invalid data size: negative byte count"); + } if (buffered_size_ == 0) { - while (size > 0 && size >= next_required_size_) { + std::span data_span(data, static_cast(size)); + while (!data_span.empty() && + data_span.size() >= static_cast(next_required_size_)) { auto used_size = next_required_size_; switch (state_) { - case State::INITIAL: - RETURN_NOT_OK(ConsumeInitialData(data, next_required_size_)); + case State::INITIAL: { + bool is_eos = (bit_util::FromLittleEndian( + util::SafeLoadAs(data_span.data())) == 0); + RETURN_NOT_OK(ConsumeInitialData(data_span.data(), next_required_size_)); + if (is_eos) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: - RETURN_NOT_OK(ConsumeMetadataLengthData(data, next_required_size_)); + } + case State::METADATA_LENGTH: { + bool is_eos = (bit_util::FromLittleEndian( + util::SafeLoadAs(data_span.data())) == 0); + RETURN_NOT_OK( + ConsumeMetadataLengthData(data_span.data(), next_required_size_)); + if (is_eos) { + return Status::OK(); + } break; + } case State::METADATA: { // We need to copy metadata because it's used in // ConsumeBody(). ConsumeBody() may be called from another @@ -704,7 +796,8 @@ class MessageDecoder::MessageDecoderImpl { // here. ARROW_ASSIGN_OR_RAISE(std::shared_ptr buffer, AllocateBuffer(next_required_size_, pool_)); - memcpy(buffer->mutable_data(), data, next_required_size_); + memcpy(buffer->mutable_data(), data_span.data(), + static_cast(next_required_size_)); RETURN_NOT_OK(ConsumeMetadataBuffer(buffer)); } break; case State::BODY: { @@ -714,21 +807,32 @@ class MessageDecoder::MessageDecoderImpl { // the passed message is needed to be valid after the // call, it's a listener_'s responsibility. The listener_ // may copy the data for it. - auto buffer = std::make_shared(data, next_required_size_); + auto buffer = std::make_shared(data_span.data(), next_required_size_); RETURN_NOT_OK(ConsumeBodyBuffer(buffer)); } break; case State::EOS: return Status::OK(); } - data += used_size; - size -= used_size; + if (data_span.size() == static_cast(used_size)) { + return Status::OK(); + } + data_span = data_span.subspan(static_cast(used_size)); + if (state_ == State::EOS) { + return Status::OK(); + } } + data = data_span.data(); + size = static_cast(data_span.size()); } if (size == 0) { return Status::OK(); } + if (state_ == State::EOS) { + return Status::OK(); + } + // We need to copy unused data because the given data for the // current ConsumeData() call may be invalid in the next // ConsumeData() call. @@ -740,16 +844,29 @@ class MessageDecoder::MessageDecoderImpl { } Status ConsumeBuffer(std::shared_ptr buffer) { + RETURN_NOT_OK(options_.Validate()); if (buffered_size_ == 0) { - while (buffer->size() >= next_required_size_) { + while (buffer->size() > 0 && buffer->size() >= next_required_size_) { auto used_size = next_required_size_; switch (state_) { - case State::INITIAL: + case State::INITIAL: { + ARROW_ASSIGN_OR_RAISE(auto token, ConsumeDataBufferInt32(buffer)); + bool is_eos = (bit_util::FromLittleEndian(token) == 0); RETURN_NOT_OK(ConsumeInitialBuffer(buffer)); + if (is_eos) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: + } + case State::METADATA_LENGTH: { + ARROW_ASSIGN_OR_RAISE(auto length, ConsumeDataBufferInt32(buffer)); + bool is_eos = (bit_util::FromLittleEndian(length) == 0); RETURN_NOT_OK(ConsumeMetadataLengthBuffer(buffer)); + if (is_eos) { + return Status::OK(); + } break; + } case State::METADATA: if (buffer->size() == next_required_size_) { return ConsumeMetadataBuffer(buffer); @@ -773,6 +890,9 @@ class MessageDecoder::MessageDecoderImpl { return Status::OK(); } buffer = SliceBuffer(buffer, used_size); + if (state_ == State::EOS) { + return Status::OK(); + } } } @@ -780,6 +900,10 @@ class MessageDecoder::MessageDecoderImpl { return Status::OK(); } + if (state_ == State::EOS) { + return Status::OK(); + } + buffered_size_ += buffer->size(); chunks_.push_back(std::move(buffer)); return ConsumeChunks(); @@ -799,12 +923,26 @@ class MessageDecoder::MessageDecoderImpl { } switch (state_) { - case State::INITIAL: - RETURN_NOT_OK(ConsumeInitialChunks()); + case State::INITIAL: { + int32_t continuation = 0; + RETURN_NOT_OK(ConsumeDataChunks(sizeof(int32_t), &continuation)); + auto token = bit_util::FromLittleEndian(continuation); + RETURN_NOT_OK(ConsumeInitial(token)); + if (token == 0) { + return Status::OK(); + } break; - case State::METADATA_LENGTH: - RETURN_NOT_OK(ConsumeMetadataLengthChunks()); + } + case State::METADATA_LENGTH: { + int32_t metadata_length = 0; + RETURN_NOT_OK(ConsumeDataChunks(sizeof(int32_t), &metadata_length)); + auto length = bit_util::FromLittleEndian(metadata_length); + RETURN_NOT_OK(ConsumeMetadataLength(length)); + if (length == 0) { + return Status::OK(); + } break; + } case State::METADATA: RETURN_NOT_OK(ConsumeMetadataChunks()); break; @@ -847,6 +985,7 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(listener_->OnEOS()); return Status::OK(); } else if (continuation > 0) { + RETURN_NOT_OK(options_.CheckMetadataLength(continuation)); state_ = State::METADATA; // ARROW-6314: Backwards compatibility for reading old IPC // messages produced prior to version 0.15.0 @@ -881,6 +1020,7 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(listener_->OnEOS()); return Status::OK(); } else if (metadata_length > 0) { + RETURN_NOT_OK(options_.CheckMetadataLength(metadata_length)); state_ = State::METADATA; next_required_size_ = metadata_length; RETURN_NOT_OK(listener_->OnMetadata()); @@ -929,6 +1069,7 @@ class MessageDecoder::MessageDecoderImpl { RETURN_NOT_OK(MaybeAlignMetadata(&metadata_)); int64_t body_length = -1; RETURN_NOT_OK(CheckMetadataAndGetBodyLength(*metadata_, &body_length)); + RETURN_NOT_OK(options_.CheckBodyLength(body_length)); state_ = State::BODY; next_required_size_ = skip_body_ ? 0 : body_length; @@ -987,7 +1128,10 @@ class MessageDecoder::MessageDecoderImpl { } Status ConsumeDataChunks(int64_t nbytes, void* out) { - size_t offset = 0; + DCHECK_GE(nbytes, 0); + DCHECK_GE(buffered_size_, nbytes); + std::span out_span(static_cast(out), static_cast(nbytes)); + int64_t offset = 0; size_t n_used_chunks = 0; auto required_size = nbytes; std::shared_ptr last_chunk; @@ -995,22 +1139,23 @@ class MessageDecoder::MessageDecoderImpl { if (!chunk->is_cpu()) { ARROW_ASSIGN_OR_RAISE(chunk, Buffer::ViewOrCopy(chunk, memory_manager_)); } - auto data = chunk->data(); - auto data_size = chunk->size(); - auto copy_size = std::min(required_size, data_size); - memcpy(static_cast(out) + offset, data, copy_size); + auto copy_size = std::min(required_size, chunk->size()); + std::memcpy( + out_span.subspan(static_cast(offset), static_cast(copy_size)) + .data(), + chunk->data(), static_cast(copy_size)); n_used_chunks++; offset += copy_size; required_size -= copy_size; if (required_size == 0) { - if (data_size != copy_size) { + if (chunk->size() != copy_size) { last_chunk = SliceBuffer(chunk, copy_size); } break; } } chunks_.erase(chunks_.begin(), chunks_.begin() + n_used_chunks); - if (last_chunk.get() != nullptr) { + if (last_chunk != nullptr) { chunks_.insert(chunks_.begin(), std::move(last_chunk)); } buffered_size_ -= offset; @@ -1018,6 +1163,7 @@ class MessageDecoder::MessageDecoderImpl { } std::shared_ptr listener_; + IpcReadOptions options_; MemoryPool* pool_; std::shared_ptr memory_manager_; State state_; @@ -1042,6 +1188,21 @@ MessageDecoder::MessageDecoder(std::shared_ptr listener, initial_next_required_size, pool, skip_body)); } +MessageDecoder::MessageDecoder(std::shared_ptr listener, + IpcReadOptions options, bool skip_body) { + impl_.reset(new MessageDecoderImpl(std::move(listener), State::INITIAL, + kMessageDecoderNextRequiredSizeInitial, + std::move(options), skip_body)); +} + +MessageDecoder::MessageDecoder(std::shared_ptr listener, + State initial_state, int64_t initial_next_required_size, + IpcReadOptions options, bool skip_body) { + impl_.reset(new MessageDecoderImpl(std::move(listener), initial_state, + initial_next_required_size, std::move(options), + skip_body)); +} + MessageDecoder::~MessageDecoder() {} Status MessageDecoder::Consume(const uint8_t* data, int64_t size) { @@ -1064,14 +1225,17 @@ MessageDecoder::State MessageDecoder::state() const { return impl_->state(); } /// \brief Implementation of MessageReader that reads from InputStream class InputStreamMessageReader : public MessageReader, public MessageDecoderListener { public: - explicit InputStreamMessageReader(io::InputStream* stream) + explicit InputStreamMessageReader(io::InputStream* stream, + IpcReadOptions options = IpcReadOptions::Defaults()) : stream_(stream), owned_stream_(), message_(), - decoder_(std::shared_ptr(this, [](void*) {})) {} + decoder_(std::shared_ptr(this, [](void*) {}), + std::move(options)) {} - explicit InputStreamMessageReader(const std::shared_ptr& owned_stream) - : InputStreamMessageReader(owned_stream.get()) { + explicit InputStreamMessageReader(const std::shared_ptr& owned_stream, + IpcReadOptions options = IpcReadOptions::Defaults()) + : InputStreamMessageReader(owned_stream.get(), std::move(options)) { owned_stream_ = owned_stream; } @@ -1112,10 +1276,20 @@ std::unique_ptr MessageReader::Open(io::InputStream* stream) { return std::make_unique(stream); } +std::unique_ptr MessageReader::Open(io::InputStream* stream, + const IpcReadOptions& options) { + return std::make_unique(stream, options); +} + std::unique_ptr MessageReader::Open( const std::shared_ptr& owned_stream) { return std::make_unique(owned_stream); } +std::unique_ptr MessageReader::Open( + const std::shared_ptr& owned_stream, const IpcReadOptions& options) { + return std::make_unique(owned_stream, options); +} + } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/message.h b/cpp/src/arrow/ipc/message.h index 90346bd1972..cfba9dc89ae 100644 --- a/cpp/src/arrow/ipc/message.h +++ b/cpp/src/arrow/ipc/message.h @@ -26,6 +26,7 @@ #include #include "arrow/io/type_fwd.h" +#include "arrow/ipc/options.h" #include "arrow/ipc/type_fwd.h" #include "arrow/result.h" #include "arrow/status.h" @@ -289,6 +290,28 @@ class ARROW_EXPORT MessageDecoder { int64_t initial_next_required_size, MemoryPool* pool = default_memory_pool(), bool skip_body = false); + /// \brief Construct a message decoder with options. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] options options for reading IPC messages + /// \param[in] skip_body if true the body will be skipped even if the message has a body + explicit MessageDecoder(std::shared_ptr listener, + IpcReadOptions options, bool skip_body = false); + + /// \brief Construct a message decoder with the specified state and options. + /// + /// \param[in] listener a MessageDecoderListener that responds events from + /// the decoder + /// \param[in] initial_state an initial state of the decoder + /// \param[in] initial_next_required_size the number of bytes needed + /// to run the next action + /// \param[in] options options for reading IPC messages + /// \param[in] skip_body if true the body will be skipped even if the message has a body + MessageDecoder(std::shared_ptr listener, State initial_state, + int64_t initial_next_required_size, IpcReadOptions options, + bool skip_body = false); + virtual ~MessageDecoder(); /// \brief Feed data to the decoder as a raw data. @@ -439,10 +462,15 @@ class ARROW_EXPORT MessageReader { /// \brief Create MessageReader that reads from InputStream static std::unique_ptr Open(io::InputStream* stream); + static std::unique_ptr Open(io::InputStream* stream, + const IpcReadOptions& options); /// \brief Create MessageReader that reads from owned InputStream static std::unique_ptr Open( const std::shared_ptr& owned_stream); + static std::unique_ptr Open( + const std::shared_ptr& owned_stream, + const IpcReadOptions& options); /// \brief Read next Message from the interface /// @@ -499,6 +527,12 @@ Result> ReadMessage( io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(), const FieldsLoaderFunction& fields_loader = {}); +ARROW_EXPORT +Result> ReadMessage( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, const IpcReadOptions& options, + const FieldsLoaderFunction& fields_loader = {}); + /// \brief Read encapsulated IPC message from cached buffers /// /// The buffers should contain an entire message. Partial reads are not handled. @@ -518,11 +552,21 @@ Result> ReadMessage( ARROW_EXPORT Result> ReadMessage( std::shared_ptr metadata, std::shared_ptr body); +ARROW_EXPORT Result> ReadMessage( + std::shared_ptr metadata, std::shared_ptr body, + const IpcReadOptions& options); + ARROW_EXPORT Future> ReadMessageAsync( const int64_t offset, const int32_t metadata_length, const int64_t body_length, io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context()); +ARROW_EXPORT +Future> ReadMessageAsync( + const int64_t offset, const int32_t metadata_length, const int64_t body_length, + io::RandomAccessFile* file, const IpcReadOptions& options, + const io::IOContext& context = io::default_io_context()); + /// \brief Advance stream to an 8-byte offset if its position is not a multiple /// of 8 already /// \param[in] stream an input stream @@ -558,6 +602,10 @@ ARROW_EXPORT Result> ReadMessage(io::InputStream* stream, MemoryPool* pool = default_memory_pool()); +ARROW_EXPORT +Result> ReadMessage(io::InputStream* stream, + const IpcReadOptions& options); + /// \brief Feed data from InputStream to MessageDecoder to decode an /// encapsulated IPC message (metadata and body) /// diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc index 02134aa805a..1a5950141d1 100644 --- a/cpp/src/arrow/ipc/metadata_internal.cc +++ b/cpp/src/arrow/ipc/metadata_internal.cc @@ -1288,8 +1288,8 @@ Result> GetKeyValueMetadata( auto metadata = std::make_shared(); - metadata->reserve(fb_metadata->size()); for (const auto pair : *fb_metadata) { + CHECK_FLATBUFFERS_NOT_NULL(pair, "custom_metadata.pair"); CHECK_FLATBUFFERS_NOT_NULL(pair->key(), "custom_metadata.key"); CHECK_FLATBUFFERS_NOT_NULL(pair->value(), "custom_metadata.value"); metadata->Append(pair->key()->str(), pair->value()->str()); @@ -1474,11 +1474,16 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr* type if (tensor == nullptr) { return Status::IOError("Header-type of flatbuffer-encoded Message is not Tensor."); } + CHECK_FLATBUFFERS_NOT_NULL(tensor->shape(), "Tensor.shape"); flatbuffers::uoffset_t ndim = tensor->shape()->size(); for (flatbuffers::uoffset_t i = 0; i < ndim; ++i) { auto dim = tensor->shape()->Get(i); + CHECK_FLATBUFFERS_NOT_NULL(dim, "Tensor.shape[i]"); + if (dim->size() < 0) { + return Status::Invalid("Invalid tensor dimension size: ", dim->size()); + } shape->push_back(dim->size()); dim_names->push_back(StringFromFlatbuffers(dim->name())); @@ -1551,11 +1556,13 @@ Status GetSparseTensorMetadata(const Buffer& metadata, std::shared_ptr return Status::IOError( "Header-type of flatbuffer-encoded Message is not SparseTensor."); } + CHECK_FLATBUFFERS_NOT_NULL(sparse_tensor->shape(), "SparseTensor.shape"); int ndim = static_cast(sparse_tensor->shape()->size()); if (shape || dim_names) { for (int i = 0; i < ndim; ++i) { auto dim = sparse_tensor->shape()->Get(i); + CHECK_FLATBUFFERS_NOT_NULL(dim, "SparseTensor.shape[i]"); if (shape) { if (dim->size() < 0) { diff --git a/cpp/src/arrow/ipc/options.cc b/cpp/src/arrow/ipc/options.cc index e5b14a47fac..24b4701fa7d 100644 --- a/cpp/src/arrow/ipc/options.cc +++ b/cpp/src/arrow/ipc/options.cc @@ -26,6 +26,48 @@ IpcWriteOptions IpcWriteOptions::Defaults() { return IpcWriteOptions(); } IpcReadOptions IpcReadOptions::Defaults() { return IpcReadOptions(); } +Status IpcReadOptions::Validate() const { + if (max_recursion_depth < 0) { + return Status::Invalid("max_recursion_depth must be non-negative, got ", + max_recursion_depth); + } + if (max_metadata_length < -1) { + return Status::Invalid("max_metadata_length must be -1 or non-negative, got ", + max_metadata_length); + } + if (max_body_length < -1) { + return Status::Invalid("max_body_length must be -1 or non-negative, got ", + max_body_length); + } + if (max_decompression_ratio < -1) { + return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ", + max_decompression_ratio); + } + return Status::OK(); +} + +Status IpcReadOptions::CheckMetadataLength(int64_t length, const char* prefix) const { + if (length < 0) { + return Status::Invalid(prefix, length, " must be non-negative"); + } + if (max_metadata_length >= 0 && length > max_metadata_length) { + return Status::Invalid(prefix, length, " exceeds maximum allowed metadata length ", + max_metadata_length); + } + return Status::OK(); +} + +Status IpcReadOptions::CheckBodyLength(int64_t length) const { + if (length < 0) { + return Status::Invalid("IPC body length ", length, " must be non-negative"); + } + if (max_body_length >= 0 && length > max_body_length) { + return Status::Invalid("IPC body length ", length, + " exceeds maximum allowed body length ", max_body_length); + } + return Status::OK(); +} + namespace internal { Status CheckCompressionSupported(Compression::type codec) { diff --git a/cpp/src/arrow/ipc/options.h b/cpp/src/arrow/ipc/options.h index 254a4693866..5ee410f0413 100644 --- a/cpp/src/arrow/ipc/options.h +++ b/cpp/src/arrow/ipc/options.h @@ -189,6 +189,27 @@ struct ARROW_EXPORT IpcReadOptions { /// The lazy property will always be reset to true to deliver the expected behavior io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults(); + /// \brief Maximum allowed metadata length in bytes (-1 means unlimited). + /// + /// Enforced on message metadata in IPC streams and file footer metadata in IPC files. + int64_t max_metadata_length = -1; + + /// \brief Maximum allowed body length in bytes (-1 means unlimited). + int64_t max_body_length = -1; + + /// \brief Maximum allowed decompression expansion ratio (-1 means unlimited). + int32_t max_decompression_ratio = -1; + + /// \brief Validate read options. + Status Validate() const; + + /// \brief Validate that metadata length does not exceed max_metadata_length. + Status CheckMetadataLength(int64_t length, + const char* prefix = "IPC metadata length ") const; + + /// \brief Validate that body length does not exceed max_body_length. + Status CheckBodyLength(int64_t length) const; + static IpcReadOptions Defaults(); }; diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc index 811f620f40a..1bc9ed2988c 100644 --- a/cpp/src/arrow/ipc/read_write_test.cc +++ b/cpp/src/arrow/ipc/read_write_test.cc @@ -3401,6 +3401,298 @@ TEST(PreBuffering, MixedAccess) { ASSERT_EQ(2, stats.num_record_batches); } +TEST(TestMessageDecoderBounds, MalformedOrExceededHeaders) { + std::unique_ptr message; + auto listener = std::make_shared(&message); + + const uint8_t data[4] = {0}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("Invalid data size: negative byte count"), + MessageDecoder(listener).Consume(data, -1)); + + int32_t neg_token = -5; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("negative continuation token"), + MessageDecoder(listener).Consume(reinterpret_cast(&neg_token), + sizeof(int32_t))); + + int32_t neg_meta[2] = {internal::kIpcContinuationToken, -10}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("negative metadata length"), + MessageDecoder(listener).Consume(reinterpret_cast(neg_meta), + sizeof(neg_meta))); + + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 50; + int32_t exceeded_meta[2] = {internal::kIpcContinuationToken, 100}; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + MessageDecoder(listener, options) + .Consume(reinterpret_cast(exceeded_meta), + sizeof(exceeded_meta))); +} + +namespace { + +Result> MakeTestStreamBuffer() { + std::shared_ptr batch; + RETURN_NOT_OK(MakeIntRecordBatch(&batch)); + StreamWriterHelper writer; + RETURN_NOT_OK(writer.Init(batch->schema(), IpcWriteOptions::Defaults())); + RETURN_NOT_OK(writer.WriteBatch(batch)); + RETURN_NOT_OK(writer.Finish()); + return writer.buffer_; +} + +Result> MakeTestFileBuffer( + std::shared_ptr metadata = nullptr) { + std::shared_ptr batch; + RETURN_NOT_OK(MakeIntRecordBatch(&batch)); + FileWriterHelper writer; + RETURN_NOT_OK(writer.Init(batch->schema(), IpcWriteOptions::Defaults())); + RETURN_NOT_OK(writer.WriteBatch(batch, std::move(metadata))); + RETURN_NOT_OK(writer.Finish()); + return writer.buffer_; +} + +class MockCustomMessageReader : public MessageReader { + public: + explicit MockCustomMessageReader(std::vector> messages) + : messages_(std::move(messages)) {} + + Result> ReadNextMessage() override { + if (index_ < messages_.size()) { + return std::move(messages_[index_++]); + } + return nullptr; + } + + private: + std::vector> messages_; + size_t index_ = 0; +}; + +} // namespace + +TEST(TestStreamReaderLimits, MaxBodyLengthExceeded) { + ASSERT_OK_AND_ASSIGN(auto buffer, MakeTestStreamBuffer()); + + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + + // RecordBatchStreamReader + io::BufferReader stream_reader_buf(buffer); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchStreamReader::Open(&stream_reader_buf, options)); + std::shared_ptr out_batch; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadNext(&out_batch)); + + // StreamDecoder + auto listener = std::make_shared(); + StreamDecoder decoder(listener, options); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + decoder.Consume(buffer)); + + auto default_listener = std::make_shared(); + StreamDecoder default_decoder(default_listener); + ASSERT_OK(default_decoder.Consume(buffer)); + ASSERT_EQ(default_listener->num_record_batches(), 1); +} + +TEST(TestStreamReaderLimits, CustomMessageReaderLimitsExceeded) { + ASSERT_OK_AND_ASSIGN(auto buffer, MakeTestStreamBuffer()); + + auto make_custom_reader = [&]() -> std::unique_ptr { + std::vector> messages; + io::BufferReader r(buffer); + auto mr = MessageReader::Open(&r); + while (true) { + auto msg = mr->ReadNextMessage().ValueOrDie(); + if (!msg) break; + messages.push_back(std::move(msg)); + } + return std::make_unique(std::move(messages)); + }; + + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 5; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + RecordBatchStreamReader::Open(make_custom_reader(), options)); + + options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchStreamReader::Open(make_custom_reader(), options)); + std::shared_ptr out_batch; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadNext(&out_batch)); +} + +TEST(TestFileReaderLimits, MaxMetadataLengthExceeded) { + ASSERT_OK_AND_ASSIGN(auto buffer, MakeTestFileBuffer()); + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 10; + io::BufferReader buf_reader(buffer); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + RecordBatchFileReader::Open(&buf_reader, options)); + + auto metadata = key_value_metadata({"large_field"}, {std::string(2000, 'x')}); + ASSERT_OK_AND_ASSIGN(auto large_buffer, MakeTestFileBuffer(metadata)); + options.max_metadata_length = 500; + io::BufferReader large_buf_reader(large_buffer); + ASSERT_OK_AND_ASSIGN(auto reader, + RecordBatchFileReader::Open(&large_buf_reader, options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + reader->PreBufferMetadata({0})); +} + +TEST(TestFileReaderLimits, MaxBodyLengthExceeded) { + ASSERT_OK_AND_ASSIGN(auto buffer, MakeTestFileBuffer()); + auto options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + auto buf_reader = std::make_shared(buffer); + ASSERT_OK_AND_ASSIGN(auto reader, RecordBatchFileReader::Open(buf_reader, options)); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->ReadRecordBatch(0)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->PreBufferMetadata({0})); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + reader->GetRecordBatchGenerator(/*coalesce=*/true)); +} + +TEST(TestIpcReadOptions, NegativeLimitsInvalid) { + auto options = IpcReadOptions::Defaults(); + ASSERT_OK(options.Validate()); + + for (int64_t* opt : {&options.max_metadata_length, &options.max_body_length}) { + *opt = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("must be -1 or non-negative, got -2"), + options.Validate()); + *opt = -1; + } + + options.max_decompression_ratio = -2; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("max_decompression_ratio must be -1 or non-negative, got -2"), + options.Validate()); + options.max_decompression_ratio = -1; + + options.max_recursion_depth = -1; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("max_recursion_depth must be non-negative, got -1"), + options.Validate()); + + options = IpcReadOptions::Defaults(); + options.max_metadata_length = -2; + io::BufferReader empty_reader(Buffer::FromString("")); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("max_metadata_length must be -1 or non-negative"), + RecordBatchStreamReader::Open(&empty_reader, options)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("max_metadata_length must be -1 or non-negative"), + RecordBatchFileReader::Open(&empty_reader, options)); + auto empty_buf = Buffer::FromString(""); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("max_metadata_length must be -1 or non-negative"), + ReadMessage(empty_buf, nullptr, options)); +} + +TEST(TestIpcReadOptions, NegativeLengthsRejected) { + auto options = IpcReadOptions::Defaults(); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("IPC metadata length -10 must be non-negative"), + options.CheckMetadataLength(-10)); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("IPC body length -10 must be non-negative"), + options.CheckBodyLength(-10)); +} + +TEST(TestReadRecordBatchLimits, DirectRecordBatchLimitsExceeded) { + ASSERT_OK_AND_ASSIGN(auto buffer, MakeTestStreamBuffer()); + io::BufferReader stream_reader(buffer); + auto message_reader = MessageReader::Open(&stream_reader); + + ASSERT_OK_AND_ASSIGN(auto schema_message, message_reader->ReadNextMessage()); + ASSERT_NE(schema_message, nullptr); + DictionaryMemo memo; + ASSERT_OK_AND_ASSIGN(auto schema, ReadSchema(*schema_message, &memo)); + + ASSERT_OK_AND_ASSIGN(auto batch_message, message_reader->ReadNextMessage()); + ASSERT_NE(batch_message, nullptr); + + auto options = IpcReadOptions::Defaults(); + options.max_metadata_length = 5; + io::BufferReader body_reader(batch_message->body()); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed metadata length"), + ReadRecordBatch(*batch_message->metadata(), schema, &memo, options, + &body_reader)); + + options = IpcReadOptions::Defaults(); + options.max_body_length = 1; + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("exceeds maximum allowed body length"), + ReadRecordBatch(*batch_message, schema, &memo, options)); +} + +TEST(TestDecompressBufferLimits, Limits) { + if (!util::Codec::IsAvailable(Compression::LZ4_FRAME)) { + GTEST_SKIP() << "LZ4 compression not available"; + } + ASSERT_OK_AND_ASSIGN(auto codec, util::Codec::Create(Compression::LZ4_FRAME)); + + auto make_buf = [](int64_t uncompressed_size, size_t payload_len = 10) { + std::string buf_data(reinterpret_cast(&uncompressed_size), + sizeof(int64_t)); + buf_data.append(payload_len, 'x'); + return Buffer::FromString(std::move(buf_data)); + }; + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("negative capacity: -5"), + internal::DecompressBuffer(make_buf(-5, 16), IpcReadOptions::Defaults(), + codec.get())); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, + ::testing::HasSubstr( + "Compressed buffer has 0 payload bytes but non-zero uncompressed size 100"), + internal::DecompressBuffer(make_buf(100, 0), IpcReadOptions::Defaults(), + codec.get())); + + auto options = IpcReadOptions::Defaults(); + for (int ratio : {5, 0}) { + options.max_decompression_ratio = ratio; + EXPECT_RAISES_WITH_MESSAGE_THAT( + CapacityError, + ::testing::HasSubstr("exceeds maximum allowed decompression ratio"), + internal::DecompressBuffer(make_buf(1000), options, codec.get())); + } +} + +TEST(TestMetadataBounds, EmptyOrInvalidTensorMetadata) { + auto empty_buf = Buffer::FromString(""); + std::shared_ptr type; + std::vector shape; + std::vector strides; + std::vector dim_names; + EXPECT_RAISES_WITH_MESSAGE_THAT( + IOError, ::testing::HasSubstr("Invalid flatbuffers message"), + internal::GetTensorMetadata(*empty_buf, &type, &shape, &strides, &dim_names)); +} + } // namespace test } // namespace ipc } // namespace arrow diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc index 8a2c4729885..2bd2dae27c2 100644 --- a/cpp/src/arrow/ipc/reader.cc +++ b/cpp/src/arrow/ipc/reader.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -555,9 +556,14 @@ class ArrayLoader { ArrayData* out_ = nullptr; }; +} // namespace + +namespace internal { + Result> DecompressBuffer(const std::shared_ptr& buf, const IpcReadOptions& options, util::Codec* codec) { + RETURN_NOT_OK(options.Validate()); if (buf == nullptr || buf->size() == 0) { return buf; } @@ -576,12 +582,35 @@ Result> DecompressBuffer(const std::shared_ptr& return SliceBuffer(buf, sizeof(int64_t), compressed_size); } + if (compressed_size == 0 && uncompressed_size > 0) { + return Status::Invalid( + "Compressed buffer has 0 payload bytes but non-zero uncompressed size ", + uncompressed_size); + } + + if (options.max_decompression_ratio >= 0) { + int64_t max_allowed = 0; + if (ARROW_PREDICT_FALSE(::arrow::internal::MultiplyWithOverflow( + compressed_size, static_cast(options.max_decompression_ratio), + &max_allowed))) { + max_allowed = std::numeric_limits::max(); + } + if (ARROW_PREDICT_FALSE(uncompressed_size > max_allowed)) { + return Status::CapacityError("Decompressed size (", uncompressed_size, + ") exceeds maximum allowed decompression ratio (", + options.max_decompression_ratio, ")"); + } + } + + std::span compressed_payload(data + sizeof(int64_t), + static_cast(compressed_size)); + ARROW_ASSIGN_OR_RAISE(auto uncompressed, AllocateBuffer(uncompressed_size, options.memory_pool)); ARROW_ASSIGN_OR_RAISE( int64_t actual_decompressed, - codec->Decompress(compressed_size, data + sizeof(int64_t), uncompressed_size, + codec->Decompress(compressed_size, compressed_payload.data(), uncompressed_size, uncompressed->mutable_data())); if (actual_decompressed != uncompressed_size) { return Status::Invalid("Failed to fully decompress buffer, expected ", @@ -592,6 +621,12 @@ Result> DecompressBuffer(const std::shared_ptr& return uncompressed; } +} // namespace internal + +namespace { + +using internal::DecompressBuffer; + Status DecompressBuffers(Compression::type compression, const IpcReadOptions& options, ArrayDataVector* fields) { struct BufferAccumulator { @@ -795,7 +830,7 @@ Status GetCompressionExperimental(const flatbuf::Message* message, Status ReadContiguousPayload(io::InputStream* file, const IpcReadOptions& options, std::unique_ptr* message) { - ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options.memory_pool)); + ARROW_ASSIGN_OR_RAISE(*message, ReadMessage(file, options)); if (*message == nullptr) { return Status::Invalid("Unable to read metadata at offset"); } @@ -806,8 +841,11 @@ Result ReadRecordBatchInternal( const Buffer& metadata, const std::shared_ptr& schema, const std::vector& inclusion_mask, IpcReadContext& context, io::RandomAccessFile* file) { + RETURN_NOT_OK(context.options.Validate()); + RETURN_NOT_OK(context.options.CheckMetadataLength(metadata.size())); const flatbuf::Message* message = nullptr; RETURN_NOT_OK(internal::VerifyMessage(metadata.data(), metadata.size(), &message)); + RETURN_NOT_OK(context.options.CheckBodyLength(message->bodyLength())); auto batch = message->header_as_RecordBatch(); if (batch == nullptr) { return Status::IOError( @@ -1038,6 +1076,10 @@ class StreamDecoderInternal : public MessageDecoderListener { swap_endian_(false) {} Status OnMessageDecoded(std::unique_ptr message) override { + if (ARROW_PREDICT_TRUE(message != nullptr)) { + RETURN_NOT_OK(options_.CheckMetadataLength(message->metadata()->size())); + RETURN_NOT_OK(options_.CheckBodyLength(message->body_length())); + } ++stats_.num_messages; switch (state_) { case State::SCHEMA: @@ -1232,6 +1274,7 @@ class RecordBatchStreamReaderImpl : public RecordBatchStreamReader, Result> RecordBatchStreamReader::Open( std::unique_ptr message_reader, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // Private ctor auto result = std::make_shared(std::move(message_reader), options); @@ -1241,12 +1284,14 @@ Result> RecordBatchStreamReader::Open( Result> RecordBatchStreamReader::Open( io::InputStream* stream, const IpcReadOptions& options) { - return Open(MessageReader::Open(stream), options); + RETURN_NOT_OK(options.Validate()); + return Open(MessageReader::Open(stream, options), options); } Result> RecordBatchStreamReader::Open( const std::shared_ptr& stream, const IpcReadOptions& options) { - return Open(MessageReader::Open(stream), options); + RETURN_NOT_OK(options.Validate()); + return Open(MessageReader::Open(stream, options), options); } // ---------------------------------------------------------------------- @@ -1271,12 +1316,17 @@ Result FileBlockFromFlatbuffer(const flatbuf::Block* fb_block, return block; } -Status CheckAligned(const FileBlock& block) { +Status CheckBlock(const FileBlock& block, const IpcReadOptions& options) { + if (block.offset < 0) { + return Status::Invalid("negative block offset in IPC file"); + } if (!bit_util::IsMultipleOf8(block.offset) || !bit_util::IsMultipleOf8(block.metadata_length) || !bit_util::IsMultipleOf8(block.body_length)) { return Status::Invalid("Unaligned block in IPC file"); } + RETURN_NOT_OK(options.CheckMetadataLength(block.metadata_length)); + RETURN_NOT_OK(options.CheckBodyLength(block.body_length)); return Status::OK(); } @@ -1296,19 +1346,19 @@ Result CheckBodyLength(MessagePtr message, const FileBlock& block) { Result> ReadMessageFromBlock( const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, const FieldsLoaderFunction& fields_loader) { - RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlock(block, options)); ARROW_ASSIGN_OR_RAISE( auto message, ReadMessage(block.offset, block.metadata_length, block.body_length, - file, options.memory_pool, fields_loader)); + file, options, fields_loader)); return CheckBodyLength(std::move(message), block); } Future> ReadMessageFromBlockAsync( const FileBlock& block, const IpcReadOptions& options, io::RandomAccessFile* file, const io::IOContext& io_context) { - RETURN_NOT_OK(CheckAligned(block)); + RETURN_NOT_OK(CheckBlock(block, options)); return ReadMessageAsync(block.offset, block.metadata_length, block.body_length, file, - io_context.with_pool(options.memory_pool)) + options, io_context) .Then([block](std::shared_ptr message) { return CheckBodyLength(std::move(message), block); }); @@ -1501,6 +1551,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { Status Open(io::RandomAccessFile* file, int64_t footer_offset, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // The metadata_cache_ may have already been constructed with an owned file in the // owning overload of Open if (!metadata_cache_) { @@ -1530,6 +1581,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { Future<> OpenAsync(io::RandomAccessFile* file, int64_t footer_offset, const IpcReadOptions& options) { + RETURN_NOT_OK(options.Validate()); // The metadata_cache_ may have already been constructed with an owned file in the // owning overload of OpenAsync if (!metadata_cache_) { @@ -1583,6 +1635,14 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { std::shared_ptr cached_source; if (coalesce && !file_->supports_zero_copy()) { if (!owned_file_) return Status::Invalid("Cannot coalesce without an owned file"); + for (int i = 0; i < state->num_dictionaries(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto block, state->GetDictionaryBlock(i)); + RETURN_NOT_OK(CheckBlock(block, options_)); + } + for (int i = 0; i < state->num_record_batches(); ++i) { + ARROW_ASSIGN_OR_RAISE(auto block, state->GetRecordBatchBlock(i)); + RETURN_NOT_OK(CheckBlock(block, options_)); + } // Since the user is asking for all fields then we can cache the entire // file (up to the footer) cached_source = std::make_shared(file_, io_context, @@ -1602,10 +1662,11 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { all_metadata_ready.Then([this, index]() -> Result> { stats_.num_messages.fetch_add(1, std::memory_order_relaxed); ARROW_ASSIGN_OR_RAISE(FileBlock block, GetRecordBatchBlock(index)); + RETURN_NOT_OK(CheckBlock(block, options_)); ARROW_ASSIGN_OR_RAISE( std::shared_ptr metadata, metadata_cache_->Read({block.offset, block.metadata_length})); - return ReadMessage(std::move(metadata), nullptr); + return ReadMessage(std::move(metadata), nullptr, options_); }); cached_metadata_.emplace(index, metadata_loaded); } @@ -1708,6 +1769,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { // Adds all dictionaries to the range cache for (int i = 0; i < num_dictionaries(); ++i) { ARROW_ASSIGN_OR_RAISE(FileBlock block, GetDictionaryBlock(i)); + RETURN_NOT_OK(CheckBlock(block, options_)); ranges->push_back({block.offset, block.metadata_length + block.body_length}); } return Status::OK(); @@ -1717,6 +1779,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { std::vector* ranges) { for (int index : indices) { ARROW_ASSIGN_OR_RAISE(FileBlock block, GetRecordBatchBlock(index)); + RETURN_NOT_OK(CheckBlock(block, options_)); ranges->push_back({block.offset, block.metadata_length}); } return Status::OK(); @@ -1873,7 +1936,10 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { .Then([this, index](const std::shared_ptr& message_obj) -> Future> { ARROW_ASSIGN_OR_RAISE(auto block, GetRecordBatchBlock(index)); - ARROW_ASSIGN_OR_RAISE(auto message, GetFlatbufMessage(message_obj)); + RETURN_NOT_OK(CheckBlock(block, options_)); + ARROW_ASSIGN_OR_RAISE(auto checked_message, + CheckBodyLength(message_obj, block)); + ARROW_ASSIGN_OR_RAISE(auto message, GetFlatbufMessage(checked_message)); ARROW_ASSIGN_OR_RAISE(auto batch, GetBatchFromMessage(message)); ARROW_ASSIGN_OR_RAISE(auto context, GetIpcReadContext(message, batch)); @@ -1924,6 +1990,9 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader { return Status::Invalid("File is smaller than indicated metadata size"); } + RETURN_NOT_OK(self->options_.CheckMetadataLength( + footer_length, "IPC file footer metadata length ")); + // Now read the footer auto read_footer = self->file_->ReadAsync( self->footer_offset_ - footer_length - kTotalMagicSize, footer_length, @@ -2046,15 +2115,17 @@ WholeIpcFileRecordBatchGenerator::operator()() { Future> WholeIpcFileRecordBatchGenerator::ReadBlock( const FileBlock& block) { + RETURN_NOT_OK(CheckBlock(block, state_->options_)); if (cached_source_) { auto cached_source = cached_source_; io::ReadRange range{block.offset, block.metadata_length + block.body_length}; - auto pool = state_->options_.memory_pool; + auto options = state_->options_; return cached_source->WaitFor({range}).Then( - [cached_source, pool, range]() -> Result> { + [cached_source, options, range, block]() -> Result> { ARROW_ASSIGN_OR_RAISE(auto buffer, cached_source->Read(range)); io::BufferReader stream(std::move(buffer)); - return ReadMessage(&stream, pool); + ARROW_ASSIGN_OR_RAISE(auto message, ReadMessage(&stream, options)); + return CheckBodyLength(std::move(message), block); }); } else { return ReadMessageFromBlockAsync(block, state_->options_, state_->file_, io_context_); @@ -2167,7 +2238,7 @@ class StreamDecoder::StreamDecoderImpl : public StreamDecoderInternal { explicit StreamDecoderImpl(std::shared_ptr listener, IpcReadOptions options) : StreamDecoderInternal(std::move(listener), options), message_decoder_(std::shared_ptr(this, [](void*) {}), - options.memory_pool) {} + options) {} Status Consume(const uint8_t* data, int64_t size) { return message_decoder_.Consume(data, size); diff --git a/cpp/src/arrow/ipc/reader_internal.h b/cpp/src/arrow/ipc/reader_internal.h index d61ac96f0f1..984be965a95 100644 --- a/cpp/src/arrow/ipc/reader_internal.h +++ b/cpp/src/arrow/ipc/reader_internal.h @@ -29,9 +29,21 @@ namespace io { struct ReadRange; } +namespace util { +class Codec; +} + namespace ipc { +struct IpcReadOptions; + namespace internal { + +ARROW_EXPORT +Result> DecompressBuffer(const std::shared_ptr& buf, + const IpcReadOptions& options, + util::Codec* codec); + /// \class IoRecordedRandomAccessFile /// \brief An RandomAccessFile that doesn't perform real IO, but only save all the IO /// operations it receives, including read operation's , for replaying