Skip to content
Open
254 changes: 214 additions & 40 deletions cpp/src/arrow/ipc/message.cc

Large diffs are not rendered by default.

48 changes: 48 additions & 0 deletions cpp/src/arrow/ipc/message.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <utility>

#include "arrow/io/type_fwd.h"
#include "arrow/ipc/options.h"
#include "arrow/ipc/type_fwd.h"
#include "arrow/result.h"
#include "arrow/status.h"
Expand Down Expand Up @@ -289,6 +290,28 @@ class ARROW_EXPORT MessageDecoder {
int64_t initial_next_required_size,
MemoryPool* pool = default_memory_pool(), bool skip_body = false);

/// \brief Construct a message decoder with options.
///
/// \param[in] listener a MessageDecoderListener that responds events from
/// the decoder
/// \param[in] options options for reading IPC messages
/// \param[in] skip_body if true the body will be skipped even if the message has a body
explicit MessageDecoder(std::shared_ptr<MessageDecoderListener> listener,
IpcReadOptions options, bool skip_body = false);

/// \brief Construct a message decoder with the specified state and options.
///
/// \param[in] listener a MessageDecoderListener that responds events from
/// the decoder
/// \param[in] initial_state an initial state of the decoder
/// \param[in] initial_next_required_size the number of bytes needed
/// to run the next action
/// \param[in] options options for reading IPC messages
/// \param[in] skip_body if true the body will be skipped even if the message has a body
MessageDecoder(std::shared_ptr<MessageDecoderListener> listener, State initial_state,
int64_t initial_next_required_size, IpcReadOptions options,
bool skip_body = false);

virtual ~MessageDecoder();

/// \brief Feed data to the decoder as a raw data.
Expand Down Expand Up @@ -439,10 +462,15 @@ class ARROW_EXPORT MessageReader {

/// \brief Create MessageReader that reads from InputStream
static std::unique_ptr<MessageReader> Open(io::InputStream* stream);
static std::unique_ptr<MessageReader> Open(io::InputStream* stream,
const IpcReadOptions& options);

/// \brief Create MessageReader that reads from owned InputStream
static std::unique_ptr<MessageReader> Open(
const std::shared_ptr<io::InputStream>& owned_stream);
static std::unique_ptr<MessageReader> Open(
const std::shared_ptr<io::InputStream>& owned_stream,
const IpcReadOptions& options);

/// \brief Read next Message from the interface
///
Expand Down Expand Up @@ -499,6 +527,12 @@ Result<std::unique_ptr<Message>> ReadMessage(
io::RandomAccessFile* file, MemoryPool* pool = default_memory_pool(),
const FieldsLoaderFunction& fields_loader = {});

ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const IpcReadOptions& options,
const FieldsLoaderFunction& fields_loader = {});

/// \brief Read encapsulated IPC message from cached buffers
///
/// The buffers should contain an entire message. Partial reads are not handled.
Expand All @@ -518,11 +552,21 @@ Result<std::unique_ptr<Message>> ReadMessage(
ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body);

ARROW_EXPORT Result<std::unique_ptr<Message>> ReadMessage(
std::shared_ptr<Buffer> metadata, std::shared_ptr<Buffer> body,
const IpcReadOptions& options);

ARROW_EXPORT
Future<std::shared_ptr<Message>> ReadMessageAsync(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const io::IOContext& context = io::default_io_context());

ARROW_EXPORT
Future<std::shared_ptr<Message>> ReadMessageAsync(
const int64_t offset, const int32_t metadata_length, const int64_t body_length,
io::RandomAccessFile* file, const IpcReadOptions& options,
const io::IOContext& context = io::default_io_context());

/// \brief Advance stream to an 8-byte offset if its position is not a multiple
/// of 8 already
/// \param[in] stream an input stream
Expand Down Expand Up @@ -558,6 +602,10 @@ ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
MemoryPool* pool = default_memory_pool());

ARROW_EXPORT
Result<std::unique_ptr<Message>> ReadMessage(io::InputStream* stream,
const IpcReadOptions& options);

/// \brief Feed data from InputStream to MessageDecoder to decode an
/// encapsulated IPC message (metadata and body)
///
Expand Down
9 changes: 8 additions & 1 deletion cpp/src/arrow/ipc/metadata_internal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1288,8 +1288,8 @@ Result<std::shared_ptr<KeyValueMetadata>> GetKeyValueMetadata(

auto metadata = std::make_shared<KeyValueMetadata>();

metadata->reserve(fb_metadata->size());
for (const auto pair : *fb_metadata) {
CHECK_FLATBUFFERS_NOT_NULL(pair, "custom_metadata.pair");
CHECK_FLATBUFFERS_NOT_NULL(pair->key(), "custom_metadata.key");
CHECK_FLATBUFFERS_NOT_NULL(pair->value(), "custom_metadata.value");
metadata->Append(pair->key()->str(), pair->value()->str());
Expand Down Expand Up @@ -1474,11 +1474,16 @@ Status GetTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>* type
if (tensor == nullptr) {
return Status::IOError("Header-type of flatbuffer-encoded Message is not Tensor.");
}
CHECK_FLATBUFFERS_NOT_NULL(tensor->shape(), "Tensor.shape");

flatbuffers::uoffset_t ndim = tensor->shape()->size();

for (flatbuffers::uoffset_t i = 0; i < ndim; ++i) {
auto dim = tensor->shape()->Get(i);
CHECK_FLATBUFFERS_NOT_NULL(dim, "Tensor.shape[i]");
if (dim->size() < 0) {
return Status::Invalid("Invalid tensor dimension size: ", dim->size());
}

shape->push_back(dim->size());
dim_names->push_back(StringFromFlatbuffers(dim->name()));
Expand Down Expand Up @@ -1551,11 +1556,13 @@ Status GetSparseTensorMetadata(const Buffer& metadata, std::shared_ptr<DataType>
return Status::IOError(
"Header-type of flatbuffer-encoded Message is not SparseTensor.");
}
CHECK_FLATBUFFERS_NOT_NULL(sparse_tensor->shape(), "SparseTensor.shape");
int ndim = static_cast<int>(sparse_tensor->shape()->size());

if (shape || dim_names) {
for (int i = 0; i < ndim; ++i) {
auto dim = sparse_tensor->shape()->Get(i);
CHECK_FLATBUFFERS_NOT_NULL(dim, "SparseTensor.shape[i]");

if (shape) {
if (dim->size() < 0) {
Expand Down
42 changes: 42 additions & 0 deletions cpp/src/arrow/ipc/options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,48 @@ IpcWriteOptions IpcWriteOptions::Defaults() { return IpcWriteOptions(); }

IpcReadOptions IpcReadOptions::Defaults() { return IpcReadOptions(); }

Status IpcReadOptions::Validate() const {
if (max_recursion_depth < 0) {
return Status::Invalid("max_recursion_depth must be non-negative, got ",
max_recursion_depth);
}
if (max_metadata_length < -1) {
return Status::Invalid("max_metadata_length must be -1 or non-negative, got ",
max_metadata_length);
}
if (max_body_length < -1) {
return Status::Invalid("max_body_length must be -1 or non-negative, got ",
max_body_length);
}
if (max_decompression_ratio < -1) {
return Status::Invalid("max_decompression_ratio must be -1 or non-negative, got ",
max_decompression_ratio);
}
return Status::OK();
}

Status IpcReadOptions::CheckMetadataLength(int64_t length, const char* prefix) const {
if (length < 0) {
return Status::Invalid(prefix, length, " must be non-negative");
}
if (max_metadata_length >= 0 && length > max_metadata_length) {
return Status::Invalid(prefix, length, " exceeds maximum allowed metadata length ",
max_metadata_length);
}
return Status::OK();
}

Status IpcReadOptions::CheckBodyLength(int64_t length) const {
if (length < 0) {
return Status::Invalid("IPC body length ", length, " must be non-negative");
}
if (max_body_length >= 0 && length > max_body_length) {
return Status::Invalid("IPC body length ", length,
" exceeds maximum allowed body length ", max_body_length);
}
return Status::OK();
}

namespace internal {

Status CheckCompressionSupported(Compression::type codec) {
Expand Down
21 changes: 21 additions & 0 deletions cpp/src/arrow/ipc/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,27 @@ struct ARROW_EXPORT IpcReadOptions {
/// The lazy property will always be reset to true to deliver the expected behavior
io::CacheOptions pre_buffer_cache_options = io::CacheOptions::LazyDefaults();

/// \brief Maximum allowed metadata length in bytes (-1 means unlimited).
///
/// Enforced on message metadata in IPC streams and file footer metadata in IPC files.
int64_t max_metadata_length = -1;

/// \brief Maximum allowed body length in bytes (-1 means unlimited).
int64_t max_body_length = -1;

/// \brief Maximum allowed decompression expansion ratio (-1 means unlimited).
int32_t max_decompression_ratio = -1;

/// \brief Validate read options.
Status Validate() const;

/// \brief Validate that metadata length does not exceed max_metadata_length.
Status CheckMetadataLength(int64_t length,
const char* prefix = "IPC metadata length ") const;

/// \brief Validate that body length does not exceed max_body_length.
Status CheckBodyLength(int64_t length) const;

static IpcReadOptions Defaults();
};

Expand Down
Loading