Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions cpp/src/parquet/arrow/arrow_reader_writer_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5348,6 +5348,66 @@ TEST_P(TestArrowWriteDictionary, NestedDictionaryParentNullStatistics) {
}
}

TEST_P(TestArrowWriteDictionary, FallbackUsesConfiguredEncoding) {
ASSERT_OK_AND_ASSIGN(auto chunk0, ::arrow::DictionaryArray::FromArrays(
ArrayFromJSON(::arrow::int32(), "[0, 1, 0, 1]"),
ArrayFromJSON(::arrow::utf8(), R"(["a", "b"])")));
ASSERT_OK_AND_ASSIGN(auto chunk1, ::arrow::DictionaryArray::FromArrays(
ArrayFromJSON(::arrow::int32(), "[0, 1, 0, 1]"),
ArrayFromJSON(::arrow::utf8(), R"(["c", "d"])")));

auto schema = ::arrow::schema({::arrow::field("values", chunk0->type())});
auto chunked = std::make_shared<ChunkedArray>(::arrow::ArrayVector{chunk0, chunk1});
auto table = ::arrow::Table::Make(schema, {chunked});

auto sink = CreateOutputStream();
auto writer_properties = WriterProperties::Builder()
.version(ParquetVersion::PARQUET_2_6)
->data_page_version(this->GetParquetDataPageVersion())
->enable_dictionary()
->encoding("values", Encoding::DELTA_LENGTH_BYTE_ARRAY)
->build();
ASSERT_OK_AND_ASSIGN(
auto writer,
FileWriter::Open(*schema, ::arrow::default_memory_pool(), sink, writer_properties,
default_arrow_writer_properties()));
ASSERT_OK(writer->WriteTable(*table, std::numeric_limits<int64_t>::max()));
ASSERT_OK(writer->Close());
ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish());

auto parquet_reader = ParquetFileReader::Open(std::make_shared<BufferReader>(buffer));
auto col_meta = parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0);
int32_t dict_pages = 0;
int32_t rle_dict_pages = 0;
int32_t dlba_pages = 0;
int32_t plain_data_pages = 0;
for (const auto& stat : col_meta->encoding_stats()) {
if (stat.page_type == PageType::DICTIONARY_PAGE && stat.encoding == Encoding::PLAIN) {
dict_pages += stat.count;
} else if (stat.page_type == PageType::DATA_PAGE) {
if (stat.encoding == Encoding::RLE_DICTIONARY) {
rle_dict_pages += stat.count;
} else if (stat.encoding == Encoding::DELTA_LENGTH_BYTE_ARRAY) {
dlba_pages += stat.count;
} else if (stat.encoding == Encoding::PLAIN) {
plain_data_pages += stat.count;
}
}
}
EXPECT_EQ(dict_pages, 1);
EXPECT_GE(rle_dict_pages, 1);
EXPECT_GE(dlba_pages, 1);
EXPECT_EQ(plain_data_pages, 0);

ASSERT_OK_AND_ASSIGN(auto reader, OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool()));
ASSERT_OK_AND_ASSIGN(auto actual, reader->ReadTable());
auto expected = ::arrow::Table::Make(
::arrow::schema({::arrow::field("values", ::arrow::utf8())}),
{ArrayFromJSON(::arrow::utf8(), R"(["a", "b", "a", "b", "c", "d", "c", "d"])")});
::arrow::AssertTablesEqual(*expected, *actual);
}

// ----------------------------------------------------------------------
// Tests for directly reading DictionaryArray

Expand Down
96 changes: 54 additions & 42 deletions cpp/src/parquet/column_writer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -744,14 +744,16 @@ class ColumnWriterImpl {
public:
ColumnWriterImpl(ColumnChunkMetaDataBuilder* metadata,
std::unique_ptr<PageWriter> pager, const bool use_dictionary,
Encoding::type encoding, const WriterProperties* properties,
Encoding::type encoding, Encoding::type fallback_encoding,
const WriterProperties* properties,
internal::ContentDefinedChunker* content_defined_chunker)
: metadata_(metadata),
descr_(metadata->descr()),
level_info_(internal::LevelInfo::ComputeLevelInfo(metadata->descr())),
pager_(std::move(pager)),
has_dictionary_(use_dictionary),
encoding_(encoding),
fallback_encoding_(fallback_encoding),
properties_(properties),
allocator_(properties->memory_pool()),
num_buffered_values_(0),
Expand Down Expand Up @@ -857,6 +859,7 @@ class ColumnWriterImpl {

bool has_dictionary_;
Encoding::type encoding_;
const Encoding::type fallback_encoding_;
const WriterProperties* properties_;

LevelEncoder level_encoder_;
Expand Down Expand Up @@ -894,7 +897,7 @@ class ColumnWriterImpl {
// Flag to check if the Writer has been closed
bool closed_;

// Flag to infer if dictionary encoding has fallen back to PLAIN
// Flag to infer if dictionary encoding has fallen back
bool fallback_;

::arrow::BufferBuilder definition_levels_sink_;
Expand Down Expand Up @@ -1030,7 +1033,7 @@ void ColumnWriterImpl::BuildDataPageV1(int64_t definition_levels_rle_size,
int64_t first_row_index = rows_written_ - num_buffered_rows_;

// Write the page to OutputStream eagerly if there is no dictionary or
// if dictionary encoding has fallen back to PLAIN
// if dictionary encoding has fallen back
if (has_dictionary_ && !fallback_) { // Save pages until end of dictionary encoding
PARQUET_ASSIGN_OR_THROW(
auto compressed_data_copy,
Expand Down Expand Up @@ -1093,7 +1096,7 @@ void ColumnWriterImpl::BuildDataPageV2(int64_t definition_levels_rle_size,
DCHECK(!page_stats.has_null_count || page_stats.null_count == null_count);

// Write the page to OutputStream eagerly if there is no dictionary or
// if dictionary encoding has fallen back to PLAIN
// if dictionary encoding has fallen back
if (has_dictionary_ && !fallback_) { // Save pages until end of dictionary encoding
std::unique_ptr<DataPage> page_ptr = std::make_unique<DataPageV2>(
combined, num_values, null_count, num_rows, encoding_, def_levels_byte_length,
Expand Down Expand Up @@ -1287,11 +1290,18 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,

TypedColumnWriterImpl(ColumnChunkMetaDataBuilder* metadata,
std::unique_ptr<PageWriter> pager, const bool use_dictionary,
Encoding::type encoding, const WriterProperties* properties,
BloomFilter* bloom_filter,
Encoding::type encoding, Encoding::type fallback_encoding,
const WriterProperties* properties, BloomFilter* bloom_filter,
internal::ContentDefinedChunker* content_defined_chunker)
: ColumnWriterImpl(metadata, std::move(pager), use_dictionary, encoding, properties,
content_defined_chunker) {
: ColumnWriterImpl(metadata, std::move(pager), use_dictionary, encoding,
fallback_encoding, properties, content_defined_chunker) {
if (use_dictionary && fallback_encoding != Encoding::PLAIN) {
// Build the fallback encoder upfront so an unsupported encoding/type pair
// fails on ColumnWriter creation rather than mid-row-group on overflow.
fallback_encoder_ =
MakeEncoder(ParquetType::type_num, fallback_encoding,
/*use_dictionary=*/false, descr_, properties->memory_pool());
}
current_encoder_ = MakeEncoder(ParquetType::type_num, encoding, use_dictionary,
descr_, properties->memory_pool());
// We have to dynamic_cast as some compilers don't want to static_cast
Expand Down Expand Up @@ -1683,6 +1693,8 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
std::shared_ptr<geospatial::GeoStatistics> chunk_geospatial_statistics_;
std::unique_ptr<BloomFilterWriter> bloom_filter_writer_;
bool pages_change_on_record_boundaries_;
// Pre-built non-PLAIN fallback encoder, swapped in on dictionary overflow.
std::unique_ptr<Encoder> fallback_encoder_;

// If writing a sequence of ::arrow::DictionaryArray to the writer, we keep the
// dictionary passed to DictEncoder<T>::PutDictionary so we can check
Expand Down Expand Up @@ -1874,25 +1886,26 @@ class TypedColumnWriterImpl : public ColumnWriterImpl,
// Serialize the buffered Dictionary Indices
FlushBufferedDataPages();
fallback_ = true;
// Only PLAIN encoding is supported for fallback
current_encoder_ = MakeEncoder(ParquetType::type_num, Encoding::PLAIN, false,
descr_, properties_->memory_pool());
current_encoder_ = fallback_encoder_ != nullptr
? std::move(fallback_encoder_)
: MakeEncoder(ParquetType::type_num, Encoding::PLAIN, false,
descr_, properties_->memory_pool());
current_value_encoder_ = dynamic_cast<ValueEncoderType*>(current_encoder_.get());
current_dict_encoder_ = nullptr; // not using dict
encoding_ = Encoding::PLAIN;
encoding_ = fallback_encoding_;
}
}

// Checks if the Dictionary Page size limit is reached
// If the limit is reached, the Dictionary and Data Pages are serialized
// The encoding is switched to PLAIN
// The encoding is switched to fallback_encoding_ (PLAIN by default)
//
// Only one Dictionary Page is written.
// Fallback to PLAIN if dictionary page limit is reached.
// Fallback to fallback_encoding_ if dictionary page limit is reached.
void CheckDictionarySizeLimit() {
if (!has_dictionary_ || fallback_) {
// Either not using dictionary encoding, or we have already fallen back
// to PLAIN encoding because the size threshold was reached
// to the non-dictionary encoding because the size threshold was reached
return;
}

Expand Down Expand Up @@ -2733,54 +2746,53 @@ std::shared_ptr<ColumnWriter> ColumnWriter::Make(
const ColumnDescriptor* descr = metadata->descr();
const bool use_dictionary = properties->dictionary_enabled(descr->path()) &&
descr->physical_type() != Type::BOOLEAN;
Encoding::type encoding = properties->encoding(descr->path());
if (encoding == Encoding::UNKNOWN) {
encoding = (descr->physical_type() == Type::BOOLEAN &&
properties->version() != ParquetVersion::PARQUET_1_0 &&
properties->data_page_version() == ParquetDataPageVersion::V2)
? Encoding::RLE
: Encoding::PLAIN;
}
if (use_dictionary) {
encoding = properties->dictionary_index_encoding();
}
Encoding::type fallback_encoding = properties->encoding(descr->path());
if (fallback_encoding == Encoding::UNKNOWN) {
fallback_encoding = (descr->physical_type() == Type::BOOLEAN &&
properties->version() != ParquetVersion::PARQUET_1_0 &&
properties->data_page_version() == ParquetDataPageVersion::V2)
? Encoding::RLE
: Encoding::PLAIN;
}
const Encoding::type encoding =
use_dictionary ? properties->dictionary_index_encoding() : fallback_encoding;
switch (descr->physical_type()) {
case Type::BOOLEAN: {
if (bloom_filter != nullptr) {
throw ParquetException("Bloom filter is not supported for boolean type");
}
return std::make_shared<TypedColumnWriterImpl<BooleanType>>(
metadata, std::move(pager), use_dictionary, encoding, properties,
/*bloom_filter=*/nullptr, content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, /*bloom_filter=*/nullptr, content_defined_chunker);
}
case Type::INT32:
return std::make_shared<TypedColumnWriterImpl<Int32Type>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::INT64:
return std::make_shared<TypedColumnWriterImpl<Int64Type>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::INT96:
return std::make_shared<TypedColumnWriterImpl<Int96Type>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::FLOAT:
return std::make_shared<TypedColumnWriterImpl<FloatType>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::DOUBLE:
return std::make_shared<TypedColumnWriterImpl<DoubleType>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::BYTE_ARRAY:
return std::make_shared<TypedColumnWriterImpl<ByteArrayType>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
case Type::FIXED_LEN_BYTE_ARRAY:
return std::make_shared<TypedColumnWriterImpl<FLBAType>>(
metadata, std::move(pager), use_dictionary, encoding, properties, bloom_filter,
content_defined_chunker);
metadata, std::move(pager), use_dictionary, encoding, fallback_encoding,
properties, bloom_filter, content_defined_chunker);
default:
ParquetException::NYI("Column writer not implemented for type: " +
TypeToString(descr->physical_type()));
Expand Down
Loading