Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions monai/data/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1167,7 +1167,8 @@ def partition_dataset(

Args:
data: input dataset to split, expect a list of data.
ratios: a list of ratio number to split the dataset, like [8, 1, 1].
ratios: a list of ratio number to split the dataset, like [8, 1, 1]. If rounding the partition sizes
leaves some items over, they are added to the partitions that were rounded down the most.
num_partitions: expected number of the partitions to evenly split, only works when `ratios` not specified.
shuffle: whether to shuffle the original dataset before splitting.
seed: random seed to shuffle the dataset, only works when `shuffle` is True.
Expand Down Expand Up @@ -1203,9 +1204,19 @@ def partition_dataset(
if ratios:
next_idx = 0
rsum = sum(ratios)
for r in ratios:
exact = [r / rsum * data_len for r in ratios]
sizes = [int(e + 0.5) for e in exact]
# rounding each size on its own can leave items without a partition,
# give them, one each, to the partitions whose sizes were rounded down
# the most (ties go to the earlier partition, as in
# torch.utils.data.random_split), so each partition stays within one
# item of its exact share
missing = max(data_len - sum(sizes), 0)
for j in sorted(range(len(sizes)), key=lambda k: sizes[k] - exact[k])[:missing]:
sizes[j] += 1
for size in sizes:
start_idx = next_idx
next_idx = min(start_idx + int(r / rsum * data_len + 0.5), data_len)
next_idx = min(start_idx + size, data_len)
datasets.append([data[i] for i in indices[start_idx:next_idx]])
return datasets

Expand Down Expand Up @@ -1257,7 +1268,8 @@ def partition_dataset_classes(
Args:
data: input dataset to split, expect a list of data.
classes: a list of labels to help split the data, the length must match the length of data.
ratios: a list of ratio number to split the dataset, like [8, 1, 1].
ratios: a list of ratio number to split the dataset, like [8, 1, 1]. If rounding the partition sizes
leaves some items over, they are added to the partitions that were rounded down the most.
num_partitions: expected number of the partitions to evenly split, only works when no `ratios`.
shuffle: whether to shuffle the original dataset before splitting.
seed: random seed to shuffle the dataset, only works when `shuffle` is True.
Expand Down
100 changes: 98 additions & 2 deletions tests/data/test_partition_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,106 @@
]


class TestPartitionDataset(unittest.TestCase):
TEST_CASE_9 = [
{
"data": list(range(10)),
"ratios": [1, 1, 1],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]],
]

TEST_CASE_10 = [
{
"data": list(range(63)),
"ratios": [0.8, 0.1, 0.1],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
[list(range(51)), list(range(51, 57)), list(range(57, 63))],
]

TEST_CASE_11 = [
{
"data": list(range(32)),
"ratios": [1, 1, 1, 1, 1],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
[list(range(7)), list(range(7, 14)), list(range(14, 20)), list(range(20, 26)), list(range(26, 32))],
]

TEST_CASE_12 = [
{
"data": [1, 2, 3, 4, 5],
"ratios": [0.6, 0.2, 0.2],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
[[1, 2, 3], [4], [5]],
]

TEST_CASE_13 = [
{
"data": list(range(10)),
"ratios": [1, 1, 1, 0],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9], []],
]


TEST_CASE_14 = [
{
"data": list(range(52)),
"ratios": [3, 1, 3, 8],
"num_partitions": None,
"shuffle": False,
"seed": 0,
"drop_last": True,
"even_divisible": True,
},
# the largest rounding deficit is in the second partition, not the first,
# so the leftover item goes there rather than to partition 0
[list(range(10)), list(range(10, 14)), list(range(14, 24)), list(range(24, 52))],
]


class TestPartitionDataset(unittest.TestCase):
@parameterized.expand(
[TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
[
TEST_CASE_1,
TEST_CASE_2,
TEST_CASE_3,
TEST_CASE_4,
TEST_CASE_5,
TEST_CASE_6,
TEST_CASE_7,
TEST_CASE_8,
TEST_CASE_9,
TEST_CASE_10,
TEST_CASE_11,
TEST_CASE_12,
TEST_CASE_13,
TEST_CASE_14,
]
)
def test_value(self, input_param, result):
self.assertListEqual(partition_dataset(**input_param), result)
Expand Down
Loading