diff --git a/monai/data/utils.py b/monai/data/utils.py index d548ed72486..f10e572863c 100644 --- a/monai/data/utils.py +++ b/monai/data/utils.py @@ -1167,7 +1167,8 @@ def partition_dataset( Args: data: input dataset to split, expect a list of data. - ratios: a list of ratio number to split the dataset, like [8, 1, 1]. + ratios: a list of ratio number to split the dataset, like [8, 1, 1]. If rounding the partition sizes + leaves some items over, they are added to the partitions that were rounded down the most. num_partitions: expected number of the partitions to evenly split, only works when `ratios` not specified. shuffle: whether to shuffle the original dataset before splitting. seed: random seed to shuffle the dataset, only works when `shuffle` is True. @@ -1203,9 +1204,19 @@ def partition_dataset( if ratios: next_idx = 0 rsum = sum(ratios) - for r in ratios: + exact = [r / rsum * data_len for r in ratios] + sizes = [int(e + 0.5) for e in exact] + # rounding each size on its own can leave items without a partition, + # give them, one each, to the partitions whose sizes were rounded down + # the most (ties go to the earlier partition, as in + # torch.utils.data.random_split), so each partition stays within one + # item of its exact share + missing = max(data_len - sum(sizes), 0) + for j in sorted(range(len(sizes)), key=lambda k: sizes[k] - exact[k])[:missing]: + sizes[j] += 1 + for size in sizes: start_idx = next_idx - next_idx = min(start_idx + int(r / rsum * data_len + 0.5), data_len) + next_idx = min(start_idx + size, data_len) datasets.append([data[i] for i in indices[start_idx:next_idx]]) return datasets @@ -1257,7 +1268,8 @@ def partition_dataset_classes( Args: data: input dataset to split, expect a list of data. classes: a list of labels to help split the data, the length must match the length of data. - ratios: a list of ratio number to split the dataset, like [8, 1, 1]. + ratios: a list of ratio number to split the dataset, like [8, 1, 1]. If rounding the partition sizes + leaves some items over, they are added to the partitions that were rounded down the most. num_partitions: expected number of the partitions to evenly split, only works when no `ratios`. shuffle: whether to shuffle the original dataset before splitting. seed: random seed to shuffle the dataset, only works when `shuffle` is True. diff --git a/tests/data/test_partition_dataset.py b/tests/data/test_partition_dataset.py index c93a6c76821..8bcba15a9b8 100644 --- a/tests/data/test_partition_dataset.py +++ b/tests/data/test_partition_dataset.py @@ -117,10 +117,106 @@ ] -class TestPartitionDataset(unittest.TestCase): +TEST_CASE_9 = [ + { + "data": list(range(10)), + "ratios": [1, 1, 1], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + [[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]], +] + +TEST_CASE_10 = [ + { + "data": list(range(63)), + "ratios": [0.8, 0.1, 0.1], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + [list(range(51)), list(range(51, 57)), list(range(57, 63))], +] + +TEST_CASE_11 = [ + { + "data": list(range(32)), + "ratios": [1, 1, 1, 1, 1], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + [list(range(7)), list(range(7, 14)), list(range(14, 20)), list(range(20, 26)), list(range(26, 32))], +] + +TEST_CASE_12 = [ + { + "data": [1, 2, 3, 4, 5], + "ratios": [0.6, 0.2, 0.2], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + [[1, 2, 3], [4], [5]], +] +TEST_CASE_13 = [ + { + "data": list(range(10)), + "ratios": [1, 1, 1, 0], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + [[0, 1, 2, 3], [4, 5, 6], [7, 8, 9], []], +] + + +TEST_CASE_14 = [ + { + "data": list(range(52)), + "ratios": [3, 1, 3, 8], + "num_partitions": None, + "shuffle": False, + "seed": 0, + "drop_last": True, + "even_divisible": True, + }, + # the largest rounding deficit is in the second partition, not the first, + # so the leftover item goes there rather than to partition 0 + [list(range(10)), list(range(10, 14)), list(range(14, 24)), list(range(24, 52))], +] + + +class TestPartitionDataset(unittest.TestCase): @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + [ + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4, + TEST_CASE_5, + TEST_CASE_6, + TEST_CASE_7, + TEST_CASE_8, + TEST_CASE_9, + TEST_CASE_10, + TEST_CASE_11, + TEST_CASE_12, + TEST_CASE_13, + TEST_CASE_14, + ] ) def test_value(self, input_param, result): self.assertListEqual(partition_dataset(**input_param), result)