fl-data-generator 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ from .dataset import FederatedDataset, NumpyDataset
2
+ from .dirichlet import dirichlet_partition
3
+ from .hybrid import hybrid_partition
4
+ from .iid import iid_partition
5
+ from .pathological import pathological_partition
6
+ from .quantity import quantity_skew_partition
7
+
8
+ __all__ = [
9
+ "FederatedDataset",
10
+ "NumpyDataset",
11
+ "dirichlet_partition",
12
+ "hybrid_partition",
13
+ "iid_partition",
14
+ "pathological_partition",
15
+ "quantity_skew_partition",
16
+ ]
@@ -0,0 +1,60 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Sequence
4
+
5
+ import numpy as np
6
+
7
+
8
+ def validate_labels(labels: Sequence | np.ndarray) -> np.ndarray:
9
+ """Validate and cast dataset labels to a 1D int64 numpy array.
10
+
11
+ Parameters
12
+ ----------
13
+ labels : Sequence or np.ndarray
14
+ Dataset labels as an array-like or sequence.
15
+
16
+ Returns
17
+ -------
18
+ np.ndarray
19
+ A 1D numpy array with dtype np.int64.
20
+
21
+ Raises
22
+ ------
23
+ ValueError
24
+ If labels is empty, not 1-dimensional, contains NaN or Inf, or
25
+ cannot be converted to integer values without loss of information.
26
+ TypeError
27
+ If labels elements cannot be converted to numeric values.
28
+ """
29
+ arr = np.asarray(labels)
30
+
31
+ if arr.ndim != 1:
32
+ raise ValueError("labels must be 1-dimensional; consider using labels.ravel()")
33
+
34
+ if arr.size == 0:
35
+ raise ValueError("labels cannot be empty")
36
+
37
+ if not np.issubdtype(arr.dtype, np.number):
38
+ # Could be object array with strings or other non-numeric types
39
+ try:
40
+ arr = arr.astype(np.float64)
41
+ except (ValueError, TypeError) as exc:
42
+ raise TypeError("labels must contain numeric values") from exc
43
+
44
+ # Check for NaN or Inf
45
+ if np.isnan(arr).any():
46
+ raise ValueError("labels contains NaN values")
47
+
48
+ if np.isinf(arr).any():
49
+ raise ValueError("labels contains infinite values")
50
+
51
+ # Check if integer-valued
52
+ if np.issubdtype(arr.dtype, np.floating):
53
+ if not np.all(np.floor(arr) == arr):
54
+ raise ValueError("labels must be integer-valued")
55
+ arr = arr.astype(np.int64)
56
+ elif np.issubdtype(arr.dtype, np.integer):
57
+ if arr.dtype != np.int64:
58
+ arr = arr.astype(np.int64)
59
+
60
+ return arr
@@ -0,0 +1,422 @@
1
+ from typing import Any, Callable, Dict, Optional, Tuple, Union
2
+
3
+ import numpy as np
4
+
5
+ try:
6
+ import torch
7
+ from torch.utils.data import DataLoader, Dataset, Subset
8
+
9
+ _HAS_TORCH = True
10
+ except ImportError:
11
+ _HAS_TORCH = False
12
+ Dataset = object # type: ignore
13
+ DataLoader = None # type: ignore
14
+ Subset = None # type: ignore
15
+
16
+ try:
17
+ import torchvision
18
+ import torchvision.datasets as tv_datasets
19
+
20
+ _HAS_TORCHVISION = True
21
+ except ImportError:
22
+ _HAS_TORCHVISION = False
23
+ tv_datasets = None # type: ignore
24
+
25
+ from .dirichlet import dirichlet_partition
26
+ from .hybrid import hybrid_partition
27
+ from .iid import iid_partition
28
+ from .pathological import pathological_partition
29
+ from .quantity import quantity_skew_partition
30
+
31
+
32
+ class NumpyDataset(Dataset):
33
+ """Lightweight dataset wrapper for NumPy arrays or tensor tuples.
34
+
35
+ Parameters
36
+ ----------
37
+ data : Sequence or np.ndarray or torch.Tensor
38
+ Feature array or tensor.
39
+ targets : Sequence or np.ndarray or torch.Tensor
40
+ Target labels array or tensor.
41
+ transform : callable, optional
42
+ Transformation applied to feature samples.
43
+ target_transform : callable, optional
44
+ Transformation applied to target labels.
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ data: Any,
50
+ targets: Any,
51
+ transform: Optional[Callable] = None,
52
+ target_transform: Optional[Callable] = None,
53
+ ) -> None:
54
+ self.data = data
55
+ self.targets = targets
56
+ self.transform = transform
57
+ self.target_transform = target_transform
58
+
59
+ def __len__(self) -> int:
60
+ return len(self.targets)
61
+
62
+ def __getitem__(self, index: int) -> Tuple[Any, Any]:
63
+ x = self.data[index]
64
+ y = self.targets[index]
65
+
66
+ if self.transform is not None:
67
+ x = self.transform(x)
68
+ if self.target_transform is not None:
69
+ y = self.target_transform(y)
70
+
71
+ return x, y
72
+
73
+
74
+ class FederatedDataset:
75
+ """High-level Federated Dataset manager for simulated distributed learning.
76
+
77
+ Provides automatic downloading, loading, and partitioning of datasets across
78
+ simulated clients according to standard Federated Learning benchmark distributions.
79
+
80
+ Supported Partitioning Regimes
81
+ ------------------------------
82
+ - **"iid"**:
83
+ Uniform random data splitting across all clients. Every client receives roughly
84
+ equal sample quantities and an identical class distribution.
85
+
86
+ - **"dirichlet"** (Label Skew):
87
+ Continuous non-IID label distribution based on Dirichlet sampling Dir(alpha).
88
+ * **alpha -> infinity** (e.g., 100.0): Approaches uniform, near-IID class distribution.
89
+ * **alpha ≈ 0.5**: Standard benchmark setting representing realistic heterogeneous clients.
90
+ * **alpha -> 0** (e.g., 0.05, 0.01): Extreme non-IID label skew where each client predominantly
91
+ holds samples from only 1 or 2 classes.
92
+
93
+ - **"pathological"** (Shard-based Label Skew):
94
+ The classic FedAvg benchmark (McMahan et al., 2017). Samples are sorted by label,
95
+ split into shards, and each client is assigned shards from a fixed number of classes
96
+ (configured via `classes_per_client`, e.g., 2 classes per client for MNIST).
97
+
98
+ - **"quantity"** (Sample Volume Skew):
99
+ Simulates hardware/usage heterogeneity where different clients collect vastly different
100
+ volumes of data. Client sample sizes follow a Dirichlet distribution parameterized by `beta`.
101
+ * **beta -> infinity** (e.g., 100.0): Clients receive roughly equal sample volumes.
102
+ * **beta -> 0** (e.g., 0.1): Extreme volume disparity (some clients receive hundreds of samples,
103
+ while others receive single digits).
104
+
105
+ - **"hybrid"** (Joint Label & Quantity Skew):
106
+ Combines volume disparity (governed by `beta`) with label preference heterogeneity
107
+ (governed by `alpha`).
108
+
109
+ Parameters
110
+ ----------
111
+ dataset : str or Tuple[Any, Any] or Dataset
112
+ Dataset identifier or instance:
113
+ - String name for built-in torchvision datasets: `"mnist"`, `"cifar10"`,
114
+ `"cifar100"`, `"fashion_mnist"`.
115
+ - Tuple of `(X, y)` containing features and labels (NumPy arrays or lists).
116
+ - Existing PyTorch `Dataset` object.
117
+ data_dir : str, default="./data"
118
+ Directory path used to cache and download standard datasets.
119
+ num_clients : int
120
+ Number of participating federated clients.
121
+ partition : str, default="iid"
122
+ Partition strategy. Choices: `"iid"`, `"dirichlet"`, `"pathological"`,
123
+ `"quantity"`, `"hybrid"`.
124
+ alpha : float, default=0.5
125
+ Dirichlet concentration parameter for label skew. Smaller values yield stronger
126
+ label heterogeneity across clients.
127
+ beta : float, default=0.5
128
+ Dirichlet concentration parameter for quantity skew. Smaller values yield higher
129
+ variance in client sample counts.
130
+ classes_per_client : int, default=2
131
+ Number of distinct classes assigned to each client in `"pathological"` partition.
132
+ seed : int, optional
133
+ Random seed for reproducible dataset partitioning.
134
+ train : bool, default=True
135
+ Whether to load the training or test split when downloading built-in datasets.
136
+ download : bool, default=True
137
+ Whether to automatically download built-in torchvision datasets if not present locally.
138
+ transform : callable, optional
139
+ Feature transformation applied to samples.
140
+ target_transform : callable, optional
141
+ Target transformation applied to labels.
142
+
143
+ Examples
144
+ --------
145
+ >>> # Using standard MNIST with Dirichlet label skew:
146
+ >>> fed_data = FederatedDataset("mnist", num_clients=10, partition="dirichlet", alpha=0.5)
147
+ >>> client_0_loader = fed_data.get_client_dataloader(client_id=0, batch_size=32)
148
+
149
+ >>> # Using custom NumPy arrays with quantity skew:
150
+ >>> import numpy as np
151
+ >>> X = np.random.randn(1000, 20)
152
+ >>> y = np.random.randint(0, 5, size=1000)
153
+ >>> fed_data = FederatedDataset((X, y), num_clients=5, partition="quantity", beta=0.2)
154
+ >>> indices = fed_data.get_client_indices(client_id=0)
155
+ """
156
+
157
+ SUPPORTED_DATASETS = {
158
+ "mnist": "MNIST",
159
+ "cifar10": "CIFAR10",
160
+ "cifar100": "CIFAR100",
161
+ "fashion_mnist": "FashionMNIST",
162
+ "fashionmnist": "FashionMNIST",
163
+ }
164
+
165
+ SUPPORTED_PARTITIONS = {
166
+ "iid",
167
+ "dirichlet",
168
+ "pathological",
169
+ "quantity",
170
+ "hybrid",
171
+ }
172
+
173
+ def __init__(
174
+ self,
175
+ dataset: Union[str, Tuple[Any, Any], Any],
176
+ data_dir: str = "./data",
177
+ num_clients: int = 10,
178
+ partition: str = "iid",
179
+ alpha: float = 0.5,
180
+ beta: float = 0.5,
181
+ classes_per_client: int = 2,
182
+ seed: Optional[int] = None,
183
+ train: bool = True,
184
+ download: bool = True,
185
+ transform: Optional[Callable] = None,
186
+ target_transform: Optional[Callable] = None,
187
+ ) -> None:
188
+ self.data_dir = data_dir
189
+ self.num_clients = num_clients
190
+ self.partition = partition.lower()
191
+ self.alpha = alpha
192
+ self.beta = beta
193
+ self.classes_per_client = classes_per_client
194
+ self.seed = seed
195
+ self.train = train
196
+ self.download = download
197
+ self.transform = transform
198
+ self.target_transform = target_transform
199
+
200
+ if self.num_clients <= 0:
201
+ raise ValueError(
202
+ f"num_clients must be positive, got {self.num_clients}"
203
+ )
204
+ if self.partition not in self.SUPPORTED_PARTITIONS:
205
+ raise ValueError(
206
+ f"Unsupported partition '{self.partition}'. "
207
+ f"Choose from: {sorted(self.SUPPORTED_PARTITIONS)}"
208
+ )
209
+
210
+ # 1. Load the underlying base dataset and extract target labels
211
+ self.base_dataset, self.labels = self._load_dataset(dataset)
212
+
213
+ # 2. Perform the specified partitioning
214
+ self.client_dict = self._partition_data()
215
+
216
+ def _load_dataset(self, dataset: Any) -> Tuple[Any, np.ndarray]:
217
+ """Resolve dataset input into a dataset instance and NumPy labels array."""
218
+ if isinstance(dataset, str):
219
+ ds_key = dataset.lower().strip()
220
+ if ds_key not in self.SUPPORTED_DATASETS:
221
+ raise ValueError(
222
+ f"Unknown built-in dataset '{dataset}'. "
223
+ f"Supported names: {sorted(self.SUPPORTED_DATASETS.keys())}"
224
+ )
225
+ if not _HAS_TORCHVISION:
226
+ raise ImportError(
227
+ f"torchvision is required to download and load '{dataset}'. "
228
+ f"Please install torchvision or provide custom (X, y) data."
229
+ )
230
+
231
+ ds_class_name = self.SUPPORTED_DATASETS[ds_key]
232
+ ds_cls = getattr(tv_datasets, ds_class_name)
233
+
234
+ ds_instance = ds_cls(
235
+ root=self.data_dir,
236
+ train=self.train,
237
+ download=self.download,
238
+ transform=self.transform,
239
+ target_transform=self.target_transform,
240
+ )
241
+
242
+ # Extract targets
243
+ if hasattr(ds_instance, "targets"):
244
+ targets = ds_instance.targets
245
+ if hasattr(targets, "numpy"):
246
+ labels = targets.numpy()
247
+ else:
248
+ labels = np.asarray(targets)
249
+ else:
250
+ labels = np.array([ds_instance[i][1] for i in range(len(ds_instance))])
251
+
252
+ return ds_instance, labels
253
+
254
+ if isinstance(dataset, tuple) and len(dataset) == 2:
255
+ data, targets = dataset
256
+ if len(data) != len(targets):
257
+ raise ValueError(
258
+ f"Mismatch between features length ({len(data)}) and "
259
+ f"targets length ({len(targets)})"
260
+ )
261
+ labels = np.asarray(targets)
262
+ ds_instance = NumpyDataset(
263
+ data=data,
264
+ targets=targets,
265
+ transform=self.transform,
266
+ target_transform=self.target_transform,
267
+ )
268
+ return ds_instance, labels
269
+
270
+ # Generic dataset object (e.g. custom PyTorch Dataset)
271
+ if hasattr(dataset, "__len__") and hasattr(dataset, "__getitem__"):
272
+ if hasattr(dataset, "targets"):
273
+ targets = dataset.targets
274
+ labels = targets.numpy() if hasattr(targets, "numpy") else np.asarray(targets)
275
+ elif hasattr(dataset, "labels"):
276
+ labels = np.asarray(dataset.labels)
277
+ else:
278
+ labels = np.array([dataset[i][1] for i in range(len(dataset))])
279
+ return dataset, labels
280
+
281
+ raise TypeError(
282
+ f"dataset must be a string name, (X, y) tuple, or Dataset instance. Got: {type(dataset)}"
283
+ )
284
+
285
+ def _partition_data(self) -> Dict[int, np.ndarray]:
286
+ """Dispatch partitioning according to configured strategy."""
287
+ if self.partition == "iid":
288
+ return iid_partition(
289
+ labels=self.labels,
290
+ num_clients=self.num_clients,
291
+ seed=self.seed,
292
+ )
293
+ elif self.partition == "dirichlet":
294
+ return dirichlet_partition(
295
+ labels=self.labels,
296
+ num_clients=self.num_clients,
297
+ alpha=self.alpha,
298
+ seed=self.seed,
299
+ )
300
+ elif self.partition == "pathological":
301
+ return pathological_partition(
302
+ labels=self.labels,
303
+ num_clients=self.num_clients,
304
+ classes_per_client=self.classes_per_client,
305
+ seed=self.seed,
306
+ )
307
+ elif self.partition == "quantity":
308
+ return quantity_skew_partition(
309
+ labels=self.labels,
310
+ num_clients=self.num_clients,
311
+ beta=self.beta,
312
+ seed=self.seed,
313
+ )
314
+ elif self.partition == "hybrid":
315
+ return hybrid_partition(
316
+ labels=self.labels,
317
+ num_clients=self.num_clients,
318
+ alpha=self.alpha,
319
+ beta=self.beta,
320
+ seed=self.seed,
321
+ )
322
+ else:
323
+ raise ValueError(f"Unknown partition strategy: {self.partition}")
324
+
325
+ def get_client_indices(self, client_id: int) -> np.ndarray:
326
+ """Get the array of sample indices allocated to a specific client.
327
+
328
+ Parameters
329
+ ----------
330
+ client_id : int
331
+ Index of the client (0 <= client_id < num_clients).
332
+
333
+ Returns
334
+ -------
335
+ np.ndarray
336
+ 1D array of sample indices.
337
+ """
338
+ if client_id not in self.client_dict:
339
+ raise IndexError(
340
+ f"Client ID {client_id} out of bounds for num_clients={self.num_clients}"
341
+ )
342
+ return self.client_dict[client_id]
343
+
344
+ def get_client_dataset(self, client_id: int) -> Any:
345
+ """Get a Dataset instance containing only the client's assigned samples.
346
+
347
+ Parameters
348
+ ----------
349
+ client_id : int
350
+ Index of the client.
351
+
352
+ Returns
353
+ -------
354
+ Dataset or Subset
355
+ Client dataset wrapper.
356
+ """
357
+ indices = self.get_client_indices(client_id)
358
+ if _HAS_TORCH and Subset is not None:
359
+ return Subset(self.base_dataset, indices)
360
+
361
+ # Fallback subset wrapper for non-torch environments
362
+ class _NonTorchSubset:
363
+ def __init__(self, base, idxs):
364
+ self.base = base
365
+ self.indices = idxs
366
+
367
+ def __len__(self):
368
+ return len(self.indices)
369
+
370
+ def __getitem__(self, i):
371
+ return self.base[self.indices[i]]
372
+
373
+ return _NonTorchSubset(self.base_dataset, indices)
374
+
375
+ def get_client_dataloader(
376
+ self,
377
+ client_id: int,
378
+ batch_size: int = 32,
379
+ shuffle: bool = True,
380
+ **kwargs: Any,
381
+ ) -> Any:
382
+ """Get a PyTorch DataLoader for the specified client.
383
+
384
+ Parameters
385
+ ----------
386
+ client_id : int
387
+ Index of the client.
388
+ batch_size : int, default=32
389
+ Mini-batch size for client local training.
390
+ shuffle : bool, default=True
391
+ Whether to shuffle the client's data each epoch.
392
+ **kwargs : Any
393
+ Additional keyword arguments passed to `torch.utils.data.DataLoader`.
394
+
395
+ Returns
396
+ -------
397
+ torch.utils.data.DataLoader
398
+ Configured DataLoader for local client training.
399
+ """
400
+ if not _HAS_TORCH or DataLoader is None:
401
+ raise ImportError(
402
+ "PyTorch is required to create a DataLoader. "
403
+ "Install torch (`pip install torch`) or use get_client_dataset() / get_client_indices()."
404
+ )
405
+
406
+ client_ds = self.get_client_dataset(client_id)
407
+ return DataLoader(client_ds, batch_size=batch_size, shuffle=shuffle, **kwargs)
408
+
409
+ @property
410
+ def client_sizes(self) -> Dict[int, int]:
411
+ """Dictionary mapping client IDs to assigned sample counts."""
412
+ return {cid: len(indices) for cid, indices in self.client_dict.items()}
413
+
414
+ def __len__(self) -> int:
415
+ """Total number of samples in the partitioned dataset."""
416
+ return len(self.labels)
417
+
418
+ def __repr__(self) -> str:
419
+ return (
420
+ f"FederatedDataset(num_clients={self.num_clients}, "
421
+ f"partition='{self.partition}', total_samples={len(self)})"
422
+ )
@@ -0,0 +1,123 @@
1
+ from typing import Dict, Optional, Sequence, Union
2
+
3
+ import numpy as np
4
+
5
+ from ._validation import validate_labels
6
+
7
+
8
+ def dirichlet_partition(
9
+ labels: Union[Sequence, np.ndarray],
10
+ num_clients: int,
11
+ alpha: Union[float, Sequence[float]] = 0.5,
12
+ seed: Optional[int] = None,
13
+ min_size: int = 1,
14
+ max_retries: int = 100,
15
+ ) -> Dict[int, np.ndarray]:
16
+ """Partition dataset indices among clients using a Dirichlet distribution over classes.
17
+
18
+ Simulates non-IID label skew across clients in Federated Learning. Smaller alpha values
19
+ result in higher distribution skew (more non-IID), while larger alpha values approach
20
+ an IID distribution.
21
+
22
+ Parameters
23
+ ----------
24
+ labels : Sequence or np.ndarray
25
+ Dataset class labels or targets.
26
+ num_clients : int
27
+ Number of clients to partition data among.
28
+ alpha : float or Sequence[float], default=0.5
29
+ Concentration parameter for the Dirichlet distribution.
30
+ seed : int, optional
31
+ Random seed for reproducibility.
32
+ min_size : int, default=1
33
+ Minimum number of samples required per client.
34
+ max_retries : int, default=100
35
+ Maximum number of Dirichlet resampling attempts to satisfy min_size.
36
+
37
+ Returns
38
+ -------
39
+ dict of int to np.ndarray
40
+ Mapping from client ID to array of assigned sample indices.
41
+
42
+ Raises
43
+ ------
44
+ ValueError
45
+ If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
46
+ If num_clients <= 0 or alpha <= 0.
47
+ If num_clients exceeds num_samples.
48
+ If alpha is a sequence and its length does not match num_clients.
49
+ If min_size < 0, not an integer, or min_size * num_clients > num_samples.
50
+ RuntimeError
51
+ If unable to partition data such that all clients receive at least min_size samples
52
+ within max_retries attempts.
53
+ """
54
+ labels_arr = validate_labels(labels)
55
+ num_samples = len(labels_arr)
56
+
57
+ if num_clients <= 0:
58
+ raise ValueError(f"num_clients must be positive, got {num_clients}")
59
+ if num_clients > num_samples:
60
+ raise ValueError(
61
+ f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
62
+ )
63
+
64
+ if not isinstance(min_size, int) or isinstance(min_size, bool):
65
+ raise ValueError(f"min_size must be an integer, got {type(min_size).__name__}") # noqa: TRY004
66
+ if min_size < 0:
67
+ raise ValueError(f"min_size must be non-negative, got {min_size}")
68
+ if min_size * num_clients > num_samples:
69
+ raise ValueError(
70
+ f"min_size * num_clients ({min_size * num_clients}) cannot exceed number of samples ({num_samples})"
71
+ )
72
+ if not isinstance(max_retries, int) or isinstance(max_retries, bool) or max_retries <= 0:
73
+ raise ValueError(f"max_retries must be a positive integer, got {max_retries}")
74
+
75
+ if isinstance(alpha, (int, float)):
76
+ if alpha <= 0:
77
+ raise ValueError(f"alpha must be positive, got {alpha}")
78
+ alpha_vec = np.repeat(float(alpha), num_clients)
79
+ else:
80
+ alpha_vec = np.asarray(alpha, dtype=float)
81
+ if alpha_vec.shape != (num_clients,):
82
+ raise ValueError(
83
+ f"alpha length must match num_clients ({num_clients}), got {len(alpha_vec)}"
84
+ )
85
+ if (alpha_vec <= 0).any():
86
+ raise ValueError("All alpha values must be positive")
87
+
88
+ rng = np.random.default_rng(seed)
89
+ unique_classes = np.unique(labels_arr)
90
+
91
+ for _ in range(max_retries):
92
+ client_indices: Dict[int, list] = {cid: [] for cid in range(num_clients)}
93
+
94
+ for cls in unique_classes:
95
+ cls_idx = np.where(labels_arr == cls)[0]
96
+ rng.shuffle(cls_idx)
97
+
98
+ proportions = rng.dirichlet(alpha_vec)
99
+ split_points = (np.cumsum(proportions)[:-1] * len(cls_idx)).astype(int)
100
+ splits = np.split(cls_idx, split_points)
101
+
102
+ for cid, split in enumerate(splits):
103
+ if len(split) > 0:
104
+ client_indices[cid].append(split)
105
+
106
+ if all(sum(len(s) for s in client_indices[cid]) >= min_size for cid in range(num_clients)):
107
+ break
108
+ else:
109
+ raise RuntimeError(
110
+ f"Could not partition data such that all clients receive at least min_size={min_size} samples "
111
+ f"after {max_retries} retries. Try using a larger alpha or a smaller min_size."
112
+ )
113
+
114
+ result = {}
115
+ for cid in range(num_clients):
116
+ if client_indices[cid]:
117
+ merged = np.concatenate(client_indices[cid])
118
+ rng.shuffle(merged)
119
+ result[cid] = merged
120
+ else:
121
+ result[cid] = np.array([], dtype=int)
122
+
123
+ return result
@@ -0,0 +1,138 @@
1
+ from typing import Dict, Optional, Sequence, Union
2
+
3
+ import numpy as np
4
+
5
+ from ._validation import validate_labels
6
+
7
+
8
+ def hybrid_partition(
9
+ labels: Union[Sequence, np.ndarray],
10
+ num_clients: int,
11
+ alpha: float = 0.5,
12
+ beta: float = 0.5,
13
+ seed: Optional[int] = None,
14
+ ) -> Dict[int, np.ndarray]:
15
+ """Partition dataset indices combining label skew and quantity skew (experimental).
16
+
17
+ .. warning::
18
+ This method is experimental and not a standard published partitioning method.
19
+ It uses sequential allocation: client sample capacities are first determined
20
+ via Dirichlet distribution with parameter beta (quantity skew). Samples are then
21
+ allocated sequentially to clients in a randomly permuted client order based on class
22
+ preference distributions drawn from a Dirichlet distribution with parameter alpha
23
+ (label skew). Because samples are drawn without replacement from a shared finite pool,
24
+ there is a client-order effect: earlier clients in the sequence have access to the
25
+ full class distribution, while later clients may receive residual classes that deviate
26
+ from their drawn preferences.
27
+
28
+ Parameters
29
+ ----------
30
+ labels : Sequence or np.ndarray
31
+ Dataset class labels or targets.
32
+ num_clients : int
33
+ Number of clients to partition data among.
34
+ alpha : float, default=0.5
35
+ Concentration parameter for the Dirichlet distribution governing label skew.
36
+ Must be a positive scalar. Smaller values induce stronger class preference heterogeneity.
37
+ beta : float, default=0.5
38
+ Concentration parameter for the Dirichlet distribution governing quantity skew.
39
+ Must be a positive scalar. Smaller values induce greater disparity in sample volume across clients.
40
+ seed : int, optional
41
+ Random seed for reproducibility.
42
+
43
+ Returns
44
+ -------
45
+ dict of int to np.ndarray
46
+ Mapping from client ID to array of assigned sample indices.
47
+
48
+ Raises
49
+ ------
50
+ ValueError
51
+ If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
52
+ If num_clients <= 0, alpha <= 0, or beta <= 0.
53
+ If num_clients exceeds num_samples.
54
+ If alpha or beta is a sequence rather than a scalar.
55
+ """
56
+ labels_arr = validate_labels(labels)
57
+ num_samples = len(labels_arr)
58
+
59
+ if num_clients <= 0:
60
+ raise ValueError(f"num_clients must be positive, got {num_clients}")
61
+ if num_clients > num_samples:
62
+ raise ValueError(
63
+ f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
64
+ )
65
+
66
+ if isinstance(alpha, (Sequence, np.ndarray)) or np.ndim(alpha) > 0 or not isinstance(alpha, (int, float, np.number)) or isinstance(alpha, bool):
67
+ raise ValueError(f"alpha must be a scalar positive number, got {alpha}")
68
+ if alpha <= 0:
69
+ raise ValueError(f"alpha must be positive, got {alpha}")
70
+
71
+ if isinstance(beta, (Sequence, np.ndarray)) or np.ndim(beta) > 0 or not isinstance(beta, (int, float, np.number)) or isinstance(beta, bool):
72
+ raise ValueError(f"beta must be a scalar positive number, got {beta}")
73
+ if beta <= 0:
74
+ raise ValueError(f"beta must be positive, got {beta}")
75
+
76
+ rng = np.random.default_rng(seed)
77
+ unique_classes = np.unique(labels_arr)
78
+ num_classes = len(unique_classes)
79
+ cls_to_idx = {c: i for i, c in enumerate(unique_classes)}
80
+
81
+ # 1. Determine client sample sizes via quantity skew (Dirichlet with parameter beta)
82
+ q_props = rng.dirichlet(np.repeat(beta, num_clients))
83
+ remaining = num_samples - num_clients
84
+ splits = (np.cumsum(q_props)[:-1] * remaining).astype(int)
85
+ client_sizes = np.diff(np.concatenate(([0], splits, [remaining]))) + 1
86
+
87
+ # Organize available sample indices by class
88
+ class_indices: Dict[object, list] = {}
89
+ for cls in unique_classes:
90
+ idx = np.where(labels_arr == cls)[0]
91
+ rng.shuffle(idx)
92
+ class_indices[cls] = list(idx)
93
+
94
+ # 2. Allocate samples to each client proportionally based on label skew (Dirichlet alpha)
95
+ client_partitions: Dict[int, list] = {cid: [] for cid in range(num_clients)}
96
+ client_order = rng.permutation(num_clients)
97
+
98
+ for cid in client_order:
99
+ needed = client_sizes[cid]
100
+ p = rng.dirichlet(np.repeat(alpha, num_classes))
101
+
102
+ while needed > 0:
103
+ avail_classes = [c for c in unique_classes if len(class_indices[c]) > 0]
104
+ if not avail_classes:
105
+ break
106
+
107
+ avail_p = np.array([p[cls_to_idx[c]] for c in avail_classes])
108
+ p_sum = avail_p.sum()
109
+ if p_sum == 0:
110
+ avail_p = np.ones(len(avail_classes))
111
+ p_sum = len(avail_classes)
112
+ avail_p = avail_p / p_sum
113
+
114
+ counts = (avail_p * needed).astype(int)
115
+ if counts.sum() == 0:
116
+ counts[np.argmax(avail_p)] = 1
117
+
118
+ allocated_this_round = 0
119
+ for c, cnt in zip(avail_classes, counts):
120
+ take = min(cnt, len(class_indices[c]), needed)
121
+ if take > 0:
122
+ for _ in range(take):
123
+ client_partitions[cid].append(class_indices[c].pop())
124
+ needed -= take
125
+ allocated_this_round += take
126
+
127
+ if allocated_this_round == 0:
128
+ c = avail_classes[0]
129
+ client_partitions[cid].append(class_indices[c].pop())
130
+ needed -= 1
131
+
132
+ result: Dict[int, np.ndarray] = {}
133
+ for cid in range(num_clients):
134
+ arr = np.array(client_partitions[cid], dtype=int)
135
+ rng.shuffle(arr)
136
+ result[cid] = arr
137
+
138
+ return result
@@ -0,0 +1,49 @@
1
+ from typing import Dict, Optional, Sequence, Union
2
+
3
+ import numpy as np
4
+
5
+ from ._validation import validate_labels
6
+
7
+
8
+ def iid_partition(
9
+ labels: Union[Sequence, np.ndarray],
10
+ num_clients: int,
11
+ seed: Optional[int] = None,
12
+ ) -> Dict[int, np.ndarray]:
13
+ """Partition dataset indices in an IID fashion across clients.
14
+
15
+ Parameters
16
+ ----------
17
+ labels : Sequence or np.ndarray
18
+ Dataset labels or target sequence.
19
+ num_clients : int
20
+ Number of clients to partition data among.
21
+ seed : int, optional
22
+ Random seed for reproducibility.
23
+
24
+ Returns
25
+ -------
26
+ dict of int to np.ndarray
27
+ Mapping from client ID to array of assigned sample indices.
28
+
29
+ Raises
30
+ ------
31
+ ValueError
32
+ If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
33
+ If num_clients <= 0 or num_clients > num_samples.
34
+ """
35
+ labels_arr = validate_labels(labels)
36
+ num_samples = len(labels_arr)
37
+
38
+ if num_clients <= 0:
39
+ raise ValueError(f"num_clients must be positive, got {num_clients}")
40
+ if num_clients > num_samples:
41
+ raise ValueError(
42
+ f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
43
+ )
44
+
45
+ rng = np.random.default_rng(seed)
46
+ indices = rng.permutation(num_samples)
47
+ client_splits = np.array_split(indices, num_clients)
48
+
49
+ return {cid: split for cid, split in enumerate(client_splits)}
@@ -0,0 +1,137 @@
1
+ from typing import Dict, Optional, Sequence, Union
2
+
3
+ import numpy as np
4
+
5
+ from ._validation import validate_labels
6
+
7
+
8
+ def pathological_partition(
9
+ labels: Union[Sequence, np.ndarray],
10
+ num_clients: int,
11
+ classes_per_client: int = 2,
12
+ seed: Optional[int] = None,
13
+ ) -> Dict[int, np.ndarray]:
14
+ """Partition dataset indices among clients with pathological non-IID label skew.
15
+
16
+ Assigns k distinct classes per client (NIID-Bench #C=k, Li et al. 2021),
17
+ NOT the McMahan et al. 2017 sort-and-shard scheme. Each class's samples are
18
+ split into shards, and clients receive shards corresponding to their assigned
19
+ classes.
20
+
21
+ Parameters
22
+ ----------
23
+ labels : Sequence or np.ndarray
24
+ Dataset class labels or targets.
25
+ num_clients : int
26
+ Number of clients to partition data among.
27
+ classes_per_client : int, default=2
28
+ Number of distinct classes assigned to each client.
29
+ seed : int, optional
30
+ Random seed for reproducibility.
31
+
32
+ Returns
33
+ -------
34
+ dict of int to np.ndarray
35
+ Mapping from client ID to array of assigned sample indices.
36
+
37
+ Raises
38
+ ------
39
+ ValueError
40
+ If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
41
+ If num_clients <= 0 or classes_per_client <= 0.
42
+ If classes_per_client exceeds the number of unique classes.
43
+ If total client slots is less than the number of unique classes.
44
+ If total samples is less than total class shards.
45
+ If any class has fewer samples than the number of shards assigned to it.
46
+ """
47
+ labels_arr = validate_labels(labels)
48
+ num_samples = len(labels_arr)
49
+
50
+ if num_clients <= 0:
51
+ raise ValueError(f"num_clients must be positive, got {num_clients}")
52
+ if classes_per_client <= 0:
53
+ raise ValueError(
54
+ f"classes_per_client must be positive, got {classes_per_client}"
55
+ )
56
+
57
+
58
+ unique_classes = np.unique(labels_arr)
59
+ num_classes = len(unique_classes)
60
+
61
+ if classes_per_client > num_classes:
62
+ raise ValueError(
63
+ f"classes_per_client ({classes_per_client}) cannot exceed number of unique classes ({num_classes})"
64
+ )
65
+
66
+ total_slots = num_clients * classes_per_client
67
+ if total_slots < num_classes:
68
+ raise ValueError(
69
+ f"Total client class slots ({num_clients} * {classes_per_client} = {total_slots}) "
70
+ f"must be at least the number of unique classes ({num_classes}) to allocate all samples."
71
+ )
72
+
73
+ if num_samples < total_slots:
74
+ raise ValueError(
75
+ f"Total samples ({num_samples}) must be at least total class shards ({total_slots})"
76
+ )
77
+
78
+ rng = np.random.default_rng(seed)
79
+
80
+ # Distribute class slots across clients
81
+ counts = np.ones(num_classes, dtype=int)
82
+ remaining = total_slots - num_classes
83
+ base = remaining // num_classes
84
+ rem = remaining % num_classes
85
+ counts += base
86
+ counts[:rem] += 1
87
+ rng.shuffle(counts)
88
+
89
+ # Validate that every class has at least as many samples as assigned shards
90
+ for c_idx, cls_val in enumerate(unique_classes):
91
+ class_sample_count = int(np.count_nonzero(labels_arr == cls_val))
92
+ needed_shards = int(counts[c_idx])
93
+ if class_sample_count < needed_shards:
94
+ raise ValueError(
95
+ f"Class '{cls_val}' has {class_sample_count} sample(s), which is fewer than "
96
+ f"the assigned {needed_shards} shard(s). Cannot partition without producing empty shards."
97
+ )
98
+
99
+
100
+ client_classes = [set() for _ in range(num_clients)]
101
+ sorted_class_indices = np.argsort(-counts)
102
+
103
+ for c_idx in sorted_class_indices:
104
+ cls_val = unique_classes[c_idx]
105
+ needed = counts[c_idx]
106
+
107
+ candidates = [
108
+ cid
109
+ for cid in range(num_clients)
110
+ if cls_val not in client_classes[cid]
111
+ and len(client_classes[cid]) < classes_per_client
112
+ ]
113
+ candidates.sort(key=lambda cid: (len(client_classes[cid]), rng.random()))
114
+
115
+ for cid in candidates[:needed]:
116
+ client_classes[cid].add(cls_val)
117
+
118
+ # Split each class's samples into the assigned number of shards
119
+ class_shards: Dict[object, list] = {}
120
+ for c_idx, cls_val in enumerate(unique_classes):
121
+ cls_indices = np.where(labels_arr == cls_val)[0]
122
+ rng.shuffle(cls_indices)
123
+ num_shards = counts[c_idx]
124
+ class_shards[cls_val] = list(np.array_split(cls_indices, num_shards))
125
+
126
+ # Assign shards to clients
127
+ result: Dict[int, np.ndarray] = {}
128
+ for cid in range(num_clients):
129
+ cid_shards = []
130
+ for cls_val in client_classes[cid]:
131
+ cid_shards.append(class_shards[cls_val].pop())
132
+
133
+ merged = np.concatenate(cid_shards)
134
+ rng.shuffle(merged)
135
+ result[cid] = merged
136
+
137
+ return result
@@ -0,0 +1,83 @@
1
+ from typing import Dict, Optional, Sequence, Union
2
+
3
+ import numpy as np
4
+
5
+ from ._validation import validate_labels
6
+
7
+
8
+ def quantity_skew_partition(
9
+ labels: Union[Sequence, np.ndarray],
10
+ num_clients: int,
11
+ beta: Union[float, Sequence[float]] = 0.5,
12
+ seed: Optional[int] = None,
13
+ ) -> Dict[int, np.ndarray]:
14
+ """Partition dataset indices among clients with quantity skew (varying sample sizes).
15
+
16
+ Simulates realistic volume heterogeneity across clients in Federated Learning.
17
+ The proportion of total samples assigned to each client is drawn from a Dirichlet
18
+ distribution parameterized by beta. Smaller beta values produce severe volume
19
+ disparities, while larger beta values approach uniform sample sizes.
20
+
21
+ Parameters
22
+ ----------
23
+ labels : Sequence or np.ndarray
24
+ Dataset labels or target sequence.
25
+ num_clients : int
26
+ Number of clients to partition data among.
27
+ beta : float or Sequence[float], default=0.5
28
+ Concentration parameter for the Dirichlet distribution governing client sample sizes.
29
+ seed : int, optional
30
+ Random seed for reproducibility.
31
+
32
+ Returns
33
+ -------
34
+ dict of int to np.ndarray
35
+ Mapping from client ID to array of assigned sample indices.
36
+
37
+ Raises
38
+ ------
39
+ ValueError
40
+ If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
41
+ If num_clients <= 0 or beta <= 0.
42
+ If num_clients exceeds num_samples.
43
+ If beta is a sequence and its length does not match num_clients.
44
+ """
45
+ labels_arr = validate_labels(labels)
46
+ num_samples = len(labels_arr)
47
+
48
+ if num_clients <= 0:
49
+ raise ValueError(f"num_clients must be positive, got {num_clients}")
50
+ if num_clients > num_samples:
51
+
52
+ raise ValueError(
53
+ f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
54
+ )
55
+
56
+ if isinstance(beta, (int, float)):
57
+ if beta <= 0:
58
+ raise ValueError(f"beta must be positive, got {beta}")
59
+ beta_vec = np.repeat(float(beta), num_clients)
60
+ else:
61
+ beta_vec = np.asarray(beta, dtype=float)
62
+ if beta_vec.shape != (num_clients,):
63
+ raise ValueError(
64
+ f"beta length must match num_clients ({num_clients}), got {len(beta_vec)}"
65
+ )
66
+ if (beta_vec <= 0).any():
67
+ raise ValueError("All beta values must be positive")
68
+
69
+ rng = np.random.default_rng(seed)
70
+ shuffled_indices = rng.permutation(num_samples)
71
+
72
+ # Sample client proportions from Dirichlet distribution
73
+ proportions = rng.dirichlet(beta_vec)
74
+
75
+ # Allocate at least 1 sample to each client, distribute remainder according to proportions
76
+ remaining = num_samples - num_clients
77
+ split_points = (np.cumsum(proportions)[:-1] * remaining).astype(int)
78
+ counts = np.diff(np.concatenate(([0], split_points, [remaining]))) + 1
79
+
80
+ # Split shuffled indices by cumulative sample counts
81
+ index_splits = np.split(shuffled_indices, np.cumsum(counts)[:-1])
82
+
83
+ return {cid: split for cid, split in enumerate(index_splits)}
@@ -0,0 +1,233 @@
1
+ Metadata-Version: 2.4
2
+ Name: fl-data-generator
3
+ Version: 0.1.0
4
+ Summary: A lightweight, framework-agnostic library for IID and non-IID data partitioning in Federated Learning.
5
+ Author-email: Mostafa Khastkhodaei <dev.khastkhodaei@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/khastkhodaei/fl-data-generator
8
+ Project-URL: Bug Tracker, https://github.com/khastkhodaei/fl-data-generator/issues
9
+ Project-URL: Source Code, https://github.com/khastkhodaei/fl-data-generator
10
+ Keywords: federated-learning,data-partitioning,non-iid,dirichlet,label-skew,quantity-skew,machine-learning
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: numpy>=1.20.0
27
+ Provides-Extra: dev
28
+ Requires-Dist: hypothesis>=6.0.0; extra == "dev"
29
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
30
+ Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
31
+ Requires-Dist: ruff~=0.17.0; extra == "dev"
32
+ Provides-Extra: viz
33
+ Requires-Dist: matplotlib>=3.5.0; extra == "viz"
34
+ Dynamic: license-file
35
+
36
+ # fl-data-generator
37
+
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
39
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
40
+
41
+ A lightweight, framework-agnostic Python library for **IID** and **Non-IID** data partitioning in **Federated Learning (FL)**.
42
+
43
+ ---
44
+
45
+ ## Overview
46
+
47
+ In Federated Learning research and benchmarks, simulating realistic data heterogeneity across distributed clients is essential. **fl-data-generator** simplifies partitioning arbitrary datasets into realistic client distributions without tying you to any specific deep learning framework.
48
+
49
+ Whether you are using PyTorch, TensorFlow, JAX, or federated frameworks like Flower, FedML, or PySyft, `fl-data-generator` provides clean sample indices and dataset wrappers for your clients.
50
+
51
+ ---
52
+
53
+ ## Features
54
+
55
+ - **IID Partitioning**: Uniform random sample allocation across all clients.
56
+ - **Dirichlet Partitioning**: Continuous label distribution skew ($\text{Dir}(\alpha)$) over class proportions, with client minimum size constraints (`min_size`).
57
+ - **Pathological Non-IID Partitioning**: Allocation of a fixed number of classes ($k$ classes per client, following the NIID-Bench benchmark scheme).
58
+ - **Quantity Skew Partitioning**: Volume heterogeneity where client dataset sizes vary according to a Dirichlet distribution ($\text{Dir}(\beta)$).
59
+ - **Hybrid Partitioning (Experimental)**: Sequential joint label skew and quantity skew allocation combining Dirichlet $\alpha$ and $\beta$.
60
+ - **FederatedDataset Wrapper**: High-level wrapper supporting built-in computer vision datasets (`mnist`, `cifar10`, `cifar100`, `fashion_mnist`) as well as custom `(X, y)` tuples and PyTorch `Dataset` instances.
61
+
62
+ ---
63
+
64
+ ## Installation
65
+
66
+ ### From Source
67
+
68
+ ```bash
69
+ git clone https://github.com/khastkhodaei/fl-data-generator.git
70
+ cd fl-data-generator
71
+ pip install -e .
72
+ ```
73
+
74
+ ### With Optional Extras
75
+
76
+ ```bash
77
+ # Development dependencies (pytest, hypothesis, ruff, etc.)
78
+ pip install -e ".[dev]"
79
+
80
+ # Visualization support (matplotlib)
81
+ pip install -e ".[viz]"
82
+ ```
83
+
84
+ ---
85
+
86
+ ## Runnable Examples
87
+
88
+ ### 1. IID Partitioning
89
+
90
+ Uniformly partitions sample indices across clients.
91
+
92
+ ```python
93
+ import numpy as np
94
+ from fl_data_generator import iid_partition
95
+
96
+ labels = np.array([0, 1] * 50) # 100 samples
97
+ client_dict = iid_partition(labels, num_clients=4, seed=42)
98
+
99
+ print("Client sample counts:", [len(indices) for indices in client_dict.values()])
100
+ # Real Output: Client sample counts: [25, 25, 25, 25]
101
+ ```
102
+
103
+ ### 2. Dirichlet Label Skew Partitioning
104
+
105
+ Simulates non-IID label skew parameterized by concentration parameter `alpha`. Smaller $\alpha$ generates severe distribution skew, while larger $\alpha$ approaches uniform class distributions.
106
+
107
+ ```python
108
+ import numpy as np
109
+ from fl_data_generator import dirichlet_partition
110
+
111
+ labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
112
+ client_dict = dirichlet_partition(labels, num_clients=3, alpha=0.5, seed=42)
113
+
114
+ print("Client sample counts:", [len(indices) for indices in client_dict.values()])
115
+ # Real Output: Client sample counts: [114, 42, 44]
116
+ ```
117
+
118
+ ### 3. Pathological Non-IID Partitioning ($k$ Classes per Client)
119
+
120
+ Partitions data such that each client receives samples from exactly $k$ distinct classes (NIID-Bench #C=k scheme).
121
+
122
+ ```python
123
+ import numpy as np
124
+ from fl_data_generator import pathological_partition
125
+
126
+ labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
127
+ client_dict = pathological_partition(labels, num_clients=4, classes_per_client=2, seed=42)
128
+
129
+ client_0_labels = labels[client_dict[0]]
130
+ print("Client 0 distinct classes:", np.unique(client_0_labels).tolist())
131
+ # Real Output: Client 0 distinct classes: [1, 2]
132
+ ```
133
+
134
+ ### 4. Quantity Skew Partitioning (Volume Heterogeneity)
135
+
136
+ Allocates varying numbers of samples to clients according to a Dirichlet distribution parameterized by concentration parameter `beta`.
137
+
138
+ ```python
139
+ import numpy as np
140
+ from fl_data_generator import quantity_skew_partition
141
+
142
+ labels = np.arange(200) # 200 samples
143
+ client_dict = quantity_skew_partition(labels, num_clients=4, beta=0.5, seed=42)
144
+
145
+ print("Client sample counts:", [len(indices) for indices in client_dict.values()])
146
+ # Real Output: Client sample counts: [52, 72, 16, 60]
147
+ ```
148
+
149
+ ### 5. Hybrid Partitioning (Experimental)
150
+
151
+ Combines quantity skew ($\beta$) and label skew ($\alpha$) via sequential allocation.
152
+
153
+ ```python
154
+ import numpy as np
155
+ from fl_data_generator import hybrid_partition
156
+
157
+ labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
158
+ client_dict = hybrid_partition(labels, num_clients=4, alpha=0.5, beta=0.5, seed=42)
159
+
160
+ print("Client sample counts:", [len(indices) for indices in client_dict.values()])
161
+ # Real Output: Client sample counts: [56, 89, 2, 53]
162
+ ```
163
+
164
+ ### 6. FederatedDataset Interface
165
+
166
+ High-level interface for creating client-specific datasets and PyTorch DataLoaders.
167
+
168
+ ```python
169
+ import numpy as np
170
+ from fl_data_generator import FederatedDataset
171
+
172
+ X = np.random.default_rng(42).standard_normal((100, 5))
173
+ y = np.array([0, 1] * 50)
174
+
175
+ fed_data = FederatedDataset((X, y), num_clients=4, partition="iid", seed=42)
176
+ client_0_ds = fed_data.get_client_dataset(0)
177
+
178
+ print(f"Client 0 sample count: {len(client_0_ds)}")
179
+ features, label = client_0_ds[0]
180
+ print(f"Feature shape: {features.shape}, Label: {label}")
181
+ # Real Output:
182
+ # Client 0 sample count: 25
183
+ # Feature shape: (5,), Label: 1
184
+ ```
185
+
186
+ ---
187
+
188
+ ## Roadmap
189
+
190
+ The following experimental features and partition strategies are planned for future releases:
191
+ - **Feature Skew (Covariate Shift)**: Partitioning based on continuous feature space clusters or synthetic domain perturbations.
192
+ - **Power-Law Quantity Skew**: Heavy-tailed sample volume distribution across clients.
193
+ - **Asymmetric Class Topologies**: Custom directed class sharing graphs.
194
+
195
+ ---
196
+
197
+ ## Project Structure
198
+
199
+ ```text
200
+ fl-data-generator/
201
+ ├── .github/
202
+ │ └── workflows/
203
+ │ └── ci.yml # GitHub Actions CI matrix workflow
204
+ ├── src/
205
+ │ └── fl_data_generator/ # Core library package
206
+ │ ├── __init__.py # Public API exports
207
+ │ ├── _validation.py # Input validation utilities
208
+ │ ├── dataset.py # FederatedDataset interface & torchvision loaders
209
+ │ ├── dirichlet.py # Dirichlet label skew partitioner
210
+ │ ├── hybrid.py # Hybrid label & quantity skew partitioner
211
+ │ ├── iid.py # Uniform IID partitioner
212
+ │ ├── pathological.py # Pathological k-class partitioner
213
+ │ └── quantity.py # Quantity skew partitioner
214
+ ├── tests/ # Unit, integration, and property test suite
215
+ │ ├── test_dataset.py
216
+ │ ├── test_dirichlet.py
217
+ │ ├── test_hybrid.py
218
+ │ ├── test_iid.py
219
+ │ ├── test_pathological.py
220
+ │ ├── test_properties.py
221
+ │ ├── test_quantity.py
222
+ │ ├── test_readme.py
223
+ │ └── test_validation.py
224
+ ├── pyproject.toml # Package build configuration & Ruff settings
225
+ ├── LICENSE # MIT License
226
+ └── README.md # Project documentation
227
+ ```
228
+
229
+ ---
230
+
231
+ ## License
232
+
233
+ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -0,0 +1,13 @@
1
+ fl_data_generator/__init__.py,sha256=m0oFZhpYf6MWug0dqfnTv3EmMCrtHm2ge3t-LxjVr3g,466
2
+ fl_data_generator/_validation.py,sha256=5hkJELUbBskEjHOkz1PN1zodntp3M4eYaGmo7mDavXw,1823
3
+ fl_data_generator/dataset.py,sha256=4jUobhkOoewO_6LW491yH3o1me2C28mevQtjsInBkpg,15910
4
+ fl_data_generator/dirichlet.py,sha256=2dHX2FGQQgtuK03GHOAznPflIwp4Uwg_QubVeQKAQeA,4803
5
+ fl_data_generator/hybrid.py,sha256=Ol3v85V8mOWmL0WbIvk-RCnov2uxTfyns0SJrn_vfGE,5800
6
+ fl_data_generator/iid.py,sha256=f2GFubs8hq_w_CU8it1qI9XXHq4C_af0nReZBx2Jqrk,1475
7
+ fl_data_generator/pathological.py,sha256=rdh5-6SUIKl3Ij_P2c8Ud7aPpvk96onE-QSVPkuCApw,4949
8
+ fl_data_generator/quantity.py,sha256=_VEo9-GnPUHdvBLxB3GwpvO9aE_-f46sMteyBrFV0UY,3149
9
+ fl_data_generator-0.1.0.dist-info/licenses/LICENSE,sha256=hkYdGffXw9mXezr-rwHQYwTTqM1r5rvJfa5esQyyLkA,1077
10
+ fl_data_generator-0.1.0.dist-info/METADATA,sha256=lyy8zTz-KEpJpOMmHVDXRIjyiPm0KcWSLlHyR_24pMw,9225
11
+ fl_data_generator-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
12
+ fl_data_generator-0.1.0.dist-info/top_level.txt,sha256=sHZKMdxbAhWu3-VClvEPI1AaLhPsD9SPjt14IlOGByc,18
13
+ fl_data_generator-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Mostafa Khastkhodaei
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ fl_data_generator