fl-data-generator 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fl_data_generator/__init__.py +16 -0
- fl_data_generator/_validation.py +60 -0
- fl_data_generator/dataset.py +422 -0
- fl_data_generator/dirichlet.py +123 -0
- fl_data_generator/hybrid.py +138 -0
- fl_data_generator/iid.py +49 -0
- fl_data_generator/pathological.py +137 -0
- fl_data_generator/quantity.py +83 -0
- fl_data_generator-0.1.0.dist-info/METADATA +233 -0
- fl_data_generator-0.1.0.dist-info/RECORD +13 -0
- fl_data_generator-0.1.0.dist-info/WHEEL +5 -0
- fl_data_generator-0.1.0.dist-info/licenses/LICENSE +21 -0
- fl_data_generator-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from .dataset import FederatedDataset, NumpyDataset
|
|
2
|
+
from .dirichlet import dirichlet_partition
|
|
3
|
+
from .hybrid import hybrid_partition
|
|
4
|
+
from .iid import iid_partition
|
|
5
|
+
from .pathological import pathological_partition
|
|
6
|
+
from .quantity import quantity_skew_partition
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"FederatedDataset",
|
|
10
|
+
"NumpyDataset",
|
|
11
|
+
"dirichlet_partition",
|
|
12
|
+
"hybrid_partition",
|
|
13
|
+
"iid_partition",
|
|
14
|
+
"pathological_partition",
|
|
15
|
+
"quantity_skew_partition",
|
|
16
|
+
]
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Sequence
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def validate_labels(labels: Sequence | np.ndarray) -> np.ndarray:
|
|
9
|
+
"""Validate and cast dataset labels to a 1D int64 numpy array.
|
|
10
|
+
|
|
11
|
+
Parameters
|
|
12
|
+
----------
|
|
13
|
+
labels : Sequence or np.ndarray
|
|
14
|
+
Dataset labels as an array-like or sequence.
|
|
15
|
+
|
|
16
|
+
Returns
|
|
17
|
+
-------
|
|
18
|
+
np.ndarray
|
|
19
|
+
A 1D numpy array with dtype np.int64.
|
|
20
|
+
|
|
21
|
+
Raises
|
|
22
|
+
------
|
|
23
|
+
ValueError
|
|
24
|
+
If labels is empty, not 1-dimensional, contains NaN or Inf, or
|
|
25
|
+
cannot be converted to integer values without loss of information.
|
|
26
|
+
TypeError
|
|
27
|
+
If labels elements cannot be converted to numeric values.
|
|
28
|
+
"""
|
|
29
|
+
arr = np.asarray(labels)
|
|
30
|
+
|
|
31
|
+
if arr.ndim != 1:
|
|
32
|
+
raise ValueError("labels must be 1-dimensional; consider using labels.ravel()")
|
|
33
|
+
|
|
34
|
+
if arr.size == 0:
|
|
35
|
+
raise ValueError("labels cannot be empty")
|
|
36
|
+
|
|
37
|
+
if not np.issubdtype(arr.dtype, np.number):
|
|
38
|
+
# Could be object array with strings or other non-numeric types
|
|
39
|
+
try:
|
|
40
|
+
arr = arr.astype(np.float64)
|
|
41
|
+
except (ValueError, TypeError) as exc:
|
|
42
|
+
raise TypeError("labels must contain numeric values") from exc
|
|
43
|
+
|
|
44
|
+
# Check for NaN or Inf
|
|
45
|
+
if np.isnan(arr).any():
|
|
46
|
+
raise ValueError("labels contains NaN values")
|
|
47
|
+
|
|
48
|
+
if np.isinf(arr).any():
|
|
49
|
+
raise ValueError("labels contains infinite values")
|
|
50
|
+
|
|
51
|
+
# Check if integer-valued
|
|
52
|
+
if np.issubdtype(arr.dtype, np.floating):
|
|
53
|
+
if not np.all(np.floor(arr) == arr):
|
|
54
|
+
raise ValueError("labels must be integer-valued")
|
|
55
|
+
arr = arr.astype(np.int64)
|
|
56
|
+
elif np.issubdtype(arr.dtype, np.integer):
|
|
57
|
+
if arr.dtype != np.int64:
|
|
58
|
+
arr = arr.astype(np.int64)
|
|
59
|
+
|
|
60
|
+
return arr
|
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
from typing import Any, Callable, Dict, Optional, Tuple, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
import torch
|
|
7
|
+
from torch.utils.data import DataLoader, Dataset, Subset
|
|
8
|
+
|
|
9
|
+
_HAS_TORCH = True
|
|
10
|
+
except ImportError:
|
|
11
|
+
_HAS_TORCH = False
|
|
12
|
+
Dataset = object # type: ignore
|
|
13
|
+
DataLoader = None # type: ignore
|
|
14
|
+
Subset = None # type: ignore
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
import torchvision
|
|
18
|
+
import torchvision.datasets as tv_datasets
|
|
19
|
+
|
|
20
|
+
_HAS_TORCHVISION = True
|
|
21
|
+
except ImportError:
|
|
22
|
+
_HAS_TORCHVISION = False
|
|
23
|
+
tv_datasets = None # type: ignore
|
|
24
|
+
|
|
25
|
+
from .dirichlet import dirichlet_partition
|
|
26
|
+
from .hybrid import hybrid_partition
|
|
27
|
+
from .iid import iid_partition
|
|
28
|
+
from .pathological import pathological_partition
|
|
29
|
+
from .quantity import quantity_skew_partition
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class NumpyDataset(Dataset):
|
|
33
|
+
"""Lightweight dataset wrapper for NumPy arrays or tensor tuples.
|
|
34
|
+
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
data : Sequence or np.ndarray or torch.Tensor
|
|
38
|
+
Feature array or tensor.
|
|
39
|
+
targets : Sequence or np.ndarray or torch.Tensor
|
|
40
|
+
Target labels array or tensor.
|
|
41
|
+
transform : callable, optional
|
|
42
|
+
Transformation applied to feature samples.
|
|
43
|
+
target_transform : callable, optional
|
|
44
|
+
Transformation applied to target labels.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
data: Any,
|
|
50
|
+
targets: Any,
|
|
51
|
+
transform: Optional[Callable] = None,
|
|
52
|
+
target_transform: Optional[Callable] = None,
|
|
53
|
+
) -> None:
|
|
54
|
+
self.data = data
|
|
55
|
+
self.targets = targets
|
|
56
|
+
self.transform = transform
|
|
57
|
+
self.target_transform = target_transform
|
|
58
|
+
|
|
59
|
+
def __len__(self) -> int:
|
|
60
|
+
return len(self.targets)
|
|
61
|
+
|
|
62
|
+
def __getitem__(self, index: int) -> Tuple[Any, Any]:
|
|
63
|
+
x = self.data[index]
|
|
64
|
+
y = self.targets[index]
|
|
65
|
+
|
|
66
|
+
if self.transform is not None:
|
|
67
|
+
x = self.transform(x)
|
|
68
|
+
if self.target_transform is not None:
|
|
69
|
+
y = self.target_transform(y)
|
|
70
|
+
|
|
71
|
+
return x, y
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class FederatedDataset:
|
|
75
|
+
"""High-level Federated Dataset manager for simulated distributed learning.
|
|
76
|
+
|
|
77
|
+
Provides automatic downloading, loading, and partitioning of datasets across
|
|
78
|
+
simulated clients according to standard Federated Learning benchmark distributions.
|
|
79
|
+
|
|
80
|
+
Supported Partitioning Regimes
|
|
81
|
+
------------------------------
|
|
82
|
+
- **"iid"**:
|
|
83
|
+
Uniform random data splitting across all clients. Every client receives roughly
|
|
84
|
+
equal sample quantities and an identical class distribution.
|
|
85
|
+
|
|
86
|
+
- **"dirichlet"** (Label Skew):
|
|
87
|
+
Continuous non-IID label distribution based on Dirichlet sampling Dir(alpha).
|
|
88
|
+
* **alpha -> infinity** (e.g., 100.0): Approaches uniform, near-IID class distribution.
|
|
89
|
+
* **alpha ≈ 0.5**: Standard benchmark setting representing realistic heterogeneous clients.
|
|
90
|
+
* **alpha -> 0** (e.g., 0.05, 0.01): Extreme non-IID label skew where each client predominantly
|
|
91
|
+
holds samples from only 1 or 2 classes.
|
|
92
|
+
|
|
93
|
+
- **"pathological"** (Shard-based Label Skew):
|
|
94
|
+
The classic FedAvg benchmark (McMahan et al., 2017). Samples are sorted by label,
|
|
95
|
+
split into shards, and each client is assigned shards from a fixed number of classes
|
|
96
|
+
(configured via `classes_per_client`, e.g., 2 classes per client for MNIST).
|
|
97
|
+
|
|
98
|
+
- **"quantity"** (Sample Volume Skew):
|
|
99
|
+
Simulates hardware/usage heterogeneity where different clients collect vastly different
|
|
100
|
+
volumes of data. Client sample sizes follow a Dirichlet distribution parameterized by `beta`.
|
|
101
|
+
* **beta -> infinity** (e.g., 100.0): Clients receive roughly equal sample volumes.
|
|
102
|
+
* **beta -> 0** (e.g., 0.1): Extreme volume disparity (some clients receive hundreds of samples,
|
|
103
|
+
while others receive single digits).
|
|
104
|
+
|
|
105
|
+
- **"hybrid"** (Joint Label & Quantity Skew):
|
|
106
|
+
Combines volume disparity (governed by `beta`) with label preference heterogeneity
|
|
107
|
+
(governed by `alpha`).
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
dataset : str or Tuple[Any, Any] or Dataset
|
|
112
|
+
Dataset identifier or instance:
|
|
113
|
+
- String name for built-in torchvision datasets: `"mnist"`, `"cifar10"`,
|
|
114
|
+
`"cifar100"`, `"fashion_mnist"`.
|
|
115
|
+
- Tuple of `(X, y)` containing features and labels (NumPy arrays or lists).
|
|
116
|
+
- Existing PyTorch `Dataset` object.
|
|
117
|
+
data_dir : str, default="./data"
|
|
118
|
+
Directory path used to cache and download standard datasets.
|
|
119
|
+
num_clients : int
|
|
120
|
+
Number of participating federated clients.
|
|
121
|
+
partition : str, default="iid"
|
|
122
|
+
Partition strategy. Choices: `"iid"`, `"dirichlet"`, `"pathological"`,
|
|
123
|
+
`"quantity"`, `"hybrid"`.
|
|
124
|
+
alpha : float, default=0.5
|
|
125
|
+
Dirichlet concentration parameter for label skew. Smaller values yield stronger
|
|
126
|
+
label heterogeneity across clients.
|
|
127
|
+
beta : float, default=0.5
|
|
128
|
+
Dirichlet concentration parameter for quantity skew. Smaller values yield higher
|
|
129
|
+
variance in client sample counts.
|
|
130
|
+
classes_per_client : int, default=2
|
|
131
|
+
Number of distinct classes assigned to each client in `"pathological"` partition.
|
|
132
|
+
seed : int, optional
|
|
133
|
+
Random seed for reproducible dataset partitioning.
|
|
134
|
+
train : bool, default=True
|
|
135
|
+
Whether to load the training or test split when downloading built-in datasets.
|
|
136
|
+
download : bool, default=True
|
|
137
|
+
Whether to automatically download built-in torchvision datasets if not present locally.
|
|
138
|
+
transform : callable, optional
|
|
139
|
+
Feature transformation applied to samples.
|
|
140
|
+
target_transform : callable, optional
|
|
141
|
+
Target transformation applied to labels.
|
|
142
|
+
|
|
143
|
+
Examples
|
|
144
|
+
--------
|
|
145
|
+
>>> # Using standard MNIST with Dirichlet label skew:
|
|
146
|
+
>>> fed_data = FederatedDataset("mnist", num_clients=10, partition="dirichlet", alpha=0.5)
|
|
147
|
+
>>> client_0_loader = fed_data.get_client_dataloader(client_id=0, batch_size=32)
|
|
148
|
+
|
|
149
|
+
>>> # Using custom NumPy arrays with quantity skew:
|
|
150
|
+
>>> import numpy as np
|
|
151
|
+
>>> X = np.random.randn(1000, 20)
|
|
152
|
+
>>> y = np.random.randint(0, 5, size=1000)
|
|
153
|
+
>>> fed_data = FederatedDataset((X, y), num_clients=5, partition="quantity", beta=0.2)
|
|
154
|
+
>>> indices = fed_data.get_client_indices(client_id=0)
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
SUPPORTED_DATASETS = {
|
|
158
|
+
"mnist": "MNIST",
|
|
159
|
+
"cifar10": "CIFAR10",
|
|
160
|
+
"cifar100": "CIFAR100",
|
|
161
|
+
"fashion_mnist": "FashionMNIST",
|
|
162
|
+
"fashionmnist": "FashionMNIST",
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
SUPPORTED_PARTITIONS = {
|
|
166
|
+
"iid",
|
|
167
|
+
"dirichlet",
|
|
168
|
+
"pathological",
|
|
169
|
+
"quantity",
|
|
170
|
+
"hybrid",
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
def __init__(
|
|
174
|
+
self,
|
|
175
|
+
dataset: Union[str, Tuple[Any, Any], Any],
|
|
176
|
+
data_dir: str = "./data",
|
|
177
|
+
num_clients: int = 10,
|
|
178
|
+
partition: str = "iid",
|
|
179
|
+
alpha: float = 0.5,
|
|
180
|
+
beta: float = 0.5,
|
|
181
|
+
classes_per_client: int = 2,
|
|
182
|
+
seed: Optional[int] = None,
|
|
183
|
+
train: bool = True,
|
|
184
|
+
download: bool = True,
|
|
185
|
+
transform: Optional[Callable] = None,
|
|
186
|
+
target_transform: Optional[Callable] = None,
|
|
187
|
+
) -> None:
|
|
188
|
+
self.data_dir = data_dir
|
|
189
|
+
self.num_clients = num_clients
|
|
190
|
+
self.partition = partition.lower()
|
|
191
|
+
self.alpha = alpha
|
|
192
|
+
self.beta = beta
|
|
193
|
+
self.classes_per_client = classes_per_client
|
|
194
|
+
self.seed = seed
|
|
195
|
+
self.train = train
|
|
196
|
+
self.download = download
|
|
197
|
+
self.transform = transform
|
|
198
|
+
self.target_transform = target_transform
|
|
199
|
+
|
|
200
|
+
if self.num_clients <= 0:
|
|
201
|
+
raise ValueError(
|
|
202
|
+
f"num_clients must be positive, got {self.num_clients}"
|
|
203
|
+
)
|
|
204
|
+
if self.partition not in self.SUPPORTED_PARTITIONS:
|
|
205
|
+
raise ValueError(
|
|
206
|
+
f"Unsupported partition '{self.partition}'. "
|
|
207
|
+
f"Choose from: {sorted(self.SUPPORTED_PARTITIONS)}"
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# 1. Load the underlying base dataset and extract target labels
|
|
211
|
+
self.base_dataset, self.labels = self._load_dataset(dataset)
|
|
212
|
+
|
|
213
|
+
# 2. Perform the specified partitioning
|
|
214
|
+
self.client_dict = self._partition_data()
|
|
215
|
+
|
|
216
|
+
def _load_dataset(self, dataset: Any) -> Tuple[Any, np.ndarray]:
|
|
217
|
+
"""Resolve dataset input into a dataset instance and NumPy labels array."""
|
|
218
|
+
if isinstance(dataset, str):
|
|
219
|
+
ds_key = dataset.lower().strip()
|
|
220
|
+
if ds_key not in self.SUPPORTED_DATASETS:
|
|
221
|
+
raise ValueError(
|
|
222
|
+
f"Unknown built-in dataset '{dataset}'. "
|
|
223
|
+
f"Supported names: {sorted(self.SUPPORTED_DATASETS.keys())}"
|
|
224
|
+
)
|
|
225
|
+
if not _HAS_TORCHVISION:
|
|
226
|
+
raise ImportError(
|
|
227
|
+
f"torchvision is required to download and load '{dataset}'. "
|
|
228
|
+
f"Please install torchvision or provide custom (X, y) data."
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
ds_class_name = self.SUPPORTED_DATASETS[ds_key]
|
|
232
|
+
ds_cls = getattr(tv_datasets, ds_class_name)
|
|
233
|
+
|
|
234
|
+
ds_instance = ds_cls(
|
|
235
|
+
root=self.data_dir,
|
|
236
|
+
train=self.train,
|
|
237
|
+
download=self.download,
|
|
238
|
+
transform=self.transform,
|
|
239
|
+
target_transform=self.target_transform,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# Extract targets
|
|
243
|
+
if hasattr(ds_instance, "targets"):
|
|
244
|
+
targets = ds_instance.targets
|
|
245
|
+
if hasattr(targets, "numpy"):
|
|
246
|
+
labels = targets.numpy()
|
|
247
|
+
else:
|
|
248
|
+
labels = np.asarray(targets)
|
|
249
|
+
else:
|
|
250
|
+
labels = np.array([ds_instance[i][1] for i in range(len(ds_instance))])
|
|
251
|
+
|
|
252
|
+
return ds_instance, labels
|
|
253
|
+
|
|
254
|
+
if isinstance(dataset, tuple) and len(dataset) == 2:
|
|
255
|
+
data, targets = dataset
|
|
256
|
+
if len(data) != len(targets):
|
|
257
|
+
raise ValueError(
|
|
258
|
+
f"Mismatch between features length ({len(data)}) and "
|
|
259
|
+
f"targets length ({len(targets)})"
|
|
260
|
+
)
|
|
261
|
+
labels = np.asarray(targets)
|
|
262
|
+
ds_instance = NumpyDataset(
|
|
263
|
+
data=data,
|
|
264
|
+
targets=targets,
|
|
265
|
+
transform=self.transform,
|
|
266
|
+
target_transform=self.target_transform,
|
|
267
|
+
)
|
|
268
|
+
return ds_instance, labels
|
|
269
|
+
|
|
270
|
+
# Generic dataset object (e.g. custom PyTorch Dataset)
|
|
271
|
+
if hasattr(dataset, "__len__") and hasattr(dataset, "__getitem__"):
|
|
272
|
+
if hasattr(dataset, "targets"):
|
|
273
|
+
targets = dataset.targets
|
|
274
|
+
labels = targets.numpy() if hasattr(targets, "numpy") else np.asarray(targets)
|
|
275
|
+
elif hasattr(dataset, "labels"):
|
|
276
|
+
labels = np.asarray(dataset.labels)
|
|
277
|
+
else:
|
|
278
|
+
labels = np.array([dataset[i][1] for i in range(len(dataset))])
|
|
279
|
+
return dataset, labels
|
|
280
|
+
|
|
281
|
+
raise TypeError(
|
|
282
|
+
f"dataset must be a string name, (X, y) tuple, or Dataset instance. Got: {type(dataset)}"
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def _partition_data(self) -> Dict[int, np.ndarray]:
|
|
286
|
+
"""Dispatch partitioning according to configured strategy."""
|
|
287
|
+
if self.partition == "iid":
|
|
288
|
+
return iid_partition(
|
|
289
|
+
labels=self.labels,
|
|
290
|
+
num_clients=self.num_clients,
|
|
291
|
+
seed=self.seed,
|
|
292
|
+
)
|
|
293
|
+
elif self.partition == "dirichlet":
|
|
294
|
+
return dirichlet_partition(
|
|
295
|
+
labels=self.labels,
|
|
296
|
+
num_clients=self.num_clients,
|
|
297
|
+
alpha=self.alpha,
|
|
298
|
+
seed=self.seed,
|
|
299
|
+
)
|
|
300
|
+
elif self.partition == "pathological":
|
|
301
|
+
return pathological_partition(
|
|
302
|
+
labels=self.labels,
|
|
303
|
+
num_clients=self.num_clients,
|
|
304
|
+
classes_per_client=self.classes_per_client,
|
|
305
|
+
seed=self.seed,
|
|
306
|
+
)
|
|
307
|
+
elif self.partition == "quantity":
|
|
308
|
+
return quantity_skew_partition(
|
|
309
|
+
labels=self.labels,
|
|
310
|
+
num_clients=self.num_clients,
|
|
311
|
+
beta=self.beta,
|
|
312
|
+
seed=self.seed,
|
|
313
|
+
)
|
|
314
|
+
elif self.partition == "hybrid":
|
|
315
|
+
return hybrid_partition(
|
|
316
|
+
labels=self.labels,
|
|
317
|
+
num_clients=self.num_clients,
|
|
318
|
+
alpha=self.alpha,
|
|
319
|
+
beta=self.beta,
|
|
320
|
+
seed=self.seed,
|
|
321
|
+
)
|
|
322
|
+
else:
|
|
323
|
+
raise ValueError(f"Unknown partition strategy: {self.partition}")
|
|
324
|
+
|
|
325
|
+
def get_client_indices(self, client_id: int) -> np.ndarray:
|
|
326
|
+
"""Get the array of sample indices allocated to a specific client.
|
|
327
|
+
|
|
328
|
+
Parameters
|
|
329
|
+
----------
|
|
330
|
+
client_id : int
|
|
331
|
+
Index of the client (0 <= client_id < num_clients).
|
|
332
|
+
|
|
333
|
+
Returns
|
|
334
|
+
-------
|
|
335
|
+
np.ndarray
|
|
336
|
+
1D array of sample indices.
|
|
337
|
+
"""
|
|
338
|
+
if client_id not in self.client_dict:
|
|
339
|
+
raise IndexError(
|
|
340
|
+
f"Client ID {client_id} out of bounds for num_clients={self.num_clients}"
|
|
341
|
+
)
|
|
342
|
+
return self.client_dict[client_id]
|
|
343
|
+
|
|
344
|
+
def get_client_dataset(self, client_id: int) -> Any:
|
|
345
|
+
"""Get a Dataset instance containing only the client's assigned samples.
|
|
346
|
+
|
|
347
|
+
Parameters
|
|
348
|
+
----------
|
|
349
|
+
client_id : int
|
|
350
|
+
Index of the client.
|
|
351
|
+
|
|
352
|
+
Returns
|
|
353
|
+
-------
|
|
354
|
+
Dataset or Subset
|
|
355
|
+
Client dataset wrapper.
|
|
356
|
+
"""
|
|
357
|
+
indices = self.get_client_indices(client_id)
|
|
358
|
+
if _HAS_TORCH and Subset is not None:
|
|
359
|
+
return Subset(self.base_dataset, indices)
|
|
360
|
+
|
|
361
|
+
# Fallback subset wrapper for non-torch environments
|
|
362
|
+
class _NonTorchSubset:
|
|
363
|
+
def __init__(self, base, idxs):
|
|
364
|
+
self.base = base
|
|
365
|
+
self.indices = idxs
|
|
366
|
+
|
|
367
|
+
def __len__(self):
|
|
368
|
+
return len(self.indices)
|
|
369
|
+
|
|
370
|
+
def __getitem__(self, i):
|
|
371
|
+
return self.base[self.indices[i]]
|
|
372
|
+
|
|
373
|
+
return _NonTorchSubset(self.base_dataset, indices)
|
|
374
|
+
|
|
375
|
+
def get_client_dataloader(
|
|
376
|
+
self,
|
|
377
|
+
client_id: int,
|
|
378
|
+
batch_size: int = 32,
|
|
379
|
+
shuffle: bool = True,
|
|
380
|
+
**kwargs: Any,
|
|
381
|
+
) -> Any:
|
|
382
|
+
"""Get a PyTorch DataLoader for the specified client.
|
|
383
|
+
|
|
384
|
+
Parameters
|
|
385
|
+
----------
|
|
386
|
+
client_id : int
|
|
387
|
+
Index of the client.
|
|
388
|
+
batch_size : int, default=32
|
|
389
|
+
Mini-batch size for client local training.
|
|
390
|
+
shuffle : bool, default=True
|
|
391
|
+
Whether to shuffle the client's data each epoch.
|
|
392
|
+
**kwargs : Any
|
|
393
|
+
Additional keyword arguments passed to `torch.utils.data.DataLoader`.
|
|
394
|
+
|
|
395
|
+
Returns
|
|
396
|
+
-------
|
|
397
|
+
torch.utils.data.DataLoader
|
|
398
|
+
Configured DataLoader for local client training.
|
|
399
|
+
"""
|
|
400
|
+
if not _HAS_TORCH or DataLoader is None:
|
|
401
|
+
raise ImportError(
|
|
402
|
+
"PyTorch is required to create a DataLoader. "
|
|
403
|
+
"Install torch (`pip install torch`) or use get_client_dataset() / get_client_indices()."
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
client_ds = self.get_client_dataset(client_id)
|
|
407
|
+
return DataLoader(client_ds, batch_size=batch_size, shuffle=shuffle, **kwargs)
|
|
408
|
+
|
|
409
|
+
@property
|
|
410
|
+
def client_sizes(self) -> Dict[int, int]:
|
|
411
|
+
"""Dictionary mapping client IDs to assigned sample counts."""
|
|
412
|
+
return {cid: len(indices) for cid, indices in self.client_dict.items()}
|
|
413
|
+
|
|
414
|
+
def __len__(self) -> int:
|
|
415
|
+
"""Total number of samples in the partitioned dataset."""
|
|
416
|
+
return len(self.labels)
|
|
417
|
+
|
|
418
|
+
def __repr__(self) -> str:
|
|
419
|
+
return (
|
|
420
|
+
f"FederatedDataset(num_clients={self.num_clients}, "
|
|
421
|
+
f"partition='{self.partition}', total_samples={len(self)})"
|
|
422
|
+
)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from typing import Dict, Optional, Sequence, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._validation import validate_labels
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def dirichlet_partition(
|
|
9
|
+
labels: Union[Sequence, np.ndarray],
|
|
10
|
+
num_clients: int,
|
|
11
|
+
alpha: Union[float, Sequence[float]] = 0.5,
|
|
12
|
+
seed: Optional[int] = None,
|
|
13
|
+
min_size: int = 1,
|
|
14
|
+
max_retries: int = 100,
|
|
15
|
+
) -> Dict[int, np.ndarray]:
|
|
16
|
+
"""Partition dataset indices among clients using a Dirichlet distribution over classes.
|
|
17
|
+
|
|
18
|
+
Simulates non-IID label skew across clients in Federated Learning. Smaller alpha values
|
|
19
|
+
result in higher distribution skew (more non-IID), while larger alpha values approach
|
|
20
|
+
an IID distribution.
|
|
21
|
+
|
|
22
|
+
Parameters
|
|
23
|
+
----------
|
|
24
|
+
labels : Sequence or np.ndarray
|
|
25
|
+
Dataset class labels or targets.
|
|
26
|
+
num_clients : int
|
|
27
|
+
Number of clients to partition data among.
|
|
28
|
+
alpha : float or Sequence[float], default=0.5
|
|
29
|
+
Concentration parameter for the Dirichlet distribution.
|
|
30
|
+
seed : int, optional
|
|
31
|
+
Random seed for reproducibility.
|
|
32
|
+
min_size : int, default=1
|
|
33
|
+
Minimum number of samples required per client.
|
|
34
|
+
max_retries : int, default=100
|
|
35
|
+
Maximum number of Dirichlet resampling attempts to satisfy min_size.
|
|
36
|
+
|
|
37
|
+
Returns
|
|
38
|
+
-------
|
|
39
|
+
dict of int to np.ndarray
|
|
40
|
+
Mapping from client ID to array of assigned sample indices.
|
|
41
|
+
|
|
42
|
+
Raises
|
|
43
|
+
------
|
|
44
|
+
ValueError
|
|
45
|
+
If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
|
|
46
|
+
If num_clients <= 0 or alpha <= 0.
|
|
47
|
+
If num_clients exceeds num_samples.
|
|
48
|
+
If alpha is a sequence and its length does not match num_clients.
|
|
49
|
+
If min_size < 0, not an integer, or min_size * num_clients > num_samples.
|
|
50
|
+
RuntimeError
|
|
51
|
+
If unable to partition data such that all clients receive at least min_size samples
|
|
52
|
+
within max_retries attempts.
|
|
53
|
+
"""
|
|
54
|
+
labels_arr = validate_labels(labels)
|
|
55
|
+
num_samples = len(labels_arr)
|
|
56
|
+
|
|
57
|
+
if num_clients <= 0:
|
|
58
|
+
raise ValueError(f"num_clients must be positive, got {num_clients}")
|
|
59
|
+
if num_clients > num_samples:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
if not isinstance(min_size, int) or isinstance(min_size, bool):
|
|
65
|
+
raise ValueError(f"min_size must be an integer, got {type(min_size).__name__}") # noqa: TRY004
|
|
66
|
+
if min_size < 0:
|
|
67
|
+
raise ValueError(f"min_size must be non-negative, got {min_size}")
|
|
68
|
+
if min_size * num_clients > num_samples:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
f"min_size * num_clients ({min_size * num_clients}) cannot exceed number of samples ({num_samples})"
|
|
71
|
+
)
|
|
72
|
+
if not isinstance(max_retries, int) or isinstance(max_retries, bool) or max_retries <= 0:
|
|
73
|
+
raise ValueError(f"max_retries must be a positive integer, got {max_retries}")
|
|
74
|
+
|
|
75
|
+
if isinstance(alpha, (int, float)):
|
|
76
|
+
if alpha <= 0:
|
|
77
|
+
raise ValueError(f"alpha must be positive, got {alpha}")
|
|
78
|
+
alpha_vec = np.repeat(float(alpha), num_clients)
|
|
79
|
+
else:
|
|
80
|
+
alpha_vec = np.asarray(alpha, dtype=float)
|
|
81
|
+
if alpha_vec.shape != (num_clients,):
|
|
82
|
+
raise ValueError(
|
|
83
|
+
f"alpha length must match num_clients ({num_clients}), got {len(alpha_vec)}"
|
|
84
|
+
)
|
|
85
|
+
if (alpha_vec <= 0).any():
|
|
86
|
+
raise ValueError("All alpha values must be positive")
|
|
87
|
+
|
|
88
|
+
rng = np.random.default_rng(seed)
|
|
89
|
+
unique_classes = np.unique(labels_arr)
|
|
90
|
+
|
|
91
|
+
for _ in range(max_retries):
|
|
92
|
+
client_indices: Dict[int, list] = {cid: [] for cid in range(num_clients)}
|
|
93
|
+
|
|
94
|
+
for cls in unique_classes:
|
|
95
|
+
cls_idx = np.where(labels_arr == cls)[0]
|
|
96
|
+
rng.shuffle(cls_idx)
|
|
97
|
+
|
|
98
|
+
proportions = rng.dirichlet(alpha_vec)
|
|
99
|
+
split_points = (np.cumsum(proportions)[:-1] * len(cls_idx)).astype(int)
|
|
100
|
+
splits = np.split(cls_idx, split_points)
|
|
101
|
+
|
|
102
|
+
for cid, split in enumerate(splits):
|
|
103
|
+
if len(split) > 0:
|
|
104
|
+
client_indices[cid].append(split)
|
|
105
|
+
|
|
106
|
+
if all(sum(len(s) for s in client_indices[cid]) >= min_size for cid in range(num_clients)):
|
|
107
|
+
break
|
|
108
|
+
else:
|
|
109
|
+
raise RuntimeError(
|
|
110
|
+
f"Could not partition data such that all clients receive at least min_size={min_size} samples "
|
|
111
|
+
f"after {max_retries} retries. Try using a larger alpha or a smaller min_size."
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
result = {}
|
|
115
|
+
for cid in range(num_clients):
|
|
116
|
+
if client_indices[cid]:
|
|
117
|
+
merged = np.concatenate(client_indices[cid])
|
|
118
|
+
rng.shuffle(merged)
|
|
119
|
+
result[cid] = merged
|
|
120
|
+
else:
|
|
121
|
+
result[cid] = np.array([], dtype=int)
|
|
122
|
+
|
|
123
|
+
return result
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
from typing import Dict, Optional, Sequence, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._validation import validate_labels
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def hybrid_partition(
|
|
9
|
+
labels: Union[Sequence, np.ndarray],
|
|
10
|
+
num_clients: int,
|
|
11
|
+
alpha: float = 0.5,
|
|
12
|
+
beta: float = 0.5,
|
|
13
|
+
seed: Optional[int] = None,
|
|
14
|
+
) -> Dict[int, np.ndarray]:
|
|
15
|
+
"""Partition dataset indices combining label skew and quantity skew (experimental).
|
|
16
|
+
|
|
17
|
+
.. warning::
|
|
18
|
+
This method is experimental and not a standard published partitioning method.
|
|
19
|
+
It uses sequential allocation: client sample capacities are first determined
|
|
20
|
+
via Dirichlet distribution with parameter beta (quantity skew). Samples are then
|
|
21
|
+
allocated sequentially to clients in a randomly permuted client order based on class
|
|
22
|
+
preference distributions drawn from a Dirichlet distribution with parameter alpha
|
|
23
|
+
(label skew). Because samples are drawn without replacement from a shared finite pool,
|
|
24
|
+
there is a client-order effect: earlier clients in the sequence have access to the
|
|
25
|
+
full class distribution, while later clients may receive residual classes that deviate
|
|
26
|
+
from their drawn preferences.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
labels : Sequence or np.ndarray
|
|
31
|
+
Dataset class labels or targets.
|
|
32
|
+
num_clients : int
|
|
33
|
+
Number of clients to partition data among.
|
|
34
|
+
alpha : float, default=0.5
|
|
35
|
+
Concentration parameter for the Dirichlet distribution governing label skew.
|
|
36
|
+
Must be a positive scalar. Smaller values induce stronger class preference heterogeneity.
|
|
37
|
+
beta : float, default=0.5
|
|
38
|
+
Concentration parameter for the Dirichlet distribution governing quantity skew.
|
|
39
|
+
Must be a positive scalar. Smaller values induce greater disparity in sample volume across clients.
|
|
40
|
+
seed : int, optional
|
|
41
|
+
Random seed for reproducibility.
|
|
42
|
+
|
|
43
|
+
Returns
|
|
44
|
+
-------
|
|
45
|
+
dict of int to np.ndarray
|
|
46
|
+
Mapping from client ID to array of assigned sample indices.
|
|
47
|
+
|
|
48
|
+
Raises
|
|
49
|
+
------
|
|
50
|
+
ValueError
|
|
51
|
+
If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
|
|
52
|
+
If num_clients <= 0, alpha <= 0, or beta <= 0.
|
|
53
|
+
If num_clients exceeds num_samples.
|
|
54
|
+
If alpha or beta is a sequence rather than a scalar.
|
|
55
|
+
"""
|
|
56
|
+
labels_arr = validate_labels(labels)
|
|
57
|
+
num_samples = len(labels_arr)
|
|
58
|
+
|
|
59
|
+
if num_clients <= 0:
|
|
60
|
+
raise ValueError(f"num_clients must be positive, got {num_clients}")
|
|
61
|
+
if num_clients > num_samples:
|
|
62
|
+
raise ValueError(
|
|
63
|
+
f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
if isinstance(alpha, (Sequence, np.ndarray)) or np.ndim(alpha) > 0 or not isinstance(alpha, (int, float, np.number)) or isinstance(alpha, bool):
|
|
67
|
+
raise ValueError(f"alpha must be a scalar positive number, got {alpha}")
|
|
68
|
+
if alpha <= 0:
|
|
69
|
+
raise ValueError(f"alpha must be positive, got {alpha}")
|
|
70
|
+
|
|
71
|
+
if isinstance(beta, (Sequence, np.ndarray)) or np.ndim(beta) > 0 or not isinstance(beta, (int, float, np.number)) or isinstance(beta, bool):
|
|
72
|
+
raise ValueError(f"beta must be a scalar positive number, got {beta}")
|
|
73
|
+
if beta <= 0:
|
|
74
|
+
raise ValueError(f"beta must be positive, got {beta}")
|
|
75
|
+
|
|
76
|
+
rng = np.random.default_rng(seed)
|
|
77
|
+
unique_classes = np.unique(labels_arr)
|
|
78
|
+
num_classes = len(unique_classes)
|
|
79
|
+
cls_to_idx = {c: i for i, c in enumerate(unique_classes)}
|
|
80
|
+
|
|
81
|
+
# 1. Determine client sample sizes via quantity skew (Dirichlet with parameter beta)
|
|
82
|
+
q_props = rng.dirichlet(np.repeat(beta, num_clients))
|
|
83
|
+
remaining = num_samples - num_clients
|
|
84
|
+
splits = (np.cumsum(q_props)[:-1] * remaining).astype(int)
|
|
85
|
+
client_sizes = np.diff(np.concatenate(([0], splits, [remaining]))) + 1
|
|
86
|
+
|
|
87
|
+
# Organize available sample indices by class
|
|
88
|
+
class_indices: Dict[object, list] = {}
|
|
89
|
+
for cls in unique_classes:
|
|
90
|
+
idx = np.where(labels_arr == cls)[0]
|
|
91
|
+
rng.shuffle(idx)
|
|
92
|
+
class_indices[cls] = list(idx)
|
|
93
|
+
|
|
94
|
+
# 2. Allocate samples to each client proportionally based on label skew (Dirichlet alpha)
|
|
95
|
+
client_partitions: Dict[int, list] = {cid: [] for cid in range(num_clients)}
|
|
96
|
+
client_order = rng.permutation(num_clients)
|
|
97
|
+
|
|
98
|
+
for cid in client_order:
|
|
99
|
+
needed = client_sizes[cid]
|
|
100
|
+
p = rng.dirichlet(np.repeat(alpha, num_classes))
|
|
101
|
+
|
|
102
|
+
while needed > 0:
|
|
103
|
+
avail_classes = [c for c in unique_classes if len(class_indices[c]) > 0]
|
|
104
|
+
if not avail_classes:
|
|
105
|
+
break
|
|
106
|
+
|
|
107
|
+
avail_p = np.array([p[cls_to_idx[c]] for c in avail_classes])
|
|
108
|
+
p_sum = avail_p.sum()
|
|
109
|
+
if p_sum == 0:
|
|
110
|
+
avail_p = np.ones(len(avail_classes))
|
|
111
|
+
p_sum = len(avail_classes)
|
|
112
|
+
avail_p = avail_p / p_sum
|
|
113
|
+
|
|
114
|
+
counts = (avail_p * needed).astype(int)
|
|
115
|
+
if counts.sum() == 0:
|
|
116
|
+
counts[np.argmax(avail_p)] = 1
|
|
117
|
+
|
|
118
|
+
allocated_this_round = 0
|
|
119
|
+
for c, cnt in zip(avail_classes, counts):
|
|
120
|
+
take = min(cnt, len(class_indices[c]), needed)
|
|
121
|
+
if take > 0:
|
|
122
|
+
for _ in range(take):
|
|
123
|
+
client_partitions[cid].append(class_indices[c].pop())
|
|
124
|
+
needed -= take
|
|
125
|
+
allocated_this_round += take
|
|
126
|
+
|
|
127
|
+
if allocated_this_round == 0:
|
|
128
|
+
c = avail_classes[0]
|
|
129
|
+
client_partitions[cid].append(class_indices[c].pop())
|
|
130
|
+
needed -= 1
|
|
131
|
+
|
|
132
|
+
result: Dict[int, np.ndarray] = {}
|
|
133
|
+
for cid in range(num_clients):
|
|
134
|
+
arr = np.array(client_partitions[cid], dtype=int)
|
|
135
|
+
rng.shuffle(arr)
|
|
136
|
+
result[cid] = arr
|
|
137
|
+
|
|
138
|
+
return result
|
fl_data_generator/iid.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from typing import Dict, Optional, Sequence, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._validation import validate_labels
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def iid_partition(
|
|
9
|
+
labels: Union[Sequence, np.ndarray],
|
|
10
|
+
num_clients: int,
|
|
11
|
+
seed: Optional[int] = None,
|
|
12
|
+
) -> Dict[int, np.ndarray]:
|
|
13
|
+
"""Partition dataset indices in an IID fashion across clients.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
labels : Sequence or np.ndarray
|
|
18
|
+
Dataset labels or target sequence.
|
|
19
|
+
num_clients : int
|
|
20
|
+
Number of clients to partition data among.
|
|
21
|
+
seed : int, optional
|
|
22
|
+
Random seed for reproducibility.
|
|
23
|
+
|
|
24
|
+
Returns
|
|
25
|
+
-------
|
|
26
|
+
dict of int to np.ndarray
|
|
27
|
+
Mapping from client ID to array of assigned sample indices.
|
|
28
|
+
|
|
29
|
+
Raises
|
|
30
|
+
------
|
|
31
|
+
ValueError
|
|
32
|
+
If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
|
|
33
|
+
If num_clients <= 0 or num_clients > num_samples.
|
|
34
|
+
"""
|
|
35
|
+
labels_arr = validate_labels(labels)
|
|
36
|
+
num_samples = len(labels_arr)
|
|
37
|
+
|
|
38
|
+
if num_clients <= 0:
|
|
39
|
+
raise ValueError(f"num_clients must be positive, got {num_clients}")
|
|
40
|
+
if num_clients > num_samples:
|
|
41
|
+
raise ValueError(
|
|
42
|
+
f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
rng = np.random.default_rng(seed)
|
|
46
|
+
indices = rng.permutation(num_samples)
|
|
47
|
+
client_splits = np.array_split(indices, num_clients)
|
|
48
|
+
|
|
49
|
+
return {cid: split for cid, split in enumerate(client_splits)}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
from typing import Dict, Optional, Sequence, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._validation import validate_labels
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def pathological_partition(
|
|
9
|
+
labels: Union[Sequence, np.ndarray],
|
|
10
|
+
num_clients: int,
|
|
11
|
+
classes_per_client: int = 2,
|
|
12
|
+
seed: Optional[int] = None,
|
|
13
|
+
) -> Dict[int, np.ndarray]:
|
|
14
|
+
"""Partition dataset indices among clients with pathological non-IID label skew.
|
|
15
|
+
|
|
16
|
+
Assigns k distinct classes per client (NIID-Bench #C=k, Li et al. 2021),
|
|
17
|
+
NOT the McMahan et al. 2017 sort-and-shard scheme. Each class's samples are
|
|
18
|
+
split into shards, and clients receive shards corresponding to their assigned
|
|
19
|
+
classes.
|
|
20
|
+
|
|
21
|
+
Parameters
|
|
22
|
+
----------
|
|
23
|
+
labels : Sequence or np.ndarray
|
|
24
|
+
Dataset class labels or targets.
|
|
25
|
+
num_clients : int
|
|
26
|
+
Number of clients to partition data among.
|
|
27
|
+
classes_per_client : int, default=2
|
|
28
|
+
Number of distinct classes assigned to each client.
|
|
29
|
+
seed : int, optional
|
|
30
|
+
Random seed for reproducibility.
|
|
31
|
+
|
|
32
|
+
Returns
|
|
33
|
+
-------
|
|
34
|
+
dict of int to np.ndarray
|
|
35
|
+
Mapping from client ID to array of assigned sample indices.
|
|
36
|
+
|
|
37
|
+
Raises
|
|
38
|
+
------
|
|
39
|
+
ValueError
|
|
40
|
+
If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
|
|
41
|
+
If num_clients <= 0 or classes_per_client <= 0.
|
|
42
|
+
If classes_per_client exceeds the number of unique classes.
|
|
43
|
+
If total client slots is less than the number of unique classes.
|
|
44
|
+
If total samples is less than total class shards.
|
|
45
|
+
If any class has fewer samples than the number of shards assigned to it.
|
|
46
|
+
"""
|
|
47
|
+
labels_arr = validate_labels(labels)
|
|
48
|
+
num_samples = len(labels_arr)
|
|
49
|
+
|
|
50
|
+
if num_clients <= 0:
|
|
51
|
+
raise ValueError(f"num_clients must be positive, got {num_clients}")
|
|
52
|
+
if classes_per_client <= 0:
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"classes_per_client must be positive, got {classes_per_client}"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
unique_classes = np.unique(labels_arr)
|
|
59
|
+
num_classes = len(unique_classes)
|
|
60
|
+
|
|
61
|
+
if classes_per_client > num_classes:
|
|
62
|
+
raise ValueError(
|
|
63
|
+
f"classes_per_client ({classes_per_client}) cannot exceed number of unique classes ({num_classes})"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
total_slots = num_clients * classes_per_client
|
|
67
|
+
if total_slots < num_classes:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"Total client class slots ({num_clients} * {classes_per_client} = {total_slots}) "
|
|
70
|
+
f"must be at least the number of unique classes ({num_classes}) to allocate all samples."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if num_samples < total_slots:
|
|
74
|
+
raise ValueError(
|
|
75
|
+
f"Total samples ({num_samples}) must be at least total class shards ({total_slots})"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
rng = np.random.default_rng(seed)
|
|
79
|
+
|
|
80
|
+
# Distribute class slots across clients
|
|
81
|
+
counts = np.ones(num_classes, dtype=int)
|
|
82
|
+
remaining = total_slots - num_classes
|
|
83
|
+
base = remaining // num_classes
|
|
84
|
+
rem = remaining % num_classes
|
|
85
|
+
counts += base
|
|
86
|
+
counts[:rem] += 1
|
|
87
|
+
rng.shuffle(counts)
|
|
88
|
+
|
|
89
|
+
# Validate that every class has at least as many samples as assigned shards
|
|
90
|
+
for c_idx, cls_val in enumerate(unique_classes):
|
|
91
|
+
class_sample_count = int(np.count_nonzero(labels_arr == cls_val))
|
|
92
|
+
needed_shards = int(counts[c_idx])
|
|
93
|
+
if class_sample_count < needed_shards:
|
|
94
|
+
raise ValueError(
|
|
95
|
+
f"Class '{cls_val}' has {class_sample_count} sample(s), which is fewer than "
|
|
96
|
+
f"the assigned {needed_shards} shard(s). Cannot partition without producing empty shards."
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
client_classes = [set() for _ in range(num_clients)]
|
|
101
|
+
sorted_class_indices = np.argsort(-counts)
|
|
102
|
+
|
|
103
|
+
for c_idx in sorted_class_indices:
|
|
104
|
+
cls_val = unique_classes[c_idx]
|
|
105
|
+
needed = counts[c_idx]
|
|
106
|
+
|
|
107
|
+
candidates = [
|
|
108
|
+
cid
|
|
109
|
+
for cid in range(num_clients)
|
|
110
|
+
if cls_val not in client_classes[cid]
|
|
111
|
+
and len(client_classes[cid]) < classes_per_client
|
|
112
|
+
]
|
|
113
|
+
candidates.sort(key=lambda cid: (len(client_classes[cid]), rng.random()))
|
|
114
|
+
|
|
115
|
+
for cid in candidates[:needed]:
|
|
116
|
+
client_classes[cid].add(cls_val)
|
|
117
|
+
|
|
118
|
+
# Split each class's samples into the assigned number of shards
|
|
119
|
+
class_shards: Dict[object, list] = {}
|
|
120
|
+
for c_idx, cls_val in enumerate(unique_classes):
|
|
121
|
+
cls_indices = np.where(labels_arr == cls_val)[0]
|
|
122
|
+
rng.shuffle(cls_indices)
|
|
123
|
+
num_shards = counts[c_idx]
|
|
124
|
+
class_shards[cls_val] = list(np.array_split(cls_indices, num_shards))
|
|
125
|
+
|
|
126
|
+
# Assign shards to clients
|
|
127
|
+
result: Dict[int, np.ndarray] = {}
|
|
128
|
+
for cid in range(num_clients):
|
|
129
|
+
cid_shards = []
|
|
130
|
+
for cls_val in client_classes[cid]:
|
|
131
|
+
cid_shards.append(class_shards[cls_val].pop())
|
|
132
|
+
|
|
133
|
+
merged = np.concatenate(cid_shards)
|
|
134
|
+
rng.shuffle(merged)
|
|
135
|
+
result[cid] = merged
|
|
136
|
+
|
|
137
|
+
return result
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from typing import Dict, Optional, Sequence, Union
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._validation import validate_labels
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def quantity_skew_partition(
|
|
9
|
+
labels: Union[Sequence, np.ndarray],
|
|
10
|
+
num_clients: int,
|
|
11
|
+
beta: Union[float, Sequence[float]] = 0.5,
|
|
12
|
+
seed: Optional[int] = None,
|
|
13
|
+
) -> Dict[int, np.ndarray]:
|
|
14
|
+
"""Partition dataset indices among clients with quantity skew (varying sample sizes).
|
|
15
|
+
|
|
16
|
+
Simulates realistic volume heterogeneity across clients in Federated Learning.
|
|
17
|
+
The proportion of total samples assigned to each client is drawn from a Dirichlet
|
|
18
|
+
distribution parameterized by beta. Smaller beta values produce severe volume
|
|
19
|
+
disparities, while larger beta values approach uniform sample sizes.
|
|
20
|
+
|
|
21
|
+
Parameters
|
|
22
|
+
----------
|
|
23
|
+
labels : Sequence or np.ndarray
|
|
24
|
+
Dataset labels or target sequence.
|
|
25
|
+
num_clients : int
|
|
26
|
+
Number of clients to partition data among.
|
|
27
|
+
beta : float or Sequence[float], default=0.5
|
|
28
|
+
Concentration parameter for the Dirichlet distribution governing client sample sizes.
|
|
29
|
+
seed : int, optional
|
|
30
|
+
Random seed for reproducibility.
|
|
31
|
+
|
|
32
|
+
Returns
|
|
33
|
+
-------
|
|
34
|
+
dict of int to np.ndarray
|
|
35
|
+
Mapping from client ID to array of assigned sample indices.
|
|
36
|
+
|
|
37
|
+
Raises
|
|
38
|
+
------
|
|
39
|
+
ValueError
|
|
40
|
+
If labels is invalid (empty, not 1D, non-integer, or contains NaN/Inf).
|
|
41
|
+
If num_clients <= 0 or beta <= 0.
|
|
42
|
+
If num_clients exceeds num_samples.
|
|
43
|
+
If beta is a sequence and its length does not match num_clients.
|
|
44
|
+
"""
|
|
45
|
+
labels_arr = validate_labels(labels)
|
|
46
|
+
num_samples = len(labels_arr)
|
|
47
|
+
|
|
48
|
+
if num_clients <= 0:
|
|
49
|
+
raise ValueError(f"num_clients must be positive, got {num_clients}")
|
|
50
|
+
if num_clients > num_samples:
|
|
51
|
+
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"num_clients ({num_clients}) cannot exceed number of samples ({num_samples})"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
if isinstance(beta, (int, float)):
|
|
57
|
+
if beta <= 0:
|
|
58
|
+
raise ValueError(f"beta must be positive, got {beta}")
|
|
59
|
+
beta_vec = np.repeat(float(beta), num_clients)
|
|
60
|
+
else:
|
|
61
|
+
beta_vec = np.asarray(beta, dtype=float)
|
|
62
|
+
if beta_vec.shape != (num_clients,):
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"beta length must match num_clients ({num_clients}), got {len(beta_vec)}"
|
|
65
|
+
)
|
|
66
|
+
if (beta_vec <= 0).any():
|
|
67
|
+
raise ValueError("All beta values must be positive")
|
|
68
|
+
|
|
69
|
+
rng = np.random.default_rng(seed)
|
|
70
|
+
shuffled_indices = rng.permutation(num_samples)
|
|
71
|
+
|
|
72
|
+
# Sample client proportions from Dirichlet distribution
|
|
73
|
+
proportions = rng.dirichlet(beta_vec)
|
|
74
|
+
|
|
75
|
+
# Allocate at least 1 sample to each client, distribute remainder according to proportions
|
|
76
|
+
remaining = num_samples - num_clients
|
|
77
|
+
split_points = (np.cumsum(proportions)[:-1] * remaining).astype(int)
|
|
78
|
+
counts = np.diff(np.concatenate(([0], split_points, [remaining]))) + 1
|
|
79
|
+
|
|
80
|
+
# Split shuffled indices by cumulative sample counts
|
|
81
|
+
index_splits = np.split(shuffled_indices, np.cumsum(counts)[:-1])
|
|
82
|
+
|
|
83
|
+
return {cid: split for cid, split in enumerate(index_splits)}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fl-data-generator
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight, framework-agnostic library for IID and non-IID data partitioning in Federated Learning.
|
|
5
|
+
Author-email: Mostafa Khastkhodaei <dev.khastkhodaei@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/khastkhodaei/fl-data-generator
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/khastkhodaei/fl-data-generator/issues
|
|
9
|
+
Project-URL: Source Code, https://github.com/khastkhodaei/fl-data-generator
|
|
10
|
+
Keywords: federated-learning,data-partitioning,non-iid,dirichlet,label-skew,quantity-skew,machine-learning
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: numpy>=1.20.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: hypothesis>=6.0.0; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff~=0.17.0; extra == "dev"
|
|
32
|
+
Provides-Extra: viz
|
|
33
|
+
Requires-Dist: matplotlib>=3.5.0; extra == "viz"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# fl-data-generator
|
|
37
|
+
|
|
38
|
+
[](https://opensource.org/licenses/MIT)
|
|
39
|
+
[](https://www.python.org/downloads/)
|
|
40
|
+
|
|
41
|
+
A lightweight, framework-agnostic Python library for **IID** and **Non-IID** data partitioning in **Federated Learning (FL)**.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Overview
|
|
46
|
+
|
|
47
|
+
In Federated Learning research and benchmarks, simulating realistic data heterogeneity across distributed clients is essential. **fl-data-generator** simplifies partitioning arbitrary datasets into realistic client distributions without tying you to any specific deep learning framework.
|
|
48
|
+
|
|
49
|
+
Whether you are using PyTorch, TensorFlow, JAX, or federated frameworks like Flower, FedML, or PySyft, `fl-data-generator` provides clean sample indices and dataset wrappers for your clients.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
|
|
55
|
+
- **IID Partitioning**: Uniform random sample allocation across all clients.
|
|
56
|
+
- **Dirichlet Partitioning**: Continuous label distribution skew ($\text{Dir}(\alpha)$) over class proportions, with client minimum size constraints (`min_size`).
|
|
57
|
+
- **Pathological Non-IID Partitioning**: Allocation of a fixed number of classes ($k$ classes per client, following the NIID-Bench benchmark scheme).
|
|
58
|
+
- **Quantity Skew Partitioning**: Volume heterogeneity where client dataset sizes vary according to a Dirichlet distribution ($\text{Dir}(\beta)$).
|
|
59
|
+
- **Hybrid Partitioning (Experimental)**: Sequential joint label skew and quantity skew allocation combining Dirichlet $\alpha$ and $\beta$.
|
|
60
|
+
- **FederatedDataset Wrapper**: High-level wrapper supporting built-in computer vision datasets (`mnist`, `cifar10`, `cifar100`, `fashion_mnist`) as well as custom `(X, y)` tuples and PyTorch `Dataset` instances.
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
### From Source
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
git clone https://github.com/khastkhodaei/fl-data-generator.git
|
|
70
|
+
cd fl-data-generator
|
|
71
|
+
pip install -e .
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### With Optional Extras
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
# Development dependencies (pytest, hypothesis, ruff, etc.)
|
|
78
|
+
pip install -e ".[dev]"
|
|
79
|
+
|
|
80
|
+
# Visualization support (matplotlib)
|
|
81
|
+
pip install -e ".[viz]"
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Runnable Examples
|
|
87
|
+
|
|
88
|
+
### 1. IID Partitioning
|
|
89
|
+
|
|
90
|
+
Uniformly partitions sample indices across clients.
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import numpy as np
|
|
94
|
+
from fl_data_generator import iid_partition
|
|
95
|
+
|
|
96
|
+
labels = np.array([0, 1] * 50) # 100 samples
|
|
97
|
+
client_dict = iid_partition(labels, num_clients=4, seed=42)
|
|
98
|
+
|
|
99
|
+
print("Client sample counts:", [len(indices) for indices in client_dict.values()])
|
|
100
|
+
# Real Output: Client sample counts: [25, 25, 25, 25]
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### 2. Dirichlet Label Skew Partitioning
|
|
104
|
+
|
|
105
|
+
Simulates non-IID label skew parameterized by concentration parameter `alpha`. Smaller $\alpha$ generates severe distribution skew, while larger $\alpha$ approaches uniform class distributions.
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
import numpy as np
|
|
109
|
+
from fl_data_generator import dirichlet_partition
|
|
110
|
+
|
|
111
|
+
labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
|
|
112
|
+
client_dict = dirichlet_partition(labels, num_clients=3, alpha=0.5, seed=42)
|
|
113
|
+
|
|
114
|
+
print("Client sample counts:", [len(indices) for indices in client_dict.values()])
|
|
115
|
+
# Real Output: Client sample counts: [114, 42, 44]
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### 3. Pathological Non-IID Partitioning ($k$ Classes per Client)
|
|
119
|
+
|
|
120
|
+
Partitions data such that each client receives samples from exactly $k$ distinct classes (NIID-Bench #C=k scheme).
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
import numpy as np
|
|
124
|
+
from fl_data_generator import pathological_partition
|
|
125
|
+
|
|
126
|
+
labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
|
|
127
|
+
client_dict = pathological_partition(labels, num_clients=4, classes_per_client=2, seed=42)
|
|
128
|
+
|
|
129
|
+
client_0_labels = labels[client_dict[0]]
|
|
130
|
+
print("Client 0 distinct classes:", np.unique(client_0_labels).tolist())
|
|
131
|
+
# Real Output: Client 0 distinct classes: [1, 2]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### 4. Quantity Skew Partitioning (Volume Heterogeneity)
|
|
135
|
+
|
|
136
|
+
Allocates varying numbers of samples to clients according to a Dirichlet distribution parameterized by concentration parameter `beta`.
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import numpy as np
|
|
140
|
+
from fl_data_generator import quantity_skew_partition
|
|
141
|
+
|
|
142
|
+
labels = np.arange(200) # 200 samples
|
|
143
|
+
client_dict = quantity_skew_partition(labels, num_clients=4, beta=0.5, seed=42)
|
|
144
|
+
|
|
145
|
+
print("Client sample counts:", [len(indices) for indices in client_dict.values()])
|
|
146
|
+
# Real Output: Client sample counts: [52, 72, 16, 60]
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### 5. Hybrid Partitioning (Experimental)
|
|
150
|
+
|
|
151
|
+
Combines quantity skew ($\beta$) and label skew ($\alpha$) via sequential allocation.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import numpy as np
|
|
155
|
+
from fl_data_generator import hybrid_partition
|
|
156
|
+
|
|
157
|
+
labels = np.repeat(np.arange(4), 50) # 200 samples across 4 classes
|
|
158
|
+
client_dict = hybrid_partition(labels, num_clients=4, alpha=0.5, beta=0.5, seed=42)
|
|
159
|
+
|
|
160
|
+
print("Client sample counts:", [len(indices) for indices in client_dict.values()])
|
|
161
|
+
# Real Output: Client sample counts: [56, 89, 2, 53]
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### 6. FederatedDataset Interface
|
|
165
|
+
|
|
166
|
+
High-level interface for creating client-specific datasets and PyTorch DataLoaders.
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
import numpy as np
|
|
170
|
+
from fl_data_generator import FederatedDataset
|
|
171
|
+
|
|
172
|
+
X = np.random.default_rng(42).standard_normal((100, 5))
|
|
173
|
+
y = np.array([0, 1] * 50)
|
|
174
|
+
|
|
175
|
+
fed_data = FederatedDataset((X, y), num_clients=4, partition="iid", seed=42)
|
|
176
|
+
client_0_ds = fed_data.get_client_dataset(0)
|
|
177
|
+
|
|
178
|
+
print(f"Client 0 sample count: {len(client_0_ds)}")
|
|
179
|
+
features, label = client_0_ds[0]
|
|
180
|
+
print(f"Feature shape: {features.shape}, Label: {label}")
|
|
181
|
+
# Real Output:
|
|
182
|
+
# Client 0 sample count: 25
|
|
183
|
+
# Feature shape: (5,), Label: 1
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## Roadmap
|
|
189
|
+
|
|
190
|
+
The following experimental features and partition strategies are planned for future releases:
|
|
191
|
+
- **Feature Skew (Covariate Shift)**: Partitioning based on continuous feature space clusters or synthetic domain perturbations.
|
|
192
|
+
- **Power-Law Quantity Skew**: Heavy-tailed sample volume distribution across clients.
|
|
193
|
+
- **Asymmetric Class Topologies**: Custom directed class sharing graphs.
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## Project Structure
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
fl-data-generator/
|
|
201
|
+
├── .github/
|
|
202
|
+
│ └── workflows/
|
|
203
|
+
│ └── ci.yml # GitHub Actions CI matrix workflow
|
|
204
|
+
├── src/
|
|
205
|
+
│ └── fl_data_generator/ # Core library package
|
|
206
|
+
│ ├── __init__.py # Public API exports
|
|
207
|
+
│ ├── _validation.py # Input validation utilities
|
|
208
|
+
│ ├── dataset.py # FederatedDataset interface & torchvision loaders
|
|
209
|
+
│ ├── dirichlet.py # Dirichlet label skew partitioner
|
|
210
|
+
│ ├── hybrid.py # Hybrid label & quantity skew partitioner
|
|
211
|
+
│ ├── iid.py # Uniform IID partitioner
|
|
212
|
+
│ ├── pathological.py # Pathological k-class partitioner
|
|
213
|
+
│ └── quantity.py # Quantity skew partitioner
|
|
214
|
+
├── tests/ # Unit, integration, and property test suite
|
|
215
|
+
│ ├── test_dataset.py
|
|
216
|
+
│ ├── test_dirichlet.py
|
|
217
|
+
│ ├── test_hybrid.py
|
|
218
|
+
│ ├── test_iid.py
|
|
219
|
+
│ ├── test_pathological.py
|
|
220
|
+
│ ├── test_properties.py
|
|
221
|
+
│ ├── test_quantity.py
|
|
222
|
+
│ ├── test_readme.py
|
|
223
|
+
│ └── test_validation.py
|
|
224
|
+
├── pyproject.toml # Package build configuration & Ruff settings
|
|
225
|
+
├── LICENSE # MIT License
|
|
226
|
+
└── README.md # Project documentation
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## License
|
|
232
|
+
|
|
233
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
fl_data_generator/__init__.py,sha256=m0oFZhpYf6MWug0dqfnTv3EmMCrtHm2ge3t-LxjVr3g,466
|
|
2
|
+
fl_data_generator/_validation.py,sha256=5hkJELUbBskEjHOkz1PN1zodntp3M4eYaGmo7mDavXw,1823
|
|
3
|
+
fl_data_generator/dataset.py,sha256=4jUobhkOoewO_6LW491yH3o1me2C28mevQtjsInBkpg,15910
|
|
4
|
+
fl_data_generator/dirichlet.py,sha256=2dHX2FGQQgtuK03GHOAznPflIwp4Uwg_QubVeQKAQeA,4803
|
|
5
|
+
fl_data_generator/hybrid.py,sha256=Ol3v85V8mOWmL0WbIvk-RCnov2uxTfyns0SJrn_vfGE,5800
|
|
6
|
+
fl_data_generator/iid.py,sha256=f2GFubs8hq_w_CU8it1qI9XXHq4C_af0nReZBx2Jqrk,1475
|
|
7
|
+
fl_data_generator/pathological.py,sha256=rdh5-6SUIKl3Ij_P2c8Ud7aPpvk96onE-QSVPkuCApw,4949
|
|
8
|
+
fl_data_generator/quantity.py,sha256=_VEo9-GnPUHdvBLxB3GwpvO9aE_-f46sMteyBrFV0UY,3149
|
|
9
|
+
fl_data_generator-0.1.0.dist-info/licenses/LICENSE,sha256=hkYdGffXw9mXezr-rwHQYwTTqM1r5rvJfa5esQyyLkA,1077
|
|
10
|
+
fl_data_generator-0.1.0.dist-info/METADATA,sha256=lyy8zTz-KEpJpOMmHVDXRIjyiPm0KcWSLlHyR_24pMw,9225
|
|
11
|
+
fl_data_generator-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
12
|
+
fl_data_generator-0.1.0.dist-info/top_level.txt,sha256=sHZKMdxbAhWu3-VClvEPI1AaLhPsD9SPjt14IlOGByc,18
|
|
13
|
+
fl_data_generator-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mostafa Khastkhodaei
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fl_data_generator
|