cesnet-datazoo 0.1.14__py3-none-any.whl → 0.1.15__py3-none-any.whl

cesnet_datazoo/config.py

@@ -166,7 +166,8 @@ class DatasetConfig():
         val_workers: Number of workers for loading validation data. `0` means that the data will be loaded in the main process. `Default: 1`
         batch_size: Number of samples per batch. `Default: 192`
         test_batch_size: Number of samples per batch for loading validation and test data. `Default: 2048`
-        preload_val: Whether to dump the validation set with `numpy.savez_compressed` and preload it in future runs. Useful when running a lot of experiments with the same dataset configuration. `Default: False`
+        preload_train: Whether to dump the train set with `numpy.savez_compressed` and preload it in future runs. Useful when running a lot of experiments with the same dataset configuration. `Default: False`
+        preload_val: Whether to dump the validation set with `numpy.savez_compressed` and preload it in future runs. `Default: False`
         preload_test: Whether to dump the test set with `numpy.savez_compressed` and preload it in future runs. `Default: False`
         train_size: Size of the train set. See [instructions][config.DatasetConfig--how-to-configure-train-validation-and-test-sets]. `Default: all`
         val_known_size: Size of the validation set. See [instructions][config.DatasetConfig--how-to-configure-train-validation-and-test-sets]. `Default: all`
@@ -244,6 +245,7 @@ class DatasetConfig():
     val_workers: int = 1
     batch_size: int = 192
     test_batch_size: int = 2048
+    preload_train: bool = False
     preload_val: bool = False
     preload_test: bool = False
     train_size: int | Literal["all"] = "all"

cesnet_datazoo/datasets/cesnet_dataset.py

@@ -642,7 +642,9 @@ class CesnetDataset():
                 flowstats_transform=dataset_config.flowstats_transform,
                 flowstats_phist_transform=dataset_config.flowstats_phist_transform,
                 target_transform=label_encoder_fn,
-                return_tensors=dataset_config.return_tensors,)
+                return_tensors=dataset_config.return_tensors,
+                preload=dataset_config.preload_train,
+                preload_blob=os.path.join(dataset_config._get_train_data_path(), "preload", f"train_dataset-{dataset_config.train_size}.npz"),)
         if dataset_config.need_val_set:
             assert val_data_path is not None
             val_dataset = PyTablesDataset(

{cesnet_datazoo-0.1.14.dist-info → cesnet_datazoo-0.1.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cesnet-datazoo
-Version: 0.1.14
+Version: 0.1.15
 Summary: A toolkit for large network traffic datasets
 Author-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
 Maintainer-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>

{cesnet_datazoo-0.1.14.dist-info → cesnet_datazoo-0.1.15.dist-info}/RECORD

@@ -1,8 +1,8 @@
 cesnet_datazoo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cesnet_datazoo/config.py,sha256=AoYMX_drWqb6K6MhdTQRyvUPf_6kZeL3ie04hUgJzgE,38768
+cesnet_datazoo/config.py,sha256=OEVdJQP9p7RYIgHxbXnwKsLZ85xJzlsA1CCoJ8SO4Ps,38933
 cesnet_datazoo/constants.py,sha256=6GhcIyjVnWYrVnxRgTlGuiWRtvwZL1KqyzMJS26ge2E,1481
 cesnet_datazoo/datasets/__init__.py,sha256=8ziQ3EUzUh5fMfWWXwk0cqYk0lOUNU7zbi0Gom3bLnI,443
-cesnet_datazoo/datasets/cesnet_dataset.py,sha256=AgxdiEwtJrOug9F8LJcMwttc7NMrligkq7cy4lRmLAs,47817
+cesnet_datazoo/datasets/cesnet_dataset.py,sha256=zP97SwDHHvpRsFVP8kdOntV9tNkXYX_YpYpy1qLt8Zc,48016
 cesnet_datazoo/datasets/datasets.py,sha256=Bn4SU1k5og6AsUlnPapFPeu4uGlpRH-IaOSafz0ZT2k,3617
 cesnet_datazoo/datasets/datasets_constants.py,sha256=1P54Ns8wCQMemdKNe8OH7cVUfkxs3vL29ugSmOLXceI,29154
 cesnet_datazoo/datasets/loaders.py,sha256=9KgRY-Y8CcgtXbgqWpAaG7gyOAsSf278w7b1eHwTSyE,1854
@@ -23,8 +23,8 @@ cesnet_datazoo/utils/class_info.py,sha256=H5UgyRqXIepBJmkLQ1gAIXV4owKSoIllguRiqF
 cesnet_datazoo/utils/download.py,sha256=hG5V1ZYZGtqCzlVV76NMgOZkSKOywdOFiq9Lagkgego,1441
 cesnet_datazoo/utils/fileutils.py,sha256=XA_VWDuTiCXnoOgHPUzsmbnLFgrlxOo5cvUY_OBJUR8,642
 cesnet_datazoo/utils/random.py,sha256=Dqgm_T25ljbew-OJozK90PsiXKnd4Kw6lcUexxF6vIc,575
-cesnet_datazoo-0.1.14.dist-info/licenses/LICENCE,sha256=69Wc69APiM1YKrFOIipG7jjU2lk89WQuO_U0AXKU8KE,1541
-cesnet_datazoo-0.1.14.dist-info/METADATA,sha256=yp-ld51MKd-oTx-z86x0BwbJEvdY2zlc1fIbfYIDKWY,12574
-cesnet_datazoo-0.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cesnet_datazoo-0.1.14.dist-info/top_level.txt,sha256=bu1Z8zaI_1Id_ZaYyvJnxIBa87OSrdlZ8J2OBMggK5o,15
-cesnet_datazoo-0.1.14.dist-info/RECORD,,
+cesnet_datazoo-0.1.15.dist-info/licenses/LICENCE,sha256=69Wc69APiM1YKrFOIipG7jjU2lk89WQuO_U0AXKU8KE,1541
+cesnet_datazoo-0.1.15.dist-info/METADATA,sha256=iVq2zImXujDGrI8Jca4cVpUY0LqWrBv7oGKVOVTJZ8k,12574
+cesnet_datazoo-0.1.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cesnet_datazoo-0.1.15.dist-info/top_level.txt,sha256=bu1Z8zaI_1Id_ZaYyvJnxIBa87OSrdlZ8J2OBMggK5o,15
+cesnet_datazoo-0.1.15.dist-info/RECORD,,