cesnet-datazoo 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

cesnet_datazoo/config.py

@@ -8,6 +8,7 @@ import warnings
 from dataclasses import InitVar, field
 from datetime import datetime
 from enum import Enum
+from importlib.metadata import version
 from typing import TYPE_CHECKING, Callable, Literal, Optional
 
 import yaml
@@ -83,6 +84,7 @@ class DataLoaderOrder(Enum):
 
 @dataclass(frozen=True)
 class TrainDataParams():
+    datazoo_version: str
     database_filename: str
     train_period_name: str
     train_tables_paths: list[str]
@@ -97,6 +99,7 @@ class TrainDataParams():
 
 @dataclass(frozen=True)
 class TestDataParams():
+    datazoo_version: str
     database_filename: str
     test_period_name: str
     test_tables_paths: list[str]
@@ -497,14 +500,12 @@ class DatasetConfig():
         return params_hash
 
     def _get_train_data_path(self) -> str:
-        if self.need_train_set:
-            params_hash = self._get_train_data_hash()
-            return os.path.join(self.data_root, "train-data", f"{params_hash}_{self.random_state}", f"fold_{self.fold_id}")
-        else:
-            return os.path.join(self.data_root, "train-data", "default")
+        params_hash = self._get_train_data_hash()
+        return os.path.join(self.data_root, "train-data", f"{params_hash}_{self.random_state}", f"fold_{self.fold_id}")
 
     def _get_train_data_params(self) -> TrainDataParams:
         return TrainDataParams(
+            datazoo_version=version("cesnet_datazoo"),
             database_filename=self.database_filename,
             train_period_name=self.train_period_name,
             train_tables_paths=self._get_train_tables_paths(),
@@ -520,6 +521,7 @@ class DatasetConfig():
     def _get_val_data_params_and_path(self, known_apps: list[str], unknown_apps: list[str]) -> tuple[TestDataParams, str]:
         assert self.val_approach == ValidationApproach.VALIDATION_DATES
         val_data_params = TestDataParams(
+            datazoo_version=version("cesnet_datazoo"),
             database_filename=self.database_filename,
             test_period_name=self.val_period_name,
             test_tables_paths=self._get_val_tables_paths(),
@@ -532,6 +534,7 @@ class DatasetConfig():
 
     def _get_test_data_params_and_path(self, known_apps: list[str], unknown_apps: list[str]) -> tuple[TestDataParams, str]:
         test_data_params = TestDataParams(
+            datazoo_version=version("cesnet_datazoo"),
             database_filename=self.database_filename,
             test_period_name=self.test_period_name,
             test_tables_paths=self._get_test_tables_paths(),

cesnet_datazoo/datasets/cesnet_dataset.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 import tables as tb
 import torch
+from numpy.lib.recfunctions import repack_fields
 from sklearn.model_selection import train_test_split
 from sklearn.preprocessing import LabelEncoder
 from torch.utils.data import BatchSampler, DataLoader, RandomSampler, Sampler, SequentialSampler
@@ -17,7 +18,8 @@ from typing_extensions import assert_never
 
 from cesnet_datazoo.config import AppSelection, DataLoaderOrder, DatasetConfig, ValidationApproach
 from cesnet_datazoo.constants import (APP_COLUMN, CATEGORY_COLUMN, DATASET_SIZES, INDICES_APP_FIELD,
-                                      SERVICEMAP_FILE, UNKNOWN_STR_LABEL)
+                                      INDICES_INDEX_FIELD, INDICES_TABLE_FIELD, SERVICEMAP_FILE,
+                                      UNKNOWN_STR_LABEL)
 from cesnet_datazoo.datasets.loaders import collate_fn_simple, create_df_from_dataloader
 from cesnet_datazoo.datasets.metadata.dataset_metadata import DatasetMetadata, load_metadata
 from cesnet_datazoo.datasets.statistics import compute_dataset_statistics
@@ -619,7 +621,7 @@ class CesnetDataset():
             train_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_train_tables_paths(),
-                indices=dataset_indices.train_indices,
+                indices=repack_fields(dataset_indices.train_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,
@@ -638,7 +640,7 @@ class CesnetDataset():
             val_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_val_tables_paths(),
-                indices=dataset_indices.val_known_indices,
+                indices=repack_fields(dataset_indices.val_known_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,
@@ -659,7 +661,7 @@ class CesnetDataset():
             test_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_test_tables_paths(),
-                indices=test_combined_indices,
+                indices=repack_fields(test_combined_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,

cesnet_datazoo/pytables_data/data_scalers.py

@@ -46,18 +46,20 @@ def fit_scalers(dataset_config: DatasetConfig, train_indices: np.ndarray) -> Non
         data_ppi = data_ppi.transpose(0, 2, 1).reshape(-1, ppi_channels)
         padding_mask = data_ppi[:, DIR_POS] == 0 # Mask of padded packets
         # Fit IPT scaler
-        train_ipt = data_ppi[:, IPT_POS].clip(max=clip_and_scale_ppi_transform.ipt_max, min=clip_and_scale_ppi_transform.ipt_min)
-        train_ipt[padding_mask] = np.nan # NaNs are ignored in sklearn scalers
-        if isinstance(clip_and_scale_ppi_transform.ipt_scaler, MinMaxScaler):
-            # Let zero be the minimum for minmax scaling
-            train_ipt = np.concatenate((train_ipt, [0]))
-        clip_and_scale_ppi_transform.ipt_scaler.fit(train_ipt.reshape(-1, 1))
+        if clip_and_scale_ppi_transform.ipt_scaler:
+            train_ipt = data_ppi[:, IPT_POS].clip(max=clip_and_scale_ppi_transform.ipt_max, min=clip_and_scale_ppi_transform.ipt_min)
+            train_ipt[padding_mask] = np.nan # NaNs are ignored in sklearn scalers
+            if isinstance(clip_and_scale_ppi_transform.ipt_scaler, MinMaxScaler):
+                # Let zero be the minimum for minmax scaling
+                train_ipt = np.concatenate((train_ipt, [0]))
+            clip_and_scale_ppi_transform.ipt_scaler.fit(train_ipt.reshape(-1, 1))
         # Fit packet sizes scaler
-        train_psizes = data_ppi[:, SIZE_POS].clip(max=clip_and_scale_ppi_transform.psizes_max, min=clip_and_scale_ppi_transform.pszies_min)
-        train_psizes[padding_mask] = np.nan
-        if isinstance(clip_and_scale_ppi_transform.psizes_scaler, MinMaxScaler):
-            train_psizes = np.concatenate((train_psizes, [0]))
-        clip_and_scale_ppi_transform.psizes_scaler.fit(train_psizes.reshape(-1, 1))
+        if clip_and_scale_ppi_transform.psizes_scaler:
+            train_psizes = data_ppi[:, SIZE_POS].clip(max=clip_and_scale_ppi_transform.psizes_max, min=clip_and_scale_ppi_transform.pszies_min)
+            train_psizes[padding_mask] = np.nan
+            if isinstance(clip_and_scale_ppi_transform.psizes_scaler, MinMaxScaler):
+                train_psizes = np.concatenate((train_psizes, [0]))
+            clip_and_scale_ppi_transform.psizes_scaler.fit(train_psizes.reshape(-1, 1))
         clip_and_scale_ppi_transform.needs_fitting = False
         json.dump(clip_and_scale_ppi_transform.to_dict(), open(os.path.join(train_data_path, "transforms", "ppi-transform.json"), "w"), indent=4)
 

cesnet_datazoo/pytables_data/pytables_dataset.py

@@ -158,11 +158,9 @@ class PyTablesDataset(Dataset):
         for i in range(len(tables)):
             base_labels[i] = tables[i].read(field=APP_COLUMN)
             base_indices[i] = np.nonzero(np.isin(base_labels[i], disabled_apps_ids, invert=True))[0]
-        indices = np.column_stack((
+        indices = np.array(list(zip(
             np.concatenate([[table_id] * len(base_indices[table_id]) for table_id in tables]),
-            np.concatenate(list(base_indices.values())),
-            np.concatenate(list(base_labels.values()))
-        )).astype(np.int32)
+            np.concatenate(list(base_indices.values())))), dtype=[field for field in INDICES_DTYPE if field[0] in [INDICES_INDEX_FIELD, INDICES_TABLE_FIELD]])
         self.indices = indices
         database.close()
 

{cesnet_datazoo-0.1.4.dist-info → cesnet_datazoo-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cesnet-datazoo
-Version: 0.1.4
+Version: 0.1.6
 Summary: A toolkit for large network traffic datasets
 Author-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
 Maintainer-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>

{cesnet_datazoo-0.1.4.dist-info → cesnet_datazoo-0.1.6.dist-info}/RECORD

@@ -1,8 +1,8 @@
 cesnet_datazoo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cesnet_datazoo/config.py,sha256=x8bugBZmBZ9PNd0D5TNHLPHbvx4ZTCQGwQzXPypenjc,38406
+cesnet_datazoo/config.py,sha256=wkpD_OL3gRXX2t0WDfDMsBD2A3vEdAjSm4yXhzsR8T0,38536
 cesnet_datazoo/constants.py,sha256=6GhcIyjVnWYrVnxRgTlGuiWRtvwZL1KqyzMJS26ge2E,1481
 cesnet_datazoo/datasets/__init__.py,sha256=8ziQ3EUzUh5fMfWWXwk0cqYk0lOUNU7zbi0Gom3bLnI,443
-cesnet_datazoo/datasets/cesnet_dataset.py,sha256=DtC597aRm4n8rlbVknG25yd9rsCqRG94jn7xMxZoC1g,46635
+cesnet_datazoo/datasets/cesnet_dataset.py,sha256=14uKWWSGIkH3GM_BDUSYyCIoOh1L-I4bH0zu0m3DkkQ,46988
 cesnet_datazoo/datasets/datasets.py,sha256=Bn4SU1k5og6AsUlnPapFPeu4uGlpRH-IaOSafz0ZT2k,3617
 cesnet_datazoo/datasets/datasets_constants.py,sha256=1P54Ns8wCQMemdKNe8OH7cVUfkxs3vL29ugSmOLXceI,29154
 cesnet_datazoo/datasets/loaders.py,sha256=9KgRY-Y8CcgtXbgqWpAaG7gyOAsSf278w7b1eHwTSyE,1854
@@ -15,16 +15,16 @@ cesnet_datazoo/metrics/classification_report.py,sha256=kqVW35uEctTiWpMqxhWzOmmDk
 cesnet_datazoo/metrics/provider_metrics.py,sha256=zoX0ps8BzEs3ml70g9dWWeLPflNAKUGYOEFYqdBbNY4,1374
 cesnet_datazoo/pytables_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cesnet_datazoo/pytables_data/apps_split.py,sha256=RjLFomrlBCmnBn08FDw1IzL3PuQf4914yJQzwhiXH_E,1411
-cesnet_datazoo/pytables_data/data_scalers.py,sha256=ednTRVl-sjrFLX6vwzCuPLJDpFuwNWDlJz7msV3yM9M,5083
+cesnet_datazoo/pytables_data/data_scalers.py,sha256=gW75d-DGBokMKNUwM_5A3W3XCZ12WYXefGtpD8xYf1Y,5236
 cesnet_datazoo/pytables_data/indices_setup.py,sha256=M5J2BevkQK8fuC22vUauKyKAEVwYg8xRz9JJK8E1VX8,13717
-cesnet_datazoo/pytables_data/pytables_dataset.py,sha256=NkN0PKKUIiAhrGFM9OCR0s48TH66JLzZLiaIOE6d7AE,19413
+cesnet_datazoo/pytables_data/pytables_dataset.py,sha256=YGbzYKrSklCu3J52Xbdcs3zZsYroBBtP8ulgS1c5Fnw,19431
 cesnet_datazoo/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cesnet_datazoo/utils/class_info.py,sha256=H5UgyRqXIepBJmkLQ1gAIXV4owKSoIllguRiqFTu5XU,2462
 cesnet_datazoo/utils/download.py,sha256=hG5V1ZYZGtqCzlVV76NMgOZkSKOywdOFiq9Lagkgego,1441
 cesnet_datazoo/utils/fileutils.py,sha256=XA_VWDuTiCXnoOgHPUzsmbnLFgrlxOo5cvUY_OBJUR8,642
 cesnet_datazoo/utils/random.py,sha256=Dqgm_T25ljbew-OJozK90PsiXKnd4Kw6lcUexxF6vIc,575
-cesnet_datazoo-0.1.4.dist-info/LICENCE,sha256=69Wc69APiM1YKrFOIipG7jjU2lk89WQuO_U0AXKU8KE,1541
-cesnet_datazoo-0.1.4.dist-info/METADATA,sha256=8HeN2mch9VTCpeBr66ZgwrdJyrS53y4INsiU0Rhgcts,12964
-cesnet_datazoo-0.1.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cesnet_datazoo-0.1.4.dist-info/top_level.txt,sha256=bu1Z8zaI_1Id_ZaYyvJnxIBa87OSrdlZ8J2OBMggK5o,15
-cesnet_datazoo-0.1.4.dist-info/RECORD,,
+cesnet_datazoo-0.1.6.dist-info/LICENCE,sha256=69Wc69APiM1YKrFOIipG7jjU2lk89WQuO_U0AXKU8KE,1541
+cesnet_datazoo-0.1.6.dist-info/METADATA,sha256=UDz2g74cJrx_NLbCyPvvLEwE02x4HI8-qhDMgutKb8Q,12964
+cesnet_datazoo-0.1.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cesnet_datazoo-0.1.6.dist-info/top_level.txt,sha256=bu1Z8zaI_1Id_ZaYyvJnxIBa87OSrdlZ8J2OBMggK5o,15
+cesnet_datazoo-0.1.6.dist-info/RECORD,,