cesnet-datazoo 0.1.4__tar.gz → 0.1.5__tar.gz

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cesnet-datazoo
-Version: 0.1.4
+Version: 0.1.5
 Summary: A toolkit for large network traffic datasets
 Author-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
 Maintainer-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/cesnet_datazoo/datasets/cesnet_dataset.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 import tables as tb
 import torch
+from numpy.lib.recfunctions import repack_fields
 from sklearn.model_selection import train_test_split
 from sklearn.preprocessing import LabelEncoder
 from torch.utils.data import BatchSampler, DataLoader, RandomSampler, Sampler, SequentialSampler
@@ -17,7 +18,8 @@ from typing_extensions import assert_never
 
 from cesnet_datazoo.config import AppSelection, DataLoaderOrder, DatasetConfig, ValidationApproach
 from cesnet_datazoo.constants import (APP_COLUMN, CATEGORY_COLUMN, DATASET_SIZES, INDICES_APP_FIELD,
-                                      SERVICEMAP_FILE, UNKNOWN_STR_LABEL)
+                                      INDICES_INDEX_FIELD, INDICES_TABLE_FIELD, SERVICEMAP_FILE,
+                                      UNKNOWN_STR_LABEL)
 from cesnet_datazoo.datasets.loaders import collate_fn_simple, create_df_from_dataloader
 from cesnet_datazoo.datasets.metadata.dataset_metadata import DatasetMetadata, load_metadata
 from cesnet_datazoo.datasets.statistics import compute_dataset_statistics
@@ -619,7 +621,7 @@ class CesnetDataset():
             train_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_train_tables_paths(),
-                indices=dataset_indices.train_indices,
+                indices=repack_fields(dataset_indices.train_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,
@@ -638,7 +640,7 @@ class CesnetDataset():
             val_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_val_tables_paths(),
-                indices=dataset_indices.val_known_indices,
+                indices=repack_fields(dataset_indices.val_known_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,
@@ -659,7 +661,7 @@ class CesnetDataset():
             test_dataset = PyTablesDataset(
                 database_path=dataset_config.database_path,
                 tables_paths=dataset_config._get_test_tables_paths(),
-                indices=test_combined_indices,
+                indices=repack_fields(test_combined_indices[[INDICES_TABLE_FIELD, INDICES_INDEX_FIELD]]), # type: ignore
                 tables_app_enum=self._tables_app_enum,
                 tables_cat_enum=self._tables_cat_enum,
                 flowstats_features=dataset_config.flowstats_features,

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/cesnet_datazoo/pytables_data/data_scalers.py

@@ -46,18 +46,20 @@ def fit_scalers(dataset_config: DatasetConfig, train_indices: np.ndarray) -> Non
         data_ppi = data_ppi.transpose(0, 2, 1).reshape(-1, ppi_channels)
         padding_mask = data_ppi[:, DIR_POS] == 0 # Mask of padded packets
         # Fit IPT scaler
-        train_ipt = data_ppi[:, IPT_POS].clip(max=clip_and_scale_ppi_transform.ipt_max, min=clip_and_scale_ppi_transform.ipt_min)
-        train_ipt[padding_mask] = np.nan # NaNs are ignored in sklearn scalers
-        if isinstance(clip_and_scale_ppi_transform.ipt_scaler, MinMaxScaler):
-            # Let zero be the minimum for minmax scaling
-            train_ipt = np.concatenate((train_ipt, [0]))
-        clip_and_scale_ppi_transform.ipt_scaler.fit(train_ipt.reshape(-1, 1))
+        if clip_and_scale_ppi_transform.ipt_scaler:
+            train_ipt = data_ppi[:, IPT_POS].clip(max=clip_and_scale_ppi_transform.ipt_max, min=clip_and_scale_ppi_transform.ipt_min)
+            train_ipt[padding_mask] = np.nan # NaNs are ignored in sklearn scalers
+            if isinstance(clip_and_scale_ppi_transform.ipt_scaler, MinMaxScaler):
+                # Let zero be the minimum for minmax scaling
+                train_ipt = np.concatenate((train_ipt, [0]))
+            clip_and_scale_ppi_transform.ipt_scaler.fit(train_ipt.reshape(-1, 1))
         # Fit packet sizes scaler
-        train_psizes = data_ppi[:, SIZE_POS].clip(max=clip_and_scale_ppi_transform.psizes_max, min=clip_and_scale_ppi_transform.pszies_min)
-        train_psizes[padding_mask] = np.nan
-        if isinstance(clip_and_scale_ppi_transform.psizes_scaler, MinMaxScaler):
-            train_psizes = np.concatenate((train_psizes, [0]))
-        clip_and_scale_ppi_transform.psizes_scaler.fit(train_psizes.reshape(-1, 1))
+        if clip_and_scale_ppi_transform.psizes_scaler:
+            train_psizes = data_ppi[:, SIZE_POS].clip(max=clip_and_scale_ppi_transform.psizes_max, min=clip_and_scale_ppi_transform.pszies_min)
+            train_psizes[padding_mask] = np.nan
+            if isinstance(clip_and_scale_ppi_transform.psizes_scaler, MinMaxScaler):
+                train_psizes = np.concatenate((train_psizes, [0]))
+            clip_and_scale_ppi_transform.psizes_scaler.fit(train_psizes.reshape(-1, 1))
         clip_and_scale_ppi_transform.needs_fitting = False
         json.dump(clip_and_scale_ppi_transform.to_dict(), open(os.path.join(train_data_path, "transforms", "ppi-transform.json"), "w"), indent=4)
 

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/cesnet_datazoo/pytables_data/pytables_dataset.py

@@ -158,11 +158,9 @@ class PyTablesDataset(Dataset):
         for i in range(len(tables)):
             base_labels[i] = tables[i].read(field=APP_COLUMN)
             base_indices[i] = np.nonzero(np.isin(base_labels[i], disabled_apps_ids, invert=True))[0]
-        indices = np.column_stack((
+        indices = np.array(list(zip(
             np.concatenate([[table_id] * len(base_indices[table_id]) for table_id in tables]),
-            np.concatenate(list(base_indices.values())),
-            np.concatenate(list(base_labels.values()))
-        )).astype(np.int32)
+            np.concatenate(list(base_indices.values())))), dtype=[field for field in INDICES_DTYPE if field[0] in [INDICES_INDEX_FIELD, INDICES_TABLE_FIELD]])
         self.indices = indices
         database.close()
 

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/cesnet_datazoo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cesnet-datazoo
-Version: 0.1.4
+Version: 0.1.5
 Summary: A toolkit for large network traffic datasets
 Author-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
 Maintainer-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>

{cesnet_datazoo-0.1.4 → cesnet_datazoo-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cesnet-datazoo"
-version = "0.1.4"
+version = "0.1.5"
 authors = [
   {name = "Jan Luxemburk", email = "luxemburk@cesnet.cz"},
   {name = "Karel Hynek", email = "hynekkar@cesnet.cz"},