cesnet-datazoo 0.0.17__tar.gz → 0.1.0__tar.gz

{cesnet-datazoo-0.0.17 → cesnet-datazoo-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cesnet-datazoo
-Version: 0.0.17
+Version: 0.1.0
 Summary: A toolkit for large network traffic datasets
 Author-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
 Maintainer-email: Jan Luxemburk <luxemburk@cesnet.cz>, Karel Hynek <hynekkar@cesnet.cz>
@@ -16,6 +16,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENCE
+Requires-Dist: cesnet_models
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas

{cesnet-datazoo-0.0.17 → cesnet-datazoo-0.1.0}/cesnet_datazoo/config.py

@@ -8,12 +8,11 @@ import warnings
 from dataclasses import InitVar, field
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Literal, Optional
+from typing import TYPE_CHECKING, Callable, Literal, Optional
 
 import yaml
 from pydantic import model_validator
 from pydantic.dataclasses import dataclass
-from sklearn.preprocessing import MinMaxScaler, RobustScaler, StandardScaler
 
 from cesnet_datazoo.constants import (PHIST_BIN_COUNT, PPI_MAX_LEN, SELECTED_TCP_FLAGS,
                                       TCP_PPI_CHANNELS, UDP_PPI_CHANNELS)
@@ -21,19 +20,6 @@ from cesnet_datazoo.constants import (PHIST_BIN_COUNT, PPI_MAX_LEN, SELECTED_TCP
 if TYPE_CHECKING:
     from cesnet_datazoo.datasets.cesnet_dataset import CesnetDataset
 
-Scaler = RobustScaler | StandardScaler | MinMaxScaler | None
-
-class ScalerEnum(Enum):
-    """Available scalers for flow statistics, packet sizes, and inter-packet times."""
-    STANDARD = "standard"
-    """Standardize features by removing the mean and scaling to unit variance - [`sklearn.preprocessing.StandardScaler`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html)."""
-    ROBUST = "robust"
-    """Robust scaling with the median and the interquartile range - [`sklearn.preprocessing.RobustScaler`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html)."""
-    MINMAX = "minmax"
-    """Scaling to a (0, 1) range - [`sklearn.preprocessing.MinMaxScaler`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html)."""
-    NO_SCALER = "no-scaler"
-    """No scaling."""
-    def __str__(self): return self.value
 
 class Protocol(Enum):
     TLS = "TLS"
@@ -48,25 +34,23 @@ class ValidationApproach(Enum):
     is used to create a random stratified validation set. The fraction of validation samples is defined in `train_val_split_fraction`."""
     VALIDATION_DATES = "validation-dates"
     """Use separate validation dates to create a validation set. Validation dates need to be specified in `val_dates`, and the name of the validation period in `val_period_name`."""
-    NO_VALIDATION = "no-validation"
-    """Do not use validation. The validation dataloader and dataframe will not be available."""
     def __str__(self): return self.value
 
 class AppSelection(Enum):
     """
     Applications can be divided into *known* and *unknown* classes. To use a dataset in the standard closed-world setting, use `ALL_KNOWN ` to select all the applications as *known*.
-    Use `TOPX_KNOWN` or `EXPLICIT_UNKNOWN` for the open-world setting and evaluation of out-of-distribution or open-set recognition methods.
-    The `LONGTERM_FIXED` is for long-term measurements when it is desired to use the same applications for multiple subsequent train and test periods.
+    Use `TOPX_KNOWN` or `BACKGROUND_UNKNOWN` for the open-world setting and evaluation of out-of-distribution or open-set recognition methods.
+    The `FIXED` is for manual selection of *known* and *unknown* applications.
     """
     ALL_KNOWN = "all-known"
     """Use all applications as *known*."""
     TOPX_KNOWN = "topx-known"
     """Use the first X (`apps_selection_topx`) most frequent (with the most samples) applications as *known*, and the rest as *unknown*.
     Applications with the same provider are never separated, i.e., all applications of a given provider are either *known* or *unknown*."""
-    EXPLICIT_UNKNOWN = "explicit-unknown"
-    """Use the provided list of applications (`apps_selection_explicit_unknown`) as *unknown*, and the rest as *known*."""
-    LONGTERM_FIXED = "longterm-fixed"
-    """Use fixed application selection. Provide a tuple of `(known_apps_database_enum, unknown_apps_database_enum)` in `apps_selection_fixed_longterm`."""
+    BACKGROUND_UNKNOWN = "background-unknown"
+    """Use the list of background traffic classes (`apps_selection_background_unknown`) as *unknown*, and the rest as *known*."""
+    FIXED = "fixed"
+    """Manual application selection. Provide lists of *known* applications (`apps_selection_fixed_known`) and *unknown* applications (`apps_selection_fixed_unknown`)."""
     def __str__(self): return self.value
 
 class MinTrainSamplesCheck(Enum):
@@ -103,8 +87,9 @@ class TrainDataParams():
     train_tables_paths: list[str]
     apps_selection: AppSelection
     apps_selection_topx: int
-    apps_selection_explicit_unknown: list[str]
-    apps_selection_fixed_longterm: Optional[tuple[dict[int, str], dict[int, str]]]
+    apps_selection_background_unknown: list[str]
+    apps_selection_fixed_known: list[str]
+    apps_selection_fixed_unknown: list[str]
     disabled_apps: list[str]
     min_train_samples_check: MinTrainSamplesCheck
     min_train_samples_per_app: int
@@ -114,8 +99,8 @@ class TestDataParams():
     database_filename: str
     test_period_name: str
     test_tables_paths: list[str]
-    known_apps_database_enum: dict[int, str]
-    unknown_apps_database_enum: dict[int, str]
+    known_apps: list[str]
+    unknown_apps: list[str]
 
 class C:
     arbitrary_types_allowed = True
@@ -128,38 +113,43 @@ class DatasetConfig():
 
     - Train, validation, test sets (dates, sizes, validation approach).
     - Application selection — either the standard closed-world setting (only *known* classes) or the open-world setting (*known* and *unknown* classes).
-    - Feature scaling. See the [data features][features] page for more information.
+    - Feature scaling. See the [data features][features] page for more information. DOCS_TODO
     - Dataloader options like batch sizes, order of loading, or number of workers.
 
     When initializing this class, pass a [`CesnetDataset`][datasets.cesnet_dataset.CesnetDataset] instance to be configured and the desired configuration. Available options are [here][config.DatasetConfig--configuration-options].
 
     Attributes:
-        dataset: The dataset instance to be configured
-        data_root: Taken from the dataset instance
-        database_filename: Taken from the dataset instance
-        database_path: Taken from the dataset instance
-        servicemap_path: Taken from the dataset instance
-        flowstats_features: Taken from `dataset.metadata.flowstats_features`
-        other_fields: Taken from `dataset.metadata.other_fields` if `return_other_fields` is true, otherwise an empty list
+        dataset: The dataset instance to be configured.
+        data_root: Taken from the dataset instance.
+        database_filename: Taken from the dataset instance.
+        database_path: Taken from the dataset instance.
+        servicemap_path: Taken from the dataset instance.
+        flowstats_features: Taken from `dataset.metadata.flowstats_features`.
+        flowstats_features_boolean: Taken from `dataset.metadata.flowstats_features_boolean`.
+        flowstats_features_phist: Taken from `dataset.metadata.packet_histograms` if `use_packet_histograms` is true, otherwise an empty list.
+        other_fields: Taken from `dataset.metadata.other_fields` if `return_other_fields` is true, otherwise an empty list.
 
     # Configuration options
 
     Attributes:
+        need_train_set: Use to disable the train set. `Default: True`
+        need_val_set: Use to disable the validation set. When `need_train_set` is false, the validation set will also be disabled. `Default: True`
+        need_test_set: Use to disable the test set. `Default: True`
         train_period_name: Name of the train period. See [instructions][config.DatasetConfig--how-to-configure-train-validation-and-test-sets].
         train_dates: Dates used for creating a train set.
         train_dates_weigths: To use a non-uniform distribution of samples across train dates.
-        val_approach: How a validation set should be created. Either split train data into train and validation, have a separate validation period, or no validation at all. `Default: SPLIT_FROM_TRAIN`
+        val_approach: How a validation set should be created. Either split train data into train and validation or have a separate validation period. `Default: SPLIT_FROM_TRAIN`
         train_val_split_fraction: The fraction of validation samples when splitting from the train set. `Default: 0.2`
         val_period_name: Name of the validation period. See [instructions][config.DatasetConfig--how-to-configure-train-validation-and-test-sets].
         val_dates: Dates used for creating a validation set.
-        no_test_set: Disable the test set. `Default: False`
         test_period_name: Name of the test period. See [instructions][config.DatasetConfig--how-to-configure-train-validation-and-test-sets].
         test_dates: Dates used for creating a test set.
 
         apps_selection: How to select application classes. `Default: ALL_KNOWN`
         apps_selection_topx: Take top X as known.
-        apps_selection_explicit_unknown: Provide a list of unknown applications.
-        apps_selection_fixed_longterm: Provide enums of known and unknown applications. This is suitable for long-term measurements.
+        apps_selection_background_unknown: Provide a list of background traffic classes to be used as unknown.
+        apps_selection_fixed_known: Provide a list of manually selected known applications.
+        apps_selection_fixed_unknown: Provide a list of manually selected unknown applications.
         disabled_apps: List of applications to be disabled and not used at all.
         min_train_samples_check: How to handle applications with *not enough* training samples. `Default: DISABLE_APPS`
         min_train_samples_per_app: Defines the threshold for *not enough*. `Default: 100`
@@ -182,22 +172,14 @@ class DatasetConfig():
         train_dataloader_seed: Seed for loading train data in random order. `Default: None`
 
         return_other_fields: Whether to return [auxiliary fields][other-fields], such as communicating hosts, flow times, and more fields extracted from the ClientHello message. `Default: False`
-        return_torch: Use for returning `torch.Tensor` from dataloaders. Dataframes are not available when this option is used. `Default: False`
-        raw_output: Return raw output without data scaling, clipping, and normalization. `Default: False`
+        return_tensors: Use for returning `torch.Tensor` from dataloaders. Dataframes are not available when this option is used. `Default: False`
         use_packet_histograms: Whether to use packet histogram features, if available in the dataset. `Default: True`
-        normalize_packet_histograms: Whether to normalize packet histograms. If true, bins contain fractions instead of absolute numbers. `Default: True`
         use_tcp_features: Whether to use TCP features, if available in the dataset. `Default: True`
         use_push_flags: Whether to use push flags in packet sequences, if available in the dataset. `Default: False`
-        zero_ppi_start: Zeroing out the first N packets of each packet sequence. `Default: 0`
-        fit_scalers_samples: Fraction of train samples used for fitting feature scalers, if float. The absolute number of samples otherwise. `Default: 0.25`
-        flowstats_scaler: Which scaler to use for flow statistics. Options are [`ROBUST`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html) | [`STANDARD`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html) | [`MINMAX`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) | `NO_SCALER`. `Default: ROBUST`
-        psizes_scaler: Which scaler to use for packet sizes. Options are [`ROBUST`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html) | [`STANDARD`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html) | [`MINMAX`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) | `NO_SCALER`. `Default: STANDARD`
-        ipt_scaler: Which scaler to use for inter-packet times. Options are [`ROBUST`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.RobustScaler.html) | [`STANDARD`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html) | [`MINMAX`](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html) | `NO_SCALER`. `Default: STANDARD`
-        scalers_attrs: Load data scalers from numeric values in this dict rather than from pickled files. `Default: None`
-        flowstats_clip: Quantile clip before the scaling of flow statistics. Should limit the influence of outliers. Set to `1` to disable. `Default: 0.99`
-        psizes_max: Max clip packet sizes before scaling. `Default: 1500`
-        ipt_min: Min clip inter-packet times before scaling. `Default: 0`
-        ipt_max: Max clip inter-packet times before scaling. `Default: 65000`
+        fit_scalers_samples: Fraction of train samples used for fitting feature scalers, if float. The absolute number of samples otherwise. `Default: 0.25` DOCS_TODO
+        ppi_transform: Transform function for PPI sequences. `Default: None` DOCS_TODO
+        flowstats_transform: Transform function for flow statistics. `Default: None`
+        flowstats_phist_transform: Transform function for packet histograms. `Default: None`
 
     # How to configure train, validation, and test sets
     There are three options for how to define train/validation/test dates.
@@ -223,8 +205,13 @@ class DatasetConfig():
     database_path: str =  field(init=False)
     servicemap_path: str = field(init=False)
     flowstats_features: list[str] = field(init=False)
+    flowstats_features_boolean: list[str] = field(init=False)
+    flowstats_features_phist: list[str] = field(init=False)
     other_fields: list[str] = field(init=False)
 
+    need_train_set: bool = True
+    need_val_set: bool = True
+    need_test_set: bool = True
     train_period_name: str = ""
     train_dates: list[str] = field(default_factory=list)
     train_dates_weigths: Optional[list[int]] = None
@@ -232,14 +219,14 @@ class DatasetConfig():
     train_val_split_fraction: float = 0.2
     val_period_name: str = ""
     val_dates: list[str] = field(default_factory=list)
-    no_test_set: bool = False
     test_period_name: str = ""
     test_dates: list[str] = field(default_factory=list)
 
     apps_selection: AppSelection = AppSelection.ALL_KNOWN
     apps_selection_topx: int = 0
-    apps_selection_explicit_unknown: list[str] = field(default_factory=list)
-    apps_selection_fixed_longterm: Optional[tuple[dict[int, str], dict[int, str]]] = None
+    apps_selection_background_unknown: list[str] = field(default_factory=list)
+    apps_selection_fixed_known: list[str] = field(default_factory=list)
+    apps_selection_fixed_unknown: list[str] = field(default_factory=list)
     disabled_apps: list[str] = field(default_factory=list)
     min_train_samples_check: MinTrainSamplesCheck = MinTrainSamplesCheck.DISABLE_APPS
     min_train_samples_per_app: int = 100
@@ -262,22 +249,14 @@ class DatasetConfig():
     train_dataloader_seed: Optional[int] = None
 
     return_other_fields: bool = False
-    return_torch: bool = False
-    raw_output: bool = False
-    use_packet_histograms: bool = True
-    normalize_packet_histograms: bool = True
-    use_tcp_features: bool = True
+    return_tensors: bool = False
+    use_packet_histograms: bool = False
+    use_tcp_features: bool = False
     use_push_flags: bool = False
-    zero_ppi_start: int = 0
     fit_scalers_samples: int | float = 0.25
-    flowstats_scaler: ScalerEnum = ScalerEnum.ROBUST
-    psizes_scaler: ScalerEnum = ScalerEnum.STANDARD
-    ipt_scaler: ScalerEnum = ScalerEnum.STANDARD
-    scalers_attrs: Optional[dict] = None
-    flowstats_clip: float = 0.99
-    psizes_max: int = 1500
-    ipt_min: int = 0
-    ipt_max: int = 65000
+    ppi_transform: Optional[Callable] = None
+    flowstats_transform: Optional[Callable] = None
+    flowstats_phist_transform: Optional[Callable] = None
 
     def __post_init__(self, dataset: CesnetDataset):
         """
@@ -287,23 +266,28 @@ class DatasetConfig():
         self.servicemap_path = dataset.servicemap_path
         self.database_filename = dataset.database_filename
         self.database_path = dataset.database_path
-        self.flowstats_features = dataset.metadata.flowstats_features
-        self.other_fields = dataset.metadata.other_fields if self.return_other_fields else []
 
-        # Configure train dates
-        if len(self.train_dates) > 0 and self.train_period_name == "":
-            raise ValueError("train_period_name has to be specified when train_dates are set")
-        if len(self.train_dates) == 0 and self.train_period_name != "":
-            if self.train_period_name not in dataset.time_periods:
-                raise ValueError(f"Unknown train_period_name {self.train_period_name}. Use time period available in dataset.time_periods")
-            self.train_dates = dataset.time_periods[self.train_period_name]
-        if len(self.train_dates) == 0 and self.test_period_name == "":
-            self.train_period_name = dataset.default_train_period_name
-            self.train_dates = dataset.time_periods[dataset.default_train_period_name]
+        if not self.need_train_set:
+            self.need_val_set = False
+            if self.apps_selection != AppSelection.FIXED:
+                raise ValueError("Application selection has to be fixed when need_train_set is false")
+            if (len(self.train_dates) > 0 or self.train_period_name != ""):
+                raise ValueError("train_dates and train_period_name cannot be specified when need_train_set is false")
+        else:
+            # Configure train dates
+            if len(self.train_dates) > 0 and self.train_period_name == "":
+                raise ValueError("train_period_name has to be specified when train_dates are set")
+            if len(self.train_dates) == 0 and self.train_period_name != "":
+                if self.train_period_name not in dataset.time_periods:
+                    raise ValueError(f"Unknown train_period_name {self.train_period_name}. Use time period available in dataset.time_periods")
+                self.train_dates = dataset.time_periods[self.train_period_name]
+            if len(self.train_dates) == 0 and self.test_period_name == "":
+                self.train_period_name = dataset.default_train_period_name
+                self.train_dates = dataset.time_periods[dataset.default_train_period_name]
         # Configure test dates
-        if self.no_test_set:
+        if not self.need_test_set:
             if (len(self.test_dates) > 0 or self.test_period_name != ""):
-                raise ValueError("test_dates and test_period_name cannot be specified when no_test_set is true")
+                raise ValueError("test_dates and test_period_name cannot be specified when need_test_set is false")
         else:
             if len(self.test_dates) > 0 and self.test_period_name == "":
                 raise ValueError("test_period_name has to be specified when test_dates are set")
@@ -315,8 +299,8 @@ class DatasetConfig():
                 self.test_period_name = dataset.default_test_period_name
                 self.test_dates = dataset.time_periods[dataset.default_test_period_name]
         # Configure val dates
-        if (self.val_approach == ValidationApproach.NO_VALIDATION or self.val_approach == ValidationApproach.SPLIT_FROM_TRAIN) and (len(self.val_dates) > 0 or self.val_period_name != ""):
-            raise ValueError("val_dates and val_period_name cannot be specified when val_approach is no-validation or split-from-train")
+        if (not self.need_val_set or self.val_approach == ValidationApproach.SPLIT_FROM_TRAIN) and (len(self.val_dates) > 0 or self.val_period_name != ""):
+            raise ValueError("val_dates and val_period_name cannot be specified when need_val_set is false or the validation approach is split-from-train")
         if self.val_approach == ValidationApproach.VALIDATION_DATES:
             if len(self.val_dates) > 0 and self.val_period_name == "":
                 raise ValueError("val_period_name has to be specified when val_dates are set")
@@ -325,57 +309,58 @@ class DatasetConfig():
                     raise ValueError(f"Unknown val_period_name {self.val_period_name}. Use time period available in dataset.time_periods")
                 self.val_dates = dataset.time_periods[self.val_period_name]
             if len(self.val_dates) == 0 and self.val_period_name == "":
-                raise ValueError("val_period_name and val_dates (or val_period_name from dataset.time_periods) have to be specified when val_approach is validation-dates")
+                raise ValueError("val_period_name and val_dates (or val_period_name from dataset.time_periods) have to be specified when the validation approach is validation-dates")
         # Check if train, val, and test dates are available in the dataset
-        if dataset.available_dates:
-            unknown_train_dates = [t for t in self.train_dates if t not in dataset.available_dates]
-            unknown_val_dates = [t for t in self.val_dates if t not in dataset.available_dates]
-            unknown_test_dates = [t for t in self.test_dates if t not in dataset.available_dates]
-            if len(unknown_train_dates) > 0:
-                raise ValueError(f"Unknown train dates {unknown_train_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
-                                + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
-            if len(unknown_val_dates) > 0:
-                raise ValueError(f"Unknown validation dates {unknown_val_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
-                                + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
-            if len(unknown_test_dates) > 0:
-                raise ValueError(f"Unknown test dates {unknown_test_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
-                                + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
+        bad_train_dates = [t for t in self.train_dates if t not in dataset.available_dates]
+        bad_val_dates = [t for t in self.val_dates if t not in dataset.available_dates]
+        bad_test_dates = [t for t in self.test_dates if t not in dataset.available_dates]
+        if len(bad_train_dates) > 0:
+            raise ValueError(f"Bad train dates {bad_train_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
+                            + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
+        if len(bad_val_dates) > 0:
+            raise ValueError(f"Bad validation dates {bad_val_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
+                            + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
+        if len(bad_test_dates) > 0:
+            raise ValueError(f"Bad test dates {bad_test_dates}. Use dates available in dataset.available_dates (collection period {dataset.metadata.collection_period})" \
+                            + (f". These dates are missing from the dataset collection period {dataset.metadata.missing_dates_in_collection_period}" if dataset.metadata.missing_dates_in_collection_period else ""))
         # Check time order of train, val, and test periods
         train_dates = [datetime.strptime(date_str, "%Y%m%d").date() for date_str in self.train_dates]
         test_dates = [datetime.strptime(date_str, "%Y%m%d").date() for date_str in self.test_dates]
-        if not self.no_test_set and min(test_dates) <= max(train_dates):
+        if len(train_dates) > 0 and len(test_dates) > 0  and min(test_dates) <= max(train_dates):
             warnings.warn(f"Some test dates ({min(test_dates).strftime('%Y%m%d')}) are before or equal to the last train date ({max(train_dates).strftime('%Y%m%d')}). This might lead to improper evaluation and should be avoided.")
         if self.val_approach == ValidationApproach.VALIDATION_DATES:
+            # Train dates are guaranteed to be set
             val_dates = [datetime.strptime(date_str, "%Y%m%d").date() for date_str in self.val_dates]
             if min(val_dates) <= max(train_dates):
                 warnings.warn(f"Some validation dates ({min(val_dates).strftime('%Y%m%d')}) are before or equal to the last train date ({max(train_dates).strftime('%Y%m%d')}). This might lead to improper evaluation and should be avoided.")
-            if not self.no_test_set and min(test_dates) <= max(val_dates):
+            if len(test_dates) > 0 and min(test_dates) <= max(val_dates):
                 warnings.warn(f"Some test dates ({min(test_dates).strftime('%Y%m%d')}) are before or equal to the last validation date ({max(val_dates).strftime('%Y%m%d')}). This might lead to improper evaluation and should be avoided.")
         # Configure features
-        if self.raw_output:
-            self.normalize_packet_histograms = False
-            self.flowstats_scaler = ScalerEnum.NO_SCALER
-            self.flowstats_clip = 1.0
-            self.psizes_scaler = ScalerEnum.NO_SCALER
-            self.psizes_max = 1500
-            self.ipt_scaler = ScalerEnum.NO_SCALER
-            self.ipt_min = 0
-            self.ipt_max = 65000
-        if dataset.metadata.protocol == Protocol.TLS and self.use_tcp_features:
-            self.flowstats_features = self.flowstats_features + SELECTED_TCP_FLAGS
-            if self.use_push_flags and "PUSH_FLAG" not in dataset.metadata.features_in_packet_sequences:
-                raise ValueError("This TLS dataset does not support use_push_flags")
+        self.flowstats_features = dataset.metadata.flowstats_features
+        self.flowstats_features_boolean = dataset.metadata.flowstats_features_boolean
+        self.other_fields = dataset.metadata.other_fields if self.return_other_fields else []
         if self.use_packet_histograms:
-            if len(dataset.metadata.packet_histogram_features) > 0:
-                self.flowstats_features = self.flowstats_features + dataset.metadata.packet_histogram_features
-            else:
-                self.use_packet_histograms = False
+            if len(dataset.metadata.packet_histograms) == 0:
+                raise ValueError("This dataset does not support use_packet_histograms")
+            self.flowstats_features_phist = dataset.metadata.packet_histograms
+        else:
+            self.flowstats_features_phist = []
+            if self.flowstats_phist_transform is not None:
+                raise ValueError("flowstats_phist_transform cannot be specified when use_packet_histograms is false")
+        if dataset.metadata.protocol == Protocol.TLS:
+            if self.use_tcp_features:
+                self.flowstats_features_boolean = self.flowstats_features_boolean + SELECTED_TCP_FLAGS
+            if self.use_push_flags and "PUSH_FLAG" not in dataset.metadata.ppi_features:
+                raise ValueError("This TLS dataset does not support use_push_flags")
         if dataset.metadata.protocol == Protocol.QUIC:
-            self.use_tcp_features = False
+            if self.use_tcp_features:
+                raise ValueError("QUIC datasets do not support use_tcp_features")
             if self.use_push_flags:
                 raise ValueError("QUIC datasets do not support use_push_flags")
         # When train_dates_weigths are used, train_size and val_known_size have to be specified
         if self.train_dates_weigths is not None:
+            if not self.need_train_set:
+                raise ValueError("train_dates_weigths cannot be specified when need_train_set is false")
             if len(self.train_dates_weigths) != len(self.train_dates):
                 raise ValueError("train_dates_weigths has to have the same length as train_dates")
             if self.train_size == "all":
@@ -386,59 +371,75 @@ class DatasetConfig():
         if self.apps_selection == AppSelection.ALL_KNOWN:
             self.val_unknown_size = 0
             self.test_unknown_size = 0
-            if self.apps_selection_topx != 0 or len(self.apps_selection_explicit_unknown) > 0 or self.apps_selection_fixed_longterm is not None:
-                raise ValueError("apps_selection_topx, apps_selection_explicit_unknown, and apps_selection_fixed_longterm cannot be specified when apps_selection is all-known")
-        if self.apps_selection == AppSelection.TOPX_KNOWN and self.apps_selection_topx == 0:
-            raise ValueError("apps_selection_topx has to be greater than 0 when apps_selection is top-x-known")
-        if self.apps_selection == AppSelection.EXPLICIT_UNKNOWN and len(self.apps_selection_explicit_unknown) == 0:
-            raise ValueError("apps_selection_explicit_unknown has to be specified when apps_selection is explicit-unknown")
-        if self.apps_selection == AppSelection.LONGTERM_FIXED:
-            if self.apps_selection_fixed_longterm is None:
-                raise ValueError("apps_selection_fixed_longterm, a tuple of (known_apps_database_enum, unknown_apps_database_enum), has to be specified when apps_selection is longterm-fixed")
+            if self.apps_selection_topx != 0 or len(self.apps_selection_background_unknown) > 0 or len(self.apps_selection_fixed_known) > 0 or len(self.apps_selection_fixed_unknown) > 0:
+                raise ValueError("apps_selection_topx, apps_selection_background_unknown, apps_selection_fixed_known, and apps_selection_fixed_unknown cannot be specified when application selection is all-known")
+        if self.apps_selection == AppSelection.TOPX_KNOWN:
+            if self.apps_selection_topx == 0:
+                raise ValueError("apps_selection_topx has to be greater than 0 when application selection is top-x-known")
+            if len(self.apps_selection_background_unknown) > 0 or len(self.apps_selection_fixed_known) > 0 or len(self.apps_selection_fixed_unknown) > 0:
+                raise ValueError("apps_selection_background_unknown, apps_selection_fixed_known, and apps_selection_fixed_unknown cannot be specified when application selection is top-x-known")
+        if self.apps_selection == AppSelection.BACKGROUND_UNKNOWN:
+            if len(self.apps_selection_background_unknown) == 0:
+                raise ValueError("apps_selection_background_unknown has to be specified when application selection is background-unknown")
+            bad_apps = [a for a in self.apps_selection_background_unknown if a not in dataset.available_classes]
+            if len(bad_apps) > 0:
+                raise ValueError(f"Bad applications in apps_selection_background_unknown {bad_apps}. Use applications available in dataset.available_classes")
+            if self.apps_selection_topx != 0 or len(self.apps_selection_fixed_known) > 0 or len(self.apps_selection_fixed_unknown) > 0:
+                raise ValueError("apps_selection_topx, apps_selection_fixed_known, and apps_selection_fixed_unknown cannot be specified when application selection is background-unknown")
+        if self.apps_selection == AppSelection.FIXED:
+            if len(self.apps_selection_fixed_known) == 0:
+                raise ValueError("apps_selection_fixed_known has to be specified when application selection is fixed")
+            bad_apps = [a for a in self.apps_selection_fixed_known + self.apps_selection_fixed_unknown if a not in dataset.available_classes]
+            if len(bad_apps) > 0:
+                raise ValueError(f"Bad applications in apps_selection_fixed_known or apps_selection_fixed_unknown {bad_apps}. Use applications available in dataset.available_classes")
             if len(self.disabled_apps) > 0:
-                raise ValueError("disabled_apps cannot be specified when apps_selection is longterm-fixed")
-            if self.min_train_samples_per_app != 0:
-                raise ValueError("min_train_samples_per_app has to be 0 when apps_selection is longterm-fixed")
-        if sum((self.apps_selection_topx != 0, len(self.apps_selection_explicit_unknown) > 0, self.apps_selection_fixed_longterm is not None)) > 1:
-            raise ValueError("apps_selection_topx, apps_selection_explicit_unknown, and apps_selection_fixed_longterm should not be specified at the same time")
+                raise ValueError("disabled_apps cannot be specified when application selection is fixed")
+            if self.min_train_samples_per_app != 0 and self.min_train_samples_per_app != 100:
+                warnings.warn("min_train_samples_per_app is not used when application selection is fixed")
+            if self.apps_selection_topx != 0 or len(self.apps_selection_background_unknown) > 0:
+                raise ValueError("apps_selection_topx and apps_selection_background_unknown cannot be specified when application selection is fixed")
         # More asserts
-        if self.zero_ppi_start > PPI_MAX_LEN:
-            raise ValueError(f"zero_ppi_start has to be <= {PPI_MAX_LEN}")
+        bad_disabled_apps = [a for a in self.disabled_apps if a not in dataset.available_classes]
+        if len(bad_disabled_apps) > 0:
+            raise ValueError(f"Bad applications in disabled_apps {bad_disabled_apps}. Use applications available in dataset.available_classes")
         if isinstance(self.fit_scalers_samples, float) and (self.fit_scalers_samples <= 0 or self.fit_scalers_samples > 1):
             raise ValueError("fit_scalers_samples has to be either float between 0 and 1 (giving the fraction of training samples used for fitting scalers) or an integer")
 
     def get_flowstats_features_len(self) -> int:
         """Gets the number of flow statistics features."""
-        n = 0
-        for f in self.flowstats_features:
-            if f.startswith("PHIST_"):
-                n += PHIST_BIN_COUNT
-            else:
-                n += 1
-        return n
+        return len(self.flowstats_features) + len(self.flowstats_features_boolean) + PHIST_BIN_COUNT * len(self.flowstats_features_phist)
 
     def get_flowstats_feature_names_expanded(self, shorter_names: bool = False) -> list[str]:
         """Gets names of flow statistics features. Packet histograms are expanded into bin features."""
-        name_mapping = {
+        phist_mapping = {
             "PHIST_SRC_SIZES": [f"PSIZE_BIN{i}" for i in range(1, PHIST_BIN_COUNT + 1)],
             "PHIST_DST_SIZES": [f"PSIZE_BIN{i}_REV" for i in range(1, PHIST_BIN_COUNT + 1)],
             "PHIST_SRC_IPT": [f"IPT_BIN{i}" for i in range(1, PHIST_BIN_COUNT + 1)],
             "PHIST_DST_IPT": [f"IPT_BIN{i}_REV" for i in range(1, PHIST_BIN_COUNT + 1)],
-            "FLOW_ENDREASON_IDLE": "FEND_IDLE" if shorter_names else "FLOW_ENDREASON_IDLE",
-            "FLOW_ENDREASON_ACTIVE": "FEND_ACTIVE" if shorter_names else "FLOW_ENDREASON_ACTIVE",
-            "FLOW_ENDREASON_END": "FEND_END" if shorter_names else "FLOW_ENDREASON_END",
-            "FLOW_ENDREASON_OTHER": "FEND_OTHER" if shorter_names else "FLOW_ENDREASON_OTHER",
         }
-        feature_names = []
-        for f in self.flowstats_features:
-            if f not in name_mapping:
-                if shorter_names and f.startswith("FLAG"):
-                    f = "F" + f.lstrip("FLAG")
-                feature_names.append(f)
-            elif isinstance(name_mapping[f], list):
-                feature_names.extend(name_mapping[f])
+        short_names_mapping = {
+            "FLOW_ENDREASON_IDLE": "FEND_IDLE",
+            "FLOW_ENDREASON_ACTIVE": "FEND_ACTIVE",
+            "FLOW_ENDREASON_END": "FEND_END",
+            "FLOW_ENDREASON_OTHER": "FEND_OTHER",
+            "FLAG_CWR": "F_CWR",
+            "FLAG_CWR_REV": "F_CWR_REV",
+            "FLAG_ECE": "F_ECE",
+            "FLAG_ECE_REV": "F_ECE_REV",
+            "FLAG_PSH_REV": "F_PSH_REV",
+            "FLAG_RST": "F_RST",
+            "FLAG_RST_REV": "F_RST_REV",
+            "FLAG_FIN": "F_FIN",
+            "FLAG_FIN_REV": "F_FIN_REV",
+        }
+        feature_names = self.flowstats_features[:]
+        for f in self.flowstats_features_boolean:
+            if shorter_names and f in short_names_mapping:
+                feature_names.append(short_names_mapping[f])
             else:
-                feature_names.append(name_mapping[f])
+                feature_names.append(f)
+        for f in self.flowstats_features_phist:
+            feature_names.extend(phist_mapping[f])
         assert len(feature_names) == self.get_flowstats_features_len()
         return feature_names
 
@@ -451,8 +452,8 @@ class DatasetConfig():
             ppi_feature_names += [f"PUSH_{i}" for i in range(1, PPI_MAX_LEN + 1)]
         return ppi_feature_names
 
-    def get_ppi_channels(self) -> int:
-        """Gets the number of features (channels) in PPI."""
+    def get_ppi_channels(self) -> list[int]:
+        """Gets the available features (channels) in PPI sequences."""
         if self.use_push_flags:
             return TCP_PPI_CHANNELS
         else:
@@ -487,8 +488,11 @@ class DatasetConfig():
         return params_hash
 
     def _get_train_data_path(self) -> str:
-        params_hash = self._get_train_data_hash()
-        return os.path.join(self.data_root, "train-data", f"{params_hash}_{self.random_state}", f"fold_{self.fold_id}")
+        if self.need_train_set:
+            params_hash = self._get_train_data_hash()
+            return os.path.join(self.data_root, "train-data", f"{params_hash}_{self.random_state}", f"fold_{self.fold_id}")
+        else:
+            return os.path.join(self.data_root, "train-data", "default")
 
     def _get_train_data_params(self) -> TrainDataParams:
         return TrainDataParams(
@@ -497,32 +501,33 @@ class DatasetConfig():
             train_tables_paths=self._get_train_tables_paths(),
             apps_selection=self.apps_selection,
             apps_selection_topx=self.apps_selection_topx,
-            apps_selection_explicit_unknown=self.apps_selection_explicit_unknown,
-            apps_selection_fixed_longterm=self.apps_selection_fixed_longterm,
+            apps_selection_background_unknown=self.apps_selection_background_unknown,
+            apps_selection_fixed_known=self.apps_selection_fixed_known,
+            apps_selection_fixed_unknown=self.apps_selection_fixed_unknown,
             disabled_apps=self.disabled_apps,
             min_train_samples_per_app=self.min_train_samples_per_app,
             min_train_samples_check=self.min_train_samples_check,)
 
-    def _get_val_data_params_and_path(self, known_apps_database_enum: dict[int, str], unknown_apps_database_enum: dict[int, str]) -> tuple[TestDataParams, str]:
+    def _get_val_data_params_and_path(self, known_apps: list[str], unknown_apps: list[str]) -> tuple[TestDataParams, str]:
         assert self.val_approach == ValidationApproach.VALIDATION_DATES
         val_data_params = TestDataParams(
             database_filename=self.database_filename,
             test_period_name=self.val_period_name,
             test_tables_paths=self._get_val_tables_paths(),
-            known_apps_database_enum=known_apps_database_enum,
-            unknown_apps_database_enum=unknown_apps_database_enum,)
+            known_apps=known_apps,
+            unknown_apps=unknown_apps,)
         params_hash = hashlib.sha256(json.dumps(dataclasses.asdict(val_data_params), sort_keys=True).encode()).hexdigest()
         params_hash = params_hash[:10]
         val_data_path = os.path.join(self.data_root, "val-data", f"{params_hash}_{self.random_state}")
         return val_data_params, val_data_path
 
-    def _get_test_data_params_and_path(self, known_apps_database_enum: dict[int, str], unknown_apps_database_enum: dict[int, str]) -> tuple[TestDataParams, str]:
+    def _get_test_data_params_and_path(self, known_apps: list[str], unknown_apps: list[str]) -> tuple[TestDataParams, str]:
         test_data_params = TestDataParams(
             database_filename=self.database_filename,
             test_period_name=self.test_period_name,
             test_tables_paths=self._get_test_tables_paths(),
-            known_apps_database_enum=known_apps_database_enum,
-            unknown_apps_database_enum=unknown_apps_database_enum,)
+            known_apps=known_apps,
+            unknown_apps=unknown_apps,)
         params_hash = hashlib.sha256(json.dumps(dataclasses.asdict(test_data_params), sort_keys=True).encode()).hexdigest()
         params_hash = params_hash[:10]
         test_data_path = os.path.join(self.data_root, "test-data", f"{params_hash}_{self.random_state}")

{cesnet-datazoo-0.0.17 → cesnet-datazoo-0.1.0}/cesnet_datazoo/constants.py

@@ -6,20 +6,18 @@ DATASET_SIZES = {
 }
 
 # Per-packet information (PPI) constants
+PPI_MAX_LEN = 30
 IPT_POS = 0
 DIR_POS = 1
 SIZE_POS = 2
 PUSH_FLAGS_POS = 3
-PPI_MAX_LEN = 30
-TCP_PPI_CHANNELS = 4
-UDP_PPI_CHANNELS = 3
+TCP_PPI_CHANNELS = [IPT_POS, DIR_POS, SIZE_POS, PUSH_FLAGS_POS]
+UDP_PPI_CHANNELS = [IPT_POS, DIR_POS, SIZE_POS]
 
 # Features
 FLOWSTATS_TO_SCALE =  ["BYTES", "BYTES_REV", "PACKETS", "PACKETS_REV", "PPI_LEN", "PPI_ROUNDTRIPS", "PPI_DURATION", "DURATION"]
-FLOWSTATS_NO_CLIP = ["PPI_LEN", "PPI_ROUNDTRIPS", "PPI_DURATION", "DURATION"]
+FLOWSTATS_NO_CLIP = ["DURATION", "PPI_LEN", "PPI_ROUNDTRIPS", "PPI_DURATION"]
 SELECTED_TCP_FLAGS = ["FLAG_CWR", "FLAG_CWR_REV", "FLAG_ECE", "FLAG_ECE_REV", "FLAG_PSH_REV", "FLAG_RST", "FLAG_RST_REV", "FLAG_FIN", "FLAG_FIN_REV"]
-FLOWEND_REASON_FEATURES = ["FLOW_ENDREASON_IDLE", "FLOW_ENDREASON_ACTIVE", "FLOW_ENDREASON_END", "FLOW_ENDREASON_OTHER"]
-PHISTS_FEATURES = ["PHIST_SRC_SIZES", "PHIST_DST_SIZES", "PHIST_SRC_IPT", "PHIST_DST_IPT"]
 PHIST_BIN_COUNT = 8
 
 # Column names