DetectMateDataset 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ from logdatasets._datasets._dataset_op.postprocess import PostProcess
2
+ from logdatasets._datasets._dataset_op.labels import LabelSelection
3
+ from logdatasets._datasets._dataset_op.cache import Cache
4
+ from logdatasets._datasets._dataset_op.stats import Stats
5
+
6
+ from logdatasets._datasets.list_datasets import overall_datasets
7
+ from logdatasets._metrics import do_metrics
8
+ import logdatasets._op as op
9
+
10
+ from detectmateperformance.match_tree import TreeMatcher, LogTemplates
11
+ from detectmateperformance.autoparser import AutoParse
12
+
13
+ from tabulate import tabulate
14
+ import polars as pl
15
+ import numpy as np
16
+
17
+
18
+ class Dataset:
19
+ def __init__(self, name: str, df: pl.DataFrame | pl.LazyFrame) -> None:
20
+ self.name = name
21
+ self.data = df
22
+ self.size = 0
23
+
24
+ def __str__(self) -> str:
25
+ line = "\033[45m" + "".join([" " for _ in range(100)]) + "\033[0m\n"
26
+ return f"\033[45m<Dataset: {self.name}>\033[0m\n{line}{self.get_stats()}\n{line}"
27
+
28
+ def get_dict(self) -> dict[str, str]:
29
+ if isinstance(data := self.get_data(), pl.DataFrame):
30
+ return Stats.from_df(data, n_memory=self.size)
31
+ return Stats.from_lazy(data, n_memory=self.size)
32
+
33
+ def __len__(self) -> int:
34
+ if isinstance(self.get_data(), pl.LazyFrame):
35
+ return self.get_data().select(pl.len()).collect().item()
36
+ return len(self.get_data())
37
+
38
+ def get_metrics(
39
+ self, pred: list[bool], do_print: bool = True, window_size: int = 1, skip: int = 0
40
+ ) -> dict:
41
+ """`skip` drops the first `skip` rows from the scoring window, so a detector's
42
+ warm-up rows can be excluded. skip=0 scores everything (streaming behaviour).
43
+ Only Labels goes on: the metric counts filter the frame, and on an eager frame a
44
+ filter copies every column (HDFS: +2 GB per scoring step)."""
45
+ df = self.get_data().select("Labels")
46
+ if skip:
47
+ df, pred = df.slice(skip), pred[skip:]
48
+ return do_metrics(df=df, pred=pred, do_print=do_print, window_size=window_size)
49
+
50
+ def get_stats(self) -> str:
51
+ return tabulate(self.get_dict(), tablefmt="double_outline")
52
+
53
+ def get_data(self) -> pl.DataFrame | pl.LazyFrame:
54
+ return self.data
55
+
56
+ def get_not_found(self) -> list[str]:
57
+ data = self.data.filter(pl.col("Templates") == "template not found")
58
+ return data["Content"].unique().to_list()
59
+
60
+ def remove_anomalies(self, from_: int, to_: int) -> int:
61
+ n = self.get_data().slice(from_, to_ - from_).filter(pl.col("Labels") == 1)
62
+ n = len(n) if isinstance(n, pl.DataFrame) else n.select(pl.len()).collect().item()
63
+
64
+ row_indices = pl.int_range(0, pl.len())
65
+ self.data = self.data.remove(
66
+ ((row_indices >= from_) & (row_indices < to_) & (pl.col("Labels") == 1))
67
+ )
68
+ return n
69
+
70
+
71
+ def process_dataset(dataset: dict[str, str], path: str, name: str) -> None:
72
+ log_file = path + dataset["Paths"]
73
+ if dataset["Multiple logs"]:
74
+ log_file = op.get_paths(log_file, dataset["Pattern"])
75
+
76
+ if "regex" in dataset:
77
+ regex = dataset["regex"]
78
+ tree_matcher= TreeMatcher(LogTemplates(dataset["templates"]))
79
+
80
+ else:
81
+ tree_matcher, regex = AutoParse()(log_file, log_type=dataset["LogType"])
82
+ df = tree_matcher(
83
+ log_file, get_var=True, regex=regex, n_workers=10, batch=int(3e+6)
84
+ )
85
+
86
+ # TODO: until there is a better metod to dial with DNSmasq exception, this stay here
87
+ if "DNSmasq" in name and len(df.filter(pl.col("Templates") == "template not found")) == len(df):
88
+ regex = r'(?P<Month>\S+) (?P<Day>\S+) (?P<Time>\S+) (?P<Value>\S+) (?P<Content>.*)'
89
+ df = tree_matcher(
90
+ log_file, get_var=True, regex=regex, n_workers=10, batch=int(3e+6)
91
+ )
92
+ if dataset.get("LogType", "") in dir(PostProcess):
93
+ df: pl.DataFrame = getattr(PostProcess, dataset["LogType"])(df)
94
+ if dataset.get("Labels_type", "") in dir(LabelSelection) and dataset["Labels"] is not None:
95
+ df: pl.DataFrame = getattr(LabelSelection, dataset["Labels_type"])(df, path + dataset["Labels"])
96
+
97
+ print("⚙️ Processing and sorting by time...")
98
+ time = df.with_columns(pl.col("Timestamp").cast(float))["Timestamp"].to_numpy()
99
+ idx = np.argsort(time, descending=False)
100
+ df = df[idx]
101
+
102
+ Cache.save(name, df)
103
+ print("✅ Data saved in cache!", name)
104
+
105
+
106
+ class MainDataset(Dataset):
107
+ def __init__(
108
+ self, name: str, use_cache: bool = True, sub_name: str = "", do_download: bool = True
109
+ ) -> None:
110
+
111
+ name_ = name + "_" + sub_name if sub_name != "" else name
112
+ if use_cache and (data := Cache.check_and_load(name_)) is not None:
113
+ print("✅ Dataset was already downloaded before!")
114
+ elif do_download:
115
+ if "PrePath" in overall_datasets[name]:
116
+ print("🗃️ Loading presaved dataset")
117
+ self._process(name)
118
+ else:
119
+ print("⬇️ Dataset not found, starting download...")
120
+ self._download(name)
121
+ data = Cache.check_and_load(name_)
122
+ else:
123
+ print("ℹ️ Dataset not downloaded")
124
+ data = pl.DataFrame()
125
+
126
+ super().__init__(name_, df=data)
127
+ self.size = Cache.get_size(name_)
128
+
129
+ @classmethod
130
+ def from_Dataset(
131
+ cls, name: str, sub_name: str, dataset: Dataset, save_cache: bool = False
132
+ ) -> "MainDataset":
133
+ cls_dataset = cls(name=name, use_cache=False, sub_name=sub_name, do_download=False)
134
+ cls_dataset.data = dataset.get_data()
135
+
136
+ if save_cache:
137
+ Cache.save(cls_dataset.name, cls_dataset.get_data())
138
+ print("✅ Data saved in cache!", name)
139
+
140
+ return cls_dataset
141
+
142
+ def _download(self, name: str) -> None:
143
+ op.create_temp()
144
+ path = op.download_file(overall_datasets[name]["URL"])
145
+ path = op.unzip(path)
146
+ path = ".temp/unzip_folder"
147
+
148
+ if "List" not in overall_datasets[name]:
149
+ process_dataset(dataset=overall_datasets[name], path=path, name=name)
150
+ else:
151
+ for dataset in overall_datasets[name]["List"]:
152
+ process_dataset(dataset=dataset, path=path, name=name + "_" + dataset["Name"])
153
+
154
+ op.remove_temp()
155
+
156
+ def _process(self, name: str) -> None:
157
+ path = op.unzip(overall_datasets[name]["PrePath"])
158
+ path = path + overall_datasets[name]["Paths"]
159
+
160
+ Cache.save(name, pl.read_parquet(path))
161
+ print("✅ Data saved in cache!", name)
@@ -0,0 +1,168 @@
1
+ from logdatasets._datasets.templates import drone
2
+
3
+ from pathlib import Path
4
+
5
+
6
+ ait_files: list[dict[str, str]] = [
7
+ {
8
+ "Name": "VPN-OpenVPN",
9
+ "LogType": "OpenVPN",
10
+ "Paths": "/gather/vpn/logs/openvpn.log",
11
+ "Multiple logs": False,
12
+ "Labels": "/labels/vpn/logs/openvpn.log",
13
+ "Labels_type": "AIT_v2",
14
+ },
15
+ {
16
+ "Name": "VPN-Audit",
17
+ "LogType": "Audit",
18
+ "Paths": "/gather/vpn/logs/audit/audit.log",
19
+ "Multiple logs": False,
20
+ "Labels": None,
21
+ "Labels_type": "AIT_v2",
22
+ },
23
+ {
24
+ "Name": "VPN-Syslog",
25
+ "LogType": "SysLog",
26
+ "Paths": "/gather/vpn/logs/",
27
+ "Pattern": "syslog",
28
+ "Multiple logs": True,
29
+ "Labels": None,
30
+ "Labels_type": "AIT_v2",
31
+ },
32
+ {
33
+ "Name": "Intranet-Audit",
34
+ "LogType": "Audit",
35
+ "Paths": "/gather/intranet_server/logs/audit/audit.log",
36
+ "Multiple logs": False,
37
+ "Labels": "/labels/intranet_server/logs/audit/audit.log",
38
+ "Labels_type": "AIT_v2",
39
+ },
40
+ {
41
+ "Name": "Intranet-Apache",
42
+ "LogType": "Apache",
43
+ "Paths": "/gather/intranet_server/logs/apache2",
44
+ "Pattern": "-access.log",
45
+ "Multiple logs": True,
46
+ "Labels": "/labels/intranet_server/logs/apache2/",
47
+ "Labels_type": "AIT_v2",
48
+ },
49
+ {
50
+ "Name": "Intranet-Syslog",
51
+ "LogType": "SysLog",
52
+ "Paths": "/gather/intranet_server/logs/",
53
+ "Pattern": "syslog",
54
+ "Multiple logs": True,
55
+ "Labels": None,
56
+ "Labels_type": "AIT_v2",
57
+ },
58
+ {
59
+ "Name": "FileShare-Audit",
60
+ "LogType": "Audit",
61
+ "Paths": "/gather/internal_share/logs/audit/audit.log",
62
+ "Multiple logs": False,
63
+ "Labels": "/labels/internal_share/logs/audit/audit.log",
64
+ "Labels_type": "AIT_v2",
65
+
66
+ },
67
+ {
68
+ "Name": "FileShare-Syslog",
69
+ "LogType": "SysLog",
70
+ "Paths": "/gather/internal_share/logs/",
71
+ "Pattern": "syslog",
72
+ "Multiple logs": True,
73
+ "Labels": None,
74
+ "Labels_type": "AIT_v2",
75
+ },
76
+ {
77
+ "Name": "Firewall-Audit",
78
+ "LogType": "Audit",
79
+ "Paths": "/gather/inet-firewall/logs/audit/audit.log",
80
+ "Multiple logs": False,
81
+ "Labels": None,
82
+ "Labels_type": "AIT_v2",
83
+ },
84
+ {
85
+ "Name": "Firewall-Syslog",
86
+ "LogType": "SysLog",
87
+ "Paths": "/gather/inet-firewall/logs/",
88
+ "Pattern": "syslog",
89
+ "Multiple logs": True,
90
+ "Labels": None,
91
+ "Labels_type": "AIT_v2",
92
+ },
93
+ {
94
+ "Name": "Firewall-DNSmasq",
95
+ "LogType": "DNSmasq",
96
+ "Paths": "/gather/inet-firewall/logs/dnsmasq.log",
97
+ "Multiple logs": False,
98
+ "Labels": "/labels/inet-firewall/logs/dnsmasq.log",
99
+ "Labels_type": "AIT_v2",
100
+ },
101
+ ]
102
+
103
+ def ait_config(name: str) -> dict[str, str | list]:
104
+ return {
105
+ "List": ait_files,
106
+ "URL": f"https://zenodo.org/records/5789064/files/{name}.zip?download=1"
107
+ }
108
+
109
+
110
+ overall_datasets: dict[str, dict[str, str | list] | dict[str, str]] = {
111
+ "AIT_v2_fox" : ait_config("fox"),
112
+ "AIT_v2_harrison": ait_config("harrison"),
113
+ "AIT_v2_russellmitchell": ait_config("russellmitchell"),
114
+ "AIT_v2_shaw": ait_config("shaw"),
115
+ "AIT_v2_santos": ait_config("santos"),
116
+ "AIT_v2_wardbeck": ait_config("wardbeck"),
117
+ "AIT_v2_wheeler": ait_config("wheeler"),
118
+ "AIT_v2_wilson": ait_config("wilson"),
119
+ "LogHub_HDFS": {
120
+ "Name": "LogHub_HDFS",
121
+ "LogType": "HDFS",
122
+ "Paths": "/HDFS.log",
123
+ "Multiple logs": False,
124
+ "Labels": "/preprocessed/anomaly_label.csv",
125
+ "Labels_type": "HDFS",
126
+ "URL": "https://zenodo.org/records/8196385/files/HDFS_v1.zip?download=1"
127
+ },
128
+ "LogHub_BGL": {
129
+ "Name": "LogHub_BGL",
130
+ "LogType": "BGL",
131
+ "Paths": "/BGL.log",
132
+ "Multiple logs": False,
133
+ "Labels": None,
134
+ "URL": "https://zenodo.org/records/8196385/files/BGL.zip?download=1"
135
+ },
136
+ "LogHub_Thunderbird": {
137
+ "Name": "LogHub_Thunderbird",
138
+ "LogType": "BGL",
139
+ "Paths": "/Thunderbird.log",
140
+ "Multiple logs": False,
141
+ "Labels": None,
142
+ "URL": "https://zenodo.org/records/8196385/files/Thunderbird.tar.gz?download=1"
143
+ },
144
+ "AInception": {
145
+ "URL": "https://zenodo.org/records/17659656/files/SL100.zip?download=1",
146
+ "List": [
147
+ {
148
+ "Name": "Drone-Audit",
149
+ "LogType": "Audit",
150
+ "Paths": "/SL100/sl100-raw-data/2nd Data Extraction_with_suricata_netflows/audit.log",
151
+ "Multiple logs": False,
152
+ "Labels": "/SL100/sl100-labeled-data/audit.log.json",
153
+ "Labels_type": "SL100",
154
+ },
155
+ {
156
+ "Name": "Drone-Logs",
157
+ "LogType": "DroneSL100",
158
+ "Paths": "/SL100/sl100-raw-data/2nd Data Extraction_with_suricata_netflows/drone_flight_logs.log",
159
+ "Multiple logs": False,
160
+ "Labels": "/SL100/sl100-labeled-data/drone_flight_logs.log.json",
161
+ "Labels_type": "SL100",
162
+ "regex": r'(?P<Date>\S+) (?P<Time>\S+) (?P<Content>.*)',
163
+ "templates": drone.templates
164
+ },
165
+ ]
166
+ }
167
+ }
168
+
File without changes
@@ -0,0 +1,67 @@
1
+
2
+ templates="""
3
+ height <*>
4
+ waypoint <*>
5
+ Got COMMAND_ACK: <*>: <*>
6
+ Setting out to : <*>
7
+ Checking for <*> : <*>
8
+ Force 32-bit build : <*>
9
+ CXX Compiler : <*>
10
+ AP: Mission: <*>
11
+ AP: Reached command <*>
12
+ Unit tests : <*>
13
+ Autoconfiguration : <*>
14
+ Mode <*>
15
+ Embedding file <*>
16
+ Waf: Entering directory <*>
17
+ Waf: Leaving directory <*>
18
+ Build directory: <*>
19
+ GUIDED> Mode GUIDED
20
+ DC_DSDL compiler in :<*>
21
+ Telemetry log: mav.tlog
22
+ MAV> Detected vehicle <*> on link <*>
23
+ Removing target_list file <*>
24
+ online system <*>
25
+ AP: <*>
26
+ AP: Terrain: clamping offset <*> to <*>
27
+ Update submodules : <*>
28
+ AUTO> AP: Mission: <*> Takeoff
29
+ Saved <*> parameters to <*>
30
+ BUILD SUMMARY
31
+ [1342/1342] checking symbols <*>
32
+ Configured VSCode Intellisense: : <*>
33
+ bin/arducopter <*> Not Applicable Not Applicable
34
+ Password:
35
+ Benchmarks
36
+ Scripting runtime checks : <*>
37
+ 'configure' finished successfully (1.357s)
38
+ Debug build : <*>
39
+ Coverage build : <*>
40
+ No target_list file found, creating
41
+ Setting board to : <*>
42
+ RiTW: Window access not found, logging to <*>
43
+ Log Directory:
44
+ Unable to access the X Display, is $DISPLAY set properly?
45
+ Source is git repository
46
+ uild commands will be stored in <*>
47
+ Connect tcp:<*> source_system=<*>
48
+ Using toolchain : <*>
49
+ Enabled firmware ID checking : <*>
50
+ RiTW: Starting ArduCopter : <*> -S --model + --speedup <*> --slave <*> --defaults <*> --sim-address=<*> -I0 --home <*>
51
+ Waiting for heartbeat from <*>
52
+ Source is git repository : <*>
53
+ Generating compile_commands.json
54
+ Enabled custom controller : <*>
55
+ 'build' finished successfully <*>
56
+ Loaded module console
57
+ Setting top to : <*>
58
+ Enabled OpenDroneID : <*>
59
+ [1341/1342] Linking build/sitl/bin/arducopter
60
+ Benchmarks : <*>
61
+ Scripting : <*>
62
+ Enabling -Werror : <*>
63
+ AUTO> Mode AUTO
64
+ ERROR in command <*>: map not ready
65
+ Received <*> parameters (ftp)
66
+ GPS Debug Logging : <*>
67
+ """.strip().splitlines()
@@ -0,0 +1,227 @@
1
+ from tabulate import tabulate
2
+ from typing import overload
3
+ import polars as pl
4
+
5
+ from scipy.stats import norm
6
+ import math
7
+
8
+
9
+ def wilson_score_interval(
10
+ successes: int, total: int, confidence: float = 0.99
11
+ ) -> tuple[float, float]:
12
+
13
+ if total == 0:
14
+ return 0.0, 0.0
15
+
16
+ p = successes / total
17
+ z = norm.ppf(1 - (1 - confidence) / 2)
18
+
19
+ denominator = 1 + (z**2 / total)
20
+ centre_adjusted_probability = p + (z**2 / (2 * total))
21
+ adjusted_variance = z * math.sqrt((p * (1 - p) / total) + (z**2 / (4 * total**2)))
22
+
23
+ lower = (centre_adjusted_probability - adjusted_variance) / denominator
24
+ upper = (centre_adjusted_probability + adjusted_variance) / denominator
25
+
26
+ return float(max(0.0, lower)), float(min(1.0, upper))
27
+
28
+
29
+ class PredCond:
30
+ TP = "TP"
31
+ FN = "FN"
32
+ FP = "FP"
33
+ TN = "TN"
34
+
35
+
36
+ class _Metric:
37
+ @staticmethod
38
+ def n(df: pl.DataFrame | pl.LazyFrame) -> int:
39
+ return df.shape[0] if isinstance(df, pl.DataFrame) else df.select(pl.len()).collect().item()
40
+
41
+ @staticmethod
42
+ def n_p(df: pl.DataFrame | pl.LazyFrame) -> int:
43
+ return _Metric.n(df.filter(
44
+ (pl.col("Pred.") == PredCond.TP) | (pl.col("Pred.") == PredCond.FN)
45
+ ))
46
+
47
+ @staticmethod
48
+ def n_n(df: pl.DataFrame | pl.LazyFrame) -> int:
49
+ return _Metric.n(df.filter(
50
+ (pl.col("Pred.") == PredCond.TN) | (pl.col("Pred.") == PredCond.FP)
51
+ ))
52
+
53
+ @staticmethod
54
+ def n_tp(df: pl.DataFrame | pl.LazyFrame) -> int:
55
+ return _Metric.n(df.filter(pl.col("Pred.") == PredCond.TP))
56
+
57
+ @staticmethod
58
+ def n_fn(df: pl.DataFrame | pl.LazyFrame) -> int:
59
+ return _Metric.n(df.filter(pl.col("Pred.") == PredCond.FN))
60
+
61
+ @staticmethod
62
+ def n_tn(df: pl.DataFrame | pl.LazyFrame) -> int:
63
+ return _Metric.n(df.filter(pl.col("Pred.") == PredCond.TN))
64
+
65
+ @staticmethod
66
+ def n_fp(df: pl.DataFrame | pl.LazyFrame) -> int:
67
+ return _Metric.n(df.filter(pl.col("Pred.") == PredCond.FP))
68
+ @staticmethod
69
+ def tpr(df: pl.DataFrame | pl.LazyFrame) -> float:
70
+ return _Metric.n_tp(df) / (_Metric.n_tp(df) + _Metric.n_fn(df) + 1e-10)
71
+
72
+ @staticmethod
73
+ def tnr(df: pl.DataFrame | pl.LazyFrame) -> float:
74
+ return _Metric.n_tn(df) / (_Metric.n_tn(df) + _Metric.n_fp(df) + 1e-10)
75
+
76
+ @staticmethod
77
+ def balance_accuracy(df: pl.DataFrame | pl.LazyFrame) -> float:
78
+ return (_Metric.tpr(df) + _Metric.tnr(df)) / 2
79
+
80
+ @staticmethod
81
+ def f1(df: pl.DataFrame | pl.LazyFrame) -> float:
82
+ tp2 = 2 * _Metric.n_tp(df)
83
+ return tp2 / (tp2 + _Metric.n_fp(df) + _Metric.n_fn(df) + 1e-10)
84
+
85
+ @staticmethod
86
+ def recall(df: pl.DataFrame | pl.LazyFrame) -> float:
87
+ return _Metric.tpr(df)
88
+
89
+ @staticmethod
90
+ def recall_confidence_interval(
91
+ df: pl.DataFrame | pl.LazyFrame, confidence: float = 0.95
92
+ ) -> tuple[float, float]:
93
+ return wilson_score_interval(
94
+ successes=(tp := _Metric.n_tp(df)), total=tp + _Metric.n_fn(df), confidence=confidence
95
+ )
96
+
97
+ @staticmethod
98
+ def precision(df: pl.DataFrame | pl.LazyFrame) -> float:
99
+ tp = _Metric.n_tp(df)
100
+ return tp / (tp + _Metric.n_fp(df) + 1e-10)
101
+
102
+ @staticmethod
103
+ def precision_confidence_interval(
104
+ df: pl.DataFrame | pl.LazyFrame, confidence: float = 0.95
105
+ ) -> tuple[float, float]:
106
+ return wilson_score_interval(
107
+ successes=(tp := _Metric.n_tp(df)), total=tp + _Metric.n_fp(df), confidence=confidence
108
+ )
109
+
110
+ @staticmethod
111
+ def jaccard_index(df: pl.DataFrame | pl.LazyFrame) -> float:
112
+ tp = _Metric.n_tp(df)
113
+ return tp / (tp + _Metric.n_fn(df) + _Metric.n_fp(df) + 1e-10)
114
+
115
+
116
+ @overload
117
+ def add_pred(df: pl.DataFrame, pred: list[bool], col_label: str = "Labels") -> pl.DataFrame:
118
+ ...
119
+
120
+
121
+ @overload
122
+ def add_pred(df: pl.LazyFrame, pred: list[bool], col_label: str = "Labels") -> pl.LazyFrame:
123
+ ...
124
+
125
+
126
+ def add_pred(
127
+ df: pl.DataFrame | pl.LazyFrame, pred: list[bool], col_label: str = "Labels"
128
+ ) -> pl.DataFrame | pl.LazyFrame:
129
+
130
+ return df.clone().with_columns(pl.Series("a", pred)).with_columns(
131
+ pl.when(pl.col(col_label) == 1).then(
132
+ pl.when(pl.col("a") == 1).then(pl.lit(PredCond.TP)).otherwise(pl.lit(PredCond.FN))
133
+ ).otherwise(
134
+ pl.when(pl.col("a") == 1).then(pl.lit(PredCond.FP)).otherwise(pl.lit(PredCond.TN))
135
+ ).alias("Pred.")
136
+ ).drop("a")
137
+
138
+
139
+ @overload
140
+ def concat_by_window(df: pl.DataFrame, window_size: int, col_label: str = "Labels") -> pl.DataFrame:
141
+ ...
142
+
143
+
144
+ @overload
145
+ def concat_by_window(df: pl.LazyFrame, window_size: int, col_label: str = "Labels") -> pl.LazyFrame:
146
+ ...
147
+
148
+
149
+ def concat_by_window(
150
+ df: pl.DataFrame | pl.LazyFrame, window_size: int, col_label: str = "Labels"
151
+ ) -> pl.DataFrame | pl.LazyFrame:
152
+
153
+ return df.with_columns(pl.col(col_label).rolling_sum(window_size=window_size, min_samples=1))
154
+
155
+
156
+ def metrics_calc(
157
+ df: pl.DataFrame | pl.LazyFrame,
158
+ pred: list[bool],
159
+ col_label: str = "Labels",
160
+ window_size: int = 1,
161
+ confidence: float = 0.95,
162
+ ) -> dict:
163
+
164
+ df = add_pred(df, pred=pred, col_label=col_label)
165
+ if window_size > 1:
166
+ df = concat_by_window(df, window_size=window_size, col_label=col_label)
167
+
168
+ prec_inter = _Metric.precision_confidence_interval(df, confidence=confidence)
169
+ rec_inter = _Metric.recall_confidence_interval(df, confidence=confidence)
170
+
171
+ return {
172
+ "Quantity": {
173
+ "N. Elements": _Metric.n(df),
174
+ "N. Positives": _Metric.n_p(df),
175
+ "N. Negatives": _Metric.n_n(df),
176
+ },
177
+ "Raw predictions": {
178
+ "TP": _Metric.n_tp(df),
179
+ "TN": _Metric.n_tn(df),
180
+ "FP": _Metric.n_fp(df),
181
+ "FN": _Metric.n_fn(df),
182
+ },
183
+ "Main metrics": {
184
+ "F1": round(_Metric.f1(df), 2),
185
+ "Balance accuracy": round(_Metric.balance_accuracy(df), 2),
186
+ "Recall": round(_Metric.recall(df), 2),
187
+ "Precision": round(_Metric.precision(df), 2),
188
+ "Jaccard Index": round(_Metric.jaccard_index(df), 2),
189
+ },
190
+ "Confidence Intervals": {
191
+ "Recall conf. min": round(rec_inter[0], 2),
192
+ "Recall conf. max": round(rec_inter[1], 2),
193
+ "Precision conf. min": round(prec_inter[0], 2),
194
+ "Precision conf. max": round(prec_inter[1], 2),
195
+ },
196
+ }
197
+
198
+
199
+ def do_metrics(
200
+ df: pl.DataFrame | pl.LazyFrame,
201
+ pred: list[bool],
202
+ col_label: str = "Labels",
203
+ do_print: bool = True,
204
+ window_size: int = 1,
205
+ confidence: float = 0.95,
206
+ ) -> dict:
207
+
208
+ results = metrics_calc(
209
+ df=df, pred=pred, col_label=col_label, window_size=window_size, confidence=confidence
210
+ )
211
+
212
+ if do_print:
213
+ print("\n\033[46m Results \033[0m")
214
+ print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m\n")
215
+
216
+ for type_ in results:
217
+ print("\033[4m" + type_ + "\033[0m\n")
218
+ print(tabulate(
219
+ {k:[v] for k,v in results[type_].items()},
220
+ headers=results[type_].keys(),
221
+ tablefmt="double_outline"
222
+ ))
223
+ print("\n")
224
+
225
+ print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m")
226
+
227
+ return results