DetectMateDataset 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- detectmatedataset-0.1.0.dist-info/METADATA +66 -0
- detectmatedataset-0.1.0.dist-info/RECORD +21 -0
- detectmatedataset-0.1.0.dist-info/WHEEL +5 -0
- detectmatedataset-0.1.0.dist-info/entry_points.txt +2 -0
- detectmatedataset-0.1.0.dist-info/licenses/LICENSE.md +287 -0
- detectmatedataset-0.1.0.dist-info/top_level.txt +1 -0
- logdatasets/__init__.py +5 -0
- logdatasets/_datasets/__init__.py +0 -0
- logdatasets/_datasets/_dataset_op/__init__.py +0 -0
- logdatasets/_datasets/_dataset_op/cache.py +54 -0
- logdatasets/_datasets/_dataset_op/labels.py +85 -0
- logdatasets/_datasets/_dataset_op/postprocess.py +81 -0
- logdatasets/_datasets/_dataset_op/stats.py +50 -0
- logdatasets/_datasets/dataset.py +161 -0
- logdatasets/_datasets/list_datasets.py +168 -0
- logdatasets/_datasets/templates/__init__.py +0 -0
- logdatasets/_datasets/templates/drone.py +67 -0
- logdatasets/_metrics.py +227 -0
- logdatasets/_op.py +209 -0
- logdatasets/api.py +68 -0
- logdatasets/cli.py +88 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
from logdatasets._datasets._dataset_op.postprocess import PostProcess
|
|
2
|
+
from logdatasets._datasets._dataset_op.labels import LabelSelection
|
|
3
|
+
from logdatasets._datasets._dataset_op.cache import Cache
|
|
4
|
+
from logdatasets._datasets._dataset_op.stats import Stats
|
|
5
|
+
|
|
6
|
+
from logdatasets._datasets.list_datasets import overall_datasets
|
|
7
|
+
from logdatasets._metrics import do_metrics
|
|
8
|
+
import logdatasets._op as op
|
|
9
|
+
|
|
10
|
+
from detectmateperformance.match_tree import TreeMatcher, LogTemplates
|
|
11
|
+
from detectmateperformance.autoparser import AutoParse
|
|
12
|
+
|
|
13
|
+
from tabulate import tabulate
|
|
14
|
+
import polars as pl
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Dataset:
|
|
19
|
+
def __init__(self, name: str, df: pl.DataFrame | pl.LazyFrame) -> None:
|
|
20
|
+
self.name = name
|
|
21
|
+
self.data = df
|
|
22
|
+
self.size = 0
|
|
23
|
+
|
|
24
|
+
def __str__(self) -> str:
|
|
25
|
+
line = "\033[45m" + "".join([" " for _ in range(100)]) + "\033[0m\n"
|
|
26
|
+
return f"\033[45m<Dataset: {self.name}>\033[0m\n{line}{self.get_stats()}\n{line}"
|
|
27
|
+
|
|
28
|
+
def get_dict(self) -> dict[str, str]:
|
|
29
|
+
if isinstance(data := self.get_data(), pl.DataFrame):
|
|
30
|
+
return Stats.from_df(data, n_memory=self.size)
|
|
31
|
+
return Stats.from_lazy(data, n_memory=self.size)
|
|
32
|
+
|
|
33
|
+
def __len__(self) -> int:
|
|
34
|
+
if isinstance(self.get_data(), pl.LazyFrame):
|
|
35
|
+
return self.get_data().select(pl.len()).collect().item()
|
|
36
|
+
return len(self.get_data())
|
|
37
|
+
|
|
38
|
+
def get_metrics(
|
|
39
|
+
self, pred: list[bool], do_print: bool = True, window_size: int = 1, skip: int = 0
|
|
40
|
+
) -> dict:
|
|
41
|
+
"""`skip` drops the first `skip` rows from the scoring window, so a detector's
|
|
42
|
+
warm-up rows can be excluded. skip=0 scores everything (streaming behaviour).
|
|
43
|
+
Only Labels goes on: the metric counts filter the frame, and on an eager frame a
|
|
44
|
+
filter copies every column (HDFS: +2 GB per scoring step)."""
|
|
45
|
+
df = self.get_data().select("Labels")
|
|
46
|
+
if skip:
|
|
47
|
+
df, pred = df.slice(skip), pred[skip:]
|
|
48
|
+
return do_metrics(df=df, pred=pred, do_print=do_print, window_size=window_size)
|
|
49
|
+
|
|
50
|
+
def get_stats(self) -> str:
|
|
51
|
+
return tabulate(self.get_dict(), tablefmt="double_outline")
|
|
52
|
+
|
|
53
|
+
def get_data(self) -> pl.DataFrame | pl.LazyFrame:
|
|
54
|
+
return self.data
|
|
55
|
+
|
|
56
|
+
def get_not_found(self) -> list[str]:
|
|
57
|
+
data = self.data.filter(pl.col("Templates") == "template not found")
|
|
58
|
+
return data["Content"].unique().to_list()
|
|
59
|
+
|
|
60
|
+
def remove_anomalies(self, from_: int, to_: int) -> int:
|
|
61
|
+
n = self.get_data().slice(from_, to_ - from_).filter(pl.col("Labels") == 1)
|
|
62
|
+
n = len(n) if isinstance(n, pl.DataFrame) else n.select(pl.len()).collect().item()
|
|
63
|
+
|
|
64
|
+
row_indices = pl.int_range(0, pl.len())
|
|
65
|
+
self.data = self.data.remove(
|
|
66
|
+
((row_indices >= from_) & (row_indices < to_) & (pl.col("Labels") == 1))
|
|
67
|
+
)
|
|
68
|
+
return n
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def process_dataset(dataset: dict[str, str], path: str, name: str) -> None:
|
|
72
|
+
log_file = path + dataset["Paths"]
|
|
73
|
+
if dataset["Multiple logs"]:
|
|
74
|
+
log_file = op.get_paths(log_file, dataset["Pattern"])
|
|
75
|
+
|
|
76
|
+
if "regex" in dataset:
|
|
77
|
+
regex = dataset["regex"]
|
|
78
|
+
tree_matcher= TreeMatcher(LogTemplates(dataset["templates"]))
|
|
79
|
+
|
|
80
|
+
else:
|
|
81
|
+
tree_matcher, regex = AutoParse()(log_file, log_type=dataset["LogType"])
|
|
82
|
+
df = tree_matcher(
|
|
83
|
+
log_file, get_var=True, regex=regex, n_workers=10, batch=int(3e+6)
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# TODO: until there is a better metod to dial with DNSmasq exception, this stay here
|
|
87
|
+
if "DNSmasq" in name and len(df.filter(pl.col("Templates") == "template not found")) == len(df):
|
|
88
|
+
regex = r'(?P<Month>\S+) (?P<Day>\S+) (?P<Time>\S+) (?P<Value>\S+) (?P<Content>.*)'
|
|
89
|
+
df = tree_matcher(
|
|
90
|
+
log_file, get_var=True, regex=regex, n_workers=10, batch=int(3e+6)
|
|
91
|
+
)
|
|
92
|
+
if dataset.get("LogType", "") in dir(PostProcess):
|
|
93
|
+
df: pl.DataFrame = getattr(PostProcess, dataset["LogType"])(df)
|
|
94
|
+
if dataset.get("Labels_type", "") in dir(LabelSelection) and dataset["Labels"] is not None:
|
|
95
|
+
df: pl.DataFrame = getattr(LabelSelection, dataset["Labels_type"])(df, path + dataset["Labels"])
|
|
96
|
+
|
|
97
|
+
print("⚙️ Processing and sorting by time...")
|
|
98
|
+
time = df.with_columns(pl.col("Timestamp").cast(float))["Timestamp"].to_numpy()
|
|
99
|
+
idx = np.argsort(time, descending=False)
|
|
100
|
+
df = df[idx]
|
|
101
|
+
|
|
102
|
+
Cache.save(name, df)
|
|
103
|
+
print("✅ Data saved in cache!", name)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class MainDataset(Dataset):
|
|
107
|
+
def __init__(
|
|
108
|
+
self, name: str, use_cache: bool = True, sub_name: str = "", do_download: bool = True
|
|
109
|
+
) -> None:
|
|
110
|
+
|
|
111
|
+
name_ = name + "_" + sub_name if sub_name != "" else name
|
|
112
|
+
if use_cache and (data := Cache.check_and_load(name_)) is not None:
|
|
113
|
+
print("✅ Dataset was already downloaded before!")
|
|
114
|
+
elif do_download:
|
|
115
|
+
if "PrePath" in overall_datasets[name]:
|
|
116
|
+
print("🗃️ Loading presaved dataset")
|
|
117
|
+
self._process(name)
|
|
118
|
+
else:
|
|
119
|
+
print("⬇️ Dataset not found, starting download...")
|
|
120
|
+
self._download(name)
|
|
121
|
+
data = Cache.check_and_load(name_)
|
|
122
|
+
else:
|
|
123
|
+
print("ℹ️ Dataset not downloaded")
|
|
124
|
+
data = pl.DataFrame()
|
|
125
|
+
|
|
126
|
+
super().__init__(name_, df=data)
|
|
127
|
+
self.size = Cache.get_size(name_)
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def from_Dataset(
|
|
131
|
+
cls, name: str, sub_name: str, dataset: Dataset, save_cache: bool = False
|
|
132
|
+
) -> "MainDataset":
|
|
133
|
+
cls_dataset = cls(name=name, use_cache=False, sub_name=sub_name, do_download=False)
|
|
134
|
+
cls_dataset.data = dataset.get_data()
|
|
135
|
+
|
|
136
|
+
if save_cache:
|
|
137
|
+
Cache.save(cls_dataset.name, cls_dataset.get_data())
|
|
138
|
+
print("✅ Data saved in cache!", name)
|
|
139
|
+
|
|
140
|
+
return cls_dataset
|
|
141
|
+
|
|
142
|
+
def _download(self, name: str) -> None:
|
|
143
|
+
op.create_temp()
|
|
144
|
+
path = op.download_file(overall_datasets[name]["URL"])
|
|
145
|
+
path = op.unzip(path)
|
|
146
|
+
path = ".temp/unzip_folder"
|
|
147
|
+
|
|
148
|
+
if "List" not in overall_datasets[name]:
|
|
149
|
+
process_dataset(dataset=overall_datasets[name], path=path, name=name)
|
|
150
|
+
else:
|
|
151
|
+
for dataset in overall_datasets[name]["List"]:
|
|
152
|
+
process_dataset(dataset=dataset, path=path, name=name + "_" + dataset["Name"])
|
|
153
|
+
|
|
154
|
+
op.remove_temp()
|
|
155
|
+
|
|
156
|
+
def _process(self, name: str) -> None:
|
|
157
|
+
path = op.unzip(overall_datasets[name]["PrePath"])
|
|
158
|
+
path = path + overall_datasets[name]["Paths"]
|
|
159
|
+
|
|
160
|
+
Cache.save(name, pl.read_parquet(path))
|
|
161
|
+
print("✅ Data saved in cache!", name)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from logdatasets._datasets.templates import drone
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
ait_files: list[dict[str, str]] = [
|
|
7
|
+
{
|
|
8
|
+
"Name": "VPN-OpenVPN",
|
|
9
|
+
"LogType": "OpenVPN",
|
|
10
|
+
"Paths": "/gather/vpn/logs/openvpn.log",
|
|
11
|
+
"Multiple logs": False,
|
|
12
|
+
"Labels": "/labels/vpn/logs/openvpn.log",
|
|
13
|
+
"Labels_type": "AIT_v2",
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"Name": "VPN-Audit",
|
|
17
|
+
"LogType": "Audit",
|
|
18
|
+
"Paths": "/gather/vpn/logs/audit/audit.log",
|
|
19
|
+
"Multiple logs": False,
|
|
20
|
+
"Labels": None,
|
|
21
|
+
"Labels_type": "AIT_v2",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"Name": "VPN-Syslog",
|
|
25
|
+
"LogType": "SysLog",
|
|
26
|
+
"Paths": "/gather/vpn/logs/",
|
|
27
|
+
"Pattern": "syslog",
|
|
28
|
+
"Multiple logs": True,
|
|
29
|
+
"Labels": None,
|
|
30
|
+
"Labels_type": "AIT_v2",
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"Name": "Intranet-Audit",
|
|
34
|
+
"LogType": "Audit",
|
|
35
|
+
"Paths": "/gather/intranet_server/logs/audit/audit.log",
|
|
36
|
+
"Multiple logs": False,
|
|
37
|
+
"Labels": "/labels/intranet_server/logs/audit/audit.log",
|
|
38
|
+
"Labels_type": "AIT_v2",
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"Name": "Intranet-Apache",
|
|
42
|
+
"LogType": "Apache",
|
|
43
|
+
"Paths": "/gather/intranet_server/logs/apache2",
|
|
44
|
+
"Pattern": "-access.log",
|
|
45
|
+
"Multiple logs": True,
|
|
46
|
+
"Labels": "/labels/intranet_server/logs/apache2/",
|
|
47
|
+
"Labels_type": "AIT_v2",
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"Name": "Intranet-Syslog",
|
|
51
|
+
"LogType": "SysLog",
|
|
52
|
+
"Paths": "/gather/intranet_server/logs/",
|
|
53
|
+
"Pattern": "syslog",
|
|
54
|
+
"Multiple logs": True,
|
|
55
|
+
"Labels": None,
|
|
56
|
+
"Labels_type": "AIT_v2",
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"Name": "FileShare-Audit",
|
|
60
|
+
"LogType": "Audit",
|
|
61
|
+
"Paths": "/gather/internal_share/logs/audit/audit.log",
|
|
62
|
+
"Multiple logs": False,
|
|
63
|
+
"Labels": "/labels/internal_share/logs/audit/audit.log",
|
|
64
|
+
"Labels_type": "AIT_v2",
|
|
65
|
+
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"Name": "FileShare-Syslog",
|
|
69
|
+
"LogType": "SysLog",
|
|
70
|
+
"Paths": "/gather/internal_share/logs/",
|
|
71
|
+
"Pattern": "syslog",
|
|
72
|
+
"Multiple logs": True,
|
|
73
|
+
"Labels": None,
|
|
74
|
+
"Labels_type": "AIT_v2",
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"Name": "Firewall-Audit",
|
|
78
|
+
"LogType": "Audit",
|
|
79
|
+
"Paths": "/gather/inet-firewall/logs/audit/audit.log",
|
|
80
|
+
"Multiple logs": False,
|
|
81
|
+
"Labels": None,
|
|
82
|
+
"Labels_type": "AIT_v2",
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"Name": "Firewall-Syslog",
|
|
86
|
+
"LogType": "SysLog",
|
|
87
|
+
"Paths": "/gather/inet-firewall/logs/",
|
|
88
|
+
"Pattern": "syslog",
|
|
89
|
+
"Multiple logs": True,
|
|
90
|
+
"Labels": None,
|
|
91
|
+
"Labels_type": "AIT_v2",
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"Name": "Firewall-DNSmasq",
|
|
95
|
+
"LogType": "DNSmasq",
|
|
96
|
+
"Paths": "/gather/inet-firewall/logs/dnsmasq.log",
|
|
97
|
+
"Multiple logs": False,
|
|
98
|
+
"Labels": "/labels/inet-firewall/logs/dnsmasq.log",
|
|
99
|
+
"Labels_type": "AIT_v2",
|
|
100
|
+
},
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
def ait_config(name: str) -> dict[str, str | list]:
|
|
104
|
+
return {
|
|
105
|
+
"List": ait_files,
|
|
106
|
+
"URL": f"https://zenodo.org/records/5789064/files/{name}.zip?download=1"
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
overall_datasets: dict[str, dict[str, str | list] | dict[str, str]] = {
|
|
111
|
+
"AIT_v2_fox" : ait_config("fox"),
|
|
112
|
+
"AIT_v2_harrison": ait_config("harrison"),
|
|
113
|
+
"AIT_v2_russellmitchell": ait_config("russellmitchell"),
|
|
114
|
+
"AIT_v2_shaw": ait_config("shaw"),
|
|
115
|
+
"AIT_v2_santos": ait_config("santos"),
|
|
116
|
+
"AIT_v2_wardbeck": ait_config("wardbeck"),
|
|
117
|
+
"AIT_v2_wheeler": ait_config("wheeler"),
|
|
118
|
+
"AIT_v2_wilson": ait_config("wilson"),
|
|
119
|
+
"LogHub_HDFS": {
|
|
120
|
+
"Name": "LogHub_HDFS",
|
|
121
|
+
"LogType": "HDFS",
|
|
122
|
+
"Paths": "/HDFS.log",
|
|
123
|
+
"Multiple logs": False,
|
|
124
|
+
"Labels": "/preprocessed/anomaly_label.csv",
|
|
125
|
+
"Labels_type": "HDFS",
|
|
126
|
+
"URL": "https://zenodo.org/records/8196385/files/HDFS_v1.zip?download=1"
|
|
127
|
+
},
|
|
128
|
+
"LogHub_BGL": {
|
|
129
|
+
"Name": "LogHub_BGL",
|
|
130
|
+
"LogType": "BGL",
|
|
131
|
+
"Paths": "/BGL.log",
|
|
132
|
+
"Multiple logs": False,
|
|
133
|
+
"Labels": None,
|
|
134
|
+
"URL": "https://zenodo.org/records/8196385/files/BGL.zip?download=1"
|
|
135
|
+
},
|
|
136
|
+
"LogHub_Thunderbird": {
|
|
137
|
+
"Name": "LogHub_Thunderbird",
|
|
138
|
+
"LogType": "BGL",
|
|
139
|
+
"Paths": "/Thunderbird.log",
|
|
140
|
+
"Multiple logs": False,
|
|
141
|
+
"Labels": None,
|
|
142
|
+
"URL": "https://zenodo.org/records/8196385/files/Thunderbird.tar.gz?download=1"
|
|
143
|
+
},
|
|
144
|
+
"AInception": {
|
|
145
|
+
"URL": "https://zenodo.org/records/17659656/files/SL100.zip?download=1",
|
|
146
|
+
"List": [
|
|
147
|
+
{
|
|
148
|
+
"Name": "Drone-Audit",
|
|
149
|
+
"LogType": "Audit",
|
|
150
|
+
"Paths": "/SL100/sl100-raw-data/2nd Data Extraction_with_suricata_netflows/audit.log",
|
|
151
|
+
"Multiple logs": False,
|
|
152
|
+
"Labels": "/SL100/sl100-labeled-data/audit.log.json",
|
|
153
|
+
"Labels_type": "SL100",
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"Name": "Drone-Logs",
|
|
157
|
+
"LogType": "DroneSL100",
|
|
158
|
+
"Paths": "/SL100/sl100-raw-data/2nd Data Extraction_with_suricata_netflows/drone_flight_logs.log",
|
|
159
|
+
"Multiple logs": False,
|
|
160
|
+
"Labels": "/SL100/sl100-labeled-data/drone_flight_logs.log.json",
|
|
161
|
+
"Labels_type": "SL100",
|
|
162
|
+
"regex": r'(?P<Date>\S+) (?P<Time>\S+) (?P<Content>.*)',
|
|
163
|
+
"templates": drone.templates
|
|
164
|
+
},
|
|
165
|
+
]
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
|
|
2
|
+
templates="""
|
|
3
|
+
height <*>
|
|
4
|
+
waypoint <*>
|
|
5
|
+
Got COMMAND_ACK: <*>: <*>
|
|
6
|
+
Setting out to : <*>
|
|
7
|
+
Checking for <*> : <*>
|
|
8
|
+
Force 32-bit build : <*>
|
|
9
|
+
CXX Compiler : <*>
|
|
10
|
+
AP: Mission: <*>
|
|
11
|
+
AP: Reached command <*>
|
|
12
|
+
Unit tests : <*>
|
|
13
|
+
Autoconfiguration : <*>
|
|
14
|
+
Mode <*>
|
|
15
|
+
Embedding file <*>
|
|
16
|
+
Waf: Entering directory <*>
|
|
17
|
+
Waf: Leaving directory <*>
|
|
18
|
+
Build directory: <*>
|
|
19
|
+
GUIDED> Mode GUIDED
|
|
20
|
+
DC_DSDL compiler in :<*>
|
|
21
|
+
Telemetry log: mav.tlog
|
|
22
|
+
MAV> Detected vehicle <*> on link <*>
|
|
23
|
+
Removing target_list file <*>
|
|
24
|
+
online system <*>
|
|
25
|
+
AP: <*>
|
|
26
|
+
AP: Terrain: clamping offset <*> to <*>
|
|
27
|
+
Update submodules : <*>
|
|
28
|
+
AUTO> AP: Mission: <*> Takeoff
|
|
29
|
+
Saved <*> parameters to <*>
|
|
30
|
+
BUILD SUMMARY
|
|
31
|
+
[1342/1342] checking symbols <*>
|
|
32
|
+
Configured VSCode Intellisense: : <*>
|
|
33
|
+
bin/arducopter <*> Not Applicable Not Applicable
|
|
34
|
+
Password:
|
|
35
|
+
Benchmarks
|
|
36
|
+
Scripting runtime checks : <*>
|
|
37
|
+
'configure' finished successfully (1.357s)
|
|
38
|
+
Debug build : <*>
|
|
39
|
+
Coverage build : <*>
|
|
40
|
+
No target_list file found, creating
|
|
41
|
+
Setting board to : <*>
|
|
42
|
+
RiTW: Window access not found, logging to <*>
|
|
43
|
+
Log Directory:
|
|
44
|
+
Unable to access the X Display, is $DISPLAY set properly?
|
|
45
|
+
Source is git repository
|
|
46
|
+
uild commands will be stored in <*>
|
|
47
|
+
Connect tcp:<*> source_system=<*>
|
|
48
|
+
Using toolchain : <*>
|
|
49
|
+
Enabled firmware ID checking : <*>
|
|
50
|
+
RiTW: Starting ArduCopter : <*> -S --model + --speedup <*> --slave <*> --defaults <*> --sim-address=<*> -I0 --home <*>
|
|
51
|
+
Waiting for heartbeat from <*>
|
|
52
|
+
Source is git repository : <*>
|
|
53
|
+
Generating compile_commands.json
|
|
54
|
+
Enabled custom controller : <*>
|
|
55
|
+
'build' finished successfully <*>
|
|
56
|
+
Loaded module console
|
|
57
|
+
Setting top to : <*>
|
|
58
|
+
Enabled OpenDroneID : <*>
|
|
59
|
+
[1341/1342] Linking build/sitl/bin/arducopter
|
|
60
|
+
Benchmarks : <*>
|
|
61
|
+
Scripting : <*>
|
|
62
|
+
Enabling -Werror : <*>
|
|
63
|
+
AUTO> Mode AUTO
|
|
64
|
+
ERROR in command <*>: map not ready
|
|
65
|
+
Received <*> parameters (ftp)
|
|
66
|
+
GPS Debug Logging : <*>
|
|
67
|
+
""".strip().splitlines()
|
logdatasets/_metrics.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
from tabulate import tabulate
|
|
2
|
+
from typing import overload
|
|
3
|
+
import polars as pl
|
|
4
|
+
|
|
5
|
+
from scipy.stats import norm
|
|
6
|
+
import math
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def wilson_score_interval(
|
|
10
|
+
successes: int, total: int, confidence: float = 0.99
|
|
11
|
+
) -> tuple[float, float]:
|
|
12
|
+
|
|
13
|
+
if total == 0:
|
|
14
|
+
return 0.0, 0.0
|
|
15
|
+
|
|
16
|
+
p = successes / total
|
|
17
|
+
z = norm.ppf(1 - (1 - confidence) / 2)
|
|
18
|
+
|
|
19
|
+
denominator = 1 + (z**2 / total)
|
|
20
|
+
centre_adjusted_probability = p + (z**2 / (2 * total))
|
|
21
|
+
adjusted_variance = z * math.sqrt((p * (1 - p) / total) + (z**2 / (4 * total**2)))
|
|
22
|
+
|
|
23
|
+
lower = (centre_adjusted_probability - adjusted_variance) / denominator
|
|
24
|
+
upper = (centre_adjusted_probability + adjusted_variance) / denominator
|
|
25
|
+
|
|
26
|
+
return float(max(0.0, lower)), float(min(1.0, upper))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class PredCond:
|
|
30
|
+
TP = "TP"
|
|
31
|
+
FN = "FN"
|
|
32
|
+
FP = "FP"
|
|
33
|
+
TN = "TN"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _Metric:
|
|
37
|
+
@staticmethod
|
|
38
|
+
def n(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
39
|
+
return df.shape[0] if isinstance(df, pl.DataFrame) else df.select(pl.len()).collect().item()
|
|
40
|
+
|
|
41
|
+
@staticmethod
|
|
42
|
+
def n_p(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
43
|
+
return _Metric.n(df.filter(
|
|
44
|
+
(pl.col("Pred.") == PredCond.TP) | (pl.col("Pred.") == PredCond.FN)
|
|
45
|
+
))
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def n_n(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
49
|
+
return _Metric.n(df.filter(
|
|
50
|
+
(pl.col("Pred.") == PredCond.TN) | (pl.col("Pred.") == PredCond.FP)
|
|
51
|
+
))
|
|
52
|
+
|
|
53
|
+
@staticmethod
|
|
54
|
+
def n_tp(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
55
|
+
return _Metric.n(df.filter(pl.col("Pred.") == PredCond.TP))
|
|
56
|
+
|
|
57
|
+
@staticmethod
|
|
58
|
+
def n_fn(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
59
|
+
return _Metric.n(df.filter(pl.col("Pred.") == PredCond.FN))
|
|
60
|
+
|
|
61
|
+
@staticmethod
|
|
62
|
+
def n_tn(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
63
|
+
return _Metric.n(df.filter(pl.col("Pred.") == PredCond.TN))
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def n_fp(df: pl.DataFrame | pl.LazyFrame) -> int:
|
|
67
|
+
return _Metric.n(df.filter(pl.col("Pred.") == PredCond.FP))
|
|
68
|
+
@staticmethod
|
|
69
|
+
def tpr(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
70
|
+
return _Metric.n_tp(df) / (_Metric.n_tp(df) + _Metric.n_fn(df) + 1e-10)
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def tnr(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
74
|
+
return _Metric.n_tn(df) / (_Metric.n_tn(df) + _Metric.n_fp(df) + 1e-10)
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def balance_accuracy(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
78
|
+
return (_Metric.tpr(df) + _Metric.tnr(df)) / 2
|
|
79
|
+
|
|
80
|
+
@staticmethod
|
|
81
|
+
def f1(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
82
|
+
tp2 = 2 * _Metric.n_tp(df)
|
|
83
|
+
return tp2 / (tp2 + _Metric.n_fp(df) + _Metric.n_fn(df) + 1e-10)
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def recall(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
87
|
+
return _Metric.tpr(df)
|
|
88
|
+
|
|
89
|
+
@staticmethod
|
|
90
|
+
def recall_confidence_interval(
|
|
91
|
+
df: pl.DataFrame | pl.LazyFrame, confidence: float = 0.95
|
|
92
|
+
) -> tuple[float, float]:
|
|
93
|
+
return wilson_score_interval(
|
|
94
|
+
successes=(tp := _Metric.n_tp(df)), total=tp + _Metric.n_fn(df), confidence=confidence
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def precision(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
99
|
+
tp = _Metric.n_tp(df)
|
|
100
|
+
return tp / (tp + _Metric.n_fp(df) + 1e-10)
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def precision_confidence_interval(
|
|
104
|
+
df: pl.DataFrame | pl.LazyFrame, confidence: float = 0.95
|
|
105
|
+
) -> tuple[float, float]:
|
|
106
|
+
return wilson_score_interval(
|
|
107
|
+
successes=(tp := _Metric.n_tp(df)), total=tp + _Metric.n_fp(df), confidence=confidence
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
@staticmethod
|
|
111
|
+
def jaccard_index(df: pl.DataFrame | pl.LazyFrame) -> float:
|
|
112
|
+
tp = _Metric.n_tp(df)
|
|
113
|
+
return tp / (tp + _Metric.n_fn(df) + _Metric.n_fp(df) + 1e-10)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@overload
|
|
117
|
+
def add_pred(df: pl.DataFrame, pred: list[bool], col_label: str = "Labels") -> pl.DataFrame:
|
|
118
|
+
...
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@overload
|
|
122
|
+
def add_pred(df: pl.LazyFrame, pred: list[bool], col_label: str = "Labels") -> pl.LazyFrame:
|
|
123
|
+
...
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def add_pred(
|
|
127
|
+
df: pl.DataFrame | pl.LazyFrame, pred: list[bool], col_label: str = "Labels"
|
|
128
|
+
) -> pl.DataFrame | pl.LazyFrame:
|
|
129
|
+
|
|
130
|
+
return df.clone().with_columns(pl.Series("a", pred)).with_columns(
|
|
131
|
+
pl.when(pl.col(col_label) == 1).then(
|
|
132
|
+
pl.when(pl.col("a") == 1).then(pl.lit(PredCond.TP)).otherwise(pl.lit(PredCond.FN))
|
|
133
|
+
).otherwise(
|
|
134
|
+
pl.when(pl.col("a") == 1).then(pl.lit(PredCond.FP)).otherwise(pl.lit(PredCond.TN))
|
|
135
|
+
).alias("Pred.")
|
|
136
|
+
).drop("a")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@overload
|
|
140
|
+
def concat_by_window(df: pl.DataFrame, window_size: int, col_label: str = "Labels") -> pl.DataFrame:
|
|
141
|
+
...
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@overload
|
|
145
|
+
def concat_by_window(df: pl.LazyFrame, window_size: int, col_label: str = "Labels") -> pl.LazyFrame:
|
|
146
|
+
...
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def concat_by_window(
|
|
150
|
+
df: pl.DataFrame | pl.LazyFrame, window_size: int, col_label: str = "Labels"
|
|
151
|
+
) -> pl.DataFrame | pl.LazyFrame:
|
|
152
|
+
|
|
153
|
+
return df.with_columns(pl.col(col_label).rolling_sum(window_size=window_size, min_samples=1))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def metrics_calc(
|
|
157
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
158
|
+
pred: list[bool],
|
|
159
|
+
col_label: str = "Labels",
|
|
160
|
+
window_size: int = 1,
|
|
161
|
+
confidence: float = 0.95,
|
|
162
|
+
) -> dict:
|
|
163
|
+
|
|
164
|
+
df = add_pred(df, pred=pred, col_label=col_label)
|
|
165
|
+
if window_size > 1:
|
|
166
|
+
df = concat_by_window(df, window_size=window_size, col_label=col_label)
|
|
167
|
+
|
|
168
|
+
prec_inter = _Metric.precision_confidence_interval(df, confidence=confidence)
|
|
169
|
+
rec_inter = _Metric.recall_confidence_interval(df, confidence=confidence)
|
|
170
|
+
|
|
171
|
+
return {
|
|
172
|
+
"Quantity": {
|
|
173
|
+
"N. Elements": _Metric.n(df),
|
|
174
|
+
"N. Positives": _Metric.n_p(df),
|
|
175
|
+
"N. Negatives": _Metric.n_n(df),
|
|
176
|
+
},
|
|
177
|
+
"Raw predictions": {
|
|
178
|
+
"TP": _Metric.n_tp(df),
|
|
179
|
+
"TN": _Metric.n_tn(df),
|
|
180
|
+
"FP": _Metric.n_fp(df),
|
|
181
|
+
"FN": _Metric.n_fn(df),
|
|
182
|
+
},
|
|
183
|
+
"Main metrics": {
|
|
184
|
+
"F1": round(_Metric.f1(df), 2),
|
|
185
|
+
"Balance accuracy": round(_Metric.balance_accuracy(df), 2),
|
|
186
|
+
"Recall": round(_Metric.recall(df), 2),
|
|
187
|
+
"Precision": round(_Metric.precision(df), 2),
|
|
188
|
+
"Jaccard Index": round(_Metric.jaccard_index(df), 2),
|
|
189
|
+
},
|
|
190
|
+
"Confidence Intervals": {
|
|
191
|
+
"Recall conf. min": round(rec_inter[0], 2),
|
|
192
|
+
"Recall conf. max": round(rec_inter[1], 2),
|
|
193
|
+
"Precision conf. min": round(prec_inter[0], 2),
|
|
194
|
+
"Precision conf. max": round(prec_inter[1], 2),
|
|
195
|
+
},
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def do_metrics(
|
|
200
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
201
|
+
pred: list[bool],
|
|
202
|
+
col_label: str = "Labels",
|
|
203
|
+
do_print: bool = True,
|
|
204
|
+
window_size: int = 1,
|
|
205
|
+
confidence: float = 0.95,
|
|
206
|
+
) -> dict:
|
|
207
|
+
|
|
208
|
+
results = metrics_calc(
|
|
209
|
+
df=df, pred=pred, col_label=col_label, window_size=window_size, confidence=confidence
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
if do_print:
|
|
213
|
+
print("\n\033[46m Results \033[0m")
|
|
214
|
+
print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m\n")
|
|
215
|
+
|
|
216
|
+
for type_ in results:
|
|
217
|
+
print("\033[4m" + type_ + "\033[0m\n")
|
|
218
|
+
print(tabulate(
|
|
219
|
+
{k:[v] for k,v in results[type_].items()},
|
|
220
|
+
headers=results[type_].keys(),
|
|
221
|
+
tablefmt="double_outline"
|
|
222
|
+
))
|
|
223
|
+
print("\n")
|
|
224
|
+
|
|
225
|
+
print("\033[46m" + "".join([" " for _ in range(100)]) + "\033[0m")
|
|
226
|
+
|
|
227
|
+
return results
|