DetectMateDataset 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
logdatasets/_op.py ADDED
@@ -0,0 +1,209 @@
1
+
2
+ from datetime import datetime
3
+ from tqdm import tqdm
4
+
5
+ from pathlib import Path
6
+
7
+ import requests
8
+ import zipfile
9
+ import tarfile
10
+ import shutil
11
+ import json
12
+ import os
13
+
14
+ import polars as pl
15
+
16
+ temp_folder = ".temp"
17
+
18
+
19
+ def create_temp() -> None:
20
+ if not os.path.exists(temp_folder):
21
+ os.mkdir(temp_folder)
22
+
23
+
24
+ def remove_temp() -> None:
25
+ if os.path.exists(temp_folder):
26
+ shutil.rmtree(temp_folder)
27
+
28
+
29
+ def unzip(zip_file: str) -> str:
30
+
31
+ print("\033[44m >>>> Unziping data \033[0m")
32
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m")
33
+ path = f"{temp_folder}/unzip_folder"
34
+
35
+ # download_file always saves as "data.zip" regardless of real format, so
36
+ # detect by content, not extension. tarfile check first: a .tar.gz is not a zip.
37
+ if tarfile.is_tarfile(zip_file):
38
+ with tarfile.open(zip_file, 'r:*') as tar_ref:
39
+ members = tar_ref.getmembers()
40
+ for member in tqdm(
41
+ members, desc="\033[44m \033[0mExtracting files ⏳",
42
+ unit="file",
43
+ bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}'
44
+ ):
45
+ tar_ref.extract(member, path, filter='data')
46
+ print(f"\033[44m \033[0m✅ All {len(members)} files extracted to {path}!")
47
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m \n")
48
+ return path
49
+
50
+ try:
51
+ with zipfile.ZipFile(zip_file, 'r') as zip_ref:
52
+ if zip_ref.testzip() is not None:
53
+ raise Exception("File corrupt")
54
+
55
+ file_list = zip_ref.namelist()
56
+ total_files = len(file_list)
57
+
58
+ for file in tqdm(
59
+ file_list, desc="\033[44m \033[0mExtracting files ⏳",
60
+ unit="file",
61
+ bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}'
62
+ ):
63
+ zip_ref.extract(file, path)
64
+ print(f"\033[44m \033[0m✅ All {total_files} files extracted to {path}!")
65
+
66
+ except zipfile.BadZipFile:
67
+ print(f"\033[44m \033[0m ⚠️ Can't uncompress the file, try manually")
68
+ print(f"\033[44m \033[0m ⚠️ please, save it as unzip_folder")
69
+ print(f"\033[44m \033[0m ⚠️ folder -> .temp")
70
+ while (msg := input("\033[44m \033[0m Y: when done, N: to abort: ")) != "Y":
71
+ if msg == "N":
72
+ raise Exception("❌ Abort process")
73
+
74
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m \n")
75
+
76
+ return path
77
+
78
+
79
+ def split_file(path: str) -> list[str]:
80
+ print("\033[44m >>>> Spliting long file \033[0m")
81
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m")
82
+
83
+ print("\033[44m \033[0mCalculating number of lines ⏳")
84
+ with open(path, "rb") as input_file:
85
+ size = sum([1 for _ in input_file])
86
+ print("\033[44m \033[0m✅ Number of lines:", size)
87
+
88
+ files_ = []
89
+ with open(path) as input_file:
90
+ for i in tqdm(
91
+ range(int(size / 10e6)),
92
+ desc="\033[44m \033[0m Spliting in files ⏳",
93
+ unit="logs",
94
+ bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}'
95
+ ):
96
+ files_.append(f"{temp_folder}/{i}.log")
97
+ head = []
98
+ for __ in range(int(10e6)):
99
+ try:
100
+ head.append(next(input_file))
101
+ except UnicodeDecodeError:
102
+ print(f"\033[44m \033[0m ⚠️ Unable to read line")
103
+ with open(files_[-1], "w") as f:
104
+ f.writelines(head)
105
+
106
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m\n")
107
+ return files_
108
+
109
+
110
+ def load_file(path_logs) -> list[str]:
111
+ with open(path_logs, "r") as f:
112
+ return f.readlines()
113
+
114
+
115
+ def load_json(path: str) -> dict:
116
+ with open(path, 'r', encoding='utf-8') as file:
117
+ return json.load(file)
118
+
119
+
120
+ def download_file(url: str, chunk_size: int = 4000 ) -> str:
121
+
122
+ print("\033[44m >>>> Downloading data \033[0m")
123
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m")
124
+
125
+ path = os.path.join(temp_folder, "data.zip")
126
+ flag = False
127
+ for _ in range(10):
128
+ try:
129
+ with requests.get(url, stream=True) as r:
130
+ r.raise_for_status()
131
+ total_size = int(r.headers.get('content-length', 0))
132
+
133
+ print("\033[44m \033[0mURL:", url)
134
+ with open(path, 'wb') as f, tqdm(
135
+ desc="\033[44m \033[0m Downloading ⏳",
136
+ total=total_size,
137
+ unit='B',
138
+ unit_scale=True,
139
+ unit_divisor=1024,
140
+ bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}'
141
+ ) as bar:
142
+ for chunk in r.iter_content(chunk_size=chunk_size):
143
+ f.write(chunk)
144
+ bar.update(len(chunk))
145
+ flag = True
146
+ break
147
+ except requests.exceptions.ChunkedEncodingError:
148
+ print(f"\033[44m \033[0m ⚠️ Fail, trying again")
149
+ continue
150
+
151
+ if not flag:
152
+ raise Exception("❌ Fail to download")
153
+ print(f"\033[44m \033[0m✅ Download complete!")
154
+ print("\033[44m" + "".join([" " for _ in range(100)]) + "\033[0m \n")
155
+
156
+ return path
157
+
158
+
159
+ class AInception:
160
+ @staticmethod
161
+ def is_malicious(label: dict) -> bool:
162
+ return bool(label["malicious"])
163
+
164
+ @staticmethod
165
+ def get_timestamp(label: dict) -> str:
166
+ dt = datetime.fromisoformat(label["@timestamp"])
167
+ return dt.timestamp()
168
+
169
+ @staticmethod
170
+ def assign_labels(df: pl.DataFrame, labels: dict) -> pl.DataFrame:
171
+
172
+ print("\033[45m >>>> Assigning data \033[0m")
173
+ print("\033[45m" + "".join([" " for _ in range(100)]) + "\033[0m")
174
+
175
+ df = df.with_row_index()
176
+ df_2 = df.clone().insert_column(-1, pl.col("Time").cast(float).alias("Time_"))
177
+
178
+ indices = []
179
+ for label in tqdm(
180
+ labels,
181
+ desc="\033[45m \033[0mAssigning labels to logs ⏳",
182
+ bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}'
183
+ ):
184
+ if AInception.is_malicious(label):
185
+ df_ = df_2.filter(pl.col("Content") == label["message"])
186
+ idx = (df_.with_columns(
187
+ pl.col("Time_") - AInception.get_timestamp(label)
188
+ )["Time_"]).abs().arg_min()
189
+ indices.extend(df_[idx]["index"].to_list())
190
+
191
+ indices = list(set(indices))
192
+
193
+ df = df.with_columns(
194
+ pl.when(pl.col("index").is_in(indices)).then(1).otherwise(0).alias("Labels")
195
+ )
196
+ df.drop("index")
197
+ print(f"\033[45m \033[0m✅ Assign labels complete!")
198
+ print("\033[45m" + "".join([" " for _ in range(100)]) + "\033[0m \n")
199
+
200
+ return df
201
+
202
+
203
+ def get_paths(path_: str, pattern: str) -> str:
204
+ path = Path(path_)
205
+ files = [
206
+ str(f) for f in path.iterdir()
207
+ if f.is_file() and pattern in str(f) and os.path.getsize(str(f)) != 0
208
+ ]
209
+ return [ fil for fil in files ]
logdatasets/api.py ADDED
@@ -0,0 +1,68 @@
1
+ from logdatasets._datasets.list_datasets import overall_datasets
2
+ from logdatasets._datasets.dataset import MainDataset
3
+
4
+ import tabulate
5
+
6
+
7
+ def load_dataset(
8
+ dataset: str, sub_name: str = "", do_cache: bool = True
9
+ ) -> MainDataset:
10
+
11
+ print(db := MainDataset(dataset, sub_name=sub_name, use_cache=do_cache))
12
+ return db
13
+
14
+
15
+ def status_dataset(dataset: str) -> None:
16
+ def get_values(sub_name: str) -> tuple[list[str], list[str]]:
17
+ db = MainDataset(dataset, sub_name=sub_name, do_download=False)
18
+ dict_ = db.get_dict()
19
+ downloaded = "✅"
20
+ if len(db) == 0:
21
+ downloaded = "❌"
22
+
23
+ values = [sub_name, downloaded] + dict_["Values"]
24
+ fields = ["Name", "Downloaded"] + dict_["Field"]
25
+
26
+ return fields, values
27
+
28
+ if "List" in overall_datasets[dataset]:
29
+ values = []
30
+ for name in overall_datasets[dataset]["List"]:
31
+ fields, values_row = get_values(name["Name"])
32
+ values.append(values_row)
33
+
34
+ else:
35
+ fields, values = get_values("")
36
+ values = [values]
37
+
38
+ print(dataset)
39
+ print(tabulate.tabulate(values, headers=fields, tablefmt="grid"))
40
+
41
+
42
+ def status_all_datasets() -> None:
43
+ for dataset in overall_datasets:
44
+ status_dataset(dataset)
45
+
46
+
47
+ def get_dataset_names() -> list[str]:
48
+ print("Datasets:")
49
+ for name in overall_datasets:
50
+ print(" ->", name)
51
+
52
+ return overall_datasets
53
+
54
+
55
+ def get_sub_dataset_names(dataset: str) -> list[str]:
56
+ print("From dataset", dataset)
57
+ sel_dataset = overall_datasets[dataset]
58
+
59
+ sub_datsets = []
60
+ if "List" in sel_dataset:
61
+ for name in sel_dataset["List"]:
62
+ print(" ->", name["Name"])
63
+ sub_datsets.append(name["Name"])
64
+ else:
65
+ print("<None>")
66
+ sub_datsets.append("")
67
+
68
+ return sub_datsets
logdatasets/cli.py ADDED
@@ -0,0 +1,88 @@
1
+ from logdatasets import api
2
+
3
+ import argparse
4
+
5
+
6
+ def status_dataset_func(args: argparse.Namespace) -> None:
7
+ api.status_dataset(args.dataset)
8
+
9
+
10
+ def status_datasets_func(args: argparse.Namespace) -> None:
11
+ api.status_all_datasets()
12
+
13
+
14
+ def get_names_datasets(args: argparse.Namespace) -> None:
15
+ api.get_dataset_names()
16
+
17
+
18
+ def get_names_datasets_sub(args: argparse.Namespace) -> None:
19
+ api.get_sub_dataset_names(args.dataset)
20
+
21
+
22
+ def load_datset_func(args: argparse.Namespace) -> None:
23
+ api.load_dataset(args.dataset, args.sub_dataset, do_cache=args.force_download)
24
+
25
+ def main():
26
+ parser = argparse.ArgumentParser()
27
+ commands = parser.add_subparsers(
28
+ dest="exec",
29
+ required=True,
30
+ help="Command to execute in the API"
31
+ )
32
+
33
+ status_dataset = commands.add_parser(
34
+ "status_dataset",
35
+ help="Check the status of a dataset",
36
+ )
37
+ status_dataset.add_argument(
38
+ "dataset",
39
+ help="Dataset to inspect"
40
+ )
41
+ status_dataset.set_defaults(func=status_dataset_func)
42
+
43
+ status_datasets = commands.add_parser(
44
+ "status_all_dataset",
45
+ help="Check the status of all the dataset",
46
+ )
47
+ status_datasets.set_defaults(func=status_datasets_func)
48
+
49
+ get_names = commands.add_parser(
50
+ "dataset_names",
51
+ help="Get the names of the datasets",
52
+ )
53
+ get_names.set_defaults(func=get_names_datasets)
54
+
55
+ get_sub_names = commands.add_parser(
56
+ "sub_dataset_names",
57
+ help="Get the names of the subdataset",
58
+ )
59
+ get_sub_names.add_argument(
60
+ "dataset",
61
+ help="Dataset to inspect"
62
+ )
63
+ get_sub_names.set_defaults(func=get_names_datasets_sub)
64
+
65
+ load_dataset = commands.add_parser(
66
+ "load_dataset",
67
+ help="Load a dataset",
68
+ )
69
+ load_dataset.add_argument(
70
+ "dataset",
71
+ help="Dataset to load"
72
+ )
73
+ load_dataset.add_argument(
74
+ "sub_dataset",
75
+ nargs="?",
76
+ help="sub dataset to load",
77
+ default=""
78
+ )
79
+ load_dataset.add_argument(
80
+ "--force_download",
81
+ action="store_false",
82
+ help="Ignore cache, always download"
83
+ )
84
+ load_dataset.set_defaults(func=load_datset_func)
85
+
86
+ args = parser.parse_args()
87
+ args.func(args)
88
+