logwarp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- logwarp/__init__.py +4 -0
- logwarp/_version.py +24 -0
- logwarp/errors.py +4 -0
- logwarp/logfile.py +500 -0
- logwarp/tests/__init__.py +0 -0
- logwarp/tests/test_logfile.py +317 -0
- logwarp/tests/test_utils.py +388 -0
- logwarp/utils.py +300 -0
- logwarp-0.1.0.dist-info/METADATA +81 -0
- logwarp-0.1.0.dist-info/RECORD +13 -0
- logwarp-0.1.0.dist-info/WHEEL +4 -0
- logwarp-0.1.0.dist-info/entry_points.txt +2 -0
- logwarp-0.1.0.dist-info/licenses/LICENSE +26 -0
logwarp/__init__.py
ADDED
logwarp/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
logwarp/errors.py
ADDED
logwarp/logfile.py
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2026 Lukasz Tuz <lukasz@bitsandpixels.io>
|
|
2
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
3
|
+
|
|
4
|
+
"""Implements a representation of a log file that can be searched and analyzed
|
|
5
|
+
by other modules.
|
|
6
|
+
|
|
7
|
+
This module provides a class, `LogFile`, that represents a log file as a list
|
|
8
|
+
of `LogLine` objects. The `LogFile` objects are subscriptable and iterable,
|
|
9
|
+
allowing for using integers, slices, and lists as indices.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
from collections import UserString
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import overload
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import scipy
|
|
21
|
+
|
|
22
|
+
from logwarp.errors import PatternGroupError
|
|
23
|
+
from logwarp.utils import SliceableArray, Vectorizer
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class LogSignalProcessor:
|
|
27
|
+
"""Implements signal processing methods for log files."""
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def smooth(signal: np.ndarray, window_size: int = 10) -> np.ndarray:
|
|
31
|
+
"""Smooths a signal convolution with a rectangular window."""
|
|
32
|
+
return np.convolve(signal, np.ones(window_size) / window_size, mode="valid")
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def tune_peak_detection(signal: np.ndarray) -> tuple[float, float]:
|
|
36
|
+
"""Tunes parameters for peak detection in a signal.
|
|
37
|
+
|
|
38
|
+
Based on signals peak-to-peak value, calculates prominence and height
|
|
39
|
+
for `scipy.signal.find_peaks()`
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
tuple[float, float]
|
|
44
|
+
Tuple of prominence and height
|
|
45
|
+
"""
|
|
46
|
+
ptp = np.ptp(signal)
|
|
47
|
+
return 0.66 * ptp, 0.33 * ptp
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LogLine(UserString):
|
|
51
|
+
_number: int
|
|
52
|
+
|
|
53
|
+
def __init__(self, line: str, /, n: int = -1):
|
|
54
|
+
self._number = n
|
|
55
|
+
super().__init__(line)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def number(self):
|
|
59
|
+
return self._number
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class LogFileSplit(dict):
|
|
63
|
+
"""Dict-like storage for split log file buckets. Guarantees existence of the
|
|
64
|
+
'outliers' bucket."""
|
|
65
|
+
|
|
66
|
+
OUTLIERS = "outliers"
|
|
67
|
+
|
|
68
|
+
def __init__(self, *args, **kwargs):
|
|
69
|
+
super().__init__(*args, **kwargs)
|
|
70
|
+
self[LogFileSplit.OUTLIERS] = []
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class LogFile:
|
|
74
|
+
_content: list[str | LogLine]
|
|
75
|
+
_vectorized: SliceableArray[float]
|
|
76
|
+
_hashes: dict
|
|
77
|
+
|
|
78
|
+
def __init__(self, log_file, **kwargs):
|
|
79
|
+
"""Main class for interacting with a log file.
|
|
80
|
+
|
|
81
|
+
Implements indexing by integers (single line), slices, and lists (including
|
|
82
|
+
non-contiguous ranges of lines).
|
|
83
|
+
|
|
84
|
+
Parameters
|
|
85
|
+
----------
|
|
86
|
+
log_file : str | Path | list | tuple
|
|
87
|
+
Path to the log file to load (`str` or `Path`) or content of a log
|
|
88
|
+
file in form of an iterable with strings.
|
|
89
|
+
|
|
90
|
+
Keyword Arguments
|
|
91
|
+
-----------------
|
|
92
|
+
kwargs : Any
|
|
93
|
+
Arguments passed directly to the `SliceableArray` constructor.
|
|
94
|
+
|
|
95
|
+
Examples
|
|
96
|
+
--------
|
|
97
|
+
>>> log_file = LogFile(["Line 1", "Line 2", "Line 3", "Line 4"])
|
|
98
|
+
>>> log_file[0]
|
|
99
|
+
'Line 1'
|
|
100
|
+
>>> log_file[1:3] # Slice returns list of lists
|
|
101
|
+
['Line 2', 'Line 3']
|
|
102
|
+
>>> log_file[[0, 2]]
|
|
103
|
+
['Line 1', 'Line 3']
|
|
104
|
+
>>> log_file[::2]
|
|
105
|
+
['Line 1', 'Line 3']
|
|
106
|
+
"""
|
|
107
|
+
if isinstance(log_file, (str, Path)):
|
|
108
|
+
self._content = self.from_file(log_file)
|
|
109
|
+
elif isinstance(log_file, (list, tuple)):
|
|
110
|
+
self._content = self.from_list(log_file)
|
|
111
|
+
else:
|
|
112
|
+
raise TypeError(f"Don't know how to make a LogFile from {type(log_file)}")
|
|
113
|
+
|
|
114
|
+
# Store intermittent results, traces, etc. here
|
|
115
|
+
self._vectorized = SliceableArray([])
|
|
116
|
+
self._hashes = {}
|
|
117
|
+
|
|
118
|
+
def __getitem__(self, key):
|
|
119
|
+
if isinstance(key, list):
|
|
120
|
+
return [line for idx, line in enumerate(self) if idx in key]
|
|
121
|
+
return self._content[key]
|
|
122
|
+
|
|
123
|
+
def __setitem__(self, key, value):
|
|
124
|
+
raise AttributeError("LogFile is read only")
|
|
125
|
+
|
|
126
|
+
def __len__(self):
|
|
127
|
+
return len(self._content)
|
|
128
|
+
|
|
129
|
+
def from_file(self, log_file: str | Path):
|
|
130
|
+
"""Creates a LogFile object from a file path."""
|
|
131
|
+
with open(log_file, "r", encoding="utf-8") as lf:
|
|
132
|
+
return self.from_list(lf.readlines()) # type: ignore
|
|
133
|
+
|
|
134
|
+
def from_list(self, lines: list[str] | list[LogLine]):
|
|
135
|
+
"""Creates a LogFile object from a list of lines."""
|
|
136
|
+
|
|
137
|
+
def maybe_convert(line, n=-1):
|
|
138
|
+
if isinstance(line, str):
|
|
139
|
+
return LogLine(line.strip(), n=n)
|
|
140
|
+
if isinstance(line, LogLine):
|
|
141
|
+
return line
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
_logfile = [
|
|
145
|
+
converted
|
|
146
|
+
for n, line in enumerate(lines)
|
|
147
|
+
if (converted := maybe_convert(line, n=n))
|
|
148
|
+
]
|
|
149
|
+
return _logfile
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def lines(self):
|
|
153
|
+
return self._content
|
|
154
|
+
|
|
155
|
+
def vectorize(
|
|
156
|
+
self,
|
|
157
|
+
tokenize: bool = True,
|
|
158
|
+
parallel: bool = False,
|
|
159
|
+
/,
|
|
160
|
+
**kwargs,
|
|
161
|
+
) -> None:
|
|
162
|
+
"""Convert the log file into an array of floating-point numbers.
|
|
163
|
+
|
|
164
|
+
Applies the vectorization mechanism to the log file and creates an array
|
|
165
|
+
of numbers.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
tokenize : bool, optional
|
|
170
|
+
Flag controls if the log line should be split into tokens before
|
|
171
|
+
vectorization, by default True
|
|
172
|
+
parallel : bool, optional
|
|
173
|
+
Flag controlling if the vectorization process is to be executed in
|
|
174
|
+
parallel, using ProcessPoolExecutor. Defaults to False.
|
|
175
|
+
|
|
176
|
+
Returns
|
|
177
|
+
-------
|
|
178
|
+
SliceableArray
|
|
179
|
+
Instance of the `SliceableArray` class with the calculated vectors.
|
|
180
|
+
|
|
181
|
+
Raises
|
|
182
|
+
------
|
|
183
|
+
ValueError
|
|
184
|
+
Requested an invalid (negative or zero) number of parallel workers.
|
|
185
|
+
"""
|
|
186
|
+
if parallel:
|
|
187
|
+
n_workers = kwargs.get("n_workers", 4)
|
|
188
|
+
if n_workers <= 0:
|
|
189
|
+
raise ValueError(f"Cannot split the data into {n_workers} pieces.")
|
|
190
|
+
if n_workers == 1:
|
|
191
|
+
batches = self._content
|
|
192
|
+
|
|
193
|
+
split_size = len(self._content) // n_workers
|
|
194
|
+
batches = [
|
|
195
|
+
self._content[i * split_size : (i + 1) * split_size]
|
|
196
|
+
for i in range(n_workers)
|
|
197
|
+
]
|
|
198
|
+
|
|
199
|
+
with ProcessPoolExecutor(max_workers=n_workers) as pool:
|
|
200
|
+
results = pool.map(Vectorizer(tokenize), batches)
|
|
201
|
+
|
|
202
|
+
scores = []
|
|
203
|
+
for result in list(results):
|
|
204
|
+
scores.extend(result)
|
|
205
|
+
|
|
206
|
+
else:
|
|
207
|
+
v = Vectorizer(tokenize)
|
|
208
|
+
if tokenize:
|
|
209
|
+
scores = [
|
|
210
|
+
[v._vectorize(token) for token in line.split(" ")] for line in self
|
|
211
|
+
]
|
|
212
|
+
else:
|
|
213
|
+
scores = [[v._vectorize(line)] for line in self]
|
|
214
|
+
|
|
215
|
+
self._vectorized = SliceableArray(scores)
|
|
216
|
+
|
|
217
|
+
def find_similar(
|
|
218
|
+
self, index: int | slice, step: int = 0, q: float = 0.001
|
|
219
|
+
) -> list[int]:
|
|
220
|
+
"""Find log entries similar to the one at the given index.
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
index : int | slice
|
|
225
|
+
Index of the log entry to compare. When using slice, it must be
|
|
226
|
+
contiguous.
|
|
227
|
+
step : int, optional
|
|
228
|
+
How many rows to step over when iterating over the log file's lines.
|
|
229
|
+
Defaults to 0, in which case, size of the reference chunk will be
|
|
230
|
+
used.
|
|
231
|
+
q : float, optional
|
|
232
|
+
Value of quantile used to determine if a chunk is to be considered
|
|
233
|
+
similar to the reference.
|
|
234
|
+
|
|
235
|
+
Returns
|
|
236
|
+
-------
|
|
237
|
+
list[int]
|
|
238
|
+
List of indices of similar log entries.
|
|
239
|
+
|
|
240
|
+
Raises
|
|
241
|
+
------
|
|
242
|
+
ValueError
|
|
243
|
+
If the provided index is not contiguous - slice step is not 1.
|
|
244
|
+
|
|
245
|
+
AttributeError
|
|
246
|
+
When attempting to run similarity check on a log file which hasn't
|
|
247
|
+
been vectorized yet. Normally shouldn't happen, as the vectorization
|
|
248
|
+
is done automatically in the init.
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
# Slide through the entire scores array, line by line, comparing each
|
|
252
|
+
# chunk to the reference. Skip comparing the chunk at `index`. Fun part:
|
|
253
|
+
# we can do this in parallel!
|
|
254
|
+
# Comparison extracted to a separate function to catch exceptions,
|
|
255
|
+
# reporting the local variables from the list comprehension.
|
|
256
|
+
if not self._vectorized:
|
|
257
|
+
raise AttributeError(
|
|
258
|
+
"LogFile has not been vectorized yet. Run vectorize() first."
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
if isinstance(index, slice) and index.step is not None and index.step > 1:
|
|
262
|
+
raise ValueError("Only contiguous slices are supported.")
|
|
263
|
+
|
|
264
|
+
if isinstance(index, slice) and (not index.start or not index.stop):
|
|
265
|
+
raise ValueError("Can't calculate similarity to a slice without bounds.")
|
|
266
|
+
|
|
267
|
+
if isinstance(index, int):
|
|
268
|
+
index = slice(index, index + 1)
|
|
269
|
+
chunk_len = index.stop - index.start
|
|
270
|
+
|
|
271
|
+
# Vectorized reference chunk (the one we're looking for similar entries to)
|
|
272
|
+
# At this point, index should be a slice regardless of the initial type.
|
|
273
|
+
ref: SliceableArray = self._vectorized[index]
|
|
274
|
+
if not ref:
|
|
275
|
+
raise ValueError("Trying to compare to an empty array.")
|
|
276
|
+
|
|
277
|
+
# Compute similarity scores between the reference and the rest of the
|
|
278
|
+
# file.
|
|
279
|
+
chunk_step = step if step > 0 else chunk_len
|
|
280
|
+
|
|
281
|
+
# Assume it's "quick" mode whenever the step is greater than 1.
|
|
282
|
+
quick = chunk_step != 1
|
|
283
|
+
similarities = [
|
|
284
|
+
chunk % ref
|
|
285
|
+
for chunk in self._vectorized.chunks(chunk_len, stride=chunk_step)
|
|
286
|
+
]
|
|
287
|
+
threshold = np.quantile(similarities, q, axis=0)
|
|
288
|
+
|
|
289
|
+
similar_indices = [
|
|
290
|
+
idx
|
|
291
|
+
for idx, score in enumerate(similarities)
|
|
292
|
+
# Don't compare with the reference chunk...
|
|
293
|
+
if slice(idx, idx + chunk_len) != index
|
|
294
|
+
# ...don't go out of bounds...
|
|
295
|
+
and (
|
|
296
|
+
(idx + 1) * chunk_len < len(self._vectorized)
|
|
297
|
+
if quick
|
|
298
|
+
else idx + chunk_len < len(self._vectorized)
|
|
299
|
+
)
|
|
300
|
+
and score[0] < threshold[0]
|
|
301
|
+
and score[1] < threshold[1]
|
|
302
|
+
]
|
|
303
|
+
return similar_indices
|
|
304
|
+
|
|
305
|
+
def find_distinctive(
|
|
306
|
+
self,
|
|
307
|
+
**kwargs,
|
|
308
|
+
) -> "list[LogFile]":
|
|
309
|
+
"""Find log entries with prominent peaks.
|
|
310
|
+
|
|
311
|
+
Method helps find relevant information in a noisy log file. It uses a
|
|
312
|
+
median filter to reduce the noise (controlled by the `kernel_size`), and
|
|
313
|
+
then finds peaks in the resulting signal (controlled by the `distance`,
|
|
314
|
+
`prominence` and `height` parameters).
|
|
315
|
+
|
|
316
|
+
Returns a list of log entries corresponding to the detected peaks, with
|
|
317
|
+
added lines of context.
|
|
318
|
+
|
|
319
|
+
If `prominence` or `height` is not provided, it will be automatically
|
|
320
|
+
calculated based on the peak-to-peak range of the signal.
|
|
321
|
+
|
|
322
|
+
Parameters
|
|
323
|
+
----------
|
|
324
|
+
log_file : LogFile
|
|
325
|
+
Log file to find peaks in.
|
|
326
|
+
context_size : int, optional
|
|
327
|
+
Context size adds additional lines around the detected peaks.
|
|
328
|
+
Default is 5.
|
|
329
|
+
kernel_size : int, optional
|
|
330
|
+
Kernel size to use when de-noising the log file, by default 5.
|
|
331
|
+
distance : int, optional
|
|
332
|
+
Minimal distance between peaks, by default 10.
|
|
333
|
+
prominence : float, optional
|
|
334
|
+
Prominence to use when finding peaks. When not set, will be
|
|
335
|
+
automatically determined from the signal's parameters. By default
|
|
336
|
+
None.
|
|
337
|
+
height : float, optional
|
|
338
|
+
Minimal distance between peak and surrounding samples. When not set,
|
|
339
|
+
will be automatically determined from the signal's parameters. By
|
|
340
|
+
default None.
|
|
341
|
+
fn : Callable, optional
|
|
342
|
+
User-defined function for reducing the vectorized log file into a 1D
|
|
343
|
+
series. Must follow signature `fn(data: SliceableArray) ->
|
|
344
|
+
list[float] | SliceableArray`.
|
|
345
|
+
If not provided, square root of sum will be used.
|
|
346
|
+
|
|
347
|
+
Returns
|
|
348
|
+
-------
|
|
349
|
+
list[LogFile]
|
|
350
|
+
List of detected peaks.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
def rms(data):
|
|
354
|
+
return [np.sqrt(np.sum(r)) for r in data]
|
|
355
|
+
|
|
356
|
+
kernel_size: int = kwargs.get("kernel_size", 1)
|
|
357
|
+
prominence: int = kwargs.get("prominence", None)
|
|
358
|
+
distance: int = kwargs.get("distance", 1)
|
|
359
|
+
height: float | None = kwargs.get("height", None)
|
|
360
|
+
fn: Callable | None = kwargs.get("fn", rms)
|
|
361
|
+
|
|
362
|
+
# Make sure the kernel_size is an odd number
|
|
363
|
+
if kernel_size % 2 == 0:
|
|
364
|
+
kernel_size += 1
|
|
365
|
+
|
|
366
|
+
# Re-calculate scores without tokenization to create a 1D array
|
|
367
|
+
# (that's why tokenize=False)
|
|
368
|
+
reduced = fn(self._vectorized.data)
|
|
369
|
+
normalized = np.abs(reduced - np.mean(reduced))
|
|
370
|
+
# Clean up the noise from the signal.
|
|
371
|
+
# TODO: Re-implement in numpy to avoid scipy dependency
|
|
372
|
+
denoised = scipy.signal.medfilt(normalized, kernel_size=kernel_size)
|
|
373
|
+
auto_params = LogSignalProcessor.tune_peak_detection(denoised)
|
|
374
|
+
height = height if height is not None else auto_params[0]
|
|
375
|
+
prominence = prominence if prominence is not None else auto_params[1]
|
|
376
|
+
|
|
377
|
+
peaks = [
|
|
378
|
+
int(peak)
|
|
379
|
+
for peak in scipy.signal.find_peaks(
|
|
380
|
+
denoised, prominence=prominence, distance=distance, height=height
|
|
381
|
+
)[0]
|
|
382
|
+
]
|
|
383
|
+
# return [self[i - context_size : i + context_size + 1] for i in peaks]
|
|
384
|
+
return peaks
|
|
385
|
+
|
|
386
|
+
@staticmethod
|
|
387
|
+
def split_by(func):
|
|
388
|
+
def wrapper(self, data, by, /, **kwargs):
|
|
389
|
+
buckets = LogFileSplit()
|
|
390
|
+
for line in data:
|
|
391
|
+
bucket: str = func(self, line, by, **kwargs)
|
|
392
|
+
if bucket not in buckets:
|
|
393
|
+
buckets[bucket] = []
|
|
394
|
+
buckets[bucket].append(line)
|
|
395
|
+
return buckets
|
|
396
|
+
|
|
397
|
+
return wrapper
|
|
398
|
+
|
|
399
|
+
@split_by
|
|
400
|
+
def split_by_token(self, line, by: int, /, delimiter=" ") -> str:
|
|
401
|
+
# Some lines might be shorter and not have the listed index
|
|
402
|
+
try:
|
|
403
|
+
bucket = line.split(delimiter)[by]
|
|
404
|
+
except IndexError:
|
|
405
|
+
bucket = LogFileSplit.OUTLIERS
|
|
406
|
+
return bucket
|
|
407
|
+
|
|
408
|
+
@split_by
|
|
409
|
+
def split_by_pattern(self, line, by: re.Pattern) -> str:
|
|
410
|
+
# default bucket for the unmatched strings
|
|
411
|
+
m: re.Match = by.match(str(line))
|
|
412
|
+
if m is None:
|
|
413
|
+
bucket = LogFileSplit.OUTLIERS
|
|
414
|
+
else:
|
|
415
|
+
bucket = m.group(1)
|
|
416
|
+
return bucket
|
|
417
|
+
|
|
418
|
+
@split_by
|
|
419
|
+
def split_by_index(self, line, by: tuple[int, int]) -> str:
|
|
420
|
+
start, stop = by
|
|
421
|
+
bucket = line[start:stop]
|
|
422
|
+
if bucket == "":
|
|
423
|
+
# Indexing with a slice won't raise exceptions, just return an empty string.
|
|
424
|
+
bucket = LogFileSplit.OUTLIERS
|
|
425
|
+
return bucket
|
|
426
|
+
|
|
427
|
+
@overload
|
|
428
|
+
def unravel(self, by: int) -> LogFileSplit: ...
|
|
429
|
+
@overload
|
|
430
|
+
def unravel(self, by: str) -> LogFileSplit: ...
|
|
431
|
+
@overload
|
|
432
|
+
def unravel(self, by: tuple[int, int]) -> LogFileSplit: ...
|
|
433
|
+
def unravel(self, *args, **kwargs) -> LogFileSplit:
|
|
434
|
+
"""Splits an interleaved log file by unique values of the selected log
|
|
435
|
+
file fragment.
|
|
436
|
+
|
|
437
|
+
Assuming a log file is a semi-structured document, parts of each line
|
|
438
|
+
may contain a set of unique values, for example: log severity
|
|
439
|
+
information, thread ID, etc. `unravel()` groups the log file's lines
|
|
440
|
+
into buckets, with each bucket representing on of the unique values from
|
|
441
|
+
the set.
|
|
442
|
+
|
|
443
|
+
Parameters
|
|
444
|
+
----------
|
|
445
|
+
by : int | str | tuple[int, int]
|
|
446
|
+
Definition of the log file field to split the file by.
|
|
447
|
+
|
|
448
|
+
When the argument has type `int`, it is treated as index of the
|
|
449
|
+
token (group of characters separated from others by a delimiter
|
|
450
|
+
character).
|
|
451
|
+
|
|
452
|
+
When the argument is string, it is treated as a regular expression
|
|
453
|
+
pattern to match. The pattern must define exactly one match group.
|
|
454
|
+
|
|
455
|
+
When the argument is a two-value tuple, it is interpreted as indices
|
|
456
|
+
for the log line slice.
|
|
457
|
+
|
|
458
|
+
Returns
|
|
459
|
+
-------
|
|
460
|
+
LogFileSplit
|
|
461
|
+
`dict`-like object with keys storing the detected buckets and
|
|
462
|
+
instances of `LogFile` as values.
|
|
463
|
+
|
|
464
|
+
Raises
|
|
465
|
+
------
|
|
466
|
+
ValueError
|
|
467
|
+
Value of the `by` parameter is incorrect; pattern is invalid, or the
|
|
468
|
+
tuple is not a two-value.
|
|
469
|
+
PatternGroupError
|
|
470
|
+
Supplied pattern string does not define exactly one match group.
|
|
471
|
+
TypeError
|
|
472
|
+
Type of the `by` parameter cannot be used to split the log file.
|
|
473
|
+
"""
|
|
474
|
+
by = kwargs.get("by", args[0] if args else None)
|
|
475
|
+
|
|
476
|
+
if isinstance(by, tuple):
|
|
477
|
+
assert len(by) == 2, ValueError("Argument must be a two-value tuple")
|
|
478
|
+
buckets = self.split_by_index(self._content, by)
|
|
479
|
+
|
|
480
|
+
elif isinstance(by, str):
|
|
481
|
+
try:
|
|
482
|
+
pattern = re.compile(by)
|
|
483
|
+
except re.PatternError as e:
|
|
484
|
+
raise ValueError(f"Incorrect regular expression pattern: {e}") from e
|
|
485
|
+
if pattern.groups != 1:
|
|
486
|
+
raise PatternGroupError("The pattern must define exactly one group.")
|
|
487
|
+
buckets = self.split_by_pattern(self._content, pattern)
|
|
488
|
+
|
|
489
|
+
elif isinstance(by, int):
|
|
490
|
+
buckets = self.split_by_token(self._content, by)
|
|
491
|
+
else:
|
|
492
|
+
raise TypeError(f"Can't split by {type(by)} with value {by}")
|
|
493
|
+
|
|
494
|
+
for bucket, lines in buckets.items():
|
|
495
|
+
# TODO: This probably is expensive in terms of extra memcpy operations
|
|
496
|
+
# maybe convert in place?
|
|
497
|
+
buckets[bucket] = LogFile(lines)
|
|
498
|
+
return buckets
|
|
499
|
+
|
|
500
|
+
def merge(self): ...
|
|
File without changes
|