logwarp 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
logwarp/__init__.py ADDED
@@ -0,0 +1,4 @@
1
+ try:
2
+ from logwarp._version import __version__
3
+ except ImportError: # package not installed (e.g. running from source without build)
4
+ __version__ = "0.0.0.dev0"
logwarp/_version.py ADDED
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '0.1.0'
22
+ __version_tuple__ = version_tuple = (0, 1, 0)
23
+
24
+ __commit_id__ = commit_id = None
logwarp/errors.py ADDED
@@ -0,0 +1,4 @@
1
+ # SPDX-FileCopyrightText: 2026 Lukasz Tuz <lukasz@bitsandpixels.io>
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ class PatternGroupError(BaseException): ...
logwarp/logfile.py ADDED
@@ -0,0 +1,500 @@
1
+ # SPDX-FileCopyrightText: 2026 Lukasz Tuz <lukasz@bitsandpixels.io>
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ """Implements a representation of a log file that can be searched and analyzed
5
+ by other modules.
6
+
7
+ This module provides a class, `LogFile`, that represents a log file as a list
8
+ of `LogLine` objects. The `LogFile` objects are subscriptable and iterable,
9
+ allowing for using integers, slices, and lists as indices.
10
+ """
11
+
12
+ import re
13
+ from collections import UserString
14
+ from collections.abc import Callable
15
+ from concurrent.futures import ProcessPoolExecutor
16
+ from pathlib import Path
17
+ from typing import overload
18
+
19
+ import numpy as np
20
+ import scipy
21
+
22
+ from logwarp.errors import PatternGroupError
23
+ from logwarp.utils import SliceableArray, Vectorizer
24
+
25
+
26
+ class LogSignalProcessor:
27
+ """Implements signal processing methods for log files."""
28
+
29
+ @staticmethod
30
+ def smooth(signal: np.ndarray, window_size: int = 10) -> np.ndarray:
31
+ """Smooths a signal convolution with a rectangular window."""
32
+ return np.convolve(signal, np.ones(window_size) / window_size, mode="valid")
33
+
34
+ @staticmethod
35
+ def tune_peak_detection(signal: np.ndarray) -> tuple[float, float]:
36
+ """Tunes parameters for peak detection in a signal.
37
+
38
+ Based on signals peak-to-peak value, calculates prominence and height
39
+ for `scipy.signal.find_peaks()`
40
+
41
+ Returns
42
+ -------
43
+ tuple[float, float]
44
+ Tuple of prominence and height
45
+ """
46
+ ptp = np.ptp(signal)
47
+ return 0.66 * ptp, 0.33 * ptp
48
+
49
+
50
+ class LogLine(UserString):
51
+ _number: int
52
+
53
+ def __init__(self, line: str, /, n: int = -1):
54
+ self._number = n
55
+ super().__init__(line)
56
+
57
+ @property
58
+ def number(self):
59
+ return self._number
60
+
61
+
62
+ class LogFileSplit(dict):
63
+ """Dict-like storage for split log file buckets. Guarantees existence of the
64
+ 'outliers' bucket."""
65
+
66
+ OUTLIERS = "outliers"
67
+
68
+ def __init__(self, *args, **kwargs):
69
+ super().__init__(*args, **kwargs)
70
+ self[LogFileSplit.OUTLIERS] = []
71
+
72
+
73
+ class LogFile:
74
+ _content: list[str | LogLine]
75
+ _vectorized: SliceableArray[float]
76
+ _hashes: dict
77
+
78
+ def __init__(self, log_file, **kwargs):
79
+ """Main class for interacting with a log file.
80
+
81
+ Implements indexing by integers (single line), slices, and lists (including
82
+ non-contiguous ranges of lines).
83
+
84
+ Parameters
85
+ ----------
86
+ log_file : str | Path | list | tuple
87
+ Path to the log file to load (`str` or `Path`) or content of a log
88
+ file in form of an iterable with strings.
89
+
90
+ Keyword Arguments
91
+ -----------------
92
+ kwargs : Any
93
+ Arguments passed directly to the `SliceableArray` constructor.
94
+
95
+ Examples
96
+ --------
97
+ >>> log_file = LogFile(["Line 1", "Line 2", "Line 3", "Line 4"])
98
+ >>> log_file[0]
99
+ 'Line 1'
100
+ >>> log_file[1:3] # Slice returns list of lists
101
+ ['Line 2', 'Line 3']
102
+ >>> log_file[[0, 2]]
103
+ ['Line 1', 'Line 3']
104
+ >>> log_file[::2]
105
+ ['Line 1', 'Line 3']
106
+ """
107
+ if isinstance(log_file, (str, Path)):
108
+ self._content = self.from_file(log_file)
109
+ elif isinstance(log_file, (list, tuple)):
110
+ self._content = self.from_list(log_file)
111
+ else:
112
+ raise TypeError(f"Don't know how to make a LogFile from {type(log_file)}")
113
+
114
+ # Store intermittent results, traces, etc. here
115
+ self._vectorized = SliceableArray([])
116
+ self._hashes = {}
117
+
118
+ def __getitem__(self, key):
119
+ if isinstance(key, list):
120
+ return [line for idx, line in enumerate(self) if idx in key]
121
+ return self._content[key]
122
+
123
+ def __setitem__(self, key, value):
124
+ raise AttributeError("LogFile is read only")
125
+
126
+ def __len__(self):
127
+ return len(self._content)
128
+
129
+ def from_file(self, log_file: str | Path):
130
+ """Creates a LogFile object from a file path."""
131
+ with open(log_file, "r", encoding="utf-8") as lf:
132
+ return self.from_list(lf.readlines()) # type: ignore
133
+
134
+ def from_list(self, lines: list[str] | list[LogLine]):
135
+ """Creates a LogFile object from a list of lines."""
136
+
137
+ def maybe_convert(line, n=-1):
138
+ if isinstance(line, str):
139
+ return LogLine(line.strip(), n=n)
140
+ if isinstance(line, LogLine):
141
+ return line
142
+ return None
143
+
144
+ _logfile = [
145
+ converted
146
+ for n, line in enumerate(lines)
147
+ if (converted := maybe_convert(line, n=n))
148
+ ]
149
+ return _logfile
150
+
151
+ @property
152
+ def lines(self):
153
+ return self._content
154
+
155
+ def vectorize(
156
+ self,
157
+ tokenize: bool = True,
158
+ parallel: bool = False,
159
+ /,
160
+ **kwargs,
161
+ ) -> None:
162
+ """Convert the log file into an array of floating-point numbers.
163
+
164
+ Applies the vectorization mechanism to the log file and creates an array
165
+ of numbers.
166
+
167
+ Parameters
168
+ ----------
169
+ tokenize : bool, optional
170
+ Flag controls if the log line should be split into tokens before
171
+ vectorization, by default True
172
+ parallel : bool, optional
173
+ Flag controlling if the vectorization process is to be executed in
174
+ parallel, using ProcessPoolExecutor. Defaults to False.
175
+
176
+ Returns
177
+ -------
178
+ SliceableArray
179
+ Instance of the `SliceableArray` class with the calculated vectors.
180
+
181
+ Raises
182
+ ------
183
+ ValueError
184
+ Requested an invalid (negative or zero) number of parallel workers.
185
+ """
186
+ if parallel:
187
+ n_workers = kwargs.get("n_workers", 4)
188
+ if n_workers <= 0:
189
+ raise ValueError(f"Cannot split the data into {n_workers} pieces.")
190
+ if n_workers == 1:
191
+ batches = self._content
192
+
193
+ split_size = len(self._content) // n_workers
194
+ batches = [
195
+ self._content[i * split_size : (i + 1) * split_size]
196
+ for i in range(n_workers)
197
+ ]
198
+
199
+ with ProcessPoolExecutor(max_workers=n_workers) as pool:
200
+ results = pool.map(Vectorizer(tokenize), batches)
201
+
202
+ scores = []
203
+ for result in list(results):
204
+ scores.extend(result)
205
+
206
+ else:
207
+ v = Vectorizer(tokenize)
208
+ if tokenize:
209
+ scores = [
210
+ [v._vectorize(token) for token in line.split(" ")] for line in self
211
+ ]
212
+ else:
213
+ scores = [[v._vectorize(line)] for line in self]
214
+
215
+ self._vectorized = SliceableArray(scores)
216
+
217
+ def find_similar(
218
+ self, index: int | slice, step: int = 0, q: float = 0.001
219
+ ) -> list[int]:
220
+ """Find log entries similar to the one at the given index.
221
+
222
+ Parameters
223
+ ----------
224
+ index : int | slice
225
+ Index of the log entry to compare. When using slice, it must be
226
+ contiguous.
227
+ step : int, optional
228
+ How many rows to step over when iterating over the log file's lines.
229
+ Defaults to 0, in which case, size of the reference chunk will be
230
+ used.
231
+ q : float, optional
232
+ Value of quantile used to determine if a chunk is to be considered
233
+ similar to the reference.
234
+
235
+ Returns
236
+ -------
237
+ list[int]
238
+ List of indices of similar log entries.
239
+
240
+ Raises
241
+ ------
242
+ ValueError
243
+ If the provided index is not contiguous - slice step is not 1.
244
+
245
+ AttributeError
246
+ When attempting to run similarity check on a log file which hasn't
247
+ been vectorized yet. Normally shouldn't happen, as the vectorization
248
+ is done automatically in the init.
249
+ """
250
+
251
+ # Slide through the entire scores array, line by line, comparing each
252
+ # chunk to the reference. Skip comparing the chunk at `index`. Fun part:
253
+ # we can do this in parallel!
254
+ # Comparison extracted to a separate function to catch exceptions,
255
+ # reporting the local variables from the list comprehension.
256
+ if not self._vectorized:
257
+ raise AttributeError(
258
+ "LogFile has not been vectorized yet. Run vectorize() first."
259
+ )
260
+
261
+ if isinstance(index, slice) and index.step is not None and index.step > 1:
262
+ raise ValueError("Only contiguous slices are supported.")
263
+
264
+ if isinstance(index, slice) and (not index.start or not index.stop):
265
+ raise ValueError("Can't calculate similarity to a slice without bounds.")
266
+
267
+ if isinstance(index, int):
268
+ index = slice(index, index + 1)
269
+ chunk_len = index.stop - index.start
270
+
271
+ # Vectorized reference chunk (the one we're looking for similar entries to)
272
+ # At this point, index should be a slice regardless of the initial type.
273
+ ref: SliceableArray = self._vectorized[index]
274
+ if not ref:
275
+ raise ValueError("Trying to compare to an empty array.")
276
+
277
+ # Compute similarity scores between the reference and the rest of the
278
+ # file.
279
+ chunk_step = step if step > 0 else chunk_len
280
+
281
+ # Assume it's "quick" mode whenever the step is greater than 1.
282
+ quick = chunk_step != 1
283
+ similarities = [
284
+ chunk % ref
285
+ for chunk in self._vectorized.chunks(chunk_len, stride=chunk_step)
286
+ ]
287
+ threshold = np.quantile(similarities, q, axis=0)
288
+
289
+ similar_indices = [
290
+ idx
291
+ for idx, score in enumerate(similarities)
292
+ # Don't compare with the reference chunk...
293
+ if slice(idx, idx + chunk_len) != index
294
+ # ...don't go out of bounds...
295
+ and (
296
+ (idx + 1) * chunk_len < len(self._vectorized)
297
+ if quick
298
+ else idx + chunk_len < len(self._vectorized)
299
+ )
300
+ and score[0] < threshold[0]
301
+ and score[1] < threshold[1]
302
+ ]
303
+ return similar_indices
304
+
305
+ def find_distinctive(
306
+ self,
307
+ **kwargs,
308
+ ) -> "list[LogFile]":
309
+ """Find log entries with prominent peaks.
310
+
311
+ Method helps find relevant information in a noisy log file. It uses a
312
+ median filter to reduce the noise (controlled by the `kernel_size`), and
313
+ then finds peaks in the resulting signal (controlled by the `distance`,
314
+ `prominence` and `height` parameters).
315
+
316
+ Returns a list of log entries corresponding to the detected peaks, with
317
+ added lines of context.
318
+
319
+ If `prominence` or `height` is not provided, it will be automatically
320
+ calculated based on the peak-to-peak range of the signal.
321
+
322
+ Parameters
323
+ ----------
324
+ log_file : LogFile
325
+ Log file to find peaks in.
326
+ context_size : int, optional
327
+ Context size adds additional lines around the detected peaks.
328
+ Default is 5.
329
+ kernel_size : int, optional
330
+ Kernel size to use when de-noising the log file, by default 5.
331
+ distance : int, optional
332
+ Minimal distance between peaks, by default 10.
333
+ prominence : float, optional
334
+ Prominence to use when finding peaks. When not set, will be
335
+ automatically determined from the signal's parameters. By default
336
+ None.
337
+ height : float, optional
338
+ Minimal distance between peak and surrounding samples. When not set,
339
+ will be automatically determined from the signal's parameters. By
340
+ default None.
341
+ fn : Callable, optional
342
+ User-defined function for reducing the vectorized log file into a 1D
343
+ series. Must follow signature `fn(data: SliceableArray) ->
344
+ list[float] | SliceableArray`.
345
+ If not provided, square root of sum will be used.
346
+
347
+ Returns
348
+ -------
349
+ list[LogFile]
350
+ List of detected peaks.
351
+ """
352
+
353
+ def rms(data):
354
+ return [np.sqrt(np.sum(r)) for r in data]
355
+
356
+ kernel_size: int = kwargs.get("kernel_size", 1)
357
+ prominence: int = kwargs.get("prominence", None)
358
+ distance: int = kwargs.get("distance", 1)
359
+ height: float | None = kwargs.get("height", None)
360
+ fn: Callable | None = kwargs.get("fn", rms)
361
+
362
+ # Make sure the kernel_size is an odd number
363
+ if kernel_size % 2 == 0:
364
+ kernel_size += 1
365
+
366
+ # Re-calculate scores without tokenization to create a 1D array
367
+ # (that's why tokenize=False)
368
+ reduced = fn(self._vectorized.data)
369
+ normalized = np.abs(reduced - np.mean(reduced))
370
+ # Clean up the noise from the signal.
371
+ # TODO: Re-implement in numpy to avoid scipy dependency
372
+ denoised = scipy.signal.medfilt(normalized, kernel_size=kernel_size)
373
+ auto_params = LogSignalProcessor.tune_peak_detection(denoised)
374
+ height = height if height is not None else auto_params[0]
375
+ prominence = prominence if prominence is not None else auto_params[1]
376
+
377
+ peaks = [
378
+ int(peak)
379
+ for peak in scipy.signal.find_peaks(
380
+ denoised, prominence=prominence, distance=distance, height=height
381
+ )[0]
382
+ ]
383
+ # return [self[i - context_size : i + context_size + 1] for i in peaks]
384
+ return peaks
385
+
386
+ @staticmethod
387
+ def split_by(func):
388
+ def wrapper(self, data, by, /, **kwargs):
389
+ buckets = LogFileSplit()
390
+ for line in data:
391
+ bucket: str = func(self, line, by, **kwargs)
392
+ if bucket not in buckets:
393
+ buckets[bucket] = []
394
+ buckets[bucket].append(line)
395
+ return buckets
396
+
397
+ return wrapper
398
+
399
+ @split_by
400
+ def split_by_token(self, line, by: int, /, delimiter=" ") -> str:
401
+ # Some lines might be shorter and not have the listed index
402
+ try:
403
+ bucket = line.split(delimiter)[by]
404
+ except IndexError:
405
+ bucket = LogFileSplit.OUTLIERS
406
+ return bucket
407
+
408
+ @split_by
409
+ def split_by_pattern(self, line, by: re.Pattern) -> str:
410
+ # default bucket for the unmatched strings
411
+ m: re.Match = by.match(str(line))
412
+ if m is None:
413
+ bucket = LogFileSplit.OUTLIERS
414
+ else:
415
+ bucket = m.group(1)
416
+ return bucket
417
+
418
+ @split_by
419
+ def split_by_index(self, line, by: tuple[int, int]) -> str:
420
+ start, stop = by
421
+ bucket = line[start:stop]
422
+ if bucket == "":
423
+ # Indexing with a slice won't raise exceptions, just return an empty string.
424
+ bucket = LogFileSplit.OUTLIERS
425
+ return bucket
426
+
427
+ @overload
428
+ def unravel(self, by: int) -> LogFileSplit: ...
429
+ @overload
430
+ def unravel(self, by: str) -> LogFileSplit: ...
431
+ @overload
432
+ def unravel(self, by: tuple[int, int]) -> LogFileSplit: ...
433
+ def unravel(self, *args, **kwargs) -> LogFileSplit:
434
+ """Splits an interleaved log file by unique values of the selected log
435
+ file fragment.
436
+
437
+ Assuming a log file is a semi-structured document, parts of each line
438
+ may contain a set of unique values, for example: log severity
439
+ information, thread ID, etc. `unravel()` groups the log file's lines
440
+ into buckets, with each bucket representing on of the unique values from
441
+ the set.
442
+
443
+ Parameters
444
+ ----------
445
+ by : int | str | tuple[int, int]
446
+ Definition of the log file field to split the file by.
447
+
448
+ When the argument has type `int`, it is treated as index of the
449
+ token (group of characters separated from others by a delimiter
450
+ character).
451
+
452
+ When the argument is string, it is treated as a regular expression
453
+ pattern to match. The pattern must define exactly one match group.
454
+
455
+ When the argument is a two-value tuple, it is interpreted as indices
456
+ for the log line slice.
457
+
458
+ Returns
459
+ -------
460
+ LogFileSplit
461
+ `dict`-like object with keys storing the detected buckets and
462
+ instances of `LogFile` as values.
463
+
464
+ Raises
465
+ ------
466
+ ValueError
467
+ Value of the `by` parameter is incorrect; pattern is invalid, or the
468
+ tuple is not a two-value.
469
+ PatternGroupError
470
+ Supplied pattern string does not define exactly one match group.
471
+ TypeError
472
+ Type of the `by` parameter cannot be used to split the log file.
473
+ """
474
+ by = kwargs.get("by", args[0] if args else None)
475
+
476
+ if isinstance(by, tuple):
477
+ assert len(by) == 2, ValueError("Argument must be a two-value tuple")
478
+ buckets = self.split_by_index(self._content, by)
479
+
480
+ elif isinstance(by, str):
481
+ try:
482
+ pattern = re.compile(by)
483
+ except re.PatternError as e:
484
+ raise ValueError(f"Incorrect regular expression pattern: {e}") from e
485
+ if pattern.groups != 1:
486
+ raise PatternGroupError("The pattern must define exactly one group.")
487
+ buckets = self.split_by_pattern(self._content, pattern)
488
+
489
+ elif isinstance(by, int):
490
+ buckets = self.split_by_token(self._content, by)
491
+ else:
492
+ raise TypeError(f"Can't split by {type(by)} with value {by}")
493
+
494
+ for bucket, lines in buckets.items():
495
+ # TODO: This probably is expensive in terms of extra memcpy operations
496
+ # maybe convert in place?
497
+ buckets[bucket] = LogFile(lines)
498
+ return buckets
499
+
500
+ def merge(self): ...
File without changes