logreducer 3.5.1__tar.gz → 3.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logreducer-3.5.1 → logreducer-3.5.2}/PKG-INFO +5 -5
- {logreducer-3.5.1 → logreducer-3.5.2}/README.md +1 -1
- {logreducer-3.5.1 → logreducer-3.5.2}/pyproject.toml +26 -12
- {logreducer-3.5.1 → logreducer-3.5.2}/pyproject.toml.orig +38 -12
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/anomaly.py +3 -6
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/core.py +65 -16
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/logging_config.py +2 -1
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/memory.py +10 -12
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/patterns.py +8 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sinks.py +10 -1
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/temporal.py +21 -27
- {logreducer-3.5.1 → logreducer-3.5.2}/LICENSE +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/NOTICE +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/__init__.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/cli.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/clickhouse.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/config.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/kafka.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/py.typed +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sampling.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sources.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sql.py +0 -0
- {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/target.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logreducer
|
|
3
|
-
Version: 3.5.
|
|
3
|
+
Version: 3.5.2
|
|
4
4
|
Summary: Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection
|
|
5
5
|
Keywords: log,logging,reduction,analysis,pattern-extraction,memory-efficient,streaming,anomaly-detection
|
|
6
6
|
Author: HyperI Team
|
|
@@ -22,10 +22,10 @@ Requires-Dist: psutil>=7.2.0
|
|
|
22
22
|
Requires-Dist: loguru>=0.7.3
|
|
23
23
|
Requires-Dist: numpy>=2.0.0
|
|
24
24
|
Requires-Dist: scikit-learn>=1.5.0
|
|
25
|
-
Requires-Dist: typer>=0.
|
|
25
|
+
Requires-Dist: typer>=0.27.3
|
|
26
26
|
Requires-Dist: clickhouse-connect>=1.0 ; extra == 'clickhouse'
|
|
27
|
-
Requires-Dist: xxhash>=
|
|
28
|
-
Requires-Dist: datasketch>=
|
|
27
|
+
Requires-Dist: xxhash>=4.0.1 ; extra == 'enhanced'
|
|
28
|
+
Requires-Dist: datasketch>=2.0.0 ; extra == 'enhanced'
|
|
29
29
|
Requires-Dist: confluent-kafka>=2.14.0 ; extra == 'kafka'
|
|
30
30
|
Requires-Dist: sqlalchemy>=2.0.30 ; extra == 'sql'
|
|
31
31
|
Maintainer: HyperI Team
|
|
@@ -438,7 +438,7 @@ Individually: `uv run ruff format`, `uv run ruff check --fix`,
|
|
|
438
438
|
| Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) |
|
|
439
439
|
| Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) |
|
|
440
440
|
| Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) |
|
|
441
|
-
|
|
|
441
|
+
| Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) |
|
|
442
442
|
| Hand-edit the version | Land a conventional commit and let semantic-release do it | The old regime wrote `VERSION` by hand and took five commits to get tagging working. CONTRIBUTING makes it a standing rule |
|
|
443
443
|
|
|
444
444
|
### Where this sits
|
|
@@ -394,7 +394,7 @@ Individually: `uv run ruff format`, `uv run ruff check --fix`,
|
|
|
394
394
|
| Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) |
|
|
395
395
|
| Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) |
|
|
396
396
|
| Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) |
|
|
397
|
-
|
|
|
397
|
+
| Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) |
|
|
398
398
|
| Hand-edit the version | Land a conventional commit and let semantic-release do it | The old regime wrote `VERSION` by hand and took five commits to get tagging working. CONTRIBUTING makes it a standing rule |
|
|
399
399
|
|
|
400
400
|
### Where this sits
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "logreducer"
|
|
7
|
-
version = "3.5.
|
|
7
|
+
version = "3.5.2"
|
|
8
8
|
description = "Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -40,7 +40,7 @@ dependencies = [
|
|
|
40
40
|
"loguru>=0.7.3",
|
|
41
41
|
"numpy>=2.0.0",
|
|
42
42
|
"scikit-learn>=1.5.0",
|
|
43
|
-
"typer>=0.
|
|
43
|
+
"typer>=0.27.3",
|
|
44
44
|
]
|
|
45
45
|
|
|
46
46
|
[[project.authors]]
|
|
@@ -53,8 +53,8 @@ email = "dev@hyperi.io"
|
|
|
53
53
|
|
|
54
54
|
[project.optional-dependencies]
|
|
55
55
|
enhanced = [
|
|
56
|
-
"xxhash>=
|
|
57
|
-
"datasketch>=
|
|
56
|
+
"xxhash>=4.0.1",
|
|
57
|
+
"datasketch>=2.0.0",
|
|
58
58
|
]
|
|
59
59
|
sql = ["sqlalchemy>=2.0.30"]
|
|
60
60
|
clickhouse = ["clickhouse-connect>=1.0"]
|
|
@@ -72,6 +72,7 @@ logreducer = "logreducer.cli:main"
|
|
|
72
72
|
|
|
73
73
|
[tool.uv]
|
|
74
74
|
index-strategy = "unsafe-best-match"
|
|
75
|
+
exclude-newer = "7 days"
|
|
75
76
|
override-dependencies = []
|
|
76
77
|
|
|
77
78
|
[[tool.uv.index]]
|
|
@@ -150,6 +151,7 @@ ignore = [
|
|
|
150
151
|
fixable = ["ALL"]
|
|
151
152
|
unfixable = []
|
|
152
153
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
154
|
+
external = ["V"]
|
|
153
155
|
|
|
154
156
|
[tool.ruff.lint.per-file-ignores]
|
|
155
157
|
"tests/**" = [
|
|
@@ -164,6 +166,15 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
164
166
|
"PT012",
|
|
165
167
|
"RUF043",
|
|
166
168
|
]
|
|
169
|
+
"src/**" = [
|
|
170
|
+
"D105",
|
|
171
|
+
"D107",
|
|
172
|
+
]
|
|
173
|
+
"src/logreducer/sampling.py" = ["S608"]
|
|
174
|
+
"src/logreducer/clickhouse.py" = ["S608"]
|
|
175
|
+
"src/logreducer/core.py" = ["S311"]
|
|
176
|
+
"src/logreducer/memory.py" = ["S311"]
|
|
177
|
+
"src/logreducer/target.py" = ["S311"]
|
|
167
178
|
|
|
168
179
|
[tool.ruff.lint.pydocstyle]
|
|
169
180
|
convention = "google"
|
|
@@ -260,30 +271,33 @@ exclude = [
|
|
|
260
271
|
"scripts/",
|
|
261
272
|
]
|
|
262
273
|
|
|
274
|
+
[tool.vulture]
|
|
275
|
+
min_confidence = 80
|
|
276
|
+
|
|
263
277
|
[tool.vermin]
|
|
264
278
|
targets = ["3.12"]
|
|
265
279
|
no_tips = true
|
|
266
280
|
|
|
267
281
|
[tool.hatch.build.targets.sdist]
|
|
268
282
|
exclude = [
|
|
269
|
-
"/ci",
|
|
270
|
-
"/GEMINI.md",
|
|
271
283
|
"/hyperi-ai",
|
|
272
|
-
"/.windsurf",
|
|
273
|
-
"/CURSOR.md",
|
|
274
|
-
"/CLAUDE.md",
|
|
275
284
|
"/.claude",
|
|
276
285
|
"/.gemini",
|
|
286
|
+
"/CURSOR.md",
|
|
287
|
+
"/CLAUDE.md",
|
|
288
|
+
"/.cursor",
|
|
289
|
+
"/ci",
|
|
290
|
+
"/GEMINI.md",
|
|
277
291
|
"/.github/copilot-instructions.md",
|
|
278
292
|
"/STATE.md",
|
|
279
|
-
"/.
|
|
293
|
+
"/.windsurf",
|
|
280
294
|
]
|
|
281
295
|
|
|
282
296
|
[dependency-groups]
|
|
283
297
|
dev = [
|
|
284
298
|
"pytest>=9.0.0",
|
|
285
299
|
"pytest-cov>=7.0.0",
|
|
286
|
-
"ruff>=0.
|
|
300
|
+
"ruff>=0.16.10",
|
|
287
301
|
"ty>=0.0.34",
|
|
288
302
|
"mypy>=2.0",
|
|
289
303
|
"bandit[toml]>=1.7.0",
|
|
@@ -293,7 +307,7 @@ dev = [
|
|
|
293
307
|
"sqlalchemy>=2.0.30",
|
|
294
308
|
"clickhouse-connect>=1.0",
|
|
295
309
|
"confluent-kafka>=2.14.0",
|
|
296
|
-
"testcontainers[clickhouse,kafka,postgres,mysql]>=4.
|
|
310
|
+
"testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
|
|
297
311
|
"psycopg[binary]>=3.2",
|
|
298
312
|
"pymysql>=1.1.0",
|
|
299
313
|
"python-dotenv>=1.0.0",
|
|
@@ -6,7 +6,7 @@ build-backend = "uv_build"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "logreducer"
|
|
9
|
-
version = "3.5.
|
|
9
|
+
version = "3.5.2"
|
|
10
10
|
authors = [
|
|
11
11
|
{ name = "HyperI Team", email = "dev@hyperi.io" },
|
|
12
12
|
]
|
|
@@ -48,13 +48,13 @@ dependencies = [
|
|
|
48
48
|
"loguru>=0.7.3",
|
|
49
49
|
"numpy>=2.0.0",
|
|
50
50
|
"scikit-learn>=1.5.0",
|
|
51
|
-
"typer>=0.
|
|
51
|
+
"typer>=0.27.3",
|
|
52
52
|
]
|
|
53
53
|
|
|
54
54
|
[project.optional-dependencies]
|
|
55
55
|
enhanced = [
|
|
56
|
-
"xxhash>=
|
|
57
|
-
"datasketch>=
|
|
56
|
+
"xxhash>=4.0.1",
|
|
57
|
+
"datasketch>=2.0.0",
|
|
58
58
|
]
|
|
59
59
|
sql = [
|
|
60
60
|
"sqlalchemy>=2.0.30",
|
|
@@ -78,6 +78,7 @@ logreducer = "logreducer.cli:main"
|
|
|
78
78
|
|
|
79
79
|
[tool.uv]
|
|
80
80
|
index-strategy = "unsafe-best-match"
|
|
81
|
+
exclude-newer = "7 days"
|
|
81
82
|
override-dependencies = []
|
|
82
83
|
index = [
|
|
83
84
|
{ name = "pypi", url = "https://pypi.org/simple", default = true },
|
|
@@ -160,6 +161,9 @@ fixable = [
|
|
|
160
161
|
]
|
|
161
162
|
unfixable = []
|
|
162
163
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
164
|
+
external = [
|
|
165
|
+
"V",
|
|
166
|
+
]
|
|
163
167
|
|
|
164
168
|
[tool.ruff.lint.per-file-ignores]
|
|
165
169
|
"tests/**" = [
|
|
@@ -174,6 +178,25 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
174
178
|
"PT012",
|
|
175
179
|
"RUF043",
|
|
176
180
|
]
|
|
181
|
+
"src/**" = [
|
|
182
|
+
"D105",
|
|
183
|
+
"D107",
|
|
184
|
+
]
|
|
185
|
+
"src/logreducer/sampling.py" = [
|
|
186
|
+
"S608",
|
|
187
|
+
]
|
|
188
|
+
"src/logreducer/clickhouse.py" = [
|
|
189
|
+
"S608",
|
|
190
|
+
]
|
|
191
|
+
"src/logreducer/core.py" = [
|
|
192
|
+
"S311",
|
|
193
|
+
]
|
|
194
|
+
"src/logreducer/memory.py" = [
|
|
195
|
+
"S311",
|
|
196
|
+
]
|
|
197
|
+
"src/logreducer/target.py" = [
|
|
198
|
+
"S311",
|
|
199
|
+
]
|
|
177
200
|
|
|
178
201
|
[tool.ruff.lint.pydocstyle]
|
|
179
202
|
convention = "google"
|
|
@@ -278,6 +301,9 @@ exclude = [
|
|
|
278
301
|
"scripts/",
|
|
279
302
|
]
|
|
280
303
|
|
|
304
|
+
[tool.vulture]
|
|
305
|
+
min_confidence = 80
|
|
306
|
+
|
|
281
307
|
[tool.vermin]
|
|
282
308
|
targets = [
|
|
283
309
|
"3.12",
|
|
@@ -286,24 +312,24 @@ no_tips = true
|
|
|
286
312
|
|
|
287
313
|
[tool.hatch.build.targets.sdist]
|
|
288
314
|
exclude = [
|
|
289
|
-
"/ci",
|
|
290
|
-
"/GEMINI.md",
|
|
291
315
|
"/hyperi-ai",
|
|
292
|
-
"/.windsurf",
|
|
293
|
-
"/CURSOR.md",
|
|
294
|
-
"/CLAUDE.md",
|
|
295
316
|
"/.claude",
|
|
296
317
|
"/.gemini",
|
|
318
|
+
"/CURSOR.md",
|
|
319
|
+
"/CLAUDE.md",
|
|
320
|
+
"/.cursor",
|
|
321
|
+
"/ci",
|
|
322
|
+
"/GEMINI.md",
|
|
297
323
|
"/.github/copilot-instructions.md",
|
|
298
324
|
"/STATE.md",
|
|
299
|
-
"/.
|
|
325
|
+
"/.windsurf",
|
|
300
326
|
]
|
|
301
327
|
|
|
302
328
|
[dependency-groups]
|
|
303
329
|
dev = [
|
|
304
330
|
"pytest>=9.0.0",
|
|
305
331
|
"pytest-cov>=7.0.0",
|
|
306
|
-
"ruff>=0.
|
|
332
|
+
"ruff>=0.16.10",
|
|
307
333
|
"ty>=0.0.34",
|
|
308
334
|
"mypy>=2.0",
|
|
309
335
|
"bandit[toml]>=1.7.0",
|
|
@@ -313,7 +339,7 @@ dev = [
|
|
|
313
339
|
"sqlalchemy>=2.0.30",
|
|
314
340
|
"clickhouse-connect>=1.0",
|
|
315
341
|
"confluent-kafka>=2.14.0",
|
|
316
|
-
"testcontainers[clickhouse,kafka,postgres,mysql]>=4.
|
|
342
|
+
"testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
|
|
317
343
|
"psycopg[binary]>=3.2",
|
|
318
344
|
"pymysql>=1.1.0",
|
|
319
345
|
"python-dotenv>=1.0.0",
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Anomaly detection utilities
|
|
3
|
-
"""
|
|
1
|
+
"""Anomaly detection utilities."""
|
|
4
2
|
|
|
5
3
|
from loguru import logger
|
|
6
4
|
|
|
@@ -15,7 +13,7 @@ except ImportError:
|
|
|
15
13
|
|
|
16
14
|
|
|
17
15
|
class AnomalyDetector:
|
|
18
|
-
"""Detect anomalous log lines"""
|
|
16
|
+
"""Detect anomalous log lines."""
|
|
19
17
|
|
|
20
18
|
def __init__(self, contamination: float = 0.1):
|
|
21
19
|
self.contamination = contamination
|
|
@@ -25,8 +23,7 @@ class AnomalyDetector:
|
|
|
25
23
|
logger.warning("scikit-learn not available, anomaly detection disabled")
|
|
26
24
|
|
|
27
25
|
def detect_anomalies(self, lines: list[str]) -> tuple[list[str], list[str]]:
|
|
28
|
-
"""
|
|
29
|
-
Separate anomalous and normal lines
|
|
26
|
+
"""Separate anomalous and normal lines.
|
|
30
27
|
|
|
31
28
|
Returns:
|
|
32
29
|
(anomalous_lines, normal_lines)
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Core LogReducer implementation
|
|
3
|
-
"""
|
|
1
|
+
"""Core LogReducer implementation."""
|
|
4
2
|
|
|
5
3
|
import json
|
|
6
4
|
import os
|
|
@@ -10,7 +8,7 @@ from dataclasses import replace
|
|
|
10
8
|
from datetime import datetime
|
|
11
9
|
from enum import Enum
|
|
12
10
|
from pathlib import Path
|
|
13
|
-
from typing import Any
|
|
11
|
+
from typing import Any, Literal, overload
|
|
14
12
|
|
|
15
13
|
from .anomaly import AnomalyDetector
|
|
16
14
|
from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode, get_preset_config
|
|
@@ -24,8 +22,7 @@ from .temporal import TemporalProcessor
|
|
|
24
22
|
|
|
25
23
|
|
|
26
24
|
class LogReducer:
|
|
27
|
-
"""
|
|
28
|
-
Main log reduction class
|
|
25
|
+
"""Main log reduction class.
|
|
29
26
|
|
|
30
27
|
Example:
|
|
31
28
|
reducer = LogReducer(level="enhanced")
|
|
@@ -41,8 +38,7 @@ class LogReducer:
|
|
|
41
38
|
config: BigDialConfig | None = None,
|
|
42
39
|
**kwargs: Any,
|
|
43
40
|
) -> None:
|
|
44
|
-
"""
|
|
45
|
-
Initialize LogReducer
|
|
41
|
+
"""Initialize LogReducer.
|
|
46
42
|
|
|
47
43
|
Args:
|
|
48
44
|
level: Processing level (standard/enhanced/maximum)
|
|
@@ -134,13 +130,41 @@ class LogReducer:
|
|
|
134
130
|
else:
|
|
135
131
|
self.temporal_processor = None
|
|
136
132
|
|
|
133
|
+
@overload
|
|
134
|
+
def reduce(
|
|
135
|
+
self,
|
|
136
|
+
source: Source,
|
|
137
|
+
output_file: str | None = None,
|
|
138
|
+
return_metadata: Literal[False] = False,
|
|
139
|
+
sink: Sink | None = None,
|
|
140
|
+
) -> list[str]: ...
|
|
141
|
+
|
|
142
|
+
@overload
|
|
143
|
+
def reduce(
|
|
144
|
+
self,
|
|
145
|
+
source: Source,
|
|
146
|
+
output_file: str | None = None,
|
|
147
|
+
*,
|
|
148
|
+
return_metadata: Literal[True],
|
|
149
|
+
sink: Sink | None = None,
|
|
150
|
+
) -> dict[str, Any]: ...
|
|
151
|
+
|
|
152
|
+
@overload
|
|
137
153
|
def reduce(
|
|
138
154
|
self,
|
|
139
155
|
source: Source,
|
|
140
156
|
output_file: str | None = None,
|
|
141
157
|
return_metadata: bool = False,
|
|
142
158
|
sink: Sink | None = None,
|
|
143
|
-
) -> list[str] | dict:
|
|
159
|
+
) -> list[str] | dict[str, Any]: ...
|
|
160
|
+
|
|
161
|
+
def reduce(
|
|
162
|
+
self,
|
|
163
|
+
source: Source,
|
|
164
|
+
output_file: str | None = None,
|
|
165
|
+
return_metadata: bool = False,
|
|
166
|
+
sink: Sink | None = None,
|
|
167
|
+
) -> list[str] | dict[str, Any]:
|
|
144
168
|
"""Reduce a source of log lines to a representative sample.
|
|
145
169
|
|
|
146
170
|
This is the core entry point: it operates on an abstraction, not on IO.
|
|
@@ -241,12 +265,37 @@ class LogReducer:
|
|
|
241
265
|
}
|
|
242
266
|
return result_lines
|
|
243
267
|
|
|
268
|
+
@overload
|
|
269
|
+
def process_file(
|
|
270
|
+
self,
|
|
271
|
+
input_file: str,
|
|
272
|
+
output_file: str | None = None,
|
|
273
|
+
return_metadata: Literal[False] = False,
|
|
274
|
+
) -> list[str]: ...
|
|
275
|
+
|
|
276
|
+
@overload
|
|
277
|
+
def process_file(
|
|
278
|
+
self,
|
|
279
|
+
input_file: str,
|
|
280
|
+
output_file: str | None = None,
|
|
281
|
+
*,
|
|
282
|
+
return_metadata: Literal[True],
|
|
283
|
+
) -> dict[str, Any]: ...
|
|
284
|
+
|
|
285
|
+
@overload
|
|
286
|
+
def process_file(
|
|
287
|
+
self,
|
|
288
|
+
input_file: str,
|
|
289
|
+
output_file: str | None = None,
|
|
290
|
+
return_metadata: bool = False,
|
|
291
|
+
) -> list[str] | dict[str, Any]: ...
|
|
292
|
+
|
|
244
293
|
def process_file(
|
|
245
294
|
self,
|
|
246
295
|
input_file: str,
|
|
247
296
|
output_file: str | None = None,
|
|
248
297
|
return_metadata: bool = False,
|
|
249
|
-
) -> list[str] | dict:
|
|
298
|
+
) -> list[str] | dict[str, Any]:
|
|
250
299
|
"""Reduce a log file - convenience wrapper over reduce() + FileSource.
|
|
251
300
|
|
|
252
301
|
Args:
|
|
@@ -302,7 +351,7 @@ class LogReducer:
|
|
|
302
351
|
return unique_lines
|
|
303
352
|
|
|
304
353
|
def _process_anomaly_mode(self, source: Source) -> list[str]:
|
|
305
|
-
"""Process using anomaly detection"""
|
|
354
|
+
"""Process using anomaly detection."""
|
|
306
355
|
if not self.anomaly_detector:
|
|
307
356
|
if self.config.enable_logging:
|
|
308
357
|
self.logger.warning("Anomaly detector not available, falling back to pattern mode")
|
|
@@ -329,7 +378,7 @@ class LogReducer:
|
|
|
329
378
|
return result
|
|
330
379
|
|
|
331
380
|
def _process_temporal_mode(self, source: Source) -> list[str]:
|
|
332
|
-
"""Process using temporal analysis"""
|
|
381
|
+
"""Process using temporal analysis."""
|
|
333
382
|
if not self.temporal_processor:
|
|
334
383
|
if self.config.enable_logging:
|
|
335
384
|
self.logger.warning("Temporal processor not available, falling back to pattern mode")
|
|
@@ -356,7 +405,7 @@ class LogReducer:
|
|
|
356
405
|
return result
|
|
357
406
|
|
|
358
407
|
def _process_hybrid_mode(self, source: Source) -> list[str]:
|
|
359
|
-
"""Process using combined approach"""
|
|
408
|
+
"""Process using combined approach."""
|
|
360
409
|
if self.config.enable_logging:
|
|
361
410
|
self.logger.info("Hybrid mode: combining pattern and anomaly detection")
|
|
362
411
|
|
|
@@ -400,7 +449,7 @@ class LogReducer:
|
|
|
400
449
|
}
|
|
401
450
|
|
|
402
451
|
def _save_output(self, lines: list[str], output_file: str) -> None:
|
|
403
|
-
"""Save output to file in specified format"""
|
|
452
|
+
"""Save output to file in specified format."""
|
|
404
453
|
output_path = Path(output_file)
|
|
405
454
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
406
455
|
|
|
@@ -453,7 +502,7 @@ class LogReducer:
|
|
|
453
502
|
self.logger.info(f"Metadata saved to {meta_file}")
|
|
454
503
|
|
|
455
504
|
def _print_summary(self) -> None:
|
|
456
|
-
"""Log processing summary"""
|
|
505
|
+
"""Log processing summary."""
|
|
457
506
|
self.logger.info("\n" + "=" * 60)
|
|
458
507
|
self.logger.info("LOG REDUCTION SUMMARY")
|
|
459
508
|
self.logger.info("=" * 60)
|
|
@@ -474,7 +523,7 @@ class LogReducer:
|
|
|
474
523
|
self.logger.info("=" * 60)
|
|
475
524
|
|
|
476
525
|
def estimate_processing(self, file_path: str) -> dict:
|
|
477
|
-
"""Estimate processing requirements before running"""
|
|
526
|
+
"""Estimate processing requirements before running."""
|
|
478
527
|
file_size = os.path.getsize(file_path)
|
|
479
528
|
file_size_gb = file_size / (1024**3)
|
|
480
529
|
|
|
@@ -26,7 +26,8 @@ from typing import TYPE_CHECKING
|
|
|
26
26
|
from loguru import logger
|
|
27
27
|
|
|
28
28
|
if TYPE_CHECKING:
|
|
29
|
-
|
|
29
|
+
# Read by get_logger's quoted return annotation, which vulture does not parse.
|
|
30
|
+
from loguru import Logger # noqa: V104
|
|
30
31
|
|
|
31
32
|
# Solarized palette (https://ethanschoonover.com/solarized/) for the console
|
|
32
33
|
# sink - readable on both light and dark terminal backgrounds.
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Memory management and monitoring utilities
|
|
3
|
-
"""
|
|
1
|
+
"""Memory management and monitoring utilities."""
|
|
4
2
|
|
|
5
3
|
import gc
|
|
6
4
|
import os
|
|
@@ -13,7 +11,7 @@ import psutil
|
|
|
13
11
|
|
|
14
12
|
|
|
15
13
|
class MemoryMonitor:
|
|
16
|
-
"""Monitor and control memory usage during processing"""
|
|
14
|
+
"""Monitor and control memory usage during processing."""
|
|
17
15
|
|
|
18
16
|
def __init__(self, max_memory_gb: float = 1.0):
|
|
19
17
|
self.max_memory_gb = max_memory_gb
|
|
@@ -57,7 +55,7 @@ class MemoryMonitor:
|
|
|
57
55
|
return current_gb, is_safe
|
|
58
56
|
|
|
59
57
|
def get_current_usage_gb(self) -> float:
|
|
60
|
-
"""Get current memory usage in GB"""
|
|
58
|
+
"""Get current memory usage in GB."""
|
|
61
59
|
current = self.process.memory_info().rss
|
|
62
60
|
# Update peak usage
|
|
63
61
|
if current > self.peak_usage_bytes:
|
|
@@ -65,7 +63,7 @@ class MemoryMonitor:
|
|
|
65
63
|
return float(current / (1024**3))
|
|
66
64
|
|
|
67
65
|
def get_peak_usage_gb(self) -> float:
|
|
68
|
-
"""Get peak memory usage in GB"""
|
|
66
|
+
"""Get peak memory usage in GB."""
|
|
69
67
|
# Update current peak if needed
|
|
70
68
|
current = self.process.memory_info().rss
|
|
71
69
|
if current > self.peak_usage_bytes:
|
|
@@ -84,12 +82,12 @@ class MemoryMonitor:
|
|
|
84
82
|
return bool(current > self.max_memory_bytes)
|
|
85
83
|
|
|
86
84
|
def reset(self) -> None:
|
|
87
|
-
"""Reset peak memory tracking"""
|
|
85
|
+
"""Reset peak memory tracking."""
|
|
88
86
|
current_usage = self.process.memory_info().rss
|
|
89
87
|
self.peak_usage_bytes = current_usage
|
|
90
88
|
|
|
91
89
|
def estimate_file_strategy(self, file_size_bytes: int) -> str:
|
|
92
|
-
"""Determine processing strategy"""
|
|
90
|
+
"""Determine processing strategy."""
|
|
93
91
|
file_size_gb = file_size_bytes / (1024**3)
|
|
94
92
|
|
|
95
93
|
if file_size_gb < self.max_memory_gb * 0.2:
|
|
@@ -101,14 +99,14 @@ class MemoryMonitor:
|
|
|
101
99
|
|
|
102
100
|
|
|
103
101
|
class StreamingProcessor:
|
|
104
|
-
"""Process large files with constant memory usage"""
|
|
102
|
+
"""Process large files with constant memory usage."""
|
|
105
103
|
|
|
106
104
|
def __init__(self, memory_monitor: MemoryMonitor):
|
|
107
105
|
self.memory_monitor = memory_monitor
|
|
108
106
|
self.chunk_size = memory_monitor.safe_chunk_size
|
|
109
107
|
|
|
110
108
|
def read_file_streaming(self, file_path: str) -> Iterator[str]:
|
|
111
|
-
"""Read file with minimal memory usage"""
|
|
109
|
+
"""Read file with minimal memory usage."""
|
|
112
110
|
file_size = os.path.getsize(file_path)
|
|
113
111
|
strategy = self.memory_monitor.estimate_file_strategy(file_size)
|
|
114
112
|
|
|
@@ -223,7 +221,7 @@ class BoundedDeduplicator:
|
|
|
223
221
|
self.stats = {"total": 0, "unique": 0, "duplicates": 0}
|
|
224
222
|
|
|
225
223
|
def is_duplicate(self, line: str) -> bool:
|
|
226
|
-
"""Check if line is duplicate"""
|
|
224
|
+
"""Check if line is duplicate."""
|
|
227
225
|
self.stats["total"] += 1
|
|
228
226
|
line_hash = self.hash_func(line)
|
|
229
227
|
|
|
@@ -241,7 +239,7 @@ class BoundedDeduplicator:
|
|
|
241
239
|
return False
|
|
242
240
|
|
|
243
241
|
def deduplicate_lines(self, lines: Iterable[str]) -> Iterator[str]:
|
|
244
|
-
"""Deduplicate a stream of lines"""
|
|
242
|
+
"""Deduplicate a stream of lines."""
|
|
245
243
|
for line in lines:
|
|
246
244
|
if not self.is_duplicate(line):
|
|
247
245
|
yield line
|
|
@@ -121,6 +121,14 @@ class PatternExtractor:
|
|
|
121
121
|
if len(pattern.examples) < self.config.examples_per_pattern:
|
|
122
122
|
pattern.examples.append(line)
|
|
123
123
|
|
|
124
|
+
# Later lines widen a cluster's slots to <*>, so the stored template is
|
|
125
|
+
# re-read once the feed ends; an LRU-evicted cluster keeps its snapshot.
|
|
126
|
+
clusters = self.miner.drain.id_to_cluster
|
|
127
|
+
for cluster_id, pattern in pattern_map.items():
|
|
128
|
+
cluster = clusters.get(cluster_id)
|
|
129
|
+
if cluster is not None:
|
|
130
|
+
pattern.template = cluster.get_template()
|
|
131
|
+
|
|
124
132
|
patterns = list(pattern_map.values())
|
|
125
133
|
|
|
126
134
|
# Apply filters
|
|
@@ -30,7 +30,8 @@ class Sink(Protocol):
|
|
|
30
30
|
iterate lazily rather than materialising the whole batch.
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
|
-
def write(self, lines: Iterable[str]) -> int:
|
|
33
|
+
def write(self, lines: Iterable[str]) -> int:
|
|
34
|
+
"""Write ``lines`` to the destination and return how many were written."""
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
class FileSink:
|
|
@@ -54,6 +55,14 @@ class FileSink:
|
|
|
54
55
|
self.output_format = fmt
|
|
55
56
|
|
|
56
57
|
def write(self, lines: Iterable[str]) -> int:
|
|
58
|
+
"""Write ``lines`` to ``path`` in the configured format, replacing the file.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
lines: The reduced lines, consumed once.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
The number of lines written.
|
|
65
|
+
"""
|
|
57
66
|
output_path = Path(self.path)
|
|
58
67
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
59
68
|
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Temporal processing and time-aware pattern extraction
|
|
3
|
-
"""
|
|
1
|
+
"""Temporal processing and time-aware pattern extraction."""
|
|
4
2
|
|
|
5
3
|
import re
|
|
6
4
|
from collections import defaultdict
|
|
@@ -13,7 +11,7 @@ from drain3 import TemplateMiner
|
|
|
13
11
|
|
|
14
12
|
@dataclass
|
|
15
13
|
class LogEntry:
|
|
16
|
-
"""Parsed log entry with timestamp"""
|
|
14
|
+
"""Parsed log entry with timestamp."""
|
|
17
15
|
|
|
18
16
|
raw_line: str
|
|
19
17
|
timestamp: datetime | None = None
|
|
@@ -22,7 +20,7 @@ class LogEntry:
|
|
|
22
20
|
|
|
23
21
|
|
|
24
22
|
class TimestampParser:
|
|
25
|
-
"""Parse timestamps from various formats"""
|
|
23
|
+
"""Parse timestamps from various formats."""
|
|
26
24
|
|
|
27
25
|
PATTERNS = [
|
|
28
26
|
(r"\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"),
|
|
@@ -34,7 +32,7 @@ class TimestampParser:
|
|
|
34
32
|
LEVEL_PATTERN = re.compile(r"\b(DEBUG|INFO|WARN|ERROR|CRITICAL)\b", re.I)
|
|
35
33
|
|
|
36
34
|
def parse_line(self, line: str, line_number: int = 0) -> LogEntry:
|
|
37
|
-
"""Parse log line"""
|
|
35
|
+
"""Parse log line."""
|
|
38
36
|
entry = LogEntry(raw_line=line.strip(), line_number=line_number)
|
|
39
37
|
|
|
40
38
|
# Extract timestamp
|
|
@@ -61,7 +59,7 @@ class TimestampParser:
|
|
|
61
59
|
|
|
62
60
|
|
|
63
61
|
class TemporalProcessor:
|
|
64
|
-
"""Time-aware log processing"""
|
|
62
|
+
"""Time-aware log processing."""
|
|
65
63
|
|
|
66
64
|
def __init__(self, window_minutes: int = 60):
|
|
67
65
|
self.window_minutes = window_minutes
|
|
@@ -69,7 +67,7 @@ class TemporalProcessor:
|
|
|
69
67
|
self.miners_by_window: dict[int, TemplateMiner] = {}
|
|
70
68
|
|
|
71
69
|
def process_temporal(self, lines: list[str]) -> dict:
|
|
72
|
-
"""Process with temporal awareness"""
|
|
70
|
+
"""Process with temporal awareness."""
|
|
73
71
|
entries = [self.parser.parse_line(line, i) for i, line in enumerate(lines)]
|
|
74
72
|
|
|
75
73
|
timed = [e for e in entries if e.timestamp]
|
|
@@ -91,7 +89,7 @@ class TemporalProcessor:
|
|
|
91
89
|
return results
|
|
92
90
|
|
|
93
91
|
def _extract_temporal_patterns(self, entries: list[LogEntry]) -> list[dict]:
|
|
94
|
-
"""Extract patterns with time context"""
|
|
92
|
+
"""Extract patterns with time context."""
|
|
95
93
|
patterns = []
|
|
96
94
|
windows = defaultdict(list)
|
|
97
95
|
|
|
@@ -112,11 +110,10 @@ class TemporalProcessor:
|
|
|
112
110
|
self.miners_by_window[window_key] = TemplateMiner(config=config)
|
|
113
111
|
|
|
114
112
|
miner = self.miners_by_window[window_key]
|
|
113
|
+
cluster_ids = [miner.add_log_message(entry.raw_line)["cluster_id"] for entry in window_entries]
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
cluster_id = result["cluster_id"]
|
|
119
|
-
|
|
115
|
+
# Templates are read once the window is fed, after later lines have widened their slots.
|
|
116
|
+
for entry, cluster_id in zip(window_entries, cluster_ids, strict=True):
|
|
120
117
|
patterns.append(
|
|
121
118
|
{
|
|
122
119
|
"template": miner.drain.id_to_cluster[cluster_id].get_template(),
|
|
@@ -129,29 +126,26 @@ class TemporalProcessor:
|
|
|
129
126
|
return patterns
|
|
130
127
|
|
|
131
128
|
def _extract_timeless_patterns(self, entries: list[LogEntry]) -> list[dict]:
|
|
132
|
-
"""Extract patterns without timestamps"""
|
|
129
|
+
"""Extract patterns without timestamps."""
|
|
133
130
|
from drain3.template_miner_config import TemplateMinerConfig
|
|
134
131
|
|
|
135
132
|
config = TemplateMinerConfig()
|
|
136
133
|
config.snapshot_interval_minutes = 0 # Disable snapshots
|
|
137
134
|
config.snapshot_compress_state = False
|
|
138
135
|
miner = TemplateMiner(config=config)
|
|
139
|
-
|
|
136
|
+
cluster_ids = [miner.add_log_message(entry.raw_line)["cluster_id"] for entry in entries]
|
|
140
137
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
)
|
|
150
|
-
|
|
151
|
-
return patterns
|
|
138
|
+
# Templates are read once every entry is fed, after later lines have widened their slots.
|
|
139
|
+
return [
|
|
140
|
+
{
|
|
141
|
+
"template": miner.drain.id_to_cluster[cluster_id].get_template(),
|
|
142
|
+
"example": entry.raw_line,
|
|
143
|
+
}
|
|
144
|
+
for entry, cluster_id in zip(entries, cluster_ids, strict=True)
|
|
145
|
+
]
|
|
152
146
|
|
|
153
147
|
def _analyze_distribution(self, entries: list[LogEntry]) -> dict:
|
|
154
|
-
"""Analyze time distribution"""
|
|
148
|
+
"""Analyze time distribution."""
|
|
155
149
|
dist: dict[str, Any] = {"hourly": defaultdict(int), "levels": defaultdict(int)}
|
|
156
150
|
|
|
157
151
|
for entry in entries:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|