logreducer 3.5.1__tar.gz → 3.5.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {logreducer-3.5.1 → logreducer-3.5.2}/PKG-INFO +5 -5
  2. {logreducer-3.5.1 → logreducer-3.5.2}/README.md +1 -1
  3. {logreducer-3.5.1 → logreducer-3.5.2}/pyproject.toml +26 -12
  4. {logreducer-3.5.1 → logreducer-3.5.2}/pyproject.toml.orig +38 -12
  5. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/anomaly.py +3 -6
  6. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/core.py +65 -16
  7. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/logging_config.py +2 -1
  8. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/memory.py +10 -12
  9. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/patterns.py +8 -0
  10. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sinks.py +10 -1
  11. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/temporal.py +21 -27
  12. {logreducer-3.5.1 → logreducer-3.5.2}/LICENSE +0 -0
  13. {logreducer-3.5.1 → logreducer-3.5.2}/NOTICE +0 -0
  14. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/__init__.py +0 -0
  15. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/cli.py +0 -0
  16. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/clickhouse.py +0 -0
  17. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/config.py +0 -0
  18. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/kafka.py +0 -0
  19. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/py.typed +0 -0
  20. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sampling.py +0 -0
  21. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sources.py +0 -0
  22. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/sql.py +0 -0
  23. {logreducer-3.5.1 → logreducer-3.5.2}/src/logreducer/target.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logreducer
3
- Version: 3.5.1
3
+ Version: 3.5.2
4
4
  Summary: Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection
5
5
  Keywords: log,logging,reduction,analysis,pattern-extraction,memory-efficient,streaming,anomaly-detection
6
6
  Author: HyperI Team
@@ -22,10 +22,10 @@ Requires-Dist: psutil>=7.2.0
22
22
  Requires-Dist: loguru>=0.7.3
23
23
  Requires-Dist: numpy>=2.0.0
24
24
  Requires-Dist: scikit-learn>=1.5.0
25
- Requires-Dist: typer>=0.25.0
25
+ Requires-Dist: typer>=0.27.3
26
26
  Requires-Dist: clickhouse-connect>=1.0 ; extra == 'clickhouse'
27
- Requires-Dist: xxhash>=3.0.0 ; extra == 'enhanced'
28
- Requires-Dist: datasketch>=1.5.0 ; extra == 'enhanced'
27
+ Requires-Dist: xxhash>=4.0.1 ; extra == 'enhanced'
28
+ Requires-Dist: datasketch>=2.0.0 ; extra == 'enhanced'
29
29
  Requires-Dist: confluent-kafka>=2.14.0 ; extra == 'kafka'
30
30
  Requires-Dist: sqlalchemy>=2.0.30 ; extra == 'sql'
31
31
  Maintainer: HyperI Team
@@ -438,7 +438,7 @@ Individually: `uv run ruff format`, `uv run ruff check --fix`,
438
438
  | Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) |
439
439
  | Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) |
440
440
  | Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) |
441
- | Read `LogPattern.template` as the final Drain3 template | Treat it as a first-seen snapshot until #5 lands | `extract_patterns` captures `cluster.get_template()` when a cluster FIRST appears, before later lines widen slots to `<*>`, so a varying cluster stores its raw first line. The `<*>`-count priority boost then runs on that stale snapshot (#5, open) |
441
+ | Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) |
442
442
  | Hand-edit the version | Land a conventional commit and let semantic-release do it | The old regime wrote `VERSION` by hand and took five commits to get tagging working. CONTRIBUTING makes it a standing rule |
443
443
 
444
444
  ### Where this sits
@@ -394,7 +394,7 @@ Individually: `uv run ruff format`, `uv run ruff check --fix`,
394
394
  | Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) |
395
395
  | Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) |
396
396
  | Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) |
397
- | Read `LogPattern.template` as the final Drain3 template | Treat it as a first-seen snapshot until #5 lands | `extract_patterns` captures `cluster.get_template()` when a cluster FIRST appears, before later lines widen slots to `<*>`, so a varying cluster stores its raw first line. The `<*>`-count priority boost then runs on that stale snapshot (#5, open) |
397
+ | Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) |
398
398
  | Hand-edit the version | Land a conventional commit and let semantic-release do it | The old regime wrote `VERSION` by hand and took five commits to get tagging working. CONTRIBUTING makes it a standing rule |
399
399
 
400
400
  ### Where this sits
@@ -4,7 +4,7 @@ build-backend = "uv_build"
4
4
 
5
5
  [project]
6
6
  name = "logreducer"
7
- version = "3.5.1"
7
+ version = "3.5.2"
8
8
  description = "Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -40,7 +40,7 @@ dependencies = [
40
40
  "loguru>=0.7.3",
41
41
  "numpy>=2.0.0",
42
42
  "scikit-learn>=1.5.0",
43
- "typer>=0.25.0",
43
+ "typer>=0.27.3",
44
44
  ]
45
45
 
46
46
  [[project.authors]]
@@ -53,8 +53,8 @@ email = "dev@hyperi.io"
53
53
 
54
54
  [project.optional-dependencies]
55
55
  enhanced = [
56
- "xxhash>=3.0.0",
57
- "datasketch>=1.5.0",
56
+ "xxhash>=4.0.1",
57
+ "datasketch>=2.0.0",
58
58
  ]
59
59
  sql = ["sqlalchemy>=2.0.30"]
60
60
  clickhouse = ["clickhouse-connect>=1.0"]
@@ -72,6 +72,7 @@ logreducer = "logreducer.cli:main"
72
72
 
73
73
  [tool.uv]
74
74
  index-strategy = "unsafe-best-match"
75
+ exclude-newer = "7 days"
75
76
  override-dependencies = []
76
77
 
77
78
  [[tool.uv.index]]
@@ -150,6 +151,7 @@ ignore = [
150
151
  fixable = ["ALL"]
151
152
  unfixable = []
152
153
  dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
154
+ external = ["V"]
153
155
 
154
156
  [tool.ruff.lint.per-file-ignores]
155
157
  "tests/**" = [
@@ -164,6 +166,15 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
164
166
  "PT012",
165
167
  "RUF043",
166
168
  ]
169
+ "src/**" = [
170
+ "D105",
171
+ "D107",
172
+ ]
173
+ "src/logreducer/sampling.py" = ["S608"]
174
+ "src/logreducer/clickhouse.py" = ["S608"]
175
+ "src/logreducer/core.py" = ["S311"]
176
+ "src/logreducer/memory.py" = ["S311"]
177
+ "src/logreducer/target.py" = ["S311"]
167
178
 
168
179
  [tool.ruff.lint.pydocstyle]
169
180
  convention = "google"
@@ -260,30 +271,33 @@ exclude = [
260
271
  "scripts/",
261
272
  ]
262
273
 
274
+ [tool.vulture]
275
+ min_confidence = 80
276
+
263
277
  [tool.vermin]
264
278
  targets = ["3.12"]
265
279
  no_tips = true
266
280
 
267
281
  [tool.hatch.build.targets.sdist]
268
282
  exclude = [
269
- "/ci",
270
- "/GEMINI.md",
271
283
  "/hyperi-ai",
272
- "/.windsurf",
273
- "/CURSOR.md",
274
- "/CLAUDE.md",
275
284
  "/.claude",
276
285
  "/.gemini",
286
+ "/CURSOR.md",
287
+ "/CLAUDE.md",
288
+ "/.cursor",
289
+ "/ci",
290
+ "/GEMINI.md",
277
291
  "/.github/copilot-instructions.md",
278
292
  "/STATE.md",
279
- "/.cursor",
293
+ "/.windsurf",
280
294
  ]
281
295
 
282
296
  [dependency-groups]
283
297
  dev = [
284
298
  "pytest>=9.0.0",
285
299
  "pytest-cov>=7.0.0",
286
- "ruff>=0.15.0",
300
+ "ruff>=0.16.10",
287
301
  "ty>=0.0.34",
288
302
  "mypy>=2.0",
289
303
  "bandit[toml]>=1.7.0",
@@ -293,7 +307,7 @@ dev = [
293
307
  "sqlalchemy>=2.0.30",
294
308
  "clickhouse-connect>=1.0",
295
309
  "confluent-kafka>=2.14.0",
296
- "testcontainers[clickhouse,kafka,postgres,mysql]>=4.14.0",
310
+ "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
297
311
  "psycopg[binary]>=3.2",
298
312
  "pymysql>=1.1.0",
299
313
  "python-dotenv>=1.0.0",
@@ -6,7 +6,7 @@ build-backend = "uv_build"
6
6
 
7
7
  [project]
8
8
  name = "logreducer"
9
- version = "3.5.1"
9
+ version = "3.5.2"
10
10
  authors = [
11
11
  { name = "HyperI Team", email = "dev@hyperi.io" },
12
12
  ]
@@ -48,13 +48,13 @@ dependencies = [
48
48
  "loguru>=0.7.3",
49
49
  "numpy>=2.0.0",
50
50
  "scikit-learn>=1.5.0",
51
- "typer>=0.25.0",
51
+ "typer>=0.27.3",
52
52
  ]
53
53
 
54
54
  [project.optional-dependencies]
55
55
  enhanced = [
56
- "xxhash>=3.0.0",
57
- "datasketch>=1.5.0",
56
+ "xxhash>=4.0.1",
57
+ "datasketch>=2.0.0",
58
58
  ]
59
59
  sql = [
60
60
  "sqlalchemy>=2.0.30",
@@ -78,6 +78,7 @@ logreducer = "logreducer.cli:main"
78
78
 
79
79
  [tool.uv]
80
80
  index-strategy = "unsafe-best-match"
81
+ exclude-newer = "7 days"
81
82
  override-dependencies = []
82
83
  index = [
83
84
  { name = "pypi", url = "https://pypi.org/simple", default = true },
@@ -160,6 +161,9 @@ fixable = [
160
161
  ]
161
162
  unfixable = []
162
163
  dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
164
+ external = [
165
+ "V",
166
+ ]
163
167
 
164
168
  [tool.ruff.lint.per-file-ignores]
165
169
  "tests/**" = [
@@ -174,6 +178,25 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
174
178
  "PT012",
175
179
  "RUF043",
176
180
  ]
181
+ "src/**" = [
182
+ "D105",
183
+ "D107",
184
+ ]
185
+ "src/logreducer/sampling.py" = [
186
+ "S608",
187
+ ]
188
+ "src/logreducer/clickhouse.py" = [
189
+ "S608",
190
+ ]
191
+ "src/logreducer/core.py" = [
192
+ "S311",
193
+ ]
194
+ "src/logreducer/memory.py" = [
195
+ "S311",
196
+ ]
197
+ "src/logreducer/target.py" = [
198
+ "S311",
199
+ ]
177
200
 
178
201
  [tool.ruff.lint.pydocstyle]
179
202
  convention = "google"
@@ -278,6 +301,9 @@ exclude = [
278
301
  "scripts/",
279
302
  ]
280
303
 
304
+ [tool.vulture]
305
+ min_confidence = 80
306
+
281
307
  [tool.vermin]
282
308
  targets = [
283
309
  "3.12",
@@ -286,24 +312,24 @@ no_tips = true
286
312
 
287
313
  [tool.hatch.build.targets.sdist]
288
314
  exclude = [
289
- "/ci",
290
- "/GEMINI.md",
291
315
  "/hyperi-ai",
292
- "/.windsurf",
293
- "/CURSOR.md",
294
- "/CLAUDE.md",
295
316
  "/.claude",
296
317
  "/.gemini",
318
+ "/CURSOR.md",
319
+ "/CLAUDE.md",
320
+ "/.cursor",
321
+ "/ci",
322
+ "/GEMINI.md",
297
323
  "/.github/copilot-instructions.md",
298
324
  "/STATE.md",
299
- "/.cursor",
325
+ "/.windsurf",
300
326
  ]
301
327
 
302
328
  [dependency-groups]
303
329
  dev = [
304
330
  "pytest>=9.0.0",
305
331
  "pytest-cov>=7.0.0",
306
- "ruff>=0.15.0",
332
+ "ruff>=0.16.10",
307
333
  "ty>=0.0.34",
308
334
  "mypy>=2.0",
309
335
  "bandit[toml]>=1.7.0",
@@ -313,7 +339,7 @@ dev = [
313
339
  "sqlalchemy>=2.0.30",
314
340
  "clickhouse-connect>=1.0",
315
341
  "confluent-kafka>=2.14.0",
316
- "testcontainers[clickhouse,kafka,postgres,mysql]>=4.14.0",
342
+ "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
317
343
  "psycopg[binary]>=3.2",
318
344
  "pymysql>=1.1.0",
319
345
  "python-dotenv>=1.0.0",
@@ -1,6 +1,4 @@
1
- """
2
- Anomaly detection utilities
3
- """
1
+ """Anomaly detection utilities."""
4
2
 
5
3
  from loguru import logger
6
4
 
@@ -15,7 +13,7 @@ except ImportError:
15
13
 
16
14
 
17
15
  class AnomalyDetector:
18
- """Detect anomalous log lines"""
16
+ """Detect anomalous log lines."""
19
17
 
20
18
  def __init__(self, contamination: float = 0.1):
21
19
  self.contamination = contamination
@@ -25,8 +23,7 @@ class AnomalyDetector:
25
23
  logger.warning("scikit-learn not available, anomaly detection disabled")
26
24
 
27
25
  def detect_anomalies(self, lines: list[str]) -> tuple[list[str], list[str]]:
28
- """
29
- Separate anomalous and normal lines
26
+ """Separate anomalous and normal lines.
30
27
 
31
28
  Returns:
32
29
  (anomalous_lines, normal_lines)
@@ -1,6 +1,4 @@
1
- """
2
- Core LogReducer implementation
3
- """
1
+ """Core LogReducer implementation."""
4
2
 
5
3
  import json
6
4
  import os
@@ -10,7 +8,7 @@ from dataclasses import replace
10
8
  from datetime import datetime
11
9
  from enum import Enum
12
10
  from pathlib import Path
13
- from typing import Any
11
+ from typing import Any, Literal, overload
14
12
 
15
13
  from .anomaly import AnomalyDetector
16
14
  from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode, get_preset_config
@@ -24,8 +22,7 @@ from .temporal import TemporalProcessor
24
22
 
25
23
 
26
24
  class LogReducer:
27
- """
28
- Main log reduction class
25
+ """Main log reduction class.
29
26
 
30
27
  Example:
31
28
  reducer = LogReducer(level="enhanced")
@@ -41,8 +38,7 @@ class LogReducer:
41
38
  config: BigDialConfig | None = None,
42
39
  **kwargs: Any,
43
40
  ) -> None:
44
- """
45
- Initialize LogReducer
41
+ """Initialize LogReducer.
46
42
 
47
43
  Args:
48
44
  level: Processing level (standard/enhanced/maximum)
@@ -134,13 +130,41 @@ class LogReducer:
134
130
  else:
135
131
  self.temporal_processor = None
136
132
 
133
+ @overload
134
+ def reduce(
135
+ self,
136
+ source: Source,
137
+ output_file: str | None = None,
138
+ return_metadata: Literal[False] = False,
139
+ sink: Sink | None = None,
140
+ ) -> list[str]: ...
141
+
142
+ @overload
143
+ def reduce(
144
+ self,
145
+ source: Source,
146
+ output_file: str | None = None,
147
+ *,
148
+ return_metadata: Literal[True],
149
+ sink: Sink | None = None,
150
+ ) -> dict[str, Any]: ...
151
+
152
+ @overload
137
153
  def reduce(
138
154
  self,
139
155
  source: Source,
140
156
  output_file: str | None = None,
141
157
  return_metadata: bool = False,
142
158
  sink: Sink | None = None,
143
- ) -> list[str] | dict:
159
+ ) -> list[str] | dict[str, Any]: ...
160
+
161
+ def reduce(
162
+ self,
163
+ source: Source,
164
+ output_file: str | None = None,
165
+ return_metadata: bool = False,
166
+ sink: Sink | None = None,
167
+ ) -> list[str] | dict[str, Any]:
144
168
  """Reduce a source of log lines to a representative sample.
145
169
 
146
170
  This is the core entry point: it operates on an abstraction, not on IO.
@@ -241,12 +265,37 @@ class LogReducer:
241
265
  }
242
266
  return result_lines
243
267
 
268
+ @overload
269
+ def process_file(
270
+ self,
271
+ input_file: str,
272
+ output_file: str | None = None,
273
+ return_metadata: Literal[False] = False,
274
+ ) -> list[str]: ...
275
+
276
+ @overload
277
+ def process_file(
278
+ self,
279
+ input_file: str,
280
+ output_file: str | None = None,
281
+ *,
282
+ return_metadata: Literal[True],
283
+ ) -> dict[str, Any]: ...
284
+
285
+ @overload
286
+ def process_file(
287
+ self,
288
+ input_file: str,
289
+ output_file: str | None = None,
290
+ return_metadata: bool = False,
291
+ ) -> list[str] | dict[str, Any]: ...
292
+
244
293
  def process_file(
245
294
  self,
246
295
  input_file: str,
247
296
  output_file: str | None = None,
248
297
  return_metadata: bool = False,
249
- ) -> list[str] | dict:
298
+ ) -> list[str] | dict[str, Any]:
250
299
  """Reduce a log file - convenience wrapper over reduce() + FileSource.
251
300
 
252
301
  Args:
@@ -302,7 +351,7 @@ class LogReducer:
302
351
  return unique_lines
303
352
 
304
353
  def _process_anomaly_mode(self, source: Source) -> list[str]:
305
- """Process using anomaly detection"""
354
+ """Process using anomaly detection."""
306
355
  if not self.anomaly_detector:
307
356
  if self.config.enable_logging:
308
357
  self.logger.warning("Anomaly detector not available, falling back to pattern mode")
@@ -329,7 +378,7 @@ class LogReducer:
329
378
  return result
330
379
 
331
380
  def _process_temporal_mode(self, source: Source) -> list[str]:
332
- """Process using temporal analysis"""
381
+ """Process using temporal analysis."""
333
382
  if not self.temporal_processor:
334
383
  if self.config.enable_logging:
335
384
  self.logger.warning("Temporal processor not available, falling back to pattern mode")
@@ -356,7 +405,7 @@ class LogReducer:
356
405
  return result
357
406
 
358
407
  def _process_hybrid_mode(self, source: Source) -> list[str]:
359
- """Process using combined approach"""
408
+ """Process using combined approach."""
360
409
  if self.config.enable_logging:
361
410
  self.logger.info("Hybrid mode: combining pattern and anomaly detection")
362
411
 
@@ -400,7 +449,7 @@ class LogReducer:
400
449
  }
401
450
 
402
451
  def _save_output(self, lines: list[str], output_file: str) -> None:
403
- """Save output to file in specified format"""
452
+ """Save output to file in specified format."""
404
453
  output_path = Path(output_file)
405
454
  output_path.parent.mkdir(parents=True, exist_ok=True)
406
455
 
@@ -453,7 +502,7 @@ class LogReducer:
453
502
  self.logger.info(f"Metadata saved to {meta_file}")
454
503
 
455
504
  def _print_summary(self) -> None:
456
- """Log processing summary"""
505
+ """Log processing summary."""
457
506
  self.logger.info("\n" + "=" * 60)
458
507
  self.logger.info("LOG REDUCTION SUMMARY")
459
508
  self.logger.info("=" * 60)
@@ -474,7 +523,7 @@ class LogReducer:
474
523
  self.logger.info("=" * 60)
475
524
 
476
525
  def estimate_processing(self, file_path: str) -> dict:
477
- """Estimate processing requirements before running"""
526
+ """Estimate processing requirements before running."""
478
527
  file_size = os.path.getsize(file_path)
479
528
  file_size_gb = file_size / (1024**3)
480
529
 
@@ -26,7 +26,8 @@ from typing import TYPE_CHECKING
26
26
  from loguru import logger
27
27
 
28
28
  if TYPE_CHECKING:
29
- from loguru import Logger
29
+ # Read by get_logger's quoted return annotation, which vulture does not parse.
30
+ from loguru import Logger # noqa: V104
30
31
 
31
32
  # Solarized palette (https://ethanschoonover.com/solarized/) for the console
32
33
  # sink - readable on both light and dark terminal backgrounds.
@@ -1,6 +1,4 @@
1
- """
2
- Memory management and monitoring utilities
3
- """
1
+ """Memory management and monitoring utilities."""
4
2
 
5
3
  import gc
6
4
  import os
@@ -13,7 +11,7 @@ import psutil
13
11
 
14
12
 
15
13
  class MemoryMonitor:
16
- """Monitor and control memory usage during processing"""
14
+ """Monitor and control memory usage during processing."""
17
15
 
18
16
  def __init__(self, max_memory_gb: float = 1.0):
19
17
  self.max_memory_gb = max_memory_gb
@@ -57,7 +55,7 @@ class MemoryMonitor:
57
55
  return current_gb, is_safe
58
56
 
59
57
  def get_current_usage_gb(self) -> float:
60
- """Get current memory usage in GB"""
58
+ """Get current memory usage in GB."""
61
59
  current = self.process.memory_info().rss
62
60
  # Update peak usage
63
61
  if current > self.peak_usage_bytes:
@@ -65,7 +63,7 @@ class MemoryMonitor:
65
63
  return float(current / (1024**3))
66
64
 
67
65
  def get_peak_usage_gb(self) -> float:
68
- """Get peak memory usage in GB"""
66
+ """Get peak memory usage in GB."""
69
67
  # Update current peak if needed
70
68
  current = self.process.memory_info().rss
71
69
  if current > self.peak_usage_bytes:
@@ -84,12 +82,12 @@ class MemoryMonitor:
84
82
  return bool(current > self.max_memory_bytes)
85
83
 
86
84
  def reset(self) -> None:
87
- """Reset peak memory tracking"""
85
+ """Reset peak memory tracking."""
88
86
  current_usage = self.process.memory_info().rss
89
87
  self.peak_usage_bytes = current_usage
90
88
 
91
89
  def estimate_file_strategy(self, file_size_bytes: int) -> str:
92
- """Determine processing strategy"""
90
+ """Determine processing strategy."""
93
91
  file_size_gb = file_size_bytes / (1024**3)
94
92
 
95
93
  if file_size_gb < self.max_memory_gb * 0.2:
@@ -101,14 +99,14 @@ class MemoryMonitor:
101
99
 
102
100
 
103
101
  class StreamingProcessor:
104
- """Process large files with constant memory usage"""
102
+ """Process large files with constant memory usage."""
105
103
 
106
104
  def __init__(self, memory_monitor: MemoryMonitor):
107
105
  self.memory_monitor = memory_monitor
108
106
  self.chunk_size = memory_monitor.safe_chunk_size
109
107
 
110
108
  def read_file_streaming(self, file_path: str) -> Iterator[str]:
111
- """Read file with minimal memory usage"""
109
+ """Read file with minimal memory usage."""
112
110
  file_size = os.path.getsize(file_path)
113
111
  strategy = self.memory_monitor.estimate_file_strategy(file_size)
114
112
 
@@ -223,7 +221,7 @@ class BoundedDeduplicator:
223
221
  self.stats = {"total": 0, "unique": 0, "duplicates": 0}
224
222
 
225
223
  def is_duplicate(self, line: str) -> bool:
226
- """Check if line is duplicate"""
224
+ """Check if line is duplicate."""
227
225
  self.stats["total"] += 1
228
226
  line_hash = self.hash_func(line)
229
227
 
@@ -241,7 +239,7 @@ class BoundedDeduplicator:
241
239
  return False
242
240
 
243
241
  def deduplicate_lines(self, lines: Iterable[str]) -> Iterator[str]:
244
- """Deduplicate a stream of lines"""
242
+ """Deduplicate a stream of lines."""
245
243
  for line in lines:
246
244
  if not self.is_duplicate(line):
247
245
  yield line
@@ -121,6 +121,14 @@ class PatternExtractor:
121
121
  if len(pattern.examples) < self.config.examples_per_pattern:
122
122
  pattern.examples.append(line)
123
123
 
124
+ # Later lines widen a cluster's slots to <*>, so the stored template is
125
+ # re-read once the feed ends; an LRU-evicted cluster keeps its snapshot.
126
+ clusters = self.miner.drain.id_to_cluster
127
+ for cluster_id, pattern in pattern_map.items():
128
+ cluster = clusters.get(cluster_id)
129
+ if cluster is not None:
130
+ pattern.template = cluster.get_template()
131
+
124
132
  patterns = list(pattern_map.values())
125
133
 
126
134
  # Apply filters
@@ -30,7 +30,8 @@ class Sink(Protocol):
30
30
  iterate lazily rather than materialising the whole batch.
31
31
  """
32
32
 
33
- def write(self, lines: Iterable[str]) -> int: ...
33
+ def write(self, lines: Iterable[str]) -> int:
34
+ """Write ``lines`` to the destination and return how many were written."""
34
35
 
35
36
 
36
37
  class FileSink:
@@ -54,6 +55,14 @@ class FileSink:
54
55
  self.output_format = fmt
55
56
 
56
57
  def write(self, lines: Iterable[str]) -> int:
58
+ """Write ``lines`` to ``path`` in the configured format, replacing the file.
59
+
60
+ Args:
61
+ lines: The reduced lines, consumed once.
62
+
63
+ Returns:
64
+ The number of lines written.
65
+ """
57
66
  output_path = Path(self.path)
58
67
  output_path.parent.mkdir(parents=True, exist_ok=True)
59
68
 
@@ -1,6 +1,4 @@
1
- """
2
- Temporal processing and time-aware pattern extraction
3
- """
1
+ """Temporal processing and time-aware pattern extraction."""
4
2
 
5
3
  import re
6
4
  from collections import defaultdict
@@ -13,7 +11,7 @@ from drain3 import TemplateMiner
13
11
 
14
12
  @dataclass
15
13
  class LogEntry:
16
- """Parsed log entry with timestamp"""
14
+ """Parsed log entry with timestamp."""
17
15
 
18
16
  raw_line: str
19
17
  timestamp: datetime | None = None
@@ -22,7 +20,7 @@ class LogEntry:
22
20
 
23
21
 
24
22
  class TimestampParser:
25
- """Parse timestamps from various formats"""
23
+ """Parse timestamps from various formats."""
26
24
 
27
25
  PATTERNS = [
28
26
  (r"\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}", "%Y-%m-%d %H:%M:%S"),
@@ -34,7 +32,7 @@ class TimestampParser:
34
32
  LEVEL_PATTERN = re.compile(r"\b(DEBUG|INFO|WARN|ERROR|CRITICAL)\b", re.I)
35
33
 
36
34
  def parse_line(self, line: str, line_number: int = 0) -> LogEntry:
37
- """Parse log line"""
35
+ """Parse log line."""
38
36
  entry = LogEntry(raw_line=line.strip(), line_number=line_number)
39
37
 
40
38
  # Extract timestamp
@@ -61,7 +59,7 @@ class TimestampParser:
61
59
 
62
60
 
63
61
  class TemporalProcessor:
64
- """Time-aware log processing"""
62
+ """Time-aware log processing."""
65
63
 
66
64
  def __init__(self, window_minutes: int = 60):
67
65
  self.window_minutes = window_minutes
@@ -69,7 +67,7 @@ class TemporalProcessor:
69
67
  self.miners_by_window: dict[int, TemplateMiner] = {}
70
68
 
71
69
  def process_temporal(self, lines: list[str]) -> dict:
72
- """Process with temporal awareness"""
70
+ """Process with temporal awareness."""
73
71
  entries = [self.parser.parse_line(line, i) for i, line in enumerate(lines)]
74
72
 
75
73
  timed = [e for e in entries if e.timestamp]
@@ -91,7 +89,7 @@ class TemporalProcessor:
91
89
  return results
92
90
 
93
91
  def _extract_temporal_patterns(self, entries: list[LogEntry]) -> list[dict]:
94
- """Extract patterns with time context"""
92
+ """Extract patterns with time context."""
95
93
  patterns = []
96
94
  windows = defaultdict(list)
97
95
 
@@ -112,11 +110,10 @@ class TemporalProcessor:
112
110
  self.miners_by_window[window_key] = TemplateMiner(config=config)
113
111
 
114
112
  miner = self.miners_by_window[window_key]
113
+ cluster_ids = [miner.add_log_message(entry.raw_line)["cluster_id"] for entry in window_entries]
115
114
 
116
- for entry in window_entries:
117
- result = miner.add_log_message(entry.raw_line)
118
- cluster_id = result["cluster_id"]
119
-
115
+ # Templates are read once the window is fed, after later lines have widened their slots.
116
+ for entry, cluster_id in zip(window_entries, cluster_ids, strict=True):
120
117
  patterns.append(
121
118
  {
122
119
  "template": miner.drain.id_to_cluster[cluster_id].get_template(),
@@ -129,29 +126,26 @@ class TemporalProcessor:
129
126
  return patterns
130
127
 
131
128
  def _extract_timeless_patterns(self, entries: list[LogEntry]) -> list[dict]:
132
- """Extract patterns without timestamps"""
129
+ """Extract patterns without timestamps."""
133
130
  from drain3.template_miner_config import TemplateMinerConfig
134
131
 
135
132
  config = TemplateMinerConfig()
136
133
  config.snapshot_interval_minutes = 0 # Disable snapshots
137
134
  config.snapshot_compress_state = False
138
135
  miner = TemplateMiner(config=config)
139
- patterns = []
136
+ cluster_ids = [miner.add_log_message(entry.raw_line)["cluster_id"] for entry in entries]
140
137
 
141
- for entry in entries:
142
- result = miner.add_log_message(entry.raw_line)
143
- cluster_id = result["cluster_id"]
144
- patterns.append(
145
- {
146
- "template": miner.drain.id_to_cluster[cluster_id].get_template(),
147
- "example": entry.raw_line,
148
- }
149
- )
150
-
151
- return patterns
138
+ # Templates are read once every entry is fed, after later lines have widened their slots.
139
+ return [
140
+ {
141
+ "template": miner.drain.id_to_cluster[cluster_id].get_template(),
142
+ "example": entry.raw_line,
143
+ }
144
+ for entry, cluster_id in zip(entries, cluster_ids, strict=True)
145
+ ]
152
146
 
153
147
  def _analyze_distribution(self, entries: list[LogEntry]) -> dict:
154
- """Analyze time distribution"""
148
+ """Analyze time distribution."""
155
149
  dist: dict[str, Any] = {"hourly": defaultdict(int), "levels": defaultdict(int)}
156
150
 
157
151
  for entry in entries:
File without changes
File without changes