logreducer 3.5.2__tar.gz → 3.5.3__tar.gz

{logreducer-3.5.2 → logreducer-3.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logreducer
-Version: 3.5.2
+Version: 3.5.3
 Summary: Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection
 Keywords: log,logging,reduction,analysis,pattern-extraction,memory-efficient,streaming,anomaly-detection
 Author: HyperI Team
@@ -17,17 +17,15 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Operating System :: OS Independent
-Requires-Dist: drain3>=0.9.0
-Requires-Dist: psutil>=7.2.0
-Requires-Dist: loguru>=0.7.3
-Requires-Dist: numpy>=2.0.0
-Requires-Dist: scikit-learn>=1.5.0
+Requires-Dist: psutil>=7.2.2
+Requires-Dist: numpy>=2.5.3
+Requires-Dist: scikit-learn>=1.9.1
 Requires-Dist: typer>=0.27.3
-Requires-Dist: clickhouse-connect>=1.0 ; extra == 'clickhouse'
+Requires-Dist: clickhouse-connect>=1.9.0 ; extra == 'clickhouse'
 Requires-Dist: xxhash>=4.0.1 ; extra == 'enhanced'
 Requires-Dist: datasketch>=2.0.0 ; extra == 'enhanced'
-Requires-Dist: confluent-kafka>=2.14.0 ; extra == 'kafka'
-Requires-Dist: sqlalchemy>=2.0.30 ; extra == 'sql'
+Requires-Dist: confluent-kafka>=2.15.1 ; extra == 'kafka'
+Requires-Dist: sqlalchemy>=2.1.3 ; extra == 'sql'
 Maintainer: HyperI Team
 Maintainer-email: HyperI Team <dev@hyperi.io>
 Requires-Python: >=3.14
@@ -337,8 +335,9 @@ reducer = logreducer.LogReducer(mode="hybrid", config=cfg)
 # ...or let logreducer read your env cascade directly: prefixed beats bare.
 cfg = logreducer.BigDialConfig.from_env("MYAPP", "LOGREDUCER")   # MYAPP_MAX_MEMORY_GB wins
 
-# 2. Logging: your app owns the loguru sinks; logreducer registers nothing and
-#    its records flow through YOUR handlers, formatted by YOUR standard.
+# 2. Logging: your app owns the stdlib logging handlers; logreducer attaches
+#    nothing and its records flow through YOUR handlers, at YOUR level,
+#    formatted by YOUR standard.
 logreducer.setup_logging(enable=True, own_sinks=False)
 ```
 
@@ -368,7 +367,7 @@ Logging is off by default. Enable it and pick a format via env vars:
 LOG_LEVEL=DEBUG LOG_FORMAT=json logreducer app.log --log
 ```
 
-`LOG_FORMAT=json` emits one JSON object per line for log aggregators; the default is human-readable (coloured in a terminal, plain in CI/containers). Embedding? See `own_sinks=False` above.
+Logging is the standard library `logging` module, under the `logreducer` logger. `LOG_FORMAT=json` emits one JSON object per line for log aggregators (keys `timestamp`, `level`, `target`, `function`, `line_number`, `message`, `fields`); the default is human-readable (coloured in a terminal, plain in CI/containers). Embedding? See `own_sinks=False` above.
 
 ## Development
 
@@ -386,6 +385,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for the toolchain, test layout (including
 
 Copyright 2026 HYPERI PTY LIMITED.
 
+Credits: template mining (`src/logreducer/template_mining.py`) is derived from [Drain3](https://github.com/logpai/Drain3), Copyright (c) 2020-2022 International Business Machines and the Drain3 project contributors, MIT licence. Drain3 builds on the Drain parser in LogPAI's [logparser](https://github.com/logpai/logparser). The module header carries Drain3's licence text in full.
+
 ## Context
 
 Depth: [docs/architecture.md](docs/architecture.md).
@@ -406,6 +407,7 @@ it is given and drops the rest.
 | `src/logreducer/sources.py`, `sinks.py` | the two structural protocols that ARE the integration surface, plus `FileSource` and `FileSink` |
 | `src/logreducer/sql.py`, `clickhouse.py`, `kafka.py` | the optional adapters, one extra each |
 | `src/logreducer/patterns.py`, `anomaly.py`, `temporal.py` | Drain3 mining and fuzzy dedup, Isolation Forest, time-window grouping |
+| `src/logreducer/template_mining.py` | the Drain template miner, derived from Drain3 (credited under License) |
 | `src/logreducer/memory.py` | `MemoryMonitor`, `BoundedDeduplicator`, the file read strategies |
 | `src/logreducer/sampling.py`, `target.py` | dialect-aware SQL sampling, and the `reduce_to_target` batch loop |
 | `src/logreducer/config.py` | `BigDialConfig`, the level presets, the `from_env` cascade |
@@ -451,7 +453,7 @@ One declared edge in `dfe-infra/suite.yaml`, and it runs the surprising way.
 
 `dfe-stack suite --consumer logreducer` returns an empty edge list: no repo in
 that graph is something this library depends on. The runtime dependencies
-(drain3, psutil, loguru, numpy, scikit-learn, typer) are packages, declared in
+(psutil, numpy, scikit-learn, typer) are packages, declared in
 `pyproject.toml`, which is where they belong.
 
 The reason dfe-engine gives for not declaring the dependency -- pin it "once it

{logreducer-3.5.2 → logreducer-3.5.3}/README.md

@@ -293,8 +293,9 @@ reducer = logreducer.LogReducer(mode="hybrid", config=cfg)
 # ...or let logreducer read your env cascade directly: prefixed beats bare.
 cfg = logreducer.BigDialConfig.from_env("MYAPP", "LOGREDUCER")   # MYAPP_MAX_MEMORY_GB wins
 
-# 2. Logging: your app owns the loguru sinks; logreducer registers nothing and
-#    its records flow through YOUR handlers, formatted by YOUR standard.
+# 2. Logging: your app owns the stdlib logging handlers; logreducer attaches
+#    nothing and its records flow through YOUR handlers, at YOUR level,
+#    formatted by YOUR standard.
 logreducer.setup_logging(enable=True, own_sinks=False)
 ```
 
@@ -324,7 +325,7 @@ Logging is off by default. Enable it and pick a format via env vars:
 LOG_LEVEL=DEBUG LOG_FORMAT=json logreducer app.log --log
 ```
 
-`LOG_FORMAT=json` emits one JSON object per line for log aggregators; the default is human-readable (coloured in a terminal, plain in CI/containers). Embedding? See `own_sinks=False` above.
+Logging is the standard library `logging` module, under the `logreducer` logger. `LOG_FORMAT=json` emits one JSON object per line for log aggregators (keys `timestamp`, `level`, `target`, `function`, `line_number`, `message`, `fields`); the default is human-readable (coloured in a terminal, plain in CI/containers). Embedding? See `own_sinks=False` above.
 
 ## Development
 
@@ -342,6 +343,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for the toolchain, test layout (including
 
 Copyright 2026 HYPERI PTY LIMITED.
 
+Credits: template mining (`src/logreducer/template_mining.py`) is derived from [Drain3](https://github.com/logpai/Drain3), Copyright (c) 2020-2022 International Business Machines and the Drain3 project contributors, MIT licence. Drain3 builds on the Drain parser in LogPAI's [logparser](https://github.com/logpai/logparser). The module header carries Drain3's licence text in full.
+
 ## Context
 
 Depth: [docs/architecture.md](docs/architecture.md).
@@ -362,6 +365,7 @@ it is given and drops the rest.
 | `src/logreducer/sources.py`, `sinks.py` | the two structural protocols that ARE the integration surface, plus `FileSource` and `FileSink` |
 | `src/logreducer/sql.py`, `clickhouse.py`, `kafka.py` | the optional adapters, one extra each |
 | `src/logreducer/patterns.py`, `anomaly.py`, `temporal.py` | Drain3 mining and fuzzy dedup, Isolation Forest, time-window grouping |
+| `src/logreducer/template_mining.py` | the Drain template miner, derived from Drain3 (credited under License) |
 | `src/logreducer/memory.py` | `MemoryMonitor`, `BoundedDeduplicator`, the file read strategies |
 | `src/logreducer/sampling.py`, `target.py` | dialect-aware SQL sampling, and the `reduce_to_target` batch loop |
 | `src/logreducer/config.py` | `BigDialConfig`, the level presets, the `from_env` cascade |
@@ -407,7 +411,7 @@ One declared edge in `dfe-infra/suite.yaml`, and it runs the surprising way.
 
 `dfe-stack suite --consumer logreducer` returns an empty edge list: no repo in
 that graph is something this library depends on. The runtime dependencies
-(drain3, psutil, loguru, numpy, scikit-learn, typer) are packages, declared in
+(psutil, numpy, scikit-learn, typer) are packages, declared in
 `pyproject.toml`, which is where they belong.
 
 The reason dfe-engine gives for not declaring the dependency -- pin it "once it

{logreducer-3.5.2 → logreducer-3.5.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "logreducer"
-version = "3.5.2"
+version = "3.5.3"
 description = "Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection"
 readme = "README.md"
 license = "Apache-2.0"
@@ -35,11 +35,9 @@ keywords = [
     "anomaly-detection",
 ]
 dependencies = [
-    "drain3>=0.9.0",
-    "psutil>=7.2.0",
-    "loguru>=0.7.3",
-    "numpy>=2.0.0",
-    "scikit-learn>=1.5.0",
+    "psutil>=7.2.2",
+    "numpy>=2.5.3",
+    "scikit-learn>=1.9.1",
     "typer>=0.27.3",
 ]
 
@@ -56,9 +54,9 @@ enhanced = [
     "xxhash>=4.0.1",
     "datasketch>=2.0.0",
 ]
-sql = ["sqlalchemy>=2.0.30"]
-clickhouse = ["clickhouse-connect>=1.0"]
-kafka = ["confluent-kafka>=2.14.0"]
+sql = ["sqlalchemy>=2.1.3"]
+clickhouse = ["clickhouse-connect>=1.9.0"]
+kafka = ["confluent-kafka>=2.15.1"]
 
 [project.urls]
 Homepage = "https://github.com/hyperi-io/logreducer"
@@ -280,17 +278,17 @@ no_tips = true
 
 [tool.hatch.build.targets.sdist]
 exclude = [
-    "/hyperi-ai",
-    "/.claude",
-    "/.gemini",
     "/CURSOR.md",
-    "/CLAUDE.md",
-    "/.cursor",
-    "/ci",
     "/GEMINI.md",
+    "/.cursor",
+    "/.windsurf",
+    "/.gemini",
+    "/CLAUDE.md",
     "/.github/copilot-instructions.md",
+    "/.claude",
+    "/ci",
+    "/hyperi-ai",
     "/STATE.md",
-    "/.windsurf",
 ]
 
 [dependency-groups]
@@ -304,9 +302,9 @@ dev = [
     "pip-audit>=2.6.0",
     "vulture>=2.16",
     "vermin>=1.6.0",
-    "sqlalchemy>=2.0.30",
-    "clickhouse-connect>=1.0",
-    "confluent-kafka>=2.14.0",
+    "sqlalchemy>=2.1.3",
+    "clickhouse-connect>=1.9.0",
+    "confluent-kafka>=2.15.1",
     "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
     "psycopg[binary]>=3.2",
     "pymysql>=1.1.0",

{logreducer-3.5.2 → logreducer-3.5.3}/pyproject.toml.orig

@@ -6,7 +6,7 @@ build-backend = "uv_build"
 
 [project]
 name = "logreducer"
-version = "3.5.2"
+version = "3.5.3"
 authors = [
     { name = "HyperI Team", email = "dev@hyperi.io" },
 ]
@@ -43,11 +43,9 @@ keywords = [
     "anomaly-detection",
 ]
 dependencies = [
-    "drain3>=0.9.0",
-    "psutil>=7.2.0",
-    "loguru>=0.7.3",
-    "numpy>=2.0.0",
-    "scikit-learn>=1.5.0",
+    "psutil>=7.2.2",
+    "numpy>=2.5.3",
+    "scikit-learn>=1.9.1",
     "typer>=0.27.3",
 ]
 
@@ -57,13 +55,13 @@ enhanced = [
     "datasketch>=2.0.0",
 ]
 sql = [
-    "sqlalchemy>=2.0.30",
+    "sqlalchemy>=2.1.3",
 ]
 clickhouse = [
-    "clickhouse-connect>=1.0",
+    "clickhouse-connect>=1.9.0",
 ]
 kafka = [
-    "confluent-kafka>=2.14.0",
+    "confluent-kafka>=2.15.1",
 ]
 
 [project.urls]
@@ -312,17 +310,17 @@ no_tips = true
 
 [tool.hatch.build.targets.sdist]
 exclude = [
-    "/hyperi-ai",
-    "/.claude",
-    "/.gemini",
     "/CURSOR.md",
-    "/CLAUDE.md",
-    "/.cursor",
-    "/ci",
     "/GEMINI.md",
+    "/.cursor",
+    "/.windsurf",
+    "/.gemini",
+    "/CLAUDE.md",
     "/.github/copilot-instructions.md",
+    "/.claude",
+    "/ci",
+    "/hyperi-ai",
     "/STATE.md",
-    "/.windsurf",
 ]
 
 [dependency-groups]
@@ -336,9 +334,9 @@ dev = [
     "pip-audit>=2.6.0",
     "vulture>=2.16",
     "vermin>=1.6.0",
-    "sqlalchemy>=2.0.30",
-    "clickhouse-connect>=1.0",
-    "confluent-kafka>=2.14.0",
+    "sqlalchemy>=2.1.3",
+    "clickhouse-connect>=1.9.0",
+    "confluent-kafka>=2.15.1",
     "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",
     "psycopg[binary]>=3.2",
     "pymysql>=1.1.0",

{logreducer-3.5.2 → logreducer-3.5.3}/src/logreducer/anomaly.py

@@ -1,6 +1,8 @@
 """Anomaly detection utilities."""
 
-from loguru import logger
+import logging
+
+logger = logging.getLogger(__name__)
 
 # Try optional imports
 try:

{logreducer-3.5.2 → logreducer-3.5.3}/src/logreducer/config.py

@@ -1,5 +1,6 @@
 """Configuration and tuning parameters for LogReducer."""
 
+import logging
 import os
 import types
 import typing
@@ -7,7 +8,8 @@ from dataclasses import dataclass, fields
 from enum import Enum
 
 import psutil
-from loguru import logger
+
+logger = logging.getLogger(__name__)
 
 
 class ProcessingLevel(Enum):

logreducer-3.5.3/src/logreducer/logging_config.py

@@ -0,0 +1,237 @@
+"""Logging configuration for LogReducer, on the standard library ``logging``.
+
+Output follows one strict standard so logreducer's logs sit uniformly beside a
+host application's logs in a shared deployment: RFC 3339 timestamps, a
+``time | LEVEL | name:function:line - message`` layout, ASCII-only in files,
+solarized colours in an interactive terminal, a plain prefix in CI, and one
+JSON object per line when ``LOG_FORMAT=json``.
+
+logreducer is a library, not a service, so it only ever attaches handlers to its
+OWN ``logreducer`` logger and never touches the root logger or a host
+application's handlers - embedding logreducer inside another application must
+not disturb that application's logging.
+
+ENV overrides:
+    LOG_LEVEL=DEBUG
+    LOG_FORMAT=json|text
+    LOG_OUTPUT=stdout|stderr
+"""
+
+import json
+import logging
+import logging.handlers
+import os
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+PACKAGE_LOGGER_NAME = "logreducer"
+
+# Solarized palette (https://ethanschoonover.com/solarized/) for the console
+# sink - readable on both light and dark terminal backgrounds.
+SOLARIZED = {
+    "base01": "#586e75",
+    "green": "#859900",
+    "cyan": "#2aa198",
+    "blue": "#268bd2",
+    "yellow": "#b58900",
+    "orange": "#cb4b16",
+    "red": "#dc322f",
+}
+
+_LEVEL_COLOURS = {
+    logging.DEBUG: SOLARIZED["base01"],
+    logging.INFO: SOLARIZED["blue"],
+    logging.WARNING: SOLARIZED["yellow"],
+    logging.ERROR: SOLARIZED["orange"],
+    logging.CRITICAL: SOLARIZED["red"],
+}
+
+# Above CRITICAL, so a disabled logreducer emits nothing, not even through a host's handlers.
+_DISABLED_LEVEL = logging.CRITICAL + 1
+
+# Rotate the file sink at 10 MB and keep seven rotated files.
+_FILE_MAX_BYTES = 10 * 1024 * 1024
+_FILE_BACKUPS = 7
+
+# Every attribute a bare LogRecord carries; anything else came from the caller's ``extra=``.
+_STANDARD_RECORD_ATTRS = frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | {"message", "asctime"}
+
+# Handlers this module has attached, so re-configuring only tears down our own
+# sinks - never the host application's.
+_HANDLERS: list[logging.Handler] = []
+
+logger = logging.getLogger(__name__)
+
+
+def _is_ci() -> bool:
+    """True in a CI environment (colours off, plain prefix format)."""
+    return (
+        os.getenv("CI") == "true"
+        or os.getenv("GITHUB_ACTIONS") == "true"
+        or os.getenv("GITLAB_CI") == "true"
+        or os.getenv("JENKINS_URL") is not None
+    )
+
+
+def _is_interactive() -> bool:
+    """True only for an interactive UTF-8 terminal (never a pipe/container)."""
+    if _is_ci() or not sys.stderr.isatty():
+        return False
+    term = os.getenv("TERM", "")
+    if not term or term == "dumb":
+        return False
+    locale = (os.getenv("LC_ALL") or os.getenv("LANG") or "").upper()
+    return "UTF-8" in locale or "UTF8" in locale
+
+
+def _paint(text: str, hex_colour: str) -> str:
+    """Wrap ``text`` in a 24-bit ANSI foreground colour."""
+    red, green, blue = (int(hex_colour[i : i + 2], 16) for i in (1, 3, 5))
+    return f"\x1b[38;2;{red};{green};{blue}m{text}\x1b[0m"
+
+
+def _local_rfc3339(created: float) -> str:
+    """RFC 3339 local time with milliseconds and the UTC offset."""
+    return datetime.fromtimestamp(created).astimezone().isoformat(timespec="milliseconds")
+
+
+class _LineFormatter(logging.Formatter):
+    """Render ``[time] LEVEL name:function:line - message`` on one line (plus any traceback)."""
+
+    def __init__(self, *, with_time: bool, colour: bool, ascii_only: bool) -> None:
+        super().__init__()
+        self._with_time = with_time
+        self._colour = colour
+        self._ascii_only = ascii_only
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Render one record."""
+        level = f"{record.levelname:<8}"
+        where = f"{record.name}:{record.funcName}:{record.lineno}"
+        message = record.getMessage()
+        if record.exc_info:
+            message = f"{message}\n{self.formatException(record.exc_info)}"
+
+        if self._colour:
+            level_colour = _LEVEL_COLOURS.get(record.levelno, SOLARIZED["blue"])
+            line = (
+                f"{_paint(_local_rfc3339(record.created), SOLARIZED['green'])} | "
+                f"{_paint(level, level_colour)} | {_paint(where, SOLARIZED['cyan'])} - {_paint(message, level_colour)}"
+            )
+        elif self._with_time:
+            line = f"{_local_rfc3339(record.created)} [{level}] {where} - {message}"
+        else:
+            line = f"[{level}] {where} - {message}"
+
+        if self._ascii_only:
+            return line.encode("ascii", "backslashreplace").decode("ascii")
+        return line
+
+
+class _JsonFormatter(logging.Formatter):
+    """Render one flat JSON object per record, keyed like scalo's JSON log lines."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Render one record."""
+        timestamp = datetime.fromtimestamp(record.created, UTC).isoformat(timespec="microseconds")
+        entry: dict[str, object] = {
+            "timestamp": timestamp.replace("+00:00", "Z"),
+            "level": record.levelname,
+            "target": record.name,
+            "function": record.funcName,
+            "line_number": record.lineno,
+            "message": record.getMessage(),
+            "fields": {key: value for key, value in record.__dict__.items() if key not in _STANDARD_RECORD_ATTRS},
+        }
+        if record.exc_info:
+            entry["exception"] = self.formatException(record.exc_info)
+        return json.dumps(entry, default=str, ensure_ascii=True)
+
+
+def _attach(package_logger: logging.Logger, handler: logging.Handler, formatter: logging.Formatter) -> None:
+    handler.setFormatter(formatter)
+    package_logger.addHandler(handler)
+    _HANDLERS.append(handler)
+
+
+def setup_logging(
+    enable: bool = False,
+    log_file: str | None = None,
+    log_level: str = "INFO",
+    log_format: str = "text",
+    console: bool = False,
+    own_sinks: bool = True,
+) -> None:
+    """Configure logreducer's own logging sinks.
+
+    Records always propagate to the host application's handlers as well, so a
+    host that has configured the root logger sees logreducer's records there.
+
+    Args:
+        enable: Whether logreducer emits logs at all (default False).
+        log_file: Optional path for an ASCII file sink that rotates at 10 MB
+            and keeps seven rotated files.
+        log_level: Threshold level (overridden by ``LOG_LEVEL``).
+        log_format: ``text`` (human) or ``json`` (overridden by ``LOG_FORMAT``).
+        console: Also log to the console (stderr, or ``LOG_OUTPUT=stdout``).
+        own_sinks: When False, attach NO handlers and leave the threshold to the
+            host - logreducer's records flow to the HOST application's handlers
+            at the host's level. This is the embedding seam: a host app that owns
+            logging calls ``setup_logging(enable=True, own_sinks=False)`` and
+            logreducer's output lands in the host's sinks, formatted by the
+            host's standard.
+
+    Raises:
+        ValueError: If the resolved level is not a ``logging`` level name.
+    """
+    # Env beats caller, so a deployment can retune logging without code changes.
+    log_level = os.environ.get("LOG_LEVEL", log_level).upper()
+    fmt_selector = (os.environ.get("LOG_FORMAT") or log_format or "text").strip().lower()
+    serialize = fmt_selector == "json"
+
+    # Tear down ONLY our previously attached handlers.
+    package_logger = logging.getLogger(PACKAGE_LOGGER_NAME)
+    for handler in _HANDLERS:
+        package_logger.removeHandler(handler)
+        handler.close()
+    _HANDLERS.clear()
+
+    if not enable:
+        package_logger.setLevel(_DISABLED_LEVEL)
+        return
+
+    if not own_sinks:
+        package_logger.setLevel(logging.NOTSET)
+        return
+
+    package_logger.setLevel(log_level)
+    ci = _is_ci()
+
+    if console:
+        stream = sys.stdout if os.environ.get("LOG_OUTPUT") == "stdout" else sys.stderr
+        colour = not serialize and not ci and _is_interactive()
+        console_formatter = (
+            _JsonFormatter() if serialize else _LineFormatter(with_time=colour, colour=colour, ascii_only=False)
+        )
+        _attach(package_logger, logging.StreamHandler(stream), console_formatter)
+
+    if log_file:
+        try:
+            Path(log_file).parent.mkdir(parents=True, exist_ok=True)
+            file_handler = logging.handlers.RotatingFileHandler(
+                log_file, maxBytes=_FILE_MAX_BYTES, backupCount=_FILE_BACKUPS, encoding="utf-8"
+            )
+        except OSError as exc:
+            # A bad log path should never take the process down.
+            logger.warning("Could not create log file %s: %s", log_file, exc)
+        else:
+            file_formatter = (
+                _JsonFormatter() if serialize else _LineFormatter(with_time=True, colour=False, ascii_only=True)
+            )
+            _attach(package_logger, file_handler, file_formatter)
+
+
+def get_logger(name: str = PACKAGE_LOGGER_NAME) -> logging.Logger:
+    """Return the logger for a logreducer module name (``logreducer.<module>``)."""
+    return logging.getLogger(name)

{logreducer-3.5.2 → logreducer-3.5.3}/src/logreducer/patterns.py

@@ -1,17 +1,17 @@
 """Pattern extraction (Drain3 template mining) and fuzzy deduplication."""
 
+import logging
 from collections.abc import Iterable, Iterator
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
-from drain3 import TemplateMiner
-from drain3.masking import MaskingInstruction
-from drain3.template_miner_config import TemplateMinerConfig
-from loguru import logger
+from .template_mining import MaskingInstruction, TemplateMiner, TemplateMinerConfig
 
 if TYPE_CHECKING:
     from .config import BigDialConfig
 
+logger = logging.getLogger(__name__)
+
 # datasketch powers fuzzy dedup and ships in the optional `enhanced` extra;
 # without it FuzzyDeduplicator degrades to a pass-through (with a warning).
 try:
@@ -34,10 +34,10 @@ class LogPattern:
     metadata: dict = field(default_factory=dict)
 
 
-def _typed_masking_instructions() -> list[MaskingInstruction]:
+def _typed_masking_instructions() -> tuple[MaskingInstruction, ...]:
     """Build the curated masking set for typed template slots.
 
-    Drain3 applies these to each line before clustering, so masked values
+    The miner applies these to each line before clustering, so masked values
     become literal typed tokens (``<IP>``, ``<NUM>``, ...) rather than
     collapsing to the bare ``<*>`` wildcard. Instructions run sequentially
     over already-masked text, so the specific shapes (IP, UUID, MAC, IPv6)
@@ -70,52 +70,47 @@ def _typed_masking_instructions() -> list[MaskingInstruction]:
         # Integer (drain3's documented NUM example).
         (r"[-+]?\d+", "NUM"),
     ]
-    return [MaskingInstruction(f"{start}(?:{pattern}){end}", name) for pattern, name in shapes]
+    return tuple(MaskingInstruction(f"{start}(?:{pattern}){end}", name) for pattern, name in shapes)
 
 
 class PatternExtractor:
-    """Extract patterns using Drain3's online template miner."""
+    """Extract patterns using the online template miner (see template_mining)."""
 
     def __init__(self, config: "BigDialConfig") -> None:
         self.config = config
         self.setup_drain3()
 
     def setup_drain3(self) -> None:
-        """Configure Drain3."""
-        drain_config = TemplateMinerConfig()
-        drain_config.profiling_enabled = False
-        drain_config.drain_sim_th = self.config.drain_similarity
-        drain_config.drain_depth = 4
-        drain_config.snapshot_interval_minutes = 0  # Disable snapshots
-        drain_config.snapshot_compress_state = False
-        # Bound the template store when configured: Drain3 LRU-evicts beyond
-        # drain_max_clusters, keeping memory flat on high-cardinality logs.
-        drain_config.drain_max_clusters = self.config.max_clusters
+        """Configure the template miner."""
         # Opt-in typed masking: pre-cluster masking of well-known value shapes
         # so templates carry typed slots (<IP>, <NUM>, ...) instead of <*>.
-        if self.config.typed_masking:
-            drain_config.masking_instructions = _typed_masking_instructions()
-        self.miner = TemplateMiner(config=drain_config)
+        masking = _typed_masking_instructions() if self.config.typed_masking else ()
+        drain_config = TemplateMinerConfig(
+            drain_sim_th=self.config.drain_similarity,
+            drain_depth=4,
+            # Bound the template store when configured: the miner LRU-evicts beyond
+            # drain_max_clusters, keeping memory flat on high-cardinality logs.
+            drain_max_clusters=self.config.max_clusters,
+            masking_instructions=masking,
+        )
+        self.miner = TemplateMiner(drain_config)
 
     def extract_patterns(self, lines: Iterable[str]) -> list[LogPattern]:
         """Extract patterns from lines (accepts a stream; only clusters are held).
 
-        Feeds each line into Drain3's online miner, so the caller can pass a
+        Feeds each line into the online miner, so the caller can pass a
         generator - no need to materialise the whole line set. Memory is bounded
         by the number of clusters (see ``max_clusters``), not the line count.
         """
-        pattern_map = {}
+        pattern_map: dict[int, LogPattern] = {}
 
         for line in lines:
-            result = self.miner.add_log_message(line)
-            cluster_id = result["cluster_id"]
-
-            if cluster_id not in pattern_map:
-                cluster = self.miner.drain.id_to_cluster[cluster_id]
+            cluster = self.miner.add_log_message(line)
+            pattern = pattern_map.get(cluster.cluster_id)
+            if pattern is None:
                 pattern = LogPattern(template=cluster.get_template(), count=0)
-                pattern_map[cluster_id] = pattern
+                pattern_map[cluster.cluster_id] = pattern
 
-            pattern = pattern_map[cluster_id]
             pattern.count += 1
 
             if len(pattern.examples) < self.config.examples_per_pattern:
@@ -123,11 +118,10 @@ class PatternExtractor:
 
         # Later lines widen a cluster's slots to <*>, so the stored template is
         # re-read once the feed ends; an LRU-evicted cluster keeps its snapshot.
-        clusters = self.miner.drain.id_to_cluster
         for cluster_id, pattern in pattern_map.items():
-            cluster = clusters.get(cluster_id)
-            if cluster is not None:
-                pattern.template = cluster.get_template()
+            live_cluster = self.miner.get_cluster(cluster_id)
+            if live_cluster is not None:
+                pattern.template = live_cluster.get_template()
 
         patterns = list(pattern_map.values())
 

{logreducer-3.5.2 → logreducer-3.5.3}/src/logreducer/sql.py

@@ -143,7 +143,7 @@ class SQLSource:
         from .sampling import build_table_sample_query
 
         owns = isinstance(connectable, str)
-        engine = create_engine(connectable) if owns else connectable
+        engine = create_engine(connectable) if isinstance(connectable, str) else connectable
         preparer = engine.dialect.identifier_preparer
         query = build_table_sample_query(
             engine.dialect.name,