logreducer 3.4.0__tar.gz → 3.5.0__tar.gz

{logreducer-3.4.0 → logreducer-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logreducer
-Version: 3.4.0
+Version: 3.5.0
 Summary: Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection
 Keywords: log,logging,reduction,analysis,pattern-extraction,memory-efficient,streaming,anomaly-detection
 Author: HyperI Team

logreducer-3.5.0/pyproject.toml

@@ -0,0 +1,300 @@
+[build-system]
+requires = ["uv_build>=0.8.0,<1"]
+build-backend = "uv_build"
+
+[project]
+name = "logreducer"
+version = "3.5.0"
+description = "Reduce GB-scale logs to a representative sample - streaming Python library and CLI with pattern mining and anomaly detection"
+readme = "README.md"
+license = "Apache-2.0"
+license-files = [
+    "LICENSE",
+    "NOTICE",
+]
+requires-python = ">=3.12"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Topic :: System :: Logging",
+    "Topic :: System :: Systems Administration",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Operating System :: OS Independent",
+]
+keywords = [
+    "log",
+    "logging",
+    "reduction",
+    "analysis",
+    "pattern-extraction",
+    "memory-efficient",
+    "streaming",
+    "anomaly-detection",
+]
+dependencies = [
+    "drain3>=0.9.0",
+    "psutil>=7.2.0",
+    "loguru>=0.7.3",
+    "numpy>=2.0.0",
+    "scikit-learn>=1.5.0",
+    "typer>=0.25.0",
+]
+
+[[project.authors]]
+name = "HyperI Team"
+email = "dev@hyperi.io"
+
+[[project.maintainers]]
+name = "HyperI Team"
+email = "dev@hyperi.io"
+
+[project.optional-dependencies]
+enhanced = [
+    "xxhash>=3.0.0",
+    "datasketch>=1.5.0",
+]
+sql = ["sqlalchemy>=2.0.30"]
+clickhouse = ["clickhouse-connect>=1.0"]
+kafka = ["confluent-kafka>=2.14.0"]
+
+[project.urls]
+Homepage = "https://github.com/hyperi-io/logreducer"
+Documentation = "https://github.com/hyperi-io/logreducer#readme"
+Repository = "https://github.com/hyperi-io/logreducer"
+"Bug Reports" = "https://github.com/hyperi-io/logreducer/issues"
+Changelog = "https://github.com/hyperi-io/logreducer/blob/main/CHANGELOG.md"
+
+[project.scripts]
+logreducer = "logreducer.cli:main"
+
+[tool.uv]
+index-strategy = "unsafe-best-match"
+override-dependencies = []
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+default = true
+
+[tool.uv.build-backend]
+module-name = "logreducer"
+module-root = "src"
+
+[tool.ruff]
+target-version = "py312"
+line-length = 120
+indent-width = 4
+src = ["src"]
+extend-exclude = ["examples"]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "B",
+    "C4",
+    "UP",
+    "ARG",
+    "SIM",
+    "S",
+    "N",
+    "PT",
+    "RUF",
+    "PIE",
+    "T20",
+]
+ignore = [
+    "ARG001",
+    "ARG002",
+    "B904",
+    "B008",
+    "B017",
+    "B007",
+    "B028",
+    "E402",
+    "E722",
+    "E501",
+    "E741",
+    "SIM105",
+    "SIM108",
+    "SIM102",
+    "F821",
+    "F601",
+    "TRY300",
+    "C901",
+    "UP035",
+    "W191",
+    "S104",
+    "S110",
+    "S112",
+    "S311",
+    "S603",
+    "S607",
+    "S608",
+    "T201",
+    "RUF012",
+    "RUF059",
+    "RUF001",
+    "RUF002",
+    "RUF005",
+    "RUF006",
+    "N806",
+    "N818",
+    "PIE810",
+]
+fixable = ["ALL"]
+unfixable = []
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = [
+    "S101",
+    "S105",
+    "S106",
+    "S108",
+    "F541",
+    "RUF013",
+    "PT017",
+    "PT011",
+    "PT012",
+    "RUF043",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+[tool.mypy]
+python_version = "3.12"
+ignore_missing_imports = true
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pytest.ini_options]
+minversion = "8.0.0"
+addopts = "-ra -q --strict-markers --strict-config"
+testpaths = ["tests"]
+python_files = [
+    "test_*.py",
+    "*_test.py",
+]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+markers = [
+    """slow: marks tests as slow (deselect with '-m "not slow"')""",
+    "integration: marks tests as integration tests",
+]
+
+[tool.coverage.run]
+source = ["logreducer"]
+omit = [
+    "*/tests/*",
+    "*/test_*",
+    "*/.venv/*",
+    "*/venv/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    'class .*\bProtocol\):',
+    '@(abc\.)?abstractmethod',
+]
+
+[tool.bandit]
+exclude_dirs = [
+    "tests",
+    "test",
+    ".venv",
+    "venv",
+    ".tox",
+    "build",
+    "dist",
+    "data/samples",
+]
+skips = [
+    "B101",
+    "B603",
+]
+
+[tool.bandit.assert_used]
+skips = [
+    "*test*.py",
+    "*tests*.py",
+]
+
+[tool.ty.environment]
+python-version = "3.12"
+
+[tool.ty.src]
+exclude = [
+    "examples/",
+    "docs/",
+    "scripts/",
+]
+
+[tool.vermin]
+targets = ["3.12"]
+no_tips = true
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+    "/.github/copilot-instructions.md",
+    "/hyperi-ai",
+    "/.cursor",
+    "/CLAUDE.md",
+    "/CURSOR.md",
+    "/GEMINI.md",
+    "/.windsurf",
+    "/STATE.md",
+    "/ci",
+    "/.gemini",
+    "/.claude",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.0",
+    "pytest-cov>=7.0.0",
+    "ruff>=0.15.0",
+    "ty>=0.0.34",
+    "mypy>=2.0",
+    "bandit[toml]>=1.7.0",
+    "pip-audit>=2.6.0",
+    "vulture>=2.16",
+    "vermin>=1.6.0",
+    "sqlalchemy>=2.0.30",
+    "clickhouse-connect>=1.0",
+    "confluent-kafka>=2.14.0",
+    "testcontainers[clickhouse,kafka,postgres,mysql]>=4.14.0",
+    "psycopg[binary]>=3.2",
+    "pymysql>=1.1.0",
+    "python-dotenv>=1.0.0",
+]

logreducer-3.4.0/pyproject.toml → logreducer-3.5.0/pyproject.toml.orig

@@ -6,7 +6,7 @@ build-backend = "uv_build"
 
 [project]
 name = "logreducer"
-version = "3.4.0"
+version = "3.5.0"
 authors = [
     { name = "HyperI Team", email = "dev@hyperi.io" },
 ]
@@ -286,17 +286,17 @@ no_tips = true
 
 [tool.hatch.build.targets.sdist]
 exclude = [
+    "/.github/copilot-instructions.md",
+    "/hyperi-ai",
+    "/.cursor",
     "/CLAUDE.md",
+    "/CURSOR.md",
     "/GEMINI.md",
-    "/.cursor",
     "/.windsurf",
     "/STATE.md",
-    "/CURSOR.md",
-    "/.claude",
-    "/.github/copilot-instructions.md",
-    "/hyperi-ai",
     "/ci",
     "/.gemini",
+    "/.claude",
 ]
 
 [dependency-groups]

{logreducer-3.4.0 → logreducer-3.5.0}/src/logreducer/config.py

@@ -61,6 +61,12 @@ class BigDialConfig:
     # TF-IDF matrix on a huge unique-line set, at the cost of anomaly recall
     # (rare lines may be sampled out). None = no cap (use every unique line).
     anomaly_max_rows: int | None = None
+    # Typed Drain3 masking: replace well-known value shapes (IPv4/IPv6, MAC,
+    # UUID, hex tokens, numbers) with typed template slots (<IP>, <NUM>, ...)
+    # before clustering, instead of the bare <*>. Opt-in: masking normalises
+    # lines before Drain3 sees them, so clustering can differ from the
+    # unmasked default - off keeps behaviour byte-identical.
+    typed_masking: bool = False
 
     # Temporal Control
     temporal_window_minutes: int = 60
@@ -104,14 +110,14 @@ class BigDialConfig:
         """
         if not prefixes:
             prefixes = ("LOGREDUCER",)
-        overrides: dict[str, object] = {}
+        overrides: dict[str, typing.Any] = {}
         for field in fields(cls):
             for prefix in prefixes:
                 raw = os.environ.get(f"{prefix.rstrip('_')}_{field.name.upper()}")
                 if raw is not None:
                     overrides[field.name] = _coerce_env_value(raw, field.name)
                     break
-        return cls(**overrides)  # type: ignore[arg-type]
+        return cls(**overrides)
 
 
 def _coerce_env_value(raw: str, field_name: str) -> object:

{logreducer-3.4.0 → logreducer-3.5.0}/src/logreducer/patterns.py

@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
 from drain3 import TemplateMiner
+from drain3.masking import MaskingInstruction
 from drain3.template_miner_config import TemplateMinerConfig
 from loguru import logger
 
@@ -33,6 +34,45 @@ class LogPattern:
     metadata: dict = field(default_factory=dict)
 
 
+def _typed_masking_instructions() -> list[MaskingInstruction]:
+    """Build the curated masking set for typed template slots.
+
+    Drain3 applies these to each line before clustering, so masked values
+    become literal typed tokens (``<IP>``, ``<NUM>``, ...) rather than
+    collapsing to the bare ``<*>`` wildcard. Instructions run sequentially
+    over already-masked text, so the specific shapes (IP, UUID, MAC, IPv6)
+    precede the greedy catch-alls (HEX, NUM).
+    """
+    # Zero-width token boundaries (drain3's documented idiom): mask a value
+    # delimited by non-alphanumerics, never a substring of a word.
+    start = r"(?:(?<=[^A-Za-z0-9])|^)"
+    end = r"(?:(?=[^A-Za-z0-9])|$)"
+    hex4 = r"[0-9A-Fa-f]{1,4}"
+    shapes = [
+        # IPv4 dotted quad.
+        (r"\d{1,3}(?:\.\d{1,3}){3}", "IP"),
+        # UUID before HEX, which would otherwise eat its 8/12-char runs.
+        (r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}", "UUID"),
+        # MAC (colon or dash separated) before IPv6, whose colon-hex shape overlaps.
+        (r"[0-9A-Fa-f]{2}(?:[:-][0-9A-Fa-f]{2}){5}", "MAC"),
+        # IPv6, best-effort: the full 8-group form, a '::'-compressed form
+        # (middle or trailing), then a leading-'::' form. Plain colon runs
+        # without '::' stay unmasked - that shape also matches timestamps.
+        (
+            rf"(?:{hex4}:){{7}}{hex4}"
+            rf"|(?:{hex4}:){{1,6}}:(?:{hex4}(?::{hex4}){{0,5}})?"
+            rf"|::(?:{hex4}(?::{hex4}){{0,6}})?",
+            "IPV6",
+        ),
+        # Hex token of >= 8 hex chars: 0x-prefixed, or containing at least
+        # one a-f letter so a long pure-decimal token stays NUM.
+        (r"0[xX][0-9A-Fa-f]{8,}|(?=\d*[A-Fa-f])[0-9A-Fa-f]{8,}", "HEX"),
+        # Integer (drain3's documented NUM example).
+        (r"[-+]?\d+", "NUM"),
+    ]
+    return [MaskingInstruction(f"{start}(?:{pattern}){end}", name) for pattern, name in shapes]
+
+
 class PatternExtractor:
     """Extract patterns using Drain3's online template miner."""
 
@@ -51,6 +91,10 @@ class PatternExtractor:
         # Bound the template store when configured: Drain3 LRU-evicts beyond
         # drain_max_clusters, keeping memory flat on high-cardinality logs.
         drain_config.drain_max_clusters = self.config.max_clusters
+        # Opt-in typed masking: pre-cluster masking of well-known value shapes
+        # so templates carry typed slots (<IP>, <NUM>, ...) instead of <*>.
+        if self.config.typed_masking:
+            drain_config.masking_instructions = _typed_masking_instructions()
         self.miner = TemplateMiner(config=drain_config)
 
     def extract_patterns(self, lines: Iterable[str]) -> list[LogPattern]: