codetool-shell 0.1.1__py3-none-win_arm64.whl

codetool_shell/filters/log_template/template.py

@@ -0,0 +1,280 @@
+"""Template construction and scalar normalization helpers."""
+
+from __future__ import annotations
+
+from .constants import (
+    _KNOWN_LEVELS,
+    _MAX_CAPTURED_VALUES,
+    _MAX_VALUE_LEN,
+    _OPERATIONAL_WORDS,
+    _SECRET_KEY_PARTS,
+)
+from .types import LineTemplate
+
+
+def template_for_line(line: str) -> LineTemplate | None:
+    stripped = line.strip()
+    if not stripped:
+        return None
+
+    tokens = stripped.split()
+    log_like = _is_log_like(tokens)
+    has_key_value = any(_parse_key_value_token(token) is not None for token in tokens)
+    stable_text = _is_stable_text_candidate(tokens)
+    if not (log_like or has_key_value or stable_text):
+        return None
+
+    allow_standalone = log_like or stable_text
+    normalized_tokens: list[str] = []
+    values: list[str] = []
+
+    for token in tokens:
+        normalized = _normalize_token(token, allow_standalone=allow_standalone)
+        if normalized is None:
+            return None
+        normalized_token, captured = normalized
+        normalized_tokens.append(normalized_token)
+        values.extend(captured)
+
+    if not (1 <= len(values) <= _MAX_CAPTURED_VALUES):
+        return None
+    if any(not _is_safe_printable_value(value) for value in values):
+        return None
+
+    template = " ".join(normalized_tokens)
+    if not _has_enough_static_text(template):
+        return None
+    return LineTemplate(template=template, values=tuple(values))
+
+def _normalize_token(
+    token: str, *, allow_standalone: bool
+) -> tuple[str, tuple[str, ...]] | None:
+    leading, core, trailing = _split_wrapping_punctuation(token)
+    if not core:
+        return token, ()
+
+    key_value = _parse_key_value_core(core)
+    if key_value is not None:
+        key, value = key_value
+        if _is_secret_key(key):
+            return None
+        normalized = _normalize_value(value, key=key)
+        if normalized is None:
+            return None
+        replacement, captured = normalized
+        return f"{leading}{key}={replacement}{trailing}", (captured,)
+
+    if _looks_like_timestamp(core):
+        return f"{leading}<ts>{trailing}", (core,)
+    if _looks_like_ipv4(core):
+        return f"{leading}<ip>{trailing}", (core,)
+
+    if allow_standalone:
+        number = _split_number_with_unit(core)
+        if number is not None:
+            captured, suffix = number
+            return f"{leading}<num>{suffix}{trailing}", (captured,)
+        if _looks_like_id_value(core):
+            return f"{leading}<id>{trailing}", (core,)
+
+    return token, ()
+
+def _normalize_value(value: str, *, key: str) -> tuple[str, str] | None:
+    if not _is_safe_printable_value(value):
+        return None
+    if _looks_like_timestamp(value):
+        return "<ts>", value
+    if _looks_like_ipv4(value):
+        return "<ip>", value
+
+    number = _split_number_with_unit(value)
+    if number is not None:
+        captured, suffix = number
+        return f"<num>{suffix}", captured
+
+    if _is_id_key(key) and _is_safe_id_value(value):
+        return "<id>", value
+    if _looks_like_id_value(value):
+        return "<id>", value
+    if _is_safe_scalar_value(value):
+        return "<value>", value
+    return None
+
+def _split_wrapping_punctuation(token: str) -> tuple[str, str, str]:
+    start = 0
+    end = len(token)
+    while start < end and token[start] in "([{":
+        start += 1
+    while end > start and token[end - 1] in ".,;)]}":
+        end -= 1
+    return token[:start], token[start:end], token[end:]
+
+def _parse_key_value_token(token: str) -> tuple[str, str] | None:
+    _, core, _ = _split_wrapping_punctuation(token)
+    return _parse_key_value_core(core)
+
+def _parse_key_value_core(core: str) -> tuple[str, str] | None:
+    if "=" not in core:
+        return None
+    key, value = core.split("=", 1)
+    if not key or not value or not _is_valid_key(key):
+        return None
+    return key, value
+
+def _is_valid_key(key: str) -> bool:
+    if len(key) > 40 or not (key[0].isalpha() or key[0] == "_"):
+        return False
+    return all(char.isalnum() or char in "_.-" for char in key)
+
+def _is_secret_key(key: str) -> bool:
+    normalized = key.lower().replace("-", "_").replace(".", "_")
+    return any(part in normalized for part in _SECRET_KEY_PARTS)
+
+def _is_id_key(key: str) -> bool:
+    normalized = key.lower()
+    return any(
+        part in normalized
+        for part in ("id", "uuid", "hash", "sha", "trace", "span", "request", "job", "task")
+    )
+
+def _is_log_like(tokens: list[str]) -> bool:
+    if not tokens:
+        return False
+
+    first = tokens[0].strip("[]")
+    if _looks_like_timestamp(first):
+        if len(tokens) == 1:
+            return True
+        return _normalize_level(tokens[1].strip("[]:=-")) is not None
+
+    first_level = _normalize_level(first.strip(":=-"))
+    if first_level is not None and (
+        tokens[0].startswith("[") or tokens[0].endswith((": ", ":", "-")) or len(tokens) > 1
+    ):
+        return tokens[0].startswith("[") or tokens[0].endswith((":","-"))
+
+    if _parse_key_value_core(tokens[0]) is not None:
+        key, value = _parse_key_value_core(tokens[0]) or ("", "")
+        return key.lower() == "level" and _normalize_level(value) is not None
+    return False
+
+def _normalize_level(value: str) -> str | None:
+    upper = value.upper()
+    if upper == "WARNING":
+        upper = "WARN"
+    if upper in _KNOWN_LEVELS:
+        return upper
+    return None
+
+def _is_stable_text_candidate(tokens: list[str]) -> bool:
+    if len(tokens) < 4:
+        return False
+    text = " ".join(tokens)
+    if text.endswith(".") or len(text) > 160:
+        return False
+    lowered_words = {token.strip(".,:;()[]{}").lower() for token in tokens}
+    return bool(lowered_words & _OPERATIONAL_WORDS)
+
+def _has_enough_static_text(template: str) -> bool:
+    static = template
+    for placeholder in ("<num>", "<id>", "<ip>", "<ts>", "<value>"):
+        static = static.replace(placeholder, "")
+    static_nonspace = "".join(char for char in static if not char.isspace())
+    alpha_count = sum(1 for char in static_nonspace if char.isalpha())
+    return len(static_nonspace) >= 12 and alpha_count >= 6
+
+def _is_safe_printable_value(value: str) -> bool:
+    if not value or len(value) > _MAX_VALUE_LEN:
+        return False
+    return all(33 <= ord(char) <= 126 for char in value) and not any(
+        char in value for char in "<>{}[]/\\"
+    )
+
+def _is_safe_scalar_value(value: str) -> bool:
+    return (
+        len(value) <= 24
+        and any(char.isalpha() for char in value)
+        and all(char.isalnum() or char in "_.-" for char in value)
+    )
+
+def _is_safe_id_value(value: str) -> bool:
+    return (
+        len(value) <= _MAX_VALUE_LEN
+        and any(char.isalnum() for char in value)
+        and all(char.isalnum() or char in "_.:-" for char in value)
+    )
+
+def _looks_like_id_value(value: str) -> bool:
+    if not _is_safe_id_value(value):
+        return False
+    has_alpha = any(char.isalpha() for char in value)
+    has_digit = any(char.isdigit() for char in value)
+    if not (has_alpha and has_digit):
+        return False
+    if "-" in value or "_" in value:
+        return len(value) >= 6
+    hex_chars = set("0123456789abcdefABCDEF")
+    return len(value) >= 12 and all(char in hex_chars for char in value)
+
+def _split_number_with_unit(value: str) -> tuple[str, str] | None:
+    if not value:
+        return None
+    index = 0
+    if value[index] in "+-":
+        index += 1
+        if index >= len(value):
+            return None
+
+    digit_count = 0
+    while index < len(value) and value[index].isdigit():
+        index += 1
+        digit_count += 1
+    if index < len(value) and value[index] == ".":
+        dot_index = index
+        index += 1
+        fractional_digits = 0
+        while index < len(value) and value[index].isdigit():
+            index += 1
+            fractional_digits += 1
+        if fractional_digits == 0:
+            index = dot_index
+    if digit_count == 0:
+        return None
+
+    number = value[:index]
+    suffix = value[index:]
+    if suffix and (
+        len(suffix) > 8
+        or not all(char.isalpha() or char in "%µ" for char in suffix)
+    ):
+        return None
+    return number, suffix
+
+def _looks_like_timestamp(value: str) -> bool:
+    if len(value) < 19:
+        return False
+    if not (
+        value[4] == "-"
+        and value[7] == "-"
+        and value[10] in "T "
+        and value[13] == ":"
+        and value[16] == ":"
+    ):
+        return False
+    for index in (0, 1, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18):
+        if not value[index].isdigit():
+            return False
+    return all(char.isdigit() or char in ".,Zz+-:" for char in value[19:])
+
+def _looks_like_ipv4(value: str) -> bool:
+    parts = value.split(".")
+    if len(parts) != 4:
+        return False
+    for part in parts:
+        if not part.isdigit():
+            return False
+        number = int(part)
+        if number > 255:
+            return False
+    return True
+

codetool_shell/filters/log_template/types.py

@@ -0,0 +1,21 @@
+"""Shared types for repeated log-template compaction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class LineTemplate:
+    """A line reduced to a stable template plus captured scalar values."""
+
+    template: str
+    values: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class LogTemplateSignal:
+    """Detected repeated templates in a log-like output."""
+
+    line_templates: tuple[LineTemplate | None, ...]
+    repeated_templates: frozenset[str]

codetool_shell/filters/opaque_payload/__init__.py

@@ -0,0 +1,7 @@
+"""Opaque blob and long payload compression."""
+
+from __future__ import annotations
+
+from .reducer import compress_opaque_payload_output
+
+__all__ = ["compress_opaque_payload_output"]