codetool-shell 0.1.1__py3-none-win_amd64.whl

codetool_shell/filters/log_template/constants.py

@@ -0,0 +1,76 @@
+"""Constants for repeated log-template compaction."""
+
+from __future__ import annotations
+
+_MIN_NONBLANK_LINES = 16
+_MIN_TEMPLATE_MATCHES = 8
+_MIN_TEMPLATED_LINES = 12
+_MIN_TEMPLATED_PERCENT = 35
+_MAX_CAPTURED_VALUES = 5
+_MAX_VALUE_LEN = 48
+_MAX_SHORT_TUPLES = 6
+
+_KNOWN_LEVELS = frozenset(
+    {
+        "TRACE",
+        "DEBUG",
+        "INFO",
+        "NOTICE",
+        "WARN",
+        "WARNING",
+        "ERROR",
+        "FATAL",
+        "PANIC",
+        "CRITICAL",
+        "ALERT",
+        "EMERG",
+    }
+)
+_LOW_LEVELS = frozenset({"TRACE", "DEBUG", "INFO"})
+_IMPORTANT_KV_KEYS = frozenset({"level", "severity", "result", "outcome"})
+_IMPORTANT_KV_VALUES = frozenset(
+    {
+        "warn",
+        "warning",
+        "error",
+        "fatal",
+        "panic",
+        "critical",
+        "alert",
+        "emerg",
+        "fail",
+        "failed",
+        "failure",
+    }
+)
+_SECRET_KEY_PARTS = (
+    "password",
+    "passwd",
+    "secret",
+    "token",
+    "api_key",
+    "apikey",
+    "authorization",
+    "credential",
+    "private_key",
+    "access_key",
+)
+_OPERATIONAL_WORDS = frozenset(
+    {
+        "batch",
+        "chunk",
+        "completed",
+        "event",
+        "handled",
+        "item",
+        "iteration",
+        "job",
+        "message",
+        "poll",
+        "processed",
+        "record",
+        "request",
+        "shard",
+        "task",
+    }
+)

codetool_shell/filters/log_template/detector.py

@@ -0,0 +1,331 @@
+"""Detection and ownership guards for repeated log-template compaction."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from .constants import (
+    _IMPORTANT_KV_KEYS,
+    _IMPORTANT_KV_VALUES,
+    _LOW_LEVELS,
+    _MIN_NONBLANK_LINES,
+    _MIN_TEMPLATE_MATCHES,
+    _MIN_TEMPLATED_LINES,
+    _MIN_TEMPLATED_PERCENT,
+)
+from .template import (
+    _is_secret_key,
+    _looks_like_timestamp,
+    _normalize_level,
+    _parse_key_value_core,
+    _parse_key_value_token,
+    _split_wrapping_punctuation,
+    template_for_line,
+)
+from .types import LogTemplateSignal
+
+
+def detect_log_templates(lines: list[str]) -> LogTemplateSignal | None:
+    if _has_other_filter_ownership(lines):
+        return None
+
+    nonblank_count = sum(1 for line in lines if line.strip())
+    if nonblank_count < _MIN_NONBLANK_LINES:
+        return None
+
+    line_templates = tuple(
+        None
+        if _is_important_line(line.strip()) or _has_forbidden_fragment(line.strip())
+        else template_for_line(line)
+        for line in lines
+    )
+    counts: Counter[str] = Counter(
+        candidate.template for candidate in line_templates if candidate is not None
+    )
+    repeated = frozenset(
+        template for template, count in counts.items() if count >= _MIN_TEMPLATE_MATCHES
+    )
+    if not repeated:
+        return None
+
+    templated_total = sum(counts[template] for template in repeated)
+    if templated_total < _MIN_TEMPLATED_LINES:
+        return None
+    if templated_total * 100 < nonblank_count * _MIN_TEMPLATED_PERCENT:
+        return None
+
+    return LogTemplateSignal(
+        line_templates=line_templates,
+        repeated_templates=repeated,
+    )
+
+
+def _is_important_line(line: str) -> bool:
+    lower = line.lower()
+    if (
+        " warn" in f" {lower}"
+        or " warning" in f" {lower}"
+        or " error" in f" {lower}"
+        or " fatal" in f" {lower}"
+        or " panic" in f" {lower}"
+        or "exception" in lower
+        or "traceback" in lower
+        or "stack trace" in lower
+        or "failed" in lower
+        or "failure" in lower
+        or "exit code" in lower
+    ):
+        return True
+
+    tokens = line.split()
+    if tokens:
+        if _line_has_important_key_value(tokens):
+            return True
+        first = tokens[0].strip("[]")
+        if _looks_like_timestamp(first) and len(tokens) > 1:
+            level = _normalize_level(tokens[1].strip("[]:=-"))
+            return level is not None and level not in _LOW_LEVELS
+        level = _normalize_level(first.strip(":=-"))
+        if level is not None:
+            return level not in _LOW_LEVELS
+    return False
+
+def _line_has_important_key_value(tokens: list[str]) -> bool:
+    for index, token in enumerate(tokens):
+        _, core, _ = _split_wrapping_punctuation(token)
+        key_value = _parse_key_value_core(core)
+        if key_value is not None:
+            key, value = key_value
+            if _is_important_kv(key, value):
+                return True
+            continue
+
+        if ":" not in core:
+            continue
+        key, value = core.split(":", 1)
+        if value:
+            if _is_important_kv(key, value):
+                return True
+        elif index + 1 < len(tokens) and _is_important_kv(key, tokens[index + 1]):
+            return True
+    return False
+
+def _is_important_kv(key: str, value: str) -> bool:
+    normalized_key = key.lower().replace("-", "_").replace(".", "_")
+    if normalized_key not in _IMPORTANT_KV_KEYS:
+        return False
+    normalized_level = _normalize_level(value.strip("[],:;"))
+    if normalized_level is not None:
+        return normalized_level not in _LOW_LEVELS
+    return value.strip("[],:;").lower() in _IMPORTANT_KV_VALUES
+
+def _has_forbidden_fragment(line: str) -> bool:
+    lower = line.lower()
+    return (
+        "://" in line
+        or "www." in lower
+        or _line_has_secret_like_key(line)
+        or _line_has_path(line)
+        or any(marker in lower for marker in ("file \"", " at ", "caused by:"))
+    )
+
+def _line_has_secret_like_key(line: str) -> bool:
+    tokens = line.split()
+    for index, token in enumerate(tokens):
+        _, core, _ = _split_wrapping_punctuation(token)
+        key_value = _parse_key_value_token(token)
+        if key_value is not None and _is_secret_key(key_value[0]):
+            return True
+        if ":" not in core:
+            continue
+        key, value = core.split(":", 1)
+        if _is_secret_key(key) and (value or index + 1 < len(tokens)):
+            return True
+    return False
+
+def _line_has_path(line: str) -> bool:
+    file_suffixes = (
+        ".py",
+        ".rs",
+        ".ts",
+        ".tsx",
+        ".js",
+        ".jsx",
+        ".json",
+        ".yaml",
+        ".yml",
+        ".toml",
+        ".log",
+        ".txt",
+        ".html",
+        ".xml",
+        ".csv",
+        ".md",
+        ".c",
+        ".cpp",
+        ".go",
+        ".rb",
+        ".zip",
+        ".png",
+    )
+    lockfile_names = (
+        "cargo.lock",
+        "package-lock.json",
+        "yarn.lock",
+        "pnpm-lock.yaml",
+        "poetry.lock",
+        "go.sum",
+    )
+    for token in line.split():
+        cleaned = token.strip("\"'(),;[]")
+        if "://" in cleaned:
+            return True
+        lower = cleaned.lower()
+        if (
+            lower in lockfile_names
+            or any(lower.endswith(f"={name}") for name in lockfile_names)
+            or any(lower.endswith(f"/{name}") for name in lockfile_names)
+            or any(lower.endswith(f"\\{name}") for name in lockfile_names)
+            or any(
+                lower.endswith(suffix) or f"{suffix}:" in lower
+                for suffix in file_suffixes
+            )
+        ):
+            return True
+        if "/" not in cleaned and "\\" not in cleaned:
+            continue
+        slash_parts = [part for part in cleaned.replace("\\", "/").split("/") if part]
+        if (
+            cleaned.startswith(("/", "./", "../", "~/"))
+            or ":\\" in cleaned
+            or len(slash_parts) >= 3
+        ):
+            return True
+    return False
+
+def _has_other_filter_ownership(lines: list[str]) -> bool:
+    stripped = [line.strip() for line in lines if line.strip()]
+    if not stripped:
+        return True
+
+    lower = "\n".join(stripped).lower()
+    if any(
+        marker in lower
+        for marker in (
+            "log templates: repeated line summary",
+            "generic log:",
+            "test output:",
+            "ci log:",
+            "python traceback:",
+            "traceback (most recent call last)",
+            "short test summary info",
+            "diff --git ",
+            "build diagnostics:",
+            "package manager:",
+            "jsonl records=",
+        )
+    ):
+        return True
+    if any(line.startswith("… ") and " omitted" in line for line in stripped):
+        return True
+    if _looks_like_json_output(stripped):
+        return True
+    if _looks_like_diff(stripped):
+        return True
+    if _looks_like_test_runner(stripped, lower):
+        return True
+    if _looks_like_ci_log(stripped):
+        return True
+    if _looks_like_table(stripped):
+        return True
+    if _looks_like_path_location_output(stripped):
+        return True
+    if _looks_like_build_or_package_output(stripped, lower):
+        return True
+    if any(_line_has_secret_like_key(line) for line in stripped):
+        return True
+    return False
+
+def _looks_like_json_output(lines: list[str]) -> bool:
+    if len(lines) == 1:
+        line = lines[0]
+        return line.startswith(("{", "[")) and line.endswith(("}", "]")) and ":" in line
+
+    joined = "\n".join(lines)
+    if joined[:1] in ("{", "[") and joined[-1:] in ("}", "]") and ":" in joined:
+        return True
+    json_like = sum(
+        1 for line in lines if line.startswith("{") and line.endswith("}") and ":" in line
+    )
+    return json_like * 100 >= len(lines) * 70
+
+def _looks_like_diff(lines: list[str]) -> bool:
+    return (
+        any(line.startswith(("diff --git ", "Index: ", "@@ ")) for line in lines)
+        or (
+            any(line.startswith("--- ") for line in lines)
+            and any(line.startswith("+++ ") for line in lines)
+        )
+    )
+
+def _looks_like_test_runner(lines: list[str], lower: str) -> bool:
+    return (
+        "pytest" in lower
+        or "test session starts" in lower
+        or any(line.startswith(("Test Files", "Tests", "FAIL ")) for line in lines)
+        or any(line.startswith("test result:") for line in lines)
+        or (
+            any(line.startswith("Ran ") and " tests in " in line for line in lines)
+            and any(line == "OK" or line.startswith("FAILED (") for line in lines)
+        )
+        or any(line.startswith("Running ") and " tests using " in line for line in lines)
+    )
+
+def _looks_like_ci_log(lines: list[str]) -> bool:
+    return any(
+        line.startswith(("##[", "::group::", "::endgroup::", "::error", "::warning"))
+        or "\t" in line
+        and "T" in line
+        and "Z " in line
+        for line in lines
+    )
+
+def _looks_like_table(lines: list[str]) -> bool:
+    pipe_rows = sum(1 for line in lines if "|" in line)
+    separator_rows = sum(1 for line in lines if set(line.strip()) <= {"-", "+", "|"})
+    return pipe_rows >= 3 and (pipe_rows * 100 >= len(lines) * 55 or separator_rows >= 1)
+
+def _looks_like_path_location_output(lines: list[str]) -> bool:
+    matches = sum(1 for line in lines if _looks_like_path_location_line(line))
+    return matches >= 2 and matches * 100 >= len(lines) * 45
+
+def _looks_like_path_location_line(line: str) -> bool:
+    parts = line.split(":")
+    if len(parts) < 3:
+        return False
+    return any(part.isdigit() for part in parts[1:3]) and (
+        "/" in parts[0] or "." in parts[0]
+    )
+
+def _looks_like_build_or_package_output(lines: list[str], lower: str) -> bool:
+    markers = (
+        "compiling ",
+        "checking ",
+        "finished ",
+        "could not compile",
+        "error[",
+        "warning[",
+        "downloading ",
+        "downloaded ",
+        "installing collected packages",
+        "successfully installed",
+        "added ",
+        "packages audited",
+        "resolved ",
+        "prepared ",
+        "installed ",
+        "npm warn",
+        "npm err",
+    )
+    return any(marker in lower for marker in markers)
+

codetool_shell/filters/log_template/reducer.py

@@ -0,0 +1,78 @@
+"""Rendering and orchestration for repeated log-template compaction."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .constants import _MAX_SHORT_TUPLES
+from .detector import detect_log_templates
+from .types import LogTemplateSignal
+
+
+def compress_log_template_output(text: str) -> str:
+    """Compact repeated scalar-varied log lines into template/value blocks."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    signal = detect_log_templates(lines)
+    if signal is None:
+        return text
+
+    candidate_lines = _render_template_blocks(lines, signal)
+    if candidate_lines is None:
+        return text
+
+    candidate = join_preserving_final_newline(candidate_lines, final_newline)
+    if score(candidate) < score(text):
+        return candidate
+    return text
+
+
+def _render_template_blocks(
+    lines: list[str], signal: LogTemplateSignal
+) -> list[str] | None:
+    output: list[str] = []
+    emitted = False
+    index = 0
+
+    while index < len(lines):
+        candidate = signal.line_templates[index]
+        if candidate is None or candidate.template not in signal.repeated_templates:
+            output.append(lines[index])
+            index += 1
+            continue
+
+        run_template = candidate.template
+        run_values: list[tuple[str, ...]] = []
+        cursor = index
+        while cursor < len(lines):
+            run_candidate = signal.line_templates[cursor]
+            if run_candidate is None or run_candidate.template != run_template:
+                break
+            run_values.append(run_candidate.values)
+            cursor += 1
+
+        if len(run_values) < 2:
+            output.append(lines[index])
+            index += 1
+            continue
+
+        output.append(f"… {len(run_values)} templated lines: {run_template}")
+        output.append(f"  values: {_format_value_tuples(run_values)}")
+        emitted = True
+        index = cursor
+
+    if not emitted:
+        return None
+    return ["log templates: repeated line summary", *output]
+
+def _format_value_tuples(values: list[tuple[str, ...]]) -> str:
+    if len(values) <= _MAX_SHORT_TUPLES:
+        return "; ".join(_format_tuple(value_tuple) for value_tuple in values)
+
+    omitted = len(values) - 6
+    rendered = [_format_tuple(value_tuple) for value_tuple in values[:3]]
+    rendered.append(f"… {omitted} omitted")
+    rendered.extend(_format_tuple(value_tuple) for value_tuple in values[-3:])
+    return "; ".join(rendered)
+
+def _format_tuple(values: tuple[str, ...]) -> str:
+    return f"({', '.join(values)})"