codetool-shell 0.1.1__py3-none-win_amd64.whl

codetool_shell/filters/diff/reducer.py

@@ -0,0 +1,308 @@
+"""Reduce unified diff output while preserving semantic change lines."""
+
+from __future__ import annotations
+
+import re
+
+from ..text import choose_smaller_lines, join_preserving_final_newline, split_preserving_final_newline
+from .detector import find_diff_blocks, hunk_prefix_columns, is_hunk_header
+
+_LOCKFILE_NAMES = frozenset(
+    {
+        "Cargo.lock",
+        "uv.lock",
+        "package-lock.json",
+        "npm-shrinkwrap.json",
+        "yarn.lock",
+        "pnpm-lock.yaml",
+        "poetry.lock",
+        "Pipfile.lock",
+        "composer.lock",
+        "Gemfile.lock",
+        "go.sum",
+    }
+)
+_CONTEXT_SUMMARY_RE = re.compile(r"^… (?P<count>\d+) context lines?$")
+_BINARY_PAYLOAD_SUMMARY_RE = re.compile(
+    r"^… (?P<count>\d+) binary patch payload lines?$"
+)
+_ASCII_WHITESPACE = frozenset(" \t\n\r\f\v")
+
+
+def compress_diff_output(text: str) -> str:
+    """Compress high-confidence generic unified diff output."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    blocks = find_diff_blocks(lines)
+    if not blocks:
+        return text
+
+    output: list[str] = []
+    cursor = 0
+    for block_range in blocks:
+        output.extend(lines[cursor : block_range.start])
+        block = lines[block_range.start : block_range.end]
+        output.extend(choose_smaller_lines(block, _compress_diff_block(block)))
+        cursor = block_range.end
+    output.extend(lines[cursor:])
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _compress_diff_block(block: list[str]) -> list[str]:
+    output: list[str] = []
+    current_path = ""
+    i = 0
+
+    while i < len(block):
+        line = block[i]
+
+        if is_hunk_header(line):
+            output.append(line)
+            prefix_columns = hunk_prefix_columns(line)
+            i += 1
+            body: list[str] = []
+            while i < len(block) and _is_hunk_body_line(block[i], prefix_columns):
+                body.append(block[i])
+                i += 1
+            output.extend(_compress_hunk_body(body, current_path, prefix_columns))
+            continue
+
+        if line.startswith("index "):
+            i += 1
+            continue
+
+        if path := _path_from_metadata_line(line):
+            current_path = path
+
+        if line.startswith(("literal ", "delta ")):
+            output.append(line)
+            i += 1
+            omitted_payload = 0
+            while i < len(block) and not _is_binary_payload_end(block[i]):
+                omitted_payload += 1
+                i += 1
+            if omitted_payload:
+                plural = "" if omitted_payload == 1 else "s"
+                output.append(f"… {omitted_payload} binary patch payload line{plural}")
+            continue
+
+        output.append(line)
+        i += 1
+
+    return output
+
+
+def _compress_hunk_body(
+    body: list[str],
+    current_path: str,
+    prefix_columns: int,
+) -> list[str]:
+    stats = _hunk_stats(body, prefix_columns)
+
+    if _is_lockfile_path(current_path) and stats.changed_lines >= 20:
+        return [stats.summary("lockfile"), *stats.no_newline_markers]
+
+    if stats.is_whitespace_only:
+        return [stats.summary("whitespace-only"), *stats.no_newline_markers]
+
+    output: list[str] = []
+    omitted_context = 0
+
+    def flush_context() -> None:
+        nonlocal omitted_context
+        if omitted_context:
+            plural = "" if omitted_context == 1 else "s"
+            output.append(f"… {omitted_context} context line{plural}")
+            omitted_context = 0
+
+    for line in body:
+        context_summary = _context_summary_count(line)
+        if context_summary is not None:
+            omitted_context += context_summary
+            continue
+
+        if line.startswith("\\ No newline"):
+            flush_context()
+            output.append(line)
+            continue
+
+        prefix = line[:prefix_columns]
+        if "+" in prefix or "-" in prefix:
+            flush_context()
+            output.append(line)
+        else:
+            omitted_context += 1
+
+    flush_context()
+    return output
+
+
+class _HunkStats:
+    def __init__(
+        self,
+        *,
+        added: int,
+        deleted: int,
+        context: int,
+        old_text: str,
+        new_text: str,
+        can_check_whitespace_only: bool,
+        no_newline_markers: list[str],
+    ) -> None:
+        self.added = added
+        self.deleted = deleted
+        self.context = context
+        self.old_text = old_text
+        self.new_text = new_text
+        self.can_check_whitespace_only = can_check_whitespace_only
+        self.no_newline_markers = no_newline_markers
+
+    @property
+    def changed_lines(self) -> int:
+        return self.added + self.deleted
+
+    @property
+    def is_whitespace_only(self) -> bool:
+        if not self.can_check_whitespace_only or not self.added or not self.deleted:
+            return False
+        return _remove_ascii_whitespace(self.old_text) == _remove_ascii_whitespace(
+            self.new_text
+        )
+
+    def summary(self, kind: str) -> str:
+        return (
+            f"… {kind} hunk omitted: "
+            f"+{self.added} -{self.deleted} context={self.context}"
+        )
+
+
+def _hunk_stats(body: list[str], prefix_columns: int) -> _HunkStats:
+    added = 0
+    deleted = 0
+    context = 0
+    old_parts: list[str] = []
+    new_parts: list[str] = []
+    can_check_whitespace_only = prefix_columns == 1
+    no_newline_markers: list[str] = []
+
+    for line in body:
+        context_summary = _context_summary_count(line)
+        if context_summary is not None:
+            context += context_summary
+            can_check_whitespace_only = False
+            continue
+
+        if line.startswith("\\ No newline"):
+            no_newline_markers.append(line)
+            continue
+
+        prefix = line[:prefix_columns]
+        content = line[prefix_columns:]
+        if "+" in prefix:
+            added += 1
+            new_parts.append(content)
+        if "-" in prefix:
+            deleted += 1
+            old_parts.append(content)
+        if "+" not in prefix and "-" not in prefix:
+            context += 1
+            old_parts.append(content)
+            new_parts.append(content)
+
+    return _HunkStats(
+        added=added,
+        deleted=deleted,
+        context=context,
+        old_text="\n".join(old_parts),
+        new_text="\n".join(new_parts),
+        can_check_whitespace_only=can_check_whitespace_only,
+        no_newline_markers=no_newline_markers,
+    )
+
+
+def _is_hunk_body_line(line: str, prefix_columns: int) -> bool:
+    if _context_summary_count(line) is not None or line.startswith("\\ No newline"):
+        return True
+    if len(line) < prefix_columns:
+        return False
+    return all(char in " +-" for char in line[:prefix_columns])
+
+
+def _context_summary_count(line: str) -> int | None:
+    match = _CONTEXT_SUMMARY_RE.match(line)
+    if match is None:
+        return None
+    return int(match.group("count"))
+
+
+def _remove_ascii_whitespace(value: str) -> str:
+    return "".join(char for char in value if char not in _ASCII_WHITESPACE)
+
+
+def _path_from_metadata_line(line: str) -> str:
+    if line.startswith("diff --git "):
+        parts = line.split()
+        if len(parts) >= 4:
+            return _clean_diff_path(parts[3]) or _clean_diff_path(parts[2])
+    if line.startswith("diff --cc "):
+        return _clean_diff_path(line.removeprefix("diff --cc "))
+    if line.startswith("diff --combined "):
+        return _clean_diff_path(line.removeprefix("diff --combined "))
+    if line.startswith("Index: "):
+        return _clean_diff_path(line.removeprefix("Index: "))
+    for prefix in ("--- ", "+++ ", "rename to ", "copy to "):
+        if line.startswith(prefix):
+            return _clean_diff_path(line.removeprefix(prefix))
+    return ""
+
+
+def _clean_diff_path(raw_path: str) -> str:
+    path = raw_path.strip().split("\t", 1)[0].strip().strip('"')
+    if path == "/dev/null":
+        return ""
+    if len(path) >= 2 and path[1] == "/" and path[0] in "abciw":
+        path = path[2:]
+    return path
+
+
+def _is_lockfile_path(path: str) -> bool:
+    normalized = path.replace("\\", "/")
+    name = normalized.rsplit("/", 1)[-1]
+    return name in _LOCKFILE_NAMES
+
+
+def _is_binary_payload_end(line: str) -> bool:
+    return (
+        is_hunk_header(line)
+        or _is_diff_metadata_line(line)
+        or _BINARY_PAYLOAD_SUMMARY_RE.match(line) is not None
+    )
+
+
+def _is_diff_metadata_line(line: str) -> bool:
+    return line.startswith(
+        (
+            "diff --git ",
+            "diff --cc ",
+            "diff --combined ",
+            "Index: ",
+            "--- ",
+            "+++ ",
+            "index ",
+            "new file mode ",
+            "deleted file mode ",
+            "old mode ",
+            "new mode ",
+            "similarity index ",
+            "dissimilarity index ",
+            "rename from ",
+            "rename to ",
+            "copy from ",
+            "copy to ",
+            "Binary files ",
+            "GIT binary patch",
+            "literal ",
+            "delta ",
+        )
+    )

codetool_shell/filters/generic_log/__init__.py

@@ -0,0 +1,7 @@
+"""Generic application log compression."""
+
+from __future__ import annotations
+
+from .reducer import compress_generic_log_output
+
+__all__ = ["compress_generic_log_output"]

codetool_shell/filters/generic_log/detector.py

@@ -0,0 +1,175 @@
+"""Detect conservative generic application log shapes."""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+
+from .summary import (
+    ParsedLogLine,
+    is_important_line,
+    is_low_severity,
+    normalized_message_pattern,
+    parse_log_line,
+)
+
+
+@dataclass(frozen=True)
+class GenericLog:
+    """A detected generic app log and repeated low-severity patterns."""
+
+    parsed_lines: tuple[ParsedLogLine | None, ...]
+    repeated_low_patterns: frozenset[tuple[str, str]]
+
+
+_MIN_NONBLANK_LINES = 12
+_MIN_PREFIXED_LINES = 8
+_MIN_REPEATED_LOW_LINES = 5
+
+
+def detect_generic_log(lines: list[str]) -> GenericLog | None:
+    """Return a strong generic app-log signal, or ``None``."""
+
+    if _has_other_filter_ownership(lines):
+        return None
+
+    nonblank_indices = [index for index, line in enumerate(lines) if line.strip()]
+    if len(nonblank_indices) < _MIN_NONBLANK_LINES:
+        return None
+
+    parsed_lines = tuple(parse_log_line(line) for line in lines)
+    parsed_nonblank = [
+        parsed_lines[index]
+        for index in nonblank_indices
+        if parsed_lines[index] is not None
+    ]
+    prefixed_count = len(parsed_nonblank)
+    if prefixed_count < _MIN_PREFIXED_LINES:
+        return None
+    if prefixed_count * 100 < len(nonblank_indices) * 65:
+        return None
+
+    pattern_counts: Counter[tuple[str, str]] = Counter()
+    for index, parsed in enumerate(parsed_lines):
+        if parsed is None or not is_low_severity(parsed):
+            continue
+        if is_important_line(lines[index], parsed):
+            continue
+        pattern_counts[(parsed.level, normalized_message_pattern(parsed))] += 1
+
+    repeated = frozenset(
+        pattern for pattern, count in pattern_counts.items() if count >= 3
+    )
+    repeated_total = sum(pattern_counts[pattern] for pattern in repeated)
+    if repeated_total < _MIN_REPEATED_LOW_LINES:
+        return None
+
+    return GenericLog(
+        parsed_lines=parsed_lines,
+        repeated_low_patterns=repeated,
+    )
+
+
+def _has_other_filter_ownership(lines: list[str]) -> bool:
+    text = "\n".join(lines)
+    lower = text.lower()
+    stripped = [line.strip() for line in lines if line.strip()]
+    if not stripped:
+        return True
+
+    if any(
+        marker in lower
+        for marker in (
+            "test output:",
+            "ci log:",
+            "python traceback:",
+            "traceback (most recent call last)",
+            "test session starts",
+            "short test summary info",
+            "diff --git ",
+        )
+    ):
+        return True
+    if _looks_like_json_lines(stripped):
+        return True
+    if _looks_like_test_runner(stripped, lower):
+        return True
+    if _looks_like_ci_log(stripped):
+        return True
+    if _looks_like_table(stripped):
+        return True
+    if _looks_like_path_location_output(stripped):
+        return True
+    if _looks_like_build_or_package_output(stripped, lower):
+        return True
+    return False
+
+
+def _looks_like_json_lines(lines: list[str]) -> bool:
+    json_like = sum(
+        1 for line in lines if line.startswith("{") and line.endswith("}") and ":" in line
+    )
+    return json_like * 100 >= len(lines) * 70
+
+
+def _looks_like_test_runner(lines: list[str], lower: str) -> bool:
+    return (
+        "pytest" in lower
+        or any(line.startswith(("Test Files", "Tests", "FAIL ")) for line in lines)
+        or any(line.startswith("test result:") for line in lines)
+        or (
+            any(line.startswith("Ran ") and " tests in " in line for line in lines)
+            and any(line == "OK" or line.startswith("FAILED (") for line in lines)
+        )
+        or any(line.startswith("Running ") and " tests using " in line for line in lines)
+    )
+
+
+def _looks_like_ci_log(lines: list[str]) -> bool:
+    return any(
+        line.startswith(("##[", "::group::", "::endgroup::", "::error", "::warning"))
+        or "\t" in line
+        and "T" in line
+        and "Z " in line
+        for line in lines
+    )
+
+
+def _looks_like_table(lines: list[str]) -> bool:
+    pipe_rows = sum(1 for line in lines if "|" in line)
+    separator_rows = sum(1 for line in lines if set(line.strip()) <= {"-", "+", "|"})
+    return pipe_rows >= 3 and (pipe_rows * 100 >= len(lines) * 55 or separator_rows >= 1)
+
+
+def _looks_like_path_location_output(lines: list[str]) -> bool:
+    matches = sum(1 for line in lines if _looks_like_path_location_line(line))
+    return matches >= 2 and matches * 100 >= len(lines) * 45
+
+
+def _looks_like_path_location_line(line: str) -> bool:
+    parts = line.split(":")
+    if len(parts) < 3:
+        return False
+    return any(part.isdigit() for part in parts[1:3]) and (
+        "/" in parts[0] or "." in parts[0]
+    )
+
+
+def _looks_like_build_or_package_output(lines: list[str], lower: str) -> bool:
+    build_markers = (
+        "compiling ",
+        "checking ",
+        "finished ",
+        "could not compile",
+        "error[",
+        "warning[",
+        "downloading ",
+        "downloaded ",
+        "installing collected packages",
+        "successfully installed",
+        "added ",
+        "packages audited",
+    )
+    return any(marker in lower for marker in build_markers) and not any(
+        parse_log_line(line) is not None for line in lines
+    )

codetool_shell/filters/generic_log/reducer.py

@@ -0,0 +1,99 @@
+"""Conservative reducer for generic application logs."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import GenericLog, detect_generic_log
+from .summary import (
+    is_important_line,
+    is_low_severity,
+    normalize_selected_line,
+    normalized_message_pattern,
+)
+
+_CONTEXT_LINES = 2
+
+
+def compress_generic_log_output(text: str) -> str:
+    """Compress repeated low-severity application log lines."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    signal = detect_generic_log(lines)
+    if signal is None:
+        return text
+
+    candidate_lines = _reduce_lines(lines, signal)
+    if len(candidate_lines) < 3:
+        return text
+
+    candidate = join_preserving_final_newline(candidate_lines, final_newline)
+    if score(candidate) < score(text):
+        return candidate
+    return text
+
+
+def _reduce_lines(lines: list[str], signal: GenericLog) -> list[str]:
+    nonblank_indices = [index for index, line in enumerate(lines) if line.strip()]
+    head = set(nonblank_indices[:_CONTEXT_LINES])
+    tail = set(nonblank_indices[-_CONTEXT_LINES:])
+    selected = ["generic log: application summary"]
+    omitted: Counter[tuple[str, str]] = Counter()
+
+    def flush_omitted() -> None:
+        if not omitted:
+            return
+        selected.extend(_format_omitted(omitted))
+        omitted.clear()
+
+    for index, line in enumerate(lines):
+        if not line.strip():
+            continue
+
+        parsed = signal.parsed_lines[index]
+        pattern: tuple[str, str] | None = None
+        if parsed is not None and is_low_severity(parsed):
+            pattern = (parsed.level, normalized_message_pattern(parsed))
+
+        should_omit = (
+            pattern is not None
+            and pattern in signal.repeated_low_patterns
+            and index not in head
+            and index not in tail
+            and not is_important_line(line, parsed)
+        )
+        if should_omit and pattern is not None:
+            omitted[pattern] += 1
+            continue
+
+        flush_omitted()
+        selected.append(normalize_selected_line(line))
+
+    flush_omitted()
+    return _drop_adjacent_duplicates(selected)
+
+
+def _format_omitted(omitted: Counter[tuple[str, str]]) -> list[str]:
+    lines: list[str] = []
+    for (level, pattern), count in omitted.most_common(3):
+        plural = "" if count == 1 else "s"
+        lines.append(f"… {count} repeated {level} line{plural} omitted: {pattern}")
+    remaining_patterns = len(omitted) - 3
+    if remaining_patterns > 0:
+        remaining_count = sum(count for _, count in omitted.most_common()[3:])
+        plural = "" if remaining_count == 1 else "s"
+        lines.append(
+            f"… {remaining_count} repeated low-severity line{plural} omitted "
+            f"across {remaining_patterns} more patterns"
+        )
+    return lines
+
+
+def _drop_adjacent_duplicates(lines: list[str]) -> list[str]:
+    output: list[str] = []
+    for line in lines:
+        if output and output[-1] == line:
+            continue
+        output.append(line)
+    return output