codetool-shell 0.1.1__py3-none-win_amd64.whl

codetool_shell/filters/ci_job_log/reducer.py

@@ -0,0 +1,99 @@
+"""Conservative reducer for CI/job log output."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import CiLogSignal, detect_ci_job_log
+from .summary import (
+    extract_group_step,
+    is_group_end,
+    is_important_line,
+    normalize_selected_line,
+    parse_ci_line,
+    step_label_for_line,
+)
+
+
+def compress_ci_job_log_output(text: str) -> str:
+    """Compress GitHub Actions / gh-run style job logs."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    signal = detect_ci_job_log(lines)
+    if signal is None:
+        return text
+
+    candidate_lines = _reduce_lines(lines, signal)
+    if len(candidate_lines) < 2:
+        return text
+
+    candidate = join_preserving_final_newline(candidate_lines, final_newline)
+    if score(candidate) < score(text):
+        return candidate
+    return text
+
+
+def _reduce_lines(lines: list[str], signal: CiLogSignal) -> list[str]:
+    outcome = "failure" if signal.failed else "warning" if signal.warning else "success"
+    selected = [f"ci log: github-actions {outcome}"]
+    current_step: str | None = None
+    in_group = False
+    omitted = 0
+
+    def flush_omitted() -> None:
+        nonlocal omitted
+        if omitted:
+            plural = "" if omitted == 1 else "s"
+            selected.append(f"… {omitted} noisy line{plural} omitted")
+            omitted = 0
+
+    def start_step(step: str) -> None:
+        nonlocal current_step
+        clean_step = step.strip()
+        if not clean_step or clean_step == current_step:
+            return
+        flush_omitted()
+        selected.append(f"step: {clean_step}")
+        current_step = clean_step
+
+    for raw_line in lines:
+        parsed = parse_ci_line(raw_line)
+        message = parsed.message.strip()
+        if not message:
+            continue
+
+        group_step = extract_group_step(message)
+        if group_step is not None:
+            start_step(group_step)
+            in_group = True
+            continue
+
+        prefixed_step = step_label_for_line(parsed)
+        if prefixed_step is not None and not in_group:
+            start_step(prefixed_step)
+            if prefixed_step == message:
+                continue
+
+        if is_group_end(message):
+            in_group = False
+            continue
+
+        if is_important_line(message):
+            flush_omitted()
+            selected.append(normalize_selected_line(message))
+            continue
+
+        omitted += 1
+
+    flush_omitted()
+    return _drop_blank_and_adjacent_duplicates(selected)
+
+
+def _drop_blank_and_adjacent_duplicates(lines: list[str]) -> list[str]:
+    output: list[str] = []
+    for line in lines:
+        if not line:
+            continue
+        if output and output[-1] == line:
+            continue
+        output.append(line)
+    return output

codetool_shell/filters/ci_job_log/summary.py

@@ -0,0 +1,243 @@
+"""Line parsing and classification for CI/job logs."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class CiLine:
+    """A parsed CI log line with noisy runner prefix removed."""
+
+    message: str
+    job: str | None
+    step: str | None
+    had_ci_prefix: bool
+
+
+def parse_ci_line(line: str) -> CiLine:
+    """Return CI prefix metadata and the visible log message."""
+
+    tab_parts = line.split("\t", 3)
+    if len(tab_parts) == 4 and _is_github_timestamp(tab_parts[2].strip()):
+        return CiLine(
+            message=tab_parts[3].strip(),
+            job=tab_parts[0].strip() or None,
+            step=tab_parts[1].strip() or None,
+            had_ci_prefix=True,
+        )
+    if len(tab_parts) == 3:
+        timestamp, separator, rest = tab_parts[2].strip().partition(" ")
+        if separator and _is_github_timestamp(timestamp):
+            return CiLine(
+                message=rest.strip(),
+                job=tab_parts[0].strip() or None,
+                step=tab_parts[1].strip() or None,
+                had_ci_prefix=True,
+            )
+
+    stripped = line.lstrip()
+    first, separator, rest = stripped.partition(" ")
+    if separator and _is_github_timestamp(first):
+        return CiLine(
+            message=rest.strip(),
+            job=None,
+            step=None,
+            had_ci_prefix=True,
+        )
+
+    return CiLine(message=line.strip(), job=None, step=None, had_ci_prefix=False)
+
+
+def extract_group_step(message: str) -> str | None:
+    """Return a GitHub Actions group title if ``message`` opens a group."""
+
+    stripped = message.strip()
+    for prefix in ("##[group]", "::group::"):
+        if stripped.startswith(prefix):
+            title = stripped[len(prefix) :].strip()
+            return title or None
+    return None
+
+
+def is_group_end(message: str) -> bool:
+    """Return true for a GitHub Actions group terminator."""
+
+    stripped = message.strip()
+    return stripped.startswith(("##[endgroup]", "::endgroup::"))
+
+
+def step_label_for_line(parsed: CiLine) -> str | None:
+    """Return a compact step label from parsed prefix data or a run marker."""
+
+    if parsed.step:
+        if parsed.job and parsed.job != parsed.step:
+            return f"{parsed.job} / {parsed.step}"
+        return parsed.step
+
+    message = parsed.message.strip()
+    if _is_step_start_message(message):
+        return message
+    return None
+
+
+def is_actions_marker(message: str) -> bool:
+    """Return true for GitHub Actions control/annotation markers."""
+
+    stripped = message.strip()
+    return stripped.startswith("##[") or stripped.startswith(
+        (
+            "::group::",
+            "::endgroup::",
+            "::error",
+            "::warning",
+            "::notice",
+            "::debug",
+        )
+    )
+
+
+def is_error_annotation(message: str) -> bool:
+    """Return true for GitHub Actions error annotations."""
+
+    stripped = message.strip()
+    return stripped.startswith(("::error", "##[error]"))
+
+
+def is_warning_annotation(message: str) -> bool:
+    """Return true for GitHub Actions warning annotations."""
+
+    stripped = message.strip()
+    return stripped.startswith(("::warning", "##[warning]"))
+
+
+def is_warning_line(message: str) -> bool:
+    """Return true for warning lines worth preserving."""
+
+    stripped = message.strip()
+    return is_warning_annotation(stripped) or stripped.startswith(("Warning:", "WARNING:"))
+
+
+def is_failure_line(message: str) -> bool:
+    """Return true for failure/exit lines worth preserving."""
+
+    stripped = message.strip()
+    lower = stripped.lower()
+    return (
+        is_error_annotation(stripped)
+        or stripped.startswith(("Error:", "ERROR:", "Failed", "FAILED", "fatal:"))
+        or "process completed with exit code" in lower
+        or "failed with exit code" in lower
+        or "exit code 1" in lower
+        or "status: failure" in lower
+        or "result: failure" in lower
+        or "conclusion: failure" in lower
+    )
+
+
+def is_conclusion_line(message: str) -> bool:
+    """Return true for final job/workflow status lines."""
+
+    lower = message.strip().lower()
+    return (
+        "workflow completed" in lower
+        or "job completed" in lower
+        or "completed with status" in lower
+        or "completed with result" in lower
+        or "completed successfully" in lower
+        or "conclusion:" in lower
+    )
+
+
+def is_reference_line(message: str) -> bool:
+    """Return true for URLs and artifact/path references."""
+
+    stripped = message.strip()
+    lower = stripped.lower()
+    if "https://" in lower or "http://" in lower:
+        return True
+    if "artifact" in lower:
+        return True
+    if any(token in lower for token in _REFERENCE_TOKENS):
+        return True
+    return _looks_like_path_reference(stripped)
+
+
+def is_path_location_line(message: str) -> bool:
+    """Return true for source file locations like ``src/app.ts:12:3``."""
+
+    stripped = message.strip().removeprefix("at ").strip()
+    parts = stripped.split(":")
+    if len(parts) < 2:
+        return False
+    for index in range(1, len(parts)):
+        if parts[index].isdigit():
+            path = ":".join(parts[:index])
+            return _looks_like_path(path)
+    return False
+
+
+def is_important_line(message: str) -> bool:
+    """Return true when a CI log line must be preserved."""
+
+    stripped = message.strip()
+    return (
+        is_warning_line(stripped)
+        or is_failure_line(stripped)
+        or is_conclusion_line(stripped)
+        or is_reference_line(stripped)
+        or is_path_location_line(stripped)
+    )
+
+
+def normalize_selected_line(message: str) -> str:
+    """Normalize a selected CI log message."""
+
+    return message.strip()
+
+
+def _is_step_start_message(message: str) -> bool:
+    return message.startswith(("Run ", "Post ", "Complete job", "Set up job", "Prepare "))
+
+
+def _is_github_timestamp(value: str) -> bool:
+    return _GITHUB_TIMESTAMP_RE.match(value) is not None
+
+
+def _looks_like_path_reference(line: str) -> bool:
+    lower = line.lower()
+    if not any(suffix in lower for suffix in _REFERENCE_SUFFIXES):
+        return False
+    return "/" in line or "\\" in line
+
+
+def _looks_like_path(path: str) -> bool:
+    if not path or path[0].isspace() or "://" in path:
+        return False
+    return "/" in path or "\\" in path or "." in path
+
+
+_GITHUB_TIMESTAMP_RE = re.compile(
+    r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z$"
+)
+_REFERENCE_TOKENS = (
+    "test-results/",
+    "playwright-report/",
+    "coverage/",
+    "dist/",
+    "build/",
+    "logs/",
+)
+_REFERENCE_SUFFIXES = (
+    ".zip",
+    ".tar.gz",
+    ".log",
+    ".txt",
+    ".html",
+    ".xml",
+    ".json",
+    ".png",
+    ".jpg",
+    ".jpeg",
+)

codetool_shell/filters/diff/__init__.py

@@ -0,0 +1,7 @@
+"""Generic unified diff and diff-noise compression."""
+
+from __future__ import annotations
+
+from .reducer import compress_diff_output
+
+__all__ = ["compress_diff_output"]

codetool_shell/filters/diff/detector.py

@@ -0,0 +1,136 @@
+"""Conservative unified-diff block detection."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class DiffBlock:
+    """Line range for one high-confidence diff block."""
+
+    start: int
+    end: int
+
+
+_NORMAL_HUNK_RE = re.compile(
+    r"^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@(?: .*)?$"
+)
+_COMBINED_HUNK_RE = re.compile(
+    r"^(?P<ats>@{3,}) (?:-\d+(?:,\d+)? )+\+\d+(?:,\d+)? (?P=ats)(?: .*)?$"
+)
+_DIFF_HEADER_PREFIXES = ("diff --git ", "diff --cc ", "diff --combined ")
+_DIFF_METADATA_PREFIXES = (
+    "new file mode ",
+    "deleted file mode ",
+    "old mode ",
+    "new mode ",
+    "similarity index ",
+    "dissimilarity index ",
+    "rename from ",
+    "rename to ",
+    "copy from ",
+    "copy to ",
+)
+
+
+def find_diff_blocks(lines: list[str]) -> tuple[DiffBlock, ...]:
+    """Return high-confidence diff block ranges."""
+
+    blocks: list[DiffBlock] = []
+    i = 0
+    while i < len(lines):
+        kind = _block_start_kind(lines, i)
+        if kind is None:
+            i += 1
+            continue
+
+        end = _find_block_end(lines, i, kind)
+        block = lines[i:end]
+        if _is_valid_diff_block(block, kind):
+            blocks.append(DiffBlock(i, end))
+            i = end
+        else:
+            i += 1
+
+    return tuple(blocks)
+
+
+def is_hunk_header(line: str) -> bool:
+    """Return whether ``line`` is a normal or combined hunk header."""
+
+    return _NORMAL_HUNK_RE.match(line) is not None or _COMBINED_HUNK_RE.match(line) is not None
+
+
+def hunk_prefix_columns(header: str) -> int:
+    """Return body prefix column count for a hunk header."""
+
+    if header.startswith("@@@"):
+        at_count = len(header) - len(header.lstrip("@"))
+        return max(2, at_count - 1)
+    return 1
+
+
+def _block_start_kind(lines: list[str], index: int) -> str | None:
+    line = lines[index]
+    if line.startswith(_DIFF_HEADER_PREFIXES):
+        return "diff"
+    if line.startswith("Index: "):
+        return "svn"
+    if _has_unified_file_header_pair_at(lines, index):
+        return "standalone"
+    return None
+
+
+def _find_block_end(lines: list[str], start: int, kind: str) -> int:
+    i = start + 1
+    while i < len(lines):
+        if lines[i].startswith(_DIFF_HEADER_PREFIXES) or lines[i].startswith("Index: "):
+            return i
+        if kind == "diff" and _is_git_patch_preamble_line(lines[i]):
+            return i
+        if kind == "standalone" and _has_unified_file_header_pair_at(lines, i):
+            return i
+        i += 1
+    return len(lines)
+
+
+def _has_unified_file_header_pair_at(lines: list[str], index: int) -> bool:
+    return (
+        index + 2 < len(lines)
+        and lines[index].startswith("--- ")
+        and lines[index + 1].startswith("+++ ")
+        and is_hunk_header(lines[index + 2])
+    )
+
+
+def _is_valid_diff_block(block: list[str], kind: str) -> bool:
+    has_hunk = any(is_hunk_header(line) for line in block)
+    has_file_headers = any(
+        block[index].startswith("--- ") and block[index + 1].startswith("+++ ")
+        for index in range(max(0, len(block) - 1))
+    )
+    has_binary = any(
+        line.startswith(("GIT binary patch", "Binary files ", "literal ", "delta "))
+        for line in block
+    )
+    has_metadata = any(line.startswith(_DIFF_METADATA_PREFIXES) for line in block)
+
+    if kind == "diff":
+        return has_hunk or has_binary or has_file_headers or has_metadata
+    return has_hunk and has_file_headers
+
+
+def _is_git_patch_preamble_line(line: str) -> bool:
+    if line.startswith("commit "):
+        token = line.removeprefix("commit ").split(maxsplit=1)[0]
+        return _is_hex_prefix(token)
+    if line.startswith("From "):
+        token = line.removeprefix("From ").split(maxsplit=1)[0]
+        return _is_hex_prefix(token)
+    return False
+
+
+def _is_hex_prefix(token: str) -> bool:
+    return len(token) >= 7 and all(char in "0123456789abcdefABCDEF" for char in token)