codetool-shell 0.1.1__py3-none-macosx_10_12_x86_64.whl

codetool_shell/__init__.py

@@ -0,0 +1,11 @@
+"""Public Python interface for shell output text compression."""
+
+from .api import compress_text
+from .rust_backend import RustBackendError, RustBackendUnavailable, is_rust_available
+
+__all__ = [
+    "RustBackendError",
+    "RustBackendUnavailable",
+    "compress_text",
+    "is_rust_available",
+]

codetool_shell/api.py

@@ -0,0 +1,59 @@
+"""Stable public API and backend selection for text compression."""
+
+from __future__ import annotations
+
+from .python_backend import compress_text_python
+from .rust_backend import (
+    RustBackendError,
+    RustBackendUnavailable,
+    is_rust_available,
+    compress_text_rust,
+)
+
+BackendName = str
+
+
+def compress_text(
+    text: str,
+    *,
+    max_blank_lines: int = 1,
+    backend: BackendName = "auto",
+) -> str:
+    """Compress plain shell-output text.
+
+    This initial scaffold applies one conservative, semantics-preserving pass:
+    normalize line endings, trim trailing whitespace on each line, and collapse
+    repeated blank lines. Future pipeline variants can layer on this API.
+    """
+
+    _validate_text(text)
+    max_blank_lines = _validate_max_blank_lines(max_blank_lines)
+
+    if backend == "python":
+        return compress_text_python(text, max_blank_lines=max_blank_lines)
+
+    if backend == "rust":
+        return compress_text_rust(text, max_blank_lines=max_blank_lines)
+
+    if backend == "auto":
+        if is_rust_available():
+            try:
+                return compress_text_rust(text, max_blank_lines=max_blank_lines)
+            except (RustBackendUnavailable, RustBackendError):
+                pass
+        return compress_text_python(text, max_blank_lines=max_blank_lines)
+
+    raise ValueError("backend must be one of: 'auto', 'python', 'rust'")
+
+
+def _validate_text(text: str) -> None:
+    if not isinstance(text, str):
+        raise TypeError("text must be a string")
+
+
+def _validate_max_blank_lines(max_blank_lines: int) -> int:
+    if isinstance(max_blank_lines, bool) or not isinstance(max_blank_lines, int):
+        raise TypeError("max_blank_lines must be an integer")
+    if max_blank_lines < 0:
+        raise ValueError("max_blank_lines must be greater than or equal to 0")
+    return max_blank_lines

codetool_shell/filters/__init__.py

@@ -0,0 +1,14 @@
+"""Dedicated shell-output filters."""
+
+from __future__ import annotations
+
+from .git import compress_git_output
+from .pipeline import TextFilter, apply_filter_pipeline
+from .rg import compress_rg_output
+
+__all__ = [
+    "TextFilter",
+    "apply_filter_pipeline",
+    "compress_git_output",
+    "compress_rg_output",
+]

codetool_shell/filters/build_compiler/__init__.py

@@ -0,0 +1,7 @@
+"""Build/compiler diagnostic shellion."""
+
+from __future__ import annotations
+
+from .reducer import compress_build_compiler_diagnostics_output
+
+__all__ = ["compress_build_compiler_diagnostics_output"]

codetool_shell/filters/build_compiler/detector.py

@@ -0,0 +1,412 @@
+"""Detect conservative build/compiler diagnostic shapes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .summary import (
+    has_error_signal,
+    has_warning_signal,
+    is_biome_diagnostic_line,
+    is_biome_summary_line,
+    is_c_family_diagnostic_line,
+    is_c_family_source_context_line,
+    is_eslint_file_header,
+    is_eslint_issue_line,
+    is_eslint_summary_line,
+    is_golangci_lint_issue_line,
+    is_golangci_lint_runner_error_line,
+    is_golangci_lint_summary_line,
+    is_go_build_diagnostic_line,
+    is_go_build_package_header,
+    is_final_summary_line,
+    is_make_ninja_cmake_diagnostic_line,
+    is_markdownlint_diagnostic_line,
+    is_maven_gradle_diagnostic_line,
+    is_msbuild_diagnostic_line,
+    is_msbuild_summary_line,
+    is_mypy_diagnostic_line,
+    is_nx_turbo_diagnostic_line,
+    is_oxlint_diagnostic_line,
+    is_oxlint_frame_line,
+    is_oxlint_summary_line,
+    is_pyright_diagnostic_line,
+    is_pyright_summary_line,
+    is_progress_line,
+    is_ruff_diagnostic_line,
+    is_ruff_summary_line,
+    is_rust_diagnostic_line,
+    is_rust_source_bar_line,
+    is_rust_span_line,
+    is_shellcheck_diagnostic_line,
+    is_shellcheck_gcc_diagnostic_line,
+    is_shellcheck_header_line,
+    is_swift_xcode_failure_line,
+    is_tsc_diagnostic_line,
+    is_yamllint_diagnostic_line,
+    is_yamllint_file_header,
+    strip_ansi,
+)
+
+
+@dataclass(frozen=True)
+class BuildCompilerDiagnostics:
+    """Detected diagnostic metadata."""
+
+    kind: str
+    failed: bool
+    warning: bool
+
+
+@dataclass(frozen=True)
+class _DiagnosticCandidate:
+    kind: str
+    score: int
+
+
+def detect_build_compiler_diagnostics(
+    lines: list[str],
+) -> BuildCompilerDiagnostics | None:
+    """Return a strong compiler/static-analysis signal, or ``None``."""
+
+    clean_lines = [strip_ansi(line).rstrip() for line in lines]
+    nonblank = [line for line in clean_lines if line.strip()]
+    if len(nonblank) < 2:
+        return None
+
+    text = "\n".join(nonblank)
+    lower = text.lower()
+    if _looks_like_excluded_output(lower):
+        return None
+
+    candidates = [
+        candidate
+        for candidate in (
+            _classify_rust(nonblank, lower),
+            _classify_tsc(nonblank),
+            _classify_eslint(nonblank),
+            _classify_mypy(nonblank),
+            _classify_ruff(nonblank),
+            _classify_pyright(nonblank),
+            _classify_biome(nonblank),
+            _classify_oxlint(nonblank),
+            _classify_shellcheck(nonblank),
+            _classify_markdownlint(nonblank, lower),
+            _classify_yamllint(nonblank, lower),
+            _classify_golangci_lint(nonblank, lower),
+            _classify_c_family(nonblank, lower),
+            _classify_go_build(nonblank),
+            _classify_msbuild_dotnet(nonblank, lower),
+            _classify_maven_gradle(nonblank, lower),
+            _classify_make_ninja_cmake(nonblank),
+            _classify_nx_turbo(nonblank, lower),
+            _classify_swift_xcode(nonblank, lower),
+        )
+        if candidate is not None
+    ]
+    if not candidates:
+        return None
+
+    kind = max(
+        candidates,
+        key=lambda candidate: (candidate.score, candidate.kind),
+    ).kind
+    failed = has_error_signal(nonblank) or _kind_defaults_to_failure(kind, nonblank)
+    warning = has_warning_signal(nonblank) or _kind_defaults_to_warning(kind, nonblank)
+    return BuildCompilerDiagnostics(
+        kind=kind,
+        failed=failed,
+        warning=warning,
+    )
+
+
+def _candidate(kind: str, enabled: bool, score: int) -> _DiagnosticCandidate | None:
+    if not enabled:
+        return None
+    return _DiagnosticCandidate(kind=kind, score=score)
+
+
+def _classify_rust(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    rust_headers = sum(1 for line in lines if is_rust_diagnostic_line(line))
+    rust_spans = sum(1 for line in lines if is_rust_span_line(line))
+    rust_bars = sum(1 for line in lines if is_rust_source_bar_line(line))
+    progress_lines = sum(1 for line in lines if is_progress_line(line))
+    signal = rust_headers > 0 and (
+        rust_spans > 0
+        or rust_bars > 0
+        or "could not compile" in lower
+        or "aborting due to" in lower
+    )
+    return _candidate(
+        "cargo/rustc",
+        signal,
+        rust_headers * 3 + rust_spans * 2 + rust_bars + progress_lines,
+    )
+
+
+def _classify_tsc(lines: list[str]) -> _DiagnosticCandidate | None:
+    tsc_lines = sum(1 for line in lines if is_tsc_diagnostic_line(line))
+    return _candidate("tsc", tsc_lines > 0, tsc_lines * 4)
+
+
+def _classify_eslint(lines: list[str]) -> _DiagnosticCandidate | None:
+    eslint_issues = sum(1 for line in lines if is_eslint_issue_line(line))
+    eslint_headers = sum(1 for line in lines if is_eslint_file_header(line))
+    eslint_summaries = sum(1 for line in lines if is_eslint_summary_line(line))
+    return _candidate(
+        "eslint",
+        eslint_issues > 0 and (eslint_headers > 0 or eslint_summaries > 0),
+        eslint_issues * 3 + eslint_headers + eslint_summaries * 2,
+    )
+
+
+def _classify_mypy(lines: list[str]) -> _DiagnosticCandidate | None:
+    mypy_lines = sum(1 for line in lines if is_mypy_diagnostic_line(line))
+    mypy_summaries = sum(1 for line in lines if _is_mypy_summary_or_code_line(line))
+    return _candidate(
+        "mypy",
+        mypy_lines > 0 and (mypy_summaries > 0 or mypy_lines >= 3),
+        mypy_lines * 3 + mypy_summaries * 2,
+    )
+
+
+def _classify_ruff(lines: list[str]) -> _DiagnosticCandidate | None:
+    ruff_lines = sum(1 for line in lines if is_ruff_diagnostic_line(line))
+    ruff_summaries = sum(1 for line in lines if is_ruff_summary_line(line))
+    return _candidate(
+        "ruff",
+        ruff_lines > 0 and (ruff_summaries > 0 or ruff_lines >= 2),
+        ruff_lines * 4 + ruff_summaries * 2,
+    )
+
+
+def _classify_pyright(lines: list[str]) -> _DiagnosticCandidate | None:
+    pyright_lines = sum(1 for line in lines if is_pyright_diagnostic_line(line))
+    pyright_summaries = sum(1 for line in lines if is_pyright_summary_line(line))
+    return _candidate(
+        "pyright",
+        pyright_lines > 0 and (pyright_summaries > 0 or pyright_lines >= 2),
+        pyright_lines * 4 + pyright_summaries * 2,
+    )
+
+
+def _classify_biome(lines: list[str]) -> _DiagnosticCandidate | None:
+    biome_lines = sum(1 for line in lines if is_biome_diagnostic_line(line))
+    biome_summaries = sum(1 for line in lines if is_biome_summary_line(line))
+    return _candidate(
+        "biome",
+        biome_lines > 0 and (biome_summaries > 0 or biome_lines >= 2),
+        biome_lines * 3 + biome_summaries * 2,
+    )
+
+
+def _classify_oxlint(lines: list[str]) -> _DiagnosticCandidate | None:
+    oxlint_lines = sum(1 for line in lines if is_oxlint_diagnostic_line(line))
+    oxlint_frames = sum(1 for line in lines if is_oxlint_frame_line(line))
+    oxlint_summaries = sum(1 for line in lines if is_oxlint_summary_line(line))
+    return _candidate(
+        "oxlint",
+        oxlint_lines > 0
+        and (oxlint_summaries > 0 or oxlint_frames > 0 or oxlint_lines >= 2),
+        oxlint_lines * 4 + oxlint_frames * 2 + oxlint_summaries * 2,
+    )
+
+
+def _classify_shellcheck(lines: list[str]) -> _DiagnosticCandidate | None:
+    shellcheck_headers = sum(1 for line in lines if is_shellcheck_header_line(line))
+    shellcheck_lines = sum(1 for line in lines if is_shellcheck_diagnostic_line(line))
+    shellcheck_gcc_lines = sum(
+        1 for line in lines if is_shellcheck_gcc_diagnostic_line(line)
+    )
+    return _candidate(
+        "shellcheck",
+        shellcheck_lines > 0
+        and (
+            shellcheck_headers > 0
+            or shellcheck_gcc_lines > 0
+            or shellcheck_lines >= 2
+        ),
+        shellcheck_lines * 4 + shellcheck_headers * 2,
+    )
+
+
+def _classify_markdownlint(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    markdownlint_lines = sum(
+        1 for line in lines if is_markdownlint_diagnostic_line(line)
+    )
+    return _candidate(
+        "markdownlint",
+        markdownlint_lines > 0
+        and (
+            markdownlint_lines >= 2
+            or "markdownlint" in lower
+            or any(is_final_summary_line(line) for line in lines)
+        ),
+        markdownlint_lines * 4,
+    )
+
+
+def _classify_yamllint(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    yamllint_lines = sum(1 for line in lines if is_yamllint_diagnostic_line(line))
+    yamllint_headers = sum(1 for line in lines if is_yamllint_file_header(line))
+    return _candidate(
+        "yamllint",
+        yamllint_lines > 0
+        and (yamllint_headers > 0 or yamllint_lines >= 2 or "yamllint" in lower),
+        yamllint_lines * 4 + yamllint_headers,
+    )
+
+
+def _classify_golangci_lint(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    golangci_lines = sum(1 for line in lines if is_golangci_lint_issue_line(line))
+    golangci_runner_errors = sum(
+        1 for line in lines if is_golangci_lint_runner_error_line(line)
+    )
+    golangci_summaries = sum(
+        1 for line in lines if is_golangci_lint_summary_line(line)
+    )
+    progress_lines = sum(1 for line in lines if is_progress_line(line))
+    signal = (
+        golangci_lines > 0
+        and (
+            golangci_lines >= 2
+            or golangci_summaries > 0
+            or golangci_runner_errors > 0
+            or "golangci-lint" in lower
+        )
+    ) or (
+        golangci_runner_errors > 0
+        and ("golangci-lint" in lower or "level=info" in lower or progress_lines > 0)
+    )
+    return _candidate(
+        "golangci-lint",
+        signal,
+        golangci_lines * 4 + golangci_runner_errors * 3 + golangci_summaries,
+    )
+
+
+def _classify_c_family(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    diagnostics = sum(1 for line in lines if is_c_family_diagnostic_line(line))
+    context = sum(1 for line in lines if is_c_family_source_context_line(line))
+    summary = int("build failed" in lower or "compilation terminated" in lower)
+    signal = diagnostics > 0 and (diagnostics >= 2 or context > 0 or summary > 0)
+    return _candidate(
+        "gcc/clang",
+        signal,
+        diagnostics * 4 + context + summary * 2,
+    )
+
+
+def _classify_go_build(lines: list[str]) -> _DiagnosticCandidate | None:
+    headers = sum(1 for line in lines if is_go_build_package_header(line))
+    diagnostics = sum(1 for line in lines if is_go_build_diagnostic_line(line))
+    return _candidate(
+        "go-build",
+        headers > 0 and diagnostics > 0,
+        headers * 2 + diagnostics * 4,
+    )
+
+
+def _classify_msbuild_dotnet(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    diagnostics = sum(1 for line in lines if is_msbuild_diagnostic_line(line))
+    summaries = sum(1 for line in lines if is_msbuild_summary_line(line))
+    marker = "build failed" in lower or "msbuild" in lower or "dotnet build" in lower
+    return _candidate(
+        "msbuild/dotnet",
+        diagnostics > 0 and (summaries > 0 or marker),
+        diagnostics * 4 + summaries * 2 + int(marker),
+    )
+
+
+def _classify_maven_gradle(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    diagnostics = sum(1 for line in lines if is_maven_gradle_diagnostic_line(line))
+    marker = any(
+        token in lower
+        for token in (
+            "maven-surefire-plugin",
+            "[error] build failure",
+            "build failure",
+            "gradle",
+            "> task ",
+            "execution failed for task",
+        )
+    )
+    return _candidate(
+        "maven/gradle",
+        diagnostics > 0 and marker,
+        diagnostics * 3 + int(marker) * 2,
+    )
+
+
+def _classify_make_ninja_cmake(lines: list[str]) -> _DiagnosticCandidate | None:
+    diagnostics = sum(1 for line in lines if is_make_ninja_cmake_diagnostic_line(line))
+    return _candidate(
+        "make/ninja/cmake",
+        diagnostics > 0,
+        diagnostics * 4,
+    )
+
+
+def _classify_nx_turbo(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    diagnostics = sum(1 for line in lines if is_nx_turbo_diagnostic_line(line))
+    marker = "nx " in lower or "nx   " in lower or "turbo" in lower
+    return _candidate(
+        "nx/turbo",
+        diagnostics > 0 and marker,
+        diagnostics * 3 + int(marker) * 2,
+    )
+
+
+def _classify_swift_xcode(lines: list[str], lower: str) -> _DiagnosticCandidate | None:
+    swift_diagnostics = sum(
+        1 for line in lines if is_c_family_diagnostic_line(line) and ".swift:" in line
+    )
+    xcode_failures = sum(1 for line in lines if is_swift_xcode_failure_line(line))
+    marker = "xcodebuild" in lower or "swiftcompile" in lower or "** build failed **" in lower
+    return _candidate(
+        "swift/xcode",
+        (swift_diagnostics > 0 and marker) or xcode_failures > 0,
+        swift_diagnostics * 4 + xcode_failures * 3 + int(marker),
+    )
+
+
+def _looks_like_excluded_output(lower: str) -> bool:
+    return (
+        lower.startswith("test output:")
+        or "test session starts" in lower
+        or "short test summary info" in lower
+        or "test result:" in lower
+        or "traceback (most recent call last)" in lower
+    )
+
+
+def _is_mypy_summary_or_code_line(line: str) -> bool:
+    lower = line.lower()
+    return (
+        ("found " in lower and " error" in lower and " file" in lower)
+        or "success: no issues found" in lower
+        or ("[" in line and "]" in line and is_mypy_diagnostic_line(line))
+        or is_final_summary_line(line)
+    )
+
+
+def _kind_defaults_to_failure(kind: str, lines: list[str]) -> bool:
+    """Classify linter issue-only formats without treating warnings as failures."""
+
+    if kind in {"markdownlint", "golangci-lint"}:
+        return True
+    if kind == "go-build":
+        return any(is_go_build_diagnostic_line(line) for line in lines)
+    if kind == "ruff":
+        return any(is_ruff_diagnostic_line(line) for line in lines)
+    if kind in {"biome", "oxlint"} and not has_warning_signal(lines):
+        return True
+    return False
+
+
+def _kind_defaults_to_warning(kind: str, lines: list[str]) -> bool:
+    if kind in {"shellcheck", "yamllint"}:
+        return True
+    if kind in {"biome", "oxlint"} and has_warning_signal(lines):
+        return True
+    return False

codetool_shell/filters/build_compiler/reducer.py

@@ -0,0 +1,166 @@
+"""Conservative reducer for build/compiler diagnostics."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import BuildCompilerDiagnostics, detect_build_compiler_diagnostics
+from .summary import (
+    diagnostic_context_budget,
+    is_eslint_issue_line,
+    is_diagnostic_file_header,
+    is_final_summary_line,
+    is_important_diagnostic_line,
+    is_mypy_diagnostic_line,
+    is_progress_line,
+    is_safe_snippet_line,
+    normalize_line,
+    parse_golangci_lint_issue_line,
+    parse_mypy_diagnostic_line,
+    strip_ansi,
+)
+
+
+def compress_build_compiler_diagnostics_output(text: str) -> str:
+    """Compress compiler/build/static-analysis diagnostic output."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    signal = detect_build_compiler_diagnostics(lines)
+    if signal is None:
+        return text
+
+    if signal.kind == "mypy":
+        candidate_lines = _reduce_mypy_lines(lines, signal)
+    else:
+        candidate_lines = _reduce_lines(lines, signal)
+    if len(candidate_lines) < 2:
+        return text
+
+    candidate = join_preserving_final_newline(candidate_lines, final_newline)
+    if score(candidate) < score(text):
+        return candidate
+    return text
+
+
+def _reduce_lines(
+    lines: list[str], signal: BuildCompilerDiagnostics
+) -> list[str]:
+    selected = [_header(signal)]
+    progress_omitted = 0
+    snippet_remaining = 0
+
+    def flush_progress() -> None:
+        nonlocal progress_omitted
+        if progress_omitted:
+            plural = "" if progress_omitted == 1 else "s"
+            selected.append(f"… {progress_omitted} build progress line{plural} omitted")
+            progress_omitted = 0
+
+    for raw_line in lines:
+        line = strip_ansi(raw_line).rstrip()
+        stripped = line.strip()
+        if not stripped:
+            continue
+
+        if is_progress_line(stripped):
+            progress_omitted += 1
+            continue
+
+        if is_important_diagnostic_line(stripped) or is_diagnostic_file_header(
+            signal.kind, stripped
+        ):
+            flush_progress()
+            selected.append(_normalize_for_signal(line, signal))
+            snippet_remaining = diagnostic_context_budget(signal.kind, stripped)
+            continue
+
+        if snippet_remaining and is_safe_snippet_line(line):
+            flush_progress()
+            selected.append(line.rstrip())
+            snippet_remaining -= 1
+            continue
+
+        if signal.failed and is_final_summary_line(stripped):
+            flush_progress()
+            selected.append(_normalize_for_signal(line, signal))
+            continue
+
+        progress_omitted += 1
+
+    flush_progress()
+    return _drop_adjacent_duplicates(selected)
+
+
+def _reduce_mypy_lines(
+    lines: list[str], signal: BuildCompilerDiagnostics
+) -> list[str]:
+    selected = [_header(signal)]
+    progress_omitted = 0
+    current_path: str | None = None
+
+    def flush_progress() -> None:
+        nonlocal progress_omitted
+        if progress_omitted:
+            plural = "" if progress_omitted == 1 else "s"
+            selected.append(f"… {progress_omitted} build progress line{plural} omitted")
+            progress_omitted = 0
+
+    for raw_line in lines:
+        line = strip_ansi(raw_line).rstrip()
+        stripped = line.strip()
+        if not stripped:
+            continue
+
+        parsed = parse_mypy_diagnostic_line(stripped)
+        if parsed is not None:
+            path, location, detail = parsed
+            flush_progress()
+            if path != current_path:
+                selected.append(path)
+                current_path = path
+            selected.append(f" {location}: {detail}")
+            continue
+
+        if is_progress_line(stripped):
+            progress_omitted += 1
+            continue
+
+        if is_final_summary_line(stripped) or (
+            is_mypy_diagnostic_line(stripped) and "[" in stripped and "]" in stripped
+        ):
+            flush_progress()
+            selected.append(normalize_line(line))
+            continue
+
+        progress_omitted += 1
+
+    flush_progress()
+    return _drop_adjacent_duplicates(selected)
+
+
+def _header(signal: BuildCompilerDiagnostics) -> str:
+    outcome = "failure" if signal.failed else "warning" if signal.warning else "success"
+    return f"build diagnostics: {signal.kind} {outcome}"
+
+
+def _normalize_for_signal(line: str, signal: BuildCompilerDiagnostics) -> str:
+    if signal.kind == "eslint" and is_eslint_issue_line(line):
+        return " ".join(line.strip().split())
+    if signal.kind == "golangci-lint":
+        parsed = parse_golangci_lint_issue_line(line)
+        if parsed is not None:
+            path, location, detail = parsed
+            return f"{path} {location}: {detail}"
+    if signal.kind == "shellcheck":
+        return line.rstrip()
+    return normalize_line(line)
+
+
+def _drop_adjacent_duplicates(lines: list[str]) -> list[str]:
+    output: list[str] = []
+    for line in lines:
+        if not line:
+            continue
+        if output and output[-1] == line:
+            continue
+        output.append(line)
+    return output