codetool-shell 0.1.1__py3-none-win_amd64.whl

codetool_shell/filters/system_output/reducer.py

@@ -0,0 +1,331 @@
+"""Reduce high-confidence system command outputs."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import (
+    DfOutput,
+    DuOutput,
+    EnvOutput,
+    PingOutput,
+    PsOutput,
+    StatBlock,
+    SystemctlStatusOutput,
+    WcOutput,
+    parse_df_output,
+    parse_du_output,
+    parse_env_output,
+    parse_ping_output,
+    parse_ps_output,
+    parse_stat_blocks,
+    parse_systemctl_status_output,
+    parse_wc_output,
+)
+from .summary import (
+    SAFE_ENV_KEYS,
+    format_env_var,
+    is_opaque_value,
+    is_sensitive_key,
+    omitted_line,
+    redact_sensitive_text,
+)
+
+_HEAD_ROWS = 2
+_TAIL_ROWS = 2
+_DF_HIGH_USE_THRESHOLD = 85
+_PS_HIGH_CPU_THRESHOLD = 10.0
+_PS_HIGH_MEM_THRESHOLD = 5.0
+_TOP_DU_ROWS = 5
+_PING_EDGE_REPLIES = 2
+
+
+def compress_system_output(text: str) -> str:
+    """Compress clear system command outputs, otherwise return ``text``."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    if not lines:
+        return text
+
+    for reducer in (
+        _reduce_env,
+        _reduce_df,
+        _reduce_ps,
+        _reduce_ping,
+        _reduce_systemctl,
+        _reduce_du,
+        _reduce_wc,
+        _reduce_stat,
+    ):
+        candidate_lines = reducer(lines)
+        if candidate_lines is None:
+            continue
+        candidate = join_preserving_final_newline(candidate_lines, final_newline)
+        if score(candidate) < score(text):
+            return candidate
+    return text
+
+
+def _reduce_env(lines: list[str]) -> list[str] | None:
+    parsed = parse_env_output(lines)
+    if parsed is None:
+        return None
+
+    vars_by_key = {var.key: var.value for var in parsed.vars}
+    selected = [
+        (key, vars_by_key[key]) for key in SAFE_ENV_KEYS if key in vars_by_key
+    ]
+    if not selected:
+        return None
+
+    redacted = sum(
+        1
+        for var in parsed.vars
+        if is_sensitive_key(var.key) or (var.key != "PATH" and is_opaque_value(var.value))
+    )
+    omitted = len(parsed.vars) - len(selected)
+    output = [
+        f"system env: vars={len(parsed.vars)} redacted={redacted} omitted={omitted}"
+    ]
+    output.extend(format_env_var(key, value) for key, value in selected)
+    if omitted > 0:
+        output.append(omitted_line(omitted, "env", "var"))
+    return output
+
+
+def _reduce_df(lines: list[str]) -> list[str] | None:
+    parsed = parse_df_output(lines)
+    if parsed is None:
+        return None
+
+    high_indices = [
+        index
+        for index, row in enumerate(parsed.rows)
+        if row.usage >= _DF_HIGH_USE_THRESHOLD
+    ]
+    important_indices = [
+        index for index, row in enumerate(parsed.rows) if _is_important_mount(row.mount)
+    ]
+    selected = _edge_indices(len(parsed.rows))
+    selected.extend(high_indices)
+    selected.extend(important_indices)
+
+    output = [
+        f"df: filesystems={len(parsed.rows)} high_use={len(high_indices)}",
+        parsed.header,
+    ]
+    output.extend(
+        _selected_rows(
+            len(parsed.rows),
+            selected,
+            lambda index: parsed.rows[index].line,
+            "df",
+        )
+    )
+    return output
+
+
+def _reduce_du(lines: list[str]) -> list[str] | None:
+    parsed = parse_du_output(lines)
+    if parsed is None:
+        return None
+
+    selected = _edge_indices(len(parsed.rows))
+    selected.extend(
+        index
+        for index, _row in sorted(
+            enumerate(parsed.rows),
+            key=lambda item: item[1].size,
+            reverse=True,
+        )[:_TOP_DU_ROWS]
+    )
+    selected.extend(index for index, row in enumerate(parsed.rows) if row.is_total)
+
+    output = [f"du: entries={len(parsed.rows)}"]
+    output.extend(
+        _selected_rows(
+            len(parsed.rows),
+            selected,
+            lambda index: parsed.rows[index].line,
+            "du",
+        )
+    )
+    return output
+
+
+def _reduce_ps(lines: list[str]) -> list[str] | None:
+    parsed = parse_ps_output(lines)
+    if parsed is None:
+        return None
+
+    high_indices = [
+        index
+        for index, row in enumerate(parsed.rows)
+        if (row.cpu is not None and row.cpu >= _PS_HIGH_CPU_THRESHOLD)
+        or (row.mem is not None and row.mem >= _PS_HIGH_MEM_THRESHOLD)
+    ]
+    interesting_indices = [
+        index
+        for index, row in enumerate(parsed.rows)
+        if _is_interesting_command(row.command)
+    ]
+    selected = _edge_indices(len(parsed.rows))
+    selected.extend(high_indices)
+    selected.extend(interesting_indices)
+
+    output = [
+        f"ps: processes={len(parsed.rows)} high={len(high_indices)} interesting={len(interesting_indices)}",
+        parsed.header,
+    ]
+    output.extend(
+        _selected_rows(
+            len(parsed.rows),
+            selected,
+            lambda index: redact_sensitive_text(parsed.rows[index].line),
+            "ps",
+        )
+    )
+    return output
+
+
+def _reduce_ping(lines: list[str]) -> list[str] | None:
+    parsed = parse_ping_output(lines)
+    if parsed is None:
+        return None
+
+    selected = [0]
+    selected.extend(parsed.reply_indices[:_PING_EDGE_REPLIES])
+    selected.extend(parsed.reply_indices[-_PING_EDGE_REPLIES:])
+    selected.extend(parsed.event_indices)
+    selected.extend(parsed.summary_indices)
+
+    output = [
+        f"ping: replies={len(parsed.reply_indices)} events={len(parsed.event_indices)}"
+    ]
+    output.extend(
+        _selected_rows(
+            len(parsed.lines),
+            selected,
+            lambda index: parsed.lines[index],
+            "ping",
+            noun="line",
+        )
+    )
+    return output
+
+
+def _reduce_systemctl(lines: list[str]) -> list[str] | None:
+    parsed = parse_systemctl_status_output(lines)
+    if parsed is None:
+        return None
+
+    selected = [*parsed.core_indices, *parsed.alert_indices]
+    output = [
+        f"systemctl status: lines={len(parsed.lines)} alerts={len(parsed.alert_indices)}"
+    ]
+    output.extend(
+        _selected_rows(
+            len(parsed.lines),
+            selected,
+            lambda index: redact_sensitive_text(parsed.lines[index]),
+            "status",
+            noun="line",
+        )
+    )
+    return output
+
+
+def _reduce_wc(lines: list[str]) -> list[str] | None:
+    parsed = parse_wc_output(lines)
+    if parsed is None:
+        return None
+
+    file_indices = [index for index, row in enumerate(parsed.rows) if not row.is_total]
+    selected = file_indices[:3] + file_indices[-3:] + [parsed.total_index]
+    output = [f"wc: files={len(file_indices)}"]
+    output.extend(
+        _selected_rows(
+            len(parsed.rows),
+            selected,
+            lambda index: parsed.rows[index].line,
+            "wc",
+        )
+    )
+    return output
+
+
+def _reduce_stat(lines: list[str]) -> list[str] | None:
+    blocks = parse_stat_blocks(lines)
+    if blocks is None:
+        return None
+    output = [f"stat: files={len(blocks)}"]
+    output.extend(_format_stat_block(block) for block in blocks)
+    return output
+
+
+def _selected_rows(
+    total: int,
+    selected: list[int],
+    line_for_index,
+    label: str,
+    *,
+    noun: str = "row",
+) -> list[str]:
+    output: list[str] = []
+    selected_sorted = sorted({index for index in selected if 0 <= index < total})
+    cursor = 0
+    for index in selected_sorted:
+        omitted = index - cursor
+        if omitted > 0:
+            output.append(omitted_line(omitted, label, noun))
+        output.append(line_for_index(index))
+        cursor = index + 1
+    if cursor < total:
+        output.append(omitted_line(total - cursor, label, noun))
+    return output
+
+
+def _edge_indices(total: int) -> list[int]:
+    return [
+        *range(min(_HEAD_ROWS, total)),
+        *range(max(_HEAD_ROWS, total - _TAIL_ROWS), total),
+    ]
+
+
+def _is_important_mount(mount: str) -> bool:
+    return mount in {"/", "/home", "/var", "/tmp", "/boot", "/data"} or mount.startswith(
+        ("/var/", "/boot/", "/data/")
+    )
+
+
+def _is_interesting_command(command: str) -> bool:
+    lower = command.lower()
+    return any(
+        marker in lower
+        for marker in (
+            "python",
+            "pytest",
+            "cargo",
+            "rustc",
+            "node",
+            "npm",
+            "java",
+            "gradle",
+            "mvn",
+            "postgres",
+            "redis",
+            "nginx",
+            "gunicorn",
+            "uvicorn",
+            "docker",
+            "kubectl",
+            "systemd",
+            "sshd",
+        )
+    )
+
+
+def _format_stat_block(block: StatBlock) -> str:
+    return (
+        f"{block.file} | size={block.size} | mode={block.mode} | "
+        f"owner={block.owner} | group={block.group} | modify={block.modify}"
+    )

codetool_shell/filters/system_output/summary.py

@@ -0,0 +1,164 @@
+"""Formatting and redaction helpers for system-shellion."""
+
+from __future__ import annotations
+
+import re
+
+SAFE_ENV_KEYS: tuple[str, ...] = (
+    "USER",
+    "LOGNAME",
+    "HOME",
+    "PWD",
+    "SHELL",
+    "TERM",
+    "LANG",
+    "LC_ALL",
+    "VIRTUAL_ENV",
+    "CONDA_DEFAULT_ENV",
+    "PYTHON_VERSION",
+    "PATH",
+)
+
+_SENSITIVE_KEY_PARTS = (
+    "secret",
+    "token",
+    "password",
+    "passwd",
+    "credential",
+    "private",
+    "api_key",
+    "apikey",
+    "access_key",
+    "session_key",
+    "auth",
+    "bearer",
+)
+
+
+def omitted_line(count: int, label: str, noun: str = "row") -> str:
+    plural = "" if count == 1 else "s"
+    return f"… {count} {label} {noun}{plural} omitted"
+
+
+def is_sensitive_key(key: str) -> bool:
+    lower = key.lower().replace("-", "_")
+    return (
+        lower == "key"
+        or lower.endswith("_key")
+        or any(part in lower for part in _SENSITIVE_KEY_PARTS)
+    )
+
+
+def is_opaque_value(value: str) -> bool:
+    if len(value) < 64:
+        return False
+    compact = value.strip()
+    if "://" in compact:
+        return True
+    allowed = sum(
+        1 for char in compact if char.isalnum() or char in "+/=_-.:"
+    )
+    return allowed * 100 >= len(compact) * 95 and not any(
+        char.isspace() for char in compact
+    )
+
+
+def format_env_var(key: str, value: str) -> str:
+    if key == "PATH":
+        return f"{key}={_format_path_value(value)}"
+    if is_sensitive_key(key) or is_opaque_value(value):
+        return f"{key}=[redacted]"
+    return f"{key}={value}"
+
+
+def redact_sensitive_text(text: str) -> str:
+    """Redact obvious secret-bearing argv/env tokens without exposing values."""
+
+    tokens = text.split()
+    if not tokens:
+        return text
+
+    output: list[str] = []
+    redact_next = False
+    changed = False
+    for token in tokens:
+        if redact_next:
+            output.append("[redacted]")
+            redact_next = False
+            changed = True
+            continue
+
+        assignment = _sensitive_assignment_prefix(token)
+        if assignment is not None:
+            key_prefix, value = assignment
+            output.append(f"{key_prefix}=[redacted]")
+            if value.lower() in {"bearer", "basic"}:
+                redact_next = True
+            changed = True
+            continue
+
+        if token.lower() == "bearer":
+            output.append(token)
+            redact_next = True
+            continue
+
+        if token.startswith("--"):
+            if "=" in token:
+                option, _value = token.split("=", 1)
+                if is_sensitive_key(option.lstrip("-")):
+                    output.append(f"{option}=[redacted]")
+                    changed = True
+                    continue
+            elif is_sensitive_key(token.lstrip("-")):
+                output.append(token)
+                redact_next = True
+                continue
+
+        output.append(token)
+
+    if not changed:
+        return text
+    return " ".join(output)
+
+
+def _format_path_value(value: str) -> str:
+    separator = (
+        ";"
+        if ";" in value and re.search(r"(?:^|;)[A-Za-z]:[\\/]", value)
+        else ";" if value.count(";") > value.count(":") else ":"
+    )
+    entries = [entry for entry in value.split(separator) if entry]
+    if not entries:
+        return "[]"
+    if len(entries) == 1:
+        return entries[0]
+    return (
+        f"[path entries={len(entries)} first={entries[0]} last={entries[-1]}]"
+    )
+
+
+def _sensitive_assignment_prefix(token: str) -> tuple[str, str] | None:
+    if "=" not in token:
+        return None
+    parts = token.split("=")
+    for index in range(len(parts) - 1):
+        key = parts[index].rsplit(",", 1)[-1]
+        if key.startswith("--"):
+            key = key.lstrip("-")
+        value = "=".join(parts[index + 1 :])
+        if not value:
+            continue
+        if _is_assignment_key(key) and (
+            is_sensitive_key(key) or _is_sensitive_url_assignment(key, value)
+        ):
+            prefix = "=".join(parts[: index + 1])
+            return prefix, value
+    return None
+
+
+def _is_assignment_key(key: str) -> bool:
+    return bool(key) and key.replace("_", "").replace("-", "").isalnum()
+
+
+def _is_sensitive_url_assignment(key: str, value: str) -> bool:
+    return key.lower().endswith("_url") and "://" in value

codetool_shell/filters/table/__init__.py

@@ -0,0 +1,7 @@
+"""Structured table shellion."""
+
+from __future__ import annotations
+
+from .reducer import compress_table_output
+
+__all__ = ["compress_table_output"]