codetool-shell 0.1.1__py3-none-win_arm64.whl

codetool_shell/filters/json_payload/reducer.py

@@ -0,0 +1,81 @@
+"""Reduce JSON/JSONL payload output."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import parse_jsonl_payload, parse_whole_json_payload
+from .summary import summarize_json_array, summarize_jsonl_records
+
+
+def compress_json_payload_output(text: str) -> str:
+    """Compress whole JSON or object-record JSONL, otherwise return unchanged."""
+
+    best = text
+    final_newline = text.endswith("\n")
+
+    value = parse_whole_json_payload(text)
+    if value is not None:
+        minified = _with_final_newline(_minify_json_source(text.strip()), final_newline)
+        best = _choose_smaller(best, minified)
+        summary = summarize_json_array(value, final_newline)
+        if summary is not None:
+            best = _choose_smaller(best, summary)
+        return best
+
+    lines, final_newline = split_preserving_final_newline(text)
+    payload = parse_jsonl_payload(lines)
+    if payload is None:
+        return text
+
+    minified_lines = [
+        _minify_json_source(line.strip()) if line.strip() else "" for line in lines
+    ]
+    best = _choose_smaller(
+        best,
+        join_preserving_final_newline(minified_lines, final_newline),
+    )
+    summary = summarize_jsonl_records(payload.records, final_newline)
+    if summary is not None:
+        best = _choose_smaller(best, summary)
+    return best
+
+
+def _choose_smaller(current: str, candidate: str) -> str:
+    if score(candidate) < score(current):
+        return candidate
+    return current
+
+
+def _with_final_newline(text: str, final_newline: bool) -> str:
+    if final_newline:
+        return f"{text}\n"
+    return text
+
+
+def _minify_json_source(source: str) -> str:
+    """Remove insignificant JSON whitespace while preserving raw string content."""
+
+    output: list[str] = []
+    in_string = False
+    escaped = False
+
+    for char in source:
+        if in_string:
+            output.append(char)
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+
+        if char == '"':
+            in_string = True
+            output.append(char)
+        elif char in " \t\r\n":
+            continue
+        else:
+            output.append(char)
+
+    return "".join(output)

codetool_shell/filters/json_payload/summary.py

@@ -0,0 +1,233 @@
+"""Summaries for large JSON object-record collections."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from ..text import join_preserving_final_newline
+
+_MIN_SUMMARY_ROWS = 10
+_HEAD_ROWS = 3
+_TAIL_ROWS = 3
+_STATUS_KEYS = {"status", "result", "outcome"}
+_OK_WORDS = {
+    "ok",
+    "success",
+    "successful",
+    "pass",
+    "passed",
+    "true",
+    "done",
+    "complete",
+    "completed",
+    "healthy",
+    "0",
+}
+_ALERT_RE = re.compile(
+    r"\b(?:error|errors|warn|warning|warnings|fail|failed|failure|fatal|critical|exception|traceback)\b",
+    re.IGNORECASE,
+)
+_PATH_RE = re.compile(
+    r"(^|[\s|])(?:\.{0,2}/|/|~/|[A-Za-z]:[\\/]|[A-Za-z0-9_.@+-]+/[A-Za-z0-9_.@+/\-]+)"
+)
+_UUID_RE = re.compile(
+    r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
+)
+_LONG_HEX_RE = re.compile(r"\b[0-9a-fA-F]{12,}\b")
+_ISSUE_ID_RE = re.compile(r"\b[A-Z]{2,}-\d{2,}\b")
+_ID_VALUE_RE = re.compile(r"^[A-Za-z0-9_.-]{8,}$")
+
+
+def summarize_json_array(value: Any, final_newline: bool) -> str | None:
+    """Summarize a top-level JSON array of object records."""
+
+    if not isinstance(value, list) or len(value) < _MIN_SUMMARY_ROWS:
+        return None
+    if not all(isinstance(row, dict) for row in value):
+        return None
+    return _summarize_records(
+        header_prefix="json array",
+        count_label="rows",
+        omitted_label="JSON rows",
+        records=value,
+        final_newline=final_newline,
+    )
+
+
+def summarize_jsonl_records(
+    records: tuple[dict[str, Any], ...],
+    final_newline: bool,
+) -> str | None:
+    """Summarize JSONL object records."""
+
+    if len(records) < _MIN_SUMMARY_ROWS:
+        return None
+    return _summarize_records(
+        header_prefix="jsonl",
+        count_label="records",
+        omitted_label="JSONL records",
+        records=list(records),
+        final_newline=final_newline,
+    )
+
+
+def _summarize_records(
+    *,
+    header_prefix: str,
+    count_label: str,
+    omitted_label: str,
+    records: list[dict[str, Any]],
+    final_newline: bool,
+) -> str:
+    keys = _schema_keys(records)
+    selected_indices = _select_record_indices(records)
+    if header_prefix == "jsonl":
+        header = f"{header_prefix} {count_label}={len(records)} keys={','.join(keys)}"
+    else:
+        header = f"{header_prefix}: {count_label}={len(records)} keys={','.join(keys)}"
+    lines = [header]
+    cursor = 0
+
+    for index in selected_indices:
+        omitted = index - cursor
+        if omitted > 0:
+            lines.append(f"… {omitted} {omitted_label} omitted")
+        lines.append(_format_record(records[index]))
+        cursor = index + 1
+
+    if cursor < len(records):
+        lines.append(f"… {len(records) - cursor} {omitted_label} omitted")
+
+    return join_preserving_final_newline(lines, final_newline)
+
+
+def _schema_keys(records: list[dict[str, Any]]) -> list[str]:
+    keys: list[str] = []
+    seen: set[str] = set()
+    for record in records:
+        for key in record:
+            if key not in seen:
+                seen.add(key)
+                keys.append(key)
+    return keys
+
+
+def _select_record_indices(records: list[dict[str, Any]]) -> list[int]:
+    total = len(records)
+    selected = set(range(min(_HEAD_ROWS, total)))
+    selected.update(range(max(_HEAD_ROWS, total - _TAIL_ROWS), total))
+    baseline_keys = set(records[0]) if records else set()
+    selected.update(
+        index
+        for index, record in enumerate(records)
+        if _is_salient_record(record, baseline_keys)
+    )
+    return sorted(selected)
+
+
+def _is_salient_record(record: dict[str, Any], baseline_keys: set[str]) -> bool:
+    return (
+        set(record) != baseline_keys
+        or _has_non_ok_status(record)
+        or _has_named_id_field(record)
+        or any(_is_interesting_text(value) for value in _iter_string_values(record))
+    )
+
+
+def _format_record(record: dict[str, Any]) -> str:
+    return json.dumps(record, ensure_ascii=False, separators=(",", ":"))
+
+
+def _iter_string_values(value: Any) -> list[str]:
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, dict):
+        values: list[str] = []
+        for child in value.values():
+            values.extend(_iter_string_values(child))
+        return values
+    if isinstance(value, list):
+        values = []
+        for child in value:
+            values.extend(_iter_string_values(child))
+        return values
+    return []
+
+
+def _is_interesting_text(value: str) -> bool:
+    return (
+        bool(_ALERT_RE.search(value))
+        or "://" in value
+        or bool(_PATH_RE.search(value))
+        or bool(_UUID_RE.search(value))
+        or bool(_LONG_HEX_RE.search(value))
+        or bool(_ISSUE_ID_RE.search(value))
+    )
+
+
+def _has_named_id_field(value: Any) -> bool:
+    if isinstance(value, dict):
+        for key, child in value.items():
+            if _is_named_id_key(key) and _is_named_id_value(child):
+                return True
+            if _has_named_id_field(child):
+                return True
+    elif isinstance(value, list):
+        return any(_has_named_id_field(child) for child in value)
+    return False
+
+
+def _is_named_id_key(key: str) -> bool:
+    lower = key.lower()
+    return (
+        lower == "id"
+        or lower.endswith("_id")
+        or lower.endswith("-id")
+        or "uuid" in lower
+        or "hash" in lower
+        or lower in {"sha", "sha1", "sha256", "sha512"}
+    )
+
+
+def _is_named_id_value(value: Any) -> bool:
+    if not isinstance(value, str):
+        return False
+    stripped = value.strip()
+    return (
+        len(stripped) >= 8
+        and not stripped.isdigit()
+        and bool(_ID_VALUE_RE.fullmatch(stripped))
+    )
+
+
+def _has_non_ok_status(value: Any) -> bool:
+    if isinstance(value, dict):
+        for key, child in value.items():
+            lower = key.lower()
+            if lower in _STATUS_KEYS and not _is_ok_status_value(lower, child):
+                return True
+            if _has_non_ok_status(child):
+                return True
+    elif isinstance(value, list):
+        return any(_has_non_ok_status(child) for child in value)
+    return False
+
+
+def _is_ok_status_value(key: str, value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, int | float) and not isinstance(value, bool):
+        if key == "status":
+            return value == 0 or 200 <= value < 300
+        return value == 0
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in _OK_WORDS:
+            return True
+        if key == "status" and normalized.isdigit():
+            status = int(normalized)
+            return 200 <= status < 300
+        return False
+    return False

codetool_shell/filters/listing/__init__.py

@@ -0,0 +1,7 @@
+"""File/tree/listing shellion."""
+
+from __future__ import annotations
+
+from .reducer import compress_listing_output
+
+__all__ = ["compress_listing_output"]

codetool_shell/filters/listing/detector.py

@@ -0,0 +1,294 @@
+"""Conservative parsers for file/tree/listing output."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ListingEntry:
+    """A path entry selected from listing output."""
+
+    path: str
+    suffix: str = ""
+
+
+@dataclass(frozen=True)
+class ListingParse:
+    """Parsed listing output ready for tree rendering."""
+
+    entries: tuple[ListingEntry, ...]
+    summary_lines: tuple[str, ...] = ()
+
+
+_MIN_PATH_LIST_LINES = 4
+_TREE_SUMMARY_RE = re.compile(
+    r"^(?:\d+ director(?:y|ies)(?:, \d+ files?)?|\d+ files?)$"
+)
+_RG_STYLE_RE = re.compile(r"^(.+?):([1-9]\d*)(?::([1-9]\d*))?:(.*)$")
+_PERMISSION_ROW_RE = re.compile(r"^[bcdlps-][rwxStTs-]{9}\s+")
+_FILE_EXTENSION_RE = re.compile(r"(^|/)[A-Za-z0-9_.@+-]+\.[A-Za-z0-9][A-Za-z0-9_.+-]*/?$")
+
+
+def parse_listing_lines(lines: list[str]) -> ListingParse | None:
+    """Return a conservative listing parse, or ``None``."""
+
+    for parser in (_parse_path_list, _parse_tree_output, _parse_ls_r_output):
+        parsed = parser(lines)
+        if parsed is not None and parsed.entries:
+            return parsed
+    return None
+
+
+def _parse_path_list(lines: list[str]) -> ListingParse | None:
+    if len(lines) < _MIN_PATH_LIST_LINES or any(not line for line in lines):
+        return None
+
+    entries: list[ListingEntry] = []
+    for line in lines:
+        if not _is_safe_path_line(line):
+            return None
+        entries.append(_entry_from_path(line))
+
+    return ListingParse(tuple(entries))
+
+
+def _parse_tree_output(lines: list[str]) -> ListingParse | None:
+    non_blank = [line for line in lines if line.strip()]
+    if len(non_blank) < 3:
+        return None
+
+    root_prefix: str | None = None
+    branch_count = 0
+    entries: list[ListingEntry] = []
+    summary_lines: list[str] = []
+    stack: list[str] = []
+
+    for raw_line in lines:
+        stripped = raw_line.strip()
+        if not stripped:
+            continue
+
+        if _is_tree_summary_line(stripped):
+            summary_lines.append(stripped)
+            continue
+
+        branch = _parse_tree_branch_line(raw_line)
+        if branch is None:
+            if branch_count == 0 and root_prefix is None and _is_safe_tree_root(stripped):
+                root_prefix = None if stripped in {".", "./"} else stripped
+                continue
+            return None
+
+        depth, label = branch
+        if depth > len(stack) or not _is_safe_tree_label(label):
+            return None
+
+        branch_count += 1
+        clean_label, suffix = _split_directory_suffix(label)
+        stack = stack[:depth]
+        path_parts = []
+        if root_prefix is not None:
+            path_parts.append(root_prefix)
+        path_parts.extend(stack)
+        path_parts.append(clean_label)
+        entries.append(ListingEntry("/".join(path_parts), suffix))
+        stack.append(clean_label)
+
+    if branch_count < 3:
+        return None
+    return ListingParse(tuple(_dedupe_entries(entries)), tuple(summary_lines))
+
+
+def _parse_ls_r_output(lines: list[str]) -> ListingParse | None:
+    if len([line for line in lines if line.strip()]) < 4:
+        return None
+
+    current_dir: str | None = None
+    headers: list[str] = []
+    child_names: list[str] = []
+    entries: list[ListingEntry] = []
+
+    for raw_line in lines:
+        line = raw_line.strip()
+        if not line:
+            continue
+        if raw_line != line:
+            return None
+
+        if line.endswith(":"):
+            header = line[:-1]
+            if not _is_safe_ls_header(header):
+                return None
+            current_dir = header
+            headers.append(header)
+            if header not in {".", "./"}:
+                entries.append(ListingEntry(header, "/"))
+            continue
+
+        if current_dir is None or not _is_safe_ls_child(line):
+            return None
+
+        child_names.append(line)
+        child, suffix = _split_directory_suffix(line)
+        entries.append(ListingEntry(_join_listing_path(current_dir, child), suffix))
+
+    if len(headers) < 2 or len(child_names) < 2:
+        return None
+    if not _has_strong_ls_r_signal(headers, child_names):
+        return None
+
+    return ListingParse(tuple(_dedupe_entries(entries)))
+
+
+def _is_safe_path_line(line: str) -> bool:
+    if not line or line.strip() != line:
+        return False
+    if line in {".", "./"}:
+        return True
+    if line in {"..", "../"}:
+        return False
+    if len(line) > 300:
+        return False
+    if any(char in line for char in "\t|"):
+        return False
+    if "://" in line or " " in line:
+        return False
+    if _PERMISSION_ROW_RE.match(line) or _is_rg_style_line(line):
+        return False
+    if ":" in line and not re.match(r"^[A-Za-z]:[\\/]", line):
+        return False
+    if line.startswith(("Traceback", "File ", "at ", "├", "└", "|--", "`--")):
+        return False
+    return _looks_like_path(line)
+
+
+def _looks_like_path(value: str) -> bool:
+    if value in {".", "./"}:
+        return True
+    if value.startswith(("./", "../", "/", "~/", "~\\")):
+        return True
+    if "/" in value or "\\" in value:
+        return True
+    return _FILE_EXTENSION_RE.search(value) is not None
+
+
+def _is_rg_style_line(line: str) -> bool:
+    match = _RG_STYLE_RE.match(line)
+    return match is not None and _looks_like_path(match.group(1))
+
+
+def _entry_from_path(path: str) -> ListingEntry:
+    clean, suffix = _split_directory_suffix(path)
+    return ListingEntry(clean, suffix)
+
+
+def _split_directory_suffix(path: str) -> tuple[str, str]:
+    if path not in {"/", "\\"} and path.endswith(("/", "\\")):
+        return path.rstrip("/\\"), "/"
+    return path, ""
+
+
+def _is_tree_summary_line(line: str) -> bool:
+    return _TREE_SUMMARY_RE.match(line) is not None
+
+
+def _parse_tree_branch_line(line: str) -> tuple[int, str] | None:
+    for marker in ("├── ", "└── ", "|-- ", "`-- "):
+        index = line.find(marker)
+        if index < 0:
+            continue
+        prefix = line[:index]
+        if index % 4 != 0 or not _is_tree_prefix(prefix):
+            return None
+        label = line[index + len(marker) :].strip()
+        if not label:
+            return None
+        return index // 4, label
+    return None
+
+
+def _is_tree_prefix(prefix: str) -> bool:
+    return all(
+        chunk in {"│   ", "|   ", "    "}
+        for chunk in (prefix[index : index + 4] for index in range(0, len(prefix), 4))
+    )
+
+
+def _is_safe_tree_root(value: str) -> bool:
+    if value in {".", "./"}:
+        return True
+    if any(char in value for char in "\t|:"):
+        return False
+    if "://" in value or " " in value:
+        return False
+    return bool(re.match(r"^[A-Za-z0-9_.@+~/-]+$", value))
+
+
+def _is_safe_tree_label(label: str) -> bool:
+    if not label or label in {".", ".."}:
+        return False
+    if label.startswith("[") or " -> " in label:
+        return False
+    if any(char in label for char in "\t:"):
+        return False
+    if _PERMISSION_ROW_RE.match(label):
+        return False
+    return True
+
+
+def _is_safe_ls_header(header: str) -> bool:
+    if not header or header in {"..", "../"}:
+        return False
+    if header.strip() != header:
+        return False
+    if any(char in header for char in "\t|:"):
+        return False
+    if "://" in header or " " in header:
+        return False
+    return bool(re.match(r"^[A-Za-z0-9_.@+~/-]+$", header))
+
+
+def _is_safe_ls_child(child: str) -> bool:
+    if child in {".", ".."} or child.startswith("total "):
+        return False
+    if child.strip() != child:
+        return False
+    if _PERMISSION_ROW_RE.match(child):
+        return False
+    clean_child = child[:-1] if child.endswith("/") else child
+    if any(char in clean_child for char in "\t|:/\\"):
+        return False
+    if "://" in clean_child or " " in clean_child:
+        return False
+    return bool(re.match(r"^[A-Za-z0-9_.@+-]+$", clean_child))
+
+
+def _has_strong_ls_r_signal(headers: list[str], child_names: list[str]) -> bool:
+    return any(
+        header in {".", "./"}
+        or header.startswith(("./", "../", "/", "~/"))
+        or "/" in header
+        or "\\" in header
+        or _looks_like_path(header)
+        for header in headers
+    ) or any(_looks_like_path(child) for child in child_names)
+
+
+def _join_listing_path(parent: str, child: str) -> str:
+    if parent in {".", "./"}:
+        return child
+    return f"{parent.rstrip('/')}/{child}"
+
+
+def _dedupe_entries(entries: list[ListingEntry]) -> list[ListingEntry]:
+    seen: set[tuple[str, str]] = set()
+    output: list[ListingEntry] = []
+    for entry in entries:
+        key = (entry.path, entry.suffix)
+        if key in seen:
+            continue
+        seen.add(key)
+        output.append(entry)
+    return output

codetool_shell/filters/listing/reducer.py

@@ -0,0 +1,30 @@
+"""Reduce safe file/tree/listing output to compact path trees."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from ..tree import TreeRow, format_tree
+from .detector import parse_listing_lines
+
+
+def compress_listing_output(text: str) -> str:
+    """Compress safe file listing output, otherwise return ``text`` unchanged."""
+
+    lines, final_newline = split_preserving_final_newline(text)
+    parsed = parse_listing_lines(lines)
+    if parsed is None:
+        return text
+
+    rows = [TreeRow(entry.path, suffix=entry.suffix) for entry in parsed.entries]
+    tree_text = format_tree(rows)
+    if not tree_text:
+        return text
+
+    candidate_lines = [*parsed.summary_lines, *tree_text.splitlines()]
+    if len(candidate_lines) < 2:
+        return text
+
+    candidate = join_preserving_final_newline(candidate_lines, final_newline)
+    if score(candidate) < score(text):
+        return candidate
+    return text

codetool_shell/filters/log_template/__init__.py

@@ -0,0 +1,7 @@
+"""Repeated log-template compaction."""
+
+from __future__ import annotations
+
+from .reducer import compress_log_template_output
+
+__all__ = ["compress_log_template_output"]