PyPI - codetool-shell - Versions diffs - 0.1.1__py3-none-win_amd64.whl - Mend

codetool-shell 0.1.1__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (72) hide show

codetool_shell/__init__.py +11 -0
codetool_shell/api.py +59 -0
codetool_shell/bin/windows-x86_64/codetool-shell-rust.exe +0 -0
codetool_shell/filters/__init__.py +14 -0
codetool_shell/filters/build_compiler/__init__.py +7 -0
codetool_shell/filters/build_compiler/detector.py +412 -0
codetool_shell/filters/build_compiler/reducer.py +166 -0
codetool_shell/filters/build_compiler/summary.py +617 -0
codetool_shell/filters/ci_job_log/__init__.py +7 -0
codetool_shell/filters/ci_job_log/detector.py +64 -0
codetool_shell/filters/ci_job_log/reducer.py +99 -0
codetool_shell/filters/ci_job_log/summary.py +243 -0
codetool_shell/filters/diff/__init__.py +7 -0
codetool_shell/filters/diff/detector.py +136 -0
codetool_shell/filters/diff/reducer.py +308 -0
codetool_shell/filters/generic_log/__init__.py +7 -0
codetool_shell/filters/generic_log/detector.py +175 -0
codetool_shell/filters/generic_log/reducer.py +99 -0
codetool_shell/filters/generic_log/summary.py +161 -0
codetool_shell/filters/git.py +514 -0
codetool_shell/filters/html_cleanup/__init__.py +7 -0
codetool_shell/filters/html_cleanup/detector.py +136 -0
codetool_shell/filters/html_cleanup/reducer.py +27 -0
codetool_shell/filters/html_cleanup/summary.py +422 -0
codetool_shell/filters/json_payload/__init__.py +7 -0
codetool_shell/filters/json_payload/detector.py +62 -0
codetool_shell/filters/json_payload/reducer.py +81 -0
codetool_shell/filters/json_payload/summary.py +233 -0
codetool_shell/filters/listing/__init__.py +7 -0
codetool_shell/filters/listing/detector.py +294 -0
codetool_shell/filters/listing/reducer.py +30 -0
codetool_shell/filters/log_template/__init__.py +7 -0
codetool_shell/filters/log_template/constants.py +76 -0
codetool_shell/filters/log_template/detector.py +331 -0
codetool_shell/filters/log_template/reducer.py +78 -0
codetool_shell/filters/log_template/template.py +280 -0
codetool_shell/filters/log_template/types.py +21 -0
codetool_shell/filters/opaque_payload/__init__.py +7 -0
codetool_shell/filters/opaque_payload/detector.py +563 -0
codetool_shell/filters/opaque_payload/reducer.py +142 -0
codetool_shell/filters/opaque_payload/summary.py +61 -0
codetool_shell/filters/package_manager/__init__.py +7 -0
codetool_shell/filters/package_manager/detector.py +220 -0
codetool_shell/filters/package_manager/reducer.py +110 -0
codetool_shell/filters/package_manager/summary.py +172 -0
codetool_shell/filters/pipeline.py +65 -0
codetool_shell/filters/rg.py +250 -0
codetool_shell/filters/system_output/__init__.py +7 -0
codetool_shell/filters/system_output/detector.py +600 -0
codetool_shell/filters/system_output/reducer.py +331 -0
codetool_shell/filters/system_output/summary.py +164 -0
codetool_shell/filters/table/__init__.py +7 -0
codetool_shell/filters/table/detector.py +244 -0
codetool_shell/filters/table/reducer.py +57 -0
codetool_shell/filters/table/summary.py +37 -0
codetool_shell/filters/test_runner/__init__.py +7 -0
codetool_shell/filters/test_runner/ansi.py +80 -0
codetool_shell/filters/test_runner/detector.py +409 -0
codetool_shell/filters/test_runner/reducer.py +288 -0
codetool_shell/filters/test_runner/summary.py +449 -0
codetool_shell/filters/text.py +38 -0
codetool_shell/filters/traceback/__init__.py +7 -0
codetool_shell/filters/traceback/detector.py +209 -0
codetool_shell/filters/traceback/reducer.py +141 -0
codetool_shell/filters/traceback/summary.py +122 -0
codetool_shell/filters/tree.py +59 -0
codetool_shell/py.typed +0 -0
codetool_shell/python_backend.py +38 -0
codetool_shell/rust_backend.py +254 -0
codetool_shell-0.1.1.dist-info/METADATA +152 -0
codetool_shell-0.1.1.dist-info/RECORD +72 -0
codetool_shell-0.1.1.dist-info/WHEEL +4 -0

codetool_shell/filters/html_cleanup/detector.py ADDED Viewed

@@ -0,0 +1,136 @@
+"""Conservative full-document HTML detection."""
+from __future__ import annotations
+from dataclasses import dataclass
+_MIN_HTML_BYTES = 4096
+_MAX_CONTROL_RATIO = 0.01
+@dataclass(frozen=True)
+class HttpBody:
+    """HTTP response headers split from a body."""
+    headers: str
+    separator: str
+    body: str
+@dataclass(frozen=True)
+class HtmlPayload:
+    """A detected HTML document, optionally wrapped by HTTP headers."""
+    headers: str
+    separator: str
+    body: str
+def detect_html_payload(text: str) -> HtmlPayload | None:
+    """Return a clear full HTML document body, or ``None``."""
+    http_body = split_http_response(text)
+    if http_body is not None:
+        body = http_body.body
+        headers = http_body.headers
+        separator = http_body.separator
+    else:
+        body = text
+        headers = ""
+        separator = ""
+    if not _is_clear_full_html_document(body):
+        return None
+    return HtmlPayload(headers=headers, separator=separator, body=body)
+def split_http_response(text: str) -> HttpBody | None:
+    """Split clear HTTP response headers from a body."""
+    header_end = text.find("\r\n\r\n")
+    separator = "\r\n\r\n"
+    if header_end < 0:
+        header_end = text.find("\n\n")
+        separator = "\n\n"
+    if header_end <= 0:
+        return None
+    headers = text[:header_end]
+    normalized_headers = headers.replace("\r\n", "\n")
+    first_line = normalized_headers.split("\n", 1)[0]
+    if not first_line.startswith("HTTP/"):
+        return None
+    header_lines = normalized_headers.split("\n")[1:]
+    if header_lines and not any(":" in line for line in header_lines):
+        return None
+    return HttpBody(
+        headers=headers,
+        separator=separator,
+        body=text[header_end + len(separator) :],
+    )
+def _is_clear_full_html_document(body: str) -> bool:
+    if len(body.encode("utf-8")) < _MIN_HTML_BYTES:
+        return False
+    if _is_control_heavy(body):
+        return False
+    stripped = body.strip()
+    lowered = stripped.lower()
+    if not (lowered.startswith("<!doctype html") or lowered.startswith("<html")):
+        return False
+    if not lowered.endswith("</html>"):
+        return False
+    if not all(marker in lowered for marker in ("<html", "</html>", "<body", "</body>")):
+        return False
+    if lowered.find("<body") > lowered.find("</body>"):
+        return False
+    if lowered.find("</body>") > lowered.rfind("</html>"):
+        return False
+    if _looks_like_non_html_source(body):
+        return False
+    return True
+def _is_control_heavy(text: str) -> bool:
+    if "\x00" in text:
+        return True
+    controls = sum(
+        1 for char in text if ord(char) < 32 and char not in {"\n", "\r", "\t"}
+    )
+    return controls / max(len(text), 1) > _MAX_CONTROL_RATIO
+def _looks_like_non_html_source(text: str) -> bool:
+    lowered = text.lower()
+    if any(marker in text for marker in ("```", "{{", "{%", "<%", "<?php")):
+        return True
+    if any(
+        marker in text
+        for marker in (
+            "className=",
+            "export default",
+            "const App",
+            "function App(",
+            "React.",
+            "</>",
+        )
+    ):
+        return True
+    if "Traceback (most recent call last):" in text:
+        return True
+    if "\ndiff --git " in text or text.startswith("diff --git "):
+        return True
+    if "\n@@ " in text or "\n--- " in text or "\n+++ " in text:
+        return True
+    if "\n --> " in text and ("error:" in lowered or "warning:" in lowered):
+        return True
+    if lowered.startswith(("error[", "error:", "warning:")):
+        return True
+    return False

codetool_shell/filters/html_cleanup/reducer.py ADDED Viewed

@@ -0,0 +1,27 @@
+"""Reduce full HTML documents to semantic visible content."""
+from __future__ import annotations
+from ..text import score
+from .detector import detect_html_payload
+from .summary import format_html_summary, summarize_html_document
+def compress_html_cleanup_output(text: str) -> str:
+    """Compress a clear full HTML document, otherwise return unchanged."""
+    payload = detect_html_payload(text)
+    if payload is None:
+        return text
+    summary = summarize_html_document(payload.body)
+    if summary is None:
+        return text
+    candidate = f"{payload.headers}{payload.separator}" + format_html_summary(
+        summary,
+        final_newline=payload.body.endswith("\n"),
+    )
+    if score(candidate) < score(text):
+        return candidate
+    return text

codetool_shell/filters/html_cleanup/summary.py ADDED Viewed

@@ -0,0 +1,422 @@
+"""Extract and format semantic HTML document summaries."""
+from __future__ import annotations
+import html
+import re
+from dataclasses import dataclass
+_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+_SKIP_TAG_RE = re.compile(
+    r"<(script|style|noscript|svg|template)\b[^>]*>.*?</\1\s*>",
+    re.IGNORECASE | re.DOTALL,
+)
+_TITLE_RE = re.compile(r"<title\b[^>]*>(.*?)</title\s*>", re.IGNORECASE | re.DOTALL)
+_BODY_OPEN_RE = re.compile(r"<body\b[^>]*>", re.IGNORECASE)
+_BODY_CLOSE_RE = re.compile(r"</body\s*>", re.IGNORECASE)
+_HEADING_RE = re.compile(
+    r"<h([1-6])\b[^>]*>(.*?)</h\1\s*>", re.IGNORECASE | re.DOTALL
+)
+_ANCHOR_RE = re.compile(r"<a\b([^>]*)>(.*?)</a\s*>", re.IGNORECASE | re.DOTALL)
+_HREF_RE = re.compile(
+    r"""href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))""",
+    re.IGNORECASE,
+)
+_TAG_RE = re.compile(r"<[^>]+>")
+_BLOCK_TAG_RE = re.compile(
+    r"</?(?:address|article|aside|blockquote|br|dd|details|div|dl|dt|figcaption|"
+    r"figure|footer|form|h[1-6]|header|hr|li|main|nav|ol|p|pre|section|table|"
+    r"tbody|td|tfoot|th|thead|tr|ul)\b[^>]*>",
+    re.IGNORECASE,
+)
+_DATA_URL_RE = re.compile(r"data:[^\s\"'<>`)]+", re.IGNORECASE)
+_LONG_DATA_URL_MIN_CHARS = 128
+_MAX_TITLE_CHARS = 80
+_MAX_HEADING_CHARS = 120
+_MAX_LINK_LABEL_CHARS = 80
+_MAX_HREF_CHARS = 160
+_MAX_TEXT_BLOCK_CHARS = 240
+_MAX_HEADINGS = 8
+_MAX_LINKS = 8
+_MAX_SMALL_TEXT_BLOCKS = 8
+_FIRST_TEXT_BLOCKS = 3
+_LAST_TEXT_BLOCKS = 2
+_MAX_SALIENT_TEXT_BLOCKS = 3
+_SALIENT_TERMS = (
+    "error",
+    "warning",
+    "failed",
+    "failure",
+    "exception",
+    "traceback",
+    "fatal",
+    "critical",
+    "denied",
+    "404",
+    "500",
+)
+_SAFE_URL_SCHEMES = {"http", "https", "mailto", "tel"}
+_COMMON_HREF_ENTITY_RE = re.compile(
+    r"&(newline|tab|colon);", re.IGNORECASE
+)
+@dataclass(frozen=True)
+class RemovalCounts:
+    """Counts of non-semantic HTML payloads removed by cleanup."""
+    comments: int = 0
+    scripts: int = 0
+    styles: int = 0
+    svg: int = 0
+    data_urls: int = 0
+    @property
+    def any(self) -> bool:
+        return any((self.comments, self.scripts, self.styles, self.svg, self.data_urls))
+@dataclass(frozen=True)
+class Link:
+    """A visible anchor label and safe href."""
+    label: str
+    href: str
+@dataclass(frozen=True)
+class Heading:
+    """A visible heading."""
+    level: str
+    text: str
+@dataclass(frozen=True)
+class HtmlSummary:
+    """Extracted semantic HTML content."""
+    byte_count: int
+    title: str | None
+    headings: list[Heading]
+    links: list[Link]
+    text_blocks: list[str]
+    removals: RemovalCounts
+    @property
+    def has_semantic_content(self) -> bool:
+        return bool(self.headings or self.links or self.text_blocks)
+def summarize_html_document(body: str) -> HtmlSummary | None:
+    """Extract a bounded semantic summary from a full HTML document."""
+    cleaned, removals = _remove_nonsemantic_html(body)
+    title = _extract_title(cleaned)
+    body_html = _extract_body(cleaned)
+    if body_html is None:
+        return None
+    headings = _extract_headings(body_html)
+    links = _extract_links(body_html)
+    text_blocks = _extract_text_blocks(body_html)
+    summary = HtmlSummary(
+        byte_count=len(body.encode("utf-8")),
+        title=title,
+        headings=headings,
+        links=links,
+        text_blocks=text_blocks,
+        removals=removals,
+    )
+    if not summary.has_semantic_content:
+        return None
+    return summary
+def format_html_summary(summary: HtmlSummary, *, final_newline: bool) -> str:
+    """Format an HTML summary in a stable, bounded layout."""
+    title = f" title={_quote(summary.title)}" if summary.title else ""
+    lines = [f"html document: bytes={summary.byte_count}{title}"]
+    if summary.headings:
+        lines.append("headings:")
+        for heading in summary.headings[:_MAX_HEADINGS]:
+            lines.append(f"- h{heading.level}: {heading.text}")
+        omitted = len(summary.headings) - _MAX_HEADINGS
+        if omitted > 0:
+            lines.append(f"… {omitted} headings omitted")
+    if summary.links:
+        lines.append("links:")
+        for link in summary.links[:_MAX_LINKS]:
+            lines.append(f"- {link.label} -> {link.href}")
+        omitted = len(summary.links) - _MAX_LINKS
+        if omitted > 0:
+            lines.append(f"… {omitted} links omitted")
+    if summary.text_blocks:
+        lines.append("text:")
+        lines.extend(_format_text_blocks(summary.text_blocks))
+    if summary.removals.any:
+        lines.append(
+            "[html cleanup removed: "
+            f"comments={summary.removals.comments} "
+            f"scripts={summary.removals.scripts} "
+            f"styles={summary.removals.styles} "
+            f"svg={summary.removals.svg} "
+            f"data_urls={summary.removals.data_urls}]"
+        )
+    output = "\n".join(lines)
+    if final_newline:
+        output += "\n"
+    return output
+def _remove_nonsemantic_html(text: str) -> tuple[str, RemovalCounts]:
+    comments = 0
+    scripts = 0
+    styles = 0
+    svg = 0
+    def replace_comment(match: re.Match[str]) -> str:
+        nonlocal comments
+        comments += 1
+        return " "
+    without_comments = _COMMENT_RE.sub(replace_comment, text)
+    def replace_skipped(match: re.Match[str]) -> str:
+        nonlocal scripts, styles, svg
+        tag = match.group(1).lower()
+        if tag == "style":
+            styles += 1
+        elif tag == "svg":
+            svg += 1
+        else:
+            scripts += 1
+        return " "
+    cleaned = _SKIP_TAG_RE.sub(replace_skipped, without_comments)
+    data_urls = sum(
+        1
+        for match in _DATA_URL_RE.finditer(cleaned)
+        if len(match.group(0)) >= _LONG_DATA_URL_MIN_CHARS
+    )
+    return cleaned, RemovalCounts(comments, scripts, styles, svg, data_urls)
+def _extract_title(text: str) -> str | None:
+    match = _TITLE_RE.search(text)
+    if match is None:
+        return None
+    title = _visible_text(match.group(1))
+    if not title:
+        return None
+    return _truncate(title, _MAX_TITLE_CHARS)
+def _extract_body(text: str) -> str | None:
+    open_match = _BODY_OPEN_RE.search(text)
+    if open_match is None:
+        return None
+    close_match = _BODY_CLOSE_RE.search(text, open_match.end())
+    if close_match is None:
+        return None
+    return text[open_match.end() : close_match.start()]
+def _extract_headings(body_html: str) -> list[Heading]:
+    headings: list[Heading] = []
+    seen: set[tuple[str, str]] = set()
+    for match in _HEADING_RE.finditer(body_html):
+        text = _visible_text(match.group(2))
+        if not text:
+            continue
+        heading = Heading(match.group(1), _truncate(text, _MAX_HEADING_CHARS))
+        key = (heading.level, heading.text)
+        if key in seen:
+            continue
+        seen.add(key)
+        headings.append(heading)
+    return headings
+def _extract_links(body_html: str) -> list[Link]:
+    links: list[Link] = []
+    seen: set[tuple[str, str]] = set()
+    for match in _ANCHOR_RE.finditer(body_html):
+        raw_href = _extract_href(match.group(1))
+        href = _normalize_safe_href(raw_href) if raw_href is not None else None
+        if not href:
+            continue
+        label = _visible_text(match.group(2))
+        if not label:
+            label = href
+        link = Link(
+            label=_truncate(label, _MAX_LINK_LABEL_CHARS),
+            href=_truncate(href, _MAX_HREF_CHARS),
+        )
+        key = (link.label, link.href)
+        if key in seen:
+            continue
+        seen.add(key)
+        links.append(link)
+    return links
+def _extract_href(attrs: str) -> str | None:
+    match = _HREF_RE.search(attrs)
+    if match is None:
+        return None
+    for group in match.groups():
+        if group is not None:
+            return group.strip()
+    return None
+def _normalize_safe_href(raw_href: str) -> str | None:
+    """Decode and allow only relative, fragment, and explicitly safe hrefs."""
+    decoded = _decode_href_entities(raw_href).strip()
+    if not decoded:
+        return None
+    scheme = _normalized_explicit_scheme(decoded)
+    if scheme is not None and scheme not in _SAFE_URL_SCHEMES:
+        return None
+    href = _sanitize_snippet(decoded).strip()
+    return href or None
+def _decode_href_entities(value: str) -> str:
+    """Decode href entities needed for robust scheme detection."""
+    def replace_common(match: re.Match[str]) -> str:
+        entity = match.group(1).lower()
+        if entity == "newline":
+            return "\n"
+        if entity == "tab":
+            return "\t"
+        return ":"
+    return html.unescape(_COMMON_HREF_ENTITY_RE.sub(replace_common, value))
+def _normalized_explicit_scheme(href: str) -> str | None:
+    """Return a normalized explicit URL scheme, or None for relative URLs."""
+    trimmed = href.lstrip()
+    if not trimmed or trimmed.startswith("#"):
+        return None
+    prefix_chars: list[str] = []
+    for char in trimmed:
+        if char in "/?#":
+            return None
+        if char == ":":
+            scheme = "".join(
+                candidate
+                for candidate in prefix_chars
+                if not _is_ascii_whitespace_or_control(candidate)
+            ).lower()
+            if not scheme:
+                return ""
+            if not re.fullmatch(r"[a-z][a-z0-9+.-]*", scheme):
+                return ""
+            return scheme
+        prefix_chars.append(char)
+    return None
+def _is_ascii_whitespace_or_control(char: str) -> bool:
+    codepoint = ord(char)
+    return codepoint <= 32 or codepoint == 127
+def _extract_text_blocks(body_html: str) -> list[str]:
+    with_breaks = _BLOCK_TAG_RE.sub("\n", body_html)
+    without_tags = _TAG_RE.sub(" ", with_breaks)
+    blocks: list[str] = []
+    seen: set[str] = set()
+    for raw_block in without_tags.splitlines():
+        block = _normalize_visible_text(raw_block)
+        if not block or not _is_semantic_text_block(block):
+            continue
+        block = _truncate(block, _MAX_TEXT_BLOCK_CHARS)
+        if block in seen:
+            continue
+        seen.add(block)
+        blocks.append(block)
+    return blocks
+def _format_text_blocks(blocks: list[str]) -> list[str]:
+    if len(blocks) <= _MAX_SMALL_TEXT_BLOCKS:
+        return [f"- {block}" for block in blocks]
+    selected = set(range(_FIRST_TEXT_BLOCKS))
+    selected.update(range(max(_FIRST_TEXT_BLOCKS, len(blocks) - _LAST_TEXT_BLOCKS), len(blocks)))
+    salient = [
+        index
+        for index, block in enumerate(blocks)
+        if index not in selected and _is_salient_text(block)
+    ][:_MAX_SALIENT_TEXT_BLOCKS]
+    selected.update(salient)
+    output: list[str] = []
+    previous = -1
+    for index in sorted(selected):
+        omitted = index - previous - 1
+        if omitted > 0:
+            output.append(f"… {omitted} text blocks omitted")
+        output.append(f"- {blocks[index]}")
+        previous = index
+    omitted = len(blocks) - previous - 1
+    if omitted > 0:
+        output.append(f"… {omitted} text blocks omitted")
+    return output
+def _visible_text(fragment: str) -> str:
+    return _normalize_visible_text(_TAG_RE.sub(" ", fragment))
+def _normalize_visible_text(text: str) -> str:
+    unescaped = html.unescape(text)
+    collapsed = re.sub(r"\s+", " ", unescaped).strip()
+    return _sanitize_snippet(collapsed)
+def _sanitize_snippet(text: str) -> str:
+    return _DATA_URL_RE.sub("[data-url omitted]", text)
+def _is_semantic_text_block(text: str) -> bool:
+    alnum_count = sum(1 for char in text if char.isalnum())
+    if alnum_count < 3:
+        return False
+    words = [part for part in re.split(r"\W+", text) if part]
+    if len(words) >= 2:
+        return True
+    return _is_salient_text(text)
+def _is_salient_text(text: str) -> bool:
+    lowered = text.lower()
+    return any(term in lowered for term in _SALIENT_TERMS)
+def _truncate(text: str, max_chars: int) -> str:
+    if len(text) <= max_chars:
+        return text
+    return f"{text[: max_chars - 1]}…"
+def _quote(value: str) -> str:
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'

codetool_shell/filters/json_payload/__init__.py ADDED Viewed

@@ -0,0 +1,7 @@
+"""JSON and JSONL payload compression."""
+from __future__ import annotations
+from .reducer import compress_json_payload_output
+__all__ = ["compress_json_payload_output"]

codetool_shell/filters/json_payload/detector.py ADDED Viewed

@@ -0,0 +1,62 @@
+"""Conservative JSON/JSONL payload detection."""
+from __future__ import annotations
+import json
+from dataclasses import dataclass
+from typing import Any
+@dataclass(frozen=True)
+class JsonlPayload:
+    """Parsed JSONL object records."""
+    records: tuple[dict[str, Any], ...]
+def parse_whole_json_payload(text: str) -> Any | None:
+    """Return a parsed whole-output JSON object/array, or ``None``."""
+    body = text.strip()
+    if not _looks_like_json_container(body):
+        return None
+    try:
+        value = json.loads(body, parse_constant=_reject_json_constant)
+    except (json.JSONDecodeError, ValueError):
+        return None
+    if not isinstance(value, (dict, list)):
+        return None
+    return value
+def parse_jsonl_payload(lines: list[str]) -> JsonlPayload | None:
+    """Return parsed object-record JSONL when every non-empty line is JSON."""
+    records: list[dict[str, Any]] = []
+    for line in lines:
+        body = line.strip()
+        if not body:
+            continue
+        if not (body.startswith("{") and body.endswith("}")):
+            return None
+        try:
+            value = json.loads(body, parse_constant=_reject_json_constant)
+        except (json.JSONDecodeError, ValueError):
+            return None
+        if not isinstance(value, dict):
+            return None
+        records.append(value)
+    if not records:
+        return None
+    return JsonlPayload(tuple(records))
+def _looks_like_json_container(body: str) -> bool:
+    return (body.startswith("{") and body.endswith("}")) or (
+        body.startswith("[") and body.endswith("]")
+    )
+def _reject_json_constant(value: str) -> None:
+    raise ValueError(f"non-standard JSON constant {value}")