codetool-shell 0.1.1__py3-none-win_arm64.whl

codetool_shell/filters/opaque_payload/detector.py

@@ -0,0 +1,563 @@
+"""Conservative opaque payload detectors."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from .summary import byte_len
+
+_BASE64_ALPHABET = frozenset(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=_-"
+)
+_BASE64_STANDARD_ALPHABET = frozenset(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+)
+_BASE64_URLSAFE_ALPHABET = frozenset(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_="
+)
+_HEX_ALPHABET = frozenset("0123456789abcdefABCDEF")
+_STRUCTURAL_PAYLOAD_CHARS = frozenset("{}[]\"':,<>;")
+_BEGIN_RE = re.compile(r"^-----BEGIN ([A-Z0-9][A-Z0-9 ._-]*?)-----$")
+_TITLE_RE = re.compile(r"(?is)<title[^>]*>(.*?)</title>")
+_SIMPLE_KEY_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$")
+_MIME_RE = re.compile(r"^[A-Za-z0-9.+-]+/[A-Za-z0-9.+-]+$")
+_BASE64_SINGLE_TOKEN_MIN_LENGTH = 1024
+_BASE64_BLOCK_MIN_LINES = 4
+_BASE64_BLOCK_MIN_CHARS = 2048
+_BASE64_RUN_LINE_MIN_LENGTH = 32
+_BASE64_VALID_ALPHABET_RATIO = 0.98
+_PEM_BODY_MIN_LINES = 4
+_PEM_BODY_MIN_BYTES = 512
+_PEM_BODY_LINE_MIN_LENGTH = 16
+_DATA_URL_METADATA_MAX_LENGTH = 256
+_DATA_URL_PAYLOAD_MIN_LENGTH = 2048
+_DATA_URL_MIN_BYTES = 4096
+_DATA_URL_BASE64_ALPHABET_RATIO = 0.95
+_OPAQUE_JSON_STRING_MIN_BYTES = 8192
+_OPAQUE_JSON_MIN_OPAQUE_RATIO_PERCENT = 70
+_WHOLE_PAYLOAD_MIN_BYTES = 32768
+_WHOLE_PAYLOAD_MAX_LINES = 3
+_WHOLE_PAYLOAD_MAX_LINE_MIN_LENGTH = 8192
+_MINIFIED_PAYLOAD_MAX_WHITESPACE_RATIO = 0.10
+_OPAQUE_STRING_MAX_WHITESPACE_RATIO = 0.01
+
+
+@dataclass(frozen=True)
+class LinePayloadSpan:
+    """A line-oriented opaque payload span."""
+
+    start: int
+    end: int
+    kind: str
+    byte_count: int
+    line_count: int
+    label: str | None = None
+
+
+@dataclass(frozen=True)
+class DataUrlSpan:
+    """A character span for a large data URL."""
+
+    start: int
+    end: int
+    mime: str
+    encoding: str
+    byte_count: int
+
+
+@dataclass(frozen=True)
+class JsonPayloadSummary:
+    """Whole JSON payload summary metadata."""
+
+    kind: str
+    keys: list[str]
+    byte_count: int
+
+
+@dataclass(frozen=True)
+class HtmlPayloadSummary:
+    """Whole HTML payload summary metadata."""
+
+    byte_count: int
+    title: str | None
+
+
+@dataclass(frozen=True)
+class MinifiedJsSummary:
+    """Whole minified JavaScript payload summary metadata."""
+
+    byte_count: int
+
+
+@dataclass(frozen=True)
+class HttpBody:
+    """HTTP response headers split from a body."""
+
+    headers: str
+    separator: str
+    body: str
+
+
+def split_http_response(text: str) -> HttpBody | None:
+    """Split clear HTTP response headers from a body."""
+
+    header_end = text.find("\r\n\r\n")
+    separator = "\r\n\r\n"
+    if header_end < 0:
+        header_end = text.find("\n\n")
+        separator = "\n\n"
+    if header_end <= 0:
+        return None
+
+    headers = text[:header_end]
+    normalized_headers = headers.replace("\r\n", "\n")
+    first_line = normalized_headers.split("\n", 1)[0]
+    if not first_line.startswith("HTTP/"):
+        return None
+
+    header_lines = normalized_headers.split("\n")[1:]
+    if header_lines and not any(":" in line for line in header_lines):
+        return None
+
+    return HttpBody(
+        headers=headers,
+        separator=separator,
+        body=text[header_end + len(separator) :],
+    )
+
+
+def find_line_payload_spans(lines: list[str]) -> list[LinePayloadSpan]:
+    """Return non-overlapping PEM/base64 line spans."""
+
+    spans: list[LinePayloadSpan] = []
+    index = 0
+    while index < len(lines):
+        pem_span = _parse_pem_span(lines, index)
+        if pem_span is not None:
+            spans.append(pem_span)
+            index = pem_span.end
+            continue
+
+        run_end, total_chars, concatenated = _base64_run(lines, index)
+        if (
+            run_end - index >= _BASE64_BLOCK_MIN_LINES
+            and total_chars >= _BASE64_BLOCK_MIN_CHARS
+            and not _is_hex_only(concatenated)
+        ):
+            spans.append(
+                LinePayloadSpan(
+                    start=index,
+                    end=run_end,
+                    kind="base64",
+                    byte_count=_line_span_byte_count(lines, index, run_end),
+                    line_count=run_end - index,
+                )
+            )
+            index = run_end
+            continue
+
+        line = lines[index].strip()
+        if lines[index] == line and _is_base64_token(
+            line, min_length=_BASE64_SINGLE_TOKEN_MIN_LENGTH
+        ):
+            spans.append(
+                LinePayloadSpan(
+                    start=index,
+                    end=index + 1,
+                    kind="base64",
+                    byte_count=byte_len(line),
+                    line_count=1,
+                )
+            )
+        index += 1
+
+    return spans
+
+
+def find_data_url_spans(text: str) -> list[DataUrlSpan]:
+    """Return large data URL spans in ``text``."""
+
+    spans: list[DataUrlSpan] = []
+    index = 0
+    while True:
+        start = text.find("data:", index)
+        if start < 0:
+            break
+
+        span = _parse_data_url(text, start)
+        if span is None:
+            index = start + len("data:")
+            continue
+
+        spans.append(span)
+        index = span.end
+
+    return spans
+
+
+def summarize_json_payload(text: str) -> JsonPayloadSummary | None:
+    """Summarize valid JSON source maps or JSON dominated by opaque strings."""
+
+    body = text.strip()
+    if not (body.startswith("{") and body.endswith("}")):
+        return None
+
+    try:
+        value = json.loads(body, parse_constant=_reject_json_constant)
+    except (json.JSONDecodeError, ValueError):
+        return None
+    if not isinstance(value, dict):
+        return None
+
+    keys = [key for key in value if _SIMPLE_KEY_RE.fullmatch(key)]
+    if not keys:
+        return None
+
+    try:
+        serialized = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+    except (TypeError, ValueError):
+        return None
+
+    serialized_bytes = byte_len(serialized)
+    opaque_bytes = _opaque_string_bytes(value)
+    if (
+        opaque_bytes < _OPAQUE_JSON_STRING_MIN_BYTES
+        or opaque_bytes * 100 < serialized_bytes * _OPAQUE_JSON_MIN_OPAQUE_RATIO_PERCENT
+    ):
+        return None
+
+    kind = "json-source-map" if _is_source_map_signature(value) else "json-opaque"
+    return JsonPayloadSummary(kind=kind, keys=keys, byte_count=byte_len(text))
+
+
+def summarize_html_payload(text: str) -> HtmlPayloadSummary | None:
+    """Summarize a clear, large HTML document body."""
+
+    body_bytes = byte_len(text)
+    lines = text.split("\n")
+    max_line_len = max((len(line) for line in lines), default=0)
+    if (
+        body_bytes < _WHOLE_PAYLOAD_MIN_BYTES
+        and max_line_len < _WHOLE_PAYLOAD_MAX_LINE_MIN_LENGTH
+    ):
+        return None
+
+    stripped = text.strip()
+    lowered = stripped.lower()
+    if not (("<!doctype html" in lowered or "<html" in lowered) and "</html>" in lowered):
+        return None
+    if not (lowered.startswith("<!doctype html") or lowered.startswith("<html")):
+        return None
+    if not lowered.endswith("</html>"):
+        return None
+
+    return HtmlPayloadSummary(byte_count=body_bytes, title=_extract_title(text))
+
+
+def summarize_minified_js_payload(text: str) -> MinifiedJsSummary | None:
+    """Summarize a large, clearly minified JavaScript body."""
+
+    body_bytes = byte_len(text)
+    if body_bytes < _WHOLE_PAYLOAD_MIN_BYTES:
+        return None
+
+    lines = text.splitlines()
+    nonblank_lines = [line for line in lines if line.strip()]
+    max_line_len = max((len(line) for line in nonblank_lines), default=0)
+    if (
+        len(nonblank_lines) > _WHOLE_PAYLOAD_MAX_LINES
+        and max_line_len < _WHOLE_PAYLOAD_MAX_LINE_MIN_LENGTH
+    ):
+        return None
+    if _whitespace_ratio(text) > _MINIFIED_PAYLOAD_MAX_WHITESPACE_RATIO:
+        return None
+
+    stripped = text.strip()
+    lowered = stripped.lower()
+    if not stripped.startswith(
+        ("(()=>", "(function", "!function", "function", "var ", "let ", "const ")
+    ):
+        return None
+    has_anchor = (
+        "function" in lowered
+        or "webpack" in lowered
+        or "//# sourcemappingurl" in lowered
+        or "=>" in text
+        or "(()=>" in text
+        or "!function" in text
+    )
+    if not has_anchor:
+        return None
+
+    return MinifiedJsSummary(byte_count=body_bytes)
+
+
+def _parse_pem_span(lines: list[str], start: int) -> LinePayloadSpan | None:
+    match = _BEGIN_RE.fullmatch(lines[start].strip())
+    if match is None:
+        return None
+
+    label = match.group(1)
+    end_marker = f"-----END {label}-----"
+    cursor = start + 1
+    while cursor < len(lines):
+        if lines[cursor].strip() == end_marker:
+            body_lines = lines[start + 1 : cursor]
+            if _is_pem_body(body_lines):
+                return LinePayloadSpan(
+                    start=start,
+                    end=cursor + 1,
+                    kind="pem",
+                    label=label,
+                    byte_count=_line_span_byte_count(lines, start, cursor + 1),
+                    line_count=cursor + 1 - start,
+                )
+            return None
+        cursor += 1
+
+    return None
+
+
+def _is_pem_body(lines: list[str]) -> bool:
+    nonblank = [line.strip() for line in lines if line.strip()]
+    body_bytes = byte_len("\n".join(lines))
+    if len(nonblank) < _PEM_BODY_MIN_LINES and body_bytes < _PEM_BODY_MIN_BYTES:
+        return False
+
+    payload_lines = 0
+    for line in nonblank:
+        if ":" in line and len(line) <= 80:
+            continue
+        if not _is_base64ish_line(line, min_length=_PEM_BODY_LINE_MIN_LENGTH):
+            return False
+        payload_lines += 1
+
+    return payload_lines > 0
+
+
+def _base64_run(lines: list[str], start: int) -> tuple[int, int, str]:
+    cursor = start
+    total_chars = 0
+    chunks: list[str] = []
+    while cursor < len(lines):
+        line = lines[cursor].strip()
+        if lines[cursor] != line or not _is_base64ish_line(
+            line, min_length=_BASE64_RUN_LINE_MIN_LENGTH, allow_diff_prefix=False
+        ):
+            break
+        chunks.append(line)
+        total_chars += len(line)
+        cursor += 1
+    return cursor, total_chars, "".join(chunks)
+
+
+def _is_base64_token(text: str, *, min_length: int) -> bool:
+    return (
+        len(text) >= min_length
+        and not any(char.isspace() for char in text)
+        and not _looks_like_path_or_diff_line(text)
+        and not _has_structural_payload_chars(text)
+        and _valid_base64_shape(text)
+        and not _is_hex_only(text)
+    )
+
+
+def _is_base64ish_line(
+    text: str, *, min_length: int, allow_diff_prefix: bool = True
+) -> bool:
+    return (
+        len(text) >= min_length
+        and not any(char.isspace() for char in text)
+        and (allow_diff_prefix or not _looks_like_path_or_diff_line(text))
+        and not _has_structural_payload_chars(text)
+        and _valid_base64_shape(text)
+    )
+
+
+def _base64_ratio(text: str) -> float:
+    if not text:
+        return 0.0
+    allowed = sum(1 for char in text if char in _BASE64_ALPHABET)
+    return allowed / len(text)
+
+
+def _is_hex_only(text: str) -> bool:
+    stripped = text.strip()
+    return bool(stripped) and all(char in _HEX_ALPHABET for char in stripped)
+
+
+def _has_structural_payload_chars(text: str) -> bool:
+    return any(char in _STRUCTURAL_PAYLOAD_CHARS for char in text)
+
+
+def _looks_like_path_or_diff_line(text: str) -> bool:
+    if text.startswith(("+", "-")):
+        return True
+    slash_count = text.count("/")
+    if slash_count >= 2 and slash_count * 8 >= len(text):
+        return True
+    if slash_count >= 3 and all(part for part in text.split("/")):
+        return True
+    return False
+
+
+def _valid_base64_shape(text: str) -> bool:
+    if _base64_ratio(text) < _BASE64_VALID_ALPHABET_RATIO:
+        return False
+    has_standard = any(char in "+/" for char in text)
+    has_urlsafe = any(char in "-_" for char in text)
+    if has_standard and has_urlsafe:
+        return False
+    alphabet = _BASE64_URLSAFE_ALPHABET if has_urlsafe else _BASE64_STANDARD_ALPHABET
+    if not all(char in alphabet for char in text):
+        return False
+    if "=" in text.rstrip("="):
+        return False
+    payload = text.rstrip("=")
+    if len(payload) % 4 == 1:
+        return False
+    padded = payload + ("=" * ((4 - len(payload) % 4) % 4))
+    try:
+        if has_urlsafe:
+            base64.urlsafe_b64decode(padded.encode("ascii"))
+        else:
+            base64.b64decode(padded.encode("ascii"), validate=True)
+    except (binascii.Error, ValueError):
+        return False
+    return True
+
+
+def _line_span_byte_count(lines: list[str], start: int, end: int) -> int:
+    return byte_len("\n".join(lines[start:end]))
+
+
+def _parse_data_url(text: str, start: int) -> DataUrlSpan | None:
+    metadata_start = start + len("data:")
+    cursor = metadata_start
+    while cursor < len(text):
+        char = text[cursor]
+        if char == ",":
+            break
+        if (
+            _is_data_url_metadata_delimiter(char)
+            or cursor - metadata_start > _DATA_URL_METADATA_MAX_LENGTH
+        ):
+            return None
+        cursor += 1
+
+    if cursor >= len(text) or text[cursor] != ",":
+        return None
+
+    metadata = text[metadata_start:cursor]
+    parts = metadata.split(";") if metadata else []
+    mime = parts[0].lower() if parts else ""
+    if not _MIME_RE.fullmatch(mime):
+        return None
+
+    encoding = "base64" if any(part.lower() == "base64" for part in parts[1:]) else "urlencoded"
+    payload_start = cursor + 1
+    payload_end = payload_start
+    while payload_end < len(text) and not _is_data_url_payload_delimiter(
+        text[payload_end], encoding=encoding
+    ):
+        payload_end += 1
+
+    payload = text[payload_start:payload_end]
+    if not payload:
+        return None
+
+    byte_count = byte_len(text[start:payload_end])
+    if len(payload) < _DATA_URL_PAYLOAD_MIN_LENGTH and byte_count < _DATA_URL_MIN_BYTES:
+        return None
+    if encoding == "base64" and _base64_ratio(payload) < _DATA_URL_BASE64_ALPHABET_RATIO:
+        return None
+
+    return DataUrlSpan(
+        start=start,
+        end=payload_end,
+        mime=mime,
+        encoding=encoding,
+        byte_count=byte_count,
+    )
+
+
+def _is_data_url_metadata_delimiter(char: str) -> bool:
+    return char.isspace() or char in {'"', "'", "<", ">", ")", "]", "}", "`"}
+
+
+def _is_data_url_payload_delimiter(char: str, *, encoding: str) -> bool:
+    if _is_data_url_metadata_delimiter(char):
+        return True
+    if encoding == "base64":
+        return char not in _BASE64_ALPHABET
+    return False
+
+
+def _reject_json_constant(value: str) -> None:
+    raise ValueError(f"non-standard JSON constant {value}")
+
+
+def _is_source_map_signature(value: dict[str, Any]) -> bool:
+    keys = {key.lower() for key in value}
+    return "version" in keys and "mappings" in keys and (
+        "sources" in keys or "names" in keys or "file" in keys
+    )
+
+
+def _opaque_string_bytes(value: Any, key_hint: str = "") -> int:
+    if isinstance(value, str):
+        if len(value) >= _OPAQUE_JSON_STRING_MIN_BYTES and _is_opaque_string(value, key_hint):
+            return byte_len(value)
+        return 0
+    if isinstance(value, dict):
+        return sum(_opaque_string_bytes(child, str(key)) for key, child in value.items())
+    if isinstance(value, list):
+        return sum(_opaque_string_bytes(child, key_hint) for child in value)
+    return 0
+
+
+def _is_opaque_string(value: str, key_hint: str) -> bool:
+    key = key_hint.lower()
+    return (
+        key == "mappings"
+        or value.startswith("data:")
+        or _is_base64_token(value, min_length=_BASE64_SINGLE_TOKEN_MIN_LENGTH)
+        or _looks_like_minified_js_string(value)
+        or _whitespace_ratio(value) <= _OPAQUE_STRING_MAX_WHITESPACE_RATIO
+    )
+
+
+def _looks_like_minified_js_string(value: str) -> bool:
+    if _whitespace_ratio(value) > _MINIFIED_PAYLOAD_MAX_WHITESPACE_RATIO:
+        return False
+    lowered = value.lower()
+    return (
+        "function" in lowered
+        or "webpack" in lowered
+        or "=>" in value
+        or "(()=>" in value
+        or "!function" in value
+    )
+
+
+def _whitespace_ratio(value: str) -> float:
+    if not value:
+        return 0.0
+    return sum(1 for char in value if char.isspace()) / len(value)
+
+
+def _extract_title(text: str) -> str | None:
+    match = _TITLE_RE.search(text)
+    if match is None:
+        return None
+    title = re.sub(r"\s+", " ", match.group(1)).strip()
+    if not title:
+        return None
+    if len(title) > 80:
+        return f"{title[:77]}..."
+    return title

codetool_shell/filters/opaque_payload/reducer.py

@@ -0,0 +1,142 @@
+"""Reduce opaque blobs and long payloads."""
+
+from __future__ import annotations
+
+from ..text import join_preserving_final_newline, score, split_preserving_final_newline
+from .detector import (
+    LinePayloadSpan,
+    find_data_url_spans,
+    find_line_payload_spans,
+    split_http_response,
+    summarize_html_payload,
+    summarize_json_payload,
+    summarize_minified_js_payload,
+)
+from .summary import (
+    format_base64_summary,
+    format_data_url_summary,
+    format_html_summary,
+    format_json_summary,
+    format_minified_js_summary,
+    format_pem_summary,
+    with_final_newline,
+)
+
+
+def compress_opaque_payload_output(text: str) -> str:
+    """Summarize high-confidence opaque payloads, otherwise return unchanged."""
+
+    best = text
+
+    whole_payload = _summarize_whole_payload(text)
+    if whole_payload is not None:
+        best = _choose_smaller(best, whole_payload)
+
+    span_replacements = _replace_line_payloads(text)
+    span_replacements = _replace_data_urls(span_replacements)
+    best = _choose_smaller(best, span_replacements)
+
+    return best
+
+
+def _summarize_whole_payload(text: str) -> str | None:
+    http_body = split_http_response(text)
+    if http_body is not None:
+        body_summary = _summarize_body(http_body.body)
+        if body_summary is None:
+            return None
+        return f"{http_body.headers}{http_body.separator}{body_summary}"
+
+    return _summarize_body(text)
+
+
+def _summarize_body(body: str) -> str | None:
+    final_newline = body.endswith("\n")
+
+    json_summary = summarize_json_payload(body)
+    if json_summary is not None:
+        return with_final_newline(
+            format_json_summary(
+                kind=json_summary.kind,
+                keys=json_summary.keys,
+                byte_count=json_summary.byte_count,
+            ),
+            final_newline,
+        )
+
+    html_summary = summarize_html_payload(body)
+    if html_summary is not None:
+        return with_final_newline(
+            format_html_summary(
+                byte_count=html_summary.byte_count,
+                title=html_summary.title,
+            ),
+            final_newline,
+        )
+
+    js_summary = summarize_minified_js_payload(body)
+    if js_summary is not None:
+        return with_final_newline(
+            format_minified_js_summary(byte_count=js_summary.byte_count),
+            final_newline,
+        )
+
+    return None
+
+
+def _replace_line_payloads(text: str) -> str:
+    lines, final_newline = split_preserving_final_newline(text)
+    spans = find_line_payload_spans(lines)
+    if not spans:
+        return text
+
+    output: list[str] = []
+    cursor = 0
+    for span in spans:
+        output.extend(lines[cursor : span.start])
+        output.append(_format_line_span(span))
+        cursor = span.end
+    output.extend(lines[cursor:])
+
+    return join_preserving_final_newline(output, final_newline)
+
+
+def _replace_data_urls(text: str) -> str:
+    spans = find_data_url_spans(text)
+    if not spans:
+        return text
+
+    output: list[str] = []
+    cursor = 0
+    for span in spans:
+        output.append(text[cursor : span.start])
+        output.append(
+            format_data_url_summary(
+                mime=span.mime,
+                encoding=span.encoding,
+                byte_count=span.byte_count,
+            )
+        )
+        cursor = span.end
+    output.append(text[cursor:])
+    return "".join(output)
+
+
+def _format_line_span(span: LinePayloadSpan) -> str:
+    if span.kind == "pem":
+        assert span.label is not None
+        return format_pem_summary(
+            label=span.label,
+            line_count=span.line_count,
+            byte_count=span.byte_count,
+        )
+    return format_base64_summary(
+        line_count=span.line_count,
+        byte_count=span.byte_count,
+    )
+
+
+def _choose_smaller(current: str, candidate: str) -> str:
+    if score(candidate) < score(current):
+        return candidate
+    return current