python-token-killer 0.1.0__py3-none-any.whl

ptk/__init__.py

@@ -0,0 +1,166 @@
+"""ptk — Python Token Killer.
+
+Minimize LLM tokens from Python objects in one call.
+
+    import ptk
+    ptk.minimize({"users": [{"name": "Alice", "age": 30}]})
+    ptk(some_dict)  # shorthand
+
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ptk._base import MinResult, _serialize
+from ptk._types import ContentType, detect
+from ptk.minimizers import (
+    CodeMinimizer,
+    DictMinimizer,
+    DiffMinimizer,
+    ListMinimizer,
+    LogMinimizer,
+    TextMinimizer,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "minimize",
+    "stats",
+    "detect_type",
+    "MinResult",
+    "ContentType",
+    # minimizer classes (for direct use / subclassing)
+    "DictMinimizer",
+    "ListMinimizer",
+    "CodeMinimizer",
+    "LogMinimizer",
+    "DiffMinimizer",
+    "TextMinimizer",
+]
+
+# ── singleton minimizer instances (created once, reused) ────────────────
+
+_ROUTER: dict[ContentType, Any] = {
+    ContentType.DICT: DictMinimizer(),
+    ContentType.LIST: ListMinimizer(),
+    ContentType.CODE: CodeMinimizer(),
+    ContentType.LOG: LogMinimizer(),
+    ContentType.DIFF: DiffMinimizer(),
+    ContentType.TEXT: TextMinimizer(),
+}
+
+
+# ── public API ──────────────────────────────────────────────────────────
+
+
+def minimize(
+    obj: Any,
+    *,
+    aggressive: bool = False,
+    content_type: ContentType | str | None = None,
+    **kw: Any,
+) -> str:
+    """Minimize tokens from any Python object.
+
+    Args:
+        obj: dict, list, str (code/log/diff/text), or anything with __str__.
+        aggressive: Apply maximum compression (may lose some fidelity).
+        content_type: Force a content type instead of auto-detecting.
+            Accepts ContentType enum or string ("dict", "code", "log", etc.)
+        **kw: Forwarded to the minimizer (format, mode, errors_only, etc.)
+
+    Returns:
+        Minimized string representation.
+    """
+    ct = _resolve_type(obj, content_type)
+    result: str = _ROUTER[ct].run(obj, aggressive=aggressive, **kw).output
+    return result
+
+
+def stats(
+    obj: Any,
+    *,
+    aggressive: bool = False,
+    content_type: ContentType | str | None = None,
+    **kw: Any,
+) -> dict[str, Any]:
+    """Return compression statistics without discarding the result.
+
+    Returns:
+        {
+            "output": <minimized string>,
+            "original_len": int,     # character count
+            "minimized_len": int,    # character count
+            "savings_pct": float,    # e.g. 73.2
+            "content_type": str,     # e.g. "dict"
+            "original_tokens": int | None,  # if tiktoken available
+            "minimized_tokens": int | None,
+        }
+    """
+    ct = _resolve_type(obj, content_type)
+    result = _ROUTER[ct].run(obj, aggressive=aggressive, **kw)
+
+    original_str = _serialize(obj)
+    orig_tok, min_tok = _estimate_tokens(original_str, result.output)
+
+    return {
+        "output": result.output,
+        "original_len": result.original_len,
+        "minimized_len": result.minimized_len,
+        "savings_pct": result.savings_pct,
+        "content_type": ct.name.lower(),
+        "original_tokens": orig_tok,
+        "minimized_tokens": min_tok,
+    }
+
+
+def detect_type(obj: Any) -> str:
+    """Return the auto-detected content type as a lowercase string."""
+    return detect(obj).name.lower()
+
+
+# ── callable module trick ───────────────────────────────────────────────
+# Allows `import ptk; ptk(obj)` as shorthand for `ptk.minimize(obj)`.
+
+import sys as _sys  # noqa: E402
+import types as _types  # noqa: E402
+
+
+class _CallableModule(_types.ModuleType):
+    """Module that's also callable — `ptk(obj)` works."""
+
+    def __call__(self, obj: Any, **kw: Any) -> str:
+        return minimize(obj, **kw)
+
+    def __repr__(self) -> str:
+        return f"<module 'ptk' v{__version__}>"
+
+
+# Swap this module's class so `ptk(...)` works
+_self = _sys.modules[__name__]
+_self.__class__ = _CallableModule
+
+
+# ── private helpers ─────────────────────────────────────────────────────
+
+
+def _resolve_type(obj: Any, hint: ContentType | str | None) -> ContentType:
+    if hint is None:
+        return detect(obj)
+    if isinstance(hint, ContentType):
+        return hint
+    # string lookup
+    return ContentType[hint.upper()]
+
+
+def _estimate_tokens(original: str, minimized: str) -> tuple[int | None, int | None]:
+    """Try tiktoken for accurate counts, fall back to len//4 heuristic."""
+    try:
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        return len(enc.encode(original)), len(enc.encode(minimized))
+    except ImportError:
+        # fast heuristic: ~4 chars per token for English
+        return len(original) // 4, len(minimized) // 4

ptk/_base.py

@@ -0,0 +1,137 @@
+"""Base minimizer protocol + shared utilities."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class MinResult:
+    """Immutable result from a minimizer pass."""
+
+    output: str
+    original_len: int
+    minimized_len: int
+
+    @property
+    def savings_pct(self) -> float:
+        if self.original_len == 0:
+            return 0.0
+        return round((1 - self.minimized_len / self.original_len) * 100, 1)
+
+
+class Minimizer(ABC):
+    """Base class all minimizers inherit from.
+
+    Subclasses only need to implement `_minimize`.
+    The public `run()` method handles result wrapping.
+    """
+
+    @abstractmethod
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        """Return the minimized string representation."""
+
+    def run(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> MinResult:
+        original = _serialize(obj)
+        try:
+            minimized = self._minimize(obj, aggressive=aggressive, **kw)
+        except (RecursionError, ValueError, TypeError, OverflowError):
+            # graceful degradation: fall back to str() if minimizer can't handle the input
+            minimized = str(obj)
+        return MinResult(
+            output=minimized,
+            original_len=len(original),
+            minimized_len=len(minimized),
+        )
+
+
+# ── shared helpers (used across minimizers) ─────────────────────────────
+
+
+def _serialize(obj: Any) -> str:
+    """Cheaply serialize an object to string for length measurement.
+
+    Must NEVER raise — this is used for metrics, not output.
+    Handles circular refs, non-serializable types, tuple keys, etc.
+    """
+    if isinstance(obj, str):
+        return obj
+    if isinstance(obj, (dict, list, tuple)):
+        import json
+
+        try:
+            return json.dumps(obj, separators=(",", ":"), default=str)
+        except (ValueError, TypeError, OverflowError):
+            # circular reference, non-string keys, or other json failure
+            return str(obj)
+    return str(obj)
+
+
+def _is_nullish(v: object) -> bool:
+    """Check if a value is 'empty' — None, "", [], or {}.
+
+    Type-checks first to avoid hashing unhashable types.
+    """
+    if v is None:
+        return True
+    if isinstance(v, str):
+        return v == ""
+    if isinstance(v, list):
+        return len(v) == 0
+    if isinstance(v, dict):
+        return len(v) == 0
+    return False
+
+
+def strip_nullish(d: dict[str, Any]) -> dict[str, Any]:
+    """Recursively strip None, empty string, empty list, empty dict values."""
+    out: dict[str, Any] = {}
+    for k, v in d.items():
+        if _is_nullish(v):
+            continue
+        if isinstance(v, dict):
+            cleaned = strip_nullish(v)
+            if cleaned:
+                out[k] = cleaned
+        elif isinstance(v, list):
+            cleaned_list = [
+                strip_nullish(i) if isinstance(i, dict) else i for i in v if not _is_nullish(i)
+            ]
+            if cleaned_list:
+                out[k] = cleaned_list
+        else:
+            out[k] = v
+    return out
+
+
+def dedup_lines(text: str, *, threshold: int = 2) -> str:
+    """Collapse consecutive duplicate lines into `<line> (xN)`.
+
+    Uses a single-pass algorithm — O(n) time, O(1) extra per group.
+    """
+    lines = text.split("\n")
+    if len(lines) <= 1:
+        return text
+
+    result: list[str] = []
+    prev = lines[0]
+    count = 1
+
+    for line in lines[1:]:
+        if line == prev:
+            count += 1
+        else:
+            _flush(result, prev, count, threshold)
+            prev = line
+            count = 1
+    _flush(result, prev, count, threshold)
+    return "\n".join(result)
+
+
+def _flush(result: list[str], line: str, count: int, threshold: int) -> None:
+    if count >= threshold:
+        result.append(f"{line} (x{count})")
+    else:
+        result.extend([line] * count)

ptk/_types.py

@@ -0,0 +1,126 @@
+"""Content type detection — pure builtins, no deps."""
+
+from __future__ import annotations
+
+from enum import Enum, auto
+
+
+class ContentType(Enum):
+    DICT = auto()
+    LIST = auto()
+    CODE = auto()
+    LOG = auto()
+    DIFF = auto()
+    TEXT = auto()
+
+
+# ── fast heuristics (order matters — first match wins) ──────────────────
+
+_CODE_MARKERS = frozenset(
+    {
+        "def ",
+        "class ",
+        "import ",
+        "from ",
+        "function ",
+        "const ",
+        "let ",
+        "var ",
+        "public ",
+        "private ",
+        "async ",
+        "await ",
+        "return ",
+        "if __name__",
+        "#!/",
+        "package ",
+        "func ",
+        "fn ",
+        "impl ",
+        "module ",
+        "export ",
+        "interface ",
+        "struct ",
+    }
+)
+
+_LOG_PATTERNS = frozenset(
+    {
+        # structured log levels
+        "[INFO]",
+        "[WARN]",
+        "[ERROR]",
+        "[DEBUG]",
+        "[TRACE]",
+        " INFO ",
+        " WARN ",
+        " ERROR ",
+        " DEBUG ",
+        " TRACE ",
+        "INFO:",
+        "WARN:",
+        "ERROR:",
+        "DEBUG:",
+        "TRACE:",
+        "WARNING:",
+        "CRITICAL:",
+        # test runner output (pytest, cargo test, go test, jest)
+        "PASSED",
+        "FAILED",
+        "ERRORS",
+        "--- PASS:",
+        "--- FAIL:",  # go test
+        "test result: ",  # cargo test
+        " passed",
+        " failed",  # pytest summary
+        "✓",
+        "✗",
+        "✕",  # jest / vitest
+    }
+)
+
+
+def _looks_like_diff(head: str) -> bool:
+    """Detect unified diff format. Requires @@ hunk header + file headers or diff --git."""
+    if "@@" not in head:
+        return False
+    # strong signal: diff --git header
+    if head.startswith("diff --git") or "\ndiff --git" in head:
+        return True
+    # also accept: --- line followed by +++ line (unified diff without git header)
+    has_minus = head.startswith("--- ") or "\n--- " in head
+    has_plus = "\n+++ " in head
+    return has_minus and has_plus
+
+
+def detect(obj: object) -> ContentType:
+    """Detect content type from a Python object. O(1) for non-str types."""
+    if isinstance(obj, dict):
+        return ContentType.DICT
+    if isinstance(obj, (list, tuple)):
+        return ContentType.LIST
+    if not isinstance(obj, str):
+        # fallback: stringify anything else and treat as text
+        return ContentType.TEXT
+
+    # ── string heuristics (check first ~2KB for speed) ──────────────
+    head = obj[:2048]
+
+    # diff detection — requires real unified diff structure, not just --- or @@
+    if _looks_like_diff(head):
+        return ContentType.DIFF
+
+    # log detection runs BEFORE code — log patterns are more specific.
+    # e.g. pytest output contains 'def test_foo():' which would trigger
+    # code detection, but the PASSED/FAILED markers identify it as log first.
+    if any(m in head for m in _LOG_PATTERNS):
+        return ContentType.LOG
+
+    # code detection — any code keyword at start of a line
+    lines = head.split("\n", 30)  # only check first ~30 lines
+    for line in lines:
+        stripped = line.lstrip()
+        if any(stripped.startswith(k) for k in _CODE_MARKERS):
+            return ContentType.CODE
+
+    return ContentType.TEXT

ptk/minimizers/__init__.py

@@ -0,0 +1,17 @@
+"""Minimizer registry — auto-imports all strategies."""
+
+from ptk.minimizers._code import CodeMinimizer
+from ptk.minimizers._dict import DictMinimizer
+from ptk.minimizers._diff import DiffMinimizer
+from ptk.minimizers._list import ListMinimizer
+from ptk.minimizers._log import LogMinimizer
+from ptk.minimizers._text import TextMinimizer
+
+__all__ = [
+    "DictMinimizer",
+    "ListMinimizer",
+    "CodeMinimizer",
+    "LogMinimizer",
+    "DiffMinimizer",
+    "TextMinimizer",
+]

ptk/minimizers/_code.py

@@ -0,0 +1,156 @@
+"""Code minimizer — comment stripping, whitespace normalization, signature extraction."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ptk._base import Minimizer
+
+# ── precompiled regexes (compiled once at import time) ──────────────────
+
+_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
+# Match strings first (to skip), then comments. Group 1 = string (keep), Group 2 = comment (strip).
+_STRING_OR_COMMENT_C = re.compile(
+    r"""("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')"""  # group 1: quoted string
+    r"|"  # OR
+    r"(//.*$)",  # group 2: C-style inline comment
+    re.MULTILINE,
+)
+_STRING_OR_COMMENT_PY = re.compile(
+    r"""("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')"""  # group 1: quoted string
+    r"|"  # OR
+    r"(#.*$)",  # group 2: Python inline comment
+    re.MULTILINE,
+)
+_DOCSTRING = re.compile(r"(\"\"\"[\s\S]*?\"\"\"|\'\'\'[\s\S]*?\'\'\')")
+_BLANK_LINES = re.compile(r"\n{3,}")
+_TRAILING_WS = re.compile(r"[ \t]+$", re.MULTILINE)
+
+# pragma/directive comments that must be preserved
+_PRAGMA_KEYWORDS: frozenset[str] = frozenset(
+    {
+        "noqa",
+        "type: ignore",
+        "type:ignore",
+        "TODO",
+        "FIXME",
+        "HACK",
+        "XXX",
+        "pragma",
+        "pylint:",
+        "fmt:",
+        "eslint-disable",
+        "eslint-enable",
+        "@ts-ignore",
+        "@ts-expect-error",
+        "noinspection",
+    }
+)
+
+# signature patterns for common languages
+_PY_SIG = re.compile(
+    r"^([ \t]*(?:async\s+)?(?:def|class)\s+\w+.*?:)\s*$",
+    re.MULTILINE,
+)
+_JS_SIG = re.compile(
+    r"^([ \t]*(?:export\s+)?(?:async\s+)?(?:function\s+\w+|(?:const|let|var)\s+\w+\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>)[^{]*)",
+    re.MULTILINE,
+)
+_RUST_SIG = re.compile(
+    r"^([ \t]*(?:pub\s+)?(?:async\s+)?fn\s+\w+[^{]*)",
+    re.MULTILINE,
+)
+_GO_SIG = re.compile(
+    r"^([ \t]*func\s+(?:\([^)]*\)\s+)?\w+[^{]*)",
+    re.MULTILINE,
+)
+
+_SIG_PATTERNS = [_PY_SIG, _JS_SIG, _RUST_SIG, _GO_SIG]
+
+
+class CodeMinimizer(Minimizer):
+    """Compress code by stripping comments, normalizing whitespace, extracting signatures.
+
+    Modes (via `mode` kwarg):
+      - "clean"      (default) — strip comments + normalize whitespace
+      - "signatures"  — extract function/class signatures only (huge savings)
+    """
+
+    def _minimize(self, obj: Any, *, aggressive: bool = False, **kw: Any) -> str:
+        code = obj if isinstance(obj, str) else str(obj)
+        mode = kw.get("mode", "signatures" if aggressive else "clean")
+
+        if mode == "signatures":
+            return _extract_signatures(code)
+        return _clean(code)
+
+
+def _has_pragma(comment: str) -> bool:
+    """Check if a comment contains a pragma/directive that must be preserved."""
+    return any(kw in comment for kw in _PRAGMA_KEYWORDS)
+
+
+def _strip_string_or_comment_c(m: re.Match[str]) -> str:
+    """Handle string-or-comment match: keep strings, strip comments (unless pragma)."""
+    if m.group(1):  # it's a quoted string — keep it
+        return m.group(1)
+    comment = m.group(2)  # it's a // comment
+    return comment if _has_pragma(comment) else ""
+
+
+def _strip_string_or_comment_py(m: re.Match[str]) -> str:
+    """Handle string-or-comment match: keep strings, strip comments (unless pragma)."""
+    if m.group(1):  # it's a quoted string — keep it
+        return m.group(1)
+    comment = m.group(2)  # it's a # comment
+    return comment if _has_pragma(comment) else ""
+
+
+def _strip_block_comment_if_safe(m: re.Match[str]) -> str:
+    """Remove a block comment unless it contains a pragma."""
+    return m.group(0) if _has_pragma(m.group(0)) else ""
+
+
+def _collapse_docstring(m: re.Match[str]) -> str:
+    """Collapse a multi-line docstring to its first line only."""
+    full = m.group(0)
+    # detect the quote style
+    quote = full[:3]
+    inner = full[3:-3].strip()
+    lines = inner.split("\n")
+    first_line = lines[0].strip() if lines else ""
+    if not first_line:
+        return ""
+    # single-line docstrings or summaries — keep as one-liner
+    return f"{quote}{first_line}{quote}"
+
+
+def _clean(code: str) -> str:
+    """Strip comments and normalize whitespace — language-agnostic.
+
+    Preserves pragma comments (noqa, type: ignore, TODO, eslint-disable, etc.)
+    and collapses multi-line docstrings to first line.
+    """
+    out = _BLOCK_COMMENT.sub(_strip_block_comment_if_safe, code)
+    # strip docstrings BEFORE inline comments so triple-quotes are handled first
+    out = _DOCSTRING.sub(_collapse_docstring, out)
+    # use string-aware patterns to avoid stripping // or # inside string literals
+    out = _STRING_OR_COMMENT_C.sub(_strip_string_or_comment_c, out)
+    out = _STRING_OR_COMMENT_PY.sub(_strip_string_or_comment_py, out)
+    out = _TRAILING_WS.sub("", out)
+    out = _BLANK_LINES.sub("\n\n", out)
+    return out.strip()
+
+
+def _extract_signatures(code: str) -> str:
+    """Pull out only function/class signatures — works across Python, JS, Rust, Go."""
+    sigs: list[str] = []
+    for pattern in _SIG_PATTERNS:
+        sigs.extend(m.group(1).strip() for m in pattern.finditer(code))
+
+    if not sigs:
+        # fallback: return cleaned code if no signatures found
+        return _clean(code)
+
+    return "\n".join(sigs)