powerailabs-squeeze 0.1.0__tar.gz

powerailabs_squeeze-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+.uv/
+.ruff_cache/
+.pytest_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+.idea/
+.vscode/
+.DS_Store
+*.log

powerailabs_squeeze-0.1.0/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.4
+Name: powerailabs-squeeze
+Version: 0.1.0
+Summary: Compress: shrink verbose context (JSON/logs/prose) 60-90% — reversibly. compress() returns a handle; expand() restores the original.
+Author: Raghav Mishra
+License-Expression: MIT
+Requires-Python: >=3.11
+Requires-Dist: powerailabs-core<0.2,>=0.1
+Description-Content-Type: text/markdown
+
+# powerailabs-squeeze
+
+Shrink verbose context — JSON, logs, prose — without throwing anything away. Compression returns
+a *handle*; the original is always restorable. Content-aware: each type gets a purpose-built,
+deterministic compressor (no LLM).
+
+**80% smaller, 100% reversible.**
+
+![status](https://img.shields.io/badge/status-building-yellow) ![license](https://img.shields.io/badge/license-MIT-blue)
+
+🚧 building (v0) · `pip install powerailabs-squeeze` · `from powerailabs.squeeze import compress`
+
+```python
+from powerailabs.squeeze import compress
+
+small, handle = compress(huge_json, kind="auto")          # detect + route (JSON/logs/prose)
+small, handle = compress(logs, kind="logs", target_tokens=400)   # compress to a budget
+original = handle.expand()                                 # restore on demand, byte-for-byte
+```
+
+**Inbound** — usually you don't call it directly; `contextkit` does, when a block is marked
+`evict="compress"` (`pip install powerailabs-contextkit[squeeze]`). It satisfies core's
+`Compressor` protocol by shape, so contextkit never imports squeeze. Call it directly to shrink a
+single known-huge blob (e.g. a 50k-token tool response) before it ever enters the window.
+Reversibility comes from a content-addressed store that keeps each original keyed by hash —
+structural compressors (JSON folding, log dedup) are deterministic; prose is extractive.
+
+See [`docs/squeeze.md`](../../docs/squeeze.md). *Part of the PowerAI Labs stack — github.com/PowerAI-Labs/powerailabs.*

powerailabs_squeeze-0.1.0/README.md

@@ -0,0 +1,28 @@
+# powerailabs-squeeze
+
+Shrink verbose context — JSON, logs, prose — without throwing anything away. Compression returns
+a *handle*; the original is always restorable. Content-aware: each type gets a purpose-built,
+deterministic compressor (no LLM).
+
+**80% smaller, 100% reversible.**
+
+![status](https://img.shields.io/badge/status-building-yellow) ![license](https://img.shields.io/badge/license-MIT-blue)
+
+🚧 building (v0) · `pip install powerailabs-squeeze` · `from powerailabs.squeeze import compress`
+
+```python
+from powerailabs.squeeze import compress
+
+small, handle = compress(huge_json, kind="auto")          # detect + route (JSON/logs/prose)
+small, handle = compress(logs, kind="logs", target_tokens=400)   # compress to a budget
+original = handle.expand()                                 # restore on demand, byte-for-byte
+```
+
+**Inbound** — usually you don't call it directly; `contextkit` does, when a block is marked
+`evict="compress"` (`pip install powerailabs-contextkit[squeeze]`). It satisfies core's
+`Compressor` protocol by shape, so contextkit never imports squeeze. Call it directly to shrink a
+single known-huge blob (e.g. a 50k-token tool response) before it ever enters the window.
+Reversibility comes from a content-addressed store that keeps each original keyed by hash —
+structural compressors (JSON folding, log dedup) are deterministic; prose is extractive.
+
+See [`docs/squeeze.md`](../../docs/squeeze.md). *Part of the PowerAI Labs stack — github.com/PowerAI-Labs/powerailabs.*

powerailabs_squeeze-0.1.0/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "powerailabs-squeeze"
+version = "0.1.0"
+description = "Compress: shrink verbose context (JSON/logs/prose) 60-90% — reversibly. compress() returns a handle; expand() restores the original."
+requires-python = ">=3.11"
+license = "MIT"
+authors = [{ name = "Raghav Mishra" }]
+readme = "README.md"
+dependencies = ["powerailabs-core>=0.1,<0.2"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/powerailabs"]     # contributes powerailabs/squeeze only — NEVER add src/powerailabs/__init__.py

powerailabs_squeeze-0.1.0/src/powerailabs/squeeze/__init__.py

@@ -0,0 +1,238 @@
+"""powerailabs.squeeze — content-aware, reversible context compression.
+
+Shrink verbose context without throwing anything away: :func:`compress` returns ``(small, handle)``
+and ``handle.expand()`` restores the original on demand. Content is routed by type — JSON, logs,
+and prose each get a purpose-built, deterministic compressor (no LLM). Reversibility is guaranteed
+by a **content-addressed store** (CCR): every original is kept keyed by its hash, deduped across
+calls, so ``expand()`` is always exact no matter how hard we squeeze.
+
+Satisfies ``powerailabs.core.protocols.Compressor`` by shape, so ``contextkit`` uses it for
+``Block(evict="compress")`` via the ``contextkit[squeeze]`` extra — without importing this package.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+from powerailabs.core import tokens
+
+__all__ = ["compress", "decompress", "detect", "Handle", "SqueezeCompressor"]
+
+# Content-addressed store: sha256(original) -> original. Deduped; the basis of reversibility.
+_STORE: dict[str, str] = {}
+
+
+@dataclass
+class Handle:
+    """Restore handle for a compression. ``expand()`` returns the exact original. (docs §5)"""
+
+    id: str
+    kind: str
+    original_ref: str  # CCR key into the local content store
+    restore_map: dict = field(default_factory=dict)
+
+    def expand(self) -> str:
+        """Return the original content, byte-for-byte."""
+        return _STORE[self.original_ref]
+
+
+def _store(original: str) -> str:
+    key = hashlib.sha256(original.encode("utf-8")).hexdigest()
+    _STORE.setdefault(key, original)
+    return key
+
+
+# --------------------------------------------------------------------------- detection
+
+_TS = re.compile(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b")
+_UUID = re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b")
+_LEVEL = re.compile(r"\b(?:DEBUG|INFO|WARN|WARNING|ERROR|CRITICAL|TRACE|FATAL)\b")
+
+
+def detect(content: str) -> str:
+    """Detect the content kind: ``"json"`` | ``"logs"`` | ``"prose"``. docs/squeeze.md §4."""
+    s = content.strip()
+    if not s:
+        return "prose"
+    if s[0] in "{[":
+        try:
+            json.loads(s)
+            return "json"
+        except (ValueError, TypeError):
+            pass
+    if _looks_like_logs(s):
+        return "logs"
+    return "prose"
+
+
+def _looks_like_logs(s: str) -> bool:
+    lines = [ln for ln in s.splitlines() if ln.strip()]
+    if len(lines) < 3:
+        return False
+    hits = sum(1 for ln in lines if _TS.search(ln) or _LEVEL.search(ln))
+    return hits >= len(lines) * 0.5
+
+
+# --------------------------------------------------------------------------- public API
+
+
+def compress(
+    content: Any,
+    kind: str = "auto",
+    target_tokens: int | None = None,
+    model: str = "gpt-4o",
+) -> tuple[str, Handle]:
+    """Compress ``content`` and return ``(small, handle)``. ``handle.expand()`` restores it.
+
+    Args:
+        content: A string, or a JSON-serializable object (dict/list).
+        kind: ``"auto"`` (detect) or one of ``"json"`` | ``"logs"`` | ``"prose"``.
+        target_tokens: If given, compress *to* this budget (best effort, never exceeds it).
+        model: Model id used for token counting.
+    """
+    if isinstance(content, str):
+        original = content
+    else:
+        original = json.dumps(content, ensure_ascii=False, separators=(",", ":"))
+        if kind == "auto":
+            kind = "json"
+
+    if kind == "auto":
+        kind = detect(original)
+
+    if kind == "json":
+        small, restore_map = _compress_json(original, target_tokens, model)
+    elif kind == "logs":
+        small, restore_map = _compress_logs(original, target_tokens, model)
+    else:
+        small, restore_map = _compress_prose(original, target_tokens, model)
+
+    handle = Handle(
+        id=uuid.uuid4().hex,
+        kind=kind,
+        original_ref=_store(original),
+        restore_map=restore_map,
+    )
+    return small, handle
+
+
+def decompress(handle: Handle) -> str:
+    """Restore the original content for a handle (same as ``handle.expand()``)."""
+    return handle.expand()
+
+
+class SqueezeCompressor:
+    """Object form satisfying ``core.protocols.Compressor`` (delegates to :func:`compress`)."""
+
+    def compress(
+        self,
+        content: Any,
+        *,
+        target_tokens: int | None = None,
+        model: str | None = None,
+        kind: str = "auto",
+    ) -> tuple[str, Handle]:
+        return compress(content, kind=kind, target_tokens=target_tokens, model=model or "gpt-4o")
+
+
+# --------------------------------------------------------------------------- compressors
+
+
+def _strip_nulls(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        return {k: _strip_nulls(v) for k, v in obj.items() if v is not None}
+    if isinstance(obj, list):
+        return [_strip_nulls(v) for v in obj]
+    return obj
+
+
+def _compress_json(text: str, target_tokens: int | None, model: str) -> tuple[str, dict]:
+    """Lossless-ish: minify whitespace and drop null-valued keys. Original kept in the CCR store."""
+    try:
+        obj = json.loads(text)
+    except (ValueError, TypeError):
+        return _compress_prose(text, target_tokens, model)
+    small = json.dumps(_strip_nulls(obj), ensure_ascii=False, separators=(",", ":"))
+    if target_tokens is not None and tokens.count(small, model) > target_tokens:
+        small = _truncate_to_tokens(small, target_tokens, model)
+    return small, {"technique": "minify+dropnulls"}
+
+
+def _compress_logs(text: str, target_tokens: int | None, model: str) -> tuple[str, dict]:
+    """Normalize volatile fields (timestamps/UUIDs) then dedup repeated lines into ``(×N)``."""
+    counts: dict[str, int] = {}
+    order: list[str] = []
+    for line in text.splitlines():
+        norm = _UUID.sub("<uuid>", _TS.sub("<ts>", line))
+        if norm not in counts:
+            order.append(norm)
+        counts[norm] = counts.get(norm, 0) + 1
+
+    if target_tokens is not None:
+        order.sort(key=lambda ln: counts[ln], reverse=True)  # keep the noisiest patterns first
+
+    out: list[str] = []
+    for norm in order:
+        rendered = f"{norm} (×{counts[norm]})" if counts[norm] > 1 else norm
+        if target_tokens is not None:
+            candidate = "\n".join([*out, rendered])
+            if tokens.count(candidate, model) > target_tokens:
+                break
+        out.append(rendered)
+    return "\n".join(out), {"technique": "normalize+dedup", "patterns": len(order)}
+
+
+_SENT = re.compile(r"(?<=[.!?])\s+")
+_STOP = frozenset(
+    "the a an and or but of to in on for with is are was were be been it this that as at by".split()
+)
+
+
+def _compress_prose(text: str, target_tokens: int | None, model: str) -> tuple[str, dict]:
+    """Extractive: rank sentences by keyword density, keep the top ones in original order."""
+    sentences = [s for s in _SENT.split(text.strip()) if s.strip()]
+    if len(sentences) <= 1:
+        return text, {"technique": "extractive", "kept": len(sentences)}
+
+    freq: dict[str, int] = {}
+    for word in re.findall(r"[a-zA-Z']+", text.lower()):
+        if word not in _STOP:
+            freq[word] = freq.get(word, 0) + 1
+
+    def score(sentence: str) -> float:
+        words = re.findall(r"[a-zA-Z']+", sentence.lower())
+        if not words:
+            return 0.0
+        return sum(freq.get(w, 0) for w in words) / len(words)
+
+    ranked = sorted(range(len(sentences)), key=lambda i: score(sentences[i]), reverse=True)
+
+    if target_tokens is not None:
+        keep: set[int] = set()
+        for i in ranked:
+            trial = " ".join(sentences[j] for j in sorted(keep | {i}))
+            if tokens.count(trial, model) > target_tokens and keep:
+                break
+            keep.add(i)
+    else:
+        keep = set(ranked[: max(1, len(sentences) // 2)])  # default: keep the top half
+
+    small = " ".join(sentences[i] for i in sorted(keep))
+    return small, {"technique": "extractive", "kept": len(keep), "of": len(sentences)}
+
+
+def _truncate_to_tokens(text: str, target: int, model: str) -> str:
+    if target <= 0:
+        return ""
+    if tokens.count(text, model) <= target:
+        return text
+    ratio = max(1, len(text)) / max(1, tokens.count(text, model))
+    cut = text[: int(target * ratio)]
+    while cut and tokens.count(cut, model) > target:
+        cut = cut[: int(len(cut) * 0.9)]
+    return cut

powerailabs_squeeze-0.1.0/tests/test_squeeze.py

@@ -0,0 +1,96 @@
+"""Compression is content-aware, deterministic, and 100% reversible. No network."""
+
+import json
+
+import pytest
+from powerailabs.core import protocols, tokens
+from powerailabs.squeeze import SqueezeCompressor, compress, decompress, detect
+
+
+@pytest.fixture(autouse=True)
+def _heuristic_tokens(monkeypatch):
+    monkeypatch.setattr(tokens, "_tiktoken_encoding", lambda model: None)
+    yield
+
+
+def test_detect():
+    assert detect('{"a": 1}') == "json"
+    assert detect("[1, 2, 3]") == "json"
+    logs = "\n".join(f"2026-06-01T00:00:0{i} INFO started worker" for i in range(5))
+    assert detect(logs) == "logs"
+    assert detect("The cat sat on the mat. It was a sunny day.") == "prose"
+
+
+def test_json_compression_is_smaller_and_reversible():
+    obj = {"name": "alice", "age": 30, "note": None, "tags": ["x", "y"], "extra": None}
+    pretty = json.dumps(obj, indent=4)
+    small, handle = compress(pretty, kind="auto")
+    assert detect(pretty) == "json"
+    assert len(small) < len(pretty)  # whitespace + nulls gone
+    assert "note" not in small and "null" not in small  # nulls dropped
+    assert handle.expand() == pretty  # exact original restored
+
+
+def test_logs_dedup_collapses_repeats():
+    logs = "\n".join(["2026-06-01T00:00:00Z INFO retry attempt"] * 20)
+    small, handle = compress(logs, kind="logs")
+    assert "(×20)" in small
+    assert tokens.count(small, "gpt-4o") < tokens.count(logs, "gpt-4o")
+    assert handle.expand() == logs
+
+
+def test_prose_extractive_hits_target_tokens():
+    text = (
+        "Refunds are processed within five business days. "
+        "The weather today is mild and pleasant. "
+        "Customers must contact support to request a refund. "
+        "Our office cat is named Mittens. "
+        "Refund eligibility depends on the purchase date."
+    )
+    target = 20
+    small, handle = compress(text, kind="prose", target_tokens=target)
+    assert tokens.count(small, "gpt-4o") <= target
+    assert len(small) < len(text)
+    assert handle.expand() == text  # original always restorable
+
+
+def test_object_input_serialized_and_restored():
+    small, handle = compress({"k": "v", "n": None}, kind="auto")
+    assert "null" not in small
+    restored = json.loads(handle.expand())
+    assert restored == {"k": "v", "n": None}
+
+
+def test_decompress_matches_expand():
+    small, handle = compress("hello world. goodbye world.", kind="prose")
+    assert decompress(handle) == handle.expand()
+
+
+def test_ccr_store_dedupes_identical_originals():
+    from powerailabs.squeeze import _STORE
+
+    before = len(_STORE)
+    _, h1 = compress("identical content here", kind="prose")
+    _, h2 = compress("identical content here", kind="prose")
+    assert h1.original_ref == h2.original_ref  # same hash key
+    assert len(_STORE) == before + 1  # stored once
+
+
+def test_satisfies_core_compressor_protocol():
+    assert isinstance(SqueezeCompressor(), protocols.Compressor)
+    small, handle = SqueezeCompressor().compress("a. b. c. d.", target_tokens=5, model="gpt-4o")
+    assert isinstance(handle.expand(), str)
+
+
+def test_contextkit_compress_routes_through_squeeze():
+    # End-to-end: contextkit discovers squeeze by shape via the [squeeze] extra (both installed).
+    from powerailabs.contextkit import Block, Context
+
+    ctx = Context(budget_tokens=30, model="gpt-4o")
+    ctx.add(Block("keep me", priority=10, role="system"))
+    big = " ".join(f"Sentence number {i} about refunds and billing." for i in range(20))
+    ctx.add(Block(big, priority=1, role="user", evict="compress"))
+    ctx.assemble()
+    decision = next(d for d in ctx.report().decisions if d.role == "user")
+    assert decision.action == "compressed"
+    assert decision.tokens_after < decision.tokens_before