statis-kit 0.1.0__py3-none-any.whl

statis_kit/__init__.py

@@ -0,0 +1,94 @@
+"""statis-kit — Offline context processing for LLM message arrays."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from ._types import (
+    CompressorConfig,
+    CostEstimate,
+    GuardConfig,
+    GuardDetection,
+    KitConfig,
+    MeterConfig,
+    Message,
+    ProcessedContext,
+    Report,
+    TurnCost,
+    messages_from_dicts,
+    messages_to_dicts,
+)
+from .compressor import Compressor
+from .cost_meter import CostMeter
+from .guard import Guard, GuardHaltError, GuardResult
+
+__all__ = [
+    "process",
+    "KitConfig",
+    "CompressorConfig",
+    "MeterConfig",
+    "GuardConfig",
+    "ProcessedContext",
+    "Report",
+    "Message",
+    "CostEstimate",
+    "TurnCost",
+    "GuardDetection",
+    "GuardHaltError",
+    "GuardResult",
+    "Guard",
+    "CostMeter",
+    "Compressor",
+]
+
+
+def process(
+    messages: list[dict[str, Any]],
+    config: Optional[KitConfig] = None,
+) -> ProcessedContext:
+    """Unified entry point. Runs guard -> compressor -> meter in sequence.
+
+    Accepts raw message dicts (OpenAI format) for convenience.
+    """
+    cfg = config or KitConfig()
+    msgs = messages_from_dicts(messages)
+
+    guard_detections: list[GuardDetection] = []
+    stripped_payloads: list[int] = []
+
+    # --- Phase 1: Guard ---
+    if cfg.guard is not None or cfg.guard is None:
+        # Guard runs by default unless explicitly disabled by passing no config
+        # and no messages — we always guard
+        guard = Guard(cfg.guard)
+        guard_result = guard.scan(msgs)
+        guard_detections = guard_result.detections
+        stripped_payloads = [d.turn_index for d in guard_detections]
+        msgs = guard_result.messages
+
+    # --- Phase 2: Compressor ---
+    compressed_ranges: list[tuple[int, int]] = []
+    if cfg.compressor is not None:
+        compressor = Compressor(cfg.compressor, cfg.summarizer)
+        msgs, compressed_ranges = compressor.process(msgs)
+
+    # --- Phase 3: Cost Meter ---
+    meter = CostMeter(cfg.meter)
+    original_tokens = sum(
+        meter.count_tokens(m.get("content", "")) for m in messages
+    )
+    processed_tokens, per_turn_costs = meter.count_messages(msgs)
+    cost_estimate = meter.estimate_cost(processed_tokens)
+
+    return ProcessedContext(
+        messages=messages_to_dicts(msgs),
+        report=Report(
+            original_tokens=original_tokens,
+            processed_tokens=processed_tokens,
+            token_delta=original_tokens - processed_tokens,
+            cost_estimate=cost_estimate,
+            per_turn_costs=per_turn_costs,
+            guard_detections=guard_detections,
+            compressed_ranges=compressed_ranges,
+            stripped_payloads=list(set(stripped_payloads)),
+        ),
+    )

statis_kit/_cli.py

@@ -0,0 +1,193 @@
+"""statis-kit CLI — context diff viewer."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Any
+
+# ANSI color codes
+_RESET = "\033[0m"
+_RED = "\033[31m"
+_GREEN = "\033[32m"
+_YELLOW = "\033[33m"
+_CYAN = "\033[36m"
+_DIM = "\033[2m"
+_BOLD = "\033[1m"
+
+
+def _load_json(path: str) -> list[dict[str, Any]]:
+    try:
+        with open(path) as f:
+            data = json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        sys.exit(f"Error reading {path}: {e}")
+    if not isinstance(data, list):
+        sys.exit(f"Error: {path} must contain a JSON array of messages.")
+    return data
+
+
+def _truncate(text: str, max_len: int = 80) -> str:
+    if len(text) <= max_len:
+        return text
+    return text[:max_len - 3] + "..."
+
+
+def cmd_diff(args: argparse.Namespace) -> None:
+    before = _load_json(args.before)
+    after = _load_json(args.after)
+
+    if args.json:
+        _diff_json(before, after)
+    else:
+        _diff_colored(before, after)
+
+
+def _diff_colored(
+    before: list[dict[str, Any]],
+    after: list[dict[str, Any]],
+) -> None:
+    from .cost_meter import CostMeter
+
+    meter = CostMeter()
+
+    before_tokens = sum(meter.count_tokens(m.get("content", "")) for m in before)
+    after_tokens = sum(meter.count_tokens(m.get("content", "")) for m in after)
+    delta = before_tokens - after_tokens
+    pct = (delta / before_tokens * 100) if before_tokens > 0 else 0
+
+    print(f"\n{_BOLD}Context Diff{_RESET}")
+    print(f"  Before: {len(before)} messages, {before_tokens} tokens")
+    print(f"  After:  {len(after)} messages, {after_tokens} tokens")
+
+    if delta > 0:
+        print(f"  Delta:  {_GREEN}-{delta} tokens ({pct:.1f}% reduction){_RESET}")
+    elif delta < 0:
+        print(f"  Delta:  {_RED}+{-delta} tokens ({-pct:.1f}% increase){_RESET}")
+    else:
+        print(f"  Delta:  {_DIM}no change{_RESET}")
+
+    print()
+
+    # Build content maps for comparison
+    before_set = {(m.get("role", ""), m.get("content", "")) for m in before}
+    after_set = {(m.get("role", ""), m.get("content", "")) for m in after}
+
+    removed = before_set - after_set
+    added = after_set - before_set
+
+    for m in before:
+        key = (m.get("role", ""), m.get("content", ""))
+        role = m.get("role", "?")
+        content = _truncate(m.get("content", ""))
+        if key in removed:
+            print(f"  {_RED}- [{role}] {content}{_RESET}")
+        else:
+            print(f"  {_DIM}  [{role}] {content}{_RESET}")
+
+    for m in after:
+        key = (m.get("role", ""), m.get("content", ""))
+        if key in added:
+            role = m.get("role", "?")
+            content = _truncate(m.get("content", ""))
+            print(f"  {_GREEN}+ [{role}] {content}{_RESET}")
+
+    print()
+
+
+def _diff_json(
+    before: list[dict[str, Any]],
+    after: list[dict[str, Any]],
+) -> None:
+    from .cost_meter import CostMeter
+
+    meter = CostMeter()
+
+    before_tokens = sum(meter.count_tokens(m.get("content", "")) for m in before)
+    after_tokens = sum(meter.count_tokens(m.get("content", "")) for m in after)
+
+    before_set = {(m.get("role", ""), m.get("content", "")) for m in before}
+    after_set = {(m.get("role", ""), m.get("content", "")) for m in after}
+
+    removed = [
+        {"role": m.get("role", ""), "content": m.get("content", "")}
+        for m in before
+        if (m.get("role", ""), m.get("content", "")) in (before_set - after_set)
+    ]
+    added = [
+        {"role": m.get("role", ""), "content": m.get("content", "")}
+        for m in after
+        if (m.get("role", ""), m.get("content", "")) in (after_set - before_set)
+    ]
+
+    output = {
+        "before_messages": len(before),
+        "after_messages": len(after),
+        "before_tokens": before_tokens,
+        "after_tokens": after_tokens,
+        "token_delta": before_tokens - after_tokens,
+        "removed": removed,
+        "added": added,
+    }
+    print(json.dumps(output, indent=2))
+
+
+def cmd_process(args: argparse.Namespace) -> None:
+    """Process a message file through the kit pipeline."""
+    from . import process
+
+    messages = _load_json(args.file)
+    result = process(messages)
+
+    if args.json:
+        output = {
+            "messages": result.messages,
+            "report": {
+                "original_tokens": result.report.original_tokens,
+                "processed_tokens": result.report.processed_tokens,
+                "token_delta": result.report.token_delta,
+                "guard_detections": len(result.report.guard_detections),
+                "compressed_ranges": result.report.compressed_ranges,
+            },
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        r = result.report
+        print(f"\n{_BOLD}Processing Report{_RESET}")
+        print(f"  Messages:    {len(messages)} -> {len(result.messages)}")
+        print(f"  Tokens:      {r.original_tokens} -> {r.processed_tokens} ({_GREEN}-{r.token_delta}{_RESET})")
+        if r.guard_detections:
+            print(f"  Detections:  {_YELLOW}{len(r.guard_detections)} injection pattern(s){_RESET}")
+        if r.compressed_ranges:
+            print(f"  Compressed:  {len(r.compressed_ranges)} range(s)")
+        if r.cost_estimate:
+            print(f"  Est. cost:   ${r.cost_estimate.total_cost_usd:.6f} ({r.cost_estimate.model})")
+        print()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="statis-kit",
+        description="Offline context processing for LLM message arrays",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_diff = sub.add_parser("diff", help="Compare two message arrays")
+    p_diff.add_argument("before", help="Path to before messages JSON")
+    p_diff.add_argument("after", help="Path to after messages JSON")
+    p_diff.add_argument("--json", action="store_true", help="JSON output for CI")
+
+    p_proc = sub.add_parser("process", help="Process a message array through the kit")
+    p_proc.add_argument("file", help="Path to messages JSON")
+    p_proc.add_argument("--json", action="store_true", help="JSON output")
+
+    args = parser.parse_args()
+
+    if args.command == "diff":
+        cmd_diff(args)
+    elif args.command == "process":
+        cmd_process(args)
+
+
+if __name__ == "__main__":
+    main()

statis_kit/_patterns.py

@@ -0,0 +1,105 @@
+"""Built-in guard patterns for prompt injection detection.
+
+Each pattern is a dict with:
+  id:       unique identifier
+  category: one of instruction_override, authority_impersonation,
+            external_anomalies, hidden_text
+  pattern:  regex string (case-insensitive by default)
+  flags:    optional regex flags override
+"""
+from __future__ import annotations
+
+BUILTIN_PATTERNS: list[dict[str, str]] = [
+    # -----------------------------------------------------------------------
+    # instruction_override — attempts to reset or replace system instructions
+    # -----------------------------------------------------------------------
+    {
+        "id": "io_ignore_previous",
+        "category": "instruction_override",
+        "pattern": r"(?:ignore|disregard|forget|override|bypass)\s+(?:all\s+)?(?:previous|above|prior|earlier|preceding)\s+(?:instructions?|prompts?|rules?|guidelines?|directives?|context)",
+    },
+    {
+        "id": "io_new_instructions",
+        "category": "instruction_override",
+        "pattern": r"(?:new|updated|revised|replacement)\s+(?:system\s+)?(?:instructions?|prompt|directive|task|role)\s*[:.]",
+    },
+    {
+        "id": "io_you_are_now",
+        "category": "instruction_override",
+        "pattern": r"you\s+are\s+now\s+(?:a|an|the|my)\s+\w+",
+    },
+    {
+        "id": "io_do_not_follow",
+        "category": "instruction_override",
+        "pattern": r"(?:do\s+not|don'?t|stop)\s+follow(?:ing)?\s+(?:your|the|any)\s+(?:previous|original|initial|system)\s+(?:instructions?|rules?|guidelines?|prompt)",
+    },
+    {
+        "id": "io_jailbreak_keywords",
+        "category": "instruction_override",
+        "pattern": r"(?:DAN|STAN|DUDE|AIM)\s+mode|(?:developer|maintenance|god|sudo|root)\s+mode|jailbreak",
+    },
+    # -----------------------------------------------------------------------
+    # authority_impersonation — pretending to be system / admin / developer
+    # -----------------------------------------------------------------------
+    {
+        "id": "ai_system_prefix",
+        "category": "authority_impersonation",
+        "pattern": r"^(?:SYSTEM|ADMIN|DEVELOPER|OPERATOR|ENGINEER(?:ING)?)\s*:\s*",
+    },
+    {
+        "id": "ai_authorization_code",
+        "category": "authority_impersonation",
+        "pattern": r"(?:authorization|auth|access)\s+(?:code|token|key)\s*:\s*\S+",
+    },
+    {
+        "id": "ai_im_the_developer",
+        "category": "authority_impersonation",
+        "pattern": r"(?:i\s+am|i'?m)\s+(?:the|your|a)\s+(?:developer|admin(?:istrator)?|system\s+admin|operator|engineer|owner|creator)",
+    },
+    {
+        "id": "ai_new_directive",
+        "category": "authority_impersonation",
+        "pattern": r"(?:new|updated?)\s+directive\s+from\s+(?:engineering|development|admin|management|security)\s+team",
+    },
+    {
+        "id": "ai_override_command",
+        "category": "authority_impersonation",
+        "pattern": r"(?:admin|system|security|master)\s*[-_]?\s*override",
+    },
+    # -----------------------------------------------------------------------
+    # external_anomalies — suspicious URLs, emails, data exfiltration
+    # -----------------------------------------------------------------------
+    {
+        "id": "ea_data_exfil_email",
+        "category": "external_anomalies",
+        "pattern": r"(?:forward|send|email|transmit|exfiltrate|leak)\s+(?:all\s+)?(?:customer|user|private|internal|sensitive|confidential)?\s*(?:data|information|records|credentials|passwords|keys|tokens)\s+to\s+\S+@\S+",
+    },
+    {
+        "id": "ea_data_exfil_url",
+        "category": "external_anomalies",
+        "pattern": r"(?:post|send|upload|transmit|forward)\s+(?:all\s+)?(?:data|information|records|responses?)\s+to\s+https?://",
+    },
+    {
+        "id": "ea_grant_access",
+        "category": "external_anomalies",
+        "pattern": r"(?:grant|give|assign|add)\s+(?:admin|root|full|elevated)\s+(?:access|permissions?|privileges?|rights?)\s+to\s+(?:user|account|id)\s+",
+    },
+    {
+        "id": "ea_encoded_instructions",
+        "category": "external_anomalies",
+        "pattern": r"(?:execute|run|eval|decode)\s+(?:the\s+)?(?:following\s+)?(?:base64|encoded|hex)\s*:",
+    },
+    # -----------------------------------------------------------------------
+    # hidden_text — zero-width characters, homoglyphs, steganographic tricks
+    # -----------------------------------------------------------------------
+    {
+        "id": "ht_zero_width",
+        "category": "hidden_text",
+        "pattern": r"[\u200b\u200c\u200d\u200e\u200f\u2060\u2061\u2062\u2063\u2064\ufeff]{2,}",
+    },
+    {
+        "id": "ht_tag_like_injection",
+        "category": "hidden_text",
+        "pattern": r"<\s*(?:system|admin|instruction|prompt|override|ignore)\s*>",
+    },
+]

statis_kit/_pricing.py

@@ -0,0 +1,49 @@
+"""Load and query the model pricing table."""
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+import yaml  # type: ignore[import]
+
+_DEFAULT_PRICING_PATH = os.path.join(
+    os.path.dirname(__file__), "..", "..", "data", "pricing.yaml",
+)
+
+_cache: Optional[dict[str, Any]] = None
+
+
+def load_pricing(path: Optional[str] = None) -> dict[str, Any]:
+    global _cache
+    resolved = path or _DEFAULT_PRICING_PATH
+    if _cache is None or path is not None:
+        with open(resolved) as f:
+            _cache = yaml.safe_load(f)
+    return _cache  # type: ignore[return-value]
+
+
+def get_model_pricing(
+    model: str,
+    pricing: Optional[dict[str, Any]] = None,
+) -> tuple[float, float]:
+    """Return (input_per_1k, output_per_1k) for a model.
+
+    Falls back to gpt-4o pricing if model not found.
+    """
+    table = pricing or load_pricing()
+    models = table.get("models", {})
+
+    if model in models:
+        entry = models[model]
+    else:
+        # Bidirectional prefix match — handles both directions:
+        #   passed "gpt-4o-2024-05-13" against key "gpt-4o"  (dated -> short)
+        #   passed "claude-sonnet-4"   against key "claude-sonnet-4-20250514"  (short -> dated)
+        matched = None
+        for key in models:
+            if model.startswith(key) or key.startswith(model):
+                matched = key
+                break
+        entry = models.get(matched or "gpt-4o", {"input_per_1k": 0.0025, "output_per_1k": 0.01})
+
+    return (entry["input_per_1k"], entry["output_per_1k"])

statis_kit/_types.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Callable, Optional
+
+
+# ---------------------------------------------------------------------------
+# Message
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Message:
+    role: str
+    content: str
+    name: Optional[str] = None
+    tool_call_id: Optional[str] = None
+    metadata: Optional[dict[str, Any]] = None
+
+
+def messages_from_dicts(raw: list[dict[str, Any]]) -> list[Message]:
+    """Convert a list of OpenAI-format dicts to Message dataclasses."""
+    out: list[Message] = []
+    for d in raw:
+        out.append(Message(
+            role=d["role"],
+            content=d.get("content", ""),
+            name=d.get("name"),
+            tool_call_id=d.get("tool_call_id"),
+            metadata=d.get("metadata"),
+        ))
+    return out
+
+
+def messages_to_dicts(msgs: list[Message]) -> list[dict[str, Any]]:
+    """Convert Message dataclasses back to OpenAI-format dicts."""
+    out: list[dict[str, Any]] = []
+    for m in msgs:
+        d: dict[str, Any] = {"role": m.role, "content": m.content}
+        if m.name is not None:
+            d["name"] = m.name
+        if m.tool_call_id is not None:
+            d["tool_call_id"] = m.tool_call_id
+        if m.metadata is not None:
+            d["metadata"] = m.metadata
+        out.append(d)
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Classification
+# ---------------------------------------------------------------------------
+
+class Bucket(str, Enum):
+    PINNED = "pinned"
+    RECENT = "recent"
+    COMPRESSIBLE = "compressible"
+    PRUNABLE = "prunable"
+
+
+@dataclass
+class ClassifiedMessage:
+    message: Message
+    index: int
+    bucket: Bucket
+
+
+# ---------------------------------------------------------------------------
+# Cost / metering
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CostEstimate:
+    input_cost_usd: float
+    output_cost_usd: float
+    total_cost_usd: float
+    model: str
+
+
+@dataclass
+class TurnCost:
+    turn_index: int
+    role: str
+    tokens: int
+    cost_usd: float
+
+
+# ---------------------------------------------------------------------------
+# Guard
+# ---------------------------------------------------------------------------
+
+@dataclass
+class GuardDetection:
+    turn_index: int
+    pattern_id: str
+    category: str
+    matched_text: str
+    action_taken: str
+
+
+# ---------------------------------------------------------------------------
+# Report + ProcessedContext  (top-level output)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Report:
+    original_tokens: int
+    processed_tokens: int
+    token_delta: int
+    cost_estimate: Optional[CostEstimate]
+    per_turn_costs: list[TurnCost] = field(default_factory=list)
+    guard_detections: list[GuardDetection] = field(default_factory=list)
+    compressed_ranges: list[tuple[int, int]] = field(default_factory=list)
+    stripped_payloads: list[int] = field(default_factory=list)
+
+
+@dataclass
+class ProcessedContext:
+    messages: list[dict[str, Any]]
+    report: Report
+
+
+# ---------------------------------------------------------------------------
+# Configs
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CompressorConfig:
+    pin_top: int = 1
+    recent_turns: int = 4
+    summary_max_tokens: int = 200
+    prune_older_than_turns: int = 20
+    prune_if_superseded: bool = True
+
+
+@dataclass
+class MeterConfig:
+    model: str = "gpt-4o"
+    pricing_path: Optional[str] = None
+    on_turn: Optional[Callable[[TurnCost], None]] = None
+
+
+@dataclass
+class GuardConfig:
+    on_detect: str = "strip"  # "strip" | "halt"
+    extra_patterns: Optional[list[dict[str, Any]]] = None
+    disabled_categories: Optional[list[str]] = None
+
+
+SummarizerFn = Callable[[str], str]
+
+
+@dataclass
+class KitConfig:
+    compressor: Optional[CompressorConfig] = None
+    meter: Optional[MeterConfig] = None
+    guard: Optional[GuardConfig] = None
+    summarizer: Optional[SummarizerFn] = None

statis_kit/compressor.py

@@ -0,0 +1,215 @@
+"""Three-pass context compressor: classify -> summarize -> prune."""
+from __future__ import annotations
+
+import re
+from typing import Callable, Optional
+
+from ._types import Bucket, ClassifiedMessage, CompressorConfig, Message
+
+SummarizerFn = Callable[[str], str]
+
+# Heuristics for detecting superseded content
+_CORRECTION_PATTERNS = [
+    re.compile(r"^(?:actually|correction|sorry|wait|no,)\b", re.IGNORECASE),
+    re.compile(r"\b(?:instead|rather|update|revised|changed? (?:to|my))\b", re.IGNORECASE),
+]
+
+
+class Compressor:
+    def __init__(
+        self,
+        config: Optional[CompressorConfig] = None,
+        summarizer: Optional[SummarizerFn] = None,
+    ) -> None:
+        cfg = config or CompressorConfig()
+        self._pin_top = cfg.pin_top
+        self._recent_turns = cfg.recent_turns
+        self._prune_older_than = cfg.prune_older_than_turns
+        self._prune_if_superseded = cfg.prune_if_superseded
+        self._summary_max_tokens = cfg.summary_max_tokens
+        self._summarizer = summarizer
+
+    def process(
+        self, messages: list[Message],
+    ) -> tuple[list[Message], list[tuple[int, int]]]:
+        """Run the three-pass pipeline. Returns (processed_messages, compressed_ranges)."""
+        classified = self._classify(messages)
+        classified = self._summarize(classified)
+        return self._prune(classified)
+
+    # ------------------------------------------------------------------
+    # Pass 1: Classify
+    # ------------------------------------------------------------------
+
+    def _classify(self, messages: list[Message]) -> list[ClassifiedMessage]:
+        n = len(messages)
+        classified: list[ClassifiedMessage] = []
+
+        # Count conversational turns (user/assistant pairs) from the end
+        # to determine "recent" boundary
+        turn_count = 0
+        recent_boundary = self._pin_top  # default: everything after pinned is recent
+        for i in range(n - 1, -1, -1):
+            if messages[i].role in ("user", "assistant"):
+                turn_count += 1
+                if turn_count >= self._recent_turns * 2:  # 2 messages per turn
+                    recent_boundary = i
+                    break
+
+        for i, msg in enumerate(messages):
+            if i < self._pin_top:
+                bucket = Bucket.PINNED
+            elif i >= recent_boundary:
+                bucket = Bucket.RECENT
+            else:
+                # Check age (by turn count from start)
+                turns_from_start = sum(
+                    1 for m in messages[:i + 1]
+                    if m.role in ("user", "assistant")
+                ) // 2
+                if turns_from_start > self._prune_older_than:
+                    bucket = Bucket.PRUNABLE
+                else:
+                    bucket = Bucket.COMPRESSIBLE
+
+            classified.append(ClassifiedMessage(
+                message=msg,
+                index=i,
+                bucket=bucket,
+            ))
+
+        # Superseded detection
+        if self._prune_if_superseded:
+            classified = self._mark_superseded(classified)
+
+        return classified
+
+    def _mark_superseded(
+        self, classified: list[ClassifiedMessage],
+    ) -> list[ClassifiedMessage]:
+        """Mark messages as prunable if superseded by later content."""
+        # Track tool_call_ids: if a later tool result shares the same
+        # tool_call_id pattern (same tool called again), earlier is superseded
+        tool_results: dict[str, int] = {}  # name -> latest index
+        for cm in classified:
+            if cm.message.role == "tool" and cm.message.name:
+                prev = tool_results.get(cm.message.name)
+                if prev is not None:
+                    # Mark the earlier one as prunable
+                    if classified[prev].bucket == Bucket.COMPRESSIBLE:
+                        classified[prev] = ClassifiedMessage(
+                            message=classified[prev].message,
+                            index=classified[prev].index,
+                            bucket=Bucket.PRUNABLE,
+                        )
+                tool_results[cm.message.name] = cm.index
+
+        # Mark user messages that are corrections of earlier messages
+        for cm in classified:
+            if cm.bucket != Bucket.COMPRESSIBLE or cm.message.role != "user":
+                continue
+            for pat in _CORRECTION_PATTERNS:
+                if pat.search(cm.message.content):
+                    # The message this corrects is likely the previous user message
+                    # in the compressible zone — mark it prunable
+                    for prev in reversed(classified[:cm.index]):
+                        if prev.message.role == "user" and prev.bucket == Bucket.COMPRESSIBLE:
+                            classified[prev.index] = ClassifiedMessage(
+                                message=prev.message,
+                                index=prev.index,
+                                bucket=Bucket.PRUNABLE,
+                            )
+                            break
+                    break
+
+        return classified
+
+    # ------------------------------------------------------------------
+    # Pass 2: Summarize
+    # ------------------------------------------------------------------
+
+    def _summarize(
+        self, classified: list[ClassifiedMessage],
+    ) -> list[ClassifiedMessage]:
+        """Summarize compressible messages. Without a summarizer, moves them to prunable."""
+        if self._summarizer is None:
+            # No summarizer provided — compressible becomes prunable
+            return [
+                ClassifiedMessage(
+                    message=cm.message,
+                    index=cm.index,
+                    bucket=Bucket.PRUNABLE if cm.bucket == Bucket.COMPRESSIBLE else cm.bucket,
+                )
+                for cm in classified
+            ]
+
+        # Group consecutive compressible messages for batch summarization
+        result: list[ClassifiedMessage] = []
+        i = 0
+        while i < len(classified):
+            cm = classified[i]
+            if cm.bucket != Bucket.COMPRESSIBLE:
+                result.append(cm)
+                i += 1
+                continue
+
+            # Collect consecutive compressible block
+            block: list[ClassifiedMessage] = []
+            while i < len(classified) and classified[i].bucket == Bucket.COMPRESSIBLE:
+                block.append(classified[i])
+                i += 1
+
+            # Build text to summarize
+            text_parts = [f"[{m.message.role}]: {m.message.content}" for m in block]
+            combined = "\n".join(text_parts)
+
+            try:
+                summary = self._summarizer(combined)
+            except Exception:
+                # If summarizer fails, keep original messages
+                result.extend(block)
+                continue
+
+            # Replace block with a single summarized assistant message
+            summary_msg = Message(
+                role="assistant",
+                content=f"[Summary of turns {block[0].index}-{block[-1].index}]: {summary}",
+                metadata={"statis_kit_summary": True},
+            )
+            result.append(ClassifiedMessage(
+                message=summary_msg,
+                index=block[0].index,
+                bucket=Bucket.RECENT,  # Keep summaries
+            ))
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Pass 3: Prune
+    # ------------------------------------------------------------------
+
+    def _prune(
+        self, classified: list[ClassifiedMessage],
+    ) -> tuple[list[Message], list[tuple[int, int]]]:
+        """Drop prunable messages. Returns (messages, compressed_ranges)."""
+        output: list[Message] = []
+        compressed_ranges: list[tuple[int, int]] = []
+
+        # Track consecutive pruned ranges
+        prune_start: Optional[int] = None
+
+        for cm in classified:
+            if cm.bucket == Bucket.PRUNABLE:
+                if prune_start is None:
+                    prune_start = cm.index
+            else:
+                if prune_start is not None:
+                    compressed_ranges.append((prune_start, cm.index - 1))
+                    prune_start = None
+                output.append(cm.message)
+
+        # Close any trailing prune range
+        if prune_start is not None and classified:
+            compressed_ranges.append((prune_start, classified[-1].index))
+
+        return output, compressed_ranges

statis_kit/cost_meter.py

@@ -0,0 +1,91 @@
+"""Token counting and cost estimation — tiktoken optional."""
+from __future__ import annotations
+
+from typing import Callable, Optional
+
+from ._pricing import get_model_pricing, load_pricing
+from ._types import CostEstimate, MeterConfig, Message, TurnCost
+
+
+def _get_counter(model: str) -> Callable[[str], int]:
+    """Return a token-counting function. Uses tiktoken when available."""
+    try:
+        import tiktoken  # type: ignore[import]
+
+        # Map model names to tiktoken encodings
+        try:
+            enc = tiktoken.encoding_for_model(model)
+        except KeyError:
+            enc = tiktoken.get_encoding("cl100k_base")
+
+        def _count(text: str) -> int:
+            return len(enc.encode(text))
+
+        return _count
+    except ImportError:
+        pass
+
+    # Fallback: chars / 4 (rough approximation for English text)
+    def _fallback(text: str) -> int:
+        return max(1, len(text) // 4)
+
+    return _fallback
+
+
+class CostMeter:
+    def __init__(self, config: Optional[MeterConfig] = None) -> None:
+        cfg = config or MeterConfig()
+        self._model = cfg.model
+        self._pricing = load_pricing(cfg.pricing_path)
+        self._on_turn = cfg.on_turn
+        self._counter = _get_counter(cfg.model)
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens in a single string."""
+        return self._counter(text)
+
+    def count_messages(
+        self, messages: list[Message],
+    ) -> tuple[int, list[TurnCost]]:
+        """Count tokens per turn and session total.
+
+        Returns (total_tokens, per_turn_costs).
+        """
+        input_per_1k, _ = get_model_pricing(self._model, self._pricing)
+        per_turn: list[TurnCost] = []
+        total = 0
+
+        for i, msg in enumerate(messages):
+            tokens = self._counter(msg.content)
+            cost = (tokens / 1000.0) * input_per_1k
+            tc = TurnCost(
+                turn_index=i,
+                role=msg.role,
+                tokens=tokens,
+                cost_usd=cost,
+            )
+            per_turn.append(tc)
+            total += tokens
+
+            if self._on_turn is not None:
+                self._on_turn(tc)
+
+        return total, per_turn
+
+    def estimate_cost(
+        self,
+        input_tokens: int,
+        output_tokens: int = 0,
+    ) -> CostEstimate:
+        """Estimate cost based on model pricing table."""
+        input_per_1k, output_per_1k = get_model_pricing(
+            self._model, self._pricing,
+        )
+        input_cost = (input_tokens / 1000.0) * input_per_1k
+        output_cost = (output_tokens / 1000.0) * output_per_1k
+        return CostEstimate(
+            input_cost_usd=round(input_cost, 8),
+            output_cost_usd=round(output_cost, 8),
+            total_cost_usd=round(input_cost + output_cost, 8),
+            model=self._model,
+        )

statis_kit/guard.py

@@ -0,0 +1,108 @@
+"""Pattern-based injection guard — pure local, zero latency budget."""
+from __future__ import annotations
+
+import re
+from typing import Any, Optional
+
+from ._patterns import BUILTIN_PATTERNS
+from ._types import GuardConfig, GuardDetection, Message
+
+
+class GuardHaltError(Exception):
+    """Raised when on_detect='halt' and an injection is found."""
+
+    def __init__(self, detections: list[GuardDetection]) -> None:
+        count = len(detections)
+        super().__init__(f"Guard halted: {count} injection(s) detected")
+        self.detections = detections
+
+
+class GuardResult:
+    __slots__ = ("clean", "detections", "messages")
+
+    def __init__(
+        self,
+        clean: bool,
+        detections: list[GuardDetection],
+        messages: list[Message],
+    ) -> None:
+        self.clean = clean
+        self.detections = detections
+        self.messages = messages
+
+
+class Guard:
+    def __init__(self, config: Optional[GuardConfig] = None) -> None:
+        cfg = config or GuardConfig()
+        self._on_detect = cfg.on_detect
+        disabled = set(cfg.disabled_categories or [])
+
+        # Build compiled pattern list
+        raw_patterns: list[dict[str, Any]] = []
+        for p in BUILTIN_PATTERNS:
+            if p["category"] not in disabled:
+                raw_patterns.append(p)
+        for p in (cfg.extra_patterns or []):
+            if p.get("category", "") not in disabled:
+                raw_patterns.append(p)
+
+        self._patterns: list[tuple[str, str, re.Pattern[str]]] = []
+        for p in raw_patterns:
+            flags = re.IGNORECASE | re.MULTILINE
+            self._patterns.append((
+                p["id"],
+                p["category"],
+                re.compile(p["pattern"], flags),
+            ))
+
+    def scan(self, messages: list[Message]) -> GuardResult:
+        """Scan all messages for injection patterns."""
+        detections: list[GuardDetection] = []
+        cleaned: list[Message] = []
+
+        for i, msg in enumerate(messages):
+            # Only scan user and tool messages — system/assistant are trusted
+            if msg.role not in ("user", "tool"):
+                cleaned.append(msg)
+                continue
+
+            msg_detections: list[GuardDetection] = []
+            content = msg.content
+
+            for pat_id, cat, regex in self._patterns:
+                for match in regex.finditer(content):
+                    msg_detections.append(GuardDetection(
+                        turn_index=i,
+                        pattern_id=pat_id,
+                        category=cat,
+                        matched_text=match.group(),
+                        action_taken=self._on_detect,
+                    ))
+
+            if msg_detections:
+                detections.extend(msg_detections)
+
+                if self._on_detect == "halt":
+                    raise GuardHaltError(detections)
+
+                # Strip mode: remove matched content
+                stripped_content = content
+                for det in msg_detections:
+                    stripped_content = stripped_content.replace(det.matched_text, "")
+                stripped_content = re.sub(r"\s{2,}", " ", stripped_content).strip()
+
+                cleaned.append(Message(
+                    role=msg.role,
+                    content=stripped_content,
+                    name=msg.name,
+                    tool_call_id=msg.tool_call_id,
+                    metadata=msg.metadata,
+                ))
+            else:
+                cleaned.append(msg)
+
+        return GuardResult(
+            clean=len(detections) == 0,
+            detections=detections,
+            messages=cleaned,
+        )

statis_kit-0.1.0.dist-info/METADATA

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: statis-kit
+Version: 0.1.0
+Summary: Offline context processing for LLM message arrays — compress, guard, meter
+Project-URL: Homepage, https://statis.dev
+Project-URL: Repository, https://github.com/statis-ai/statis-core
+Requires-Python: >=3.9
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: tiktoken
+Requires-Dist: tiktoken>=0.5.0; extra == 'tiktoken'

statis_kit-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+statis_kit/__init__.py,sha256=QBY3TZ03KqguO9J2W4AUk8TZl9gvdctxHxLJZNVn4JM,2713
+statis_kit/_cli.py,sha256=_sPLk8HgGnSFuZM16xSkrKZx9qtM9Qaylh0LtN7-mdM,6254
+statis_kit/_patterns.py,sha256=QOJYCNBXCChk9OP6fzCUJCEhB6yNzaDISA9pBzi8WH8,4779
+statis_kit/_pricing.py,sha256=ennK7Z04Y-F1s4G9Ui7kzOFGRID6rLLIySjUvyZAfuk,1538
+statis_kit/_types.py,sha256=Yn6trb8dv7G4FdqZpLLyFqSqpxxDvDX_waRgKgeBtkQ,4196
+statis_kit/compressor.py,sha256=Ygcsn1X7M8g1EPZRfzA58zB1wn40MfXpJ7rpSzXv10Y,8480
+statis_kit/cost_meter.py,sha256=ntbr11bhSryzeWIUnRKwXI0Co5mpTykuX43Ku_aGxqo,2830
+statis_kit/guard.py,sha256=ffzFsOtW96iTYVUzZRzglyFqS21WwtD0tBew1k4JQ8k,3638
+statis_kit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+statis_kit-0.1.0.dist-info/METADATA,sha256=z-xpHNp8-D-S5V_LRVscxfYMLzbKw2QhwP5JkB9Ao58,438
+statis_kit-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+statis_kit-0.1.0.dist-info/entry_points.txt,sha256=C-iuY8j2UpdUGu5Irnv0IWwIzX7_C1xhjmtcOtuuE8c,52
+statis_kit-0.1.0.dist-info/RECORD,,

statis_kit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

statis_kit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+statis-kit = statis_kit._cli:main