trimwise 0.1.0__py3-none-any.whl

trimwise/__init__.py

@@ -0,0 +1,19 @@
+"""Expose Trimwise's stable public interface."""
+
+from trimwise.models import (
+    BudgetUnit,
+    SemanticBackendError,
+    Strategy,
+    TrimConfig,
+    TrimResult,
+)
+from trimwise.trimmer import Trimmer
+
+__all__ = [
+    "BudgetUnit",
+    "SemanticBackendError",
+    "Strategy",
+    "TrimConfig",
+    "TrimResult",
+    "Trimmer",
+]

trimwise/measurement.py

@@ -0,0 +1,150 @@
+"""Measure text consistently and derive source-preserving fitting prefixes."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+import tiktoken
+
+from trimwise.models import BudgetUnit
+
+if TYPE_CHECKING:
+    from tiktoken import Encoding
+
+TokenCounter = Callable[[str], int]
+
+
+class Measurer:
+    """Measure text in one configured unit and expose lexical token IDs."""
+
+    def __init__(
+        self,
+        unit: BudgetUnit,
+        encoding_name: str,
+        token_counter: TokenCounter | None,
+    ) -> None:
+        """Store measurement settings without eagerly loading unused encodings.
+
+        Args:
+            unit: Unit used for public budget counts.
+            encoding_name: Tiktoken encoding used for tokens and ranking.
+            token_counter: Optional caller-supplied token counter.
+        """
+        self.unit = unit
+        self._encoding_name = encoding_name
+        self._token_counter = token_counter
+        self._encoding: Encoding | None = None
+
+    def count(self, text: str) -> int:
+        """Measure text and validate custom counter output.
+
+        Args:
+            text: Text to measure.
+
+        Returns:
+            Nonnegative size in the configured unit.
+
+        Raises:
+            ValueError: If a custom counter returns an invalid value.
+        """
+        if self.unit is BudgetUnit.CHARACTERS:
+            return len(text)
+        if self.unit is BudgetUnit.WORDS:
+            return len(text.split())
+        if self._token_counter is None:
+            return len(self._get_encoding().encode(text, disallowed_special=()))
+
+        count = self._token_counter(text)
+        if isinstance(count, bool) or not isinstance(count, int) or count < 0:
+            raise ValueError("token_counter must return a nonnegative integer")
+        return count
+
+    def token_ids(self, text: str) -> list[int]:
+        """Tokenize normalized ranking text with the configured encoding.
+
+        Args:
+            text: Ranking-only text.
+
+        Returns:
+            Tiktoken subword identifiers.
+        """
+        return self._get_encoding().encode(text, disallowed_special=())
+
+    def fitting_prefix(self, text: str, limit: int) -> str:
+        """Return the longest source prefix found within a measurement limit.
+
+        Args:
+            text: Source text whose prefix may be retained.
+            limit: Maximum measured size.
+
+        Returns:
+            A source-preserving prefix, possibly empty.
+        """
+        if limit < 0:
+            return ""
+        if self.count(text) <= limit:
+            return text
+        if self.unit is BudgetUnit.CHARACTERS:
+            return text[:limit]
+        if self.unit is BudgetUnit.WORDS:
+            matches = list(re.finditer(r"\S+(?:\s+|$)", text))
+            return text[: matches[limit - 1].end()] if limit else ""
+        if self._token_counter is None:
+            return self._fitting_encoded_prefix(text, limit)
+        return self._fitting_scanned_prefix(text, limit)
+
+    def _fitting_encoded_prefix(self, text: str, limit: int) -> str:
+        """Fit a source prefix around the configured encoding's token boundary.
+
+        Args:
+            text: Oversized source text.
+            limit: Maximum encoded token count.
+
+        Returns:
+            Longest fitting source prefix around the first excluded token.
+        """
+        encoding = self._get_encoding()
+        tokens = encoding.encode(text, disallowed_special=())
+        decoded, offsets = encoding.decode_with_offsets(tokens)
+        if decoded != text:
+            return self._fitting_scanned_prefix(text, limit)
+
+        boundary = offsets[limit]
+        upper = next(
+            (offset for offset in offsets[limit + 1 :] if offset > boundary),
+            len(text),
+        )
+        fitting_end = 0
+        for end in range(max(0, boundary - 1), upper + 1):
+            if self.count(text[:end]) <= limit:
+                fitting_end = end
+        return text[:fitting_end]
+
+    def _fitting_scanned_prefix(self, text: str, limit: int) -> str:
+        """Find an exact prefix for a counter with no token-offset API.
+
+        Args:
+            text: Oversized source text.
+            limit: Maximum custom token count.
+
+        Returns:
+            Longest fitting source prefix, possibly empty.
+        """
+        # ponytail: custom counters may be non-monotonic; keep their rare fallback exact until
+        # callers can supply a source-offset API.
+        for end in range(len(text) - 1, -1, -1):
+            if self.count(text[:end]) <= limit:
+                return text[:end]
+        return ""
+
+    def _get_encoding(self) -> Encoding:
+        """Load and cache the tiktoken encoding on first lexical use.
+
+        Returns:
+            Configured tiktoken encoding.
+        """
+        if self._encoding is None:
+            self._encoding = tiktoken.get_encoding(self._encoding_name)
+        return self._encoding

trimwise/models.py

@@ -0,0 +1,116 @@
+"""Define Trimwise's public value objects and validation rules."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from enum import Enum
+from types import MappingProxyType
+from typing import Any
+
+
+class Strategy(str, Enum):
+    """Select the evidence-ranking method used during truncation."""
+
+    AUTO = "auto"
+    STRUCTURAL = "structural"
+    LEXICAL = "lexical"
+    SEMANTIC = "semantic"
+    HYBRID = "hybrid"
+
+
+class BudgetUnit(str, Enum):
+    """Define how an input and output budget is measured."""
+
+    TOKENS = "tokens"
+    WORDS = "words"
+    CHARACTERS = "characters"
+
+
+class SemanticBackendError(RuntimeError):
+    """Report an optional semantic backend failure without hiding its cause."""
+
+
+@dataclass(frozen=True, slots=True)
+class TrimConfig:
+    """Configure reusable measurement, ranking, and omission behavior.
+
+    Attributes:
+        token_encoding: Tiktoken encoding used for token budgets and lexical ranking.
+        embedding_model: FastEmbed model used by semantic strategies.
+        fastembed_options: Additional keyword arguments for ``TextEmbedding``.
+        embedding_batch_size: Passage batch size used during inference.
+        mmr_lambda: Balance between relevance and diversity.
+        omission_marker: Text inserted where source content was omitted.
+    """
+
+    token_encoding: str = "o200k_base"
+    embedding_model: str = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+    fastembed_options: Mapping[str, Any] = field(default_factory=dict)
+    embedding_batch_size: int = 256
+    mmr_lambda: float = 0.7
+    omission_marker: str = "[…omitted…]"
+
+    def __post_init__(self) -> None:
+        """Validate configuration and freeze a defensive options copy.
+
+        Raises:
+            TypeError: If FastEmbed options are not a mapping.
+            ValueError: If a configured value is outside its supported domain.
+        """
+        if not isinstance(self.token_encoding, str):
+            raise TypeError("token_encoding must be a string")
+        if not self.token_encoding.strip():
+            raise ValueError("token_encoding must not be blank")
+        if not isinstance(self.embedding_model, str):
+            raise TypeError("embedding_model must be a string")
+        if not self.embedding_model.strip():
+            raise ValueError("embedding_model must not be blank")
+        if not isinstance(self.omission_marker, str):
+            raise TypeError("omission_marker must be a string")
+        if not self.omission_marker.strip():
+            raise ValueError("omission_marker must not be blank")
+        if (
+            isinstance(self.embedding_batch_size, bool)
+            or not isinstance(self.embedding_batch_size, int)
+            or self.embedding_batch_size <= 0
+        ):
+            raise ValueError("embedding_batch_size must be a positive integer")
+        if (
+            isinstance(self.mmr_lambda, bool)
+            or not isinstance(self.mmr_lambda, (int, float))
+            or not 0 <= self.mmr_lambda <= 1
+        ):
+            raise ValueError("mmr_lambda must be between 0 and 1")
+        if not isinstance(self.fastembed_options, Mapping):
+            raise TypeError("fastembed_options must be a mapping")
+
+        options = dict(self.fastembed_options)
+        if any(not isinstance(key, str) for key in options):
+            raise ValueError("fastembed_options keys must be strings")
+        if "model_name" in options:
+            raise ValueError("model_name belongs in embedding_model")
+        object.__setattr__(self, "fastembed_options", MappingProxyType(options))
+
+
+@dataclass(frozen=True, slots=True)
+class TrimResult:
+    """Describe the resolved strategy and measured truncation result.
+
+    Attributes:
+        text: Extractive output that fits the requested limit.
+        input_count: Measured size of the original input.
+        output_count: Measured size of the returned output.
+        limit: Requested maximum output size.
+        unit: Unit used for all counts.
+        strategy: Concrete strategy used after resolving ``auto``.
+        trimmed: Whether the output differs from the input.
+    """
+
+    text: str
+    input_count: int
+    output_count: int
+    limit: int
+    unit: BudgetUnit
+    strategy: Strategy
+    trimmed: bool