agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/algorithms/curate/llmlingua/controlled.py

@@ -0,0 +1,175 @@
+"""LongLLMLingua with immutable shared-comparison identities and settings."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import asdict, dataclass, replace
+from typing import Any
+
+from agentinputlab.contracts import (
+    AlgorithmOutputError,
+    ExecutionBudget,
+    ExternalDependencyError,
+    FrozenAgentState,
+    InputPatch,
+    InterventionContext,
+    SemanticFamily,
+    TraceEvent,
+    TracePhase,
+    ValidationError,
+)
+from agentinputlab.contracts.serialization import content_hash, freeze_mapping
+
+from .adapters import LongLLMLinguaConfig
+from .budget import parse_context_budget
+
+_DEFAULT_CONFIG = LongLLMLinguaConfig()
+
+
+@dataclass(frozen=True, slots=True)
+class LongLLMLinguaCompressionRequest:
+    state: FrozenAgentState
+    config: LongLLMLinguaConfig
+    seed: int
+
+    def __post_init__(self):
+        if (not isinstance(self.state, FrozenAgentState) or not isinstance(self.config, LongLLMLinguaConfig)
+                or type(self.seed) is not int or self.seed < 0):
+            raise ValidationError("LongLLMLingua requires a typed compression request")
+
+
+def _identity(value: Any, label: str) -> Mapping[str, Any]:
+    if not isinstance(value, Mapping) or not value:
+        raise ValidationError(f"LongLLMLingua {label} fingerprint must be nonempty")
+    try:
+        result = freeze_mapping(value)
+        content_hash(result)
+    except (TypeError, ValueError, OverflowError) as exc:
+        raise ValidationError(f"LongLLMLingua {label} fingerprint must be canonical JSON") from exc
+    return result
+
+
+def _identities(context: InterventionContext) -> dict[str, Mapping[str, Any]]:
+    values = {}
+    for name, dependency, operation in (
+        ("compressor", context.external_runtime, "run"),
+        ("reader", context.backend, "generate"),
+    ):
+        fingerprint = getattr(dependency, "fingerprint", None)
+        if not callable(fingerprint) or not callable(getattr(dependency, operation, None)):
+            raise ExternalDependencyError(
+                f"controlled LongLLMLingua requires {name} with {operation}() and fingerprint()"
+            )
+        values[name] = _identity(fingerprint(), name)
+    return values
+
+
+@dataclass(frozen=True, slots=True)
+class LongLLMLinguaControlledBindings:
+    """Operator identities, not a certificate of weights or a paired experiment.
+
+    The compressor identity includes its helper model and tokenizer. The reader
+    is checked but not called here: its benchmark driver must preserve the same
+    reader, generation policy and remaining budget for all comparison arms.
+    """
+
+    compressor: Mapping[str, Any]
+    reader: Mapping[str, Any]
+    budget: ExecutionBudget
+    config: LongLLMLinguaConfig = _DEFAULT_CONFIG
+    seed: int = 0
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.budget, ExecutionBudget) or not isinstance(self.config, LongLLMLinguaConfig):
+            raise ValidationError("controlled LongLLMLingua requires typed budget/config")
+        if type(self.seed) is not int or self.seed < 0:
+            raise ValidationError("controlled LongLLMLingua seed must be nonnegative")
+        object.__setattr__(self, "config", replace(self.config,
+            context_budget=parse_context_budget(self.config.context_budget)))
+        for name in ("compressor", "reader"):
+            object.__setattr__(self, name, _identity(getattr(self, name), name))
+        for name in ("model", "tokenizer", "source"):
+            _identity(self.compressor.get(name), f"compressor {name}")
+
+    @classmethod
+    def from_context(cls, context: InterventionContext, *, budget: ExecutionBudget,
+                     config: LongLLMLinguaConfig = _DEFAULT_CONFIG) -> LongLLMLinguaControlledBindings:
+        if not isinstance(context, InterventionContext):
+            raise ValidationError("controlled LongLLMLingua requires an InterventionContext")
+        return cls(**_identities(context), budget=budget, config=config, seed=context.seed)
+
+    @classmethod
+    def from_mapping(cls, value: Mapping[str, Any]) -> LongLLMLinguaControlledBindings:
+        required = {"compressor", "reader", "budget"}
+        if not required.issubset(value) or set(value) - required - {"config", "seed"}:
+            raise ValidationError("invalid controlled LongLLMLingua binding fields")
+        try:
+            return cls(compressor=value["compressor"], reader=value["reader"],
+                budget=ExecutionBudget(**value["budget"]),
+                config=LongLLMLinguaConfig(**value.get("config", {})), seed=value.get("seed", 0))
+        except TypeError as exc:
+            raise ValidationError("invalid controlled LongLLMLingua binding settings") from exc
+
+    @property
+    def binding_hash(self) -> str:
+        return content_hash(self)
+
+    def require_match(self, context: InterventionContext) -> None:
+        if context.seed != self.seed:
+            raise ValidationError("controlled LongLLMLingua seed differs from binding")
+        for name, value in _identities(context).items():
+            if content_hash(value) != content_hash(getattr(self, name)):
+                raise ValidationError(f"controlled LongLLMLingua {name} differs from binding")
+
+
+@dataclass(frozen=True, slots=True)
+class LongLLMLinguaControlled:
+    bindings: LongLLMLinguaControlledBindings | Mapping[str, Any]
+    profile_kind: str = "controlled_adapter"
+    method_id: str = "longllmlingua"
+    release_id: str = "curate.longllmlingua.controlled"
+    family: SemanticFamily = SemanticFamily.CURATE
+
+    def __post_init__(self) -> None:
+        if isinstance(self.bindings, Mapping):
+            object.__setattr__(self, "bindings", LongLLMLinguaControlledBindings.from_mapping(self.bindings))
+        if not isinstance(self.bindings, LongLLMLinguaControlledBindings):
+            raise ValidationError("controlled LongLLMLingua requires frozen bindings")
+        if self.profile_kind not in {"controlled_adapter", "factorial_adapter"}:
+            raise ValidationError("controlled LongLLMLingua requires controlled or factorial mode")
+        if (self.method_id != "longllmlingua" or self.release_id != "curate.longllmlingua.controlled"
+                or self.family is not SemanticFamily.CURATE):
+            raise ValidationError("controlled LongLLMLingua identity differs from its registry")
+
+    def propose(self, state: FrozenAgentState, context: InterventionContext) -> InputPatch:
+        if state.budget != self.bindings.budget:
+            raise ValidationError("controlled LongLLMLingua state budget differs from binding")
+        self.bindings.require_match(context)
+        result = context.external_runtime.run(LongLLMLinguaCompressionRequest(
+            state, self.bindings.config, context.seed))
+        native = result.get("patch") if isinstance(result, Mapping) else None
+        self.bindings.require_match(context)
+        if (not isinstance(native, InputPatch) or native.method_id != self.method_id
+                or native.release_id != "curate.longllmlingua.official"
+                or native.source_state_hash != state.state_hash):
+            raise AlgorithmOutputError("LongLLMLingua runtime returned a different native patch identity")
+        configs = [e.payload.get("config") for e in native.trace.events if e.name == "official_runtime_config"]
+        if len(configs) != 1 or content_hash(configs[0]) != content_hash(asdict(self.bindings.config)):
+            raise AlgorithmOutputError("LongLLMLingua runtime used different method settings")
+        model = self.bindings.compressor["model"]
+        if (native.metadata.get("model_name") != model.get("locator")
+                or native.metadata.get("model_revision") != model.get("revision")):
+            raise AlgorithmOutputError("LongLLMLingua native patch uses a different helper model")
+        return replace(native, release_id=self.release_id, profile_kind=self.profile_kind,
+            trace=replace(native.trace, release_id=self.release_id, events=(
+                TraceEvent(TracePhase.PREPARE, "controlled_dependency_binding", {
+                    "binding_sha256": self.bindings.binding_hash, "budget": self.bindings.budget,
+                    "seed": self.bindings.seed, "identity_verification": "operator_fingerprint_equality",
+                    "reader_bound_not_invoked": True}), *native.trace.events)),
+            metadata={**native.metadata, "native_source_release": native.release_id,
+                "controlled_binding_sha256": self.bindings.binding_hash,
+                "reader_generation_performed": False, "helper_compute_accounting": "separate_from_reader_generation",
+                "original_paper_result_parity_claimed": False})
+
+
+__all__ = ["LongLLMLinguaCompressionRequest", "LongLLMLinguaControlled", "LongLLMLinguaControlledBindings"]

agentinputlab/algorithms/curate/llmlingua/multilingual.py

@@ -0,0 +1,105 @@
+"""Official LLMLingua-2 multilingual BERT, with an exact asset closure.
+
+The token classifier is a fixed auxiliary model, independent of the evaluated
+reader. No XLM-R weights or downstream-backbone weights are substituted.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from agentinputlab.artifacts import sha256_file
+from agentinputlab.contracts import ExternalDependencyError, ValidationError
+
+from .adapters import LLMLingua2OfficialAdapter, _OfficialAdapterBase
+
+BERT_MULTILINGUAL_MODEL = "microsoft/llmlingua-2-bert-base-multilingual-cased-meetingbank"
+BERT_MULTILINGUAL_REVISION = "5f0c82792b7ea14c6484e015b6a072009496b7f2"
+BERT_MULTILINGUAL_FILES = (
+    ("config.json", "e6c33ec2f099e659e125efaa8b7bb07a6e65b7d9fc36211c6fa3206e1c399085"),
+    ("tokenizer_config.json", "214f78f6c5d034a8b405632d301ee3710eb6f075b3f5faf8241a273a13908746"),
+    ("tokenizer.json", "bf1b59b7b11c95f194f51708d918eea378e09d05f84c0e1656dc5180e8117088"),
+    ("vocab.txt", "fe0fda7c425b48c516fc8f160d594c8022a0808447475c1a7c6d6479763f310c"),
+    ("special_tokens_map.json", "b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3"),
+    ("model.safetensors", "22b9ecde52fec5c97e8c54a293be768727df95a81c6c8dccb03f262a50c58324"),
+)
+
+
+@lru_cache(maxsize=8)
+def _verify_snapshot(snapshot: str, identities: tuple[tuple[str, int, int], ...]) -> None:
+    """Hash immutable weights once per local path/size/mtime identity."""
+    del identities
+    for name, expected in BERT_MULTILINGUAL_FILES:
+        if sha256_file(Path(snapshot) / name) != expected:
+            raise ExternalDependencyError(f"multilingual BERT asset checksum differs: {name}")
+
+
+@dataclass(slots=True)
+class LLMLingua2BertMultilingualOfficialAdapter(LLMLingua2OfficialAdapter):
+    """Pinned author checkpoint and unchanged upstream compression dispatcher.
+
+    Pass an existing snapshot as ``model_name`` for offline use. Otherwise the
+    built-in loader downloads only the six pinned model/tokenizer files. An
+    injected compressor factory is caller-owned code, not accepted-model proof.
+    """
+
+    model_name: str = BERT_MULTILINGUAL_MODEL
+    model_revision: str = BERT_MULTILINGUAL_REVISION
+    release_id: str = "curate.llmlingua2.bert_multilingual_official"
+
+    def __post_init__(self) -> None:
+        _OfficialAdapterBase.__post_init__(self)
+        if self.model_revision != BERT_MULTILINGUAL_REVISION:
+            raise ValidationError("multilingual BERT requires its pinned checkpoint revision")
+        if self.release_id != "curate.llmlingua2.bert_multilingual_official":
+            raise ValidationError("multilingual BERT release identity cannot be replaced")
+        if self.profile_kind not in {"official_native", "factorial_adapter"}:
+            raise ValidationError("multilingual BERT requires an official or factorial profile")
+        allowed = {"revision", "trust_remote_code", "local_files_only", "torch_dtype"}
+        if set(self.model_config) - allowed:
+            raise ValidationError("unsupported multilingual BERT loading option")
+        if self.model_config.get("revision", self.model_revision) != self.model_revision:
+            raise ValidationError("multilingual BERT revision cannot be overridden")
+        if self.model_config.get("trust_remote_code") not in {None, False}:
+            raise ValidationError("multilingual BERT disables remote code")
+        if not isinstance(self.model_config.get("local_files_only", False), bool):
+            raise ValidationError("local_files_only must be boolean")
+        self._validate_torch_dtype(self.model_config.get("torch_dtype", "auto"))
+        if self.model_name != BERT_MULTILINGUAL_MODEL and not Path(self.model_name).is_dir():
+            raise ValidationError("model_name must be the official BERT ID or a local snapshot")
+
+    def _load(self, *, use_llmlingua2: bool) -> Any:
+        if not use_llmlingua2:
+            raise ValidationError("multilingual BERT requires LLMLingua-2 token classification")
+        if self.compressor_factory is None:
+            try:
+                snapshot = Path(self.model_name)
+                if self.model_name == BERT_MULTILINGUAL_MODEL:
+                    from huggingface_hub import snapshot_download
+
+                    snapshot = Path(snapshot_download(
+                        self.model_name,
+                        revision=self.model_revision,
+                        allow_patterns=[name for name, _ in BERT_MULTILINGUAL_FILES],
+                        local_files_only=self.model_config.get("local_files_only", False),
+                    ))
+                identities = tuple(
+                    (name, (snapshot / name).stat().st_size, (snapshot / name).stat().st_mtime_ns)
+                    for name, _ in BERT_MULTILINGUAL_FILES
+                )
+                _verify_snapshot(str(snapshot.resolve()), identities)
+            except (ImportError, OSError, ValueError) as exc:
+                raise ExternalDependencyError(
+                    "multilingual BERT needs its complete pinned safetensors/tokenizer snapshot "
+                    "and `pip install agentinputlab[compression]`"
+                ) from exc
+        return _OfficialAdapterBase._load(self, use_llmlingua2=True)
+
+
+__all__ = [
+    "BERT_MULTILINGUAL_FILES", "BERT_MULTILINGUAL_MODEL", "BERT_MULTILINGUAL_REVISION",
+    "LLMLingua2BertMultilingualOfficialAdapter",
+]

agentinputlab/algorithms/curate/llmlingua/span_map.py

@@ -0,0 +1,167 @@
+"""Conservative exact-span provenance for compressed context."""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_left
+from collections.abc import Sequence
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+
+from agentinputlab.contracts import ContextBlock
+
+# Character-level difflib is useful for short inputs but quadratic on the long,
+# repetitive contexts these compressors are meant to handle.  Longer pairs use
+# literal indexed anchors instead; this changes provenance choices, not text.
+_DIFF_PRODUCT_LIMIT = 2_000_000
+_TOKENS = re.compile(r"[A-Za-z0-9_]+|[^\s]")
+
+
+def _indexed_matches(
+    source: str, output: str, output_tokens: Sequence[tuple[int, int]], minimum_match: int,
+) -> list[tuple[int, int, int]]:
+    """Find exact runs without a source-by-output character comparison matrix.
+
+    One-, two- and three-token anchors include the original intervening spaces.
+    Sorted occurrence lists keep even repeated anchors cheap.  Reordered source
+    text is allowed; partial words fall back to indexed character runs.  Neither
+    path normalizes text or attributes a span to a merely similar substring.
+    """
+
+    source_tokens = [match.span() for match in _TOKENS.finditer(source)]
+    anchors: list[dict[str, list[int]]] = [{}, {}, {}]
+    for width, index in enumerate(anchors, start=1):
+        for offset in range(len(source_tokens) - width + 1):
+            start = source_tokens[offset][0]
+            end = source_tokens[offset + width - 1][1]
+            index.setdefault(source[start:end], []).append(start)
+    character_anchors: dict[str, list[int]] | None = None
+    character_width = min(3, minimum_match)
+    matches: list[tuple[int, int, int]] = []
+
+    def append(source_start: int, output_start: int, size: int) -> None:
+        if matches:
+            last_source, last_output, last_size = matches[-1]
+            source_end, output_end = last_source + last_size, last_output + last_size
+            if (source_start >= source_end and output_start >= output_end
+                    and source_start - source_end == output_start - output_end
+                    and source[source_end:source_start] == output[output_end:output_start]):
+                matches[-1] = (last_source, last_output, output_start + size - last_output)
+                return
+        matches.append((source_start, output_start, size))
+
+    source_cursor = 0
+    offset = 0
+    while offset < len(output_tokens):
+        for width in range(min(3, len(output_tokens) - offset), 0, -1):
+            start, end = output_tokens[offset][0], output_tokens[offset + width - 1][1]
+            positions = anchors[width - 1].get(output[start:end])
+            if positions:
+                position_index = bisect_left(positions, source_cursor)
+                source_start = positions[position_index if position_index < len(positions) else 0]
+                append(source_start, start, end - start)
+                source_cursor = source_start + end - start
+                offset += width
+                break
+        else:
+            # Token compression can keep only part of a word.  Short literal
+            # anchors locate those runs without rescanning the source per word.
+            if character_anchors is None:
+                character_anchors = {}
+                for position in range(len(source) - character_width + 1):
+                    key = source[position:position + character_width]
+                    character_anchors.setdefault(key, []).append(position)
+            start, end = output_tokens[offset]
+            output_start = start
+            while output_start + character_width <= end:
+                positions = character_anchors.get(output[output_start:output_start + character_width])
+                if not positions:
+                    output_start += 1
+                    continue
+                position_index = bisect_left(positions, source_cursor)
+                source_start = positions[position_index if position_index < len(positions) else 0]
+                size = character_width
+                while (output_start + size < end and source_start + size < len(source)
+                       and output[output_start + size] == source[source_start + size]):
+                    size += 1
+                append(source_start, output_start, size)
+                source_cursor = source_start + size
+                output_start += size
+            offset += 1
+    return matches
+
+
+@dataclass(frozen=True, slots=True)
+class SpanMapping:
+    output_start: int
+    output_end: int
+    source_block_id: str
+    source_start: int
+    source_end: int
+    exact: bool = True
+
+
+def build_span_map(
+    compressed_text: str,
+    source_blocks: Sequence[ContextBlock],
+    *,
+    minimum_match: int = 3,
+) -> tuple[SpanMapping, ...]:
+    """Map only exact matching spans and leave generated/unmatched text unmapped.
+
+    Per-block matching supports context reorder.  Overlapping output matches are
+    resolved longest-first; the function never invents provenance for a token
+    that cannot be located exactly in a source block.  Large pairs use indexed
+    literal anchors instead of quadratic character diffing; ambiguous locations
+    can therefore differ from the short-input matcher.  This is provenance, not
+    a reconstruction of the compressor's internal token-selection decisions.
+    """
+
+    if minimum_match < 1:
+        raise ValueError("minimum_match must be positive")
+    if not compressed_text:
+        return ()
+    output_tokens: list[tuple[int, int]] | None = None
+    candidates: list[SpanMapping] = []
+    for index, block in enumerate(source_blocks):
+        block_id = block.block_id or f"context-{index}"
+        contained = block.text.find(compressed_text)
+        if contained >= 0:
+            matches = [(contained, 0, len(compressed_text))]
+        elif len(block.text) * len(compressed_text) <= _DIFF_PRODUCT_LIMIT:
+            matcher = SequenceMatcher(None, block.text, compressed_text, autojunk=False)
+            matches = matcher.get_matching_blocks()
+        else:
+            if output_tokens is None:
+                output_tokens = [match.span() for match in _TOKENS.finditer(compressed_text)]
+            matches = _indexed_matches(block.text, compressed_text, output_tokens, minimum_match)
+        for source_start, output_start, size in matches:
+            if size < minimum_match:
+                continue
+            candidates.append(
+                SpanMapping(
+                    output_start=output_start,
+                    output_end=output_start + size,
+                    source_block_id=block_id,
+                    source_start=source_start,
+                    source_end=source_start + size,
+                )
+            )
+    candidates.sort(
+        key=lambda item: (
+            -(item.output_end - item.output_start),
+            item.output_start,
+            item.source_block_id,
+            item.source_start,
+        )
+    )
+    occupied = bytearray(len(compressed_text))
+    selected: list[SpanMapping] = []
+    for candidate in candidates:
+        if any(occupied[candidate.output_start : candidate.output_end]):
+            continue
+        occupied[candidate.output_start : candidate.output_end] = b"\x01" * (
+            candidate.output_end - candidate.output_start
+        )
+        selected.append(candidate)
+    return tuple(sorted(selected, key=lambda item: (item.output_start, item.output_end)))

agentinputlab/algorithms/curate/recomp/__init__.py

@@ -0,0 +1,47 @@
+"""RECOMP compressor-component adapters."""
+
+from .external import (
+    RECOMP_ABSTRACTIVE_INPUT_TEMPLATE,
+    RECOMP_ABSTRACTIVE_INPUT_TEMPLATE_SHA256,
+    RECOMP_CANDIDATE_UNIT_KIND,
+    RECOMP_COMMON_SUPPORT_RELEASE,
+    RECOMP_COMMON_SUPPORT_SPLITTER,
+    RECOMP_UPSTREAM_COMMIT,
+    RECOMPAbstractiveControls,
+    RECOMPAbstractiveExternal,
+    RECOMPExtractiveCommonSupport,
+    RECOMPExtractiveControls,
+    RECOMPExtractiveExternal,
+    RECOMPSentenceScore,
+    prepare_recomp_sentence_state,
+)
+from .selective import (
+    RECOMP_QUERY_DECODER_ID,
+    RECOMP_QUERY_DECODER_SHA256,
+    RECOMP_SELECTIVE_ASSEMBLY,
+    RECOMP_SELECTIVE_RELEASE,
+    RECOMPSelectiveRuntime,
+    parse_recomp_query_reader_answer,
+)
+
+__all__ = [
+    "RECOMP_ABSTRACTIVE_INPUT_TEMPLATE",
+    "RECOMP_ABSTRACTIVE_INPUT_TEMPLATE_SHA256",
+    "RECOMP_CANDIDATE_UNIT_KIND",
+    "RECOMP_COMMON_SUPPORT_RELEASE",
+    "RECOMP_COMMON_SUPPORT_SPLITTER",
+    "RECOMP_QUERY_DECODER_ID",
+    "RECOMP_QUERY_DECODER_SHA256",
+    "RECOMP_SELECTIVE_ASSEMBLY",
+    "RECOMP_SELECTIVE_RELEASE",
+    "RECOMP_UPSTREAM_COMMIT",
+    "RECOMPAbstractiveControls",
+    "RECOMPAbstractiveExternal",
+    "RECOMPExtractiveCommonSupport",
+    "RECOMPExtractiveControls",
+    "RECOMPExtractiveExternal",
+    "RECOMPSelectiveRuntime",
+    "RECOMPSentenceScore",
+    "parse_recomp_query_reader_answer",
+    "prepare_recomp_sentence_state",
+]