synapse-cli-agent 0.1.13__py3-none-any.whl

synapse/tool_output/__init__.py

@@ -0,0 +1,59 @@
+"""Reversible tool-output transformation domain."""
+
+from synapse.tool_output.detection import detect_content_type
+from synapse.tool_output.metrics import clear_metrics_notifier, set_metrics_notifier
+from synapse.tool_output.models import (
+    CompressionStageEvent,
+    ContentType,
+    Detection,
+    ModelRequestCompressionEvent,
+    ToolOutputRecord,
+    TransformContext,
+    TransformEvent,
+    TransformResult,
+)
+from synapse.tool_output.pipeline import ToolOutputTransformPipeline
+from synapse.tool_output.repository import ToolOutputRepository, content_to_text
+from synapse.tool_output.transformers import (
+    CodeTransformer,
+    DiffTransformer,
+    GenericTransformer,
+    GitSummaryTransformer,
+    JsonTransformer,
+    LogTransformer,
+    NativeTransformer,
+    PathListTransformer,
+    SearchTransformer,
+    ToolOutputTransformer,
+    load_native_transformers,
+    load_transformer_plugins,
+)
+
+__all__ = [
+    "CodeTransformer",
+    "CompressionStageEvent",
+    "ContentType",
+    "Detection",
+    "DiffTransformer",
+    "GenericTransformer",
+    "GitSummaryTransformer",
+    "JsonTransformer",
+    "LogTransformer",
+    "ModelRequestCompressionEvent",
+    "NativeTransformer",
+    "PathListTransformer",
+    "SearchTransformer",
+    "ToolOutputRecord",
+    "ToolOutputRepository",
+    "ToolOutputTransformer",
+    "ToolOutputTransformPipeline",
+    "TransformContext",
+    "TransformEvent",
+    "TransformResult",
+    "clear_metrics_notifier",
+    "content_to_text",
+    "detect_content_type",
+    "load_native_transformers",
+    "load_transformer_plugins",
+    "set_metrics_notifier",
+]

synapse/tool_output/detection.py

@@ -0,0 +1,170 @@
+"""Content classification rules for tool-output transformation."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from synapse.tool_output.models import ContentType, Detection
+
+_SEARCH_LINE = re.compile(
+    r"^(?P<path>.+?)(?P<separator>[:\-])(?P<line>\d+)(?P=separator)(?P<body>.*)$"
+)
+_ERROR_LINE = re.compile(
+    r"\b(error|fatal|failed|failure|exception|traceback|critical)\b", re.I
+)
+_LOG_SUMMARY = re.compile(r"\b(passed|failed|skipped|collected|tests? run|exit code)\b", re.I)
+_NUMBERED_SOURCE_LINE = re.compile(r"^(?P<indent>\s*)\d+(?:\.\d+)?\t(?P<body>.*)$")
+_PATH_ROW = re.compile(r"^(?:(?:[A-Za-z]:)?[/\\]|\.{0,2}[/\\])?(?:[^/\\\s:]+[/\\])+[^/\\\s:]+$")
+_CODE_SUFFIXES = frozenset(
+    {
+        ".c", ".cc", ".cpp", ".cs", ".go", ".h", ".hpp", ".java", ".js", ".jsx",
+        ".kt", ".php", ".py", ".pyi", ".rb", ".rs", ".scala", ".sh", ".swift", ".ts",
+        ".tsx",
+    }
+)
+
+def _strip_numbered_source_lines(content: str) -> tuple[str, int, int]:
+    """Return a detection-only view without read_file's cat-n line prefixes."""
+    normalized: list[str] = []
+    numbered = 0
+    non_empty = 0
+    for line in content.splitlines():
+        if line.strip():
+            non_empty += 1
+        match = _NUMBERED_SOURCE_LINE.match(line)
+        if match:
+            numbered += 1
+            normalized.append(match.group("indent") + match.group("body"))
+        else:
+            normalized.append(line)
+    return "\n".join(normalized), numbered, non_empty
+
+
+def _code_marker_count(content: str) -> int:
+    return sum(
+        bool(
+            re.match(
+                r"^\s*(?:async\s+def|def|class|function|func|fn|import|from|use)\b",
+                line,
+            )
+        )
+        for line in content.splitlines()[:200]
+    )
+
+
+def _diff_bloat_metadata(content: str) -> dict[str, Any]:
+    total_lines = 0
+    change_lines = 0
+    context_lines = 0
+    in_hunk = False
+    for line in content.splitlines():
+        total_lines += 1
+        if line.startswith("@@"):
+            in_hunk = True
+            continue
+        if line.startswith("diff --git"):
+            in_hunk = False
+            continue
+        if not in_hunk or line.startswith(("+++", "---")):
+            continue
+        if line.startswith(("+", "-")):
+            change_lines += 1
+        elif line.startswith(" "):
+            context_lines += 1
+    denominator = context_lines + change_lines
+    context_ratio = context_lines / denominator if denominator else 0.0
+    normal_context_ratio = 0.6
+    bloat_score = (
+        max(0.0, min(1.0, (context_ratio - normal_context_ratio) / (1 - normal_context_ratio)))
+        if denominator
+        else 0.0
+    )
+    return {
+        "total_lines": total_lines,
+        "change_lines": change_lines,
+        "context_lines": context_lines,
+        "context_ratio": round(context_ratio, 4),
+        "bloat_score": round(bloat_score, 4),
+        "dense_diff": bool(total_lines >= 50 and context_ratio <= normal_context_ratio),
+    }
+
+
+def detect_content_type(content: str) -> Detection:
+    lines = content.splitlines()
+    if not lines:
+        return Detection(ContentType.TEXT, 1.0)
+    search_count = sum(bool(_SEARCH_LINE.match(line)) for line in lines[:100])
+    timestamp_count = sum(
+        bool(re.match(r"^\d{4}-\d{2}-\d{2}(?:[ T]|$)", line)) for line in lines[:100]
+    )
+    enough_search_lines = search_count >= max(3, len(lines[:100]) // 3)
+    mostly_timestamped = timestamp_count >= max(3, len(lines[:100]) // 2)
+    if enough_search_lines and not mostly_timestamped:
+        return Detection(
+            ContentType.SEARCH, min(1.0, search_count / max(1, len(lines[:100])) + 0.3)
+        )
+    path_rows = sum(bool(_PATH_ROW.match(line)) for line in lines)
+    if len(lines) >= 3 and path_rows == len(lines):
+        return Detection(ContentType.PATHS, 1.0)
+    if any(line.startswith(("diff --git", "--- a/", "+++ b/", "@@")) for line in lines[:20]):
+        return Detection(ContentType.DIFF, 0.95)
+    git_summary_markers = sum(
+        bool(
+            re.match(
+                r"^(?:Merge made by the .+ strategy\.|(?:create|delete) mode \d+ |"
+                r"rename .+ => .+|\s*\d+ files? changed(?:,|$)|\s*\d+ insertions?\(\+\)|"
+                r"\s*\d+ deletions?\(-\)| .+\s+\|\s+\d+\s+[+\-]+$)",
+                line,
+            )
+        )
+        for line in lines[:200]
+    )
+    if git_summary_markers >= 2:
+        return Detection(ContentType.GIT_SUMMARY, min(0.95, 0.45 + git_summary_markers / 20))
+    sampled = lines[:100]
+    log_markers = sum(
+        bool(_ERROR_LINE.search(line) or _LOG_SUMMARY.search(line)) for line in sampled
+    )
+    timestamped = timestamp_count
+    if log_markers >= 3 or (
+        timestamped >= max(3, len(sampled) // 2) and any(_ERROR_LINE.search(line) for line in lines)
+    ):
+        return Detection(ContentType.LOG, 0.8)
+    code_markers = _code_marker_count(content)
+    if code_markers >= 3:
+        return Detection(ContentType.CODE, min(0.95, 0.4 + code_markers / max(1, len(lines[:200]))))
+    try:
+        parsed = json.loads(content)
+        if isinstance(parsed, (list, dict)):
+            return Detection(ContentType.JSON, 0.9)
+    except (ValueError, TypeError):
+        pass
+    return Detection(ContentType.TEXT, 0.3)
+
+
+def _critical_lines(content: str, content_type: ContentType) -> list[str]:
+    lines = content.splitlines()
+    if content_type is ContentType.LOG:
+        return [
+            line
+            for line in lines
+            if _ERROR_LINE.search(line) or re.search(r"\b[A-Za-z_]+(?:Error|Exception):", line)
+        ]
+    if content_type is ContentType.DIFF:
+        return [
+            line
+            for line in lines
+            if line.startswith(("+", "-")) and not line.startswith(("+++", "---"))
+        ]
+    if content_type is ContentType.GIT_SUMMARY:
+        return [
+            line
+            for line in lines
+            if re.match(
+                r"^(?:Merge made by the .+ strategy\.|\s*\d+ files? changed(?:,|$)|"
+                r"\s*\d+ insertions?\(\+\)|\s*\d+ deletions?\(-\))",
+                line,
+            )
+        ]
+    return []

synapse/tool_output/metrics.py

@@ -0,0 +1,32 @@
+"""Best-effort process-local notifications for tool-output metrics."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+
+_metrics_notifier: Callable[[str], None] | None = None
+_metrics_notifier_lock = threading.RLock()
+
+
+def set_metrics_notifier(notifier: Callable[[str], None] | None) -> None:
+    """Install a callback invoked after metrics-changing persistence writes."""
+    global _metrics_notifier
+    with _metrics_notifier_lock:
+        _metrics_notifier = notifier
+
+
+def clear_metrics_notifier() -> None:
+    """Remove the active process-local metrics callback."""
+    set_metrics_notifier(None)
+
+
+def notify_metrics_changed(thread_id: str) -> None:
+    """Notify an observer without allowing UI failures to affect tool execution."""
+    with _metrics_notifier_lock:
+        notifier = _metrics_notifier
+    if notifier is not None:
+        try:
+            notifier(thread_id)
+        except Exception:  # noqa: BLE001
+            pass

synapse/tool_output/models.py

@@ -0,0 +1,173 @@
+"""Data models for reversible tool-output transformation."""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import StrEnum
+from typing import Any
+
+
+class ContentType(StrEnum):
+    SEARCH = "search"
+    PATHS = "paths"
+    LOG = "log"
+    DIFF = "diff"
+    GIT_SUMMARY = "git-summary"
+    JSON = "json"
+    CODE = "code"
+    TEXT = "text"
+
+
+@dataclass(frozen=True)
+class Detection:
+    content_type: ContentType
+    confidence: float
+
+
+@dataclass(frozen=True)
+class TransformContext:
+    tool_name: str
+    status: str
+    query: str = ""
+    tool_args: dict[str, Any] = field(default_factory=dict)
+    file_path: str = ""
+    file_suffix: str = ""
+    fresh_read_source: bool = True
+
+
+@dataclass(frozen=True)
+class CompressionStageEvent:
+    """One observable stage in a tool-output compression decision."""
+
+    phase: str
+    algorithm: str
+    applied: bool
+    reason_code: str
+    input_bytes: int = 0
+    output_bytes: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    duration_ms: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class TransformResult:
+    content: str
+    transformer: str
+    content_type: ContentType
+    critical_total: int
+    critical_retained: int
+    metadata: dict[str, Any] = field(default_factory=dict)
+    stages: tuple[CompressionStageEvent, ...] = ()
+
+
+@dataclass(frozen=True)
+class ToolOutputRecord:
+    ref: str
+    thread_id: str
+    checkpoint_ns: str
+    tool_call_id: str
+    tool_name: str
+    status: str
+    content: str
+    size_bytes: int
+    sha256: str
+    created_at: str
+
+
+@dataclass(frozen=True)
+class ModelRequestCompressionEvent:
+    """One model call's compression and provider-safety accounting."""
+
+    request_id: str
+    provider: str
+    api_style: str
+    auth_mode: str
+    model: str
+    input_tokens_before: int
+    input_tokens_after: int
+    provider_input_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_write_tokens: int = 0
+    uncached_input_tokens: int = 0
+    output_tokens: int = 0
+    tool_output_saved_tokens: int = 0
+    prompt_saved_tokens: int = 0
+    summarization_saved_tokens: int = 0
+    total_saved_tokens: int = 0
+    candidate_blocks: int = 0
+    transformed_blocks: int = 0
+    protected_tokens_by_reason: dict[str, int] = field(default_factory=dict)
+    content_breakdown: dict[str, int] = field(default_factory=dict)
+    opportunity_tokens_by_reason: dict[str, int] = field(default_factory=dict)
+    turn_id: str = ""
+    turn_index: int = 0
+    model_call_index: int = 0
+    live_zone_plan: list[dict[str, Any]] = field(default_factory=list)
+    live_zone_tokens: dict[str, int] = field(default_factory=dict)
+    wire_fingerprints: dict[str, Any] = field(default_factory=dict)
+    cache_diagnostics: dict[str, Any] = field(default_factory=dict)
+    tool_schema_profiles: list[dict[str, Any]] = field(default_factory=list)
+    token_count_method: str = "langchain_approximate"
+    duration_ms: float = 0.0
+
+    def as_dict(self) -> dict[str, Any]:
+        data = asdict(self)
+        whole_denominator = self.input_tokens_after + self.total_saved_tokens
+        new_input = self.uncached_input_tokens + self.cache_write_tokens
+        data["whole_request_savings_ratio"] = (
+            round(self.total_saved_tokens / whole_denominator, 4) if whole_denominator else 0.0
+        )
+        data["new_input_savings_ratio"] = (
+            round(self.total_saved_tokens / (new_input + self.total_saved_tokens), 4)
+            if new_input > 0
+            else 0.0
+        )
+        return data
+
+
+@dataclass(frozen=True)
+class TransformEvent:
+    content_type: str
+    transformer: str
+    outcome: str
+    original_bytes: int
+    visible_bytes: int
+    duration_ms: float
+    critical_total: int
+    critical_retained: int
+    ref_created: bool
+    execution_path: str = "python_only"
+    estimated_original_tokens: int = 0
+    estimated_visible_tokens: int = 0
+    decision: str = "transformed"
+    reason_code: str = "compressed"
+    reason_detail: str = ""
+    eligible: bool = True
+    source_kind: str = "tool-output"
+    detection_confidence: float = 0.0
+    threshold_bytes: int = 0
+    tool_call_id: str = ""
+    tool_name: str = ""
+    status: str = "success"
+    checkpoint_ns: str = ""
+    message_id: str = ""
+    algorithm_output_bytes: int = 0
+    algorithm_output_tokens: int = 0
+    token_count_method: str = "langchain_approximate"
+    content_sha256: str = ""
+    stages: tuple[CompressionStageEvent, ...] = ()
+
+    def as_dict(self) -> dict[str, Any]:
+        data = asdict(self)
+        data["saved_bytes"] = max(0, self.original_bytes - self.visible_bytes)
+        data["estimated_saved_tokens"] = max(
+            0, self.estimated_original_tokens - self.estimated_visible_tokens
+        )
+        data["savings_ratio"] = (
+            round(1 - self.visible_bytes / self.original_bytes, 4) if self.original_bytes else 0.0
+        )
+        return data