code-puppy-core-plugins 0.0.77__tar.gz → 0.0.78__tar.gz

{code_puppy_core_plugins-0.0.77 → code_puppy_core_plugins-0.0.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: code-puppy-core-plugins
-Version: 0.0.77
+Version: 0.0.78
 Summary: Official plugin bundle for Code Puppy
 Project-URL: Repository, https://github.com/mpfaffenberger/code_puppy_core_plugins
 Project-URL: Homepage, https://github.com/mpfaffenberger/code_puppy_core_plugins

{code_puppy_core_plugins-0.0.77 → code_puppy_core_plugins-0.0.78}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "code-puppy-core-plugins"
-version = "0.0.77"
+version = "0.0.78"
 description = "Official plugin bundle for Code Puppy"
 readme = "README.md"
 requires-python = ">=3.11,<3.15"
@@ -122,7 +122,6 @@ subagent_panel = "code_puppy_core_plugins.subagent_panel.register_callbacks"
 switch_agent_resume = "code_puppy_core_plugins.switch_agent_resume.register_callbacks"
 theme = "code_puppy_core_plugins.theme.register_callbacks"
 timestamp_heartbeat = "code_puppy_core_plugins.timestamp_heartbeat.register_callbacks"
-token_ratio_learner = "code_puppy_core_plugins.token_ratio_learner.register_callbacks"
 universal_constructor = "code_puppy_core_plugins.universal_constructor.register_callbacks"
 web_retriever_skill = "code_puppy_core_plugins.web_retriever_skill.register_callbacks"
 wiggum = "code_puppy_core_plugins.wiggum.register_callbacks"

code_puppy_core_plugins-0.0.77/code_puppy_core_plugins/token_ratio_learner/__init__.py

@@ -1,5 +0,0 @@
-"""Token ratio learner plugin — per-model chars/token ratio learning.
-
-Provides ``count_tokens()`` using learned ratios, falling back to a safe
-overestimate when no ratio has been learned yet for a given model.
-"""

code_puppy_core_plugins-0.0.77/code_puppy_core_plugins/token_ratio_learner/ratios.py

@@ -1,208 +0,0 @@
-"""Learned token-per-character ratios for accurate token estimation.
-
-When the API reports real prompt_tokens we can compute the actual
-chars-per-token ratio for that model and store it keyed by the bare
-model name (the part after ``:``).  Future ``count_tokens`` calls use the
-stored ratio for that model, falling back to a hardcoded default.
-
-The default (2.5) matches the classic char/token heuristic used across
-most models.  Real tokenizers vary (GPT-4 ~3.5, Claude ~2.8), but 2.5
-provides a reasonable middle ground.  The max clamp (3.5) prevents
-absurdly high ratios from inflating token estimates.
-
-Storage lives at ``~/.code_puppy/token_ratios.json``, overridable via
-the env var ``CODE_PUPPY_TOKEN_RATIOS_PATH``.
-"""
-
-import json
-import logging
-import os
-import tempfile
-import threading
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-# Constants
-
-# Default 2.5 chars/token slightly overestimates for safer compaction.
-_DEFAULT_RATIO = 2.5
-
-# Clamp learned ratios to [1.5, 3.5]; extremes imply implausible tokenization and
-# can collapse estimates or risk context overflow.
-_MIN_RATIO = 1.5
-_MAX_RATIO = 3.5
-
-# Lazy in-memory cache, populated by ``_ensure_ratios_loaded()``.
-
-_LEARNED_RATIOS: dict[str, float] = {}
-_ratios_loaded: bool = False
-_ratios_lock = threading.Lock()
-
-# Learned-ratio storage path.
-
-_TOKEN_RATIOS_PATH: Path = Path(
-    os.path.expanduser(
-        os.environ.get(
-            "CODE_PUPPY_TOKEN_RATIOS_PATH",
-            str(Path.home() / ".code_puppy" / "token_ratios.json"),
-        )
-    )
-)
-
-
-# Internal helpers
-
-
-def _ensure_ratios_loaded() -> None:
-    """Populate ``_LEARNED_RATIOS`` from disk on first call (idempotent)."""
-    global _ratios_loaded
-    with _ratios_lock:
-        if _ratios_loaded:
-            return
-        _LEARNED_RATIOS.clear()
-        _LEARNED_RATIOS.update(_load_learned_ratios())
-        _ratios_loaded = True
-
-
-def _load_learned_ratios() -> dict[str, float]:
-    """Load learned token ratios from the on-disk JSON file.
-
-    Returns an empty dict when the file doesn't exist or is corrupt —
-    callers fall back to ``_DEFAULT_RATIO``.
-
-    Each loaded value is clamped to ``[_MIN_RATIO, _MAX_RATIO]`` so that
-    poisoned or corrupted values cannot cause token-count collapse.
-    """
-    try:
-        if _TOKEN_RATIOS_PATH.is_file():
-            data = json.loads(_TOKEN_RATIOS_PATH.read_text(encoding="utf-8"))
-            if isinstance(data, dict):
-                clamped: dict[str, float] = {}
-                for k, v in data.items():
-                    if isinstance(v, (int, float)) and v > 0:
-                        # Lowercase keys for case-insensitive matching.
-                        clamped[k.lower()] = max(_MIN_RATIO, min(_MAX_RATIO, float(v)))
-                return clamped
-    except (OSError, json.JSONDecodeError, ValueError):
-        pass
-    return {}
-
-
-def _save_learned_ratios(ratios: dict[str, float]) -> None:
-    """Persist learned token ratios to disk atomically (best-effort, never raises)."""
-    try:
-        parent = _TOKEN_RATIOS_PATH.parent
-        parent.mkdir(parents=True, exist_ok=True)
-        fd, tmp_name = tempfile.mkstemp(dir=parent, suffix=".tmp")
-        try:
-            with os.fdopen(fd, "w", encoding="utf-8") as tmp:
-                json.dump(ratios, tmp, indent=2)
-            os.replace(tmp_name, str(_TOKEN_RATIOS_PATH))
-        except OSError:
-            try:
-                os.unlink(tmp_name)
-            except OSError:
-                pass
-            raise
-    except OSError:
-        pass  # Silently ignore write failures (permissions, disk full, etc.)
-
-
-def _record_token_ratio(model: str, char_count: int, token_count: int) -> None:
-    """Record an observed chars-per-token ratio for *model*.
-
-    Called after a successful API response where we know both the input
-    character count and the actual prompt token count.  Stores a running
-    average keyed by the bare model name (the part after ``:``) so that
-    the same model accessed through different providers shares the estimate.
-
-    The running average uses a 70/30 blend: 70% old, 30% new.  This lets
-    the estimate adapt to the real tokenizer without swinging wildly on
-    every call.
-    """
-    if char_count <= 0 or token_count <= 0:
-        return
-
-    # Strip the provider prefix and lowercase the model name.
-    model_name = model.split(":", 1)[1] if ":" in model else model
-    model_name = model_name.lower()
-
-    _ensure_ratios_loaded()
-
-    new_ratio = char_count / token_count
-    new_ratio = max(_MIN_RATIO, min(_MAX_RATIO, new_ratio))
-
-    with _ratios_lock:
-        old = _LEARNED_RATIOS.get(model_name)
-        if old is not None:
-            blended = 0.7 * old + 0.3 * new_ratio
-        else:
-            blended = new_ratio
-        _LEARNED_RATIOS[model_name] = round(blended, 4)
-        _save_learned_ratios(_LEARNED_RATIOS)
-
-
-# Public API
-
-
-def count_tokens(text: str, model: str | None = None) -> int:
-    """Estimate token count using a character-based heuristic.
-
-    If we have a *learned* chars-per-token ratio for *model*, use it.
-    Otherwise fall back to ``_DEFAULT_RATIO`` (2.5).
-
-    Args:
-        text: The text content whose token count we need.
-        model: Optional model identifier string (e.g.
-               ``"anthropic:claude-sonnet"``).  The provider prefix is
-               stripped before lookup, so ``"wafer:glm5.1"`` and bare
-               ``"glm5.1"`` resolve to the same key.
-
-    Returns:
-        Estimated token count (always >= 1 for non-empty input).
-    """
-    _ensure_ratios_loaded()
-
-    if not text:
-        return 0
-
-    # Look up by bare model name (strip provider prefix, lowercase).
-    model_name = None
-    if model:
-        model_name = model.split(":", 1)[1] if ":" in model else model
-        model_name = model_name.lower()
-
-    raw_ratio = _LEARNED_RATIOS.get(model_name or "", _DEFAULT_RATIO)
-    ratio = max(_MIN_RATIO, min(_MAX_RATIO, raw_ratio))
-
-    result = round(len(text) / ratio)
-    return result if result > 0 else 1
-
-
-def get_ratio_for_model(model: str) -> float:
-    """Return the current chars/token ratio for *model* (or default)."""
-    _ensure_ratios_loaded()
-    model_name = model.split(":", 1)[1] if ":" in model else model
-    model_name = model_name.lower()
-    raw = _LEARNED_RATIOS.get(model_name, _DEFAULT_RATIO)
-    return max(_MIN_RATIO, min(_MAX_RATIO, raw))
-
-
-def list_known_ratios() -> dict[str, float]:
-    """Return a copy of all learned ratios (for debugging/stats)."""
-    _ensure_ratios_loaded()
-    with _ratios_lock:
-        return dict(_LEARNED_RATIOS)
-
-
-# Storage path override (for tests)
-
-
-def set_ratios_path(path: str | Path) -> None:
-    """Override the token ratios file path (mostly for testing)."""
-    global _TOKEN_RATIOS_PATH, _ratios_loaded, _LEARNED_RATIOS
-    _TOKEN_RATIOS_PATH = Path(path)
-    with _ratios_lock:
-        _LEARNED_RATIOS.clear()
-        _ratios_loaded = False

code_puppy_core_plugins-0.0.77/code_puppy_core_plugins/token_ratio_learner/register_callbacks.py

@@ -1,218 +0,0 @@
-"""Register callbacks for the token-ratio-learner plugin.
-
-On startup we monkeypatch four things:
-
-1. ``_history.estimate_tokens`` — replaces the hardcoded
-   ``max(1, floor(len(text)/2.5))`` with a learned-ratio lookup, falling
-   back to ``_DEFAULT_RATIO`` (2.5) which matches the classic heuristic.
-
-2. ``_history.estimate_tokens_for_message`` — replaces per-part char/2.5
-   summing + ``model_token_multiplier`` with direct
-   ``ratios.count_tokens(part_str, model=model_name)`` per part.  The
-   learned ratio is already model-calibrated, so the multiplier becomes
-   redundant.
-
-3. ``_runtime.run_with_mcp`` — wraps the orchestration so that after every
-   successful agent run we extract ``result.usage.input_tokens`` and
-   the character count of the input, then call
-   ``ratios._record_token_ratio()`` to update the running average.
-
-4. ``subagent_stream_handler._estimate_tokens`` — same replacement as
-   (1) so that subagent streaming token counts use learned ratios.
-
-We do **not** touch ``event_stream_handler`` inlining — that code is for
-progress display only, not for compaction decisions.
-"""
-
-import logging
-from typing import Any, Optional, Union
-
-from code_puppy.callbacks import register_callback
-
-logger = logging.getLogger(__name__)
-
-# Original references, populated during startup.
-
-_original_estimate_tokens = None
-_original_estimate_tokens_for_message = None
-_original_run_with_mcp = None
-_original_subagent_estimate_tokens = None
-
-
-# Patched ``estimate_tokens`` (root text-level)
-
-
-def _patched_estimate_tokens(text: str) -> int:
-    """Drop-in replacement that uses learned char/token ratios.
-
-    No model argument — uses the default ratio.  Callers that need
-    model-specific calibration go through ``estimate_tokens_for_message``.
-    """
-    from code_puppy_core_plugins.token_ratio_learner.ratios import count_tokens
-
-    return count_tokens(text, model=None)
-
-
-# Patched ``estimate_tokens_for_message``
-
-
-def _patched_estimate_tokens_for_message(
-    message: Any,
-    model_name: Optional[str] = None,
-) -> int:
-    """Per-message token estimation using learned model-specific ratios.
-
-    Iterates over ``message.parts``, stringifying each with
-    ``stringify_part``, then calls ``ratios.count_tokens(part_str, model)``.
-    Bypasses the old ``estimate_tokens`` + ``model_token_multiplier`` chain
-    entirely — the learned ratio *is* the calibration.
-    """
-    # Import stringify_part from the same module the original lives in
-    from code_puppy.agents._history import stringify_part
-    from code_puppy_core_plugins.token_ratio_learner.ratios import count_tokens
-
-    total = 0
-    for part in getattr(message, "parts", []) or []:
-        part_str = stringify_part(part)
-        if part_str:
-            total += count_tokens(part_str, model=model_name)
-    return max(1, total)
-
-
-# Patched ``run_with_mcp`` records ratios after each API call.
-
-
-def _compute_input_char_count(
-    agent: Any, prompt: Union[str, list], attachments: Any, link_attachments: Any
-) -> int:
-    """Compute a reasonable character count for the input to the model.
-
-    Counts the prompt text (or list items) plus the stringified message
-    history.  Binary/URL attachments aren't tokenized as text so they're
-    skipped.
-    """
-    char_count = 0
-
-    if isinstance(prompt, str):
-        char_count += len(prompt)
-    elif isinstance(prompt, list):
-        for item in prompt:
-            if isinstance(item, str):
-                char_count += len(item)
-
-    try:
-        history = agent._message_history
-        from code_puppy.agents._history import stringify_part
-
-        for msg in history:
-            for part in getattr(msg, "parts", []) or []:
-                part_str = stringify_part(part)
-                if part_str:
-                    char_count += len(part_str)
-    except Exception:
-        pass
-
-    return char_count
-
-
-async def _patched_run_with_mcp(
-    agent: Any,
-    prompt: str,
-    *,
-    attachments: Optional[Any] = None,
-    link_attachments: Optional[Any] = None,
-    output_type: Optional[Any] = None,
-    **kwargs: Any,
-) -> Any:
-    """Wrap ``run_with_mcp`` to learn token ratios from real API responses."""
-    global _original_run_with_mcp
-
-    input_char_count = _compute_input_char_count(
-        agent, prompt, attachments, link_attachments
-    )
-
-    result = await _original_run_with_mcp(
-        agent,
-        prompt,
-        attachments=attachments,
-        link_attachments=link_attachments,
-        output_type=output_type,
-        **kwargs,
-    )
-
-    # Try to extract token usage from the result
-    try:
-        usage = getattr(result, "usage", None)
-        # pydantic-ai 2.31.0 exposes usage as a property. A callable value
-        # belongs to an older method-style host; ignore it rather than
-        # triggering the deprecated call path.
-        if callable(usage):
-            return result
-        input_tokens = getattr(usage, "input_tokens", 0) or getattr(
-            usage, "request_tokens", 0
-        )
-        if input_tokens > 0 and input_char_count > 0:
-            from code_puppy_core_plugins.token_ratio_learner.ratios import (
-                _record_token_ratio,
-            )
-
-            model = agent.get_model_name()
-            if model:
-                _record_token_ratio(model, input_char_count, input_tokens)
-    except Exception:
-        pass  # Never let ratio recording break the agent run
-
-    return result
-
-
-# Patched subagent ``_estimate_tokens``
-
-
-def _patched_subagent_estimate_tokens(content: str) -> int:
-    """Drop-in replacement for subagent token estimation."""
-    from code_puppy_core_plugins.token_ratio_learner.ratios import count_tokens
-
-    return count_tokens(content, model=None)
-
-
-# Startup hook
-
-
-def _on_startup() -> None:
-    """Apply all monkeypatches on startup."""
-    global _original_estimate_tokens, _original_estimate_tokens_for_message
-    global _original_run_with_mcp, _original_subagent_estimate_tokens
-
-    # 1. Patch _history.estimate_tokens (root text-level)
-    from code_puppy.agents import _history
-
-    _original_estimate_tokens = _history.estimate_tokens
-    _history.estimate_tokens = _patched_estimate_tokens
-    logger.info("token_ratio_learner: patched _history.estimate_tokens")
-
-    # 2. Patch _history.estimate_tokens_for_message
-    _original_estimate_tokens_for_message = _history.estimate_tokens_for_message
-    _history.estimate_tokens_for_message = _patched_estimate_tokens_for_message
-    logger.info("token_ratio_learner: patched _history.estimate_tokens_for_message")
-
-    # 3. Patch _runtime.run_with_mcp
-    from code_puppy.agents import _runtime
-
-    _original_run_with_mcp = _runtime.run_with_mcp
-    _runtime.run_with_mcp = _patched_run_with_mcp
-    logger.info("token_ratio_learner: patched _runtime.run_with_mcp")
-
-    # 4. Patch subagent_stream_handler._estimate_tokens
-    # NOTE: ``subagent_stream_handler`` re-exports a function, not its module;
-    # import the actual module via ``importlib`` before patching.
-    import importlib
-
-    ssh_mod = importlib.import_module("code_puppy.agents.subagent_stream_handler")
-
-    _original_subagent_estimate_tokens = ssh_mod._estimate_tokens
-    ssh_mod._estimate_tokens = _patched_subagent_estimate_tokens
-    logger.info("token_ratio_learner: patched subagent_stream_handler._estimate_tokens")
-
-
-# Register the startup callback
-register_callback("startup", _on_startup)