geny-executor 2.41.0__tar.gz → 2.42.0__tar.gz

{geny_executor-2.41.0 → geny_executor-2.42.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: geny-executor
-Version: 2.41.0
+Version: 2.42.0
 Summary: Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API
 Project-URL: Homepage, https://github.com/CocoRoF/geny-executor
 Project-URL: Repository, https://github.com/CocoRoF/geny-executor

{geny_executor-2.41.0 → geny_executor-2.42.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "geny-executor"
-version = "2.41.0"
+version = "2.42.0"
 description = "Harness-engineered agent pipeline library with 21-stage dual-abstraction architecture, built on the Anthropic API"
 readme = "README.md"
 license = "Apache-2.0"

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/core/environment.py

@@ -80,6 +80,16 @@ class ToolsSnapshot:
     manifest only records *which provider-backed names are active* for
     this environment. The pipeline resolves each name against the
     ``adhoc_providers`` passed to :meth:`Pipeline.from_manifest`.
+
+    ``core_overrides`` (2.42.0) flips individual tools between *core*
+    (schema shipped to the LLM on every request) and *deferred*
+    (registered + dispatchable, but only discoverable at runtime via
+    the ``ToolSearch`` built-in). Defaults when a name is absent:
+    framework built-ins are core, everything else (external / provider
+    / MCP tools) is deferred. Keys are exact tool names; a trailing
+    ``*`` matches by prefix (e.g. ``"mcp__github__*": true`` promotes a
+    whole MCP server whose tool names are only known after discovery).
+    Exact keys win over wildcard keys.
     """
 
     built_in: List[str] = field(default_factory=list)
@@ -87,6 +97,7 @@ class ToolsSnapshot:
     mcp_servers: List[Dict[str, Any]] = field(default_factory=list)
     external: List[str] = field(default_factory=list)
     scope: Dict[str, Any] = field(default_factory=dict)
+    core_overrides: Dict[str, bool] = field(default_factory=dict)
 
     def to_dict(self) -> Dict[str, Any]:
         return {
@@ -95,6 +106,7 @@ class ToolsSnapshot:
             "mcp_servers": list(self.mcp_servers),
             "external": list(self.external),
             "scope": dict(self.scope),
+            "core_overrides": dict(self.core_overrides),
         }
 
     @classmethod
@@ -105,6 +117,9 @@ class ToolsSnapshot:
             mcp_servers=data.get("mcp_servers", []),
             external=data.get("external", []),
             scope=data.get("scope", {}),
+            core_overrides={
+                str(k): bool(v) for k, v in (data.get("core_overrides") or {}).items()
+            },
         )
 
 

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/core/pipeline.py

@@ -444,6 +444,64 @@ class DroppedStage:
     error: str
 
 
+def _resolve_core_flag(
+    name: str,
+    overrides: Mapping[str, bool],
+    default: bool,
+) -> bool:
+    """Resolve a tool's core/deferred exposure from ``manifest.tools.core_overrides``.
+
+    Exact-name keys win; otherwise the longest matching trailing-``*``
+    prefix key applies (``"mcp__github__*"`` — MCP tool names are only
+    known after discovery, so per-server toggles need the wildcard);
+    otherwise *default* (built-ins: core, everything else: deferred).
+    """
+    if name in overrides:
+        return bool(overrides[name])
+    best_len = -1
+    best_val = default
+    for key, val in overrides.items():
+        if key.endswith("*") and name.startswith(key[:-1]) and len(key) > best_len:
+            best_len = len(key)
+            best_val = bool(val)
+    return best_val
+
+
+def _core_overrides_from_manifest(manifest: "EnvironmentManifest") -> Dict[str, bool]:
+    """Read ``manifest.tools.core_overrides`` defensively (old manifests lack it)."""
+    raw = getattr(manifest.tools, "core_overrides", None) or {}
+    return {str(k): bool(v) for k, v in raw.items()} if isinstance(raw, Mapping) else {}
+
+
+def _ensure_tool_search_reachable(registry: "ToolRegistry") -> None:
+    """Guarantee deferred tools stay discoverable.
+
+    A registry that holds deferred (non-core) tools without an exposed
+    ``ToolSearch`` would strand them — the LLM could never learn they
+    exist. Auto-register the built-in ``ToolSearch`` as core in that
+    case, and force it back to core if a manifest override demoted it
+    while deferred tools remain.
+    """
+    if not registry.list_deferred():
+        return
+    existing = registry.get("ToolSearch")
+    if existing is None:
+        from geny_executor.tools.built_in.tool_search_tool import ToolSearchTool
+
+        registry.register(ToolSearchTool(), core=True)
+        logger.info(
+            "ToolSearch auto-registered (core): %d deferred tool(s) need a discovery path",
+            len(registry.list_deferred()),
+        )
+    elif not registry.is_core("ToolSearch"):
+        registry.set_core("ToolSearch", True)
+        logger.warning(
+            "ToolSearch forced back to core — %d deferred tool(s) would be "
+            "unreachable with it deferred/demoted",
+            len(registry.list_deferred()),
+        )
+
+
 def _register_built_in_tools(
     manifest: "EnvironmentManifest",
     registry: "ToolRegistry",
@@ -484,6 +542,8 @@ def _register_built_in_tools(
     if names == ["*"]:
         names = list(BUILT_IN_TOOL_CLASSES.keys())
 
+    core_overrides = _core_overrides_from_manifest(manifest)
+
     for name in names:
         cls = BUILT_IN_TOOL_CLASSES.get(name)
         if cls is None:
@@ -499,7 +559,9 @@ def _register_built_in_tools(
             if report is not None:
                 report.shadowed.append(name)
             continue
-        registry.register(cls())
+        # Framework built-ins are core (upfront schema) by default;
+        # manifest.tools.core_overrides can defer individual names.
+        registry.register(cls(), core=_resolve_core_flag(name, core_overrides, True))
         if report is not None:
             report.resolved.append(name)
 
@@ -541,6 +603,8 @@ def _register_external_tools(
     if not raw_entries:
         return
 
+    core_overrides = _core_overrides_from_manifest(manifest)
+
     # Normalize entries → (name, required). Malformed entries warn and
     # count as unresolved-but-unnamed; they cannot be required because
     # we cannot even tell what the author wanted.
@@ -627,7 +691,13 @@ def _register_external_tools(
                 # intentionally hardened by the host).
                 report.shadowed.append(name)
             report.resolved.append(name)
-        registry.register(tool)
+        # Host-supplied (external) tools are deferred by default — the
+        # LLM discovers them via ToolSearch. core_overrides opts a name
+        # into upfront exposure; an external entry shadowing a framework
+        # built-in keeps the built-in's core default so the replacement
+        # is invisible to the model.
+        default_core = registry.is_core(name) if registry.get(name) is not None else False
+        registry.register(tool, core=_resolve_core_flag(name, core_overrides, default_core))
 
 
 def _gate_unconfigured_tools(
@@ -1248,6 +1318,9 @@ class Pipeline:
         # (no-op when satisfied_config is None). Runs after both registration
         # passes so it sees the full built-in + external surface.
         _gate_unconfigured_tools(registry, satisfied_config, resolution_report)
+        # Deferred tools need a live discovery path; from_manifest_async
+        # re-checks after providers + MCP land more registrations.
+        _ensure_tool_search_reachable(registry)
         pipeline.tool_resolution_report = resolution_report
         pipeline._tool_registry = registry
         # Retain the adhoc providers so the self-modifying-environment controller
@@ -1430,9 +1503,16 @@ class Pipeline:
         # MCP adapters arrive. Name collisions at this layer are logged
         # by ``register_providers``; MCP registrations would still fail
         # cleanly via the registry's own dedupe if they conflicted.
+        # Provider-shipped tools are deferred by default (ToolSearch
+        # discovery); manifest.tools.core_overrides opts names in.
+        core_overrides = _core_overrides_from_manifest(manifest)
         started_providers: List["ToolProvider"] = []
         if tool_providers:
-            started_providers = await register_providers(list(tool_providers), registry)
+            started_providers = await register_providers(
+                list(tool_providers),
+                registry,
+                core_resolver=lambda name: _resolve_core_flag(name, core_overrides, False),
+            )
 
         manager = MCPManager()
 
@@ -1481,8 +1561,15 @@ class Pipeline:
                 try:
                     await manager.connect_all(configs)
                     adapters = await manager.discover_all()
+                    # MCP tools are deferred by default (ToolSearch
+                    # discovery); core_overrides opts names in — the
+                    # trailing-* form covers whole servers whose tool
+                    # names are only known after discovery.
                     for adapter in adapters:
-                        registry.register(adapter)
+                        registry.register(
+                            adapter,
+                            core=_resolve_core_flag(adapter.name, core_overrides, False),
+                        )
                 except BaseException:
                     # Unwind providers if MCP bring-up fails mid-flight so no
                     # half-started resources leak out.
@@ -1492,6 +1579,10 @@ class Pipeline:
                     await manager.disconnect_all()
                     raise
 
+        # Providers + MCP may have added deferred tools after the sync
+        # from_manifest pass — re-check the discovery path.
+        _ensure_tool_search_reachable(registry)
+
         pipeline._mcp_manager = manager
         pipeline._tool_registry = registry
         pipeline._tool_providers = started_providers

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/stages/s03_system/artifact/default/stage.py

@@ -164,14 +164,24 @@ class SystemStage(Stage[Any, Any]):
         # Register tools in state if registry provided. Snapshotted on the first
         # turn, then rebuilt only when the live registry's version moves — so a
         # tool/skill enabled/disabled/created mid-session (self-modifying
-        # environment), or an MCP re-seed, takes effect on the next turn. The
-        # version guard keeps the steady-state cost at one int compare per turn.
+        # environment), an MCP re-seed, or a ToolSearch activation takes effect
+        # on the next iteration. The version guard keeps the steady-state cost
+        # at one int compare per turn.
+        #
+        # Only *exposed* tools ship to the model: core tools plus deferred
+        # tools a ToolSearch hit activated. Deferred tools stay registered
+        # (Stage 10 can still dispatch them) but their schemas stay out of
+        # the request payload until discovered — that's the token contract.
         if self._tool_registry is not None:
             reg_version = getattr(self._tool_registry, "version", None)
             if not state.tools or (
                 reg_version is not None and reg_version != state.tools_version
             ):
-                state.tools = self._tool_registry.to_api_format()
+                try:
+                    state.tools = self._tool_registry.to_api_format(exposed_only=True)
+                except TypeError:
+                    # Host passed a registry-alike without exposure support.
+                    state.tools = self._tool_registry.to_api_format()
                 if reg_version is not None:
                     state.tools_version = reg_version
 

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/stages/s10_tool/artifact/default/stage.py

@@ -208,6 +208,9 @@ class ToolStage(Stage[Any, Any]):
             # can see the live tool descriptors + shared state. Tools
             # MUST NOT mutate — use state_mutations / state_apply instead.
             state_view=state,
+            # The live registry — ToolSearch searches the FULL catalogue
+            # through it (deferred tools included) and activates matches.
+            tool_registry=self._registry,
             # Phase 5: propagate the host-attached HookRunner so the
             # router can fire PRE_TOOL_USE / POST_TOOL_USE / POST_TOOL_
             # FAILURE around dispatch. ``None`` is the default no-op.

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/tools/base.py

@@ -157,6 +157,12 @@ class ToolContext:
     # means the host did not enable self-modification. Typed ``Any`` to avoid a
     # core import cycle.
     environment: Optional[Any] = None
+    # The live ToolRegistry backing this session (set by Stage 10). ToolSearch
+    # reads the FULL catalogue from it — including deferred tools whose schemas
+    # are not in ``state.tools`` — and promotes matches via ``activate()``.
+    # ``None`` (default) preserves the state_view/built-in fallback path.
+    # Typed ``Any`` to avoid importing the registry from this base module.
+    tool_registry: Optional[Any] = None
     extras: Dict[str, Any] = field(default_factory=dict)
 
 

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/tools/built_in/tool_search_tool.py

@@ -1,23 +1,35 @@
-"""ToolSearch — discovery helper for the tools available this turn.
+"""ToolSearch — deferred-tool discovery + activation.
 
 Cycle 20260424 executor uplift — Phase 3 Week 6.
-
-When a pipeline has dozens of tools wired up (executor built-ins + MCP
-servers + skill bundles + host-specific custom tools) the LLM often
-needs help finding the right one. ``ToolSearch`` is a lightweight
-introspection tool — pass it a query, get back a ranked list of
-matching tool names + their one-line descriptions.
-
-Source of truth:
-
-1. If ``ToolContext.state_view`` is set (the usual Stage 10 path), we
-   read ``state_view.tools`` — the live list of API-format tool
-   descriptors the LLM already sees. This includes MCP tools, custom
-   host tools, everything.
-2. Otherwise (e.g. a test harness running a bare executor without a
-   Stage), we fall back to the built-in catalogue
-   (``BUILT_IN_TOOL_CLASSES``) so the tool still produces useful
-   output in isolation.
+2.42.0 — upgraded from a list-introspection helper to the discovery
+half of the core/deferred tool contract.
+
+The pipeline no longer ships every tool schema to the LLM upfront.
+*Core* tools (framework built-ins by default, plus anything the
+manifest's ``tools.core_overrides`` promotes) are always in the
+request payload; everything else — host/external tools, provider
+bundles, MCP adapters — is registered but *deferred*: dispatchable,
+yet invisible to the model until discovered here.
+
+``ToolSearch`` is that discovery path. Pass it a keyword query and it:
+
+1. Searches the FULL catalogue via ``ToolContext.tool_registry``
+   (Stage 10 binds the live :class:`ToolRegistry`), deferred tools
+   included.
+2. Ranks matches (name > description > schema, see :func:`_rank`).
+3. **Activates** every deferred match — ``registry.activate(name)``
+   bumps the registry version, so Stage 3 rebuilds ``state.tools`` on
+   the next loop iteration and the activated schemas reach the model
+   on its very next step within the same turn.
+
+Fallback sources when no registry handle is bound (bare harnesses,
+older hosts):
+
+1. ``ToolContext.state_view.tools`` — the live API-format descriptors
+   the LLM already sees (pre-2.42 behaviour; search only, nothing to
+   activate).
+2. The built-in catalogue (``BUILT_IN_TOOL_CLASSES``) so the tool
+   still produces useful output in isolation.
 
 Ranking is simple: a hit on the tool ``name`` beats a hit on the
 ``description``; exact name match beats substring; case-insensitive
@@ -34,16 +46,16 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from geny_executor.tools.base import Tool, ToolCapabilities, ToolContext, ToolResult
 
-_DEFAULT_LIMIT = 20
+_DEFAULT_LIMIT = 10
 _HARD_LIMIT = 100
 
 
 def _fallback_descriptors() -> List[Dict[str, Any]]:
     """Build descriptors from the built-in catalogue.
 
-    Only used when ``state_view`` is absent. Mirrors the shape of the
-    Anthropic API tool descriptor so rank logic can treat both sources
-    the same.
+    Only used when neither ``tool_registry`` nor ``state_view`` is
+    available. Mirrors the shape of the Anthropic API tool descriptor
+    so rank logic can treat every source the same.
     """
     # Local import to avoid the package-init circular dependency.
     from geny_executor.tools.built_in import BUILT_IN_TOOL_CLASSES
@@ -120,17 +132,19 @@ def _rank(descriptor: Dict[str, Any], query: str) -> int:
 
 
 class ToolSearchTool(Tool):
-    """Find tools matching a keyword query.
+    """Discover tools from the full catalogue — and activate deferred ones.
 
-    Usage: the LLM calls ``ToolSearch({"query": "grep files"})`` and
+    Usage: the LLM calls ``ToolSearch({"query": "spreadsheet"})`` and
     gets back something like::
 
-        Matching 2 tools for 'grep files':
-        1. Grep — Search file contents with ripgrep regex syntax.
-        2. Glob — Match file paths with shell globbing.
+        Matching 2 tool(s) for 'spreadsheet':
+        1. xlsx_edit — Edit XLSX workbooks in place. [activated]
+        2. Read — Read a file from the filesystem. [available]
+        Activated tool schemas become available on your next step.
 
-    The host decides which tools to include by deciding what to put on
-    ``state.tools`` each turn — this tool just surfaces that list.
+    Matches already exposed to the model are tagged ``[available]``;
+    deferred matches are tagged ``[activated]`` — their full schemas
+    enter the tool list on the next loop iteration.
     """
 
     @property
@@ -140,9 +154,11 @@ class ToolSearchTool(Tool):
     @property
     def description(self) -> str:
         return (
-            "Find tools available this turn by keyword. Returns ranked "
-            "matches with name + description. Useful when unsure which "
-            "tool to call for a task."
+            "Search the full tool catalog by keyword — it contains more "
+            "tools than your current tool list. Matching tools that are "
+            "not yet in your tool list are activated automatically and "
+            "become callable on your next step. Use this whenever no "
+            "visible tool fits the task."
         )
 
     @property
@@ -155,15 +171,18 @@ class ToolSearchTool(Tool):
                     "description": (
                         "Keyword query. Multi-word queries require every "
                         "token to match somewhere (name / description / "
-                        "input schema)."
+                        "input schema). An exact tool name is the most "
+                        "precise query."
                     ),
                     "minLength": 1,
                 },
                 "limit": {
                     "type": "integer",
                     "description": (
-                        f"Maximum number of matches to return. Default "
-                        f"{_DEFAULT_LIMIT}, hard cap {_HARD_LIMIT}."
+                        f"Maximum number of matches to return (and thus to "
+                        f"activate). Default {_DEFAULT_LIMIT}, hard cap "
+                        f"{_HARD_LIMIT}. Keep it small to avoid bloating "
+                        f"your tool list."
                     ),
                     "exclusiveMinimum": 0,
                 },
@@ -186,7 +205,8 @@ class ToolSearchTool(Tool):
         limit = int(input.get("limit", _DEFAULT_LIMIT))
         limit = max(1, min(_HARD_LIMIT, limit))
 
-        descriptors = self._collect_descriptors(context)
+        registry = getattr(context, "tool_registry", None)
+        descriptors = self._collect_descriptors(context, registry)
         ranked: List[Tuple[int, Dict[str, Any]]] = []
         for desc in descriptors:
             score = _rank(desc, query)
@@ -206,13 +226,31 @@ class ToolSearchTool(Tool):
                 },
             )
 
+        # Promote deferred matches: activate() bumps the registry
+        # version, Stage 3 rebuilds state.tools next iteration, and the
+        # schemas reach the model on its next step within this turn.
+        activated: List[str] = []
+        if registry is not None:
+            for _, desc in top:
+                name = str(desc.get("name", ""))
+                if desc.get("_deferred") and self._activate(registry, name):
+                    activated.append(name)
+
         lines = [f"Matching {len(top)} tool(s) for {query!r}:"]
         for i, (score, desc) in enumerate(top, 1):
             name = desc.get("name", "?")
             d = str(desc.get("description", "")).strip()
             # Keep descriptions on one line for easy scanning.
             one_liner = d.splitlines()[0] if d else "(no description)"
-            lines.append(f"{i}. {name} — {one_liner}")
+            tag = ""
+            if registry is not None:
+                tag = " [activated]" if name in activated else " [available]"
+            lines.append(f"{i}. {name} — {one_liner}{tag}")
+        if activated:
+            lines.append(
+                "Activated tool schemas become available on your next step — "
+                "call them then."
+            )
 
         return ToolResult(
             content="\n".join(lines),
@@ -223,18 +261,51 @@ class ToolSearchTool(Tool):
                     {"name": d.get("name"), "score": s, "description": d.get("description")}
                     for s, d in top
                 ],
+                "activated": activated,
                 "searched": len(descriptors),
             },
         )
 
-    def _collect_descriptors(self, context: ToolContext) -> List[Dict[str, Any]]:
-        """Prefer live pipeline tools (via ``state_view``), fall back to built-ins.
-
-        The ``state_view`` path is what Stage 10 configures at runtime —
-        it reflects MCP servers, custom tools, skills, everything. The
-        fallback is just a graceful default for test harnesses that run
-        executors without a Stage wrapper.
+    @staticmethod
+    def _activate(registry: Any, name: str) -> bool:
+        """Best-effort ``registry.activate(name)`` — never breaks the search."""
+        activate = getattr(registry, "activate", None)
+        if not callable(activate):
+            return False
+        try:
+            return bool(activate(name))
+        except Exception:
+            return False
+
+    def _collect_descriptors(
+        self, context: ToolContext, registry: Optional[Any]
+    ) -> List[Dict[str, Any]]:
+        """Prefer the live registry (full catalogue), then ``state_view``, then built-ins.
+
+        The registry path is what Stage 10 configures at runtime — it
+        covers MCP tools, custom host tools, skills, everything,
+        *including deferred tools whose schemas the model has not seen*.
+        Each descriptor is annotated with a private ``_deferred`` flag
+        so ``execute`` knows which matches need activation. The
+        ``state_view`` path is the pre-2.42 behaviour (exposed tools
+        only); the built-in catalogue is a graceful default for test
+        harnesses running executors without a Stage wrapper.
         """
+        if registry is not None:
+            try:
+                out: List[Dict[str, Any]] = []
+                for tool in registry.list_all():
+                    desc = tool.to_api_format()
+                    if isinstance(desc, dict):
+                        is_exposed = getattr(registry, "is_exposed", None)
+                        desc["_deferred"] = (
+                            not bool(is_exposed(tool.name)) if callable(is_exposed) else False
+                        )
+                        out.append(desc)
+                if out:
+                    return out
+            except Exception:
+                pass  # fall through to the view/built-in sources
         view: Optional[Any] = getattr(context, "state_view", None)
         if view is not None:
             tools = getattr(view, "tools", None)

{geny_executor-2.41.0 → geny_executor-2.42.0}/src/geny_executor/tools/provider.py

@@ -38,7 +38,7 @@ from __future__ import annotations
 
 import logging
 from abc import ABC, abstractmethod
-from typing import Iterable, List, Optional
+from typing import Callable, Iterable, List, Optional
 
 from geny_executor.tools.base import Tool
 
@@ -151,6 +151,8 @@ class BuiltInToolProvider(ToolProvider):
 async def register_providers(
     providers: List[ToolProvider],
     registry: "ToolRegistry",  # type: ignore[name-defined]  # noqa: F821
+    *,
+    core_resolver: Optional[Callable[[str], bool]] = None,
 ) -> List[ToolProvider]:
     """Start each provider and register its tools into ``registry``.
 
@@ -165,6 +167,12 @@ async def register_providers(
           is logged at WARNING but not rejected. This keeps a partial
           build usable instead of failing the whole pipeline.
 
+    ``core_resolver`` maps a tool name to its core flag (upfront schema
+    exposure vs ToolSearch-deferred). ``None`` keeps the pre-2.42
+    behaviour: everything registers as core. The manifest build path
+    passes a resolver that defaults provider tools to deferred and
+    applies ``manifest.tools.core_overrides``.
+
     Returns the list of providers that were successfully started, in
     the order they were registered — callers can pass this to
     :func:`shutdown_providers` to unwind cleanly.
@@ -196,7 +204,8 @@ async def register_providers(
                         tool.name,
                     )
                     continue
-                registry.register(tool)
+                core = True if core_resolver is None else bool(core_resolver(tool.name))
+                registry.register(tool, core=core)
     except BaseException:
         await shutdown_providers(started)
         raise