agently-devtools 0.0.1a0__tar.gz

agently_devtools-0.0.1a0/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.2
+Name: agently-devtools
+Version: 0.0.1a0
+Summary: Observation and developer tooling companion package for Agently
+Author-email: Agently Team <developer@agently.tech>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: agently<4.1.0,>=4.0.9
+Requires-Dist: fastapi<1.0,>=0.104
+Requires-Dist: httpx<0.29.0,>=0.28.1
+Requires-Dist: uvicorn<1.0,>=0.30
+Provides-Extra: dev
+Requires-Dist: anyio<5,>=4; extra == "dev"
+Requires-Dist: build<2.0,>=1.2; extra == "dev"
+Requires-Dist: pytest<9.0.0,>=8.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio<2.0.0,>=1.0.0; extra == "dev"
+
+# agently-devtools
+
+`agently-devtools` is the local observation, evaluation, and playground companion package for `agently`.
+
+## Install
+
+```bash
+pip install -U agently agently-devtools
+```
+
+Python `>=3.10`
+
+## Start
+
+```bash
+agently-devtools start
+```
+
+Default local address:
+
+- Console: `http://127.0.0.1:15596/`
+- Ingest: `http://127.0.0.1:15596/observation/ingest`
+
+## Observation Bridge
+
+```python
+import os
+
+from agently import Agently
+from agently_devtools import ObservationBridge
+
+bridge = ObservationBridge(
+    os.environ["AGENTLY_DEVTOOLS_INGEST_URL"],
+    app_id="your_app_id",
+    group_id="your_group_id",
+)
+bridge.register(Agently.event_center)
+```
+
+TriggerFlow executions automatically publish their flow definition, so the execution graph can show the full static structure, including branches that have not been hit yet.
+
+## Scenario Evaluations
+
+```python
+import os
+
+from agently_devtools import EvaluationBridge, EvaluationCase, EvaluationRunner
+
+bridge = EvaluationBridge(
+    base_url=os.environ["AGENTLY_DEVTOOLS_BASE_URL"],
+    app_id="your_app_id",
+    group_id="your_group_id",
+)
+
+binding = bridge.bind_agent(
+    agent,
+    suite_id="your_suite_id",
+    target_name="your_target_name",
+)
+
+report = EvaluationRunner(bridge=bridge).run(
+    binding,
+    cases=[EvaluationCase(case_id="case-1", input="your scenario")],
+    rounds=3,
+)
+```

agently_devtools-0.0.1a0/README.md

@@ -0,0 +1,66 @@
+# agently-devtools
+
+`agently-devtools` is the local observation, evaluation, and playground companion package for `agently`.
+
+## Install
+
+```bash
+pip install -U agently agently-devtools
+```
+
+Python `>=3.10`
+
+## Start
+
+```bash
+agently-devtools start
+```
+
+Default local address:
+
+- Console: `http://127.0.0.1:15596/`
+- Ingest: `http://127.0.0.1:15596/observation/ingest`
+
+## Observation Bridge
+
+```python
+import os
+
+from agently import Agently
+from agently_devtools import ObservationBridge
+
+bridge = ObservationBridge(
+    os.environ["AGENTLY_DEVTOOLS_INGEST_URL"],
+    app_id="your_app_id",
+    group_id="your_group_id",
+)
+bridge.register(Agently.event_center)
+```
+
+TriggerFlow executions automatically publish their flow definition, so the execution graph can show the full static structure, including branches that have not been hit yet.
+
+## Scenario Evaluations
+
+```python
+import os
+
+from agently_devtools import EvaluationBridge, EvaluationCase, EvaluationRunner
+
+bridge = EvaluationBridge(
+    base_url=os.environ["AGENTLY_DEVTOOLS_BASE_URL"],
+    app_id="your_app_id",
+    group_id="your_group_id",
+)
+
+binding = bridge.bind_agent(
+    agent,
+    suite_id="your_suite_id",
+    target_name="your_target_name",
+)
+
+report = EvaluationRunner(bridge=bridge).run(
+    binding,
+    cases=[EvaluationCase(case_id="case-1", input="your scenario")],
+    rounds=3,
+)
+```

agently_devtools-0.0.1a0/agently_devtools/__init__.py

@@ -0,0 +1,21 @@
+from .app import DEFAULT_LOCAL_API_KEY, LocalObservationBinding, create_local_observation_app
+from .bridge import ObservationBridge
+from .evaluations import (
+    EvaluationBinding,
+    EvaluationBridge,
+    EvaluationCase,
+    EvaluationCheckResult,
+    EvaluationReport,
+    EvaluationRoundRecord,
+    EvaluationRule,
+    EvaluationRunner,
+    EvaluationVariant,
+)
+from .logs import LogEventQuery, RuntimeLogService, RuntimeLogStats
+from .observation import ObservationEventQuery, ObservationRunQuery, ObservationSubscriptionFilter, RunTreeNode
+from .observation import (
+    RunRecord,
+    RuntimeObservationBackend,
+    RuntimeObservationService,
+    RuntimeObservationSubscription,
+)

agently_devtools-0.0.1a0/agently_devtools/_agently_compat.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    class RunContext:
+        run_id: str
+        root_run_id: str | None
+        parent_run_id: str | None
+        run_kind: str
+        agent_id: str | None
+        agent_name: str | None
+        session_id: str | None
+        response_id: str | None
+        execution_id: str | None
+        meta: dict[str, Any]
+
+        @classmethod
+        def create(cls, *args: Any, **kwargs: Any) -> "RunContext": ...
+
+        def create_child(self, *args: Any, **kwargs: Any) -> "RunContext": ...
+
+        def model_copy(self, *args: Any, **kwargs: Any) -> "RunContext": ...
+
+        def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ...
+
+
+    class RuntimeEvent:
+        event_id: str
+        event_type: str
+        source: str
+        level: str
+        message: str | None
+        payload: Any
+        error: Any
+        run: RunContext | None
+        meta: dict[str, Any]
+        timestamp: int
+
+        def __init__(self, *args: Any, **kwargs: Any) -> None: ...
+
+        def model_copy(self, *args: Any, **kwargs: Any) -> "RuntimeEvent": ...
+
+        def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]: ...
+
+        @classmethod
+        def model_validate(cls, *args: Any, **kwargs: Any) -> "RuntimeEvent": ...
+
+else:
+    try:
+        from agently.types.data import RunContext, RuntimeEvent
+    except ImportError:  # Compatibility for published agently wheel variants.
+        from agently.types.data.event import RunContext, RuntimeEvent
+
+__all__ = ["RunContext", "RuntimeEvent"]

agently_devtools-0.0.1a0/agently_devtools/app.py

@@ -0,0 +1,197 @@
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, cast
+from urllib.parse import urlparse
+
+from agently import Agently
+
+from ._agently_compat import RunContext
+from .bridge import _create_observation_service_hooker
+from .integrations import ObservationFastAPI
+from .observation import RuntimeObservationService
+
+if TYPE_CHECKING:
+    from agently.core import EventCenter
+    from agently.types.plugins import EventHooker
+
+DEFAULT_LOCAL_API_KEY = ""
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 15596
+DEFAULT_ROUTE_PREFIX = "/observation"
+DEFAULT_LOG_ROOT = Path.home() / ".agently-devtools" / "logs"
+_LOCAL_SERVER_LOCK = threading.Lock()
+_LOCAL_SERVER_THREADS: dict[tuple[str, int, str], threading.Thread] = {}
+
+
+def _ensure_local_auth(api_key: str | None):
+    if Agently.settings.get("observation.buffer_limit", None) is None:
+        Agently.set_settings("observation.buffer_limit", 1200)
+    if Agently.settings.get("observation.channel_buffer_limit", None) is None:
+        Agently.set_settings("observation.channel_buffer_limit", 160)
+    if Agently.settings.get("logs.enabled", None) is None:
+        Agently.set_settings("logs.enabled", True)
+    if Agently.settings.get("logs.storage.path", None) is None:
+        Agently.set_settings("logs.storage.path", str(DEFAULT_LOG_ROOT))
+    if Agently.settings.get("logs.storage.segment_event_limit", None) is None:
+        Agently.set_settings("logs.storage.segment_event_limit", 1000)
+    if isinstance(api_key, str) and api_key:
+        cast(Any, Agently).set_api_key(api_key)
+        Agently.set_settings("observation.api.auth.api_keys", [api_key])
+    else:
+        Agently.set_settings("observation.api.auth.api_keys", [])
+    Agently.set_settings("observation.api.auth.allow_query_api_key", True)
+    return Agently.settings
+
+
+async def emit_startup_run(
+    *,
+    source: str = "AgentlyDevtoolsCLI",
+    message: str = "Observation service is ready.",
+):
+    run = RunContext.create(
+        run_kind="request",
+        agent_name="agently-devtools",
+        meta={"component": "devtools"},
+    )
+    emitter = cast(Any, Agently.event_center).create_emitter(source)
+    await emitter.async_emit(
+        "observation.service_started",
+        message=message,
+        run=run,
+        payload={"component": "observation"},
+    )
+    await emitter.async_emit(
+        "observation.service_ready",
+        message="Observation console is listening for remote runtime uploads.",
+        run=run,
+        payload={"component": "observation"},
+    )
+
+
+class LocalObservationBinding:
+    def __init__(self, observation_service: RuntimeObservationService, hooker: type["EventHooker"]):
+        self.observation_service = observation_service
+        self._hooker = hooker
+
+    def unregister(self, event_center: "EventCenter"):
+        event_center.unregister_hooker_plugin(self._hooker)
+        return self
+
+    @property
+    def hooker(self):
+        return self._hooker
+
+
+def create_local_observation_app(
+    *,
+    api_key: str | None = DEFAULT_LOCAL_API_KEY,
+    route_prefix: str = DEFAULT_ROUTE_PREFIX,
+    title: str = "Agently DevTools",
+    fastapi_kwargs: dict[str, Any] | None = None,
+    register_event_sink: bool = True,
+):
+    settings = _ensure_local_auth(api_key)
+    observation_service = RuntimeObservationService(settings)
+    hooker = _create_observation_service_hooker(observation_service, name="LocalObservationSinkHooker")
+    if register_event_sink:
+        Agently.event_center.register_hooker_plugin(hooker)
+    binding = LocalObservationBinding(observation_service, hooker)
+    kwargs = dict(fastapi_kwargs) if fastapi_kwargs is not None else {}
+    app = ObservationFastAPI(
+        observation_service=observation_service,
+        settings=settings,
+        api_keys=[api_key] if isinstance(api_key, str) and api_key else None,
+        route_prefix=route_prefix,
+        title=title,
+        **kwargs,
+    )
+    return app, binding
+
+
+def build_local_base_url(*, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT):
+    return f"http://{host}:{port}"
+
+
+def build_observation_ingest_endpoint(*, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, route_prefix: str = DEFAULT_ROUTE_PREFIX):
+    return f"{build_local_base_url(host=host, port=port)}{route_prefix}/ingest"
+
+
+def _is_local_base_url(base_url: str):
+    parsed = urlparse(base_url)
+    return parsed.scheme in {"http", "https"} and parsed.hostname in {"127.0.0.1", "localhost"}
+
+
+def _probe_local_listener(base_url: str, *, route_prefix: str = DEFAULT_ROUTE_PREFIX, api_key: str | None = None, timeout: float = 0.35):
+    try:
+        import httpx
+
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
+        response = httpx.get(f"{base_url}{route_prefix}/runs", headers=headers, timeout=timeout)
+        return response.status_code < 500
+    except Exception:
+        return False
+
+
+def _serve_local_listener(*, host: str, port: int, api_key: str | None, route_prefix: str):
+    import uvicorn
+
+    app, _binding = create_local_observation_app(
+        api_key=api_key,
+        route_prefix=route_prefix,
+        register_event_sink=False,
+    )
+    config = uvicorn.Config(
+        app,
+        host=host,
+        port=port,
+        log_level="warning",
+        access_log=False,
+    )
+    server = uvicorn.Server(config)
+    cast(Any, server).install_signal_handlers = lambda: None
+    server.run()
+
+
+def ensure_local_devtools_listener(
+    base_url: str,
+    *,
+    api_key: str | None = None,
+    route_prefix: str = DEFAULT_ROUTE_PREFIX,
+    wait_timeout: float = 5.0,
+):
+    if not _is_local_base_url(base_url):
+        return False
+    if _probe_local_listener(base_url, route_prefix=route_prefix, api_key=api_key):
+        return False
+
+    parsed = urlparse(base_url)
+    host = parsed.hostname or DEFAULT_HOST
+    port = parsed.port or DEFAULT_PORT
+    server_key = (host, port, route_prefix)
+
+    with _LOCAL_SERVER_LOCK:
+        if _probe_local_listener(base_url, route_prefix=route_prefix, api_key=api_key):
+            return False
+        thread = _LOCAL_SERVER_THREADS.get(server_key)
+        if thread is None or not thread.is_alive():
+            thread = threading.Thread(
+                target=_serve_local_listener,
+                kwargs={
+                    "host": host,
+                    "port": port,
+                    "api_key": api_key,
+                    "route_prefix": route_prefix,
+                },
+                name=f"AgentlyDevToolsListener-{host}:{port}",
+                daemon=True,
+            )
+            thread.start()
+            _LOCAL_SERVER_THREADS[server_key] = thread
+
+    deadline = time.time() + wait_timeout
+    while time.time() < deadline:
+        if _probe_local_listener(base_url, route_prefix=route_prefix, api_key=api_key):
+            return True
+        time.sleep(0.1)
+    raise RuntimeError(f"Timed out while starting local Agently DevTools listener at {base_url}.")