agentmetrics-hermes 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentmetrics_hermes-0.2.0/PKG-INFO +89 -0
- agentmetrics_hermes-0.2.0/README.md +69 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/__init__.py +162 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/cli.py +135 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/client.py +84 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/config.py +98 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/events.py +411 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/hooks.py +748 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/pipeline.py +281 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/pricing.py +44 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/redact.py +161 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/schema.py +234 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/state.py +147 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes/wal.py +143 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/PKG-INFO +89 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/SOURCES.txt +29 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/dependency_links.txt +1 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/entry_points.txt +2 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/requires.txt +11 -0
- agentmetrics_hermes-0.2.0/agentmetrics_hermes.egg-info/top_level.txt +1 -0
- agentmetrics_hermes-0.2.0/pyproject.toml +69 -0
- agentmetrics_hermes-0.2.0/setup.cfg +4 -0
- agentmetrics_hermes-0.2.0/setup.py +57 -0
- agentmetrics_hermes-0.2.0/tests/test_client.py +97 -0
- agentmetrics_hermes-0.2.0/tests/test_config.py +51 -0
- agentmetrics_hermes-0.2.0/tests/test_hooks.py +151 -0
- agentmetrics_hermes-0.2.0/tests/test_integration.py +89 -0
- agentmetrics_hermes-0.2.0/tests/test_pipeline.py +96 -0
- agentmetrics_hermes-0.2.0/tests/test_pricing.py +57 -0
- agentmetrics_hermes-0.2.0/tests/test_redact.py +95 -0
- agentmetrics_hermes-0.2.0/tests/test_wal.py +82 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agentmetrics-hermes
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: AgentMetrics observability plugin for Hermes agents
|
|
5
|
+
Author-email: AndaLabX <anda@andalabx.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/andalabx/agentmetrics
|
|
8
|
+
Project-URL: Repository, https://github.com/andalabx/agentmetrics.git
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: agentmetrics-shared>=0.2.0
|
|
12
|
+
Provides-Extra: crypto
|
|
13
|
+
Requires-Dist: cryptography>=41.0; extra == "crypto"
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
16
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
17
|
+
Requires-Dist: ruff>=0.11.0; extra == "dev"
|
|
18
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
19
|
+
Requires-Dist: hypothesis>=6.0; extra == "dev"
|
|
20
|
+
|
|
21
|
+
# agentmetrics-hermes
|
|
22
|
+
|
|
23
|
+
[](https://pypi.org/project/agentmetrics-hermes)
|
|
24
|
+
[](../../LICENSE)
|
|
25
|
+
|
|
26
|
+
AgentMetrics observability plugin for [Hermes](https://github.com/andalabx/hermes) agents. Auto-discovered via the `hermes_agent.plugins` entry-point — no code changes needed.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install agentmetrics-hermes
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Configuration
|
|
39
|
+
|
|
40
|
+
Create `~/.config/hermes/agentmetrics.yaml`:
|
|
41
|
+
|
|
42
|
+
```yaml
|
|
43
|
+
api_key: am_your_key
|
|
44
|
+
server_url: http://localhost:4000
|
|
45
|
+
agent_id: my-hermes-agent
|
|
46
|
+
environment: production
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Or set environment variables (take precedence over the file):
|
|
50
|
+
|
|
51
|
+
| Variable | Description |
|
|
52
|
+
|---|---|
|
|
53
|
+
| `AGENTMETRICS_API_KEY` | API key |
|
|
54
|
+
| `AGENTMETRICS_SERVER_URL` | Server URL |
|
|
55
|
+
| `AGENTMETRICS_AGENT_ID` | Agent identifier |
|
|
56
|
+
| `AGENTMETRICS_ENVIRONMENT` | Deployment environment |
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## What gets tracked
|
|
61
|
+
|
|
62
|
+
| Metric | Description |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `input_tokens` / `output_tokens` | Aggregated LLM token usage |
|
|
65
|
+
| `cache_read_tokens` / `cache_write_tokens` | Prompt cache statistics |
|
|
66
|
+
| `estimated_cost_usd` | Computed from model pricing |
|
|
67
|
+
| `duration_ms` | Session wall-clock time |
|
|
68
|
+
| `tool_calls` / `tool_errors` | Per-step tool tracking |
|
|
69
|
+
| `llm_calls` | Number of model API requests |
|
|
70
|
+
| `secrets_blocked_count` | Credentials redacted before transmission |
|
|
71
|
+
| `error` | Exception message on failure (secrets scrubbed) |
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Durability
|
|
76
|
+
|
|
77
|
+
Events are written to a local WAL before transmission. If the server is unreachable the plugin retries automatically via an internal circuit breaker. Events that exhaust all retries go to a dead-letter queue and are surfaced as `audit_dlq_alert` events.
|
|
78
|
+
|
|
79
|
+
---
|
|
80
|
+
|
|
81
|
+
## Security
|
|
82
|
+
|
|
83
|
+
All event payloads are scanned for API keys, JWTs, and passwords before transmission. Set `redaction_mode: debug` in the config to disable scrubbing during local development (expires automatically after a configurable TTL).
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
[MIT](../../LICENSE)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# agentmetrics-hermes
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/agentmetrics-hermes)
|
|
4
|
+
[](../../LICENSE)
|
|
5
|
+
|
|
6
|
+
AgentMetrics observability plugin for [Hermes](https://github.com/andalabx/hermes) agents. Auto-discovered via the `hermes_agent.plugins` entry-point — no code changes needed.
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install agentmetrics-hermes
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## Configuration
|
|
19
|
+
|
|
20
|
+
Create `~/.config/hermes/agentmetrics.yaml`:
|
|
21
|
+
|
|
22
|
+
```yaml
|
|
23
|
+
api_key: am_your_key
|
|
24
|
+
server_url: http://localhost:4000
|
|
25
|
+
agent_id: my-hermes-agent
|
|
26
|
+
environment: production
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Or set environment variables (take precedence over the file):
|
|
30
|
+
|
|
31
|
+
| Variable | Description |
|
|
32
|
+
|---|---|
|
|
33
|
+
| `AGENTMETRICS_API_KEY` | API key |
|
|
34
|
+
| `AGENTMETRICS_SERVER_URL` | Server URL |
|
|
35
|
+
| `AGENTMETRICS_AGENT_ID` | Agent identifier |
|
|
36
|
+
| `AGENTMETRICS_ENVIRONMENT` | Deployment environment |
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## What gets tracked
|
|
41
|
+
|
|
42
|
+
| Metric | Description |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `input_tokens` / `output_tokens` | Aggregated LLM token usage |
|
|
45
|
+
| `cache_read_tokens` / `cache_write_tokens` | Prompt cache statistics |
|
|
46
|
+
| `estimated_cost_usd` | Computed from model pricing |
|
|
47
|
+
| `duration_ms` | Session wall-clock time |
|
|
48
|
+
| `tool_calls` / `tool_errors` | Per-step tool tracking |
|
|
49
|
+
| `llm_calls` | Number of model API requests |
|
|
50
|
+
| `secrets_blocked_count` | Credentials redacted before transmission |
|
|
51
|
+
| `error` | Exception message on failure (secrets scrubbed) |
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Durability
|
|
56
|
+
|
|
57
|
+
Events are written to a local WAL before transmission. If the server is unreachable the plugin retries automatically via an internal circuit breaker. Events that exhaust all retries go to a dead-letter queue and are surfaced as `audit_dlq_alert` events.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## Security
|
|
62
|
+
|
|
63
|
+
All event payloads are scanned for API keys, JWTs, and passwords before transmission. Set `redaction_mode: debug` in the config to disable scrubbing during local development (expires automatically after a configurable TTL).
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## License
|
|
68
|
+
|
|
69
|
+
[MIT](../../LICENSE)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""AgentMetrics observability plugin for Hermes Agent.
|
|
2
|
+
|
|
3
|
+
Drop-in Python plugin for Hermes' native plugin system. After installation:
|
|
4
|
+
|
|
5
|
+
pip install agentmetrics-hermes
|
|
6
|
+
|
|
7
|
+
Add to your Hermes config.yaml:
|
|
8
|
+
|
|
9
|
+
plugins:
|
|
10
|
+
agentmetrics:
|
|
11
|
+
enabled: true
|
|
12
|
+
endpoint: http://localhost:8099
|
|
13
|
+
api_key: am_your_key_here
|
|
14
|
+
enabled:
|
|
15
|
+
- observability/agentmetrics
|
|
16
|
+
|
|
17
|
+
Events flow automatically from the next gateway restart.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from .client import AgentMetricsClient
|
|
27
|
+
from .config import AgentMetricsConfig
|
|
28
|
+
from .hooks import AgentMetricsHooks
|
|
29
|
+
from .pipeline import EventPipeline
|
|
30
|
+
from .state import StateStore
|
|
31
|
+
from .wal import WriteAheadLog
|
|
32
|
+
|
|
33
|
+
__version__ = "0.1.0"
|
|
34
|
+
__all__ = ["register"]
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
# Module-level references so CLI commands can reach the live pipeline and store.
|
|
39
|
+
_pipeline: EventPipeline | None = None
|
|
40
|
+
_store: StateStore | None = None
|
|
41
|
+
_cfg: AgentMetricsConfig | None = None
|
|
42
|
+
|
|
43
|
+
_HOOK_NAMES = [
|
|
44
|
+
("on_session_start", "on_session_start"),
|
|
45
|
+
("on_session_end", "on_session_end"),
|
|
46
|
+
("pre_llm_call", "pre_llm_call"),
|
|
47
|
+
("post_llm_call", "post_llm_call"),
|
|
48
|
+
("pre_api_request", "pre_api_request"),
|
|
49
|
+
("post_api_request", "post_api_request"),
|
|
50
|
+
("api_request_error", "api_request_error"),
|
|
51
|
+
("pre_tool_call", "pre_tool_call"),
|
|
52
|
+
("post_tool_call", "post_tool_call"),
|
|
53
|
+
# Future hooks — registered when Hermes exposes them.
|
|
54
|
+
("on_subagent_spawn", "on_subagent_spawn"),
|
|
55
|
+
("on_subagent_end", "on_subagent_end"),
|
|
56
|
+
("on_skill_load", "on_skill_load"),
|
|
57
|
+
("on_memory_write", "on_memory_write"),
|
|
58
|
+
("on_session_search", "on_session_search"),
|
|
59
|
+
("on_cron_start", "on_cron_start"),
|
|
60
|
+
("on_cron_end", "on_cron_end"),
|
|
61
|
+
("on_gateway_connect", "on_gateway_connect"),
|
|
62
|
+
("on_gateway_disconnect", "on_gateway_disconnect"),
|
|
63
|
+
("on_gateway_reconnect", "on_gateway_reconnect"),
|
|
64
|
+
("on_retry", "on_retry"),
|
|
65
|
+
("on_timeout", "on_timeout"),
|
|
66
|
+
("on_cancel", "on_cancel"),
|
|
67
|
+
("on_failure", "on_failure"),
|
|
68
|
+
("on_before_compaction", "on_before_compaction"),
|
|
69
|
+
("on_before_reset", "on_before_reset"),
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def register(ctx: Any) -> None:
|
|
74
|
+
"""Hermes plugin entry point. Called once by the gateway on startup."""
|
|
75
|
+
global _pipeline, _store, _cfg
|
|
76
|
+
|
|
77
|
+
cfg = AgentMetricsConfig.load()
|
|
78
|
+
|
|
79
|
+
if not cfg.enabled:
|
|
80
|
+
logger.info("agentmetrics: disabled — set plugins.agentmetrics.enabled=true")
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
if not cfg.api_key:
|
|
84
|
+
logger.warning(
|
|
85
|
+
"agentmetrics: no API key configured — set AGENTMETRICS_API_KEY or "
|
|
86
|
+
"plugins.agentmetrics.api_key in config.yaml"
|
|
87
|
+
)
|
|
88
|
+
return
|
|
89
|
+
|
|
90
|
+
hermes_home = os.environ.get("HERMES_HOME") or os.path.join(
|
|
91
|
+
os.environ.get("HOME") or os.path.expanduser("~"), ".hermes"
|
|
92
|
+
)
|
|
93
|
+
wal_path = os.path.join(hermes_home, "agentmetrics-wal.jsonl")
|
|
94
|
+
dlq_path = os.path.join(hermes_home, "agentmetrics-dlq.json")
|
|
95
|
+
|
|
96
|
+
wal = WriteAheadLog.from_api_key(wal_path, cfg.api_key)
|
|
97
|
+
|
|
98
|
+
# Recover any events that were queued but not flushed before last shutdown.
|
|
99
|
+
recovered = wal.recover()
|
|
100
|
+
|
|
101
|
+
client = AgentMetricsClient(cfg)
|
|
102
|
+
store = StateStore()
|
|
103
|
+
pipeline = EventPipeline(cfg, client, wal, dlq_path)
|
|
104
|
+
pipeline.start()
|
|
105
|
+
|
|
106
|
+
if recovered:
|
|
107
|
+
logger.info("agentmetrics: recovering %d event(s) from WAL", len(recovered))
|
|
108
|
+
from .pipeline import QueueItem
|
|
109
|
+
|
|
110
|
+
for event in recovered:
|
|
111
|
+
pipeline._queue.put_nowait(QueueItem(event=event))
|
|
112
|
+
pipeline.emit_audit("wal_recovery", {"recovered_count": len(recovered)})
|
|
113
|
+
|
|
114
|
+
hooks = AgentMetricsHooks(cfg, pipeline, store)
|
|
115
|
+
|
|
116
|
+
registered = 0
|
|
117
|
+
for hermes_hook, method_name in _HOOK_NAMES:
|
|
118
|
+
handler = getattr(hooks, method_name, None)
|
|
119
|
+
if handler is None:
|
|
120
|
+
continue
|
|
121
|
+
try:
|
|
122
|
+
ctx.register_hook(hermes_hook, handler)
|
|
123
|
+
registered += 1
|
|
124
|
+
except Exception:
|
|
125
|
+
# Hermes may not expose all hooks yet — skip unknown ones silently.
|
|
126
|
+
pass
|
|
127
|
+
|
|
128
|
+
# Register CLI commands via Hermes plugin API when available.
|
|
129
|
+
_register_cli(ctx, cfg, pipeline, store)
|
|
130
|
+
|
|
131
|
+
# Store for CLI/introspection access.
|
|
132
|
+
_pipeline = pipeline
|
|
133
|
+
_store = store
|
|
134
|
+
_cfg = cfg
|
|
135
|
+
|
|
136
|
+
logger.info(
|
|
137
|
+
"agentmetrics: plugin registered — %d hooks active, %d WAL event(s) recovered",
|
|
138
|
+
registered,
|
|
139
|
+
len(recovered),
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _register_cli(ctx: Any, cfg: AgentMetricsConfig, pipeline: EventPipeline, store: StateStore) -> None:
|
|
144
|
+
"""Register `agentmetrics <command>` in Hermes CLI when the API supports it."""
|
|
145
|
+
from . import cli
|
|
146
|
+
|
|
147
|
+
commands = {
|
|
148
|
+
"status": lambda **_: cli.cmd_status(cfg, pipeline, store),
|
|
149
|
+
"flush": lambda **_: cli.cmd_flush(pipeline),
|
|
150
|
+
"tail": lambda **_: cli.cmd_tail(store),
|
|
151
|
+
"test": lambda **_: cli.cmd_test(cfg),
|
|
152
|
+
"redaction-check": lambda **_: cli.cmd_redaction_check(cfg),
|
|
153
|
+
"drain": lambda **_: cli.cmd_drain(pipeline),
|
|
154
|
+
"cost": lambda **_: cli.cmd_cost(cfg, pipeline),
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
for name, handler in commands.items():
|
|
158
|
+
try:
|
|
159
|
+
ctx.register_command(f"agentmetrics {name}", handler)
|
|
160
|
+
except (AttributeError, TypeError):
|
|
161
|
+
# Hermes version without register_command — CLI commands unavailable until updated.
|
|
162
|
+
break
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from .config import AgentMetricsConfig
|
|
8
|
+
from .pipeline import EventPipeline
|
|
9
|
+
from .state import StateStore
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def cmd_status(cfg: AgentMetricsConfig, pipeline: EventPipeline, store: StateStore) -> str:
|
|
13
|
+
"""Report plugin health, config, and live counters."""
|
|
14
|
+
key_preview = (cfg.api_key[:8] + "..." + cfg.api_key[-4:]) if len(cfg.api_key) > 12 else "not set"
|
|
15
|
+
lines = [
|
|
16
|
+
"AgentMetrics — status",
|
|
17
|
+
f" API key : {key_preview}",
|
|
18
|
+
f" Endpoint : {cfg.endpoint}",
|
|
19
|
+
f" Redaction : {cfg.redaction_mode}",
|
|
20
|
+
f" Tool names : {cfg.exported_tool_names}",
|
|
21
|
+
f" Compress payloads : {cfg.compress_payloads}",
|
|
22
|
+
f" Flush interval : {cfg.flush_interval}s",
|
|
23
|
+
f" Batch size : {cfg.batch_size}",
|
|
24
|
+
"",
|
|
25
|
+
f" Circuit breaker : {pipeline.cb.state}",
|
|
26
|
+
f" Queue depth : {pipeline._queue.qsize()} / {cfg.queue_size}",
|
|
27
|
+
f" DLQ depth : {pipeline.dlq.depth}",
|
|
28
|
+
"",
|
|
29
|
+
f" Sent : {pipeline.sent}",
|
|
30
|
+
f" Failed : {pipeline.failed}",
|
|
31
|
+
f" Dropped (overflow) : {pipeline.dropped}",
|
|
32
|
+
]
|
|
33
|
+
return "\n".join(lines)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def cmd_flush(pipeline: EventPipeline) -> str:
|
|
37
|
+
"""Force-flush all queued events immediately."""
|
|
38
|
+
result = pipeline.flush_now()
|
|
39
|
+
return f"Flushed — sent: {result['sent']}, failed: {result['failed']}"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def cmd_tail(store: StateStore) -> str:
|
|
43
|
+
"""Show active sessions and runs."""
|
|
44
|
+
lines = ["AgentMetrics — active sessions"]
|
|
45
|
+
with store._lock:
|
|
46
|
+
sessions = dict(store._sessions)
|
|
47
|
+
runs = dict(store._runs)
|
|
48
|
+
active = dict(store._active_run)
|
|
49
|
+
|
|
50
|
+
if not sessions:
|
|
51
|
+
lines.append(" (no active sessions)")
|
|
52
|
+
return "\n".join(lines)
|
|
53
|
+
|
|
54
|
+
now = time.time()
|
|
55
|
+
for sk, sess in sessions.items():
|
|
56
|
+
elapsed = int(now - sess.started_at)
|
|
57
|
+
run_id = active.get(sk)
|
|
58
|
+
run = runs.get(run_id or "")
|
|
59
|
+
lines.append(f" session: {sk[:20]} elapsed: {elapsed}s runs: {sess.run_count}")
|
|
60
|
+
if run:
|
|
61
|
+
run_elapsed = int(now - run.started_at)
|
|
62
|
+
lines.append(
|
|
63
|
+
f" ↳ run: {run.run_id[:12]} llm: {run.llm_calls} tools: {run.tool_calls}"
|
|
64
|
+
f" elapsed: {run_elapsed}s"
|
|
65
|
+
)
|
|
66
|
+
return "\n".join(lines)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_test(cfg: AgentMetricsConfig) -> str:
|
|
70
|
+
"""Send a test event to verify connectivity."""
|
|
71
|
+
from .client import AgentMetricsClient
|
|
72
|
+
|
|
73
|
+
client = AgentMetricsClient(cfg)
|
|
74
|
+
status, error = client.post_test_event()
|
|
75
|
+
if 200 <= status < 300:
|
|
76
|
+
return f"Delivered — HTTP {status}"
|
|
77
|
+
return f"Failed — HTTP {status}: {error}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def cmd_redaction_check(cfg: AgentMetricsConfig) -> str:
|
|
81
|
+
"""Show what the current redaction policy does to sample data."""
|
|
82
|
+
from .redact import active_mode, redact_tool_name, scrub_secrets
|
|
83
|
+
|
|
84
|
+
mode = active_mode(cfg)
|
|
85
|
+
samples = [
|
|
86
|
+
("OpenAI key", "sk-proj-abc123def456ghi789jkl012mno345pqr"),
|
|
87
|
+
("AgentMetrics key", "am_R4Z5nEek123456789012345678901234"),
|
|
88
|
+
("JWT token", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.abc123def456"),
|
|
89
|
+
("Password in JSON", 'password: "hunter2"'),
|
|
90
|
+
]
|
|
91
|
+
lines = [f"AgentMetrics — redaction check (mode: {mode.value})", ""]
|
|
92
|
+
lines.append(" Secret scrubbing:")
|
|
93
|
+
for label, sample in samples:
|
|
94
|
+
result = scrub_secrets(sample, mode)
|
|
95
|
+
lines.append(f" {label}: {result}")
|
|
96
|
+
|
|
97
|
+
tool_names = ["bash", "read_file", "send_email", "web_search"]
|
|
98
|
+
lines.append("")
|
|
99
|
+
lines.append(f" Tool name policy ({cfg.exported_tool_names}):")
|
|
100
|
+
for name in tool_names:
|
|
101
|
+
tool_result = redact_tool_name(name, cfg)
|
|
102
|
+
lines.append(f" {name}: {tool_result or '[not exported]'}")
|
|
103
|
+
|
|
104
|
+
return "\n".join(lines)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def cmd_drain(pipeline: EventPipeline) -> str:
|
|
108
|
+
"""Retry all events in the dead-letter queue."""
|
|
109
|
+
count = pipeline.retry_dlq()
|
|
110
|
+
if count == 0:
|
|
111
|
+
return "DLQ is empty — nothing to drain"
|
|
112
|
+
return f"Retrying {count} DLQ event(s) — check `agentmetrics flush` to confirm delivery"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def cmd_cost(cfg: AgentMetricsConfig, pipeline: EventPipeline) -> str:
|
|
116
|
+
"""Display the model pricing table and any custom overrides."""
|
|
117
|
+
from .pricing import MODEL_PRICING
|
|
118
|
+
|
|
119
|
+
lines = ["AgentMetrics — pricing table (per million tokens, USD)", ""]
|
|
120
|
+
lines.append(f" {'Model':<30} {'Input':>8} {'Output':>8} {'CR':>8} {'CW':>8}")
|
|
121
|
+
lines.append(" " + "-" * 66)
|
|
122
|
+
for model, prices in MODEL_PRICING.items():
|
|
123
|
+
inp = f"${prices[0]:.3f}" if prices[0] is not None else "—"
|
|
124
|
+
out = f"${prices[1]:.3f}" if prices[1] is not None else "—"
|
|
125
|
+
cr = f"${prices[2]:.3f}" if prices[2] is not None else "—"
|
|
126
|
+
cw = f"${prices[3]:.3f}" if prices[3] is not None else "—"
|
|
127
|
+
lines.append(f" {model:<30} {inp:>8} {out:>8} {cr:>8} {cw:>8}")
|
|
128
|
+
|
|
129
|
+
if cfg.cost_provider_table:
|
|
130
|
+
lines.append("")
|
|
131
|
+
lines.append(" Custom overrides:")
|
|
132
|
+
for model, prices in cfg.cost_provider_table.items():
|
|
133
|
+
lines.append(f" {model}: {prices}")
|
|
134
|
+
|
|
135
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import urllib.error
|
|
7
|
+
import urllib.request
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from .config import AgentMetricsConfig
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
_CONNECT_TIMEOUT = 5 # seconds
|
|
16
|
+
_READ_TIMEOUT = 15 # seconds
|
|
17
|
+
_MAX_PAYLOAD_BYTES = 10 * 1024 * 1024 # 10 MB hard cap per batch (DoS guard)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AgentMetricsClient:
|
|
21
|
+
"""HTTP transport for the AgentMetrics ingest API. Uses stdlib urllib — no extra deps."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, cfg: AgentMetricsConfig) -> None:
|
|
24
|
+
self._endpoint = cfg.endpoint.rstrip("/")
|
|
25
|
+
self._api_key = cfg.api_key
|
|
26
|
+
self._compress = cfg.compress_payloads
|
|
27
|
+
|
|
28
|
+
def post_events(self, payloads: list[dict[str, Any]]) -> tuple[int, str]:
|
|
29
|
+
"""POST a batch of event dicts to /v1/events. Returns (status_code, error_message)."""
|
|
30
|
+
url = f"{self._endpoint}/v1/events"
|
|
31
|
+
body = json.dumps(payloads).encode()
|
|
32
|
+
|
|
33
|
+
if len(body) > _MAX_PAYLOAD_BYTES:
|
|
34
|
+
logger.warning(
|
|
35
|
+
"agentmetrics: batch payload %d bytes exceeds %d byte cap — dropping",
|
|
36
|
+
len(body),
|
|
37
|
+
_MAX_PAYLOAD_BYTES,
|
|
38
|
+
)
|
|
39
|
+
return 413, "payload too large"
|
|
40
|
+
|
|
41
|
+
headers = self._headers()
|
|
42
|
+
if self._compress and len(body) > 1024:
|
|
43
|
+
body = gzip.compress(body)
|
|
44
|
+
headers["Content-Encoding"] = "gzip"
|
|
45
|
+
headers["Content-Length"] = str(len(body))
|
|
46
|
+
|
|
47
|
+
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
48
|
+
try:
|
|
49
|
+
with urllib.request.urlopen(
|
|
50
|
+
req, timeout=max(_CONNECT_TIMEOUT, _READ_TIMEOUT)
|
|
51
|
+
) as resp:
|
|
52
|
+
return resp.status, ""
|
|
53
|
+
except urllib.error.HTTPError as exc:
|
|
54
|
+
return exc.code, str(exc.reason)
|
|
55
|
+
except urllib.error.URLError as exc:
|
|
56
|
+
return 0, str(exc.reason)
|
|
57
|
+
except TimeoutError:
|
|
58
|
+
return 0, "timeout"
|
|
59
|
+
except Exception as exc:
|
|
60
|
+
return 0, str(exc)
|
|
61
|
+
|
|
62
|
+
def post_test_event(self) -> tuple[int, str]:
|
|
63
|
+
"""Send a minimal test event. Used by the `agentmetrics test` CLI command."""
|
|
64
|
+
import time
|
|
65
|
+
import uuid
|
|
66
|
+
|
|
67
|
+
payload = [
|
|
68
|
+
{
|
|
69
|
+
"event_id": str(uuid.uuid4()),
|
|
70
|
+
"event_name": "plugin_test",
|
|
71
|
+
"agent_id": "hermes",
|
|
72
|
+
"platform": "hermes",
|
|
73
|
+
"ts": int(time.time() * 1000),
|
|
74
|
+
"status": "success",
|
|
75
|
+
}
|
|
76
|
+
]
|
|
77
|
+
return self.post_events(payload)
|
|
78
|
+
|
|
79
|
+
def _headers(self) -> dict[str, str]:
|
|
80
|
+
return {
|
|
81
|
+
"Content-Type": "application/json",
|
|
82
|
+
"X-API-Key": self._api_key,
|
|
83
|
+
"User-Agent": "agentmetrics-hermes/0.1.0",
|
|
84
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from urllib.parse import urlparse
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
_DEFAULT_ENDPOINT = "http://localhost:8099"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _validate_endpoint(url: str) -> str:
|
|
14
|
+
"""Reject non-HTTP/HTTPS schemes. Warn (not block) on non-HTTPS for non-localhost URLs."""
|
|
15
|
+
try:
|
|
16
|
+
parsed = urlparse(url)
|
|
17
|
+
except Exception:
|
|
18
|
+
logger.warning("agentmetrics: invalid endpoint URL, falling back to default")
|
|
19
|
+
return _DEFAULT_ENDPOINT
|
|
20
|
+
if parsed.scheme not in ("http", "https"):
|
|
21
|
+
logger.warning(
|
|
22
|
+
"agentmetrics: endpoint scheme %r is not http/https — using default", parsed.scheme
|
|
23
|
+
)
|
|
24
|
+
return _DEFAULT_ENDPOINT
|
|
25
|
+
host = parsed.hostname or ""
|
|
26
|
+
is_local = host in ("localhost", "127.0.0.1", "::1") or host.startswith("192.168.")
|
|
27
|
+
if parsed.scheme == "http" and not is_local:
|
|
28
|
+
# Warn operators who send telemetry over plain HTTP to a remote host.
|
|
29
|
+
logger.warning(
|
|
30
|
+
"agentmetrics: endpoint %r uses plain HTTP to a non-local host — "
|
|
31
|
+
"API keys and event data will be sent unencrypted. Use HTTPS in production.",
|
|
32
|
+
url,
|
|
33
|
+
)
|
|
34
|
+
return url
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass
|
|
38
|
+
class AgentMetricsConfig:
|
|
39
|
+
enabled: bool = True
|
|
40
|
+
endpoint: str = _DEFAULT_ENDPOINT
|
|
41
|
+
api_key: str = ""
|
|
42
|
+
flush_interval: int = 10
|
|
43
|
+
batch_size: int = 100
|
|
44
|
+
queue_size: int = 10_000
|
|
45
|
+
retry_max_attempts: int = 5
|
|
46
|
+
redaction_mode: str = "strict"
|
|
47
|
+
exported_tool_names: str = "blocklist"
|
|
48
|
+
redact_tool_names: list[str] = field(default_factory=list)
|
|
49
|
+
compress_payloads: bool = False
|
|
50
|
+
cost_provider_table: dict[str, list[float | None]] = field(default_factory=dict)
|
|
51
|
+
debug_expires_at: float | None = None
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def load(cls) -> AgentMetricsConfig:
|
|
55
|
+
"""Read from Hermes config.yaml plugins.agentmetrics.* section."""
|
|
56
|
+
try:
|
|
57
|
+
from hermes_cli.config import cfg_get, load_config # type: ignore[import-not-found]
|
|
58
|
+
|
|
59
|
+
config = load_config()
|
|
60
|
+
raw = cfg_get(config, "plugins", "agentmetrics", default={})
|
|
61
|
+
if not isinstance(raw, dict):
|
|
62
|
+
return cls(enabled=False)
|
|
63
|
+
api_key = str(
|
|
64
|
+
raw.get("api_key") or os.environ.get("AGENTMETRICS_API_KEY", "")
|
|
65
|
+
)
|
|
66
|
+
raw_endpoint = str(
|
|
67
|
+
os.environ.get("AGENTMETRICS_URL")
|
|
68
|
+
or raw.get("endpoint", _DEFAULT_ENDPOINT)
|
|
69
|
+
)
|
|
70
|
+
endpoint = _validate_endpoint(raw_endpoint)
|
|
71
|
+
return cls(
|
|
72
|
+
enabled=bool(raw.get("enabled", True)),
|
|
73
|
+
endpoint=endpoint,
|
|
74
|
+
api_key=api_key,
|
|
75
|
+
flush_interval=int(raw.get("flush_interval", cls.flush_interval)),
|
|
76
|
+
batch_size=int(raw.get("batch_size", cls.batch_size)),
|
|
77
|
+
queue_size=int(raw.get("queue_size", cls.queue_size)),
|
|
78
|
+
retry_max_attempts=int(raw.get("retry_max_attempts", cls.retry_max_attempts)),
|
|
79
|
+
redaction_mode=str(raw.get("redaction_mode", cls.redaction_mode)),
|
|
80
|
+
exported_tool_names=str(
|
|
81
|
+
raw.get("exported_tool_names", cls.exported_tool_names)
|
|
82
|
+
),
|
|
83
|
+
redact_tool_names=list(raw.get("redact_tool_names") or []),
|
|
84
|
+
compress_payloads=bool(raw.get("compress_payloads", False)),
|
|
85
|
+
cost_provider_table=dict(raw.get("cost_provider_table") or {}),
|
|
86
|
+
)
|
|
87
|
+
except ImportError:
|
|
88
|
+
# Hermes not installed — config comes from env vars only.
|
|
89
|
+
api_key = os.environ.get("AGENTMETRICS_API_KEY", "")
|
|
90
|
+
raw_endpoint = os.environ.get("AGENTMETRICS_URL", _DEFAULT_ENDPOINT)
|
|
91
|
+
return cls(
|
|
92
|
+
enabled=bool(api_key),
|
|
93
|
+
endpoint=_validate_endpoint(raw_endpoint),
|
|
94
|
+
api_key=api_key,
|
|
95
|
+
)
|
|
96
|
+
except Exception:
|
|
97
|
+
logger.exception("agentmetrics: failed to load config")
|
|
98
|
+
return cls(enabled=False)
|