clio-relay 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. clio_relay-0.9.0/.gitignore +11 -0
  2. clio_relay-0.9.0/LICENSE +21 -0
  3. clio_relay-0.9.0/PKG-INFO +154 -0
  4. clio_relay-0.9.0/README.md +134 -0
  5. clio_relay-0.9.0/docs/ai/README.md +14 -0
  6. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/__init__.py +1 -0
  7. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/bounded_command/__init__.py +1 -0
  8. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/bounded_command/pkg.py +169 -0
  9. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/bounded_command/progress.py +134 -0
  10. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/mcp_call/__init__.py +1 -0
  11. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/mcp_call/pkg.py +52 -0
  12. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/mcp_call/runner.py +221 -0
  13. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/remote_agent/__init__.py +1 -0
  14. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/remote_agent/pkg.py +36 -0
  15. clio_relay-0.9.0/jarvis-packages/clio_relay/clio_relay/remote_agent/runner.py +311 -0
  16. clio_relay-0.9.0/pyproject.toml +77 -0
  17. clio_relay-0.9.0/src/clio_relay/__init__.py +5 -0
  18. clio_relay-0.9.0/src/clio_relay/bootstrap.py +497 -0
  19. clio_relay-0.9.0/src/clio_relay/cli.py +1942 -0
  20. clio_relay-0.9.0/src/clio_relay/cluster_config.py +141 -0
  21. clio_relay-0.9.0/src/clio_relay/config.py +107 -0
  22. clio_relay-0.9.0/src/clio_relay/core_queue.py +777 -0
  23. clio_relay-0.9.0/src/clio_relay/deployment.py +122 -0
  24. clio_relay-0.9.0/src/clio_relay/doctor.py +91 -0
  25. clio_relay-0.9.0/src/clio_relay/endpoint.py +990 -0
  26. clio_relay-0.9.0/src/clio_relay/errors.py +17 -0
  27. clio_relay-0.9.0/src/clio_relay/frp_check.py +48 -0
  28. clio_relay-0.9.0/src/clio_relay/http_api.py +736 -0
  29. clio_relay-0.9.0/src/clio_relay/jarvis_provider.py +415 -0
  30. clio_relay-0.9.0/src/clio_relay/live_acceptance.py +1334 -0
  31. clio_relay-0.9.0/src/clio_relay/mcp_server.py +907 -0
  32. clio_relay-0.9.0/src/clio_relay/models.py +407 -0
  33. clio_relay-0.9.0/src/clio_relay/progress_adapters.py +311 -0
  34. clio_relay-0.9.0/src/clio_relay/progress_provenance.py +59 -0
  35. clio_relay-0.9.0/src/clio_relay/relay_host.py +132 -0
  36. clio_relay-0.9.0/src/clio_relay/relay_ops.py +411 -0
  37. clio_relay-0.9.0/src/clio_relay/remote_cli.py +190 -0
  38. clio_relay-0.9.0/src/clio_relay/scheduler_status.py +206 -0
  39. clio_relay-0.9.0/src/clio_relay/session_lifecycle.py +319 -0
  40. clio_relay-0.9.0/src/clio_relay/spool.py +92 -0
  41. clio_relay-0.9.0/src/clio_relay/transport_probe.py +583 -0
  42. clio_relay-0.9.0/tests/test_bootstrap.py +140 -0
  43. clio_relay-0.9.0/tests/test_bounded_command_progress.py +125 -0
  44. clio_relay-0.9.0/tests/test_cli.py +1166 -0
  45. clio_relay-0.9.0/tests/test_config.py +163 -0
  46. clio_relay-0.9.0/tests/test_doctor_and_relay_host.py +202 -0
  47. clio_relay-0.9.0/tests/test_endpoint.py +1142 -0
  48. clio_relay-0.9.0/tests/test_frp_check.py +42 -0
  49. clio_relay-0.9.0/tests/test_http_api.py +517 -0
  50. clio_relay-0.9.0/tests/test_jarvis_provider.py +157 -0
  51. clio_relay-0.9.0/tests/test_live_acceptance.py +1367 -0
  52. clio_relay-0.9.0/tests/test_mcp_call_runner.py +163 -0
  53. clio_relay-0.9.0/tests/test_mcp_server.py +843 -0
  54. clio_relay-0.9.0/tests/test_progress_adapters.py +152 -0
  55. clio_relay-0.9.0/tests/test_queue.py +635 -0
  56. clio_relay-0.9.0/tests/test_remote_agent_runner.py +242 -0
  57. clio_relay-0.9.0/tests/test_scheduler_status.py +167 -0
  58. clio_relay-0.9.0/tests/test_session_lifecycle.py +119 -0
  59. clio_relay-0.9.0/tests/test_transport_probe.py +440 -0
  60. clio_relay-0.9.0/uv.lock +576 -0
@@ -0,0 +1,11 @@
1
+ .venv/
2
+ .ruff_cache/
3
+ .pytest_cache/
4
+ .pyright/
5
+ __pycache__/
6
+ *.pyc
7
+ .clio-relay/
8
+ .tools/
9
+ dist/
10
+ build/
11
+ *.egg-info/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Jaime Cernuda Garcia
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,154 @@
1
+ Metadata-Version: 2.4
2
+ Name: clio-relay
3
+ Version: 0.9.0
4
+ Summary: Cluster relay endpoints backed by clio-core and JARVIS-CD.
5
+ Project-URL: Homepage, https://github.com/iowarp/clio-relay
6
+ Project-URL: Repository, https://github.com/iowarp/clio-relay
7
+ Project-URL: Issues, https://github.com/iowarp/clio-relay/issues
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Requires-Python: >=3.12
11
+ Requires-Dist: fastapi>=0.115
12
+ Requires-Dist: filelock>=3.15
13
+ Requires-Dist: httpx>=0.27
14
+ Requires-Dist: pydantic>=2.8
15
+ Requires-Dist: pyyaml>=6.0
16
+ Requires-Dist: typer>=0.12
17
+ Requires-Dist: uvicorn>=0.30
18
+ Requires-Dist: websockets>=16.0
19
+ Description-Content-Type: text/markdown
20
+
21
+ <p align="center">
22
+ <img src="docs/assets/clio-relay-banner.png" alt="clio-relay banner">
23
+ </p>
24
+
25
+ <h1 align="center">clio-relay</h1>
26
+
27
+ `clio-relay` lets a desktop tool submit work to a remote cluster, follow it while it runs, and collect logs, artifacts, progress, and provenance without putting job state in the network tunnel.
28
+
29
+ It is a piece of the federation layer for [`clio-agent`](https://github.com/iowarp/clio-agent): a local CLIO experience can delegate work to a remote machine, keep observing it, detach, reconnect, and clean up after itself. The project is also designed for use outside CLIO. Any client that can call the CLI, HTTP API, or MCP tools can use the same relay model.
30
+
31
+ ## How It Works
32
+
33
+ `clio-relay` has three long-running roles:
34
+
35
+ - `desktop`: submits work and exposes CLI, HTTP, and MCP surfaces for local tools.
36
+ - `worker`: runs on a configured cluster, leases work, invokes JARVIS-CD, and records results.
37
+ - `relay-host`: carries bytes for frp deployments. It does not store jobs or queue state.
38
+
39
+ The durable boundary is `clio-core`. The filesystem queue in this repository is the development backend for that record contract. Jobs, task timelines, progress, scheduler state, gateway sessions, logs, artifacts, and provenance are recorded there so clients can detach, reconnect, and replay state. JARVIS-CD owns scheduler execution, package behavior, output collection, and provenance.
40
+
41
+ Transport is replaceable because it only carries HTTP bytes between endpoints:
42
+
43
+ - Relay mode uses frp through a public relay host. It supports WebSocket/TLS for Cloudflare-style HTTPS infrastructure and raw TCP for environments that provide a direct public port.
44
+ - NAT bypass uses frp XTCP to try a direct peer path between desktop and cluster. It is an optimization for lower-latency or higher-volume traffic, with fallback to relay mode and the durable queue.
45
+ - SSH forwarding uses local port forwarding through an existing SSH or VPN path. It is useful for closed environments that do not want a public relay.
46
+
47
+ Remote agent tasks, remote MCP calls, JARVIS pipelines, and gateway sessions all use the same queue and observation model. The transport can change without changing where state lives.
48
+
49
+ ## Install
50
+
51
+ ```powershell
52
+ uv sync
53
+ uv run clio-relay init
54
+ uv run clio-relay install-frp
55
+ ```
56
+
57
+ Add a cluster. The cluster name and agent executable are local configuration.
58
+
59
+ ```powershell
60
+ uv run clio-relay cluster add --name my-cluster --ssh-host my-cluster-login --agent-adapter exec --agent-bin agent
61
+ uv run clio-relay cluster bootstrap --cluster my-cluster
62
+ uv run clio-relay cluster install-endpoint-service --cluster my-cluster --start --enable
63
+ ```
64
+
65
+ ## Submit Work
66
+
67
+ Submit a JARVIS pipeline:
68
+
69
+ ```powershell
70
+ uv run clio-relay job submit --cluster my-cluster --jarvis-yaml .\pipeline.yaml
71
+ uv run clio-relay job watch <job-id> --cluster my-cluster
72
+ uv run clio-relay job read-log <job-id> --cluster my-cluster --stream stdout
73
+ uv run clio-relay job list-artifacts <job-id> --cluster my-cluster
74
+ ```
75
+
76
+ Expose relay tools to an agent:
77
+
78
+ ```powershell
79
+ uv run clio-relay agent render-mcp-config --output .\clio-relay-agent.config.toml
80
+ uv run clio-relay agent run --cluster my-cluster --prompt /path/on/cluster/prompt.md --mcp-config /path/on/cluster/clio-relay-agent.config.toml
81
+ ```
82
+
83
+ ## Observe Remote Agent Work
84
+
85
+ Remote agents can emit structured task timeline events while they work. This is useful when a UI needs to show discovery and planning before the final answer exists.
86
+
87
+ ```powershell
88
+ uv run clio-relay job tasks <job-id> --cluster my-cluster
89
+ uv run clio-relay job record-task-event <task-id> --cluster my-cluster --event-type dataset_found --label dataset --summary "found staged dataset" --path-ref /mnt/common/datasets/red_sea_001
90
+ uv run clio-relay job task-events <task-id> --cluster my-cluster --cursor 1
91
+ ```
92
+
93
+ The same contract is available over HTTP at `/tasks/{task_id}/events`, `/tasks/{task_id}/events/sse`, and `/tasks/{task_id}/events/ws`, and through MCP tools `relay_record_task_event` and `relay_watch_task_events`.
94
+
95
+ ## Manage Gateway Sessions
96
+
97
+ Long-running visualization services should be tracked as durable gateway sessions. A session records the scheduler job, node, logs, published or forwarded endpoint, health metadata, and reconnect hints.
98
+
99
+ ```powershell
100
+ uv run clio-relay gateway create --cluster my-cluster --name paraview-red-sea --gateway-json-file .\gateway.json
101
+ uv run clio-relay gateway update <session-id> --cluster my-cluster --state ready --scheduler-job-id 12345 --node compute-01 --gateway-json-file .\gateway-ready.json
102
+ uv run clio-relay gateway get <session-id> --cluster my-cluster
103
+ uv run clio-relay gateway close <session-id> --cluster my-cluster
104
+ ```
105
+
106
+ The HTTP API exposes `/gateway-sessions`, `/gateway-sessions/{session_id}`, and `/gateway-sessions/{session_id}/close`. MCP tools expose the same create, read, update, and close operations.
107
+
108
+ ## Choose Transport
109
+
110
+ For a public relay through Cloudflare or another HTTPS edge, use frp with `transport.protocol = "wss"`:
111
+
112
+ ```powershell
113
+ $env:CLIO_RELAY_FRP_TOKEN = "<shared-frp-token>"
114
+ $env:CLIO_RELAY_STCP_SECRET = "<shared-stcp-secret>"
115
+ uv run clio-relay cluster add --name my-cluster --ssh-host my-cluster-login --frp-server-addr relay.example.edu
116
+ uv run clio-relay relay-host render-frpc-config --cluster my-cluster --local-port 8848
117
+ uv run clio-relay relay-host render-frpc-visitor-config --cluster my-cluster --bind-port 8765
118
+ uv run clio-relay relay-host test-http-transport --cluster my-cluster --local-bind-port 18765
119
+ ```
120
+
121
+ For closed environments where SSH or VPN already exists, use SSH local forwarding:
122
+
123
+ ```powershell
124
+ uv run clio-relay relay-host test-ssh-transport --cluster my-cluster --local-bind-port 18766 --remote-api-port 8766 --session-id relay-ssh-test
125
+ ```
126
+
127
+ To leave the remote API alive for a desktop detach and later reattach:
128
+
129
+ ```powershell
130
+ uv run clio-relay session start --cluster my-cluster --session-id desktop-session --remote-api-port 8766 --replace
131
+ uv run clio-relay session status --cluster my-cluster --session-id desktop-session
132
+ uv run clio-relay session teardown --cluster my-cluster --session-id desktop-session
133
+ ```
134
+
135
+ Use `session teardown --stop-worker` only when the user chooses to clean up the persistent remote worker too.
136
+
137
+ ## Documentation
138
+
139
+ - [architecture](docs/architecture.md)
140
+ - [operations](docs/operations.md)
141
+ - [release](docs/release.md)
142
+ - [brand prompt](docs/brand.md)
143
+ - [ai context](docs/ai/README.md)
144
+
145
+ ## Development
146
+
147
+ ```powershell
148
+ uv run ruff check --fix
149
+ uv run ruff format
150
+ uv run pyright
151
+ uv run pytest
152
+ ```
153
+
154
+ The GitHub workflow runs lint, type checks, tests, package builds, and artifact validation. Publishing to PyPI is configured through trusted publishing for the `iowarp/clio-relay` repository.
@@ -0,0 +1,134 @@
1
+ <p align="center">
2
+ <img src="docs/assets/clio-relay-banner.png" alt="clio-relay banner">
3
+ </p>
4
+
5
+ <h1 align="center">clio-relay</h1>
6
+
7
+ `clio-relay` lets a desktop tool submit work to a remote cluster, follow it while it runs, and collect logs, artifacts, progress, and provenance without putting job state in the network tunnel.
8
+
9
+ It is a piece of the federation layer for [`clio-agent`](https://github.com/iowarp/clio-agent): a local CLIO experience can delegate work to a remote machine, keep observing it, detach, reconnect, and clean up after itself. The project is also designed for use outside CLIO. Any client that can call the CLI, HTTP API, or MCP tools can use the same relay model.
10
+
11
+ ## How It Works
12
+
13
+ `clio-relay` has three long-running roles:
14
+
15
+ - `desktop`: submits work and exposes CLI, HTTP, and MCP surfaces for local tools.
16
+ - `worker`: runs on a configured cluster, leases work, invokes JARVIS-CD, and records results.
17
+ - `relay-host`: carries bytes for frp deployments. It does not store jobs or queue state.
18
+
19
+ The durable boundary is `clio-core`. The filesystem queue in this repository is the development backend for that record contract. Jobs, task timelines, progress, scheduler state, gateway sessions, logs, artifacts, and provenance are recorded there so clients can detach, reconnect, and replay state. JARVIS-CD owns scheduler execution, package behavior, output collection, and provenance.
20
+
21
+ Transport is replaceable because it only carries HTTP bytes between endpoints:
22
+
23
+ - Relay mode uses frp through a public relay host. It supports WebSocket/TLS for Cloudflare-style HTTPS infrastructure and raw TCP for environments that provide a direct public port.
24
+ - NAT bypass uses frp XTCP to try a direct peer path between desktop and cluster. It is an optimization for lower-latency or higher-volume traffic, with fallback to relay mode and the durable queue.
25
+ - SSH forwarding uses local port forwarding through an existing SSH or VPN path. It is useful for closed environments that do not want a public relay.
26
+
27
+ Remote agent tasks, remote MCP calls, JARVIS pipelines, and gateway sessions all use the same queue and observation model. The transport can change without changing where state lives.
28
+
29
+ ## Install
30
+
31
+ ```powershell
32
+ uv sync
33
+ uv run clio-relay init
34
+ uv run clio-relay install-frp
35
+ ```
36
+
37
+ Add a cluster. The cluster name and agent executable are local configuration.
38
+
39
+ ```powershell
40
+ uv run clio-relay cluster add --name my-cluster --ssh-host my-cluster-login --agent-adapter exec --agent-bin agent
41
+ uv run clio-relay cluster bootstrap --cluster my-cluster
42
+ uv run clio-relay cluster install-endpoint-service --cluster my-cluster --start --enable
43
+ ```
44
+
45
+ ## Submit Work
46
+
47
+ Submit a JARVIS pipeline:
48
+
49
+ ```powershell
50
+ uv run clio-relay job submit --cluster my-cluster --jarvis-yaml .\pipeline.yaml
51
+ uv run clio-relay job watch <job-id> --cluster my-cluster
52
+ uv run clio-relay job read-log <job-id> --cluster my-cluster --stream stdout
53
+ uv run clio-relay job list-artifacts <job-id> --cluster my-cluster
54
+ ```
55
+
56
+ Expose relay tools to an agent:
57
+
58
+ ```powershell
59
+ uv run clio-relay agent render-mcp-config --output .\clio-relay-agent.config.toml
60
+ uv run clio-relay agent run --cluster my-cluster --prompt /path/on/cluster/prompt.md --mcp-config /path/on/cluster/clio-relay-agent.config.toml
61
+ ```
62
+
63
+ ## Observe Remote Agent Work
64
+
65
+ Remote agents can emit structured task timeline events while they work. This is useful when a UI needs to show discovery and planning before the final answer exists.
66
+
67
+ ```powershell
68
+ uv run clio-relay job tasks <job-id> --cluster my-cluster
69
+ uv run clio-relay job record-task-event <task-id> --cluster my-cluster --event-type dataset_found --label dataset --summary "found staged dataset" --path-ref /mnt/common/datasets/red_sea_001
70
+ uv run clio-relay job task-events <task-id> --cluster my-cluster --cursor 1
71
+ ```
72
+
73
+ The same contract is available over HTTP at `/tasks/{task_id}/events`, `/tasks/{task_id}/events/sse`, and `/tasks/{task_id}/events/ws`, and through MCP tools `relay_record_task_event` and `relay_watch_task_events`.
74
+
75
+ ## Manage Gateway Sessions
76
+
77
+ Long-running visualization services should be tracked as durable gateway sessions. A session records the scheduler job, node, logs, published or forwarded endpoint, health metadata, and reconnect hints.
78
+
79
+ ```powershell
80
+ uv run clio-relay gateway create --cluster my-cluster --name paraview-red-sea --gateway-json-file .\gateway.json
81
+ uv run clio-relay gateway update <session-id> --cluster my-cluster --state ready --scheduler-job-id 12345 --node compute-01 --gateway-json-file .\gateway-ready.json
82
+ uv run clio-relay gateway get <session-id> --cluster my-cluster
83
+ uv run clio-relay gateway close <session-id> --cluster my-cluster
84
+ ```
85
+
86
+ The HTTP API exposes `/gateway-sessions`, `/gateway-sessions/{session_id}`, and `/gateway-sessions/{session_id}/close`. MCP tools expose the same create, read, update, and close operations.
87
+
88
+ ## Choose Transport
89
+
90
+ For a public relay through Cloudflare or another HTTPS edge, use frp with `transport.protocol = "wss"`:
91
+
92
+ ```powershell
93
+ $env:CLIO_RELAY_FRP_TOKEN = "<shared-frp-token>"
94
+ $env:CLIO_RELAY_STCP_SECRET = "<shared-stcp-secret>"
95
+ uv run clio-relay cluster add --name my-cluster --ssh-host my-cluster-login --frp-server-addr relay.example.edu
96
+ uv run clio-relay relay-host render-frpc-config --cluster my-cluster --local-port 8848
97
+ uv run clio-relay relay-host render-frpc-visitor-config --cluster my-cluster --bind-port 8765
98
+ uv run clio-relay relay-host test-http-transport --cluster my-cluster --local-bind-port 18765
99
+ ```
100
+
101
+ For closed environments where SSH or VPN already exists, use SSH local forwarding:
102
+
103
+ ```powershell
104
+ uv run clio-relay relay-host test-ssh-transport --cluster my-cluster --local-bind-port 18766 --remote-api-port 8766 --session-id relay-ssh-test
105
+ ```
106
+
107
+ To leave the remote API alive for a desktop detach and later reattach:
108
+
109
+ ```powershell
110
+ uv run clio-relay session start --cluster my-cluster --session-id desktop-session --remote-api-port 8766 --replace
111
+ uv run clio-relay session status --cluster my-cluster --session-id desktop-session
112
+ uv run clio-relay session teardown --cluster my-cluster --session-id desktop-session
113
+ ```
114
+
115
+ Use `session teardown --stop-worker` only when the user chooses to clean up the persistent remote worker too.
116
+
117
+ ## Documentation
118
+
119
+ - [architecture](docs/architecture.md)
120
+ - [operations](docs/operations.md)
121
+ - [release](docs/release.md)
122
+ - [brand prompt](docs/brand.md)
123
+ - [ai context](docs/ai/README.md)
124
+
125
+ ## Development
126
+
127
+ ```powershell
128
+ uv run ruff check --fix
129
+ uv run ruff format
130
+ uv run pyright
131
+ uv run pytest
132
+ ```
133
+
134
+ The GitHub workflow runs lint, type checks, tests, package builds, and artifact validation. Publishing to PyPI is configured through trusted publishing for the `iowarp/clio-relay` repository.
@@ -0,0 +1,14 @@
1
+ # AI Context
2
+
3
+ These files are for coding agents and review agents. Human-facing docs should stay short and readable.
4
+
5
+ Read in this order:
6
+
7
+ 1. `system-context.md`
8
+ 2. `testing-context.md`
9
+ 3. `interface-context.md`
10
+ 4. `../../README.md`
11
+ 5. `../architecture.md`
12
+ 6. `../operations.md`
13
+
14
+ Do not treat examples as hardcoded product semantics. Examples show one configured target or workload.
@@ -0,0 +1 @@
1
+ """JARVIS-CD package repository for clio-relay."""
@@ -0,0 +1 @@
1
+ """Bounded command JARVIS package."""
@@ -0,0 +1,169 @@
1
+ """JARVIS-CD package for bounded relay commands."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import queue
7
+ import signal
8
+ import subprocess
9
+ import sys
10
+ import threading
11
+ import time
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from jarvis_cd.core.pkg import Application
16
+
17
+ from clio_relay.bounded_command.progress import adapter_from_config, append_progress_record
18
+
19
+ PROGRESS_FILE_ENV = "CLIO_RELAY_PROGRESS_FILE"
20
+ PROGRESS_TOKEN_ENV = "CLIO_RELAY_PROGRESS_TOKEN"
21
+
22
+
23
+ class BoundedCommand(Application):
24
+ """Execute a bounded command and let JARVIS-CD capture provenance."""
25
+
26
+ def _init(self) -> None:
27
+ """Initialize package state."""
28
+
29
+ def _configure_menu(self) -> list[dict[str, Any]]:
30
+ """Return JARVIS configurator options."""
31
+ return []
32
+
33
+ def _configure(self, **kwargs: Any) -> None:
34
+ """Store configuration provided by the pipeline YAML."""
35
+ self.config.update(kwargs)
36
+
37
+ def start(self) -> None:
38
+ """Run the configured command."""
39
+ command = self.config.get("command")
40
+ if not isinstance(command, list) or not all(isinstance(item, str) for item in command):
41
+ raise ValueError("command must be a string array")
42
+ env = os.environ.copy()
43
+ supplied_env = self.config.get("env", {})
44
+ if isinstance(supplied_env, dict):
45
+ env.update({str(key): str(value) for key, value in supplied_env.items()})
46
+ env.pop(PROGRESS_FILE_ENV, None)
47
+ env.pop(PROGRESS_TOKEN_ENV, None)
48
+ workdir_value = self.config.get("workdir")
49
+ workdir = Path(workdir_value) if isinstance(workdir_value, str) else None
50
+ timeout_value = self.config.get("timeout_seconds")
51
+ timeout = int(timeout_value) if timeout_value is not None else None
52
+ result = _run_streaming(
53
+ command,
54
+ cwd=workdir,
55
+ env=env,
56
+ timeout=timeout,
57
+ progress_config=self.config.get("progress"),
58
+ )
59
+ if result.returncode != 0:
60
+ raise RuntimeError(f"command failed with exit code {result.returncode}")
61
+
62
+ def stop(self) -> None:
63
+ """Stop hook for bounded commands."""
64
+
65
+ def clean(self) -> None:
66
+ """Clean hook for bounded commands."""
67
+
68
+
69
+ def _run_streaming(
70
+ command: list[str],
71
+ *,
72
+ cwd: Path | None,
73
+ env: dict[str, str],
74
+ timeout: int | None,
75
+ progress_config: object,
76
+ ) -> subprocess.CompletedProcess[str]:
77
+ adapter = adapter_from_config(progress_config)
78
+ process = subprocess.Popen(
79
+ command,
80
+ cwd=cwd,
81
+ env=env,
82
+ stdout=subprocess.PIPE,
83
+ stderr=subprocess.PIPE,
84
+ text=True,
85
+ encoding="utf-8",
86
+ errors="replace",
87
+ start_new_session=os.name != "nt",
88
+ creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0,
89
+ )
90
+ stdout_chunks: list[str] = []
91
+ stderr_chunks: list[str] = []
92
+ output_queue: queue.Queue[tuple[str, str | None]] = queue.Queue()
93
+
94
+ def read_stream(name: str, stream: Any) -> None:
95
+ try:
96
+ for line in stream:
97
+ output_queue.put((name, line))
98
+ finally:
99
+ output_queue.put((name, None))
100
+
101
+ assert process.stdout is not None
102
+ assert process.stderr is not None
103
+ threads = [
104
+ threading.Thread(target=read_stream, args=("stdout", process.stdout), daemon=True),
105
+ threading.Thread(target=read_stream, args=("stderr", process.stderr), daemon=True),
106
+ ]
107
+ for thread in threads:
108
+ thread.start()
109
+ deadline = None if timeout is None else time.monotonic() + timeout
110
+ closed_streams: set[str] = set()
111
+ try:
112
+ while len(closed_streams) < 2:
113
+ if deadline is not None and time.monotonic() >= deadline:
114
+ raise subprocess.TimeoutExpired(command, timeout)
115
+ try:
116
+ stream_name, line = output_queue.get(timeout=0.1)
117
+ except queue.Empty:
118
+ if process.poll() is not None and all(not thread.is_alive() for thread in threads):
119
+ break
120
+ continue
121
+ if line is None:
122
+ closed_streams.add(stream_name)
123
+ continue
124
+ if stream_name == "stdout":
125
+ stdout_chunks.append(line)
126
+ print(line, end="", flush=True)
127
+ if adapter is not None:
128
+ for record in adapter.observe_stdout(line):
129
+ append_progress_record(record)
130
+ else:
131
+ stderr_chunks.append(line)
132
+ print(line, end="", file=sys.stderr, flush=True)
133
+ returncode = process.wait(timeout=1)
134
+ except subprocess.TimeoutExpired:
135
+ _terminate_process_tree(process)
136
+ stdout, stderr = process.communicate()
137
+ stdout_chunks.append(stdout)
138
+ stderr_chunks.append(stderr)
139
+ raise
140
+ return subprocess.CompletedProcess(
141
+ command,
142
+ returncode,
143
+ stdout="".join(stdout_chunks),
144
+ stderr="".join(stderr_chunks),
145
+ )
146
+
147
+
148
+ def _terminate_process_tree(process: subprocess.Popen[str]) -> None:
149
+ if process.poll() is not None:
150
+ return
151
+ if os.name == "nt":
152
+ process.send_signal(signal.CTRL_BREAK_EVENT)
153
+ try:
154
+ process.wait(timeout=5)
155
+ return
156
+ except subprocess.TimeoutExpired:
157
+ process.kill()
158
+ return
159
+ try:
160
+ os.killpg(process.pid, signal.SIGTERM)
161
+ except ProcessLookupError:
162
+ return
163
+ try:
164
+ process.wait(timeout=10)
165
+ except subprocess.TimeoutExpired:
166
+ try:
167
+ os.killpg(process.pid, signal.SIGKILL)
168
+ except ProcessLookupError:
169
+ return
@@ -0,0 +1,134 @@
1
+ """Generic progress adapters for bounded command JARVIS packages."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+ from typing import Protocol
11
+
12
+ PROGRESS_FILE_ENV = "CLIO_RELAY_PROGRESS_FILE"
13
+ PROGRESS_TOKEN_ENV = "CLIO_RELAY_PROGRESS_TOKEN"
14
+ PACKAGE_NAME = "clio_relay.bounded_command"
15
+
16
+
17
+ class ProgressAdapter(Protocol):
18
+ """Observe application output and emit structured progress records."""
19
+
20
+ def observe_stdout(self, line: str) -> list[dict[str, object]]:
21
+ """Return zero or more progress records derived from one stdout line."""
22
+ ...
23
+
24
+
25
+ @dataclass
26
+ class GenericRegexProgressAdapter:
27
+ """Extract progress from stdout using caller-supplied regular expressions."""
28
+
29
+ pattern: re.Pattern[str]
30
+ label: str = "progress"
31
+ unit: str | None = None
32
+ current_group: str = "current"
33
+ total_group: str | None = None
34
+ message_group: str | None = None
35
+ metadata: dict[str, object] = field(default_factory=dict)
36
+
37
+ def observe_stdout(self, line: str) -> list[dict[str, object]]:
38
+ """Extract all matching progress observations from a stdout line."""
39
+ records: list[dict[str, object]] = []
40
+ for match in self.pattern.finditer(line):
41
+ current = _group_float(match, self.current_group)
42
+ total = _group_float(match, self.total_group) if self.total_group else None
43
+ message = _group_text(match, self.message_group) if self.message_group else None
44
+ records.append(
45
+ _drop_none(
46
+ {
47
+ "label": self.label,
48
+ "current": current,
49
+ "total": total,
50
+ "unit": self.unit,
51
+ "message": message,
52
+ "metadata": {
53
+ **_metadata(config=None, metadata=self.metadata),
54
+ "source": "jarvis_package",
55
+ "package_name": PACKAGE_NAME,
56
+ "package_version": "builtin",
57
+ "adapter": "regex",
58
+ },
59
+ }
60
+ )
61
+ )
62
+ return records
63
+
64
+
65
+ def adapter_from_config(config: object) -> ProgressAdapter | None:
66
+ """Build a progress adapter from bounded command package configuration."""
67
+ if config is None:
68
+ return None
69
+ if not isinstance(config, dict):
70
+ raise ValueError("progress must be an object")
71
+ adapter = str(config.get("adapter", "regex"))
72
+ if adapter == "none":
73
+ return None
74
+ if adapter == "regex":
75
+ pattern = config.get("pattern")
76
+ if not isinstance(pattern, str) or pattern == "":
77
+ raise ValueError("regex progress adapter requires pattern")
78
+ return GenericRegexProgressAdapter(
79
+ pattern=re.compile(pattern),
80
+ label=str(config.get("label", "progress")),
81
+ unit=_optional_str(config.get("unit")),
82
+ current_group=str(config.get("current_group", "current")),
83
+ total_group=_optional_str(config.get("total_group")),
84
+ message_group=_optional_str(config.get("message_group")),
85
+ metadata=_metadata(config),
86
+ )
87
+ raise ValueError(f"unsupported progress adapter: {adapter}")
88
+
89
+
90
+ def append_progress_record(record: dict[str, object]) -> None:
91
+ """Append a trusted package progress record to the relay side-channel file."""
92
+ path_value = os.getenv(PROGRESS_FILE_ENV)
93
+ token = os.getenv(PROGRESS_TOKEN_ENV)
94
+ if path_value is None or path_value == "" or token is None or token == "":
95
+ return
96
+ metadata = record.get("metadata")
97
+ if not isinstance(metadata, dict):
98
+ metadata = {}
99
+ record = {**record, "metadata": {**metadata, "relay_progress_token": token}}
100
+ path = Path(path_value)
101
+ path.parent.mkdir(parents=True, exist_ok=True)
102
+ with path.open("a", encoding="utf-8") as handle:
103
+ handle.write(json.dumps(record, sort_keys=True))
104
+ handle.write("\n")
105
+
106
+
107
+ def _group_text(match: re.Match[str], group: str | None) -> str | None:
108
+ if group is None:
109
+ return None
110
+ return match.group(int(group)) if group.isdigit() else match.group(group)
111
+
112
+
113
+ def _group_float(match: re.Match[str], group: str) -> float:
114
+ return float(_group_text(match, group))
115
+
116
+
117
+ def _optional_str(value: object) -> str | None:
118
+ return value if isinstance(value, str) and value != "" else None
119
+
120
+
121
+ def _metadata(
122
+ config: dict[str, object] | None = None,
123
+ *,
124
+ metadata: object | None = None,
125
+ ) -> dict[str, object]:
126
+ value = config.get("metadata", {}) if config is not None else metadata
127
+ if not isinstance(value, dict):
128
+ return {}
129
+ protected = {"source", "package_name", "package_version", "run_id", "execution_id", "adapter"}
130
+ return {str(key): item for key, item in value.items() if str(key) not in protected}
131
+
132
+
133
+ def _drop_none(value: dict[str, object | None]) -> dict[str, object]:
134
+ return {key: item for key, item in value.items() if item is not None}
@@ -0,0 +1 @@
1
+ """MCP call JARVIS package."""