loopotel 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopotel-0.1.0/.github/workflows/publish.yml +31 -0
- loopotel-0.1.0/.github/workflows/test.yml +36 -0
- loopotel-0.1.0/.gitignore +9 -0
- loopotel-0.1.0/LICENSE +21 -0
- loopotel-0.1.0/PKG-INFO +111 -0
- loopotel-0.1.0/PLAN.md +51 -0
- loopotel-0.1.0/PUBLISHING.md +41 -0
- loopotel-0.1.0/README.md +83 -0
- loopotel-0.1.0/STATUS.md +30 -0
- loopotel-0.1.0/SYNC.md +10 -0
- loopotel-0.1.0/examples/export_loopgym_ltf.py +67 -0
- loopotel-0.1.0/examples/grafana-dashboard.json +134 -0
- loopotel-0.1.0/loopotel/__init__.py +6 -0
- loopotel-0.1.0/loopotel/cli.py +40 -0
- loopotel-0.1.0/loopotel/exporter/__init__.py +6 -0
- loopotel-0.1.0/loopotel/exporter/jsonl.py +22 -0
- loopotel-0.1.0/loopotel/exporter/loopnet.py +35 -0
- loopotel-0.1.0/loopotel/exporter/otlp.py +57 -0
- loopotel-0.1.0/loopotel/integrations/__init__.py +5 -0
- loopotel-0.1.0/loopotel/integrations/loopgym.py +104 -0
- loopotel-0.1.0/loopotel/les.py +18 -0
- loopotel-0.1.0/loopotel/models.py +71 -0
- loopotel-0.1.0/loopotel/schemas/ltf-0.1.schema.json +105 -0
- loopotel-0.1.0/loopotel/tracer.py +235 -0
- loopotel-0.1.0/loopotel/validate.py +32 -0
- loopotel-0.1.0/pyproject.toml +57 -0
- loopotel-0.1.0/scripts/validate_ltf.py +55 -0
- loopotel-0.1.0/specs/les-timeseries.md +37 -0
- loopotel-0.1.0/specs/ltf-0.1.schema.json +127 -0
- loopotel-0.1.0/specs/otel-semconv-loop.md +71 -0
- loopotel-0.1.0/tests/test_loopgym_integration.py +35 -0
- loopotel-0.1.0/tests/test_tracer.py +42 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
id-token: write
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
publish:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.12"
|
|
20
|
+
|
|
21
|
+
- name: Install build tools
|
|
22
|
+
run: pip install hatchling build
|
|
23
|
+
|
|
24
|
+
- name: Build package
|
|
25
|
+
run: python -m build
|
|
26
|
+
|
|
27
|
+
- name: Publish to PyPI
|
|
28
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
29
|
+
with:
|
|
30
|
+
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
31
|
+
skip-existing: true
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: test-loop-observability
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, master]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main, master]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
with:
|
|
17
|
+
repository: KanakMalpani/LoopGym
|
|
18
|
+
path: deps/loopgym
|
|
19
|
+
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
|
|
24
|
+
- name: Install package and LoopGym
|
|
25
|
+
run: |
|
|
26
|
+
pip install -e ".[dev]"
|
|
27
|
+
pip install -e deps/loopgym
|
|
28
|
+
|
|
29
|
+
- name: Run tests
|
|
30
|
+
run: pytest tests/ -q
|
|
31
|
+
|
|
32
|
+
- name: Export sample LTF from LoopGym
|
|
33
|
+
run: python examples/export_loopgym_ltf.py examples/sample-trace.jsonl
|
|
34
|
+
|
|
35
|
+
- name: Validate sample trace
|
|
36
|
+
run: loopotel-validate examples/sample-trace.jsonl
|
loopotel-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 KanakMalpani
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
loopotel-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: loopotel
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Loop Trace Format (LTF) instrumentation and exporters for loop observability
|
|
5
|
+
Project-URL: Homepage, https://github.com/KanakMalpani/loop-observability
|
|
6
|
+
Project-URL: Repository, https://github.com/KanakMalpani/loop-observability
|
|
7
|
+
Project-URL: Loop Core Engineering, https://github.com/KanakMalpani/Loop-Core-Engineering
|
|
8
|
+
Project-URL: LoopGym, https://github.com/KanakMalpani/LoopGym
|
|
9
|
+
Author: Kanak Malpani
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: les,loop-engineering,ltf,observability,opentelemetry
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Requires-Dist: jsonschema>=4.21
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: loopgym>=0.1.0; extra == 'dev'
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
22
|
+
Provides-Extra: loopgym
|
|
23
|
+
Requires-Dist: loopgym>=0.1.0; extra == 'loopgym'
|
|
24
|
+
Provides-Extra: otlp
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27; extra == 'otlp'
|
|
26
|
+
Requires-Dist: opentelemetry-sdk>=1.27; extra == 'otlp'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# Loop Observability
|
|
30
|
+
|
|
31
|
+
**Loop Trace Format (LTF)** and OpenTelemetry conventions for production loop monitoring.
|
|
32
|
+
|
|
33
|
+
SREs need spans for iterations, evaluators, token burn, and LES deltas — not raw chat logs. This repo defines the format and ships `loopotel`, a minimal Python instrumentation library.
|
|
34
|
+
|
|
35
|
+
[](https://github.com/KanakMalpani/loop-observability/actions/workflows/test.yml)
|
|
36
|
+
[](https://pypi.org/project/loopotel/)
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install loopotel
|
|
42
|
+
pip install "loopotel[loopgym]" # LoopGym episode tracing
|
|
43
|
+
pip install "loopotel[otlp]" # OTLP export
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Quick start — trace a LoopGym run
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import loopgym as lg
|
|
50
|
+
from loopotel.integrations.loopgym import run_traced_episode
|
|
51
|
+
from loopotel.exporter.jsonl import JsonlExporter
|
|
52
|
+
|
|
53
|
+
env = lg.make("loopbench/code-repair-v1")
|
|
54
|
+
result, trace = run_traced_episode(env, task_id="cr-001", seed=42, enabled=True)
|
|
55
|
+
|
|
56
|
+
JsonlExporter("traces.jsonl").export(trace)
|
|
57
|
+
print(result["success"], trace["trace_id"])
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Or run the example:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install loopgym loopotel
|
|
64
|
+
python examples/export_loopgym_ltf.py
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## API
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from loopotel import LoopTracer, emit_iteration, trace_loop
|
|
71
|
+
|
|
72
|
+
with LoopTracer(loop_name="my-loop", env_id="prod/agent") as tracer:
|
|
73
|
+
emit_iteration(iteration=1, goal_score=0.55, tokens_delta=120,
|
|
74
|
+
worker_id="implementer", evaluator_id="rubric")
|
|
75
|
+
tracer.finish(outcome="success", termination_reason="goal_met")
|
|
76
|
+
|
|
77
|
+
trace = tracer.build_trace() # ltf/0.1 document
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Specs
|
|
81
|
+
|
|
82
|
+
| Document | Purpose |
|
|
83
|
+
|----------|---------|
|
|
84
|
+
| [`specs/ltf-0.1.schema.json`](specs/ltf-0.1.schema.json) | LTF JSON schema |
|
|
85
|
+
| [`specs/otel-semconv-loop.md`](specs/otel-semconv-loop.md) | `loop.*` OTel attributes |
|
|
86
|
+
| [`specs/les-timeseries.md`](specs/les-timeseries.md) | Point-in-loop LES metrics |
|
|
87
|
+
|
|
88
|
+
## Grafana
|
|
89
|
+
|
|
90
|
+
Import [`examples/grafana-dashboard.json`](examples/grafana-dashboard.json) for iteration vs goal score, cumulative LES, and token burn panels (sample data included).
|
|
91
|
+
|
|
92
|
+
## Validate
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
loopotel-validate examples/sample-trace.jsonl
|
|
96
|
+
python scripts/validate_ltf.py path/to/trace.json
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Design
|
|
100
|
+
|
|
101
|
+
- **Minimal overhead** — tracing off by default; pass `enabled=True` for SimEnv, use `trace_live_episode()` for LiveEnv
|
|
102
|
+
- **Exporters** — JSONL (built-in), OTLP (optional), LoopNet trajectory mapping
|
|
103
|
+
- **Pins** — `lss@1.0.0`, `les@1.0.0`, `ltf@0.1.0`
|
|
104
|
+
|
|
105
|
+
## Links
|
|
106
|
+
|
|
107
|
+
- [LoopNet end-to-end tutorial](https://github.com/KanakMalpani/loopnet/blob/main/guides/END-TO-END-TUTORIAL.md) — HF → replay → LoopBench
|
|
108
|
+
- [Loop Core Engineering](https://github.com/KanakMalpani/Loop-Core-Engineering) — LES / LSS
|
|
109
|
+
- [LoopGym](https://github.com/KanakMalpani/LoopGym) — instrumentation target
|
|
110
|
+
- [LoopNet](https://github.com/KanakMalpani/loopnet) — trajectory corpus export
|
|
111
|
+
- [Publishing](PUBLISHING.md) · [PyPI](https://pypi.org/project/loopotel/)
|
loopotel-0.1.0/PLAN.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# 07 — loop-observability
|
|
2
|
+
|
|
3
|
+
## One-line purpose
|
|
4
|
+
|
|
5
|
+
**Loop Trace Format (LTF)** and OpenTelemetry conventions for production loop monitoring.
|
|
6
|
+
|
|
7
|
+
## Why this repo exists
|
|
8
|
+
|
|
9
|
+
Industry adopts what they can **operate**. SREs need spans for iterations, evaluators, token burn, LES deltas — not raw chat logs.
|
|
10
|
+
|
|
11
|
+
## Scope (in scope)
|
|
12
|
+
|
|
13
|
+
- LTF JSON schema (one span per iteration, child spans per worker/evaluator)
|
|
14
|
+
- OpenTelemetry semantic conventions draft (`loop.*` attributes)
|
|
15
|
+
- Exporters: OTLP, JSONL, LoopNet trajectory export
|
|
16
|
+
- Reference instrumentation for LoopGym / LangGraph
|
|
17
|
+
- Dashboard templates (Grafana JSON v0.1)
|
|
18
|
+
- LES time-series spec (point-in-loop metrics)
|
|
19
|
+
|
|
20
|
+
## Scope (out of scope)
|
|
21
|
+
|
|
22
|
+
- Full SaaS observability product
|
|
23
|
+
- Log storage infrastructure
|
|
24
|
+
|
|
25
|
+
## Deliverables v0.1
|
|
26
|
+
|
|
27
|
+
- [x] `specs/ltf-0.1.schema.json`
|
|
28
|
+
- [x] `specs/otel-semconv-loop.md`
|
|
29
|
+
- [x] `loopotel/` Python package — `@trace_loop`, `emit_iteration()`
|
|
30
|
+
- [x] `examples/grafana-dashboard.json`
|
|
31
|
+
|
|
32
|
+
## Status
|
|
33
|
+
|
|
34
|
+
✅ v0.1 shipped — see [STATUS.md](STATUS.md)
|
|
35
|
+
|
|
36
|
+
## Dependencies
|
|
37
|
+
|
|
38
|
+
- **01-loop-engineering-core** — LES dimensions, worker/evaluator IDs
|
|
39
|
+
- **05-loopgym** — instrumentation target
|
|
40
|
+
|
|
41
|
+
## Success criteria
|
|
42
|
+
|
|
43
|
+
One LoopGym run exports valid LTF; Grafana dashboard shows iterations vs cumulative LES.
|
|
44
|
+
|
|
45
|
+
## Agent instructions
|
|
46
|
+
|
|
47
|
+
Design for **minimal overhead**; default off in SimEnv, on in LiveEnv.
|
|
48
|
+
|
|
49
|
+
## Status
|
|
50
|
+
|
|
51
|
+
🟡 Planning only
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Publishing loopotel to PyPI
|
|
2
|
+
|
|
3
|
+
## One-time setup
|
|
4
|
+
|
|
5
|
+
1. Register the project name **`loopotel`** at [pypi.org](https://pypi.org/).
|
|
6
|
+
2. **Preferred:** [trusted publishing](https://docs.pypi.org/trusted-publishers/) on PyPI:
|
|
7
|
+
- **Project:** `loopotel`
|
|
8
|
+
- **Owner:** `KanakMalpani`
|
|
9
|
+
- **Repository:** `loop-observability`
|
|
10
|
+
- **Workflow:** `publish.yml`
|
|
11
|
+
3. **Fallback:** add **`PYPI_API_TOKEN`** to [loop-observability Actions secrets](https://github.com/KanakMalpani/loop-observability/settings/secrets/actions) (reuse the same upload token as LoopGym/LoopBench).
|
|
12
|
+
|
|
13
|
+
## Publish
|
|
14
|
+
|
|
15
|
+
Release tag:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
git tag v0.1.0
|
|
19
|
+
git push origin v0.1.0
|
|
20
|
+
gh release create v0.1.0 --title "v0.1.0" --notes "Initial loopotel release"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Or **Actions → Publish to PyPI → Run workflow**.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install loopotel
|
|
29
|
+
pip install "loopotel[loopgym]" # LoopGym integration
|
|
30
|
+
pip install "loopotel[otlp]" # OTLP export
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Verify locally
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install build
|
|
37
|
+
python -m build
|
|
38
|
+
pip install dist/loopotel-*.whl
|
|
39
|
+
loopotel-validate --help
|
|
40
|
+
pytest tests/ -q
|
|
41
|
+
```
|
loopotel-0.1.0/README.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Loop Observability
|
|
2
|
+
|
|
3
|
+
**Loop Trace Format (LTF)** and OpenTelemetry conventions for production loop monitoring.
|
|
4
|
+
|
|
5
|
+
SREs need spans for iterations, evaluators, token burn, and LES deltas — not raw chat logs. This repo defines the format and ships `loopotel`, a minimal Python instrumentation library.
|
|
6
|
+
|
|
7
|
+
[](https://github.com/KanakMalpani/loop-observability/actions/workflows/test.yml)
|
|
8
|
+
[](https://pypi.org/project/loopotel/)
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install loopotel
|
|
14
|
+
pip install "loopotel[loopgym]" # LoopGym episode tracing
|
|
15
|
+
pip install "loopotel[otlp]" # OTLP export
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Quick start — trace a LoopGym run
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import loopgym as lg
|
|
22
|
+
from loopotel.integrations.loopgym import run_traced_episode
|
|
23
|
+
from loopotel.exporter.jsonl import JsonlExporter
|
|
24
|
+
|
|
25
|
+
env = lg.make("loopbench/code-repair-v1")
|
|
26
|
+
result, trace = run_traced_episode(env, task_id="cr-001", seed=42, enabled=True)
|
|
27
|
+
|
|
28
|
+
JsonlExporter("traces.jsonl").export(trace)
|
|
29
|
+
print(result["success"], trace["trace_id"])
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Or run the example:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install loopgym loopotel
|
|
36
|
+
python examples/export_loopgym_ltf.py
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## API
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from loopotel import LoopTracer, emit_iteration, trace_loop
|
|
43
|
+
|
|
44
|
+
with LoopTracer(loop_name="my-loop", env_id="prod/agent") as tracer:
|
|
45
|
+
emit_iteration(iteration=1, goal_score=0.55, tokens_delta=120,
|
|
46
|
+
worker_id="implementer", evaluator_id="rubric")
|
|
47
|
+
tracer.finish(outcome="success", termination_reason="goal_met")
|
|
48
|
+
|
|
49
|
+
trace = tracer.build_trace() # ltf/0.1 document
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Specs
|
|
53
|
+
|
|
54
|
+
| Document | Purpose |
|
|
55
|
+
|----------|---------|
|
|
56
|
+
| [`specs/ltf-0.1.schema.json`](specs/ltf-0.1.schema.json) | LTF JSON schema |
|
|
57
|
+
| [`specs/otel-semconv-loop.md`](specs/otel-semconv-loop.md) | `loop.*` OTel attributes |
|
|
58
|
+
| [`specs/les-timeseries.md`](specs/les-timeseries.md) | Point-in-loop LES metrics |
|
|
59
|
+
|
|
60
|
+
## Grafana
|
|
61
|
+
|
|
62
|
+
Import [`examples/grafana-dashboard.json`](examples/grafana-dashboard.json) for iteration vs goal score, cumulative LES, and token burn panels (sample data included).
|
|
63
|
+
|
|
64
|
+
## Validate
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
loopotel-validate examples/sample-trace.jsonl
|
|
68
|
+
python scripts/validate_ltf.py path/to/trace.json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Design
|
|
72
|
+
|
|
73
|
+
- **Minimal overhead** — tracing off by default; pass `enabled=True` for SimEnv, use `trace_live_episode()` for LiveEnv
|
|
74
|
+
- **Exporters** — JSONL (built-in), OTLP (optional), LoopNet trajectory mapping
|
|
75
|
+
- **Pins** — `lss@1.0.0`, `les@1.0.0`, `ltf@0.1.0`
|
|
76
|
+
|
|
77
|
+
## Links
|
|
78
|
+
|
|
79
|
+
- [LoopNet end-to-end tutorial](https://github.com/KanakMalpani/loopnet/blob/main/guides/END-TO-END-TUTORIAL.md) — HF → replay → LoopBench
|
|
80
|
+
- [Loop Core Engineering](https://github.com/KanakMalpani/Loop-Core-Engineering) — LES / LSS
|
|
81
|
+
- [LoopGym](https://github.com/KanakMalpani/LoopGym) — instrumentation target
|
|
82
|
+
- [LoopNet](https://github.com/KanakMalpani/loopnet) — trajectory corpus export
|
|
83
|
+
- [Publishing](PUBLISHING.md) · [PyPI](https://pypi.org/project/loopotel/)
|
loopotel-0.1.0/STATUS.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Status
|
|
2
|
+
|
|
3
|
+
| Field | Value |
|
|
4
|
+
|-------|-------|
|
|
5
|
+
| **Phase** | v0.1 shipped |
|
|
6
|
+
| **Symbol** | ✅ |
|
|
7
|
+
| **Notes** | LTF schema, loopotel package, LoopGym integration, Grafana template |
|
|
8
|
+
|
|
9
|
+
## Completion checklist
|
|
10
|
+
|
|
11
|
+
- [x] `specs/ltf-0.1.schema.json`
|
|
12
|
+
- [x] `specs/otel-semconv-loop.md`
|
|
13
|
+
- [x] `specs/les-timeseries.md`
|
|
14
|
+
- [x] `loopotel/` — `LoopTracer`, `@trace_loop`, `emit_iteration()`
|
|
15
|
+
- [x] Exporters: JSONL, OTLP (optional), LoopNet trajectory
|
|
16
|
+
- [x] `loopotel.integrations.loopgym` — `run_traced_episode()`
|
|
17
|
+
- [x] `examples/grafana-dashboard.json`
|
|
18
|
+
- [x] `examples/export_loopgym_ltf.py`
|
|
19
|
+
- [x] CI: pytest + LTF validation
|
|
20
|
+
- [ ] PyPI publish `loopotel` — add `PYPI_API_TOKEN` secret or trusted publisher, then run [Publish to PyPI](https://github.com/KanakMalpani/loop-observability/actions/workflows/publish.yml) (see [PUBLISHING.md](PUBLISHING.md))
|
|
21
|
+
|
|
22
|
+
## Success criteria
|
|
23
|
+
|
|
24
|
+
- [x] One LoopGym run exports valid LTF
|
|
25
|
+
- [x] Grafana dashboard template for iterations vs cumulative LES
|
|
26
|
+
|
|
27
|
+
## Links
|
|
28
|
+
|
|
29
|
+
- Parent workspace: [../README.md](../README.md)
|
|
30
|
+
- Plan: [PLAN.md](PLAN.md)
|
loopotel-0.1.0/SYNC.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Sync policy
|
|
2
|
+
|
|
3
|
+
| Artifact | Canonical home | This repo |
|
|
4
|
+
|----------|----------------|-----------|
|
|
5
|
+
| LSS / LES specs | [Loop Core Engineering](https://github.com/KanakMalpani/Loop-Core-Engineering) | Pins only in LTF |
|
|
6
|
+
| LTF schema | **This repo** `specs/ltf-0.1.schema.json` | Bundled copy in `loopotel/schemas/` |
|
|
7
|
+
| LoopNet trajectory | [loopnet](https://github.com/KanakMalpani/loopnet) | Export via `trajectory_from_trace()` |
|
|
8
|
+
| LoopGym runtime | [LoopGym](https://github.com/KanakMalpani/LoopGym) | Integration in `loopotel.integrations.loopgym` |
|
|
9
|
+
|
|
10
|
+
Do not fork LSS/LES schemas here. Reference pins in every LTF document via `spec_pins`.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Export a LoopGym SimEnv run as valid LTF JSONL."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
11
|
+
DEFAULT_OUT = ROOT / "examples" / "sample-trace.jsonl"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main() -> int:
|
|
15
|
+
try:
|
|
16
|
+
import loopgym as lg
|
|
17
|
+
except ImportError:
|
|
18
|
+
print("Install loopgym: pip install loopgym", file=sys.stderr)
|
|
19
|
+
return 1
|
|
20
|
+
|
|
21
|
+
from loopotel.exporter.jsonl import JsonlExporter
|
|
22
|
+
from loopotel.integrations.loopgym import run_traced_episode
|
|
23
|
+
from loopotel.validate import validate_trace
|
|
24
|
+
|
|
25
|
+
env = lg.make("loopbench/code-repair-v1")
|
|
26
|
+
_result, trace = run_traced_episode(env, task_id="cr-001", seed=42, enabled=True)
|
|
27
|
+
|
|
28
|
+
valid, errors = validate_trace(trace)
|
|
29
|
+
if not valid:
|
|
30
|
+
print("LTF validation failed:", file=sys.stderr)
|
|
31
|
+
for err in errors:
|
|
32
|
+
print(f" - {err}", file=sys.stderr)
|
|
33
|
+
return 1
|
|
34
|
+
|
|
35
|
+
out_path = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUT
|
|
36
|
+
JsonlExporter(out_path).export(trace)
|
|
37
|
+
|
|
38
|
+
iterations = sum(1 for s in trace["spans"] if s["kind"] == "iteration")
|
|
39
|
+
last_les = next(
|
|
40
|
+
(
|
|
41
|
+
s["attributes"].get("loop.les.cumulative_normalized")
|
|
42
|
+
for s in reversed(trace["spans"])
|
|
43
|
+
if s["kind"] == "iteration"
|
|
44
|
+
),
|
|
45
|
+
0,
|
|
46
|
+
)
|
|
47
|
+
print(f"Wrote LTF trace -> {out_path}")
|
|
48
|
+
print(f" trace_id: {trace['trace_id']}")
|
|
49
|
+
print(f" iterations: {iterations}")
|
|
50
|
+
print(f" cumulative LES: {last_les}")
|
|
51
|
+
print(f" outcome: {trace['outcome']}")
|
|
52
|
+
print()
|
|
53
|
+
print("Iteration series (for Grafana):")
|
|
54
|
+
for span in trace["spans"]:
|
|
55
|
+
if span["kind"] != "iteration":
|
|
56
|
+
continue
|
|
57
|
+
attrs = span["attributes"]
|
|
58
|
+
print(
|
|
59
|
+
f" iter={attrs['loop.iteration']} "
|
|
60
|
+
f"goal={attrs['loop.goal_score']:.3f} "
|
|
61
|
+
f"les={attrs['loop.les.cumulative_normalized']:.3f}"
|
|
62
|
+
)
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
{
|
|
2
|
+
"annotations": {
|
|
3
|
+
"list": [
|
|
4
|
+
{
|
|
5
|
+
"builtIn": 1,
|
|
6
|
+
"enable": true,
|
|
7
|
+
"hide": true,
|
|
8
|
+
"iconColor": "rgba(0, 211, 255, 1)",
|
|
9
|
+
"name": "Annotations & Alerts",
|
|
10
|
+
"type": "dashboard"
|
|
11
|
+
}
|
|
12
|
+
]
|
|
13
|
+
},
|
|
14
|
+
"editable": true,
|
|
15
|
+
"fiscalYearStartMonth": 0,
|
|
16
|
+
"graphTooltip": 0,
|
|
17
|
+
"id": null,
|
|
18
|
+
"links": [],
|
|
19
|
+
"panels": [
|
|
20
|
+
{
|
|
21
|
+
"datasource": { "type": "grafana-testdata-datasource", "uid": "grafana" },
|
|
22
|
+
"fieldConfig": {
|
|
23
|
+
"defaults": {
|
|
24
|
+
"color": { "mode": "palette-classic" },
|
|
25
|
+
"custom": {
|
|
26
|
+
"axisBorderShow": false,
|
|
27
|
+
"drawStyle": "line",
|
|
28
|
+
"fillOpacity": 10,
|
|
29
|
+
"lineWidth": 2,
|
|
30
|
+
"pointSize": 6,
|
|
31
|
+
"showPoints": "always"
|
|
32
|
+
},
|
|
33
|
+
"unit": "none"
|
|
34
|
+
},
|
|
35
|
+
"overrides": []
|
|
36
|
+
},
|
|
37
|
+
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
|
38
|
+
"id": 1,
|
|
39
|
+
"options": {
|
|
40
|
+
"legend": { "displayMode": "list", "placement": "bottom" },
|
|
41
|
+
"tooltip": { "mode": "single" }
|
|
42
|
+
},
|
|
43
|
+
"targets": [
|
|
44
|
+
{
|
|
45
|
+
"refId": "A",
|
|
46
|
+
"scenarioId": "csv_content",
|
|
47
|
+
"csvContent": "iteration,goal_score\n1,0.57\n2,0.69\n3,0.81\n4,0.88",
|
|
48
|
+
"csvFileName": "goal_score.csv"
|
|
49
|
+
}
|
|
50
|
+
],
|
|
51
|
+
"title": "Goal score by iteration",
|
|
52
|
+
"type": "timeseries"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"datasource": { "type": "grafana-testdata-datasource", "uid": "grafana" },
|
|
56
|
+
"fieldConfig": {
|
|
57
|
+
"defaults": {
|
|
58
|
+
"color": { "mode": "palette-classic" },
|
|
59
|
+
"custom": {
|
|
60
|
+
"axisBorderShow": false,
|
|
61
|
+
"drawStyle": "line",
|
|
62
|
+
"fillOpacity": 10,
|
|
63
|
+
"lineWidth": 2,
|
|
64
|
+
"pointSize": 6,
|
|
65
|
+
"showPoints": "always"
|
|
66
|
+
},
|
|
67
|
+
"max": 1,
|
|
68
|
+
"min": 0,
|
|
69
|
+
"unit": "none"
|
|
70
|
+
},
|
|
71
|
+
"overrides": []
|
|
72
|
+
},
|
|
73
|
+
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
|
74
|
+
"id": 2,
|
|
75
|
+
"options": {
|
|
76
|
+
"legend": { "displayMode": "list", "placement": "bottom" },
|
|
77
|
+
"tooltip": { "mode": "single" }
|
|
78
|
+
},
|
|
79
|
+
"targets": [
|
|
80
|
+
{
|
|
81
|
+
"refId": "A",
|
|
82
|
+
"scenarioId": "csv_content",
|
|
83
|
+
"csvContent": "iteration,les_cumulative\n1,0.71\n2,0.78\n3,0.84\n4,0.88",
|
|
84
|
+
"csvFileName": "les_cumulative.csv"
|
|
85
|
+
}
|
|
86
|
+
],
|
|
87
|
+
"title": "Cumulative LES (normalized)",
|
|
88
|
+
"type": "timeseries"
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"datasource": { "type": "grafana-testdata-datasource", "uid": "grafana" },
|
|
92
|
+
"fieldConfig": {
|
|
93
|
+
"defaults": {
|
|
94
|
+
"color": { "mode": "palette-classic" },
|
|
95
|
+
"custom": {
|
|
96
|
+
"axisBorderShow": false,
|
|
97
|
+
"drawStyle": "bars",
|
|
98
|
+
"fillOpacity": 80,
|
|
99
|
+
"lineWidth": 1
|
|
100
|
+
},
|
|
101
|
+
"unit": "none"
|
|
102
|
+
},
|
|
103
|
+
"overrides": []
|
|
104
|
+
},
|
|
105
|
+
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
|
|
106
|
+
"id": 3,
|
|
107
|
+
"options": {
|
|
108
|
+
"legend": { "displayMode": "list", "placement": "bottom" },
|
|
109
|
+
"tooltip": { "mode": "single" }
|
|
110
|
+
},
|
|
111
|
+
"targets": [
|
|
112
|
+
{
|
|
113
|
+
"refId": "A",
|
|
114
|
+
"scenarioId": "csv_content",
|
|
115
|
+
"csvContent": "iteration,tokens_delta\n1,159\n2,159\n3,159\n4,159",
|
|
116
|
+
"csvFileName": "tokens.csv"
|
|
117
|
+
}
|
|
118
|
+
],
|
|
119
|
+
"title": "Token burn per iteration",
|
|
120
|
+
"type": "timeseries"
|
|
121
|
+
}
|
|
122
|
+
],
|
|
123
|
+
"refresh": "",
|
|
124
|
+
"schemaVersion": 39,
|
|
125
|
+
"tags": ["loop-engineering", "ltf", "les"],
|
|
126
|
+
"templating": { "list": [] },
|
|
127
|
+
"time": { "from": "now-6h", "to": "now" },
|
|
128
|
+
"timepicker": {},
|
|
129
|
+
"timezone": "browser",
|
|
130
|
+
"title": "Loop Engineering — LTF v0.1",
|
|
131
|
+
"uid": "loop-ltf-v01",
|
|
132
|
+
"version": 1,
|
|
133
|
+
"description": "Sample dashboard for loop.iteration metrics. Replace testdata with LTF JSONL → Prometheus/Loki pipeline or Grafana Infinity datasource."
|
|
134
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""CLI for LTF validation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from loopotel.validate import validate_trace
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(argv: list[str] | None = None) -> int:
|
|
14
|
+
parser = argparse.ArgumentParser(description="Validate LTF trace JSON or JSONL")
|
|
15
|
+
parser.add_argument("path", type=Path)
|
|
16
|
+
args = parser.parse_args(argv)
|
|
17
|
+
|
|
18
|
+
if not args.path.exists():
|
|
19
|
+
print(f"Missing: {args.path}", file=sys.stderr)
|
|
20
|
+
return 1
|
|
21
|
+
|
|
22
|
+
lines = args.path.read_text(encoding="utf-8").strip().splitlines()
|
|
23
|
+
docs = [json.loads(line) for line in lines if line.strip()] if args.path.suffix == ".jsonl" else [json.loads(args.path.read_text(encoding="utf-8"))]
|
|
24
|
+
|
|
25
|
+
failed = False
|
|
26
|
+
for index, doc in enumerate(docs, start=1):
|
|
27
|
+
valid, errors = validate_trace(doc)
|
|
28
|
+
label = str(args.path) if len(docs) == 1 else f"{args.path}#{index}"
|
|
29
|
+
if valid:
|
|
30
|
+
print(f"VALID: {label}")
|
|
31
|
+
else:
|
|
32
|
+
failed = True
|
|
33
|
+
print(f"INVALID: {label}", file=sys.stderr)
|
|
34
|
+
for err in errors:
|
|
35
|
+
print(f" - {err}", file=sys.stderr)
|
|
36
|
+
return 1 if failed else 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
if __name__ == "__main__":
|
|
40
|
+
raise SystemExit(main())
|