agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Append explicit, normalized SDK usage to a private local JSONL file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from .config import ALL_AGENTS
|
|
11
|
+
from .observations import MAX_FILE_BYTES, MAX_RECORD_BYTES, identifier, validate_record
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def validate_envelope(value: Any) -> dict[str, Any]:
|
|
15
|
+
if (
|
|
16
|
+
not isinstance(value, dict)
|
|
17
|
+
or set(value) - {"version", "producer", "usage", "model", "provider", "native_source"}
|
|
18
|
+
or value.get("version") != 1
|
|
19
|
+
or type(value.get("version")) is not int
|
|
20
|
+
):
|
|
21
|
+
raise ValueError("invalid SDK envelope")
|
|
22
|
+
result = dict(value)
|
|
23
|
+
result["producer"] = identifier(value.get("producer"))
|
|
24
|
+
result["usage"] = validate_record(value.get("usage"))
|
|
25
|
+
if result["usage"]["kind"] != "usage":
|
|
26
|
+
raise ValueError("SDK envelope requires usage")
|
|
27
|
+
for key in ("model", "provider"):
|
|
28
|
+
item = value.get(key)
|
|
29
|
+
result[key] = item
|
|
30
|
+
if item is not None and (
|
|
31
|
+
not isinstance(item, str)
|
|
32
|
+
or not item
|
|
33
|
+
or len(item) > 128
|
|
34
|
+
or any(ord(c) < 32 for c in item)
|
|
35
|
+
):
|
|
36
|
+
raise ValueError(f"invalid SDK {key}")
|
|
37
|
+
native = value.get("native_source")
|
|
38
|
+
result["native_source"] = native
|
|
39
|
+
if native is not None and (native not in ALL_AGENTS or native == "generic_sdk"):
|
|
40
|
+
raise ValueError("invalid SDK native_source")
|
|
41
|
+
if len(json.dumps(result).encode()) > MAX_RECORD_BYTES:
|
|
42
|
+
raise ValueError("SDK record exceeds limit")
|
|
43
|
+
return result
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def emit_usage(
|
|
47
|
+
path: str | Path,
|
|
48
|
+
usage: dict[str, Any],
|
|
49
|
+
*,
|
|
50
|
+
producer: str,
|
|
51
|
+
model: str | None = None,
|
|
52
|
+
provider: str | None = None,
|
|
53
|
+
native_source: str | None = None,
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Append one validated record; callers supply stable identity and timestamp."""
|
|
56
|
+
envelope = validate_envelope(
|
|
57
|
+
dict(
|
|
58
|
+
version=1,
|
|
59
|
+
producer=producer,
|
|
60
|
+
usage=usage,
|
|
61
|
+
model=model,
|
|
62
|
+
provider=provider,
|
|
63
|
+
native_source=native_source,
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
payload = (json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n").encode()
|
|
67
|
+
fd = os.open(Path(path).expanduser(), os.O_CREAT | os.O_APPEND | os.O_RDWR, 0o600)
|
|
68
|
+
try:
|
|
69
|
+
size = os.fstat(fd).st_size
|
|
70
|
+
if size:
|
|
71
|
+
os.lseek(fd, -1, os.SEEK_END)
|
|
72
|
+
if os.read(fd, 1) != b"\n":
|
|
73
|
+
raise ValueError("SDK file has an incomplete trailing record")
|
|
74
|
+
if size + len(payload) > MAX_FILE_BYTES:
|
|
75
|
+
raise ValueError("SDK file exceeds limit; rotate it before appending")
|
|
76
|
+
if os.write(fd, payload) != len(payload):
|
|
77
|
+
raise OSError("incomplete SDK append")
|
|
78
|
+
finally:
|
|
79
|
+
os.close(fd)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The one way to reach the sink: every Postgres connection goes through connect(). Reads and
|
|
2
|
+
feature tables keep their SQL next to their feature; the scan's rows are written by push.py.
|
|
3
|
+
connect() and write_many() let the driver's errors through; cli.main reports them on one redacted
|
|
4
|
+
line.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import functools
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import sys
|
|
13
|
+
from typing import Any
|
|
14
|
+
from urllib.parse import parse_qs, unquote, urlparse
|
|
15
|
+
|
|
16
|
+
from .config import AutError
|
|
17
|
+
from .net import LOCAL_HOSTS
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _host_and_mode(dsn: str) -> tuple[str, str] | None:
|
|
21
|
+
"""(host, sslmode) a DSN names, "" where it names none; None for one that cannot be parsed."""
|
|
22
|
+
if "://" not in dsn:
|
|
23
|
+
keys = dict(re.findall(r"(\w+)\s*=\s*('[^']*'|\S+)", dsn))
|
|
24
|
+
return (keys.get("host") or keys.get("hostaddr") or "").strip("'"), \
|
|
25
|
+
keys.get("sslmode", "").strip("'")
|
|
26
|
+
try:
|
|
27
|
+
uri = urlparse(dsn)
|
|
28
|
+
except ValueError:
|
|
29
|
+
return None
|
|
30
|
+
query = parse_qs(uri.query)
|
|
31
|
+
return query.get("host", [unquote(uri.hostname or "")])[0], query.get("sslmode", [""])[0]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@functools.lru_cache(maxsize=None) # once per DSN per run: a scan connects many times
|
|
35
|
+
def _warn_if_plaintext(dsn: str) -> None:
|
|
36
|
+
"""libpq's default sslmode=prefer falls back to plaintext without a word. Warn, never refuse:
|
|
37
|
+
a Tailscale or tunnelled sink without TLS works today. An explicit sslmode=disable is the
|
|
38
|
+
opt-out. The warning names the host only: the DSN can hold the password."""
|
|
39
|
+
named = _host_and_mode(dsn)
|
|
40
|
+
if named is None:
|
|
41
|
+
return # let libpq report a DSN it cannot parse either
|
|
42
|
+
host, mode = named
|
|
43
|
+
host = host or os.environ.get("PGHOST") or ""
|
|
44
|
+
mode = mode or os.environ.get("PGSSLMODE") or ""
|
|
45
|
+
if not host or host.startswith("/") or host in LOCAL_HOSTS:
|
|
46
|
+
return # a local socket or loopback, e.g. an SSH tunnel
|
|
47
|
+
if mode not in ("disable", "require", "verify-ca", "verify-full"):
|
|
48
|
+
print(f"aut: warning: transport.pg_dsn reaches {host} without requiring TLS, so rows and "
|
|
49
|
+
"the password can travel in plaintext. Add sslmode=require (or sslmode=disable on a "
|
|
50
|
+
"private link such as Tailscale).", file=sys.stderr)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def connect(dsn: str, timeout: int = 15) -> Any:
|
|
54
|
+
"""A Postgres connection, or the operator's error naming the extra that installs the driver."""
|
|
55
|
+
try:
|
|
56
|
+
import psycopg
|
|
57
|
+
except ImportError as exc:
|
|
58
|
+
raise AutError("this needs the Postgres driver, which is not usable here: install "
|
|
59
|
+
f"agent-usage-tracker[postgres] ({exc})") from None
|
|
60
|
+
_warn_if_plaintext(dsn)
|
|
61
|
+
return psycopg.connect(dsn, connect_timeout=timeout)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def write_many(dsn: str, sql: str, rows: list[tuple[Any, ...]], timeout: int = 15) -> None:
|
|
65
|
+
"""One statement over many rows in one transaction. Raises on failure."""
|
|
66
|
+
if not rows:
|
|
67
|
+
return
|
|
68
|
+
with connect(dsn, timeout) as conn:
|
|
69
|
+
with conn.cursor() as cur:
|
|
70
|
+
cur.executemany(sql, rows)
|
|
71
|
+
conn.commit()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def check_ledger_zone(dsn: str, zone: str) -> None:
|
|
75
|
+
"""A sink from sink_stack records its AUT_TZ. A collector that counts days in another zone
|
|
76
|
+
would write rows a day off near midnight, so it stops instead. Older sinks record nothing."""
|
|
77
|
+
with connect(dsn) as conn:
|
|
78
|
+
row = conn.execute("SELECT current_setting('aut.ledger_tz', true)").fetchone()
|
|
79
|
+
sink_zone = row[0] if row else None
|
|
80
|
+
if sink_zone and sink_zone != zone:
|
|
81
|
+
raise AutError(f"ledger_timezone is {zone} but the sink counts days in {sink_zone}: "
|
|
82
|
+
f'set ledger_timezone = "{sink_zone}" in config.toml')
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# The aut sink
|
|
2
|
+
|
|
3
|
+
One Docker Compose stack holds your history and dashboards:
|
|
4
|
+
|
|
5
|
+
- `db`: TimescaleDB with the aut schema.
|
|
6
|
+
- `schema`: applies the schema on every `up`, then exits. A newer aut release updates an existing database.
|
|
7
|
+
- `refresh`: refreshes the derived views (active time, episodes, cost) every hour.
|
|
8
|
+
- `grafana`: the two aut dashboards, read through the `agents_ro` role, which never sees window titles.
|
|
9
|
+
|
|
10
|
+
The same stack runs in two places:
|
|
11
|
+
|
|
12
|
+
- **This machine only.** Keep the default `AUT_BIND=127.0.0.1`.
|
|
13
|
+
- **A homelab.** Run it on the server. Set `AUT_BIND` to the address your other machines reach it on, such as a LAN or Tailscale address.
|
|
14
|
+
Every laptop then sends its rows there. Never expose it to the internet.
|
|
15
|
+
|
|
16
|
+
## Start it
|
|
17
|
+
|
|
18
|
+
You need Docker with Compose v2.
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
git clone https://github.com/rhnfzl/agent-usage-tracker
|
|
22
|
+
cd agent-usage-tracker/src/agent_usage_tracker/sink_stack
|
|
23
|
+
cp env.example .env && chmod 600 .env
|
|
24
|
+
$EDITOR .env # four passwords (openssl rand -hex 24) and AUT_TZ
|
|
25
|
+
docker compose up -d --wait
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Grafana is at `http://<AUT_BIND>:<AUT_GRAFANA_PORT>` (default port 3000). Sign in as `admin` with `AUT_GRAFANA_ADMIN_PASSWORD`.
|
|
29
|
+
The dashboards are in the "Coding agents" folder.
|
|
30
|
+
|
|
31
|
+
## Connect each collector
|
|
32
|
+
|
|
33
|
+
Install aut with the PostgreSQL extra on every machine you code on:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
uv tool install "agent-usage-tracker[postgres]"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
In `~/.config/agent-usage-tracker/config.toml`:
|
|
40
|
+
|
|
41
|
+
```toml
|
|
42
|
+
machine = "laptop"
|
|
43
|
+
ledger_timezone = "Europe/Berlin" # the same value as AUT_TZ
|
|
44
|
+
|
|
45
|
+
[transport]
|
|
46
|
+
pg_dsn = "env:AUT_PG_DSN"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Set `AUT_PG_DSN=postgresql://agents_writer:<AUT_WRITER_PASSWORD>@<host>:<AUT_PG_PORT>/agents?sslmode=disable`.
|
|
50
|
+
`sslmode=disable` is only safe on this machine or a private network such as Tailscale.
|
|
51
|
+
Then run `aut scan --dry-run` and `aut scan`.
|
|
52
|
+
`aut git-days` and `aut history-steers` stop if `ledger_timezone` differs from the sink's `AUT_TZ`.
|
|
53
|
+
|
|
54
|
+
## Know this
|
|
55
|
+
|
|
56
|
+
- **Upgrade:** pull the new aut version, then `docker compose up -d --wait`. The `schema` service applies the changes and updates the TimescaleDB extension.
|
|
57
|
+
A change of Postgres major version (for example `pg17` to `pg18` in the image tag) needs a `pg_dump` and a restore; aut's release notes will say so.
|
|
58
|
+
- **Time zone:** `AUT_TZ` decides what a "day" is in the views and in Grafana's hour and day panels. Choose it once.
|
|
59
|
+
Rows already written keep the days of the zone they were counted in, so `apply.sh` refuses a new `AUT_TZ`; `AUT_TZ_FORCE=1` switches anyway.
|
|
60
|
+
The `usage_daily` rollup always counts UTC days.
|
|
61
|
+
- **Passwords:** the writer and read-only passwords follow `.env` on every `up`.
|
|
62
|
+
The Postgres admin and Grafana admin passwords apply on the first start only. To change them later, use
|
|
63
|
+
`docker compose exec db psql -U postgres -c "ALTER ROLE postgres PASSWORD '...'"` and
|
|
64
|
+
`docker compose exec grafana grafana cli admin reset-admin-password ...`, then update `.env`.
|
|
65
|
+
- **Network:** with a LAN or Tailscale `AUT_BIND`, every host that reaches it can try to log in to Postgres, including as `postgres`, and Grafana runs over plain HTTP.
|
|
66
|
+
Docker's published ports also bypass host firewalls such as ufw. Prefer a Tailscale address over a LAN address.
|
|
67
|
+
- **Cost panels:** the cost panels stay empty until `agents.model_price` has rows. aut does not fill that table yet.
|
|
68
|
+
- **Backups:** your history is in the `pgdata` volume. Back it up with `docker compose exec db pg_dump -U postgres agents`.
|
|
69
|
+
- **Your own Postgres:** skip Compose and run `apply.sh` with the libpq environment (`PGHOST`, `PGUSER`, ...) of a superuser on a TimescaleDB server.
|
|
70
|
+
It needs `psql` 15 or newer. Without `AUT_WRITER_PASSWORD` and `AUT_RO_PASSWORD`, it leaves existing role passwords alone.
|
|
71
|
+
Run `sql/refresh.sql` as that superuser at least once a day.
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Create the two login roles and apply the sink schema. Every file is idempotent, so this runs on
|
|
3
|
+
# every `docker compose up`: a newer aut release reaches an existing database, not only a new one.
|
|
4
|
+
# Needs psql 15 or newer and the libpq environment of a superuser (PGHOST, PGPORT, PGUSER,
|
|
5
|
+
# PGPASSWORD, PGDATABASE), plus AUT_TZ. AUT_WRITER_PASSWORD and AUT_RO_PASSWORD set the two role
|
|
6
|
+
# passwords; when one is unset, that role's existing password is left alone.
|
|
7
|
+
set -eu
|
|
8
|
+
here=$(dirname "$0")
|
|
9
|
+
: "${AUT_TZ:?set AUT_TZ to an IANA time zone, for example Europe/Berlin}"
|
|
10
|
+
case "$AUT_TZ" in
|
|
11
|
+
*[!A-Za-z0-9_+/-]*) echo "apply: AUT_TZ has an unexpected character: $AUT_TZ" >&2; exit 2 ;;
|
|
12
|
+
esac
|
|
13
|
+
psql="psql -X -q -v ON_ERROR_STOP=1"
|
|
14
|
+
known=$($psql -tAc "SELECT count(*) FROM pg_timezone_names WHERE name = '$AUT_TZ'")
|
|
15
|
+
if [ "$known" != 1 ]; then
|
|
16
|
+
echo "apply: Postgres does not know the time zone $AUT_TZ" >&2
|
|
17
|
+
exit 2
|
|
18
|
+
fi
|
|
19
|
+
# Rows already written keep the days of the zone they were counted in, so a change mixes days.
|
|
20
|
+
stored=$($psql -tAc "SELECT coalesce(current_setting('aut.ledger_tz', true), '')")
|
|
21
|
+
if [ -n "$stored" ] && [ "$stored" != "$AUT_TZ" ] && [ "${AUT_TZ_FORCE:-}" != 1 ]; then
|
|
22
|
+
echo "apply: this sink counts days in $stored, not $AUT_TZ. Set AUT_TZ=$stored," \
|
|
23
|
+
"or AUT_TZ_FORCE=1 to switch and accept that older rows keep their old days" >&2
|
|
24
|
+
exit 2
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
# A newer image ships a newer extension, but a database keeps the version it loaded until this.
|
|
28
|
+
# It must be the first statement of its session.
|
|
29
|
+
if [ "$($psql -tAc "SELECT count(*) FROM pg_extension WHERE extname = 'timescaledb'")" = 1 ]; then
|
|
30
|
+
$psql -c "ALTER EXTENSION timescaledb UPDATE"
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
# Roles first: the grants file skips a role that does not exist, without an error.
|
|
34
|
+
# \getenv keeps the passwords off the command line, where other users can read them.
|
|
35
|
+
$psql <<'SQL'
|
|
36
|
+
\getenv writer_pw AUT_WRITER_PASSWORD
|
|
37
|
+
\getenv ro_pw AUT_RO_PASSWORD
|
|
38
|
+
\getenv tz AUT_TZ
|
|
39
|
+
SELECT format('CREATE ROLE %I LOGIN', r) FROM unnest(ARRAY['agents_writer', 'agents_ro']) AS r
|
|
40
|
+
WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) \gexec
|
|
41
|
+
\if :{?writer_pw}
|
|
42
|
+
ALTER ROLE agents_writer PASSWORD :'writer_pw';
|
|
43
|
+
\endif
|
|
44
|
+
\if :{?ro_pw}
|
|
45
|
+
ALTER ROLE agents_ro PASSWORD :'ro_pw';
|
|
46
|
+
\endif
|
|
47
|
+
-- Collectors compare their ledger_timezone with this. Grafana's raw-table panels read hours and
|
|
48
|
+
-- months in the session zone, so agents_ro sessions use the sink's zone too.
|
|
49
|
+
SELECT format('ALTER DATABASE %I SET aut.ledger_tz = %L', current_database(), :'tz') \gexec
|
|
50
|
+
SELECT format('ALTER ROLE agents_ro IN DATABASE %I SET timezone = %L', current_database(), :'tz') \gexec
|
|
51
|
+
SQL
|
|
52
|
+
|
|
53
|
+
# Each file in its own psql run and not in one transaction: a continuous aggregate cannot be
|
|
54
|
+
# created inside a transaction block. The order matters: the cost views read the price table,
|
|
55
|
+
# and the grants name objects from every earlier file.
|
|
56
|
+
for f in 01_schema 02_price 03_effort 04_grants; do
|
|
57
|
+
if [ ! -r "$here/sql/$f.sql" ]; then
|
|
58
|
+
echo "apply: cannot read $here/sql/$f.sql" >&2
|
|
59
|
+
exit 2
|
|
60
|
+
fi
|
|
61
|
+
sed "s#@AUT_TZ@#$AUT_TZ#g" "$here/sql/$f.sql" | $psql -f - >/dev/null
|
|
62
|
+
done
|
|
63
|
+
echo "apply: aut sink schema is current (time zone $AUT_TZ)"
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# The aut sink: TimescaleDB with the aut schema, an hourly refresh of the derived views, and
|
|
2
|
+
# Grafana with the dashboards. Same file on a laptop or a homelab; see README.md beside it.
|
|
3
|
+
name: aut-sink
|
|
4
|
+
|
|
5
|
+
x-pg-admin: &pg-admin
|
|
6
|
+
PGHOST: db
|
|
7
|
+
PGUSER: postgres
|
|
8
|
+
PGDATABASE: agents
|
|
9
|
+
PGPASSWORD: ${AUT_PG_ADMIN_PASSWORD:?set AUT_PG_ADMIN_PASSWORD in .env}
|
|
10
|
+
|
|
11
|
+
services:
|
|
12
|
+
db:
|
|
13
|
+
image: timescale/timescaledb:2.32.1-pg17
|
|
14
|
+
restart: unless-stopped
|
|
15
|
+
environment:
|
|
16
|
+
POSTGRES_DB: agents
|
|
17
|
+
POSTGRES_PASSWORD: ${AUT_PG_ADMIN_PASSWORD:?set AUT_PG_ADMIN_PASSWORD in .env}
|
|
18
|
+
TIMESCALEDB_TELEMETRY: "off"
|
|
19
|
+
volumes:
|
|
20
|
+
- pgdata:/var/lib/postgresql/data
|
|
21
|
+
ports:
|
|
22
|
+
- "${AUT_BIND:-127.0.0.1}:${AUT_PG_PORT:-5432}:5432"
|
|
23
|
+
healthcheck:
|
|
24
|
+
# Over TCP: during first-start setup the server listens on its socket only, and the
|
|
25
|
+
# schema service connects over TCP.
|
|
26
|
+
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d agents"]
|
|
27
|
+
interval: 5s
|
|
28
|
+
timeout: 5s
|
|
29
|
+
retries: 30
|
|
30
|
+
start_period: 30s
|
|
31
|
+
|
|
32
|
+
# Runs once per `up`, then exits. The other services wait for it to succeed.
|
|
33
|
+
schema:
|
|
34
|
+
image: timescale/timescaledb:2.32.1-pg17
|
|
35
|
+
depends_on:
|
|
36
|
+
db:
|
|
37
|
+
condition: service_healthy
|
|
38
|
+
environment:
|
|
39
|
+
<<: *pg-admin
|
|
40
|
+
AUT_TZ: ${AUT_TZ:?set AUT_TZ in .env}
|
|
41
|
+
AUT_WRITER_PASSWORD: ${AUT_WRITER_PASSWORD:?set AUT_WRITER_PASSWORD in .env}
|
|
42
|
+
AUT_RO_PASSWORD: ${AUT_RO_PASSWORD:?set AUT_RO_PASSWORD in .env}
|
|
43
|
+
volumes:
|
|
44
|
+
- ./apply.sh:/stack/apply.sh:ro
|
|
45
|
+
- ./sql:/stack/sql:ro
|
|
46
|
+
entrypoint: ["sh", "/stack/apply.sh"]
|
|
47
|
+
restart: "no"
|
|
48
|
+
|
|
49
|
+
refresh:
|
|
50
|
+
image: timescale/timescaledb:2.32.1-pg17
|
|
51
|
+
restart: unless-stopped
|
|
52
|
+
depends_on:
|
|
53
|
+
schema:
|
|
54
|
+
condition: service_completed_successfully
|
|
55
|
+
environment:
|
|
56
|
+
<<: *pg-admin
|
|
57
|
+
AUT_REFRESH_SECONDS: ${AUT_REFRESH_SECONDS:-3600}
|
|
58
|
+
volumes:
|
|
59
|
+
- ./sql:/stack/sql:ro
|
|
60
|
+
entrypoint:
|
|
61
|
+
- sh
|
|
62
|
+
- -c
|
|
63
|
+
- |
|
|
64
|
+
while :; do
|
|
65
|
+
if psql -X -q -f /stack/sql/refresh.sql; then
|
|
66
|
+
sleep "$${AUT_REFRESH_SECONDS}" || sleep 3600
|
|
67
|
+
else
|
|
68
|
+
echo "refresh: failed, retrying in a minute" >&2
|
|
69
|
+
sleep 60
|
|
70
|
+
fi
|
|
71
|
+
done
|
|
72
|
+
|
|
73
|
+
grafana:
|
|
74
|
+
image: grafana/grafana:12.4.11
|
|
75
|
+
restart: unless-stopped
|
|
76
|
+
depends_on:
|
|
77
|
+
schema:
|
|
78
|
+
condition: service_completed_successfully
|
|
79
|
+
environment:
|
|
80
|
+
GF_SECURITY_ADMIN_PASSWORD: ${AUT_GRAFANA_ADMIN_PASSWORD:?set AUT_GRAFANA_ADMIN_PASSWORD in .env}
|
|
81
|
+
GF_USERS_ALLOW_SIGN_UP: "false"
|
|
82
|
+
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
83
|
+
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
|
84
|
+
AUT_RO_PASSWORD: ${AUT_RO_PASSWORD:?set AUT_RO_PASSWORD in .env}
|
|
85
|
+
volumes:
|
|
86
|
+
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
|
87
|
+
- ./grafana/dashboards:/etc/aut-dashboards:ro
|
|
88
|
+
- grafana:/var/lib/grafana
|
|
89
|
+
ports:
|
|
90
|
+
- "${AUT_BIND:-127.0.0.1}:${AUT_GRAFANA_PORT:-3000}:3000"
|
|
91
|
+
|
|
92
|
+
volumes:
|
|
93
|
+
pgdata:
|
|
94
|
+
grafana:
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Copy to .env beside compose.yaml, fill it in, and keep it private (chmod 600). Never commit it.
|
|
2
|
+
# Make each password with: openssl rand -hex 24
|
|
3
|
+
# Letters and digits only: Compose and Grafana expand "$", and @ : / ? break a postgresql:// DSN.
|
|
4
|
+
AUT_PG_ADMIN_PASSWORD=
|
|
5
|
+
AUT_WRITER_PASSWORD=
|
|
6
|
+
AUT_RO_PASSWORD=
|
|
7
|
+
AUT_GRAFANA_ADMIN_PASSWORD=
|
|
8
|
+
|
|
9
|
+
# The IANA time zone that decides what a "day" is in every view, for example Europe/Berlin.
|
|
10
|
+
# Set ledger_timezone in each collector's config.toml to the same value.
|
|
11
|
+
AUT_TZ=
|
|
12
|
+
|
|
13
|
+
# 127.0.0.1 keeps the database and Grafana on this machine only. On a homelab, set the address
|
|
14
|
+
# other machines reach it on, such as its LAN or Tailscale address. Never expose it to the internet.
|
|
15
|
+
AUT_BIND=127.0.0.1
|
|
16
|
+
AUT_PG_PORT=5432
|
|
17
|
+
AUT_GRAFANA_PORT=3000
|
|
18
|
+
|
|
19
|
+
# How often the derived views (active time, episodes, cost) refresh, in seconds.
|
|
20
|
+
AUT_REFRESH_SECONDS=3600
|