microcosm-emitter 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- microcosm_emitter/__init__.py +1 -0
- microcosm_emitter/core/__init__.py +1 -0
- microcosm_emitter/core/alembic/env.py +28 -0
- microcosm_emitter/core/alembic/script.py.mako +31 -0
- microcosm_emitter/core/alembic/versions/v20261007_01_initial_spool.py +85 -0
- microcosm_emitter/core/alembic/versions/v20261008_02_graph_publication.py +27 -0
- microcosm_emitter/core/auth.py +169 -0
- microcosm_emitter/core/constants.py +69 -0
- microcosm_emitter/core/database.py +34 -0
- microcosm_emitter/core/migrations.py +78 -0
- microcosm_emitter/core/models.py +145 -0
- microcosm_emitter/host/__init__.py +1 -0
- microcosm_emitter/host/__main__.py +50 -0
- microcosm_emitter/host/client.py +132 -0
- microcosm_emitter/host/constants.py +21 -0
- microcosm_emitter/host/contracts.py +35 -0
- microcosm_emitter/host/errors.py +52 -0
- microcosm_emitter/host/launcher.py +184 -0
- microcosm_emitter/host/loading.py +22 -0
- microcosm_emitter/host/process.py +20 -0
- microcosm_emitter/host/request_dispatch.py +96 -0
- microcosm_emitter/host/runtime.py +227 -0
- microcosm_emitter/orrery/__init__.py +1 -0
- microcosm_emitter/orrery/cli.py +39 -0
- microcosm_emitter/orrery/client.py +229 -0
- microcosm_emitter/orrery/constants.py +5 -0
- microcosm_emitter/orrery/contracts.py +57 -0
- microcosm_emitter/orrery/py.typed +1 -0
- microcosm_emitter/orrery/service/__init__.py +8 -0
- microcosm_emitter/orrery/service/module.py +98 -0
- microcosm_emitter/orrery/service/queue.py +447 -0
- microcosm_emitter/py.typed +1 -0
- microcosm_emitter/telemetry/__init__.py +1 -0
- microcosm_emitter/telemetry/client.py +295 -0
- microcosm_emitter/telemetry/constants.py +16 -0
- microcosm_emitter/telemetry/protocol.py +57 -0
- microcosm_emitter/telemetry/requests.py +14 -0
- microcosm_emitter/telemetry/serialization.py +24 -0
- microcosm_emitter/telemetry/service/__init__.py +14 -0
- microcosm_emitter/telemetry/service/collector.py +186 -0
- microcosm_emitter/telemetry/service/lifecycle.py +346 -0
- microcosm_emitter/telemetry/service/module.py +128 -0
- microcosm_emitter/telemetry/service/resources.py +45 -0
- microcosm_emitter/telemetry/service/sanitization.py +94 -0
- microcosm_emitter/telemetry/service/spool.py +286 -0
- microcosm_emitter/telemetry/service/timestamps.py +11 -0
- microcosm_emitter-0.2.0.dist-info/METADATA +83 -0
- microcosm_emitter-0.2.0.dist-info/RECORD +51 -0
- microcosm_emitter-0.2.0.dist-info/WHEEL +4 -0
- microcosm_emitter-0.2.0.dist-info/entry_points.txt +6 -0
- microcosm_emitter-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Local emitter host and required telemetry library, without import side effects."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared provider services; importing this package has no runtime side effects."""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Alembic environment for the local telemetry spool."""
|
|
2
|
+
|
|
3
|
+
from alembic import context
|
|
4
|
+
from sqlalchemy.engine import Connection
|
|
5
|
+
|
|
6
|
+
from microcosm_emitter.core.models import SpoolModel
|
|
7
|
+
|
|
8
|
+
_MISSING_CONNECTION_ERROR = (
|
|
9
|
+
"telemetry spool migrations require a programmatically supplied connection"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def run_migrations() -> None:
|
|
14
|
+
"""Run migrations on the connection supplied by the emitter service."""
|
|
15
|
+
|
|
16
|
+
connection = context.config.attributes.get("connection")
|
|
17
|
+
if not isinstance(connection, Connection):
|
|
18
|
+
raise RuntimeError(_MISSING_CONNECTION_ERROR)
|
|
19
|
+
context.configure(
|
|
20
|
+
connection=connection,
|
|
21
|
+
target_metadata=SpoolModel.metadata,
|
|
22
|
+
render_as_batch=True,
|
|
23
|
+
)
|
|
24
|
+
with context.begin_transaction():
|
|
25
|
+
context.run_migrations()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
run_migrations()
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""${message}
|
|
2
|
+
|
|
3
|
+
Revision ID: ${up_revision}
|
|
4
|
+
Revises: ${down_revision | comma,n}
|
|
5
|
+
Create Date: ${create_date}
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Sequence
|
|
11
|
+
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
from alembic import op
|
|
14
|
+
${imports if imports else ""}
|
|
15
|
+
|
|
16
|
+
revision: str = ${repr(up_revision)}
|
|
17
|
+
down_revision: str | None = ${repr(down_revision)}
|
|
18
|
+
branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
|
|
19
|
+
depends_on: str | Sequence[str] | None = ${repr(depends_on)}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def upgrade() -> None:
|
|
23
|
+
"""Apply this schema revision."""
|
|
24
|
+
|
|
25
|
+
${upgrades if upgrades else "pass"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def downgrade() -> None:
|
|
29
|
+
"""Reverse this schema revision."""
|
|
30
|
+
|
|
31
|
+
${downgrades if downgrades else "pass"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Create the telemetry spool schema.
|
|
2
|
+
|
|
3
|
+
Revision ID: 20261007_01
|
|
4
|
+
Revises:
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
|
|
11
|
+
import sqlalchemy as sa
|
|
12
|
+
from alembic import op
|
|
13
|
+
|
|
14
|
+
revision: str = "20261007_01"
|
|
15
|
+
down_revision: str | None = None
|
|
16
|
+
branch_labels: str | Sequence[str] | None = None
|
|
17
|
+
depends_on: str | Sequence[str] | None = None
|
|
18
|
+
|
|
19
|
+
_RUNS_TABLE = "telemetry_runs"
|
|
20
|
+
_EVENTS_TABLE = "telemetry_events"
|
|
21
|
+
_EVENTS_INDEX = "telemetry_events_run_sequence"
|
|
22
|
+
_PENDING_UPLOAD_STATE = "pending"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _create_runs_table() -> None:
|
|
26
|
+
op.create_table(
|
|
27
|
+
_RUNS_TABLE,
|
|
28
|
+
sa.Column("run_id", sa.Text(), nullable=False),
|
|
29
|
+
sa.Column("producer_id", sa.Text(), nullable=False),
|
|
30
|
+
sa.Column("registration_json", sa.Text(), nullable=False),
|
|
31
|
+
sa.Column(
|
|
32
|
+
"next_sequence",
|
|
33
|
+
sa.Integer(),
|
|
34
|
+
nullable=False,
|
|
35
|
+
server_default="1",
|
|
36
|
+
),
|
|
37
|
+
sa.Column(
|
|
38
|
+
"upload_state",
|
|
39
|
+
sa.Text(),
|
|
40
|
+
nullable=False,
|
|
41
|
+
server_default=_PENDING_UPLOAD_STATE,
|
|
42
|
+
),
|
|
43
|
+
sa.Column("local_only_reason", sa.Text(), nullable=True),
|
|
44
|
+
sa.Column("updated_at", sa.Text(), nullable=False),
|
|
45
|
+
sa.PrimaryKeyConstraint("run_id", "producer_id"),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _create_events_table() -> None:
|
|
50
|
+
op.create_table(
|
|
51
|
+
_EVENTS_TABLE,
|
|
52
|
+
sa.Column("event_id", sa.Text(), nullable=False),
|
|
53
|
+
sa.Column("run_id", sa.Text(), nullable=False),
|
|
54
|
+
sa.Column("producer_id", sa.Text(), nullable=False),
|
|
55
|
+
sa.Column("sequence", sa.Integer(), nullable=False),
|
|
56
|
+
sa.Column("payload_json", sa.Text(), nullable=False),
|
|
57
|
+
sa.Column("created_at", sa.Text(), nullable=False),
|
|
58
|
+
sa.ForeignKeyConstraint(
|
|
59
|
+
["run_id", "producer_id"],
|
|
60
|
+
[f"{_RUNS_TABLE}.run_id", f"{_RUNS_TABLE}.producer_id"],
|
|
61
|
+
name="fk_telemetry_events_run",
|
|
62
|
+
),
|
|
63
|
+
sa.PrimaryKeyConstraint("event_id"),
|
|
64
|
+
sa.UniqueConstraint(
|
|
65
|
+
"run_id",
|
|
66
|
+
"producer_id",
|
|
67
|
+
"sequence",
|
|
68
|
+
name="uq_telemetry_events_run_producer_sequence",
|
|
69
|
+
),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def upgrade() -> None:
|
|
74
|
+
"""Create a new database; existing schemas must already be versioned."""
|
|
75
|
+
_create_runs_table()
|
|
76
|
+
_create_events_table()
|
|
77
|
+
op.create_index(_EVENTS_INDEX, _EVENTS_TABLE, ["run_id", "producer_id", "sequence"])
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def downgrade() -> None:
|
|
81
|
+
"""Remove the local spool schema."""
|
|
82
|
+
|
|
83
|
+
op.drop_index(_EVENTS_INDEX, table_name=_EVENTS_TABLE)
|
|
84
|
+
op.drop_table(_EVENTS_TABLE)
|
|
85
|
+
op.drop_table(_RUNS_TABLE)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Add durable graph publication jobs without changing event retention."""
|
|
2
|
+
|
|
3
|
+
import sqlalchemy as sa
|
|
4
|
+
from alembic import op
|
|
5
|
+
|
|
6
|
+
revision = "20261008_02"
|
|
7
|
+
down_revision = "20261007_01"
|
|
8
|
+
branch_labels = None
|
|
9
|
+
depends_on = None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def upgrade() -> None:
|
|
13
|
+
op.create_table(
|
|
14
|
+
"graph_publication_jobs",
|
|
15
|
+
sa.Column("publication_id", sa.Text(), primary_key=True),
|
|
16
|
+
sa.Column("inventory", sa.Text(), nullable=False),
|
|
17
|
+
sa.Column("directory", sa.Text(), nullable=False),
|
|
18
|
+
sa.Column("status", sa.Text(), nullable=False),
|
|
19
|
+
sa.Column("attempts", sa.Integer(), nullable=False),
|
|
20
|
+
sa.Column("next_attempt_at", sa.Float(), nullable=False),
|
|
21
|
+
sa.Column("lease_until", sa.Float(), nullable=False),
|
|
22
|
+
sa.Column("receipt", sa.Text(), nullable=False),
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def downgrade() -> None:
|
|
27
|
+
op.drop_table("graph_publication_jobs")
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""HF-to-collector session shared by telemetry and graph publication."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import time
|
|
8
|
+
import urllib.error
|
|
9
|
+
import urllib.request
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from http import HTTPStatus
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import urlsplit
|
|
14
|
+
|
|
15
|
+
from huggingface_hub import get_token
|
|
16
|
+
|
|
17
|
+
from microcosm_emitter.core.constants import (
|
|
18
|
+
COLLECTOR_RESPONSE_TOO_LARGE_ERROR,
|
|
19
|
+
COLLECTOR_URL_HTTPS_ERROR,
|
|
20
|
+
COLLECTOR_URL_ORIGIN_ERROR,
|
|
21
|
+
DEFAULT_TOKEN_LIFETIME_SECONDS,
|
|
22
|
+
DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR,
|
|
23
|
+
HTTP_TIMEOUT_SECONDS,
|
|
24
|
+
HTTP_USER_AGENT,
|
|
25
|
+
LOOPBACK_HOSTS,
|
|
26
|
+
MAX_HTTP_RESPONSE_BYTES,
|
|
27
|
+
MINIMUM_TOKEN_LIFETIME_SECONDS,
|
|
28
|
+
TOKEN_EXCHANGE_PATH,
|
|
29
|
+
TOKEN_REFRESH_MARGIN_SECONDS,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
34
|
+
"""Keep bearer credentials on the explicitly configured origin."""
|
|
35
|
+
|
|
36
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str:
|
|
41
|
+
parsed = urlsplit(value)
|
|
42
|
+
loopback = parsed.hostname in LOOPBACK_HOSTS
|
|
43
|
+
valid_scheme = parsed.scheme == "https" or (
|
|
44
|
+
allow_loopback_http and loopback and parsed.scheme == "http"
|
|
45
|
+
)
|
|
46
|
+
if not parsed.hostname or not valid_scheme:
|
|
47
|
+
raise ValueError(COLLECTOR_URL_HTTPS_ERROR)
|
|
48
|
+
if (
|
|
49
|
+
parsed.username
|
|
50
|
+
or parsed.password
|
|
51
|
+
or parsed.query
|
|
52
|
+
or parsed.fragment
|
|
53
|
+
or parsed.path not in {"", "/"}
|
|
54
|
+
):
|
|
55
|
+
raise ValueError(COLLECTOR_URL_ORIGIN_ERROR)
|
|
56
|
+
return value.rstrip("/")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _development_collector_url(value: str) -> str:
|
|
60
|
+
parsed = urlsplit(value)
|
|
61
|
+
if parsed.hostname not in LOOPBACK_HOSTS:
|
|
62
|
+
raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR)
|
|
63
|
+
return _collector_origin(value, allow_loopback_http=True)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _decode_response(body: bytes) -> dict[str, Any]:
|
|
67
|
+
try:
|
|
68
|
+
response = json.loads(body) if body else {}
|
|
69
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
70
|
+
return {}
|
|
71
|
+
return response if isinstance(response, dict) else {}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _http_post(
|
|
75
|
+
url: str,
|
|
76
|
+
payload: Mapping[str, Any],
|
|
77
|
+
bearer_token: str,
|
|
78
|
+
*,
|
|
79
|
+
timeout: float = HTTP_TIMEOUT_SECONDS,
|
|
80
|
+
) -> tuple[int, dict[str, Any]]:
|
|
81
|
+
request = urllib.request.Request(
|
|
82
|
+
url,
|
|
83
|
+
data=json.dumps(payload, separators=(",", ":")).encode(),
|
|
84
|
+
headers={
|
|
85
|
+
"Authorization": f"Bearer {bearer_token}",
|
|
86
|
+
"Content-Type": "application/json",
|
|
87
|
+
"User-Agent": HTTP_USER_AGENT,
|
|
88
|
+
},
|
|
89
|
+
method="POST",
|
|
90
|
+
)
|
|
91
|
+
try:
|
|
92
|
+
opener = urllib.request.build_opener(_NoRedirectHandler)
|
|
93
|
+
with opener.open(request, timeout=timeout) as response:
|
|
94
|
+
body = response.read(MAX_HTTP_RESPONSE_BYTES + 1)
|
|
95
|
+
if len(body) > MAX_HTTP_RESPONSE_BYTES:
|
|
96
|
+
raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR)
|
|
97
|
+
return response.status, _decode_response(body)
|
|
98
|
+
except urllib.error.HTTPError as error:
|
|
99
|
+
body = error.read(MAX_HTTP_RESPONSE_BYTES + 1)
|
|
100
|
+
if len(body) > MAX_HTTP_RESPONSE_BYTES:
|
|
101
|
+
return error.code, {}
|
|
102
|
+
return error.code, _decode_response(body)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _huggingface_token() -> str | None:
|
|
106
|
+
return (
|
|
107
|
+
os.environ.get("HF_TOKEN", "").strip()
|
|
108
|
+
or os.environ.get("HUGGINGFACE_TOKEN", "").strip()
|
|
109
|
+
or get_token()
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class CollectorSession:
|
|
114
|
+
"""One thread-safe short-lived credential shared by independent publishers."""
|
|
115
|
+
|
|
116
|
+
def __init__(self, origin: str, *, post=None, credential=None):
|
|
117
|
+
import threading
|
|
118
|
+
|
|
119
|
+
self.origin = origin
|
|
120
|
+
self._post = post or _http_post
|
|
121
|
+
self._credential = credential or _huggingface_token
|
|
122
|
+
self._session_token: tuple[str, float] | None = None
|
|
123
|
+
self._lock = threading.Lock()
|
|
124
|
+
|
|
125
|
+
def invalidate(self) -> None:
|
|
126
|
+
with self._lock:
|
|
127
|
+
self._session_token = None
|
|
128
|
+
|
|
129
|
+
def credential(self) -> tuple[str, str | None]:
|
|
130
|
+
with self._lock:
|
|
131
|
+
if (
|
|
132
|
+
self._session_token is not None
|
|
133
|
+
and self._session_token[1]
|
|
134
|
+
> time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS
|
|
135
|
+
):
|
|
136
|
+
return "ok", self._session_token[0]
|
|
137
|
+
hf_token = self._credential()
|
|
138
|
+
if not hf_token:
|
|
139
|
+
return "missing", None
|
|
140
|
+
try:
|
|
141
|
+
status, response = self._post(
|
|
142
|
+
self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token
|
|
143
|
+
)
|
|
144
|
+
except (OSError, TimeoutError):
|
|
145
|
+
return "unavailable", None
|
|
146
|
+
if status == HTTPStatus.OK and isinstance(
|
|
147
|
+
response.get("access_token"), str
|
|
148
|
+
):
|
|
149
|
+
try:
|
|
150
|
+
expires_in = max(
|
|
151
|
+
MINIMUM_TOKEN_LIFETIME_SECONDS,
|
|
152
|
+
int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)),
|
|
153
|
+
)
|
|
154
|
+
except (ValueError, TypeError):
|
|
155
|
+
return "unavailable", None
|
|
156
|
+
token = response["access_token"]
|
|
157
|
+
self._session_token = (token, time.monotonic() + expires_in)
|
|
158
|
+
return "ok", token
|
|
159
|
+
if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}:
|
|
160
|
+
return "rejected", None
|
|
161
|
+
return "unavailable", None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def shared_session(services: dict, origin: str) -> CollectorSession:
|
|
165
|
+
"""Share a session among explicit modules without adding auth to the host."""
|
|
166
|
+
session = services.setdefault("collector_session", CollectorSession(origin))
|
|
167
|
+
if not isinstance(session, CollectorSession) or session.origin != origin:
|
|
168
|
+
raise ValueError("Provider modules must use the same collector origin.")
|
|
169
|
+
return session
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Configuration and protocol values for the telemetry emitter service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
RETENTION_DAYS: Final = 7
|
|
8
|
+
MAX_QUEUED_BYTES: Final = 100 * 1024 * 1024
|
|
9
|
+
BATCH_SIZE: Final = 100
|
|
10
|
+
PRODUCTION_COLLECTOR_URL: Final = (
|
|
11
|
+
"https://microcosm-telemetry-389282473430.us-central1.run.app"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
LOOPBACK_HOSTS: Final = frozenset({"localhost", "127.0.0.1", "::1"})
|
|
15
|
+
TOKEN_EXCHANGE_PATH: Final = "/v1/auth/huggingface/exchange"
|
|
16
|
+
RUN_REGISTRATION_PATH: Final = "/v1/runs"
|
|
17
|
+
RUN_EVENTS_PATH_TEMPLATE: Final = "/v1/runs/{run_id}/events"
|
|
18
|
+
HTTP_USER_AGENT: Final = "microcosm-telemetry-emitter/1"
|
|
19
|
+
MAX_HTTP_RESPONSE_BYTES: Final = 1_048_576
|
|
20
|
+
HTTP_TIMEOUT_SECONDS: Final = 5.0
|
|
21
|
+
|
|
22
|
+
UPLOAD_STATE_PENDING: Final = "pending"
|
|
23
|
+
UPLOAD_STATE_LOCAL_ONLY: Final = "local_only"
|
|
24
|
+
LOCAL_ONLY_MISSING_CREDENTIAL: Final = "missing_huggingface_credential"
|
|
25
|
+
LOCAL_ONLY_REJECTED_CREDENTIAL: Final = "huggingface_credential_rejected"
|
|
26
|
+
LOCAL_ONLY_REJECTED_REGISTRATION: Final = "run_registration_rejected"
|
|
27
|
+
LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION: Final = "collector_authorization_rejected"
|
|
28
|
+
LOCAL_ONLY_PRE_ELIGIBILITY: Final = "created_before_upload_eligibility"
|
|
29
|
+
|
|
30
|
+
NO_CREDENTIAL_MESSAGE: Final = (
|
|
31
|
+
"Microcosm telemetry is local-only: no ambient Hugging Face credential was "
|
|
32
|
+
"found. The dataset build will continue."
|
|
33
|
+
)
|
|
34
|
+
REJECTED_CREDENTIAL_MESSAGE: Final = (
|
|
35
|
+
"Microcosm telemetry is local-only for this run: the ambient Hugging Face "
|
|
36
|
+
"credential was not accepted as a PolicyEngine organization member. The "
|
|
37
|
+
"dataset build will continue."
|
|
38
|
+
)
|
|
39
|
+
COLLECTOR_URL_HTTPS_ERROR: Final = "collector URL must be an HTTPS origin"
|
|
40
|
+
COLLECTOR_URL_ORIGIN_ERROR: Final = (
|
|
41
|
+
"collector URL must be an origin without credentials or path data"
|
|
42
|
+
)
|
|
43
|
+
DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR: Final = (
|
|
44
|
+
"development collector URL must use a loopback address"
|
|
45
|
+
)
|
|
46
|
+
COLLECTOR_RESPONSE_TOO_LARGE_ERROR: Final = "collector response exceeds 1 MiB"
|
|
47
|
+
|
|
48
|
+
DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 3_600
|
|
49
|
+
MINIMUM_TOKEN_LIFETIME_SECONDS: Final = 60
|
|
50
|
+
TOKEN_REFRESH_MARGIN_SECONDS: Final = 30
|
|
51
|
+
INITIAL_RETRY_SECONDS: Final = 1.0
|
|
52
|
+
MAX_RETRY_SECONDS: Final = 60.0
|
|
53
|
+
|
|
54
|
+
DATABASE_TIMEOUT_SECONDS: Final = 5
|
|
55
|
+
PRUNE_INTERVAL_SECONDS: Final = 60.0
|
|
56
|
+
|
|
57
|
+
DEFAULT_HEARTBEAT_SECONDS: Final = 60.0
|
|
58
|
+
DEFAULT_DRAIN_SECONDS: Final = 15.0
|
|
59
|
+
MINIMUM_HEARTBEAT_SECONDS: Final = 1.0
|
|
60
|
+
SOCKET_LISTEN_BACKLOG: Final = 16
|
|
61
|
+
SOCKET_ACCEPT_TIMEOUT_SECONDS: Final = 0.5
|
|
62
|
+
SOCKET_CONNECTION_TIMEOUT_SECONDS: Final = 0.25
|
|
63
|
+
WORKER_INTERVAL_SECONDS: Final = 1.0
|
|
64
|
+
DRAIN_RETRY_SECONDS: Final = 0.5
|
|
65
|
+
|
|
66
|
+
EVENT_OBJECT_ERROR: Final = "event must be an object"
|
|
67
|
+
UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action"
|
|
68
|
+
LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB"
|
|
69
|
+
FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""SQLAlchemy engine and session construction for the telemetry spool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import URL, create_engine
|
|
8
|
+
from sqlalchemy.engine import Engine
|
|
9
|
+
from sqlalchemy.orm import Session, sessionmaker
|
|
10
|
+
|
|
11
|
+
from microcosm_emitter.core.constants import (
|
|
12
|
+
DATABASE_TIMEOUT_SECONDS,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def sqlite_database_url(path: Path | str) -> URL:
|
|
17
|
+
"""Return a SQLAlchemy URL for a local SQLite spool path."""
|
|
18
|
+
|
|
19
|
+
return URL.create("sqlite+pysqlite", database=str(Path(path)))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_spool_engine(path: Path | str) -> Engine:
|
|
23
|
+
"""Create the SQLAlchemy engine used by one emitter service process."""
|
|
24
|
+
|
|
25
|
+
return create_engine(
|
|
26
|
+
sqlite_database_url(path),
|
|
27
|
+
connect_args={"timeout": DATABASE_TIMEOUT_SECONDS},
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def create_spool_session_factory(engine: Engine) -> sessionmaker[Session]:
|
|
32
|
+
"""Create short-lived ORM sessions bound to the spool engine."""
|
|
33
|
+
|
|
34
|
+
return sessionmaker(engine, expire_on_commit=False)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Programmatic Alembic migration entry points for the telemetry spool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import fcntl
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from importlib.resources import as_file, files
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from alembic import command
|
|
12
|
+
from alembic.config import Config
|
|
13
|
+
from alembic.migration import MigrationContext
|
|
14
|
+
from alembic.script import ScriptDirectory
|
|
15
|
+
from sqlalchemy import inspect
|
|
16
|
+
from sqlalchemy.engine import Connection, Engine
|
|
17
|
+
|
|
18
|
+
from microcosm_emitter.core.database import create_spool_engine
|
|
19
|
+
|
|
20
|
+
_MIGRATION_TARGET = "head"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@contextmanager
|
|
24
|
+
def alembic_config(
|
|
25
|
+
*,
|
|
26
|
+
connection: Connection | None = None,
|
|
27
|
+
) -> Iterator[Config]:
|
|
28
|
+
"""Yield Alembic configuration with migrations on the filesystem."""
|
|
29
|
+
|
|
30
|
+
migration_resources = files(__package__).joinpath("alembic")
|
|
31
|
+
with as_file(migration_resources) as migration_directory:
|
|
32
|
+
config = Config()
|
|
33
|
+
config.set_main_option("script_location", str(migration_directory))
|
|
34
|
+
if connection is not None:
|
|
35
|
+
config.attributes["connection"] = connection
|
|
36
|
+
yield config
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def upgrade_spool_database(engine: Engine) -> None:
|
|
40
|
+
"""Apply every committed spool migration to an engine."""
|
|
41
|
+
|
|
42
|
+
# All build processes share this database. Serialize migration checks and
|
|
43
|
+
# upgrades; SQLite transaction locks alone do not serialize schema inspection.
|
|
44
|
+
path = Path(engine.url.database)
|
|
45
|
+
with path.with_suffix(".migration.lock").open("a") as lock:
|
|
46
|
+
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
47
|
+
with engine.begin() as connection:
|
|
48
|
+
names = set(inspect(connection).get_table_names())
|
|
49
|
+
revision = MigrationContext.configure(connection).get_current_revision()
|
|
50
|
+
if names and revision is None:
|
|
51
|
+
raise RuntimeError(
|
|
52
|
+
"Refusing an unversioned telemetry database; no changes made."
|
|
53
|
+
)
|
|
54
|
+
with alembic_config(connection=connection) as config:
|
|
55
|
+
history = ScriptDirectory.from_config(config)
|
|
56
|
+
if revision is not None:
|
|
57
|
+
history.get_revision(
|
|
58
|
+
revision
|
|
59
|
+
) # Reject unknown revisions before writes.
|
|
60
|
+
command.upgrade(config, _MIGRATION_TARGET)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def current_database_revision(path: Path | str) -> str | None:
|
|
64
|
+
"""Return the Alembic revision recorded by one spool database."""
|
|
65
|
+
|
|
66
|
+
engine = create_spool_engine(path)
|
|
67
|
+
try:
|
|
68
|
+
with engine.connect() as connection:
|
|
69
|
+
return MigrationContext.configure(connection).get_current_revision()
|
|
70
|
+
finally:
|
|
71
|
+
engine.dispose()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def migration_head_revision() -> str | None:
|
|
75
|
+
"""Return the single current head from the packaged migration history."""
|
|
76
|
+
|
|
77
|
+
with alembic_config() as config:
|
|
78
|
+
return ScriptDirectory.from_config(config).get_current_head()
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""SQLAlchemy ORM models for the local telemetry spool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from sqlalchemy import (
|
|
9
|
+
Float,
|
|
10
|
+
ForeignKeyConstraint,
|
|
11
|
+
Index,
|
|
12
|
+
Integer,
|
|
13
|
+
Text,
|
|
14
|
+
UniqueConstraint,
|
|
15
|
+
)
|
|
16
|
+
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
|
17
|
+
from sqlalchemy.types import TypeDecorator
|
|
18
|
+
|
|
19
|
+
from microcosm_emitter.core.constants import (
|
|
20
|
+
UPLOAD_STATE_PENDING,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class JsonObjectText(TypeDecorator[dict[str, Any]]):
|
|
25
|
+
"""Store JSON objects in the spool's existing text-column format."""
|
|
26
|
+
|
|
27
|
+
impl = Text
|
|
28
|
+
cache_ok = True
|
|
29
|
+
|
|
30
|
+
def process_bind_param(
|
|
31
|
+
self,
|
|
32
|
+
value: dict[str, Any] | None,
|
|
33
|
+
dialect,
|
|
34
|
+
) -> str | None:
|
|
35
|
+
if value is None:
|
|
36
|
+
return None
|
|
37
|
+
return _serialize_json_object(value)
|
|
38
|
+
|
|
39
|
+
def process_result_value(
|
|
40
|
+
self,
|
|
41
|
+
value: str | None,
|
|
42
|
+
dialect,
|
|
43
|
+
) -> dict[str, Any] | None:
|
|
44
|
+
if value is None:
|
|
45
|
+
return None
|
|
46
|
+
decoded = json.loads(value)
|
|
47
|
+
if not isinstance(decoded, dict):
|
|
48
|
+
raise ValueError("telemetry spool JSON must contain an object")
|
|
49
|
+
return decoded
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def serialized_json_length(value: dict[str, Any]) -> int:
|
|
53
|
+
"""Return the stored character count for one mapped JSON object."""
|
|
54
|
+
|
|
55
|
+
return len(_serialize_json_object(value))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _serialize_json_object(value: dict[str, Any]) -> str:
|
|
59
|
+
return json.dumps(
|
|
60
|
+
value,
|
|
61
|
+
separators=(",", ":"),
|
|
62
|
+
sort_keys=True,
|
|
63
|
+
allow_nan=False,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SpoolModel(DeclarativeBase):
|
|
68
|
+
"""Declarative base for the local telemetry spool."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class TelemetryRunRecord(SpoolModel):
|
|
72
|
+
"""One telemetry producer registered for one build run."""
|
|
73
|
+
|
|
74
|
+
__tablename__ = "telemetry_runs"
|
|
75
|
+
|
|
76
|
+
run_id: Mapped[str] = mapped_column(Text, primary_key=True)
|
|
77
|
+
producer_id: Mapped[str] = mapped_column(Text, primary_key=True)
|
|
78
|
+
registration: Mapped[dict[str, Any]] = mapped_column(
|
|
79
|
+
"registration_json",
|
|
80
|
+
JsonObjectText,
|
|
81
|
+
nullable=False,
|
|
82
|
+
)
|
|
83
|
+
next_sequence: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
|
84
|
+
upload_state: Mapped[str] = mapped_column(
|
|
85
|
+
Text,
|
|
86
|
+
nullable=False,
|
|
87
|
+
default=UPLOAD_STATE_PENDING,
|
|
88
|
+
)
|
|
89
|
+
local_only_reason: Mapped[str | None] = mapped_column(Text)
|
|
90
|
+
updated_at: Mapped[str] = mapped_column(Text, nullable=False)
|
|
91
|
+
events: Mapped[list[TelemetryEventRecord]] = relationship(
|
|
92
|
+
back_populates="run",
|
|
93
|
+
cascade="all, delete-orphan",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class TelemetryEventRecord(SpoolModel):
|
|
98
|
+
"""One ordered telemetry event awaiting collector acknowledgement."""
|
|
99
|
+
|
|
100
|
+
__tablename__ = "telemetry_events"
|
|
101
|
+
__table_args__ = (
|
|
102
|
+
UniqueConstraint(
|
|
103
|
+
"run_id",
|
|
104
|
+
"producer_id",
|
|
105
|
+
"sequence",
|
|
106
|
+
name="uq_telemetry_events_run_producer_sequence",
|
|
107
|
+
),
|
|
108
|
+
ForeignKeyConstraint(
|
|
109
|
+
["run_id", "producer_id"],
|
|
110
|
+
["telemetry_runs.run_id", "telemetry_runs.producer_id"],
|
|
111
|
+
name="fk_telemetry_events_run",
|
|
112
|
+
),
|
|
113
|
+
Index(
|
|
114
|
+
"telemetry_events_run_sequence",
|
|
115
|
+
"run_id",
|
|
116
|
+
"producer_id",
|
|
117
|
+
"sequence",
|
|
118
|
+
),
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
event_id: Mapped[str] = mapped_column(Text, primary_key=True)
|
|
122
|
+
run_id: Mapped[str] = mapped_column(Text, nullable=False)
|
|
123
|
+
producer_id: Mapped[str] = mapped_column(Text, nullable=False)
|
|
124
|
+
sequence: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
125
|
+
payload: Mapped[dict[str, Any]] = mapped_column(
|
|
126
|
+
"payload_json",
|
|
127
|
+
JsonObjectText,
|
|
128
|
+
nullable=False,
|
|
129
|
+
)
|
|
130
|
+
created_at: Mapped[str] = mapped_column(Text, nullable=False)
|
|
131
|
+
run: Mapped[TelemetryRunRecord] = relationship(back_populates="events")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class GraphPublicationJob(SpoolModel):
|
|
135
|
+
"""Durable artifact delivery, independent of event retention and run IDs."""
|
|
136
|
+
|
|
137
|
+
__tablename__ = "graph_publication_jobs"
|
|
138
|
+
publication_id: Mapped[str] = mapped_column(Text, primary_key=True)
|
|
139
|
+
inventory: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False)
|
|
140
|
+
directory: Mapped[str] = mapped_column(Text, nullable=False)
|
|
141
|
+
status: Mapped[str] = mapped_column(Text, nullable=False)
|
|
142
|
+
attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
143
|
+
next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False)
|
|
144
|
+
lease_until: Mapped[float] = mapped_column(Float, nullable=False)
|
|
145
|
+
receipt: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Generic local service support; importing this package has no side effects."""
|