microcosm-emitter 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. microcosm_emitter/__init__.py +1 -0
  2. microcosm_emitter/core/__init__.py +1 -0
  3. microcosm_emitter/core/alembic/env.py +28 -0
  4. microcosm_emitter/core/alembic/script.py.mako +31 -0
  5. microcosm_emitter/core/alembic/versions/v20261007_01_initial_spool.py +85 -0
  6. microcosm_emitter/core/alembic/versions/v20261008_02_graph_publication.py +27 -0
  7. microcosm_emitter/core/auth.py +169 -0
  8. microcosm_emitter/core/constants.py +69 -0
  9. microcosm_emitter/core/database.py +34 -0
  10. microcosm_emitter/core/migrations.py +78 -0
  11. microcosm_emitter/core/models.py +145 -0
  12. microcosm_emitter/host/__init__.py +1 -0
  13. microcosm_emitter/host/__main__.py +50 -0
  14. microcosm_emitter/host/client.py +132 -0
  15. microcosm_emitter/host/constants.py +21 -0
  16. microcosm_emitter/host/contracts.py +35 -0
  17. microcosm_emitter/host/errors.py +52 -0
  18. microcosm_emitter/host/launcher.py +184 -0
  19. microcosm_emitter/host/loading.py +22 -0
  20. microcosm_emitter/host/process.py +20 -0
  21. microcosm_emitter/host/request_dispatch.py +96 -0
  22. microcosm_emitter/host/runtime.py +227 -0
  23. microcosm_emitter/orrery/__init__.py +1 -0
  24. microcosm_emitter/orrery/cli.py +39 -0
  25. microcosm_emitter/orrery/client.py +229 -0
  26. microcosm_emitter/orrery/constants.py +5 -0
  27. microcosm_emitter/orrery/contracts.py +57 -0
  28. microcosm_emitter/orrery/py.typed +1 -0
  29. microcosm_emitter/orrery/service/__init__.py +8 -0
  30. microcosm_emitter/orrery/service/module.py +98 -0
  31. microcosm_emitter/orrery/service/queue.py +447 -0
  32. microcosm_emitter/py.typed +1 -0
  33. microcosm_emitter/telemetry/__init__.py +1 -0
  34. microcosm_emitter/telemetry/client.py +295 -0
  35. microcosm_emitter/telemetry/constants.py +16 -0
  36. microcosm_emitter/telemetry/protocol.py +57 -0
  37. microcosm_emitter/telemetry/requests.py +14 -0
  38. microcosm_emitter/telemetry/serialization.py +24 -0
  39. microcosm_emitter/telemetry/service/__init__.py +14 -0
  40. microcosm_emitter/telemetry/service/collector.py +186 -0
  41. microcosm_emitter/telemetry/service/lifecycle.py +346 -0
  42. microcosm_emitter/telemetry/service/module.py +128 -0
  43. microcosm_emitter/telemetry/service/resources.py +45 -0
  44. microcosm_emitter/telemetry/service/sanitization.py +94 -0
  45. microcosm_emitter/telemetry/service/spool.py +286 -0
  46. microcosm_emitter/telemetry/service/timestamps.py +11 -0
  47. microcosm_emitter-0.2.0.dist-info/METADATA +83 -0
  48. microcosm_emitter-0.2.0.dist-info/RECORD +51 -0
  49. microcosm_emitter-0.2.0.dist-info/WHEEL +4 -0
  50. microcosm_emitter-0.2.0.dist-info/entry_points.txt +6 -0
  51. microcosm_emitter-0.2.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1 @@
1
+ """Local emitter host and required telemetry library, without import side effects."""
@@ -0,0 +1 @@
1
+ """Shared provider services; importing this package has no runtime side effects."""
@@ -0,0 +1,28 @@
1
+ """Alembic environment for the local telemetry spool."""
2
+
3
+ from alembic import context
4
+ from sqlalchemy.engine import Connection
5
+
6
+ from microcosm_emitter.core.models import SpoolModel
7
+
8
+ _MISSING_CONNECTION_ERROR = (
9
+ "telemetry spool migrations require a programmatically supplied connection"
10
+ )
11
+
12
+
13
+ def run_migrations() -> None:
14
+ """Run migrations on the connection supplied by the emitter service."""
15
+
16
+ connection = context.config.attributes.get("connection")
17
+ if not isinstance(connection, Connection):
18
+ raise RuntimeError(_MISSING_CONNECTION_ERROR)
19
+ context.configure(
20
+ connection=connection,
21
+ target_metadata=SpoolModel.metadata,
22
+ render_as_batch=True,
23
+ )
24
+ with context.begin_transaction():
25
+ context.run_migrations()
26
+
27
+
28
+ run_migrations()
@@ -0,0 +1,31 @@
1
+ """${message}
2
+
3
+ Revision ID: ${up_revision}
4
+ Revises: ${down_revision | comma,n}
5
+ Create Date: ${create_date}
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Sequence
11
+
12
+ import sqlalchemy as sa
13
+ from alembic import op
14
+ ${imports if imports else ""}
15
+
16
+ revision: str = ${repr(up_revision)}
17
+ down_revision: str | None = ${repr(down_revision)}
18
+ branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
19
+ depends_on: str | Sequence[str] | None = ${repr(depends_on)}
20
+
21
+
22
+ def upgrade() -> None:
23
+ """Apply this schema revision."""
24
+
25
+ ${upgrades if upgrades else "pass"}
26
+
27
+
28
+ def downgrade() -> None:
29
+ """Reverse this schema revision."""
30
+
31
+ ${downgrades if downgrades else "pass"}
@@ -0,0 +1,85 @@
1
+ """Create the telemetry spool schema.
2
+
3
+ Revision ID: 20261007_01
4
+ Revises:
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Sequence
10
+
11
+ import sqlalchemy as sa
12
+ from alembic import op
13
+
14
+ revision: str = "20261007_01"
15
+ down_revision: str | None = None
16
+ branch_labels: str | Sequence[str] | None = None
17
+ depends_on: str | Sequence[str] | None = None
18
+
19
+ _RUNS_TABLE = "telemetry_runs"
20
+ _EVENTS_TABLE = "telemetry_events"
21
+ _EVENTS_INDEX = "telemetry_events_run_sequence"
22
+ _PENDING_UPLOAD_STATE = "pending"
23
+
24
+
25
+ def _create_runs_table() -> None:
26
+ op.create_table(
27
+ _RUNS_TABLE,
28
+ sa.Column("run_id", sa.Text(), nullable=False),
29
+ sa.Column("producer_id", sa.Text(), nullable=False),
30
+ sa.Column("registration_json", sa.Text(), nullable=False),
31
+ sa.Column(
32
+ "next_sequence",
33
+ sa.Integer(),
34
+ nullable=False,
35
+ server_default="1",
36
+ ),
37
+ sa.Column(
38
+ "upload_state",
39
+ sa.Text(),
40
+ nullable=False,
41
+ server_default=_PENDING_UPLOAD_STATE,
42
+ ),
43
+ sa.Column("local_only_reason", sa.Text(), nullable=True),
44
+ sa.Column("updated_at", sa.Text(), nullable=False),
45
+ sa.PrimaryKeyConstraint("run_id", "producer_id"),
46
+ )
47
+
48
+
49
+ def _create_events_table() -> None:
50
+ op.create_table(
51
+ _EVENTS_TABLE,
52
+ sa.Column("event_id", sa.Text(), nullable=False),
53
+ sa.Column("run_id", sa.Text(), nullable=False),
54
+ sa.Column("producer_id", sa.Text(), nullable=False),
55
+ sa.Column("sequence", sa.Integer(), nullable=False),
56
+ sa.Column("payload_json", sa.Text(), nullable=False),
57
+ sa.Column("created_at", sa.Text(), nullable=False),
58
+ sa.ForeignKeyConstraint(
59
+ ["run_id", "producer_id"],
60
+ [f"{_RUNS_TABLE}.run_id", f"{_RUNS_TABLE}.producer_id"],
61
+ name="fk_telemetry_events_run",
62
+ ),
63
+ sa.PrimaryKeyConstraint("event_id"),
64
+ sa.UniqueConstraint(
65
+ "run_id",
66
+ "producer_id",
67
+ "sequence",
68
+ name="uq_telemetry_events_run_producer_sequence",
69
+ ),
70
+ )
71
+
72
+
73
+ def upgrade() -> None:
74
+ """Create a new database; existing schemas must already be versioned."""
75
+ _create_runs_table()
76
+ _create_events_table()
77
+ op.create_index(_EVENTS_INDEX, _EVENTS_TABLE, ["run_id", "producer_id", "sequence"])
78
+
79
+
80
+ def downgrade() -> None:
81
+ """Remove the local spool schema."""
82
+
83
+ op.drop_index(_EVENTS_INDEX, table_name=_EVENTS_TABLE)
84
+ op.drop_table(_EVENTS_TABLE)
85
+ op.drop_table(_RUNS_TABLE)
@@ -0,0 +1,27 @@
1
+ """Add durable graph publication jobs without changing event retention."""
2
+
3
+ import sqlalchemy as sa
4
+ from alembic import op
5
+
6
+ revision = "20261008_02"
7
+ down_revision = "20261007_01"
8
+ branch_labels = None
9
+ depends_on = None
10
+
11
+
12
+ def upgrade() -> None:
13
+ op.create_table(
14
+ "graph_publication_jobs",
15
+ sa.Column("publication_id", sa.Text(), primary_key=True),
16
+ sa.Column("inventory", sa.Text(), nullable=False),
17
+ sa.Column("directory", sa.Text(), nullable=False),
18
+ sa.Column("status", sa.Text(), nullable=False),
19
+ sa.Column("attempts", sa.Integer(), nullable=False),
20
+ sa.Column("next_attempt_at", sa.Float(), nullable=False),
21
+ sa.Column("lease_until", sa.Float(), nullable=False),
22
+ sa.Column("receipt", sa.Text(), nullable=False),
23
+ )
24
+
25
+
26
+ def downgrade() -> None:
27
+ op.drop_table("graph_publication_jobs")
@@ -0,0 +1,169 @@
1
+ """HF-to-collector session shared by telemetry and graph publication."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import time
8
+ import urllib.error
9
+ import urllib.request
10
+ from collections.abc import Mapping
11
+ from http import HTTPStatus
12
+ from typing import Any
13
+ from urllib.parse import urlsplit
14
+
15
+ from huggingface_hub import get_token
16
+
17
+ from microcosm_emitter.core.constants import (
18
+ COLLECTOR_RESPONSE_TOO_LARGE_ERROR,
19
+ COLLECTOR_URL_HTTPS_ERROR,
20
+ COLLECTOR_URL_ORIGIN_ERROR,
21
+ DEFAULT_TOKEN_LIFETIME_SECONDS,
22
+ DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR,
23
+ HTTP_TIMEOUT_SECONDS,
24
+ HTTP_USER_AGENT,
25
+ LOOPBACK_HOSTS,
26
+ MAX_HTTP_RESPONSE_BYTES,
27
+ MINIMUM_TOKEN_LIFETIME_SECONDS,
28
+ TOKEN_EXCHANGE_PATH,
29
+ TOKEN_REFRESH_MARGIN_SECONDS,
30
+ )
31
+
32
+
33
+ class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
34
+ """Keep bearer credentials on the explicitly configured origin."""
35
+
36
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
37
+ return None
38
+
39
+
40
+ def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str:
41
+ parsed = urlsplit(value)
42
+ loopback = parsed.hostname in LOOPBACK_HOSTS
43
+ valid_scheme = parsed.scheme == "https" or (
44
+ allow_loopback_http and loopback and parsed.scheme == "http"
45
+ )
46
+ if not parsed.hostname or not valid_scheme:
47
+ raise ValueError(COLLECTOR_URL_HTTPS_ERROR)
48
+ if (
49
+ parsed.username
50
+ or parsed.password
51
+ or parsed.query
52
+ or parsed.fragment
53
+ or parsed.path not in {"", "/"}
54
+ ):
55
+ raise ValueError(COLLECTOR_URL_ORIGIN_ERROR)
56
+ return value.rstrip("/")
57
+
58
+
59
+ def _development_collector_url(value: str) -> str:
60
+ parsed = urlsplit(value)
61
+ if parsed.hostname not in LOOPBACK_HOSTS:
62
+ raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR)
63
+ return _collector_origin(value, allow_loopback_http=True)
64
+
65
+
66
+ def _decode_response(body: bytes) -> dict[str, Any]:
67
+ try:
68
+ response = json.loads(body) if body else {}
69
+ except (json.JSONDecodeError, UnicodeDecodeError):
70
+ return {}
71
+ return response if isinstance(response, dict) else {}
72
+
73
+
74
+ def _http_post(
75
+ url: str,
76
+ payload: Mapping[str, Any],
77
+ bearer_token: str,
78
+ *,
79
+ timeout: float = HTTP_TIMEOUT_SECONDS,
80
+ ) -> tuple[int, dict[str, Any]]:
81
+ request = urllib.request.Request(
82
+ url,
83
+ data=json.dumps(payload, separators=(",", ":")).encode(),
84
+ headers={
85
+ "Authorization": f"Bearer {bearer_token}",
86
+ "Content-Type": "application/json",
87
+ "User-Agent": HTTP_USER_AGENT,
88
+ },
89
+ method="POST",
90
+ )
91
+ try:
92
+ opener = urllib.request.build_opener(_NoRedirectHandler)
93
+ with opener.open(request, timeout=timeout) as response:
94
+ body = response.read(MAX_HTTP_RESPONSE_BYTES + 1)
95
+ if len(body) > MAX_HTTP_RESPONSE_BYTES:
96
+ raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR)
97
+ return response.status, _decode_response(body)
98
+ except urllib.error.HTTPError as error:
99
+ body = error.read(MAX_HTTP_RESPONSE_BYTES + 1)
100
+ if len(body) > MAX_HTTP_RESPONSE_BYTES:
101
+ return error.code, {}
102
+ return error.code, _decode_response(body)
103
+
104
+
105
+ def _huggingface_token() -> str | None:
106
+ return (
107
+ os.environ.get("HF_TOKEN", "").strip()
108
+ or os.environ.get("HUGGINGFACE_TOKEN", "").strip()
109
+ or get_token()
110
+ )
111
+
112
+
113
+ class CollectorSession:
114
+ """One thread-safe short-lived credential shared by independent publishers."""
115
+
116
+ def __init__(self, origin: str, *, post=None, credential=None):
117
+ import threading
118
+
119
+ self.origin = origin
120
+ self._post = post or _http_post
121
+ self._credential = credential or _huggingface_token
122
+ self._session_token: tuple[str, float] | None = None
123
+ self._lock = threading.Lock()
124
+
125
+ def invalidate(self) -> None:
126
+ with self._lock:
127
+ self._session_token = None
128
+
129
+ def credential(self) -> tuple[str, str | None]:
130
+ with self._lock:
131
+ if (
132
+ self._session_token is not None
133
+ and self._session_token[1]
134
+ > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS
135
+ ):
136
+ return "ok", self._session_token[0]
137
+ hf_token = self._credential()
138
+ if not hf_token:
139
+ return "missing", None
140
+ try:
141
+ status, response = self._post(
142
+ self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token
143
+ )
144
+ except (OSError, TimeoutError):
145
+ return "unavailable", None
146
+ if status == HTTPStatus.OK and isinstance(
147
+ response.get("access_token"), str
148
+ ):
149
+ try:
150
+ expires_in = max(
151
+ MINIMUM_TOKEN_LIFETIME_SECONDS,
152
+ int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)),
153
+ )
154
+ except (ValueError, TypeError):
155
+ return "unavailable", None
156
+ token = response["access_token"]
157
+ self._session_token = (token, time.monotonic() + expires_in)
158
+ return "ok", token
159
+ if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}:
160
+ return "rejected", None
161
+ return "unavailable", None
162
+
163
+
164
+ def shared_session(services: dict, origin: str) -> CollectorSession:
165
+ """Share a session among explicit modules without adding auth to the host."""
166
+ session = services.setdefault("collector_session", CollectorSession(origin))
167
+ if not isinstance(session, CollectorSession) or session.origin != origin:
168
+ raise ValueError("Provider modules must use the same collector origin.")
169
+ return session
@@ -0,0 +1,69 @@
1
+ """Configuration and protocol values for the telemetry emitter service."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Final
6
+
7
+ RETENTION_DAYS: Final = 7
8
+ MAX_QUEUED_BYTES: Final = 100 * 1024 * 1024
9
+ BATCH_SIZE: Final = 100
10
+ PRODUCTION_COLLECTOR_URL: Final = (
11
+ "https://microcosm-telemetry-389282473430.us-central1.run.app"
12
+ )
13
+
14
+ LOOPBACK_HOSTS: Final = frozenset({"localhost", "127.0.0.1", "::1"})
15
+ TOKEN_EXCHANGE_PATH: Final = "/v1/auth/huggingface/exchange"
16
+ RUN_REGISTRATION_PATH: Final = "/v1/runs"
17
+ RUN_EVENTS_PATH_TEMPLATE: Final = "/v1/runs/{run_id}/events"
18
+ HTTP_USER_AGENT: Final = "microcosm-telemetry-emitter/1"
19
+ MAX_HTTP_RESPONSE_BYTES: Final = 1_048_576
20
+ HTTP_TIMEOUT_SECONDS: Final = 5.0
21
+
22
+ UPLOAD_STATE_PENDING: Final = "pending"
23
+ UPLOAD_STATE_LOCAL_ONLY: Final = "local_only"
24
+ LOCAL_ONLY_MISSING_CREDENTIAL: Final = "missing_huggingface_credential"
25
+ LOCAL_ONLY_REJECTED_CREDENTIAL: Final = "huggingface_credential_rejected"
26
+ LOCAL_ONLY_REJECTED_REGISTRATION: Final = "run_registration_rejected"
27
+ LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION: Final = "collector_authorization_rejected"
28
+ LOCAL_ONLY_PRE_ELIGIBILITY: Final = "created_before_upload_eligibility"
29
+
30
+ NO_CREDENTIAL_MESSAGE: Final = (
31
+ "Microcosm telemetry is local-only: no ambient Hugging Face credential was "
32
+ "found. The dataset build will continue."
33
+ )
34
+ REJECTED_CREDENTIAL_MESSAGE: Final = (
35
+ "Microcosm telemetry is local-only for this run: the ambient Hugging Face "
36
+ "credential was not accepted as a PolicyEngine organization member. The "
37
+ "dataset build will continue."
38
+ )
39
+ COLLECTOR_URL_HTTPS_ERROR: Final = "collector URL must be an HTTPS origin"
40
+ COLLECTOR_URL_ORIGIN_ERROR: Final = (
41
+ "collector URL must be an origin without credentials or path data"
42
+ )
43
+ DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR: Final = (
44
+ "development collector URL must use a loopback address"
45
+ )
46
+ COLLECTOR_RESPONSE_TOO_LARGE_ERROR: Final = "collector response exceeds 1 MiB"
47
+
48
+ DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 3_600
49
+ MINIMUM_TOKEN_LIFETIME_SECONDS: Final = 60
50
+ TOKEN_REFRESH_MARGIN_SECONDS: Final = 30
51
+ INITIAL_RETRY_SECONDS: Final = 1.0
52
+ MAX_RETRY_SECONDS: Final = 60.0
53
+
54
+ DATABASE_TIMEOUT_SECONDS: Final = 5
55
+ PRUNE_INTERVAL_SECONDS: Final = 60.0
56
+
57
+ DEFAULT_HEARTBEAT_SECONDS: Final = 60.0
58
+ DEFAULT_DRAIN_SECONDS: Final = 15.0
59
+ MINIMUM_HEARTBEAT_SECONDS: Final = 1.0
60
+ SOCKET_LISTEN_BACKLOG: Final = 16
61
+ SOCKET_ACCEPT_TIMEOUT_SECONDS: Final = 0.5
62
+ SOCKET_CONNECTION_TIMEOUT_SECONDS: Final = 0.25
63
+ WORKER_INTERVAL_SECONDS: Final = 1.0
64
+ DRAIN_RETRY_SECONDS: Final = 0.5
65
+
66
+ EVENT_OBJECT_ERROR: Final = "event must be an object"
67
+ UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action"
68
+ LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB"
69
+ FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit"
@@ -0,0 +1,34 @@
1
+ """SQLAlchemy engine and session construction for the telemetry spool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ from sqlalchemy import URL, create_engine
8
+ from sqlalchemy.engine import Engine
9
+ from sqlalchemy.orm import Session, sessionmaker
10
+
11
+ from microcosm_emitter.core.constants import (
12
+ DATABASE_TIMEOUT_SECONDS,
13
+ )
14
+
15
+
16
+ def sqlite_database_url(path: Path | str) -> URL:
17
+ """Return a SQLAlchemy URL for a local SQLite spool path."""
18
+
19
+ return URL.create("sqlite+pysqlite", database=str(Path(path)))
20
+
21
+
22
+ def create_spool_engine(path: Path | str) -> Engine:
23
+ """Create the SQLAlchemy engine used by one emitter service process."""
24
+
25
+ return create_engine(
26
+ sqlite_database_url(path),
27
+ connect_args={"timeout": DATABASE_TIMEOUT_SECONDS},
28
+ )
29
+
30
+
31
+ def create_spool_session_factory(engine: Engine) -> sessionmaker[Session]:
32
+ """Create short-lived ORM sessions bound to the spool engine."""
33
+
34
+ return sessionmaker(engine, expire_on_commit=False)
@@ -0,0 +1,78 @@
1
+ """Programmatic Alembic migration entry points for the telemetry spool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import fcntl
6
+ from collections.abc import Iterator
7
+ from contextlib import contextmanager
8
+ from importlib.resources import as_file, files
9
+ from pathlib import Path
10
+
11
+ from alembic import command
12
+ from alembic.config import Config
13
+ from alembic.migration import MigrationContext
14
+ from alembic.script import ScriptDirectory
15
+ from sqlalchemy import inspect
16
+ from sqlalchemy.engine import Connection, Engine
17
+
18
+ from microcosm_emitter.core.database import create_spool_engine
19
+
20
+ _MIGRATION_TARGET = "head"
21
+
22
+
23
+ @contextmanager
24
+ def alembic_config(
25
+ *,
26
+ connection: Connection | None = None,
27
+ ) -> Iterator[Config]:
28
+ """Yield Alembic configuration with migrations on the filesystem."""
29
+
30
+ migration_resources = files(__package__).joinpath("alembic")
31
+ with as_file(migration_resources) as migration_directory:
32
+ config = Config()
33
+ config.set_main_option("script_location", str(migration_directory))
34
+ if connection is not None:
35
+ config.attributes["connection"] = connection
36
+ yield config
37
+
38
+
39
+ def upgrade_spool_database(engine: Engine) -> None:
40
+ """Apply every committed spool migration to an engine."""
41
+
42
+ # All build processes share this database. Serialize migration checks and
43
+ # upgrades; SQLite transaction locks alone do not serialize schema inspection.
44
+ path = Path(engine.url.database)
45
+ with path.with_suffix(".migration.lock").open("a") as lock:
46
+ fcntl.flock(lock, fcntl.LOCK_EX)
47
+ with engine.begin() as connection:
48
+ names = set(inspect(connection).get_table_names())
49
+ revision = MigrationContext.configure(connection).get_current_revision()
50
+ if names and revision is None:
51
+ raise RuntimeError(
52
+ "Refusing an unversioned telemetry database; no changes made."
53
+ )
54
+ with alembic_config(connection=connection) as config:
55
+ history = ScriptDirectory.from_config(config)
56
+ if revision is not None:
57
+ history.get_revision(
58
+ revision
59
+ ) # Reject unknown revisions before writes.
60
+ command.upgrade(config, _MIGRATION_TARGET)
61
+
62
+
63
+ def current_database_revision(path: Path | str) -> str | None:
64
+ """Return the Alembic revision recorded by one spool database."""
65
+
66
+ engine = create_spool_engine(path)
67
+ try:
68
+ with engine.connect() as connection:
69
+ return MigrationContext.configure(connection).get_current_revision()
70
+ finally:
71
+ engine.dispose()
72
+
73
+
74
+ def migration_head_revision() -> str | None:
75
+ """Return the single current head from the packaged migration history."""
76
+
77
+ with alembic_config() as config:
78
+ return ScriptDirectory.from_config(config).get_current_head()
@@ -0,0 +1,145 @@
1
+ """SQLAlchemy ORM models for the local telemetry spool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any
7
+
8
+ from sqlalchemy import (
9
+ Float,
10
+ ForeignKeyConstraint,
11
+ Index,
12
+ Integer,
13
+ Text,
14
+ UniqueConstraint,
15
+ )
16
+ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
17
+ from sqlalchemy.types import TypeDecorator
18
+
19
+ from microcosm_emitter.core.constants import (
20
+ UPLOAD_STATE_PENDING,
21
+ )
22
+
23
+
24
+ class JsonObjectText(TypeDecorator[dict[str, Any]]):
25
+ """Store JSON objects in the spool's existing text-column format."""
26
+
27
+ impl = Text
28
+ cache_ok = True
29
+
30
+ def process_bind_param(
31
+ self,
32
+ value: dict[str, Any] | None,
33
+ dialect,
34
+ ) -> str | None:
35
+ if value is None:
36
+ return None
37
+ return _serialize_json_object(value)
38
+
39
+ def process_result_value(
40
+ self,
41
+ value: str | None,
42
+ dialect,
43
+ ) -> dict[str, Any] | None:
44
+ if value is None:
45
+ return None
46
+ decoded = json.loads(value)
47
+ if not isinstance(decoded, dict):
48
+ raise ValueError("telemetry spool JSON must contain an object")
49
+ return decoded
50
+
51
+
52
+ def serialized_json_length(value: dict[str, Any]) -> int:
53
+ """Return the stored character count for one mapped JSON object."""
54
+
55
+ return len(_serialize_json_object(value))
56
+
57
+
58
+ def _serialize_json_object(value: dict[str, Any]) -> str:
59
+ return json.dumps(
60
+ value,
61
+ separators=(",", ":"),
62
+ sort_keys=True,
63
+ allow_nan=False,
64
+ )
65
+
66
+
67
+ class SpoolModel(DeclarativeBase):
68
+ """Declarative base for the local telemetry spool."""
69
+
70
+
71
+ class TelemetryRunRecord(SpoolModel):
72
+ """One telemetry producer registered for one build run."""
73
+
74
+ __tablename__ = "telemetry_runs"
75
+
76
+ run_id: Mapped[str] = mapped_column(Text, primary_key=True)
77
+ producer_id: Mapped[str] = mapped_column(Text, primary_key=True)
78
+ registration: Mapped[dict[str, Any]] = mapped_column(
79
+ "registration_json",
80
+ JsonObjectText,
81
+ nullable=False,
82
+ )
83
+ next_sequence: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
84
+ upload_state: Mapped[str] = mapped_column(
85
+ Text,
86
+ nullable=False,
87
+ default=UPLOAD_STATE_PENDING,
88
+ )
89
+ local_only_reason: Mapped[str | None] = mapped_column(Text)
90
+ updated_at: Mapped[str] = mapped_column(Text, nullable=False)
91
+ events: Mapped[list[TelemetryEventRecord]] = relationship(
92
+ back_populates="run",
93
+ cascade="all, delete-orphan",
94
+ )
95
+
96
+
97
+ class TelemetryEventRecord(SpoolModel):
98
+ """One ordered telemetry event awaiting collector acknowledgement."""
99
+
100
+ __tablename__ = "telemetry_events"
101
+ __table_args__ = (
102
+ UniqueConstraint(
103
+ "run_id",
104
+ "producer_id",
105
+ "sequence",
106
+ name="uq_telemetry_events_run_producer_sequence",
107
+ ),
108
+ ForeignKeyConstraint(
109
+ ["run_id", "producer_id"],
110
+ ["telemetry_runs.run_id", "telemetry_runs.producer_id"],
111
+ name="fk_telemetry_events_run",
112
+ ),
113
+ Index(
114
+ "telemetry_events_run_sequence",
115
+ "run_id",
116
+ "producer_id",
117
+ "sequence",
118
+ ),
119
+ )
120
+
121
+ event_id: Mapped[str] = mapped_column(Text, primary_key=True)
122
+ run_id: Mapped[str] = mapped_column(Text, nullable=False)
123
+ producer_id: Mapped[str] = mapped_column(Text, nullable=False)
124
+ sequence: Mapped[int] = mapped_column(Integer, nullable=False)
125
+ payload: Mapped[dict[str, Any]] = mapped_column(
126
+ "payload_json",
127
+ JsonObjectText,
128
+ nullable=False,
129
+ )
130
+ created_at: Mapped[str] = mapped_column(Text, nullable=False)
131
+ run: Mapped[TelemetryRunRecord] = relationship(back_populates="events")
132
+
133
+
134
+ class GraphPublicationJob(SpoolModel):
135
+ """Durable artifact delivery, independent of event retention and run IDs."""
136
+
137
+ __tablename__ = "graph_publication_jobs"
138
+ publication_id: Mapped[str] = mapped_column(Text, primary_key=True)
139
+ inventory: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False)
140
+ directory: Mapped[str] = mapped_column(Text, nullable=False)
141
+ status: Mapped[str] = mapped_column(Text, nullable=False)
142
+ attempts: Mapped[int] = mapped_column(Integer, nullable=False)
143
+ next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False)
144
+ lease_until: Mapped[float] = mapped_column(Float, nullable=False)
145
+ receipt: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False)
@@ -0,0 +1 @@
1
+ """Generic local service support; importing this package has no side effects."""