terrine 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. terrine/__init__.py +120 -0
  2. terrine/_time.py +21 -0
  3. terrine/adapters/__init__.py +3 -0
  4. terrine/adapters/increment.py +365 -0
  5. terrine/adapters/opentelemetry.py +63 -0
  6. terrine/agent/__init__.py +6 -0
  7. terrine/agent/definitions.py +143 -0
  8. terrine/agent/design_schema.py +220 -0
  9. terrine/agent/policy.py +144 -0
  10. terrine/agent/schemas.py +285 -0
  11. terrine/agent/tools.py +956 -0
  12. terrine/backends/__init__.py +3 -0
  13. terrine/backends/capabilities.py +117 -0
  14. terrine/backends/dialect.py +55 -0
  15. terrine/backends/extrema.py +40 -0
  16. terrine/backends/joins.py +75 -0
  17. terrine/backends/numeric.py +657 -0
  18. terrine/backends/sequences.py +40 -0
  19. terrine/backends/timestamps.py +421 -0
  20. terrine/compile/__init__.py +3 -0
  21. terrine/compile/_asof.py +165 -0
  22. terrine/compile/_predicates.py +110 -0
  23. terrine/compile/anchors.py +492 -0
  24. terrine/compile/columns.py +1353 -0
  25. terrine/compile/cost.py +110 -0
  26. terrine/compile/dispatch.py +148 -0
  27. terrine/compile/layers.py +242 -0
  28. terrine/compile/materialize.py +91 -0
  29. terrine/compile/provenance.py +120 -0
  30. terrine/compile/ranges.py +91 -0
  31. terrine/compile/retention.py +127 -0
  32. terrine/compile/shapes/__init__.py +3 -0
  33. terrine/compile/shapes/_observations.py +502 -0
  34. terrine/compile/shapes/calendar_comparison.py +370 -0
  35. terrine/compile/shapes/cohorts.py +285 -0
  36. terrine/compile/shapes/event_panel.py +46 -0
  37. terrine/compile/shapes/event_sequence.py +498 -0
  38. terrine/compile/shapes/life_table.py +244 -0
  39. terrine/compile/shapes/ltv.py +287 -0
  40. terrine/compile/shapes/matched_t0.py +75 -0
  41. terrine/compile/shapes/sequential.py +320 -0
  42. terrine/compile/shapes/state_panel.py +460 -0
  43. terrine/compile/shapes/time_to_event.py +521 -0
  44. terrine/data.py +1311 -0
  45. terrine/dataset/__init__.py +5 -0
  46. terrine/dataset/export.py +463 -0
  47. terrine/dataset/manifest.py +254 -0
  48. terrine/dataset/reference.py +582 -0
  49. terrine/dataset/splits.py +373 -0
  50. terrine/dataset/training.py +114 -0
  51. terrine/dataset/transforms.py +182 -0
  52. terrine/errors.py +98 -0
  53. terrine/execution/__init__.py +5 -0
  54. terrine/execution/_bigquery.py +318 -0
  55. terrine/execution/_metadata.py +47 -0
  56. terrine/execution/_native.py +574 -0
  57. terrine/execution/_postgres.py +468 -0
  58. terrine/execution/_publish.py +299 -0
  59. terrine/execution/_snowflake.py +684 -0
  60. terrine/execution/backend.py +349 -0
  61. terrine/execution/gates.py +90 -0
  62. terrine/execution/inputs.py +923 -0
  63. terrine/export/__init__.py +31 -0
  64. terrine/export/diff_diff.py +177 -0
  65. terrine/export/lifelines.py +219 -0
  66. terrine/export/preflight.py +243 -0
  67. terrine/export/surrogate.py +168 -0
  68. terrine/frame.py +1042 -0
  69. terrine/model.py +397 -0
  70. terrine/orchestration/__init__.py +3 -0
  71. terrine/orchestration/airflow.py +115 -0
  72. terrine/orchestration/dagster.py +103 -0
  73. terrine/py.typed +0 -0
  74. terrine/recipes.py +295 -0
  75. terrine/runtime/__init__.py +70 -0
  76. terrine/runtime/api.py +803 -0
  77. terrine/runtime/arrow.py +98 -0
  78. terrine/runtime/backends/base.py +70 -0
  79. terrine/runtime/backends/duckdb.py +645 -0
  80. terrine/runtime/changes.py +71 -0
  81. terrine/runtime/cli.py +108 -0
  82. terrine/runtime/control.py +355 -0
  83. terrine/runtime/cost.py +211 -0
  84. terrine/runtime/events.py +194 -0
  85. terrine/runtime/execute.py +932 -0
  86. terrine/runtime/gc.py +43 -0
  87. terrine/runtime/graph.py +200 -0
  88. terrine/runtime/layers.py +61 -0
  89. terrine/runtime/leases.py +165 -0
  90. terrine/runtime/materialize.py +53 -0
  91. terrine/runtime/owner.py +57 -0
  92. terrine/runtime/planner.py +441 -0
  93. terrine/runtime/planning.py +394 -0
  94. terrine/runtime/saved_plan.py +337 -0
  95. terrine/runtime/snapshot.py +541 -0
  96. terrine/runtime/state.py +585 -0
  97. terrine/runtime/status.py +360 -0
  98. terrine/runtime/synchronization.py +15 -0
  99. terrine/semantic/__init__.py +3 -0
  100. terrine/semantic/bind.py +309 -0
  101. terrine/semantic/io.py +116 -0
  102. terrine/semantic/models.py +342 -0
  103. terrine/semantic/rules.py +113 -0
  104. terrine/semantic/sql.py +185 -0
  105. terrine/semantic/time.py +42 -0
  106. terrine/simulate/__init__.py +24 -0
  107. terrine/simulate/baseline.yaml +57 -0
  108. terrine/simulate/generate.py +182 -0
  109. terrine/simulate/scale.py +234 -0
  110. terrine/simulate/world.py +75 -0
  111. terrine/spec/__init__.py +96 -0
  112. terrine/spec/anchors.py +107 -0
  113. terrine/spec/dataset.py +260 -0
  114. terrine/spec/design.py +361 -0
  115. terrine/spec/expressions.py +440 -0
  116. terrine/spec/io.py +829 -0
  117. terrine/spec/panels.py +359 -0
  118. terrine/spec/sequence.py +165 -0
  119. terrine/spec/state_panel.py +246 -0
  120. terrine/spec/trajectory.py +144 -0
  121. terrine/spec/transforms.py +260 -0
  122. terrine/spec/validate.py +758 -0
  123. terrine/temporal/__init__.py +3 -0
  124. terrine/temporal/aggregate.py +688 -0
  125. terrine/temporal/attributes.py +389 -0
  126. terrine/temporal/daily.py +481 -0
  127. terrine/temporal/first.py +239 -0
  128. terrine/temporal/lookup.py +194 -0
  129. terrine/temporal/nonadditive.py +466 -0
  130. terrine/temporal/prefix.py +104 -0
  131. terrine/temporal/ranges.py +122 -0
  132. terrine/temporal/readiness.py +41 -0
  133. terrine/temporal/retention.py +225 -0
  134. terrine/temporal/sequence.py +900 -0
  135. terrine/temporal/sessionize.py +135 -0
  136. terrine/temporal/sketches.py +329 -0
  137. terrine/windows.py +54 -0
  138. terrine-0.1.0.dev0.dist-info/METADATA +164 -0
  139. terrine-0.1.0.dev0.dist-info/RECORD +144 -0
  140. terrine-0.1.0.dev0.dist-info/WHEEL +4 -0
  141. terrine-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  142. terrine-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
  143. terrine-0.1.0.dev0.dist-info/licenses/NOTICE +4 -0
  144. terrine-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.txt +219 -0
terrine/agent/tools.py ADDED
@@ -0,0 +1,956 @@
1
+ """Owner-authorized dataset discovery, pinned execution and bounded reads."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import re
9
+ import secrets
10
+ import threading
11
+ from collections.abc import Callable
12
+ from dataclasses import replace
13
+ from datetime import date
14
+ from pathlib import Path
15
+ from time import monotonic
16
+ from typing import Any
17
+
18
+ from pydantic_core import to_jsonable_python
19
+
20
+ from terrine.agent.definitions import definition_schema, parse_definition
21
+ from terrine.agent.design_schema import design_schema
22
+ from terrine.agent.policy import _LIMIT, _SCOPE, AgentPolicy, Permission, effective_budget
23
+ from terrine.agent.schemas import (
24
+ AgentDatasetReference,
25
+ AgentExportManifest,
26
+ CancelOutput,
27
+ CapabilityOutput,
28
+ CatalogOutput,
29
+ DatasetColumn,
30
+ DefinitionOutput,
31
+ DesignOutput,
32
+ DesignRequest,
33
+ EventOutput,
34
+ EventsOutput,
35
+ MetricMetadata,
36
+ PlanReceipt,
37
+ PropertyMetadata,
38
+ Response,
39
+ RowsOutput,
40
+ RunOutput,
41
+ SchemaOutput,
42
+ SourceMetadata,
43
+ ValidationOutput,
44
+ refusal_response,
45
+ )
46
+ from terrine.data import _OUTPUT_BYTES_EXCEEDED, _ROWS_EXCEEDED, Budget, Preview
47
+ from terrine.dataset.manifest import DatasetManifest
48
+ from terrine.dataset.reference import Dataset, dataset_from_run
49
+ from terrine.errors import RefusalSpec, TerrineError
50
+ from terrine.model import Model
51
+ from terrine.runtime.api import Executor, cancel_run, get_run, read_run_events, run_serialized_plan
52
+ from terrine.runtime.changes import ChangeSet
53
+ from terrine.runtime.planning import OwnerPlanner, PlanInvocation, PlanRequest
54
+ from terrine.runtime.saved_plan import (
55
+ DestinationRef,
56
+ ModelReference,
57
+ SavedPlan,
58
+ _bind_saved_plan,
59
+ )
60
+ from terrine.runtime.state import RuntimeStore
61
+ from terrine.spec.anchors import EventAnchor, FixedDate
62
+ from terrine.spec.dataset import INVALID_DEFINITION, DatasetDefinition
63
+ from terrine.spec.design import Design
64
+ from terrine.spec.io import from_json, from_yaml, to_json
65
+ from terrine.spec.validate import ensure_runtime_shape_supported, validate_design
66
+
67
+ __all__ = ["DatasetTools"]
68
+
69
+ _INVALID = RefusalSpec("terrine.agent.invalid_request", "Use a valid typed Design payload.")
70
+ _STORAGE = RefusalSpec("terrine.agent.storage_denied", "Named Design storage is unavailable.")
71
+ _MODEL = RefusalSpec("terrine.agent.model_mismatch", "Owner model configuration has changed.")
72
+ _RUN = RefusalSpec(
73
+ "terrine.agent.run_unavailable", "Run has no matching owner-prepared invocation."
74
+ )
75
+ _OWNER = RefusalSpec(
76
+ "terrine.agent.owner_configuration", "Owner execution configuration is unavailable."
77
+ )
78
+ _RECEIPT = RefusalSpec(
79
+ "terrine.agent.receipt_unavailable", "Use an unchanged owner-prepared receipt."
80
+ )
81
+
82
+
83
+ def _diff(before: Any, after: Any, path: str = "") -> tuple[str, ...]:
84
+ if isinstance(before, dict) and isinstance(after, dict):
85
+ paths = []
86
+ for key in sorted(before.keys() | after.keys()):
87
+ escaped = key.replace("~", "~0").replace("/", "~1")
88
+ child = f"{path}/{escaped}"
89
+ if key not in before or key not in after:
90
+ paths.append(child)
91
+ else:
92
+ paths.extend(_diff(before[key], after[key], child))
93
+ return tuple(paths)
94
+ return () if before == after else (path,)
95
+
96
+
97
+ class DatasetTools:
98
+ """An owner-bound session with a shared request cap across all operations."""
99
+
100
+ def __init__(
101
+ self, model: Model, policy: AgentPolicy, *, namespace: str, executor: Executor | None = None
102
+ ) -> None:
103
+ if not isinstance(model, Model) or not isinstance(policy, AgentPolicy):
104
+ raise TypeError("model and policy must be owner-created typed values")
105
+ if not isinstance(namespace, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", namespace):
106
+ raise ValueError("namespace must be a simple identifier")
107
+ if executor is not None and executor is not policy.executor:
108
+ raise TypeError("executor must be the owner policy executor")
109
+ self.model = model
110
+ self.policy = policy
111
+ self.namespace = namespace
112
+ self._requests = 0
113
+ self._lock = threading.RLock()
114
+
115
+ def _call[T](self, permission: Permission, operation: Callable[[], T]) -> Response[T]:
116
+ try:
117
+ with self._lock:
118
+ if self._requests >= self.policy.max_requests:
119
+ raise _LIMIT.error()
120
+ self._requests += 1
121
+ self.policy.authorize(permission)
122
+ reference = self.policy.model_reference
123
+ if (
124
+ reference is not None
125
+ and ModelReference.from_model(self.model, owner_key=reference.owner_key)
126
+ != reference
127
+ ):
128
+ raise _MODEL.error()
129
+ catalog = (
130
+ reference.catalog_hash
131
+ if reference
132
+ else ModelReference.from_model(self.model, owner_key="inspection").catalog_hash
133
+ )
134
+ self.policy.authorize(permission, catalog=catalog)
135
+ return Response[T](status="ok", output=operation())
136
+ except TerrineError as error:
137
+ return refusal_response(error)
138
+
139
+ def _parse(self, serialized: str) -> Design:
140
+ try:
141
+ DesignRequest(serialized=serialized)
142
+ return (
143
+ from_json(serialized)
144
+ if serialized.lstrip().startswith("{")
145
+ else from_yaml(serialized)
146
+ )
147
+ except TerrineError:
148
+ raise
149
+ except (TypeError, ValueError) as error:
150
+ raise _INVALID.error(
151
+ route="Use get_design_schema and design_example to construct a Design."
152
+ ) from error
153
+
154
+ def _validated(self, design: Design) -> ValidationOutput:
155
+ report = validate_design(design, self.model)
156
+ self.policy.authorize("validate", sources=report.required_sources)
157
+ return ValidationOutput(
158
+ spec_hash=design.spec_hash,
159
+ required_sources=report.required_sources,
160
+ runtime_obligations=report.runtime_obligations,
161
+ )
162
+
163
+ def _design_output(self, serialized: str, base: str | None = None) -> DesignOutput:
164
+ design = self._parse(serialized)
165
+ valid = self._validated(design)
166
+ canonical = to_json(design)
167
+ paths = (
168
+ ()
169
+ if base is None
170
+ else _diff(json.loads(to_json(self._parse(base))), json.loads(canonical))
171
+ )
172
+ return DesignOutput(**valid.model_dump(), serialized=canonical, changed_paths=paths)
173
+
174
+ def inspect_catalog(self) -> Response[CatalogOutput]:
175
+ def inspect():
176
+ allowed = self.policy.allowed_sources
177
+ return CatalogOutput(
178
+ sources=tuple(
179
+ SourceMetadata(
180
+ name=name,
181
+ entities=tuple(sorted(source.entities)),
182
+ facts=tuple(sorted(source.facts)),
183
+ properties=tuple(sorted(source.properties)),
184
+ )
185
+ for name, source in sorted(self.model.sources.items())
186
+ if name in allowed
187
+ ),
188
+ metrics=tuple(
189
+ MetricMetadata(
190
+ name=name,
191
+ entity=metric.entity,
192
+ kind=metric.kind,
193
+ sources=metric.source_names,
194
+ )
195
+ for name, metric in sorted(self.model.metrics.items())
196
+ if set(metric.source_names) <= allowed
197
+ ),
198
+ properties=tuple(
199
+ PropertyMetadata(
200
+ name=name, source=prop.source, dtype=prop.dtype, as_of=prop.as_of
201
+ )
202
+ for name, prop in sorted(self.model.catalog.properties.items())
203
+ if prop.source in allowed
204
+ ),
205
+ units=tuple(
206
+ sorted(
207
+ {
208
+ unit
209
+ for name, source in self.model.sources.items()
210
+ if name in allowed
211
+ for unit in source.entities
212
+ }
213
+ )
214
+ ),
215
+ )
216
+
217
+ return self._call("inspect", inspect)
218
+
219
+ def list_capabilities(self) -> Response[CapabilityOutput]:
220
+ def capabilities():
221
+ shapes = []
222
+ for shape in ("cross_section", "panel", "sequence", "trajectory"):
223
+ try:
224
+ ensure_runtime_shape_supported(shape)
225
+ except TerrineError:
226
+ continue
227
+ shapes.append(shape)
228
+ from terrine.spec.state_panel import StateObservations, StatePanel
229
+
230
+ ensure_runtime_shape_supported(
231
+ StatePanel(StateObservations("state", ("a", "b")), max_periods=2)
232
+ )
233
+ shapes.append("StatePanel")
234
+ return CapabilityOutput(
235
+ operations=(
236
+ "inspect_catalog",
237
+ "list_capabilities",
238
+ "get_design_schema",
239
+ "get_dataset_definition_schema",
240
+ "design_example",
241
+ "propose_design",
242
+ "validate_design",
243
+ "propose_definition",
244
+ "validate_definition",
245
+ "save_design",
246
+ "load_design",
247
+ "preview_dataset",
248
+ "prepare_dataset",
249
+ "execute_plan",
250
+ "stage",
251
+ "prepare_refresh",
252
+ "refresh",
253
+ "export_parquet",
254
+ "preview_rows",
255
+ "get_run_status",
256
+ "get_run_events",
257
+ "request_cancel",
258
+ ),
259
+ runtime_shapes=tuple(shapes),
260
+ )
261
+
262
+ return self._call("inspect", capabilities)
263
+
264
+ def get_design_schema(self) -> Response[SchemaOutput]:
265
+ return self._call("inspect", lambda: SchemaOutput(document=design_schema()))
266
+
267
+ def design_example(self) -> Response[DesignOutput]:
268
+ def example():
269
+ sources = [
270
+ (name, source)
271
+ for name, source in sorted(self.model.sources.items())
272
+ if name in self.policy.allowed_sources and source.entities
273
+ ]
274
+ for name, source in sources:
275
+ for unit in sorted(source.entities):
276
+ if unit in self.model.units:
277
+ coverage = self.model.readiness.sources.get(name)
278
+ anchor = coverage.start.date() if coverage else date(2000, 1, 1)
279
+ return self._design_output(to_json(Design(unit, FixedDate(anchor))))
280
+ for name, source in sources:
281
+ if source.facts and name in self.model.readiness.sources:
282
+ design = Design(
283
+ sorted(source.entities)[0], EventAnchor(sorted(source.facts)[0])
284
+ )
285
+ return self._design_output(to_json(design))
286
+ raise _INVALID.error(
287
+ route="Configure a unit roster or allow an event fact with declared coverage."
288
+ )
289
+
290
+ return self._call("validate", example)
291
+
292
+ def propose_design(self, serialized: str, *, base: str | None = None) -> Response[DesignOutput]:
293
+ return self._call("validate", lambda: self._design_output(serialized, base))
294
+
295
+ def validate_design(self, serialized: str) -> Response[ValidationOutput]:
296
+ return self._call("validate", lambda: self._validated(self._parse(serialized)))
297
+
298
+ def _storage(self, name: str) -> tuple[int, str]:
299
+ if (
300
+ not isinstance(name, str)
301
+ or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_-]*", name)
302
+ or self.policy.design_root is None
303
+ ):
304
+ raise _STORAGE.error(route="Use a simple name under the owner-configured design root.")
305
+ root = self.policy.design_root
306
+ try:
307
+ root.mkdir(parents=True, exist_ok=True)
308
+ descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
309
+ return descriptor, f"{name}.json"
310
+ except OSError as error:
311
+ raise _STORAGE.error() from error
312
+
313
+ def save_design(self, name: str, serialized: str) -> Response[DesignOutput | DefinitionOutput]:
314
+ def save():
315
+ output = self._definition_output(serialized)
316
+ root, filename = self._storage(name)
317
+ temporary = f".design-{secrets.token_hex(12)}"
318
+ try:
319
+ try:
320
+ existing = os.open(filename, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root)
321
+ except FileNotFoundError:
322
+ pass
323
+ else:
324
+ os.close(existing)
325
+ descriptor = os.open(
326
+ temporary,
327
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
328
+ 0o600,
329
+ dir_fd=root,
330
+ )
331
+ with os.fdopen(descriptor, "w") as stream:
332
+ stream.write(output.serialized)
333
+ stream.flush()
334
+ os.fsync(stream.fileno())
335
+ os.replace(temporary, filename, src_dir_fd=root, dst_dir_fd=root)
336
+ except OSError as error:
337
+ raise _STORAGE.error() from error
338
+ finally:
339
+ try:
340
+ os.unlink(temporary, dir_fd=root)
341
+ except FileNotFoundError:
342
+ pass
343
+ os.close(root)
344
+ return output
345
+
346
+ return self._call("validate", save)
347
+
348
+ def load_design(self, name: str) -> Response[DesignOutput | DefinitionOutput]:
349
+ def load():
350
+ root, filename = self._storage(name)
351
+ try:
352
+ descriptor = os.open(filename, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root)
353
+ with os.fdopen(descriptor) as stream:
354
+ serialized = stream.read()
355
+ except OSError as error:
356
+ raise _STORAGE.error() from error
357
+ finally:
358
+ os.close(root)
359
+ return self._definition_output(serialized)
360
+
361
+ return self._call("validate", load)
362
+
363
+ def _authorize_run(self, run_id: str, permission: Permission) -> None:
364
+ if not isinstance(run_id, str) or not run_id:
365
+ raise _INVALID.error()
366
+ con = self.model.con.con
367
+ store = RuntimeStore(con, namespace=self.namespace)
368
+ tables = {
369
+ item[0]
370
+ for item in con.execute(
371
+ "SELECT table_name FROM information_schema.tables "
372
+ "WHERE lower(table_schema)=lower(?)",
373
+ [self.namespace],
374
+ ).fetchall()
375
+ }
376
+ if not {"runtime_runs", "runtime_plan_invocations"} <= tables:
377
+ raise _RUN.error()
378
+ columns = {
379
+ item[0]
380
+ for item in con.execute(
381
+ "SELECT column_name FROM information_schema.columns "
382
+ "WHERE lower(table_schema)=lower(?) AND table_name='runtime_runs'",
383
+ [self.namespace],
384
+ ).fetchall()
385
+ }
386
+ if "destination_json" not in columns:
387
+ raise _RUN.error()
388
+ row = con.execute(
389
+ f"SELECT destination_json, fingerprint FROM {store._schema}.runtime_runs "
390
+ "WHERE run_id=?",
391
+ [run_id],
392
+ ).fetchone()
393
+ if row is None or row[0] is None:
394
+ raise _RUN.error()
395
+ destination = DestinationRef(**json.loads(row[0]))
396
+ self.policy.authorize(permission, destination=destination)
397
+ invocations = con.execute(
398
+ f"SELECT invocation_json FROM {store._schema}.runtime_plan_invocations"
399
+ ).fetchall()
400
+ for record in invocations:
401
+ invocation = PlanInvocation.from_dict(json.loads(record[0]))
402
+ if invocation.destination != destination or not invocation.saved_plan_json:
403
+ continue
404
+ definition = SavedPlan.from_json(invocation.saved_plan_json)
405
+ if (
406
+ definition.model != self.policy.model_reference
407
+ or definition.namespace != self.namespace
408
+ ):
409
+ continue
410
+ if (
411
+ _bind_saved_plan(definition, namespace=self.namespace).definition.fingerprint
412
+ != row[1]
413
+ ):
414
+ continue
415
+ report = validate_design(from_json(definition.design_json), self.model)
416
+ self.policy.authorize(permission, sources=report.required_sources)
417
+ return
418
+ raise _RUN.error(route="Use a run from an owner-prepared invocation for this Model.")
419
+
420
+ def get_run_status(self, run_id: str) -> Response[RunOutput]:
421
+ def status():
422
+ self._authorize_run(run_id, "read")
423
+ result = get_run(run_id, con=self.model.con.con, namespace=self.namespace)
424
+ return RunOutput(
425
+ run_id=result.run_id,
426
+ durable_state=result.durable_state,
427
+ phase=result.phase,
428
+ error_code=result.error_code,
429
+ pinned_result_ref=result.pinned_result_ref,
430
+ latest_event_cursor=result.latest_event_cursor,
431
+ )
432
+
433
+ return self._call("read", status)
434
+
435
+ def get_run_events(
436
+ self, run_id: str, *, after: str | None = None, limit: int = 100
437
+ ) -> Response[EventsOutput]:
438
+ def events():
439
+ if (
440
+ isinstance(limit, bool)
441
+ or not isinstance(limit, int)
442
+ or not 0 < limit <= self.policy.batch_cap
443
+ ):
444
+ raise _INVALID.error(
445
+ route="Choose a positive event limit no larger than the owner batch cap."
446
+ )
447
+ self._authorize_run(run_id, "read")
448
+ page = read_run_events(
449
+ run_id, con=self.model.con.con, namespace=self.namespace, after=after, limit=limit
450
+ )
451
+ return EventsOutput(
452
+ events=tuple(
453
+ EventOutput(
454
+ sequence=event.sequence,
455
+ kind=event.kind,
456
+ state=event.payload.get("state"),
457
+ phase=event.payload.get("phase"),
458
+ )
459
+ for event in page.events
460
+ ),
461
+ next_cursor=page.next_cursor,
462
+ has_more=page.has_more,
463
+ cursor_gap=page.cursor_gap,
464
+ )
465
+
466
+ return self._call("read", events)
467
+
468
+ def request_cancel(self, run_id: str) -> Response[CancelOutput]:
469
+ def cancel():
470
+ self._authorize_run(run_id, "cancel")
471
+ receipt = cancel_run(run_id, con=self.model.con.con, namespace=self.namespace)
472
+ return CancelOutput(
473
+ run_id=run_id,
474
+ disposition=receipt.disposition,
475
+ durable_state=receipt.status.durable_state,
476
+ )
477
+
478
+ return self._call("cancel", cancel)
479
+
480
+ def get_dataset_definition_schema(self) -> Response[SchemaOutput]:
481
+ return self._call("inspect", lambda: SchemaOutput(document=definition_schema()))
482
+
483
+ def _dataset_definition_output(
484
+ self, serialized: str, base: str | None = None
485
+ ) -> DefinitionOutput:
486
+ definition = parse_definition(serialized, self.model)
487
+ valid = self._validated(definition.base_design)
488
+ canonical = definition.canonical_json().decode()
489
+ paths = (
490
+ ()
491
+ if base is None
492
+ else _diff(
493
+ json.loads(parse_definition(base, self.model).canonical_json()),
494
+ json.loads(canonical),
495
+ )
496
+ )
497
+ return DefinitionOutput(
498
+ **valid.model_dump(),
499
+ serialized=canonical,
500
+ changed_paths=paths,
501
+ definition_digest=definition.digest,
502
+ )
503
+
504
+ def propose_definition(
505
+ self, serialized: str, *, base: str | None = None
506
+ ) -> Response[DefinitionOutput]:
507
+ return self._call("validate", lambda: self._dataset_definition_output(serialized, base))
508
+
509
+ def validate_definition(self, serialized: str) -> Response[DefinitionOutput]:
510
+ def validate():
511
+ output = self._dataset_definition_output(serialized)
512
+ try:
513
+ parse_definition(serialized, self.model).build(self.model)
514
+ except (KeyError, TypeError, ValueError) as error:
515
+ if isinstance(error, TerrineError):
516
+ raise
517
+ raise _INVALID.error(
518
+ route="Use the recorded training or scoring view fields."
519
+ ) from error
520
+ return output
521
+
522
+ return self._call("validate", validate)
523
+
524
+ def _definition_output(self, serialized: str) -> DesignOutput | DefinitionOutput:
525
+ try:
526
+ payload = json.loads(serialized)
527
+ except (TypeError, ValueError):
528
+ return self._design_output(serialized)
529
+ if isinstance(payload, dict) and "base_design" in payload:
530
+ return self._dataset_definition_output(serialized)
531
+ return self._design_output(serialized)
532
+
533
+ def _dataset_definition(self, definition: DatasetDefinition | str) -> DatasetDefinition:
534
+ if not isinstance(definition, (DatasetDefinition, str)):
535
+ raise _INVALID.error(route="Use the discovered DatasetDefinition schema.")
536
+ return parse_definition(
537
+ definition.canonical_json().decode()
538
+ if isinstance(definition, DatasetDefinition)
539
+ else definition,
540
+ self.model,
541
+ )
542
+
543
+ def _resolved_definition_design(self, definition: DatasetDefinition) -> Design:
544
+ try:
545
+ return definition.build(self.model)._design
546
+ except TerrineError:
547
+ raise
548
+ except (KeyError, TypeError, ValueError, OverflowError) as error:
549
+ raise INVALID_DEFINITION.error(reason="dataset view cannot be constructed") from error
550
+
551
+ def preview_dataset(
552
+ self,
553
+ definition: DatasetDefinition | str,
554
+ *,
555
+ budget: Budget | None = None,
556
+ bounded_count: int = 0,
557
+ ) -> Response[Preview]:
558
+ def preview():
559
+ from terrine.compile import cost
560
+
561
+ if (
562
+ isinstance(bounded_count, bool)
563
+ or not isinstance(bounded_count, int)
564
+ or not 0 <= bounded_count <= self.policy.max_bounded_count
565
+ ):
566
+ raise _INVALID.error(route="Use a bounded count within the owner cap.")
567
+ parsed = self._dataset_definition(definition)
568
+ design = self._resolved_definition_design(parsed)
569
+ valid = self._validated(design)
570
+ if bounded_count:
571
+ self.policy.authorize("preview", sources=valid.required_sources)
572
+ effective = effective_budget(self.policy.budget, budget)
573
+ cost._scan_policy(effective)
574
+ result = cost.preview(design, self.model, effective, bounded_count=bounded_count)
575
+ return replace(
576
+ result,
577
+ coverage={
578
+ name: value
579
+ for name, value in result.coverage.items()
580
+ if name in valid.required_sources
581
+ },
582
+ )
583
+
584
+ return self._call("validate", preview)
585
+
586
+ def _owner_planner(self, destination: DestinationRef) -> OwnerPlanner:
587
+ authority = self.policy.snapshot_authority
588
+ if authority is None or self.policy.resolver is None or self.policy.model_reference is None:
589
+ raise _OWNER.error(route="Configure an owner resolver and retained snapshot authority.")
590
+ from terrine.runtime.owner import RuntimeOwner
591
+
592
+ try:
593
+ connected_owner = RuntimeOwner.from_connection(self.model.con)
594
+ except (TypeError, ValueError) as error:
595
+ raise _OWNER.error(
596
+ route="Use an owner-configured file-backed DuckDB connection."
597
+ ) from error
598
+ if connected_owner.reference != authority.owner.reference:
599
+ raise _OWNER.error(
600
+ route="Align the owner's configured snapshot root with the connected dataset owner."
601
+ )
602
+ if (
603
+ destination.schema != self.namespace
604
+ or Path(destination.database).resolve() != authority.owner.state_path.resolve()
605
+ ):
606
+ raise _SCOPE.error(route="Use this owner's database and runtime namespace.")
607
+ return OwnerPlanner(resolver=self.policy.resolver, snapshot_authority=authority)
608
+
609
+ @staticmethod
610
+ def _receipt(invocation: PlanInvocation) -> PlanReceipt:
611
+ saved = SavedPlan.from_json(invocation.saved_plan_json)
612
+ identity = json.dumps(invocation.to_dict(), sort_keys=True, separators=(",", ":"))
613
+ return PlanReceipt(
614
+ receipt_id=hashlib.sha256(identity.encode()).hexdigest(),
615
+ design_hash=saved.design_hash,
616
+ plan_hash=saved.plan_hash,
617
+ )
618
+
619
+ def _prepare(
620
+ self,
621
+ definition: DatasetDefinition,
622
+ *,
623
+ destination: DestinationRef,
624
+ invocation_id: str,
625
+ budget: Budget | None,
626
+ changes: ChangeSet | None = None,
627
+ expected_base_manifest: str | None = None,
628
+ bind_base_manifest: bool = False,
629
+ ) -> PlanReceipt:
630
+ valid = self._validated(definition.base_design)
631
+ self.policy.authorize("prepare", sources=valid.required_sources, destination=destination)
632
+ planner = self._owner_planner(destination)
633
+ reference = self.policy.model_reference
634
+ if reference is None:
635
+ raise _OWNER.error()
636
+ try:
637
+ request = PlanRequest(
638
+ invocation_id,
639
+ reference,
640
+ definition.base_design,
641
+ destination,
642
+ changes=changes,
643
+ budget=effective_budget(self.policy.budget, budget),
644
+ dataset_definition=definition,
645
+ expected_base_manifest=expected_base_manifest,
646
+ bind_base_manifest=bind_base_manifest,
647
+ )
648
+ except (TypeError, ValueError) as error:
649
+ raise _INVALID.error(
650
+ route="Use a safe invocation ID and typed change evidence."
651
+ ) from error
652
+ return self._receipt(
653
+ planner.plan(request, con=self.model.con.con, namespace=self.namespace)
654
+ )
655
+
656
+ def prepare_dataset(
657
+ self,
658
+ definition: DatasetDefinition | str,
659
+ *,
660
+ destination: DestinationRef,
661
+ invocation_id: str,
662
+ budget: Budget | None = None,
663
+ ) -> Response[PlanReceipt]:
664
+ return self._call(
665
+ "prepare",
666
+ lambda: self._prepare(
667
+ self._dataset_definition(definition),
668
+ destination=destination,
669
+ invocation_id=invocation_id,
670
+ budget=budget,
671
+ ),
672
+ )
673
+
674
+ def _resolve_receipt(
675
+ self, receipt: PlanReceipt, permission: Permission
676
+ ) -> tuple[PlanInvocation, SavedPlan]:
677
+ if not isinstance(receipt, PlanReceipt):
678
+ raise _RECEIPT.error()
679
+ con = self.model.con.con
680
+ exists = con.execute(
681
+ "SELECT 1 FROM information_schema.tables WHERE lower(table_schema)=lower(?) "
682
+ "AND table_name='runtime_plan_invocations'",
683
+ [self.namespace],
684
+ ).fetchone()
685
+ if exists is None:
686
+ raise _RECEIPT.error()
687
+ for (payload,) in con.execute(
688
+ f'SELECT invocation_json FROM "{self.namespace}".runtime_plan_invocations'
689
+ ).fetchall():
690
+ document = json.loads(payload)
691
+ identity = json.dumps(document, sort_keys=True, separators=(",", ":"))
692
+ if hashlib.sha256(identity.encode()).hexdigest() != receipt.receipt_id:
693
+ continue
694
+ invocation = PlanInvocation.from_dict(document)
695
+ if self._receipt(invocation) != receipt:
696
+ continue
697
+ saved = SavedPlan.from_json(invocation.saved_plan_json)
698
+ if saved.namespace != self.namespace or saved.model != self.policy.model_reference:
699
+ raise _SCOPE.error()
700
+ valid = validate_design(from_json(saved.design_json), self.model)
701
+ self.policy.authorize(
702
+ permission, sources=valid.required_sources, destination=saved.destination
703
+ )
704
+ self._owner_planner(saved.destination)
705
+ planned_budget = _bind_saved_plan(saved, namespace=self.namespace).budget
706
+ if effective_budget(self.policy.budget, planned_budget) != planned_budget:
707
+ raise _SCOPE.error(route="Prepare a new invocation under the current owner budget.")
708
+ return invocation, saved
709
+ raise _RECEIPT.error()
710
+
711
+ def _execute(self, receipt: PlanReceipt, *, request_key: str):
712
+ if not isinstance(request_key, str) or not request_key.strip() or len(request_key) > 256:
713
+ raise _INVALID.error(route="Use a nonempty request key of at most 256 characters.")
714
+ invocation, saved = self._resolve_receipt(receipt, "write")
715
+ deadline = monotonic() + self.policy.max_run_seconds
716
+ return run_serialized_plan(
717
+ invocation.saved_plan_json,
718
+ con=self.model.con.con,
719
+ namespace=self.namespace,
720
+ request_key=request_key,
721
+ destination=saved.destination,
722
+ executor=self.policy.executor,
723
+ resolver=self.policy.resolver,
724
+ snapshot_authority=self.policy.snapshot_authority,
725
+ max_active_runs=self.policy.max_concurrency,
726
+ cancelled=lambda: monotonic() >= deadline,
727
+ )
728
+
729
+ def execute_plan(self, receipt: PlanReceipt, *, request_key: str) -> Response[RunOutput]:
730
+ def execute():
731
+ result = self._execute(receipt, request_key=request_key)
732
+ status = get_run(result.run_id, con=self.model.con.con, namespace=self.namespace)
733
+ return RunOutput(
734
+ run_id=status.run_id,
735
+ durable_state=status.durable_state,
736
+ phase=status.phase,
737
+ error_code=status.error_code,
738
+ pinned_result_ref=status.pinned_result_ref,
739
+ latest_event_cursor=status.latest_event_cursor,
740
+ )
741
+
742
+ return self._call("write", execute)
743
+
744
+ @staticmethod
745
+ def _dataset_reference(dataset: Dataset) -> AgentDatasetReference:
746
+ manifest = dataset.manifest
747
+ return AgentDatasetReference(
748
+ dataset=manifest.generation.dataset_id,
749
+ generation=manifest.generation.generation_id,
750
+ definition_hash=manifest.design_hash,
751
+ model_hash=manifest.model_hash,
752
+ manifest_hash=hashlib.sha256(manifest.to_json()).hexdigest(),
753
+ definition_reference=manifest.definition.digest,
754
+ row_count=manifest.generation.row_count,
755
+ columns=tuple(
756
+ DatasetColumn(
757
+ name=column.name,
758
+ data_type=column.data_type,
759
+ role=column.role,
760
+ complete_column=column.metadata.get("complete_column"),
761
+ reason_column=column.metadata.get("reason_column"),
762
+ )
763
+ for column in manifest.columns
764
+ ),
765
+ )
766
+
767
+ def _resolve_dataset(
768
+ self, reference: AgentDatasetReference, permission: Permission
769
+ ) -> tuple[Dataset, SavedPlan]:
770
+ if not isinstance(reference, AgentDatasetReference):
771
+ raise _RECEIPT.error(route="Use the unchanged reference returned by stage or refresh.")
772
+ con = self.model.con.con
773
+ exists = con.execute(
774
+ "SELECT 1 FROM information_schema.tables WHERE lower(table_schema)=lower(?) "
775
+ "AND table_name='runtime_dataset_recipes'",
776
+ [self.namespace],
777
+ ).fetchone()
778
+ if exists is None:
779
+ raise _RECEIPT.error()
780
+ record = con.execute(
781
+ f'SELECT recipe_json,manifest_json FROM "{self.namespace}".runtime_dataset_recipes '
782
+ "WHERE dataset=? AND generation=?",
783
+ [reference.dataset, reference.generation],
784
+ ).fetchone()
785
+ if record is None:
786
+ raise _RECEIPT.error()
787
+ saved = SavedPlan.from_json(record[0])
788
+ if saved.namespace != self.namespace or saved.model != self.policy.model_reference:
789
+ raise _SCOPE.error()
790
+ valid = validate_design(from_json(saved.design_json), self.model)
791
+ self.policy.authorize(
792
+ permission, sources=valid.required_sources, destination=saved.destination
793
+ )
794
+ manifest = DatasetManifest.from_json(record[1])
795
+ if manifest.definition.digest != reference.definition_reference:
796
+ raise _RECEIPT.error()
797
+ dataset = Dataset.open_generation(
798
+ reference.dataset, reference.generation, con=self.model.con, namespace=self.namespace
799
+ )
800
+ if self._dataset_reference(dataset) != reference:
801
+ raise _RECEIPT.error()
802
+ return dataset, saved
803
+
804
+ def _stage(self, receipt: PlanReceipt, *, request_key: str) -> AgentDatasetReference:
805
+ _, saved = self._resolve_receipt(receipt, "write")
806
+ result = self._execute(receipt, request_key=request_key)
807
+ authority = self.policy.snapshot_authority
808
+ if result.state != "succeeded" or authority is None:
809
+ raise _RUN.error(route="Inspect the run state before requesting a committed dataset.")
810
+ return self._dataset_reference(
811
+ dataset_from_run(
812
+ saved,
813
+ result,
814
+ model=self.model,
815
+ namespace=self.namespace,
816
+ request_key=request_key,
817
+ snapshot_authority=authority,
818
+ )
819
+ )
820
+
821
+ def stage(self, receipt: PlanReceipt, *, request_key: str) -> Response[AgentDatasetReference]:
822
+ return self._call("write", lambda: self._stage(receipt, request_key=request_key))
823
+
824
+ def prepare_refresh(
825
+ self,
826
+ reference: AgentDatasetReference,
827
+ *,
828
+ invocation_id: str,
829
+ changes: ChangeSet | None = None,
830
+ budget: Budget | None = None,
831
+ ) -> Response[PlanReceipt]:
832
+ def prepare():
833
+ dataset, saved = self._resolve_dataset(reference, "prepare")
834
+ return self._prepare(
835
+ dataset.manifest.definition,
836
+ destination=saved.destination,
837
+ invocation_id=invocation_id,
838
+ budget=budget,
839
+ changes=changes,
840
+ expected_base_manifest=reference.generation,
841
+ bind_base_manifest=True,
842
+ )
843
+
844
+ return self._call("prepare", prepare)
845
+
846
+ def refresh(
847
+ self, reference: AgentDatasetReference, receipt: PlanReceipt, *, request_key: str
848
+ ) -> Response[AgentDatasetReference]:
849
+ def refresh():
850
+ self._resolve_dataset(reference, "write")
851
+ _, saved = self._resolve_receipt(receipt, "write")
852
+ if (
853
+ saved.expected_base_manifest != reference.generation
854
+ or saved.destination.dataset != reference.dataset
855
+ ):
856
+ raise _RECEIPT.error(
857
+ route="Use a refresh receipt prepared from this exact generation."
858
+ )
859
+ return self._stage(receipt, request_key=request_key)
860
+
861
+ return self._call("write", refresh)
862
+
863
+ def export_parquet(
864
+ self,
865
+ reference: AgentDatasetReference,
866
+ *,
867
+ destination: str,
868
+ partition_by: tuple[str, ...] = (),
869
+ budget: Budget | None = None,
870
+ ) -> Response[AgentExportManifest]:
871
+ def export():
872
+ from terrine.compile.cost import _scan_policy
873
+
874
+ if not isinstance(destination, str) or not destination:
875
+ raise _INVALID.error()
876
+ path = Path(destination).expanduser().resolve()
877
+ if path not in self.policy.allowed_exports:
878
+ raise _SCOPE.error(route="Use an exact owner-approved export destination.")
879
+ effective = effective_budget(self.policy.budget, budget)
880
+ _scan_policy(effective)
881
+ dataset, _ = self._resolve_dataset(reference, "export")
882
+ manifest = dataset.export_parquet(
883
+ destination=str(path), partition_by=partition_by, budget=effective
884
+ )
885
+ return AgentExportManifest(
886
+ dataset=manifest.dataset_id,
887
+ generation=manifest.generation_id,
888
+ row_count=manifest.row_count,
889
+ byte_count=manifest.byte_count,
890
+ shard_count=len(manifest.shards),
891
+ )
892
+
893
+ return self._call("export", export)
894
+
895
+ def preview_rows(
896
+ self, reference: AgentDatasetReference, *, limit: int, columns: tuple[str, ...]
897
+ ) -> Response[RowsOutput]:
898
+ def preview():
899
+ from terrine.compile.cost import _scan_policy
900
+
901
+ if (
902
+ isinstance(limit, bool)
903
+ or not isinstance(limit, int)
904
+ or not 0 < limit <= self.policy.max_preview_rows
905
+ ):
906
+ raise _INVALID.error(route="Use a positive row limit within the owner preview cap.")
907
+ if (
908
+ not isinstance(columns, tuple)
909
+ or not columns
910
+ or any(not isinstance(name, str) for name in columns)
911
+ or len(set(columns)) != len(columns)
912
+ or not set(columns) <= self.policy.preview_columns
913
+ ):
914
+ raise _SCOPE.error(route="Request only unique owner-approved preview columns.")
915
+ budget = self.policy.budget
916
+ _scan_policy(budget)
917
+ dataset, _ = self._resolve_dataset(reference, "preview")
918
+ if not set(columns) <= {column.name for column in dataset.manifest.columns}:
919
+ raise _SCOPE.error()
920
+ if (
921
+ budget.max_rows is not None
922
+ and min(limit, dataset.manifest.generation.row_count) > budget.max_rows
923
+ ):
924
+ raise _ROWS_EXCEEDED.error(limit=budget.max_rows)
925
+ iterator = dataset.to_arrow_batches(batch_size=min(limit, self.policy.batch_cap))
926
+ rows = []
927
+ byte_count = 0
928
+ try:
929
+ for batch in iterator:
930
+ projected = batch.select(columns).slice(0, limit - len(rows))
931
+ if (
932
+ budget.max_rows is not None
933
+ and len(rows) + projected.num_rows > budget.max_rows
934
+ ):
935
+ raise _ROWS_EXCEEDED.error(limit=budget.max_rows)
936
+ byte_count += projected.nbytes
937
+ if budget.max_output_bytes is not None and byte_count > budget.max_output_bytes:
938
+ raise _OUTPUT_BYTES_EXCEEDED.error(limit=budget.max_output_bytes)
939
+ for row in projected.to_pylist():
940
+ rows.append(
941
+ tuple(
942
+ None
943
+ if name in self.policy.redacted_columns
944
+ else to_jsonable_python(
945
+ row[name], bytes_mode="base64", inf_nan_mode="strings"
946
+ )
947
+ for name in columns
948
+ )
949
+ )
950
+ if len(rows) >= limit:
951
+ break
952
+ finally:
953
+ iterator.close()
954
+ return RowsOutput(columns=columns, rows=tuple(rows))
955
+
956
+ return self._call("preview", preview)