terrine 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terrine/__init__.py +120 -0
- terrine/_time.py +21 -0
- terrine/adapters/__init__.py +3 -0
- terrine/adapters/increment.py +365 -0
- terrine/adapters/opentelemetry.py +63 -0
- terrine/agent/__init__.py +6 -0
- terrine/agent/definitions.py +143 -0
- terrine/agent/design_schema.py +220 -0
- terrine/agent/policy.py +144 -0
- terrine/agent/schemas.py +285 -0
- terrine/agent/tools.py +956 -0
- terrine/backends/__init__.py +3 -0
- terrine/backends/capabilities.py +117 -0
- terrine/backends/dialect.py +55 -0
- terrine/backends/extrema.py +40 -0
- terrine/backends/joins.py +75 -0
- terrine/backends/numeric.py +657 -0
- terrine/backends/sequences.py +40 -0
- terrine/backends/timestamps.py +421 -0
- terrine/compile/__init__.py +3 -0
- terrine/compile/_asof.py +165 -0
- terrine/compile/_predicates.py +110 -0
- terrine/compile/anchors.py +492 -0
- terrine/compile/columns.py +1353 -0
- terrine/compile/cost.py +110 -0
- terrine/compile/dispatch.py +148 -0
- terrine/compile/layers.py +242 -0
- terrine/compile/materialize.py +91 -0
- terrine/compile/provenance.py +120 -0
- terrine/compile/ranges.py +91 -0
- terrine/compile/retention.py +127 -0
- terrine/compile/shapes/__init__.py +3 -0
- terrine/compile/shapes/_observations.py +502 -0
- terrine/compile/shapes/calendar_comparison.py +370 -0
- terrine/compile/shapes/cohorts.py +285 -0
- terrine/compile/shapes/event_panel.py +46 -0
- terrine/compile/shapes/event_sequence.py +498 -0
- terrine/compile/shapes/life_table.py +244 -0
- terrine/compile/shapes/ltv.py +287 -0
- terrine/compile/shapes/matched_t0.py +75 -0
- terrine/compile/shapes/sequential.py +320 -0
- terrine/compile/shapes/state_panel.py +460 -0
- terrine/compile/shapes/time_to_event.py +521 -0
- terrine/data.py +1311 -0
- terrine/dataset/__init__.py +5 -0
- terrine/dataset/export.py +463 -0
- terrine/dataset/manifest.py +254 -0
- terrine/dataset/reference.py +582 -0
- terrine/dataset/splits.py +373 -0
- terrine/dataset/training.py +114 -0
- terrine/dataset/transforms.py +182 -0
- terrine/errors.py +98 -0
- terrine/execution/__init__.py +5 -0
- terrine/execution/_bigquery.py +318 -0
- terrine/execution/_metadata.py +47 -0
- terrine/execution/_native.py +574 -0
- terrine/execution/_postgres.py +468 -0
- terrine/execution/_publish.py +299 -0
- terrine/execution/_snowflake.py +684 -0
- terrine/execution/backend.py +349 -0
- terrine/execution/gates.py +90 -0
- terrine/execution/inputs.py +923 -0
- terrine/export/__init__.py +31 -0
- terrine/export/diff_diff.py +177 -0
- terrine/export/lifelines.py +219 -0
- terrine/export/preflight.py +243 -0
- terrine/export/surrogate.py +168 -0
- terrine/frame.py +1042 -0
- terrine/model.py +397 -0
- terrine/orchestration/__init__.py +3 -0
- terrine/orchestration/airflow.py +115 -0
- terrine/orchestration/dagster.py +103 -0
- terrine/py.typed +0 -0
- terrine/recipes.py +295 -0
- terrine/runtime/__init__.py +70 -0
- terrine/runtime/api.py +803 -0
- terrine/runtime/arrow.py +98 -0
- terrine/runtime/backends/base.py +70 -0
- terrine/runtime/backends/duckdb.py +645 -0
- terrine/runtime/changes.py +71 -0
- terrine/runtime/cli.py +108 -0
- terrine/runtime/control.py +355 -0
- terrine/runtime/cost.py +211 -0
- terrine/runtime/events.py +194 -0
- terrine/runtime/execute.py +932 -0
- terrine/runtime/gc.py +43 -0
- terrine/runtime/graph.py +200 -0
- terrine/runtime/layers.py +61 -0
- terrine/runtime/leases.py +165 -0
- terrine/runtime/materialize.py +53 -0
- terrine/runtime/owner.py +57 -0
- terrine/runtime/planner.py +441 -0
- terrine/runtime/planning.py +394 -0
- terrine/runtime/saved_plan.py +337 -0
- terrine/runtime/snapshot.py +541 -0
- terrine/runtime/state.py +585 -0
- terrine/runtime/status.py +360 -0
- terrine/runtime/synchronization.py +15 -0
- terrine/semantic/__init__.py +3 -0
- terrine/semantic/bind.py +309 -0
- terrine/semantic/io.py +116 -0
- terrine/semantic/models.py +342 -0
- terrine/semantic/rules.py +113 -0
- terrine/semantic/sql.py +185 -0
- terrine/semantic/time.py +42 -0
- terrine/simulate/__init__.py +24 -0
- terrine/simulate/baseline.yaml +57 -0
- terrine/simulate/generate.py +182 -0
- terrine/simulate/scale.py +234 -0
- terrine/simulate/world.py +75 -0
- terrine/spec/__init__.py +96 -0
- terrine/spec/anchors.py +107 -0
- terrine/spec/dataset.py +260 -0
- terrine/spec/design.py +361 -0
- terrine/spec/expressions.py +440 -0
- terrine/spec/io.py +829 -0
- terrine/spec/panels.py +359 -0
- terrine/spec/sequence.py +165 -0
- terrine/spec/state_panel.py +246 -0
- terrine/spec/trajectory.py +144 -0
- terrine/spec/transforms.py +260 -0
- terrine/spec/validate.py +758 -0
- terrine/temporal/__init__.py +3 -0
- terrine/temporal/aggregate.py +688 -0
- terrine/temporal/attributes.py +389 -0
- terrine/temporal/daily.py +481 -0
- terrine/temporal/first.py +239 -0
- terrine/temporal/lookup.py +194 -0
- terrine/temporal/nonadditive.py +466 -0
- terrine/temporal/prefix.py +104 -0
- terrine/temporal/ranges.py +122 -0
- terrine/temporal/readiness.py +41 -0
- terrine/temporal/retention.py +225 -0
- terrine/temporal/sequence.py +900 -0
- terrine/temporal/sessionize.py +135 -0
- terrine/temporal/sketches.py +329 -0
- terrine/windows.py +54 -0
- terrine-0.1.0.dev0.dist-info/METADATA +164 -0
- terrine-0.1.0.dev0.dist-info/RECORD +144 -0
- terrine-0.1.0.dev0.dist-info/WHEEL +4 -0
- terrine-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- terrine-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- terrine-0.1.0.dev0.dist-info/licenses/NOTICE +4 -0
- terrine-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.txt +219 -0
terrine/agent/tools.py
ADDED
|
@@ -0,0 +1,956 @@
|
|
|
1
|
+
"""Owner-authorized dataset discovery, pinned execution and bounded reads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import secrets
|
|
10
|
+
import threading
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import replace
|
|
13
|
+
from datetime import date
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from time import monotonic
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from pydantic_core import to_jsonable_python
|
|
19
|
+
|
|
20
|
+
from terrine.agent.definitions import definition_schema, parse_definition
|
|
21
|
+
from terrine.agent.design_schema import design_schema
|
|
22
|
+
from terrine.agent.policy import _LIMIT, _SCOPE, AgentPolicy, Permission, effective_budget
|
|
23
|
+
from terrine.agent.schemas import (
|
|
24
|
+
AgentDatasetReference,
|
|
25
|
+
AgentExportManifest,
|
|
26
|
+
CancelOutput,
|
|
27
|
+
CapabilityOutput,
|
|
28
|
+
CatalogOutput,
|
|
29
|
+
DatasetColumn,
|
|
30
|
+
DefinitionOutput,
|
|
31
|
+
DesignOutput,
|
|
32
|
+
DesignRequest,
|
|
33
|
+
EventOutput,
|
|
34
|
+
EventsOutput,
|
|
35
|
+
MetricMetadata,
|
|
36
|
+
PlanReceipt,
|
|
37
|
+
PropertyMetadata,
|
|
38
|
+
Response,
|
|
39
|
+
RowsOutput,
|
|
40
|
+
RunOutput,
|
|
41
|
+
SchemaOutput,
|
|
42
|
+
SourceMetadata,
|
|
43
|
+
ValidationOutput,
|
|
44
|
+
refusal_response,
|
|
45
|
+
)
|
|
46
|
+
from terrine.data import _OUTPUT_BYTES_EXCEEDED, _ROWS_EXCEEDED, Budget, Preview
|
|
47
|
+
from terrine.dataset.manifest import DatasetManifest
|
|
48
|
+
from terrine.dataset.reference import Dataset, dataset_from_run
|
|
49
|
+
from terrine.errors import RefusalSpec, TerrineError
|
|
50
|
+
from terrine.model import Model
|
|
51
|
+
from terrine.runtime.api import Executor, cancel_run, get_run, read_run_events, run_serialized_plan
|
|
52
|
+
from terrine.runtime.changes import ChangeSet
|
|
53
|
+
from terrine.runtime.planning import OwnerPlanner, PlanInvocation, PlanRequest
|
|
54
|
+
from terrine.runtime.saved_plan import (
|
|
55
|
+
DestinationRef,
|
|
56
|
+
ModelReference,
|
|
57
|
+
SavedPlan,
|
|
58
|
+
_bind_saved_plan,
|
|
59
|
+
)
|
|
60
|
+
from terrine.runtime.state import RuntimeStore
|
|
61
|
+
from terrine.spec.anchors import EventAnchor, FixedDate
|
|
62
|
+
from terrine.spec.dataset import INVALID_DEFINITION, DatasetDefinition
|
|
63
|
+
from terrine.spec.design import Design
|
|
64
|
+
from terrine.spec.io import from_json, from_yaml, to_json
|
|
65
|
+
from terrine.spec.validate import ensure_runtime_shape_supported, validate_design
|
|
66
|
+
|
|
67
|
+
__all__ = ["DatasetTools"]
|
|
68
|
+
|
|
69
|
+
_INVALID = RefusalSpec("terrine.agent.invalid_request", "Use a valid typed Design payload.")
|
|
70
|
+
_STORAGE = RefusalSpec("terrine.agent.storage_denied", "Named Design storage is unavailable.")
|
|
71
|
+
_MODEL = RefusalSpec("terrine.agent.model_mismatch", "Owner model configuration has changed.")
|
|
72
|
+
_RUN = RefusalSpec(
|
|
73
|
+
"terrine.agent.run_unavailable", "Run has no matching owner-prepared invocation."
|
|
74
|
+
)
|
|
75
|
+
_OWNER = RefusalSpec(
|
|
76
|
+
"terrine.agent.owner_configuration", "Owner execution configuration is unavailable."
|
|
77
|
+
)
|
|
78
|
+
_RECEIPT = RefusalSpec(
|
|
79
|
+
"terrine.agent.receipt_unavailable", "Use an unchanged owner-prepared receipt."
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _diff(before: Any, after: Any, path: str = "") -> tuple[str, ...]:
|
|
84
|
+
if isinstance(before, dict) and isinstance(after, dict):
|
|
85
|
+
paths = []
|
|
86
|
+
for key in sorted(before.keys() | after.keys()):
|
|
87
|
+
escaped = key.replace("~", "~0").replace("/", "~1")
|
|
88
|
+
child = f"{path}/{escaped}"
|
|
89
|
+
if key not in before or key not in after:
|
|
90
|
+
paths.append(child)
|
|
91
|
+
else:
|
|
92
|
+
paths.extend(_diff(before[key], after[key], child))
|
|
93
|
+
return tuple(paths)
|
|
94
|
+
return () if before == after else (path,)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class DatasetTools:
|
|
98
|
+
"""An owner-bound session with a shared request cap across all operations."""
|
|
99
|
+
|
|
100
|
+
def __init__(
|
|
101
|
+
self, model: Model, policy: AgentPolicy, *, namespace: str, executor: Executor | None = None
|
|
102
|
+
) -> None:
|
|
103
|
+
if not isinstance(model, Model) or not isinstance(policy, AgentPolicy):
|
|
104
|
+
raise TypeError("model and policy must be owner-created typed values")
|
|
105
|
+
if not isinstance(namespace, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", namespace):
|
|
106
|
+
raise ValueError("namespace must be a simple identifier")
|
|
107
|
+
if executor is not None and executor is not policy.executor:
|
|
108
|
+
raise TypeError("executor must be the owner policy executor")
|
|
109
|
+
self.model = model
|
|
110
|
+
self.policy = policy
|
|
111
|
+
self.namespace = namespace
|
|
112
|
+
self._requests = 0
|
|
113
|
+
self._lock = threading.RLock()
|
|
114
|
+
|
|
115
|
+
def _call[T](self, permission: Permission, operation: Callable[[], T]) -> Response[T]:
|
|
116
|
+
try:
|
|
117
|
+
with self._lock:
|
|
118
|
+
if self._requests >= self.policy.max_requests:
|
|
119
|
+
raise _LIMIT.error()
|
|
120
|
+
self._requests += 1
|
|
121
|
+
self.policy.authorize(permission)
|
|
122
|
+
reference = self.policy.model_reference
|
|
123
|
+
if (
|
|
124
|
+
reference is not None
|
|
125
|
+
and ModelReference.from_model(self.model, owner_key=reference.owner_key)
|
|
126
|
+
!= reference
|
|
127
|
+
):
|
|
128
|
+
raise _MODEL.error()
|
|
129
|
+
catalog = (
|
|
130
|
+
reference.catalog_hash
|
|
131
|
+
if reference
|
|
132
|
+
else ModelReference.from_model(self.model, owner_key="inspection").catalog_hash
|
|
133
|
+
)
|
|
134
|
+
self.policy.authorize(permission, catalog=catalog)
|
|
135
|
+
return Response[T](status="ok", output=operation())
|
|
136
|
+
except TerrineError as error:
|
|
137
|
+
return refusal_response(error)
|
|
138
|
+
|
|
139
|
+
def _parse(self, serialized: str) -> Design:
|
|
140
|
+
try:
|
|
141
|
+
DesignRequest(serialized=serialized)
|
|
142
|
+
return (
|
|
143
|
+
from_json(serialized)
|
|
144
|
+
if serialized.lstrip().startswith("{")
|
|
145
|
+
else from_yaml(serialized)
|
|
146
|
+
)
|
|
147
|
+
except TerrineError:
|
|
148
|
+
raise
|
|
149
|
+
except (TypeError, ValueError) as error:
|
|
150
|
+
raise _INVALID.error(
|
|
151
|
+
route="Use get_design_schema and design_example to construct a Design."
|
|
152
|
+
) from error
|
|
153
|
+
|
|
154
|
+
def _validated(self, design: Design) -> ValidationOutput:
|
|
155
|
+
report = validate_design(design, self.model)
|
|
156
|
+
self.policy.authorize("validate", sources=report.required_sources)
|
|
157
|
+
return ValidationOutput(
|
|
158
|
+
spec_hash=design.spec_hash,
|
|
159
|
+
required_sources=report.required_sources,
|
|
160
|
+
runtime_obligations=report.runtime_obligations,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def _design_output(self, serialized: str, base: str | None = None) -> DesignOutput:
|
|
164
|
+
design = self._parse(serialized)
|
|
165
|
+
valid = self._validated(design)
|
|
166
|
+
canonical = to_json(design)
|
|
167
|
+
paths = (
|
|
168
|
+
()
|
|
169
|
+
if base is None
|
|
170
|
+
else _diff(json.loads(to_json(self._parse(base))), json.loads(canonical))
|
|
171
|
+
)
|
|
172
|
+
return DesignOutput(**valid.model_dump(), serialized=canonical, changed_paths=paths)
|
|
173
|
+
|
|
174
|
+
def inspect_catalog(self) -> Response[CatalogOutput]:
|
|
175
|
+
def inspect():
|
|
176
|
+
allowed = self.policy.allowed_sources
|
|
177
|
+
return CatalogOutput(
|
|
178
|
+
sources=tuple(
|
|
179
|
+
SourceMetadata(
|
|
180
|
+
name=name,
|
|
181
|
+
entities=tuple(sorted(source.entities)),
|
|
182
|
+
facts=tuple(sorted(source.facts)),
|
|
183
|
+
properties=tuple(sorted(source.properties)),
|
|
184
|
+
)
|
|
185
|
+
for name, source in sorted(self.model.sources.items())
|
|
186
|
+
if name in allowed
|
|
187
|
+
),
|
|
188
|
+
metrics=tuple(
|
|
189
|
+
MetricMetadata(
|
|
190
|
+
name=name,
|
|
191
|
+
entity=metric.entity,
|
|
192
|
+
kind=metric.kind,
|
|
193
|
+
sources=metric.source_names,
|
|
194
|
+
)
|
|
195
|
+
for name, metric in sorted(self.model.metrics.items())
|
|
196
|
+
if set(metric.source_names) <= allowed
|
|
197
|
+
),
|
|
198
|
+
properties=tuple(
|
|
199
|
+
PropertyMetadata(
|
|
200
|
+
name=name, source=prop.source, dtype=prop.dtype, as_of=prop.as_of
|
|
201
|
+
)
|
|
202
|
+
for name, prop in sorted(self.model.catalog.properties.items())
|
|
203
|
+
if prop.source in allowed
|
|
204
|
+
),
|
|
205
|
+
units=tuple(
|
|
206
|
+
sorted(
|
|
207
|
+
{
|
|
208
|
+
unit
|
|
209
|
+
for name, source in self.model.sources.items()
|
|
210
|
+
if name in allowed
|
|
211
|
+
for unit in source.entities
|
|
212
|
+
}
|
|
213
|
+
)
|
|
214
|
+
),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
return self._call("inspect", inspect)
|
|
218
|
+
|
|
219
|
+
def list_capabilities(self) -> Response[CapabilityOutput]:
|
|
220
|
+
def capabilities():
|
|
221
|
+
shapes = []
|
|
222
|
+
for shape in ("cross_section", "panel", "sequence", "trajectory"):
|
|
223
|
+
try:
|
|
224
|
+
ensure_runtime_shape_supported(shape)
|
|
225
|
+
except TerrineError:
|
|
226
|
+
continue
|
|
227
|
+
shapes.append(shape)
|
|
228
|
+
from terrine.spec.state_panel import StateObservations, StatePanel
|
|
229
|
+
|
|
230
|
+
ensure_runtime_shape_supported(
|
|
231
|
+
StatePanel(StateObservations("state", ("a", "b")), max_periods=2)
|
|
232
|
+
)
|
|
233
|
+
shapes.append("StatePanel")
|
|
234
|
+
return CapabilityOutput(
|
|
235
|
+
operations=(
|
|
236
|
+
"inspect_catalog",
|
|
237
|
+
"list_capabilities",
|
|
238
|
+
"get_design_schema",
|
|
239
|
+
"get_dataset_definition_schema",
|
|
240
|
+
"design_example",
|
|
241
|
+
"propose_design",
|
|
242
|
+
"validate_design",
|
|
243
|
+
"propose_definition",
|
|
244
|
+
"validate_definition",
|
|
245
|
+
"save_design",
|
|
246
|
+
"load_design",
|
|
247
|
+
"preview_dataset",
|
|
248
|
+
"prepare_dataset",
|
|
249
|
+
"execute_plan",
|
|
250
|
+
"stage",
|
|
251
|
+
"prepare_refresh",
|
|
252
|
+
"refresh",
|
|
253
|
+
"export_parquet",
|
|
254
|
+
"preview_rows",
|
|
255
|
+
"get_run_status",
|
|
256
|
+
"get_run_events",
|
|
257
|
+
"request_cancel",
|
|
258
|
+
),
|
|
259
|
+
runtime_shapes=tuple(shapes),
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
return self._call("inspect", capabilities)
|
|
263
|
+
|
|
264
|
+
def get_design_schema(self) -> Response[SchemaOutput]:
|
|
265
|
+
return self._call("inspect", lambda: SchemaOutput(document=design_schema()))
|
|
266
|
+
|
|
267
|
+
def design_example(self) -> Response[DesignOutput]:
|
|
268
|
+
def example():
|
|
269
|
+
sources = [
|
|
270
|
+
(name, source)
|
|
271
|
+
for name, source in sorted(self.model.sources.items())
|
|
272
|
+
if name in self.policy.allowed_sources and source.entities
|
|
273
|
+
]
|
|
274
|
+
for name, source in sources:
|
|
275
|
+
for unit in sorted(source.entities):
|
|
276
|
+
if unit in self.model.units:
|
|
277
|
+
coverage = self.model.readiness.sources.get(name)
|
|
278
|
+
anchor = coverage.start.date() if coverage else date(2000, 1, 1)
|
|
279
|
+
return self._design_output(to_json(Design(unit, FixedDate(anchor))))
|
|
280
|
+
for name, source in sources:
|
|
281
|
+
if source.facts and name in self.model.readiness.sources:
|
|
282
|
+
design = Design(
|
|
283
|
+
sorted(source.entities)[0], EventAnchor(sorted(source.facts)[0])
|
|
284
|
+
)
|
|
285
|
+
return self._design_output(to_json(design))
|
|
286
|
+
raise _INVALID.error(
|
|
287
|
+
route="Configure a unit roster or allow an event fact with declared coverage."
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
return self._call("validate", example)
|
|
291
|
+
|
|
292
|
+
def propose_design(self, serialized: str, *, base: str | None = None) -> Response[DesignOutput]:
|
|
293
|
+
return self._call("validate", lambda: self._design_output(serialized, base))
|
|
294
|
+
|
|
295
|
+
def validate_design(self, serialized: str) -> Response[ValidationOutput]:
|
|
296
|
+
return self._call("validate", lambda: self._validated(self._parse(serialized)))
|
|
297
|
+
|
|
298
|
+
def _storage(self, name: str) -> tuple[int, str]:
|
|
299
|
+
if (
|
|
300
|
+
not isinstance(name, str)
|
|
301
|
+
or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_-]*", name)
|
|
302
|
+
or self.policy.design_root is None
|
|
303
|
+
):
|
|
304
|
+
raise _STORAGE.error(route="Use a simple name under the owner-configured design root.")
|
|
305
|
+
root = self.policy.design_root
|
|
306
|
+
try:
|
|
307
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
308
|
+
descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
309
|
+
return descriptor, f"{name}.json"
|
|
310
|
+
except OSError as error:
|
|
311
|
+
raise _STORAGE.error() from error
|
|
312
|
+
|
|
313
|
+
def save_design(self, name: str, serialized: str) -> Response[DesignOutput | DefinitionOutput]:
|
|
314
|
+
def save():
|
|
315
|
+
output = self._definition_output(serialized)
|
|
316
|
+
root, filename = self._storage(name)
|
|
317
|
+
temporary = f".design-{secrets.token_hex(12)}"
|
|
318
|
+
try:
|
|
319
|
+
try:
|
|
320
|
+
existing = os.open(filename, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root)
|
|
321
|
+
except FileNotFoundError:
|
|
322
|
+
pass
|
|
323
|
+
else:
|
|
324
|
+
os.close(existing)
|
|
325
|
+
descriptor = os.open(
|
|
326
|
+
temporary,
|
|
327
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
328
|
+
0o600,
|
|
329
|
+
dir_fd=root,
|
|
330
|
+
)
|
|
331
|
+
with os.fdopen(descriptor, "w") as stream:
|
|
332
|
+
stream.write(output.serialized)
|
|
333
|
+
stream.flush()
|
|
334
|
+
os.fsync(stream.fileno())
|
|
335
|
+
os.replace(temporary, filename, src_dir_fd=root, dst_dir_fd=root)
|
|
336
|
+
except OSError as error:
|
|
337
|
+
raise _STORAGE.error() from error
|
|
338
|
+
finally:
|
|
339
|
+
try:
|
|
340
|
+
os.unlink(temporary, dir_fd=root)
|
|
341
|
+
except FileNotFoundError:
|
|
342
|
+
pass
|
|
343
|
+
os.close(root)
|
|
344
|
+
return output
|
|
345
|
+
|
|
346
|
+
return self._call("validate", save)
|
|
347
|
+
|
|
348
|
+
def load_design(self, name: str) -> Response[DesignOutput | DefinitionOutput]:
|
|
349
|
+
def load():
|
|
350
|
+
root, filename = self._storage(name)
|
|
351
|
+
try:
|
|
352
|
+
descriptor = os.open(filename, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=root)
|
|
353
|
+
with os.fdopen(descriptor) as stream:
|
|
354
|
+
serialized = stream.read()
|
|
355
|
+
except OSError as error:
|
|
356
|
+
raise _STORAGE.error() from error
|
|
357
|
+
finally:
|
|
358
|
+
os.close(root)
|
|
359
|
+
return self._definition_output(serialized)
|
|
360
|
+
|
|
361
|
+
return self._call("validate", load)
|
|
362
|
+
|
|
363
|
+
def _authorize_run(self, run_id: str, permission: Permission) -> None:
|
|
364
|
+
if not isinstance(run_id, str) or not run_id:
|
|
365
|
+
raise _INVALID.error()
|
|
366
|
+
con = self.model.con.con
|
|
367
|
+
store = RuntimeStore(con, namespace=self.namespace)
|
|
368
|
+
tables = {
|
|
369
|
+
item[0]
|
|
370
|
+
for item in con.execute(
|
|
371
|
+
"SELECT table_name FROM information_schema.tables "
|
|
372
|
+
"WHERE lower(table_schema)=lower(?)",
|
|
373
|
+
[self.namespace],
|
|
374
|
+
).fetchall()
|
|
375
|
+
}
|
|
376
|
+
if not {"runtime_runs", "runtime_plan_invocations"} <= tables:
|
|
377
|
+
raise _RUN.error()
|
|
378
|
+
columns = {
|
|
379
|
+
item[0]
|
|
380
|
+
for item in con.execute(
|
|
381
|
+
"SELECT column_name FROM information_schema.columns "
|
|
382
|
+
"WHERE lower(table_schema)=lower(?) AND table_name='runtime_runs'",
|
|
383
|
+
[self.namespace],
|
|
384
|
+
).fetchall()
|
|
385
|
+
}
|
|
386
|
+
if "destination_json" not in columns:
|
|
387
|
+
raise _RUN.error()
|
|
388
|
+
row = con.execute(
|
|
389
|
+
f"SELECT destination_json, fingerprint FROM {store._schema}.runtime_runs "
|
|
390
|
+
"WHERE run_id=?",
|
|
391
|
+
[run_id],
|
|
392
|
+
).fetchone()
|
|
393
|
+
if row is None or row[0] is None:
|
|
394
|
+
raise _RUN.error()
|
|
395
|
+
destination = DestinationRef(**json.loads(row[0]))
|
|
396
|
+
self.policy.authorize(permission, destination=destination)
|
|
397
|
+
invocations = con.execute(
|
|
398
|
+
f"SELECT invocation_json FROM {store._schema}.runtime_plan_invocations"
|
|
399
|
+
).fetchall()
|
|
400
|
+
for record in invocations:
|
|
401
|
+
invocation = PlanInvocation.from_dict(json.loads(record[0]))
|
|
402
|
+
if invocation.destination != destination or not invocation.saved_plan_json:
|
|
403
|
+
continue
|
|
404
|
+
definition = SavedPlan.from_json(invocation.saved_plan_json)
|
|
405
|
+
if (
|
|
406
|
+
definition.model != self.policy.model_reference
|
|
407
|
+
or definition.namespace != self.namespace
|
|
408
|
+
):
|
|
409
|
+
continue
|
|
410
|
+
if (
|
|
411
|
+
_bind_saved_plan(definition, namespace=self.namespace).definition.fingerprint
|
|
412
|
+
!= row[1]
|
|
413
|
+
):
|
|
414
|
+
continue
|
|
415
|
+
report = validate_design(from_json(definition.design_json), self.model)
|
|
416
|
+
self.policy.authorize(permission, sources=report.required_sources)
|
|
417
|
+
return
|
|
418
|
+
raise _RUN.error(route="Use a run from an owner-prepared invocation for this Model.")
|
|
419
|
+
|
|
420
|
+
def get_run_status(self, run_id: str) -> Response[RunOutput]:
|
|
421
|
+
def status():
|
|
422
|
+
self._authorize_run(run_id, "read")
|
|
423
|
+
result = get_run(run_id, con=self.model.con.con, namespace=self.namespace)
|
|
424
|
+
return RunOutput(
|
|
425
|
+
run_id=result.run_id,
|
|
426
|
+
durable_state=result.durable_state,
|
|
427
|
+
phase=result.phase,
|
|
428
|
+
error_code=result.error_code,
|
|
429
|
+
pinned_result_ref=result.pinned_result_ref,
|
|
430
|
+
latest_event_cursor=result.latest_event_cursor,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
return self._call("read", status)
|
|
434
|
+
|
|
435
|
+
def get_run_events(
|
|
436
|
+
self, run_id: str, *, after: str | None = None, limit: int = 100
|
|
437
|
+
) -> Response[EventsOutput]:
|
|
438
|
+
def events():
|
|
439
|
+
if (
|
|
440
|
+
isinstance(limit, bool)
|
|
441
|
+
or not isinstance(limit, int)
|
|
442
|
+
or not 0 < limit <= self.policy.batch_cap
|
|
443
|
+
):
|
|
444
|
+
raise _INVALID.error(
|
|
445
|
+
route="Choose a positive event limit no larger than the owner batch cap."
|
|
446
|
+
)
|
|
447
|
+
self._authorize_run(run_id, "read")
|
|
448
|
+
page = read_run_events(
|
|
449
|
+
run_id, con=self.model.con.con, namespace=self.namespace, after=after, limit=limit
|
|
450
|
+
)
|
|
451
|
+
return EventsOutput(
|
|
452
|
+
events=tuple(
|
|
453
|
+
EventOutput(
|
|
454
|
+
sequence=event.sequence,
|
|
455
|
+
kind=event.kind,
|
|
456
|
+
state=event.payload.get("state"),
|
|
457
|
+
phase=event.payload.get("phase"),
|
|
458
|
+
)
|
|
459
|
+
for event in page.events
|
|
460
|
+
),
|
|
461
|
+
next_cursor=page.next_cursor,
|
|
462
|
+
has_more=page.has_more,
|
|
463
|
+
cursor_gap=page.cursor_gap,
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
return self._call("read", events)
|
|
467
|
+
|
|
468
|
+
def request_cancel(self, run_id: str) -> Response[CancelOutput]:
|
|
469
|
+
def cancel():
|
|
470
|
+
self._authorize_run(run_id, "cancel")
|
|
471
|
+
receipt = cancel_run(run_id, con=self.model.con.con, namespace=self.namespace)
|
|
472
|
+
return CancelOutput(
|
|
473
|
+
run_id=run_id,
|
|
474
|
+
disposition=receipt.disposition,
|
|
475
|
+
durable_state=receipt.status.durable_state,
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
return self._call("cancel", cancel)
|
|
479
|
+
|
|
480
|
+
def get_dataset_definition_schema(self) -> Response[SchemaOutput]:
|
|
481
|
+
return self._call("inspect", lambda: SchemaOutput(document=definition_schema()))
|
|
482
|
+
|
|
483
|
+
def _dataset_definition_output(
|
|
484
|
+
self, serialized: str, base: str | None = None
|
|
485
|
+
) -> DefinitionOutput:
|
|
486
|
+
definition = parse_definition(serialized, self.model)
|
|
487
|
+
valid = self._validated(definition.base_design)
|
|
488
|
+
canonical = definition.canonical_json().decode()
|
|
489
|
+
paths = (
|
|
490
|
+
()
|
|
491
|
+
if base is None
|
|
492
|
+
else _diff(
|
|
493
|
+
json.loads(parse_definition(base, self.model).canonical_json()),
|
|
494
|
+
json.loads(canonical),
|
|
495
|
+
)
|
|
496
|
+
)
|
|
497
|
+
return DefinitionOutput(
|
|
498
|
+
**valid.model_dump(),
|
|
499
|
+
serialized=canonical,
|
|
500
|
+
changed_paths=paths,
|
|
501
|
+
definition_digest=definition.digest,
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
def propose_definition(
|
|
505
|
+
self, serialized: str, *, base: str | None = None
|
|
506
|
+
) -> Response[DefinitionOutput]:
|
|
507
|
+
return self._call("validate", lambda: self._dataset_definition_output(serialized, base))
|
|
508
|
+
|
|
509
|
+
def validate_definition(self, serialized: str) -> Response[DefinitionOutput]:
|
|
510
|
+
def validate():
|
|
511
|
+
output = self._dataset_definition_output(serialized)
|
|
512
|
+
try:
|
|
513
|
+
parse_definition(serialized, self.model).build(self.model)
|
|
514
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
515
|
+
if isinstance(error, TerrineError):
|
|
516
|
+
raise
|
|
517
|
+
raise _INVALID.error(
|
|
518
|
+
route="Use the recorded training or scoring view fields."
|
|
519
|
+
) from error
|
|
520
|
+
return output
|
|
521
|
+
|
|
522
|
+
return self._call("validate", validate)
|
|
523
|
+
|
|
524
|
+
def _definition_output(self, serialized: str) -> DesignOutput | DefinitionOutput:
|
|
525
|
+
try:
|
|
526
|
+
payload = json.loads(serialized)
|
|
527
|
+
except (TypeError, ValueError):
|
|
528
|
+
return self._design_output(serialized)
|
|
529
|
+
if isinstance(payload, dict) and "base_design" in payload:
|
|
530
|
+
return self._dataset_definition_output(serialized)
|
|
531
|
+
return self._design_output(serialized)
|
|
532
|
+
|
|
533
|
+
def _dataset_definition(self, definition: DatasetDefinition | str) -> DatasetDefinition:
|
|
534
|
+
if not isinstance(definition, (DatasetDefinition, str)):
|
|
535
|
+
raise _INVALID.error(route="Use the discovered DatasetDefinition schema.")
|
|
536
|
+
return parse_definition(
|
|
537
|
+
definition.canonical_json().decode()
|
|
538
|
+
if isinstance(definition, DatasetDefinition)
|
|
539
|
+
else definition,
|
|
540
|
+
self.model,
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
def _resolved_definition_design(self, definition: DatasetDefinition) -> Design:
|
|
544
|
+
try:
|
|
545
|
+
return definition.build(self.model)._design
|
|
546
|
+
except TerrineError:
|
|
547
|
+
raise
|
|
548
|
+
except (KeyError, TypeError, ValueError, OverflowError) as error:
|
|
549
|
+
raise INVALID_DEFINITION.error(reason="dataset view cannot be constructed") from error
|
|
550
|
+
|
|
551
|
+
def preview_dataset(
|
|
552
|
+
self,
|
|
553
|
+
definition: DatasetDefinition | str,
|
|
554
|
+
*,
|
|
555
|
+
budget: Budget | None = None,
|
|
556
|
+
bounded_count: int = 0,
|
|
557
|
+
) -> Response[Preview]:
|
|
558
|
+
def preview():
|
|
559
|
+
from terrine.compile import cost
|
|
560
|
+
|
|
561
|
+
if (
|
|
562
|
+
isinstance(bounded_count, bool)
|
|
563
|
+
or not isinstance(bounded_count, int)
|
|
564
|
+
or not 0 <= bounded_count <= self.policy.max_bounded_count
|
|
565
|
+
):
|
|
566
|
+
raise _INVALID.error(route="Use a bounded count within the owner cap.")
|
|
567
|
+
parsed = self._dataset_definition(definition)
|
|
568
|
+
design = self._resolved_definition_design(parsed)
|
|
569
|
+
valid = self._validated(design)
|
|
570
|
+
if bounded_count:
|
|
571
|
+
self.policy.authorize("preview", sources=valid.required_sources)
|
|
572
|
+
effective = effective_budget(self.policy.budget, budget)
|
|
573
|
+
cost._scan_policy(effective)
|
|
574
|
+
result = cost.preview(design, self.model, effective, bounded_count=bounded_count)
|
|
575
|
+
return replace(
|
|
576
|
+
result,
|
|
577
|
+
coverage={
|
|
578
|
+
name: value
|
|
579
|
+
for name, value in result.coverage.items()
|
|
580
|
+
if name in valid.required_sources
|
|
581
|
+
},
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
return self._call("validate", preview)
|
|
585
|
+
|
|
586
|
+
def _owner_planner(self, destination: DestinationRef) -> OwnerPlanner:
|
|
587
|
+
authority = self.policy.snapshot_authority
|
|
588
|
+
if authority is None or self.policy.resolver is None or self.policy.model_reference is None:
|
|
589
|
+
raise _OWNER.error(route="Configure an owner resolver and retained snapshot authority.")
|
|
590
|
+
from terrine.runtime.owner import RuntimeOwner
|
|
591
|
+
|
|
592
|
+
try:
|
|
593
|
+
connected_owner = RuntimeOwner.from_connection(self.model.con)
|
|
594
|
+
except (TypeError, ValueError) as error:
|
|
595
|
+
raise _OWNER.error(
|
|
596
|
+
route="Use an owner-configured file-backed DuckDB connection."
|
|
597
|
+
) from error
|
|
598
|
+
if connected_owner.reference != authority.owner.reference:
|
|
599
|
+
raise _OWNER.error(
|
|
600
|
+
route="Align the owner's configured snapshot root with the connected dataset owner."
|
|
601
|
+
)
|
|
602
|
+
if (
|
|
603
|
+
destination.schema != self.namespace
|
|
604
|
+
or Path(destination.database).resolve() != authority.owner.state_path.resolve()
|
|
605
|
+
):
|
|
606
|
+
raise _SCOPE.error(route="Use this owner's database and runtime namespace.")
|
|
607
|
+
return OwnerPlanner(resolver=self.policy.resolver, snapshot_authority=authority)
|
|
608
|
+
|
|
609
|
+
@staticmethod
|
|
610
|
+
def _receipt(invocation: PlanInvocation) -> PlanReceipt:
|
|
611
|
+
saved = SavedPlan.from_json(invocation.saved_plan_json)
|
|
612
|
+
identity = json.dumps(invocation.to_dict(), sort_keys=True, separators=(",", ":"))
|
|
613
|
+
return PlanReceipt(
|
|
614
|
+
receipt_id=hashlib.sha256(identity.encode()).hexdigest(),
|
|
615
|
+
design_hash=saved.design_hash,
|
|
616
|
+
plan_hash=saved.plan_hash,
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
def _prepare(
|
|
620
|
+
self,
|
|
621
|
+
definition: DatasetDefinition,
|
|
622
|
+
*,
|
|
623
|
+
destination: DestinationRef,
|
|
624
|
+
invocation_id: str,
|
|
625
|
+
budget: Budget | None,
|
|
626
|
+
changes: ChangeSet | None = None,
|
|
627
|
+
expected_base_manifest: str | None = None,
|
|
628
|
+
bind_base_manifest: bool = False,
|
|
629
|
+
) -> PlanReceipt:
|
|
630
|
+
valid = self._validated(definition.base_design)
|
|
631
|
+
self.policy.authorize("prepare", sources=valid.required_sources, destination=destination)
|
|
632
|
+
planner = self._owner_planner(destination)
|
|
633
|
+
reference = self.policy.model_reference
|
|
634
|
+
if reference is None:
|
|
635
|
+
raise _OWNER.error()
|
|
636
|
+
try:
|
|
637
|
+
request = PlanRequest(
|
|
638
|
+
invocation_id,
|
|
639
|
+
reference,
|
|
640
|
+
definition.base_design,
|
|
641
|
+
destination,
|
|
642
|
+
changes=changes,
|
|
643
|
+
budget=effective_budget(self.policy.budget, budget),
|
|
644
|
+
dataset_definition=definition,
|
|
645
|
+
expected_base_manifest=expected_base_manifest,
|
|
646
|
+
bind_base_manifest=bind_base_manifest,
|
|
647
|
+
)
|
|
648
|
+
except (TypeError, ValueError) as error:
|
|
649
|
+
raise _INVALID.error(
|
|
650
|
+
route="Use a safe invocation ID and typed change evidence."
|
|
651
|
+
) from error
|
|
652
|
+
return self._receipt(
|
|
653
|
+
planner.plan(request, con=self.model.con.con, namespace=self.namespace)
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
def prepare_dataset(
|
|
657
|
+
self,
|
|
658
|
+
definition: DatasetDefinition | str,
|
|
659
|
+
*,
|
|
660
|
+
destination: DestinationRef,
|
|
661
|
+
invocation_id: str,
|
|
662
|
+
budget: Budget | None = None,
|
|
663
|
+
) -> Response[PlanReceipt]:
|
|
664
|
+
return self._call(
|
|
665
|
+
"prepare",
|
|
666
|
+
lambda: self._prepare(
|
|
667
|
+
self._dataset_definition(definition),
|
|
668
|
+
destination=destination,
|
|
669
|
+
invocation_id=invocation_id,
|
|
670
|
+
budget=budget,
|
|
671
|
+
),
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
def _resolve_receipt(
|
|
675
|
+
self, receipt: PlanReceipt, permission: Permission
|
|
676
|
+
) -> tuple[PlanInvocation, SavedPlan]:
|
|
677
|
+
if not isinstance(receipt, PlanReceipt):
|
|
678
|
+
raise _RECEIPT.error()
|
|
679
|
+
con = self.model.con.con
|
|
680
|
+
exists = con.execute(
|
|
681
|
+
"SELECT 1 FROM information_schema.tables WHERE lower(table_schema)=lower(?) "
|
|
682
|
+
"AND table_name='runtime_plan_invocations'",
|
|
683
|
+
[self.namespace],
|
|
684
|
+
).fetchone()
|
|
685
|
+
if exists is None:
|
|
686
|
+
raise _RECEIPT.error()
|
|
687
|
+
for (payload,) in con.execute(
|
|
688
|
+
f'SELECT invocation_json FROM "{self.namespace}".runtime_plan_invocations'
|
|
689
|
+
).fetchall():
|
|
690
|
+
document = json.loads(payload)
|
|
691
|
+
identity = json.dumps(document, sort_keys=True, separators=(",", ":"))
|
|
692
|
+
if hashlib.sha256(identity.encode()).hexdigest() != receipt.receipt_id:
|
|
693
|
+
continue
|
|
694
|
+
invocation = PlanInvocation.from_dict(document)
|
|
695
|
+
if self._receipt(invocation) != receipt:
|
|
696
|
+
continue
|
|
697
|
+
saved = SavedPlan.from_json(invocation.saved_plan_json)
|
|
698
|
+
if saved.namespace != self.namespace or saved.model != self.policy.model_reference:
|
|
699
|
+
raise _SCOPE.error()
|
|
700
|
+
valid = validate_design(from_json(saved.design_json), self.model)
|
|
701
|
+
self.policy.authorize(
|
|
702
|
+
permission, sources=valid.required_sources, destination=saved.destination
|
|
703
|
+
)
|
|
704
|
+
self._owner_planner(saved.destination)
|
|
705
|
+
planned_budget = _bind_saved_plan(saved, namespace=self.namespace).budget
|
|
706
|
+
if effective_budget(self.policy.budget, planned_budget) != planned_budget:
|
|
707
|
+
raise _SCOPE.error(route="Prepare a new invocation under the current owner budget.")
|
|
708
|
+
return invocation, saved
|
|
709
|
+
raise _RECEIPT.error()
|
|
710
|
+
|
|
711
|
+
def _execute(self, receipt: PlanReceipt, *, request_key: str):
|
|
712
|
+
if not isinstance(request_key, str) or not request_key.strip() or len(request_key) > 256:
|
|
713
|
+
raise _INVALID.error(route="Use a nonempty request key of at most 256 characters.")
|
|
714
|
+
invocation, saved = self._resolve_receipt(receipt, "write")
|
|
715
|
+
deadline = monotonic() + self.policy.max_run_seconds
|
|
716
|
+
return run_serialized_plan(
|
|
717
|
+
invocation.saved_plan_json,
|
|
718
|
+
con=self.model.con.con,
|
|
719
|
+
namespace=self.namespace,
|
|
720
|
+
request_key=request_key,
|
|
721
|
+
destination=saved.destination,
|
|
722
|
+
executor=self.policy.executor,
|
|
723
|
+
resolver=self.policy.resolver,
|
|
724
|
+
snapshot_authority=self.policy.snapshot_authority,
|
|
725
|
+
max_active_runs=self.policy.max_concurrency,
|
|
726
|
+
cancelled=lambda: monotonic() >= deadline,
|
|
727
|
+
)
|
|
728
|
+
|
|
729
|
+
def execute_plan(self, receipt: PlanReceipt, *, request_key: str) -> Response[RunOutput]:
|
|
730
|
+
def execute():
|
|
731
|
+
result = self._execute(receipt, request_key=request_key)
|
|
732
|
+
status = get_run(result.run_id, con=self.model.con.con, namespace=self.namespace)
|
|
733
|
+
return RunOutput(
|
|
734
|
+
run_id=status.run_id,
|
|
735
|
+
durable_state=status.durable_state,
|
|
736
|
+
phase=status.phase,
|
|
737
|
+
error_code=status.error_code,
|
|
738
|
+
pinned_result_ref=status.pinned_result_ref,
|
|
739
|
+
latest_event_cursor=status.latest_event_cursor,
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
return self._call("write", execute)
|
|
743
|
+
|
|
744
|
+
@staticmethod
|
|
745
|
+
def _dataset_reference(dataset: Dataset) -> AgentDatasetReference:
|
|
746
|
+
manifest = dataset.manifest
|
|
747
|
+
return AgentDatasetReference(
|
|
748
|
+
dataset=manifest.generation.dataset_id,
|
|
749
|
+
generation=manifest.generation.generation_id,
|
|
750
|
+
definition_hash=manifest.design_hash,
|
|
751
|
+
model_hash=manifest.model_hash,
|
|
752
|
+
manifest_hash=hashlib.sha256(manifest.to_json()).hexdigest(),
|
|
753
|
+
definition_reference=manifest.definition.digest,
|
|
754
|
+
row_count=manifest.generation.row_count,
|
|
755
|
+
columns=tuple(
|
|
756
|
+
DatasetColumn(
|
|
757
|
+
name=column.name,
|
|
758
|
+
data_type=column.data_type,
|
|
759
|
+
role=column.role,
|
|
760
|
+
complete_column=column.metadata.get("complete_column"),
|
|
761
|
+
reason_column=column.metadata.get("reason_column"),
|
|
762
|
+
)
|
|
763
|
+
for column in manifest.columns
|
|
764
|
+
),
|
|
765
|
+
)
|
|
766
|
+
|
|
767
|
+
def _resolve_dataset(
|
|
768
|
+
self, reference: AgentDatasetReference, permission: Permission
|
|
769
|
+
) -> tuple[Dataset, SavedPlan]:
|
|
770
|
+
if not isinstance(reference, AgentDatasetReference):
|
|
771
|
+
raise _RECEIPT.error(route="Use the unchanged reference returned by stage or refresh.")
|
|
772
|
+
con = self.model.con.con
|
|
773
|
+
exists = con.execute(
|
|
774
|
+
"SELECT 1 FROM information_schema.tables WHERE lower(table_schema)=lower(?) "
|
|
775
|
+
"AND table_name='runtime_dataset_recipes'",
|
|
776
|
+
[self.namespace],
|
|
777
|
+
).fetchone()
|
|
778
|
+
if exists is None:
|
|
779
|
+
raise _RECEIPT.error()
|
|
780
|
+
record = con.execute(
|
|
781
|
+
f'SELECT recipe_json,manifest_json FROM "{self.namespace}".runtime_dataset_recipes '
|
|
782
|
+
"WHERE dataset=? AND generation=?",
|
|
783
|
+
[reference.dataset, reference.generation],
|
|
784
|
+
).fetchone()
|
|
785
|
+
if record is None:
|
|
786
|
+
raise _RECEIPT.error()
|
|
787
|
+
saved = SavedPlan.from_json(record[0])
|
|
788
|
+
if saved.namespace != self.namespace or saved.model != self.policy.model_reference:
|
|
789
|
+
raise _SCOPE.error()
|
|
790
|
+
valid = validate_design(from_json(saved.design_json), self.model)
|
|
791
|
+
self.policy.authorize(
|
|
792
|
+
permission, sources=valid.required_sources, destination=saved.destination
|
|
793
|
+
)
|
|
794
|
+
manifest = DatasetManifest.from_json(record[1])
|
|
795
|
+
if manifest.definition.digest != reference.definition_reference:
|
|
796
|
+
raise _RECEIPT.error()
|
|
797
|
+
dataset = Dataset.open_generation(
|
|
798
|
+
reference.dataset, reference.generation, con=self.model.con, namespace=self.namespace
|
|
799
|
+
)
|
|
800
|
+
if self._dataset_reference(dataset) != reference:
|
|
801
|
+
raise _RECEIPT.error()
|
|
802
|
+
return dataset, saved
|
|
803
|
+
|
|
804
|
+
def _stage(self, receipt: PlanReceipt, *, request_key: str) -> AgentDatasetReference:
|
|
805
|
+
_, saved = self._resolve_receipt(receipt, "write")
|
|
806
|
+
result = self._execute(receipt, request_key=request_key)
|
|
807
|
+
authority = self.policy.snapshot_authority
|
|
808
|
+
if result.state != "succeeded" or authority is None:
|
|
809
|
+
raise _RUN.error(route="Inspect the run state before requesting a committed dataset.")
|
|
810
|
+
return self._dataset_reference(
|
|
811
|
+
dataset_from_run(
|
|
812
|
+
saved,
|
|
813
|
+
result,
|
|
814
|
+
model=self.model,
|
|
815
|
+
namespace=self.namespace,
|
|
816
|
+
request_key=request_key,
|
|
817
|
+
snapshot_authority=authority,
|
|
818
|
+
)
|
|
819
|
+
)
|
|
820
|
+
|
|
821
|
+
def stage(self, receipt: PlanReceipt, *, request_key: str) -> Response[AgentDatasetReference]:
|
|
822
|
+
return self._call("write", lambda: self._stage(receipt, request_key=request_key))
|
|
823
|
+
|
|
824
|
+
def prepare_refresh(
|
|
825
|
+
self,
|
|
826
|
+
reference: AgentDatasetReference,
|
|
827
|
+
*,
|
|
828
|
+
invocation_id: str,
|
|
829
|
+
changes: ChangeSet | None = None,
|
|
830
|
+
budget: Budget | None = None,
|
|
831
|
+
) -> Response[PlanReceipt]:
|
|
832
|
+
def prepare():
|
|
833
|
+
dataset, saved = self._resolve_dataset(reference, "prepare")
|
|
834
|
+
return self._prepare(
|
|
835
|
+
dataset.manifest.definition,
|
|
836
|
+
destination=saved.destination,
|
|
837
|
+
invocation_id=invocation_id,
|
|
838
|
+
budget=budget,
|
|
839
|
+
changes=changes,
|
|
840
|
+
expected_base_manifest=reference.generation,
|
|
841
|
+
bind_base_manifest=True,
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
return self._call("prepare", prepare)
|
|
845
|
+
|
|
846
|
+
def refresh(
|
|
847
|
+
self, reference: AgentDatasetReference, receipt: PlanReceipt, *, request_key: str
|
|
848
|
+
) -> Response[AgentDatasetReference]:
|
|
849
|
+
def refresh():
|
|
850
|
+
self._resolve_dataset(reference, "write")
|
|
851
|
+
_, saved = self._resolve_receipt(receipt, "write")
|
|
852
|
+
if (
|
|
853
|
+
saved.expected_base_manifest != reference.generation
|
|
854
|
+
or saved.destination.dataset != reference.dataset
|
|
855
|
+
):
|
|
856
|
+
raise _RECEIPT.error(
|
|
857
|
+
route="Use a refresh receipt prepared from this exact generation."
|
|
858
|
+
)
|
|
859
|
+
return self._stage(receipt, request_key=request_key)
|
|
860
|
+
|
|
861
|
+
return self._call("write", refresh)
|
|
862
|
+
|
|
863
|
+
def export_parquet(
|
|
864
|
+
self,
|
|
865
|
+
reference: AgentDatasetReference,
|
|
866
|
+
*,
|
|
867
|
+
destination: str,
|
|
868
|
+
partition_by: tuple[str, ...] = (),
|
|
869
|
+
budget: Budget | None = None,
|
|
870
|
+
) -> Response[AgentExportManifest]:
|
|
871
|
+
def export():
|
|
872
|
+
from terrine.compile.cost import _scan_policy
|
|
873
|
+
|
|
874
|
+
if not isinstance(destination, str) or not destination:
|
|
875
|
+
raise _INVALID.error()
|
|
876
|
+
path = Path(destination).expanduser().resolve()
|
|
877
|
+
if path not in self.policy.allowed_exports:
|
|
878
|
+
raise _SCOPE.error(route="Use an exact owner-approved export destination.")
|
|
879
|
+
effective = effective_budget(self.policy.budget, budget)
|
|
880
|
+
_scan_policy(effective)
|
|
881
|
+
dataset, _ = self._resolve_dataset(reference, "export")
|
|
882
|
+
manifest = dataset.export_parquet(
|
|
883
|
+
destination=str(path), partition_by=partition_by, budget=effective
|
|
884
|
+
)
|
|
885
|
+
return AgentExportManifest(
|
|
886
|
+
dataset=manifest.dataset_id,
|
|
887
|
+
generation=manifest.generation_id,
|
|
888
|
+
row_count=manifest.row_count,
|
|
889
|
+
byte_count=manifest.byte_count,
|
|
890
|
+
shard_count=len(manifest.shards),
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
return self._call("export", export)
|
|
894
|
+
|
|
895
|
+
def preview_rows(
|
|
896
|
+
self, reference: AgentDatasetReference, *, limit: int, columns: tuple[str, ...]
|
|
897
|
+
) -> Response[RowsOutput]:
|
|
898
|
+
def preview():
|
|
899
|
+
from terrine.compile.cost import _scan_policy
|
|
900
|
+
|
|
901
|
+
if (
|
|
902
|
+
isinstance(limit, bool)
|
|
903
|
+
or not isinstance(limit, int)
|
|
904
|
+
or not 0 < limit <= self.policy.max_preview_rows
|
|
905
|
+
):
|
|
906
|
+
raise _INVALID.error(route="Use a positive row limit within the owner preview cap.")
|
|
907
|
+
if (
|
|
908
|
+
not isinstance(columns, tuple)
|
|
909
|
+
or not columns
|
|
910
|
+
or any(not isinstance(name, str) for name in columns)
|
|
911
|
+
or len(set(columns)) != len(columns)
|
|
912
|
+
or not set(columns) <= self.policy.preview_columns
|
|
913
|
+
):
|
|
914
|
+
raise _SCOPE.error(route="Request only unique owner-approved preview columns.")
|
|
915
|
+
budget = self.policy.budget
|
|
916
|
+
_scan_policy(budget)
|
|
917
|
+
dataset, _ = self._resolve_dataset(reference, "preview")
|
|
918
|
+
if not set(columns) <= {column.name for column in dataset.manifest.columns}:
|
|
919
|
+
raise _SCOPE.error()
|
|
920
|
+
if (
|
|
921
|
+
budget.max_rows is not None
|
|
922
|
+
and min(limit, dataset.manifest.generation.row_count) > budget.max_rows
|
|
923
|
+
):
|
|
924
|
+
raise _ROWS_EXCEEDED.error(limit=budget.max_rows)
|
|
925
|
+
iterator = dataset.to_arrow_batches(batch_size=min(limit, self.policy.batch_cap))
|
|
926
|
+
rows = []
|
|
927
|
+
byte_count = 0
|
|
928
|
+
try:
|
|
929
|
+
for batch in iterator:
|
|
930
|
+
projected = batch.select(columns).slice(0, limit - len(rows))
|
|
931
|
+
if (
|
|
932
|
+
budget.max_rows is not None
|
|
933
|
+
and len(rows) + projected.num_rows > budget.max_rows
|
|
934
|
+
):
|
|
935
|
+
raise _ROWS_EXCEEDED.error(limit=budget.max_rows)
|
|
936
|
+
byte_count += projected.nbytes
|
|
937
|
+
if budget.max_output_bytes is not None and byte_count > budget.max_output_bytes:
|
|
938
|
+
raise _OUTPUT_BYTES_EXCEEDED.error(limit=budget.max_output_bytes)
|
|
939
|
+
for row in projected.to_pylist():
|
|
940
|
+
rows.append(
|
|
941
|
+
tuple(
|
|
942
|
+
None
|
|
943
|
+
if name in self.policy.redacted_columns
|
|
944
|
+
else to_jsonable_python(
|
|
945
|
+
row[name], bytes_mode="base64", inf_nan_mode="strings"
|
|
946
|
+
)
|
|
947
|
+
for name in columns
|
|
948
|
+
)
|
|
949
|
+
)
|
|
950
|
+
if len(rows) >= limit:
|
|
951
|
+
break
|
|
952
|
+
finally:
|
|
953
|
+
iterator.close()
|
|
954
|
+
return RowsOutput(columns=columns, rows=tuple(rows))
|
|
955
|
+
|
|
956
|
+
return self._call("preview", preview)
|