terrine 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terrine/__init__.py +120 -0
- terrine/_time.py +21 -0
- terrine/adapters/__init__.py +3 -0
- terrine/adapters/increment.py +365 -0
- terrine/adapters/opentelemetry.py +63 -0
- terrine/agent/__init__.py +6 -0
- terrine/agent/definitions.py +143 -0
- terrine/agent/design_schema.py +220 -0
- terrine/agent/policy.py +144 -0
- terrine/agent/schemas.py +285 -0
- terrine/agent/tools.py +956 -0
- terrine/backends/__init__.py +3 -0
- terrine/backends/capabilities.py +117 -0
- terrine/backends/dialect.py +55 -0
- terrine/backends/extrema.py +40 -0
- terrine/backends/joins.py +75 -0
- terrine/backends/numeric.py +657 -0
- terrine/backends/sequences.py +40 -0
- terrine/backends/timestamps.py +421 -0
- terrine/compile/__init__.py +3 -0
- terrine/compile/_asof.py +165 -0
- terrine/compile/_predicates.py +110 -0
- terrine/compile/anchors.py +492 -0
- terrine/compile/columns.py +1353 -0
- terrine/compile/cost.py +110 -0
- terrine/compile/dispatch.py +148 -0
- terrine/compile/layers.py +242 -0
- terrine/compile/materialize.py +91 -0
- terrine/compile/provenance.py +120 -0
- terrine/compile/ranges.py +91 -0
- terrine/compile/retention.py +127 -0
- terrine/compile/shapes/__init__.py +3 -0
- terrine/compile/shapes/_observations.py +502 -0
- terrine/compile/shapes/calendar_comparison.py +370 -0
- terrine/compile/shapes/cohorts.py +285 -0
- terrine/compile/shapes/event_panel.py +46 -0
- terrine/compile/shapes/event_sequence.py +498 -0
- terrine/compile/shapes/life_table.py +244 -0
- terrine/compile/shapes/ltv.py +287 -0
- terrine/compile/shapes/matched_t0.py +75 -0
- terrine/compile/shapes/sequential.py +320 -0
- terrine/compile/shapes/state_panel.py +460 -0
- terrine/compile/shapes/time_to_event.py +521 -0
- terrine/data.py +1311 -0
- terrine/dataset/__init__.py +5 -0
- terrine/dataset/export.py +463 -0
- terrine/dataset/manifest.py +254 -0
- terrine/dataset/reference.py +582 -0
- terrine/dataset/splits.py +373 -0
- terrine/dataset/training.py +114 -0
- terrine/dataset/transforms.py +182 -0
- terrine/errors.py +98 -0
- terrine/execution/__init__.py +5 -0
- terrine/execution/_bigquery.py +318 -0
- terrine/execution/_metadata.py +47 -0
- terrine/execution/_native.py +574 -0
- terrine/execution/_postgres.py +468 -0
- terrine/execution/_publish.py +299 -0
- terrine/execution/_snowflake.py +684 -0
- terrine/execution/backend.py +349 -0
- terrine/execution/gates.py +90 -0
- terrine/execution/inputs.py +923 -0
- terrine/export/__init__.py +31 -0
- terrine/export/diff_diff.py +177 -0
- terrine/export/lifelines.py +219 -0
- terrine/export/preflight.py +243 -0
- terrine/export/surrogate.py +168 -0
- terrine/frame.py +1042 -0
- terrine/model.py +397 -0
- terrine/orchestration/__init__.py +3 -0
- terrine/orchestration/airflow.py +115 -0
- terrine/orchestration/dagster.py +103 -0
- terrine/py.typed +0 -0
- terrine/recipes.py +295 -0
- terrine/runtime/__init__.py +70 -0
- terrine/runtime/api.py +803 -0
- terrine/runtime/arrow.py +98 -0
- terrine/runtime/backends/base.py +70 -0
- terrine/runtime/backends/duckdb.py +645 -0
- terrine/runtime/changes.py +71 -0
- terrine/runtime/cli.py +108 -0
- terrine/runtime/control.py +355 -0
- terrine/runtime/cost.py +211 -0
- terrine/runtime/events.py +194 -0
- terrine/runtime/execute.py +932 -0
- terrine/runtime/gc.py +43 -0
- terrine/runtime/graph.py +200 -0
- terrine/runtime/layers.py +61 -0
- terrine/runtime/leases.py +165 -0
- terrine/runtime/materialize.py +53 -0
- terrine/runtime/owner.py +57 -0
- terrine/runtime/planner.py +441 -0
- terrine/runtime/planning.py +394 -0
- terrine/runtime/saved_plan.py +337 -0
- terrine/runtime/snapshot.py +541 -0
- terrine/runtime/state.py +585 -0
- terrine/runtime/status.py +360 -0
- terrine/runtime/synchronization.py +15 -0
- terrine/semantic/__init__.py +3 -0
- terrine/semantic/bind.py +309 -0
- terrine/semantic/io.py +116 -0
- terrine/semantic/models.py +342 -0
- terrine/semantic/rules.py +113 -0
- terrine/semantic/sql.py +185 -0
- terrine/semantic/time.py +42 -0
- terrine/simulate/__init__.py +24 -0
- terrine/simulate/baseline.yaml +57 -0
- terrine/simulate/generate.py +182 -0
- terrine/simulate/scale.py +234 -0
- terrine/simulate/world.py +75 -0
- terrine/spec/__init__.py +96 -0
- terrine/spec/anchors.py +107 -0
- terrine/spec/dataset.py +260 -0
- terrine/spec/design.py +361 -0
- terrine/spec/expressions.py +440 -0
- terrine/spec/io.py +829 -0
- terrine/spec/panels.py +359 -0
- terrine/spec/sequence.py +165 -0
- terrine/spec/state_panel.py +246 -0
- terrine/spec/trajectory.py +144 -0
- terrine/spec/transforms.py +260 -0
- terrine/spec/validate.py +758 -0
- terrine/temporal/__init__.py +3 -0
- terrine/temporal/aggregate.py +688 -0
- terrine/temporal/attributes.py +389 -0
- terrine/temporal/daily.py +481 -0
- terrine/temporal/first.py +239 -0
- terrine/temporal/lookup.py +194 -0
- terrine/temporal/nonadditive.py +466 -0
- terrine/temporal/prefix.py +104 -0
- terrine/temporal/ranges.py +122 -0
- terrine/temporal/readiness.py +41 -0
- terrine/temporal/retention.py +225 -0
- terrine/temporal/sequence.py +900 -0
- terrine/temporal/sessionize.py +135 -0
- terrine/temporal/sketches.py +329 -0
- terrine/windows.py +54 -0
- terrine-0.1.0.dev0.dist-info/METADATA +164 -0
- terrine-0.1.0.dev0.dist-info/RECORD +144 -0
- terrine-0.1.0.dev0.dist-info/WHEEL +4 -0
- terrine-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- terrine-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- terrine-0.1.0.dev0.dist-info/licenses/NOTICE +4 -0
- terrine-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.txt +219 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Portable Arrow projection for downstream dataset consumers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from types import MappingProxyType
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
from terrine.errors import RefusalSpec
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
import pyarrow as pa
|
|
15
|
+
|
|
16
|
+
from terrine.data import MaterializedFrame
|
|
17
|
+
|
|
18
|
+
__all__ = ["ArrowSurrogate", "to_surrogate_arrow"]
|
|
19
|
+
|
|
20
|
+
_INVALID = RefusalSpec(
|
|
21
|
+
"terrine.export.invalid_surrogate",
|
|
22
|
+
"Surrogate export input is invalid ({reason}).",
|
|
23
|
+
)
|
|
24
|
+
_SURROGATE_METADATA_KEY = b"terrine.surrogate_arrow.v1"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class ArrowSurrogate:
|
|
29
|
+
"""Versioned Arrow surrogate projection and portable metadata."""
|
|
30
|
+
|
|
31
|
+
data: pa.Table
|
|
32
|
+
metadata: Mapping[str, object]
|
|
33
|
+
|
|
34
|
+
def __post_init__(self) -> None:
|
|
35
|
+
import pyarrow as pa
|
|
36
|
+
|
|
37
|
+
if not isinstance(self.data, pa.Table):
|
|
38
|
+
raise TypeError("data must be a pyarrow.Table")
|
|
39
|
+
object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata)))
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_arrow(cls, table: pa.Table) -> ArrowSurrogate:
|
|
43
|
+
raw = (table.schema.metadata or {}).get(_SURROGATE_METADATA_KEY)
|
|
44
|
+
if raw is None:
|
|
45
|
+
raise ValueError("Arrow schema has no Terrine surrogate metadata")
|
|
46
|
+
metadata = json.loads(raw.decode("utf-8"))
|
|
47
|
+
if metadata.get("schema_version") != 1:
|
|
48
|
+
raise ValueError("unsupported Terrine surrogate metadata version")
|
|
49
|
+
clean = dict(table.schema.metadata or {})
|
|
50
|
+
del clean[_SURROGATE_METADATA_KEY]
|
|
51
|
+
return cls(table.replace_schema_metadata(clean or None), metadata)
|
|
52
|
+
|
|
53
|
+
def to_arrow(self) -> pa.Table:
|
|
54
|
+
metadata = json.dumps(
|
|
55
|
+
dict(self.metadata), sort_keys=True, separators=(",", ":"), allow_nan=False
|
|
56
|
+
).encode("utf-8")
|
|
57
|
+
schema_metadata = dict(self.data.schema.metadata or {})
|
|
58
|
+
schema_metadata[_SURROGATE_METADATA_KEY] = metadata
|
|
59
|
+
return self.data.replace_schema_metadata(schema_metadata)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def to_surrogate_arrow(
|
|
63
|
+
data: MaterializedFrame, *, features: tuple[str, ...], outcomes: tuple[str, ...]
|
|
64
|
+
) -> ArrowSurrogate:
|
|
65
|
+
"""Project Arrow training data with identity, missingness and provenance."""
|
|
66
|
+
from terrine.data import MaterializedFrame
|
|
67
|
+
|
|
68
|
+
if not isinstance(data, MaterializedFrame):
|
|
69
|
+
raise _INVALID.error(reason="materialized_frame_required")
|
|
70
|
+
table = data.table
|
|
71
|
+
structural = {
|
|
72
|
+
"time_to_event": ("entry_time",),
|
|
73
|
+
"ltv": ("age",),
|
|
74
|
+
"cohort_table": ("age", "at_risk", "observable"),
|
|
75
|
+
"life_table": ("time_bin", "censored_by_reason"),
|
|
76
|
+
}.get(data.provenance.shape, ())
|
|
77
|
+
requested = (*features, *outcomes)
|
|
78
|
+
if len(set(requested)) != len(requested):
|
|
79
|
+
raise _INVALID.error(reason="duplicate_surrogate_column")
|
|
80
|
+
if any(name not in table.column_names for name in requested):
|
|
81
|
+
raise _INVALID.error(reason="missing_surrogate_column")
|
|
82
|
+
identity = tuple(
|
|
83
|
+
dict.fromkeys(
|
|
84
|
+
name for name in (data.roles.unit, data.roles.anchor, data.roles.t0) if name is not None
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
role_support = tuple(
|
|
88
|
+
name
|
|
89
|
+
for name in (
|
|
90
|
+
data.roles.trial,
|
|
91
|
+
data.roles.assignment_probability,
|
|
92
|
+
data.roles.target_weight,
|
|
93
|
+
data.roles.trial_weight,
|
|
94
|
+
)
|
|
95
|
+
if name is not None
|
|
96
|
+
)
|
|
97
|
+
metadata_columns = tuple(
|
|
98
|
+
dict.fromkeys((*requested, *data.roles.outcomes, *structural, *identity, *role_support))
|
|
99
|
+
)
|
|
100
|
+
support = list(role_support)
|
|
101
|
+
for name in metadata_columns:
|
|
102
|
+
timing = data.provenance.expression_timings.get(name)
|
|
103
|
+
if timing is not None:
|
|
104
|
+
support.extend(
|
|
105
|
+
column
|
|
106
|
+
for column in (
|
|
107
|
+
timing.point_time_column,
|
|
108
|
+
timing.start_bound_column,
|
|
109
|
+
timing.end_bound_column,
|
|
110
|
+
)
|
|
111
|
+
if column is not None
|
|
112
|
+
)
|
|
113
|
+
meta = data.columns.get(name)
|
|
114
|
+
if meta is not None:
|
|
115
|
+
support.extend(
|
|
116
|
+
column
|
|
117
|
+
for column in (
|
|
118
|
+
meta.observed_from_column,
|
|
119
|
+
meta.observed_until_column,
|
|
120
|
+
meta.complete_column,
|
|
121
|
+
meta.reason_column,
|
|
122
|
+
)
|
|
123
|
+
if column is not None
|
|
124
|
+
)
|
|
125
|
+
retained = tuple(
|
|
126
|
+
dict.fromkeys((*identity, *requested, *data.roles.outcomes, *structural, *support))
|
|
127
|
+
)
|
|
128
|
+
missing = [name for name in retained if name not in table.column_names]
|
|
129
|
+
if missing:
|
|
130
|
+
raise _INVALID.error(reason=f"missing_surrogate_support_{missing[0]}")
|
|
131
|
+
projected = table.select(retained)
|
|
132
|
+
serialized = data.to_metadata()
|
|
133
|
+
role_metadata = dict(serialized["roles"])
|
|
134
|
+
projected_outcomes = [
|
|
135
|
+
name for name in (*outcomes, *data.roles.outcomes, *structural) if name in retained
|
|
136
|
+
]
|
|
137
|
+
role_metadata["features"] = [name for name in features if name in retained]
|
|
138
|
+
role_metadata["outcomes"] = list(dict.fromkeys(projected_outcomes))
|
|
139
|
+
for role in ("unit", "anchor", "t0", "treatment", "weight", "cluster_unit", "prior_state"):
|
|
140
|
+
if role in role_metadata and role_metadata[role] not in retained:
|
|
141
|
+
role_metadata[role] = None
|
|
142
|
+
provenance = serialized["provenance"]
|
|
143
|
+
for field in ("expression_timings", "feature_cutoffs"):
|
|
144
|
+
provenance[field] = {
|
|
145
|
+
name: value for name, value in provenance[field].items() if name in retained
|
|
146
|
+
}
|
|
147
|
+
metadata = {
|
|
148
|
+
"schema_version": 1,
|
|
149
|
+
"design_hash": data.provenance.design_hash,
|
|
150
|
+
"model_hash": data.provenance.model_hash,
|
|
151
|
+
"source_generations": dict(data.provenance.source_generations),
|
|
152
|
+
"execution_snapshot": serialized["provenance"]["execution_snapshot"],
|
|
153
|
+
"provenance": provenance,
|
|
154
|
+
"roles": role_metadata,
|
|
155
|
+
"feature_columns": list(features),
|
|
156
|
+
"outcome_columns": list(dict.fromkeys(projected_outcomes)),
|
|
157
|
+
"support_columns": list(dict.fromkeys(support)),
|
|
158
|
+
"columns": {
|
|
159
|
+
name: value for name, value in serialized["columns"].items() if name in retained
|
|
160
|
+
},
|
|
161
|
+
"administrative_cutoff": (
|
|
162
|
+
data.provenance.administrative_cutoff.isoformat()
|
|
163
|
+
if data.provenance.administrative_cutoff is not None
|
|
164
|
+
else None
|
|
165
|
+
),
|
|
166
|
+
"cohort_operation": serialized["provenance"].get("cohort_operation"),
|
|
167
|
+
}
|
|
168
|
+
return ArrowSurrogate(projected, metadata)
|